- Merge from HEAD

876 files changed, 50143 insertions, 36035 deletions

diff --git a/sys/amd64/acpica/Makefile b/sys/amd64/acpica/Makefile
new file mode 100644
index 0000000..7437280
--- /dev/null
+++ b/sys/amd64/acpica/Makefile
@@ -0,0 +1,33 @@
+# $FreeBSD$
+
+# Correct path for kernel builds
+# Don't rely on the kernel's .depend file
+.ifdef MAKESRCPATH
+.PATH: ${MAKESRCPATH}
+DEPENDFILE=
+.else
+MAKESRCPATH= ${.CURDIR}
+CLEANFILES= acpi_wakecode.h acpi_wakedata.h acpi_wakecode.bin acpi_wakecode.o
+.endif
+.if ${CC} == "icc"
+CFLAGS+=	-restrict
+NOSTDINC=	-X
+.else
+NOSTDINC=	-nostdinc
+.endif
+CFLAGS+=	${NOSTDINC} -include opt_global.h -I. -I${MAKESRCPATH}/../..
+
+all: acpi_wakecode.h acpi_wakedata.h
+
+acpi_wakecode.o: acpi_wakecode.S assym.s
+
+acpi_wakecode.bin: acpi_wakecode.o
+	objcopy -S -O binary acpi_wakecode.o acpi_wakecode.bin
+
+acpi_wakecode.h: acpi_wakecode.bin
+	sh ${MAKESRCPATH}/genwakecode.sh > acpi_wakecode.h
+
+acpi_wakedata.h: acpi_wakecode.bin
+	sh ${MAKESRCPATH}/genwakedata.sh > acpi_wakedata.h
+
+.include <bsd.prog.mk>
diff --git a/sys/amd64/acpica/acpi_machdep.c b/sys/amd64/acpica/acpi_machdep.c
index e9e9235..c69f14a 100644
--- a/sys/amd64/acpica/acpi_machdep.c
+++ b/sys/amd64/acpica/acpi_machdep.c
@@ -31,25 +31,51 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
+#include <sys/sysctl.h>
 
 #include <contrib/dev/acpica/acpi.h>
 #include <dev/acpica/acpivar.h>
 
 #include <machine/nexusvar.h>
 
+SYSCTL_DECL(_debug_acpi);
+
+int acpi_resume_beep;
+TUNABLE_INT("debug.acpi.resume_beep", &acpi_resume_beep);
+SYSCTL_INT(_debug_acpi, OID_AUTO, resume_beep, CTLFLAG_RW, &acpi_resume_beep,
+    0, "Beep the PC speaker when resuming");
+
+int acpi_reset_video;
+TUNABLE_INT("hw.acpi.reset_video", &acpi_reset_video);
+
 static int intr_model = ACPI_INTR_PIC;
+static struct apm_clone_data acpi_clone;
 
 int
 acpi_machdep_init(device_t dev)
 {
-	struct	acpi_softc *sc;
+	struct acpi_softc	*sc;
 
 	sc = devclass_get_softc(devclass_find("acpi"), 0);
+
+	/* Create a fake clone for /dev/acpi. */
+	STAILQ_INIT(&sc->apm_cdevs);
+	acpi_clone.cdev = sc->acpi_dev_t;
+	acpi_clone.acpi_sc = sc;
+	ACPI_LOCK(acpi);
+	STAILQ_INSERT_TAIL(&sc->apm_cdevs, &acpi_clone, entries);
+	ACPI_UNLOCK(acpi);
+	sc->acpi_clone = &acpi_clone;
 	acpi_install_wakeup_handler(sc);
 
 	if (intr_model != ACPI_INTR_PIC)
 		acpi_SetIntrModel(intr_model);
 
+	SYSCTL_ADD_UINT(&sc->acpi_sysctl_ctx,
+	    SYSCTL_CHILDREN(sc->acpi_sysctl_tree), OID_AUTO,
+	    "reset_video", CTLFLAG_RW, &acpi_reset_video, 0,
+	    "Call the VESA reset BIOS vector on the resume path");
+
 	return (0);
 }
 
diff --git a/sys/amd64/acpica/acpi_switch.S b/sys/amd64/acpica/acpi_switch.S
new file mode 100644
index 0000000..892dd11
--- /dev/null
+++ b/sys/amd64/acpica/acpi_switch.S
@@ -0,0 +1,190 @@
+/*-
+ * Copyright (c) 2001 Takanori Watanabe <takawata@jp.freebsd.org>
+ * Copyright (c) 2001 Mitsuru IWASAKI <iwasaki@jp.freebsd.org>
+ * Copyright (c) 2008-2009 Jung-uk Kim <jkim@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <machine/asmacros.h>
+#include <machine/specialreg.h>
+
+#include "acpi_wakedata.h"
+#include "assym.s"
+
+#define	WAKEUP_DECL(member)	\
+    .set WAKEUP_ ## member, wakeup_ ## member - wakeup_ctx
+
+	WAKEUP_DECL(xpcb)
+	WAKEUP_DECL(gdt)
+	WAKEUP_DECL(efer)
+	WAKEUP_DECL(pat)
+	WAKEUP_DECL(star)
+	WAKEUP_DECL(lstar)
+	WAKEUP_DECL(cstar)
+	WAKEUP_DECL(sfmask)
+	WAKEUP_DECL(cpu)
+
+#define	WAKEUP_CTX(member)	WAKEUP_ ## member (%rdi)
+#define	WAKEUP_PCB(member)	PCB_ ## member(%r11)
+#define	WAKEUP_XPCB(member)	XPCB_ ## member(%r11)
+
+ENTRY(acpi_restorecpu)
+	/* Switch to KPML4phys. */
+	movq	%rsi, %rax
+	movq	%rax, %cr3
+
+	/* Restore GDT. */
+	lgdt	WAKEUP_CTX(gdt)
+	jmp	1f
+1:
+
+	/* Fetch PCB. */
+	movq	WAKEUP_CTX(xpcb), %r11
+
+	/* Force kernel segment registers. */
+	movl	$KDSEL, %eax
+	movw	%ax, %ds
+	movw	%ax, %es
+	movw	%ax, %ss
+	movl	$KUF32SEL, %eax
+	movw	%ax, %fs
+	movl	$KUG32SEL, %eax
+	movw	%ax, %gs
+
+	movl	$MSR_FSBASE, %ecx
+	movl	WAKEUP_PCB(FSBASE), %eax
+	movl	4 + WAKEUP_PCB(FSBASE), %edx
+	wrmsr
+	movl	$MSR_GSBASE, %ecx
+	movl	WAKEUP_PCB(GSBASE), %eax
+	movl	4 + WAKEUP_PCB(GSBASE), %edx
+	wrmsr
+	movl	$MSR_KGSBASE, %ecx
+	movl	WAKEUP_XPCB(KGSBASE), %eax
+	movl	4 + WAKEUP_XPCB(KGSBASE), %edx
+	wrmsr
+
+	/* Restore EFER. */
+	movl	$MSR_EFER, %ecx
+	movl	WAKEUP_CTX(efer), %eax
+	wrmsr
+
+	/* Restore PAT. */
+	movl	$MSR_PAT, %ecx
+	movl	WAKEUP_CTX(pat), %eax
+	movl	4 + WAKEUP_CTX(pat), %edx
+	wrmsr
+
+	/* Restore fast syscall stuff. */
+	movl	$MSR_STAR, %ecx
+	movl	WAKEUP_CTX(star), %eax
+	movl	4 + WAKEUP_CTX(star), %edx
+	wrmsr
+	movl	$MSR_LSTAR, %ecx
+	movl	WAKEUP_CTX(lstar), %eax
+	movl	4 + WAKEUP_CTX(lstar), %edx
+	wrmsr
+	movl	$MSR_CSTAR, %ecx
+	movl	WAKEUP_CTX(cstar), %eax
+	movl	4 + WAKEUP_CTX(cstar), %edx
+	wrmsr
+	movl	$MSR_SF_MASK, %ecx
+	movl	WAKEUP_CTX(sfmask), %eax
+	wrmsr
+
+	/* Restore CR0, CR2 and CR4. */
+	movq	WAKEUP_XPCB(CR0), %rax
+	movq	%rax, %cr0
+	movq	WAKEUP_XPCB(CR2), %rax
+	movq	%rax, %cr2
+	movq	WAKEUP_XPCB(CR4), %rax
+	movq	%rax, %cr4
+
+	/* Restore descriptor tables. */
+	lidt	WAKEUP_XPCB(IDT)
+	lldt	WAKEUP_XPCB(LDT)
+
+#define	SDT_SYSTSS	9
+#define	SDT_SYSBSY	11
+
+	/* Clear "task busy" bit and reload TR. */
+	movq	PCPU(TSS), %rax
+	andb	$(~SDT_SYSBSY | SDT_SYSTSS), 5(%rax)
+	movw	WAKEUP_XPCB(TR), %ax
+	ltr	%ax
+
+#undef	SDT_SYSTSS
+#undef	SDT_SYSBSY
+
+	/* Restore other callee saved registers. */
+	movq	WAKEUP_PCB(R15), %r15
+	movq	WAKEUP_PCB(R14), %r14
+	movq	WAKEUP_PCB(R13), %r13
+	movq	WAKEUP_PCB(R12), %r12
+	movq	WAKEUP_PCB(RBP), %rbp
+	movq	WAKEUP_PCB(RSP), %rsp
+	movq	WAKEUP_PCB(RBX), %rbx
+
+	/* Restore debug registers. */
+	movq	WAKEUP_PCB(DR0), %rax
+	movq	%rax, %dr0
+	movq	WAKEUP_PCB(DR1), %rax
+	movq	%rax, %dr1
+	movq	WAKEUP_PCB(DR2), %rax
+	movq	%rax, %dr2
+	movq	WAKEUP_PCB(DR3), %rax
+	movq	%rax, %dr3
+	movq	WAKEUP_PCB(DR6), %rax
+	movq	%rax, %dr6
+	movq	WAKEUP_PCB(DR7), %rax
+	movq	%rax, %dr7
+
+	/* Restore return address. */
+	movq	WAKEUP_PCB(RIP), %rax
+	movq	%rax, (%rsp)
+
+	/* Indicate the CPU is resumed. */
+	xorl	%eax, %eax
+	movl	%eax, WAKEUP_CTX(cpu)
+
+	ret
+END(acpi_restorecpu)
+
+ENTRY(acpi_savecpu)
+	/* Fetch XPCB and save CPU context. */
+	movq	%rdi, %r10
+	call	savectx2
+	movq	%r10, %r11
+
+	/* Patch caller's return address and stack pointer. */
+	movq	(%rsp), %rax
+	movq	%rax, WAKEUP_PCB(RIP)
+	movq	%rsp, %rax
+	movq	%rax, WAKEUP_PCB(RSP)
+
+	movl	$1, %eax
+	ret
+END(acpi_savecpu)
diff --git a/sys/amd64/acpica/acpi_wakecode.S b/sys/amd64/acpica/acpi_wakecode.S
new file mode 100644
index 0000000..4e82f53
--- /dev/null
+++ b/sys/amd64/acpica/acpi_wakecode.S
@@ -0,0 +1,286 @@
+/*-
+ * Copyright (c) 2001 Takanori Watanabe <takawata@jp.freebsd.org>
+ * Copyright (c) 2001 Mitsuru IWASAKI <iwasaki@jp.freebsd.org>
+ * Copyright (c) 2003 Peter Wemm
+ * Copyright (c) 2008-2009 Jung-uk Kim <jkim@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#define LOCORE
+
+#include <machine/asmacros.h>
+#include <machine/specialreg.h>
+
+#include "assym.s"
+
+/*
+ * Resume entry point for real mode.
+ *
+ * If XFirmwareWakingVector is zero and FirmwareWakingVector is non-zero
+ * in FACS, the BIOS enters here in real mode after POST with CS set to
+ * (FirmwareWakingVector >> 4) and IP set to (FirmwareWakingVector & 0xf).
+ * Depending on the previous sleep state, we may need to initialize more
+ * of the system (i.e., S3 suspend-to-RAM vs. S4 suspend-to-disk).
+ *
+ * Note: If XFirmwareWakingVector is non-zero, it should disable address
+ * translation/paging and interrupts, load all segment registers with
+ * a flat 4 GB address space, and set EFLAGS.IF to zero.  Currently
+ * this mode is not supported by this code.
+ */
+
+	.data				/* So we can modify it */
+
+	ALIGN_TEXT
+wakeup_start:
+	.code16
+	/*
+	 * Set up segment registers for real mode, a small stack for
+	 * any calls we make, and clear any flags.
+	 */
+	cli				/* make sure no interrupts */
+	cld
+	mov	%cs, %ax		/* copy %cs to %ds.  Remember these */
+	mov	%ax, %ds		/* are offsets rather than selectors */
+	mov	%ax, %ss
+	movw	$PAGE_SIZE - 8, %sp
+	xorw	%ax, %ax
+	pushw	%ax
+	popfw
+
+	/* To debug resume hangs, beep the speaker if the user requested. */
+	testb	$~0, resume_beep - wakeup_start
+	jz	1f
+	movb	$0, resume_beep - wakeup_start
+	movb	$0xc0, %al
+	outb	%al, $0x42
+	movb	$0x04, %al
+	outb	%al, $0x42
+	inb	$0x61, %al
+	orb	$0x3, %al
+	outb	%al, $0x61
+1:
+
+	/* Re-initialize video BIOS if the reset_video tunable is set. */
+	testb	$~0, reset_video - wakeup_start
+	jz	1f
+	movb	$0, reset_video - wakeup_start
+	lcall	$0xc000, $3
+
+	/* Re-start in case the previous BIOS call clobbers them. */
+	jmp	wakeup_start
+1:
+
+	/*
+	 * Find relocation base and patch the gdt descript and ljmp targets
+	 */
+	xorl	%ebx, %ebx
+	mov	%cs, %bx
+	sall	$4, %ebx		/* %ebx is now our relocation base */
+
+	/*
+	 * Load the descriptor table pointer.  We'll need it when running
+	 * in 16-bit protected mode.
+	 */
+	lgdtl	bootgdtdesc - wakeup_start
+
+	/* Enable protected mode */
+	movl	$CR0_PE, %eax
+	mov	%eax, %cr0
+
+	/*
+	 * Now execute a far jump to turn on protected mode.  This
+	 * causes the segment registers to turn into selectors and causes
+	 * %cs to be loaded from the gdt.
+	 *
+	 * The following instruction is:
+	 * ljmpl $bootcode32 - bootgdt, $wakeup_32 - wakeup_start
+	 * but gas cannot assemble that.  And besides, we patch the targets
+	 * in early startup and its a little clearer what we are patching.
+	 */
+wakeup_sw32:
+	.byte	0x66			/* size override to 32 bits */
+	.byte	0xea			/* opcode for far jump */
+	.long	wakeup_32 - wakeup_start /* offset in segment */
+	.word	bootcode32 - bootgdt	/* index in gdt for 32 bit code */
+
+	/*
+	 * At this point, we are running in 32 bit legacy protected mode.
+	 */
+	.code32
+wakeup_32:
+
+	mov	$bootdata32 - bootgdt, %eax
+	mov	%ax, %ds
+
+	/* Turn on the PAE and PSE bits for when paging is enabled */
+	mov	%cr4, %eax
+	orl	$(CR4_PAE | CR4_PSE), %eax
+	mov	%eax, %cr4
+
+	/*
+	 * Enable EFER.LME so that we get long mode when all the prereqs are
+	 * in place.  In this case, it turns on when CR0_PG is finally enabled.
+	 * Pick up a few other EFER bits that we'll use need we're here.
+	 */
+	movl	$MSR_EFER, %ecx
+	rdmsr
+	orl	$EFER_LME | EFER_SCE, %eax
+	wrmsr
+
+	/*
+	 * Point to the embedded page tables for startup.  Note that this
+	 * only gets accessed after we're actually in 64 bit mode, however
+	 * we can only set the bottom 32 bits of %cr3 in this state.  This
+	 * means we are required to use a temporary page table that is below
+	 * the 4GB limit.  %ebx is still our relocation base.  We could just
+	 * subtract 3 * PAGE_SIZE, but that would be too easy.
+	 */
+	leal	wakeup_pagetables - wakeup_start(%ebx), %eax
+	movl	(%eax), %eax
+	mov	%eax, %cr3
+
+	/*
+	 * Finally, switch to long bit mode by enabling paging.  We have
+	 * to be very careful here because all the segmentation disappears
+	 * out from underneath us.  The spec says we can depend on the
+	 * subsequent pipelined branch to execute, but *only if* everthing
+	 * is still identity mapped.  If any mappings change, the pipeline
+	 * will flush.
+	 */
+	mov	%cr0, %eax
+	orl	$CR0_PG, %eax
+	mov	%eax, %cr0
+
+	/*
+	 * At this point paging is enabled, and we are in "compatability" mode.
+	 * We do another far jump to reload %cs with the 64 bit selector.
+	 * %cr3 points to a 4-level page table page.
+	 * We cannot yet jump all the way to the kernel because we can only
+	 * specify a 32 bit linear address.  So, yet another trampoline.
+	 *
+	 * The following instruction is:
+	 * ljmp $bootcode64 - bootgdt, $wakeup_64 - wakeup_start
+	 * but gas cannot assemble that.  And besides, we patch the targets
+	 * in early startup and its a little clearer what we are patching.
+	 */
+wakeup_sw64:
+	.byte	0xea			/* opcode for far jump */
+	.long	wakeup_64 - wakeup_start /* offset in segment */
+	.word	bootcode64 - bootgdt	/* index in gdt for 64 bit code */
+
+	/*
+	 * Yeehar!  We're running in 64-bit mode!  We can mostly ignore our
+	 * segment registers, and get on with it.
+	 * Note that we are running at the correct virtual address, but with
+	 * a 1:1 1GB mirrored mapping over entire address space.  We had better
+	 * switch to a real %cr3 promptly so that we can get to the direct map
+	 * space. Remember that jmp is relative and that we've been relocated,
+	 * so use an indirect jump.
+	 */
+	ALIGN_TEXT
+	.code64
+wakeup_64:
+	mov	$bootdata64 - bootgdt, %eax
+	mov	%ax, %ds
+
+	/* Restore arguments and return. */
+	movq	wakeup_ctx - wakeup_start(%rbx), %rdi
+	movq	wakeup_kpml4 - wakeup_start(%rbx), %rsi
+	movq	wakeup_retaddr - wakeup_start(%rbx), %rax
+	jmp	*%rax
+
+	.data
+
+resume_beep:
+	.byte	0
+reset_video:
+	.byte	0
+
+	ALIGN_DATA
+bootgdt:
+	.long	0x00000000
+	.long	0x00000000
+	.long	0x00000000
+	.long	0x00000000
+	.long	0x00000000
+	.long	0x00000000
+	.long	0x00000000
+	.long	0x00000000
+
+bootcode64:
+	.long	0x0000ffff
+	.long	0x00af9b00
+
+bootdata64:
+	.long	0x0000ffff
+	.long	0x00af9300
+
+bootcode32:
+	.long	0x0000ffff
+	.long	0x00cf9b00
+
+bootdata32:
+	.long	0x0000ffff
+	.long	0x00cf9300
+bootgdtend:
+
+wakeup_pagetables:
+	.long	0
+
+bootgdtdesc:
+	.word	bootgdtend - bootgdt	/* Length */
+	.long	bootgdt - wakeup_start	/* Offset plus %ds << 4 */
+
+	ALIGN_DATA
+wakeup_retaddr:
+	.quad	0
+wakeup_kpml4:
+	.quad	0
+
+wakeup_ctx:
+	.quad	0
+wakeup_xpcb:
+	.quad	0
+wakeup_gdt:
+	.word	0
+	.quad	0
+
+	ALIGN_DATA
+wakeup_efer:
+	.quad	0
+wakeup_pat:
+	.quad	0
+wakeup_star:
+	.quad	0
+wakeup_lstar:
+	.quad	0
+wakeup_cstar:
+	.quad	0
+wakeup_sfmask:
+	.quad	0
+wakeup_cpu:
+	.long	0
+dummy:
diff --git a/sys/amd64/acpica/acpi_wakeup.c b/sys/amd64/acpica/acpi_wakeup.c
index 53868e4..2f9d8a0 100644
--- a/sys/amd64/acpica/acpi_wakeup.c
+++ b/sys/amd64/acpica/acpi_wakeup.c
@@ -1,6 +1,8 @@
 /*-
  * Copyright (c) 2001 Takanori Watanabe <takawata@jp.freebsd.org>
  * Copyright (c) 2001 Mitsuru IWASAKI <iwasaki@jp.freebsd.org>
+ * Copyright (c) 2003 Peter Wemm
+ * Copyright (c) 2008-2009 Jung-uk Kim <jkim@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -29,18 +31,390 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
+#include <sys/systm.h>
 #include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/memrange.h>
+#include <sys/smp.h>
+#include <sys/types.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/intr_machdep.h>
+#include <machine/pcb.h>
+#include <machine/pmap.h>
+#include <machine/specialreg.h>
+#include <machine/vmparam.h>
+
+#ifdef SMP
+#include <machine/apicreg.h>
+#include <machine/smp.h>
+#endif
 
 #include <contrib/dev/acpica/acpi.h>
 #include <dev/acpica/acpivar.h>
 
+#include "acpi_wakecode.h"
+#include "acpi_wakedata.h"
+
+/* Make sure the code is less than a page and leave room for the stack. */
+CTASSERT(sizeof(wakecode) < PAGE_SIZE - 1024);
+
+#ifndef _SYS_CDEFS_H_
+#error this file needs sys/cdefs.h as a prerequisite
+#endif
+
+extern int		acpi_resume_beep;
+extern int		acpi_reset_video;
+
+#ifdef SMP
+extern struct xpcb	*stopxpcbs;
+#else
+static struct xpcb	*stopxpcbs;
+#endif
+
+int			acpi_restorecpu(struct xpcb *, vm_offset_t);
+int			acpi_savecpu(struct xpcb *);
+
+static void		acpi_alloc_wakeup_handler(void);
+static void		acpi_stop_beep(void *);
+
+#ifdef SMP
+static int		acpi_wakeup_ap(struct acpi_softc *, int);
+static void		acpi_wakeup_cpus(struct acpi_softc *, cpumask_t);
+#endif
+
+#define	WAKECODE_VADDR(sc)	((sc)->acpi_wakeaddr + (3 * PAGE_SIZE))
+#define	WAKECODE_PADDR(sc)	((sc)->acpi_wakephys + (3 * PAGE_SIZE))
+#define	WAKECODE_FIXUP(offset, type, val) do	{	\
+	type	*addr;					\
+	addr = (type *)(WAKECODE_VADDR(sc) + offset);	\
+	*addr = val;					\
+} while (0)
+
+/* Turn off bits 1&2 of the PIT, stopping the beep. */
+static void
+acpi_stop_beep(void *arg)
+{
+	outb(0x61, inb(0x61) & ~0x3);
+}
+
+#ifdef SMP
+static int
+acpi_wakeup_ap(struct acpi_softc *sc, int cpu)
+{
+	int		vector = (WAKECODE_PADDR(sc) >> 12) & 0xff;
+	int		apic_id = cpu_apic_ids[cpu];
+	int		ms;
+
+	WAKECODE_FIXUP(wakeup_xpcb, struct xpcb *, &stopxpcbs[cpu]);
+	WAKECODE_FIXUP(wakeup_gdt, uint16_t, stopxpcbs[cpu].xpcb_gdt.rd_limit);
+	WAKECODE_FIXUP(wakeup_gdt + 2, uint64_t,
+	    stopxpcbs[cpu].xpcb_gdt.rd_base);
+	WAKECODE_FIXUP(wakeup_cpu, int, cpu);
+
+	/* do an INIT IPI: assert RESET */
+	lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
+	    APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, apic_id);
+
+	/* wait for pending status end */
+	lapic_ipi_wait(-1);
+
+	/* do an INIT IPI: deassert RESET */
+	lapic_ipi_raw(APIC_DEST_ALLESELF | APIC_TRIGMOD_LEVEL |
+	    APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, 0);
+
+	/* wait for pending status end */
+	DELAY(10000);		/* wait ~10mS */
+	lapic_ipi_wait(-1);
+
+	/*
+	 * next we do a STARTUP IPI: the previous INIT IPI might still be
+	 * latched, (P5 bug) this 1st STARTUP would then terminate
+	 * immediately, and the previously started INIT IPI would continue. OR
+	 * the previous INIT IPI has already run. and this STARTUP IPI will
+	 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
+	 * will run.
+	 */
+
+	/* do a STARTUP IPI */
+	lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
+	    APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP |
+	    vector, apic_id);
+	lapic_ipi_wait(-1);
+	DELAY(200);		/* wait ~200uS */
+
+	/*
+	 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
+	 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
+	 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
+	 * recognized after hardware RESET or INIT IPI.
+	 */
+
+	lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
+	    APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP |
+	    vector, apic_id);
+	lapic_ipi_wait(-1);
+	DELAY(200);		/* wait ~200uS */
+
+	/* Wait up to 5 seconds for it to start. */
+	for (ms = 0; ms < 5000; ms++) {
+		if (*(int *)(WAKECODE_VADDR(sc) + wakeup_cpu) == 0)
+			return (1);	/* return SUCCESS */
+		DELAY(1000);
+	}
+	return (0);		/* return FAILURE */
+}
+
+#define	WARMBOOT_TARGET		0
+#define	WARMBOOT_OFF		(KERNBASE + 0x0467)
+#define	WARMBOOT_SEG		(KERNBASE + 0x0469)
+
+#define	CMOS_REG		(0x70)
+#define	CMOS_DATA		(0x71)
+#define	BIOS_RESET		(0x0f)
+#define	BIOS_WARM		(0x0a)
+
+static void
+acpi_wakeup_cpus(struct acpi_softc *sc, cpumask_t wakeup_cpus)
+{
+	uint32_t	mpbioswarmvec;
+	cpumask_t	map;
+	int		cpu;
+	u_char		mpbiosreason;
+
+	/* save the current value of the warm-start vector */
+	mpbioswarmvec = *((uint32_t *)WARMBOOT_OFF);
+	outb(CMOS_REG, BIOS_RESET);
+	mpbiosreason = inb(CMOS_DATA);
+
+	/* setup a vector to our boot code */
+	*((volatile u_short *)WARMBOOT_OFF) = WARMBOOT_TARGET;
+	*((volatile u_short *)WARMBOOT_SEG) = WAKECODE_PADDR(sc) >> 4;
+	outb(CMOS_REG, BIOS_RESET);
+	outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */
+
+	/* Wake up each AP. */
+	for (cpu = 1; cpu < mp_ncpus; cpu++) {
+		map = 1ul << cpu;
+		if ((wakeup_cpus & map) != map)
+			continue;
+		if (acpi_wakeup_ap(sc, cpu) == 0) {
+			/* restore the warmstart vector */
+			*(uint32_t *)WARMBOOT_OFF = mpbioswarmvec;
+			panic("acpi_wakeup: failed to resume AP #%d (PHY #%d)",
+			    cpu, cpu_apic_ids[cpu]);
+		}
+	}
+
+	/* restore the warmstart vector */
+	*(uint32_t *)WARMBOOT_OFF = mpbioswarmvec;
+
+	outb(CMOS_REG, BIOS_RESET);
+	outb(CMOS_DATA, mpbiosreason);
+}
+#endif
+
 int
 acpi_sleep_machdep(struct acpi_softc *sc, int state)
 {
-	return (0);
+	struct savefpu	*stopfpu;
+#ifdef SMP
+	cpumask_t	wakeup_cpus;
+#endif
+	register_t	cr3, rf;
+	ACPI_STATUS	status;
+	int		ret;
+
+	ret = -1;
+
+	if (sc->acpi_wakeaddr == 0ul)
+		return (ret);
+
+#ifdef SMP
+	wakeup_cpus = PCPU_GET(other_cpus);
+#endif
+
+	AcpiSetFirmwareWakingVector(WAKECODE_PADDR(sc));
+
+	rf = intr_disable();
+	intr_suspend();
+
+	/*
+	 * Temporarily switch to the kernel pmap because it provides
+	 * an identity mapping (setup at boot) for the low physical
+	 * memory region containing the wakeup code.
+	 */
+	cr3 = rcr3();
+	load_cr3(KPML4phys);
+
+	stopfpu = &stopxpcbs[0].xpcb_pcb.pcb_save;
+	if (acpi_savecpu(&stopxpcbs[0])) {
+		fpugetregs(curthread, stopfpu);
+
+#ifdef SMP
+		if (wakeup_cpus != 0 && suspend_cpus(wakeup_cpus) == 0) {
+			device_printf(sc->acpi_dev,
+			    "Failed to suspend APs: CPU mask = 0x%jx\n",
+			    (uintmax_t)(wakeup_cpus & ~stopped_cpus));
+			goto out;
+		}
+#endif
+
+		WAKECODE_FIXUP(resume_beep, uint8_t, (acpi_resume_beep != 0));
+		WAKECODE_FIXUP(reset_video, uint8_t, (acpi_reset_video != 0));
+
+		WAKECODE_FIXUP(wakeup_xpcb, struct xpcb *, &stopxpcbs[0]);
+		WAKECODE_FIXUP(wakeup_gdt, uint16_t,
+		    stopxpcbs[0].xpcb_gdt.rd_limit);
+		WAKECODE_FIXUP(wakeup_gdt + 2, uint64_t,
+		    stopxpcbs[0].xpcb_gdt.rd_base);
+		WAKECODE_FIXUP(wakeup_cpu, int, 0);
+
+		/* Call ACPICA to enter the desired sleep state */
+		if (state == ACPI_STATE_S4 && sc->acpi_s4bios)
+			status = AcpiEnterSleepStateS4bios();
+		else
+			status = AcpiEnterSleepState(state);
+
+		if (status != AE_OK) {
+			device_printf(sc->acpi_dev,
+			    "AcpiEnterSleepState failed - %s\n",
+			    AcpiFormatException(status));
+			goto out;
+		}
+
+		for (;;)
+			ia32_pause();
+	} else {
+		fpusetregs(curthread, stopfpu);
+#ifdef SMP
+		if (wakeup_cpus != 0)
+			acpi_wakeup_cpus(sc, wakeup_cpus);
+#endif
+		acpi_resync_clock(sc);
+		ret = 0;
+	}
+
+out:
+#ifdef SMP
+	if (wakeup_cpus != 0)
+		restart_cpus(wakeup_cpus);
+#endif
+
+	load_cr3(cr3);
+	intr_resume();
+	intr_restore(rf);
+
+	AcpiSetFirmwareWakingVector(0);
+
+	if (ret == 0 && mem_range_softc.mr_op != NULL &&
+	    mem_range_softc.mr_op->reinit != NULL)
+		mem_range_softc.mr_op->reinit(&mem_range_softc);
+
+	/* If we beeped, turn it off after a delay. */
+	if (acpi_resume_beep)
+		timeout(acpi_stop_beep, NULL, 3 * hz);
+
+	return (ret);
 }
 
+static vm_offset_t	acpi_wakeaddr;
+
+static void
+acpi_alloc_wakeup_handler(void)
+{
+	void		*wakeaddr;
+
+	if (!cold)
+		return;
+
+	/*
+	 * Specify the region for our wakeup code.  We want it in the low 1 MB
+	 * region, excluding video memory and above (0xa0000).  We ask for
+	 * it to be page-aligned, just to be safe.
+	 */
+	wakeaddr = contigmalloc(4 * PAGE_SIZE, M_DEVBUF, M_NOWAIT, 0, 0x9ffff,
+	    PAGE_SIZE, 0ul);
+	if (wakeaddr == NULL) {
+		printf("%s: can't alloc wake memory\n", __func__);
+		return;
+	}
+	stopxpcbs = malloc(mp_ncpus * sizeof(*stopxpcbs), M_DEVBUF, M_NOWAIT);
+	if (stopxpcbs == NULL) {
+		contigfree(wakeaddr, 4 * PAGE_SIZE, M_DEVBUF);
+		printf("%s: can't alloc CPU state memory\n", __func__);
+		return;
+	}
+	acpi_wakeaddr = (vm_offset_t)wakeaddr;
+}
+
+SYSINIT(acpiwakeup, SI_SUB_KMEM, SI_ORDER_ANY, acpi_alloc_wakeup_handler, 0);
+
 void
 acpi_install_wakeup_handler(struct acpi_softc *sc)
 {
+	uint64_t	*pt4, *pt3, *pt2;
+	int		i;
+
+	if (acpi_wakeaddr == 0ul)
+		return;
+
+	sc->acpi_wakeaddr = acpi_wakeaddr;
+	sc->acpi_wakephys = vtophys(acpi_wakeaddr);
+
+	bcopy(wakecode, (void *)WAKECODE_VADDR(sc), sizeof(wakecode));
+
+	/* Patch GDT base address, ljmp targets and page table base address. */
+	WAKECODE_FIXUP((bootgdtdesc + 2), uint32_t,
+	    WAKECODE_PADDR(sc) + bootgdt);
+	WAKECODE_FIXUP((wakeup_sw32 + 2), uint32_t,
+	    WAKECODE_PADDR(sc) + wakeup_32);
+	WAKECODE_FIXUP((wakeup_sw64 + 1), uint32_t,
+	    WAKECODE_PADDR(sc) + wakeup_64);
+	WAKECODE_FIXUP(wakeup_pagetables, uint32_t, sc->acpi_wakephys);
+
+	/* Save pointers to some global data. */
+	WAKECODE_FIXUP(wakeup_retaddr, void *, acpi_restorecpu);
+	WAKECODE_FIXUP(wakeup_kpml4, uint64_t, KPML4phys);
+	WAKECODE_FIXUP(wakeup_ctx, vm_offset_t,
+	    WAKECODE_VADDR(sc) + wakeup_ctx);
+	WAKECODE_FIXUP(wakeup_efer, uint64_t, rdmsr(MSR_EFER));
+	WAKECODE_FIXUP(wakeup_pat, uint64_t, rdmsr(MSR_PAT));
+	WAKECODE_FIXUP(wakeup_star, uint64_t, rdmsr(MSR_STAR));
+	WAKECODE_FIXUP(wakeup_lstar, uint64_t, rdmsr(MSR_LSTAR));
+	WAKECODE_FIXUP(wakeup_cstar, uint64_t, rdmsr(MSR_CSTAR));
+	WAKECODE_FIXUP(wakeup_sfmask, uint64_t, rdmsr(MSR_SF_MASK));
+
+	/* Build temporary page tables below realmode code. */
+	pt4 = (uint64_t *)acpi_wakeaddr;
+	pt3 = pt4 + (PAGE_SIZE) / sizeof(uint64_t);
+	pt2 = pt3 + (PAGE_SIZE) / sizeof(uint64_t);
+
+	/* Create the initial 1GB replicated page tables */
+	for (i = 0; i < 512; i++) {
+		/*
+		 * Each slot of the level 4 pages points
+		 * to the same level 3 page
+		 */
+		pt4[i] = (uint64_t)(sc->acpi_wakephys + PAGE_SIZE);
+		pt4[i] |= PG_V | PG_RW | PG_U;
+
+		/*
+		 * Each slot of the level 3 pages points
+		 * to the same level 2 page
+		 */
+		pt3[i] = (uint64_t)(sc->acpi_wakephys + (2 * PAGE_SIZE));
+		pt3[i] |= PG_V | PG_RW | PG_U;
+
+		/* The level 2 page slots are mapped with 2MB pages for 1GB. */
+		pt2[i] = i * (2 * 1024 * 1024);
+		pt2[i] |= PG_V | PG_RW | PG_PS | PG_U;
+	}
+
+	if (bootverbose)
+		device_printf(sc->acpi_dev, "wakeup code va %p pa %p\n",
+		    (void *)sc->acpi_wakeaddr, (void *)sc->acpi_wakephys);
 }
diff --git a/sys/amd64/acpica/genwakecode.sh b/sys/amd64/acpica/genwakecode.sh
new file mode 100755
index 0000000..c9d0077
--- /dev/null
+++ b/sys/amd64/acpica/genwakecode.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+# $FreeBSD$
+#
+file2c -sx 'static char wakecode[] = {' '};' <acpi_wakecode.bin
+
+exit 0
diff --git a/sys/amd64/acpica/genwakedata.sh b/sys/amd64/acpica/genwakedata.sh
new file mode 100755
index 0000000..6d4181e
--- /dev/null
+++ b/sys/amd64/acpica/genwakedata.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+# $FreeBSD$
+#
+nm -n --defined-only acpi_wakecode.o | while read offset dummy what
+do
+    echo "#define ${what}	0x${offset}"
+done
+
+exit 0
diff --git a/sys/amd64/amd64/amd64_mem.c b/sys/amd64/amd64/amd64_mem.c
index 2b5a73d..d7959fd 100644
--- a/sys/amd64/amd64/amd64_mem.c
+++ b/sys/amd64/amd64/amd64_mem.c
@@ -73,11 +73,13 @@ static void	amd64_mrinit(struct mem_range_softc *sc);
 static int	amd64_mrset(struct mem_range_softc *sc,
 		    struct mem_range_desc *mrd, int *arg);
 static void	amd64_mrAPinit(struct mem_range_softc *sc);
+static void	amd64_mrreinit(struct mem_range_softc *sc);
 
 static struct mem_range_ops amd64_mrops = {
 	amd64_mrinit,
 	amd64_mrset,
-	amd64_mrAPinit
+	amd64_mrAPinit,
+	amd64_mrreinit
 };
 
 /* XXX for AP startup hook */
@@ -668,6 +670,30 @@ amd64_mrAPinit(struct mem_range_softc *sc)
 	wrmsr(MSR_MTRRdefType, mtrrdef);
 }
 
+/*
+ * Re-initialise running CPU(s) MTRRs to match the ranges in the descriptor
+ * list.
+ *
+ * XXX Must be called with interrupts enabled.
+ */
+static void
+amd64_mrreinit(struct mem_range_softc *sc)
+{
+#ifdef SMP
+	/*
+	 * We should use ipi_all_but_self() to call other CPUs into a
+	 * locking gate, then call a target function to do this work.
+	 * The "proper" solution involves a generalised locking gate
+	 * implementation, not ready yet.
+	 */
+	smp_rendezvous(NULL, (void *)amd64_mrAPinit, NULL, sc);
+#else
+	disable_intr();				/* disable interrupts */
+	amd64_mrAPinit(sc);
+	enable_intr();
+#endif
+}
+
 static void
 amd64_mem_drvinit(void *unused)
 {
diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S
index 14a6f87..cebafc8 100644
--- a/sys/amd64/amd64/apic_vector.S
+++ b/sys/amd64/amd64/apic_vector.S
@@ -219,6 +219,20 @@ IDTVEC(cpustop)
 	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
 
 	call	cpustop_handler
+	jmp	doreti
+
+/*
+ * Executed by a CPU when it receives an IPI_SUSPEND from another CPU.
+ */
+	.text
+	SUPERALIGN_TEXT
+IDTVEC(cpususpend)
+	PUSH_FRAME
+
+	movq	lapic, %rax
+	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
+
+	call	cpususpend_handler
 
 	POP_FRAME
 	iretq
@@ -235,6 +249,5 @@ IDTVEC(rendezvous)
 	call	smp_rendezvous_action
 	movq	lapic, %rax
 	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
-	POP_FRAME			/* Why not doreti? */
-	iretq
+	jmp	doreti
 #endif /* SMP */
diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S
index bc1a4bb..6fc8290 100644
--- a/sys/amd64/amd64/cpu_switch.S
+++ b/sys/amd64/amd64/cpu_switch.S
@@ -75,8 +75,6 @@ ENTRY(cpu_throw)
 1:
 	movq	TD_PCB(%rdi),%r8		/* Old pcb */
 	movl	PCPU(CPUID), %eax
-	movq	PCB_FSBASE(%r8),%r9
-	movq	PCB_GSBASE(%r8),%r10
 	/* release bit from old pm_active */
 	movq	TD_PROC(%rdi), %rdx		/* oldtd->td_proc */
 	movq	P_VMSPACE(%rdx), %rdx		/* proc->p_vmspace */
@@ -110,28 +108,6 @@ ENTRY(cpu_switch)
 	movq	%rbx,PCB_RBX(%r8)
 	movq	%rax,PCB_RIP(%r8)
 
-	/*
-	 * Reread fs and gs bases. Explicit fs segment register load
-	 * by the usermode code may change actual fs base without
-	 * updating pcb_{fs,gs}base.
-	 *
-	 * %rdx still contains the mtx, save %rdx around rdmsr.
-	 */
-	movq	%rdx,%r11
-	movl	$MSR_FSBASE,%ecx
-	rdmsr
-	shlq	$32,%rdx
-	leaq	(%rax,%rdx),%r9
-	movl	$MSR_KGSBASE,%ecx
-	rdmsr
-	shlq	$32,%rdx
-	leaq	(%rax,%rdx),%r10
-	movq	%r11,%rdx
-
-	testl	$PCB_32BIT,PCB_FLAGS(%r8)
-	jnz	store_seg
-done_store_seg:
-
 	testl	$PCB_DBREGS,PCB_FLAGS(%r8)
 	jnz	store_dr			/* static predict not taken */
 done_store_dr:
@@ -192,36 +168,47 @@ sw1:
 	testl	$TDP_KTHREAD,TD_PFLAGS(%rsi)
 	jnz	do_kthread
 
-	testl	$PCB_32BIT,PCB_FLAGS(%r8)
-	jnz	load_seg
-done_load_seg:
+	/*
+	 * Load ldt register
+	 */
+	movq	TD_PROC(%rsi),%rcx
+	cmpq	$0, P_MD+MD_LDT(%rcx)
+	jne	do_ldt
+	xorl	%eax,%eax
+ld_ldt:	lldt	%ax
 
-	cmpq	PCB_FSBASE(%r8),%r9
-	jz	1f
-	/* Restore userland %fs */
-restore_fsbase:
-	movl	$MSR_FSBASE,%ecx
+	/* Restore fs base in GDT */
 	movl	PCB_FSBASE(%r8),%eax
-	movl	PCB_FSBASE+4(%r8),%edx
-	wrmsr
-1:
-	cmpq	PCB_GSBASE(%r8),%r10
-	jz	2f
-	/* Restore userland %gs */
-	movl	$MSR_KGSBASE,%ecx
+	movq	PCPU(FS32P),%rdx
+	movw	%ax,2(%rdx)
+	shrl	$16,%eax
+	movb	%al,4(%rdx)
+	shrl	$8,%eax
+	movb	%al,7(%rdx)
+
+	/* Restore gs base in GDT */
 	movl	PCB_GSBASE(%r8),%eax
-	movl	PCB_GSBASE+4(%r8),%edx
-	wrmsr
-2:
+	movq	PCPU(GS32P),%rdx
+	movw	%ax,2(%rdx)
+	shrl	$16,%eax
+	movb	%al,4(%rdx)
+	shrl	$8,%eax
+	movb	%al,7(%rdx)
 
-do_tss:
+do_kthread:
+	/* Do we need to reload tss ? */
+	movq	PCPU(TSSP),%rax
+	movq	PCB_TSSP(%r8),%rdx
+	testq	%rdx,%rdx
+	cmovzq	PCPU(COMMONTSSP),%rdx
+	cmpq	%rax,%rdx
+	jne	do_tss
+done_tss:
+	movq	%r8,PCPU(RSP0)
+	movq	%r8,PCPU(CURPCB)
 	/* Update the TSS_RSP0 pointer for the next interrupt */
-	movq	PCPU(TSSP), %rax
-	movq	%r8, PCPU(RSP0)
-	movq	%r8, PCPU(CURPCB)
-	addq	$COMMON_TSS_RSP0, %rax
-	movq	%rsi, PCPU(CURTHREAD)		/* into next thread */
-	movq	%r8, (%rax)
+	movq	%r8,COMMON_TSS_RSP0(%rdx)
+	movq	%rsi,PCPU(CURTHREAD)		/* into next thread */
 
 	/* Test if debug registers should be restored. */
 	testl	$PCB_DBREGS,PCB_FLAGS(%r8)
@@ -250,45 +237,6 @@ done_load_dr:
 	 * We use jumps rather than call in order to avoid the stack.
 	 */
 
-do_kthread:
-	/*
-	 * Copy old fs/gsbase to new kthread pcb for future switches
-	 * This maintains curpcb->pcb_[fg]sbase as caches of the MSR
-	 */
-	movq	%r9,PCB_FSBASE(%r8)
-	movq	%r10,PCB_GSBASE(%r8)
-	jmp	do_tss
-
-store_seg:
-	mov	%gs,PCB_GS(%r8)
-	testl	$PCB_GS32BIT,PCB_FLAGS(%r8)
-	jnz	2f
-1:	mov	%ds,PCB_DS(%r8)
-	mov	%es,PCB_ES(%r8)
-	mov	%fs,PCB_FS(%r8)
-	jmp	done_store_seg
-2:	movq	PCPU(GS32P),%rax
-	movq	(%rax),%rax
-	movq	%rax,PCB_GS32SD(%r8)
-	jmp	1b
-
-load_seg:
-	testl	$PCB_GS32BIT,PCB_FLAGS(%r8)
-	jnz	2f
-1:	movl	$MSR_GSBASE,%ecx
-	rdmsr
-	mov	PCB_GS(%r8),%gs
-	wrmsr
-	mov	PCB_DS(%r8),%ds
-	mov	PCB_ES(%r8),%es
-	mov	PCB_FS(%r8),%fs
-	jmp	restore_fsbase
-	/* Restore userland %gs while preserving kernel gsbase */
-2:	movq	PCPU(GS32P),%rax
-	movq	PCB_GS32SD(%r8),%rcx
-	movq	%rcx,(%rax)
-	jmp	1b
-
 store_dr:
 	movq	%dr7,%rax			/* yes, do the save */
 	movq	%dr0,%r15
@@ -326,8 +274,30 @@ load_dr:
 	movq	%rax,%dr7
 	jmp	done_load_dr
 
+do_tss:	movq	%rdx,PCPU(TSSP)
+	movq	%rdx,%rcx
+	movq	PCPU(TSS),%rax
+	movw	%rcx,2(%rax)
+	shrq	$16,%rcx
+	movb	%cl,4(%rax)
+	shrq	$8,%rcx
+	movb	%cl,7(%rax)
+	shrq	$8,%rcx
+	movl	%ecx,8(%rax)
+	movb	$0x89,5(%rax)	/* unset busy */
+	movl	$TSSSEL,%eax
+	ltr	%ax
+	jmp	done_tss
+
+do_ldt:	movq	PCPU(LDT),%rax
+	movq	P_MD+MD_LDT_SD(%rcx),%rdx
+	movq	%rdx,(%rax)
+	movq	P_MD+MD_LDT_SD+8(%rcx),%rdx
+	movq	%rdx,8(%rax)
+	movl	$LDTSEL,%eax
+	jmp	ld_ldt
 END(cpu_switch)
-	
+
 /*
  * savectx(pcb)
  * Update pcb, saving current processor state.
@@ -386,3 +356,68 @@ ENTRY(savectx)
 
 	ret
 END(savectx)
+
+/*
+ * savectx2(xpcb)
+ * Update xpcb, saving current processor state.
+ */
+ENTRY(savectx2)
+	/* Fetch XPCB. */
+	movq	%rdi,%r8
+
+	/* Save caller's return address. */
+	movq	(%rsp),%rax
+	movq	%rax,PCB_RIP(%r8)
+
+	movq	%rbx,PCB_RBX(%r8)
+	movq	%rsp,PCB_RSP(%r8)
+	movq	%rbp,PCB_RBP(%r8)
+	movq	%r12,PCB_R12(%r8)
+	movq	%r13,PCB_R13(%r8)
+	movq	%r14,PCB_R14(%r8)
+	movq	%r15,PCB_R15(%r8)
+
+	movq	%cr0,%rax
+	movq	%rax,XPCB_CR0(%r8)
+	movq	%cr2,%rax
+	movq	%rax,XPCB_CR2(%r8)
+	movq	%cr4,%rax
+	movq	%rax,XPCB_CR4(%r8)
+
+	movq	%dr0,%rax
+	movq	%rax,PCB_DR0(%r8)
+	movq	%dr1,%rax
+	movq	%rax,PCB_DR1(%r8)
+	movq	%dr2,%rax
+	movq	%rax,PCB_DR2(%r8)
+	movq	%dr3,%rax
+	movq	%rax,PCB_DR3(%r8)
+	movq	%dr6,%rax
+	movq	%rax,PCB_DR6(%r8)
+	movq	%dr7,%rax
+	movq	%rax,PCB_DR7(%r8)
+
+	sgdt	XPCB_GDT(%r8)
+	sidt	XPCB_IDT(%r8)
+	sldt	XPCB_LDT(%r8)
+	str	XPCB_TR(%r8)
+
+	movl	$MSR_FSBASE,%ecx
+	rdmsr
+	shlq	$32,%rdx
+	leaq	(%rax,%rdx),%rax
+	movq	%rax,PCB_FSBASE(%r8)
+	movl	$MSR_GSBASE,%ecx
+	rdmsr
+	shlq	$32,%rdx
+	leaq	(%rax,%rdx),%rax
+	movq	%rax,PCB_GSBASE(%r8)
+	movl	$MSR_KGSBASE,%ecx
+	rdmsr
+	shlq	$32,%rdx
+	leaq	(%rax,%rdx),%rax
+	movq	%rax,XPCB_KGSBASE(%r8)
+
+	movl	$1, %eax
+	ret
+END(savectx2)
diff --git a/sys/amd64/amd64/db_interface.c b/sys/amd64/amd64/db_interface.c
index b297616..287c236 100644
--- a/sys/amd64/amd64/db_interface.c
+++ b/sys/amd64/amd64/db_interface.c
@@ -139,7 +139,11 @@ void
 db_show_mdpcpu(struct pcpu *pc)
 {
 
-#if 0
-	db_printf("currentldt   = 0x%x\n", pc->pc_currentldt);
-#endif
+	db_printf("curpmap		= %p\n", pc->pc_curpmap);
+	db_printf("tssp		= %p\n", pc->pc_tssp);
+	db_printf("commontssp	= %p\n", pc->pc_commontssp);
+	db_printf("rsp0		= 0x%lx\n", pc->pc_rsp0);
+	db_printf("gs32p		= %p\n", pc->pc_gs32p);
+	db_printf("ldt		= %p\n", pc->pc_ldt);
+	db_printf("tss		= %p\n", pc->pc_tss);
 }
diff --git a/sys/amd64/amd64/db_trace.c b/sys/amd64/amd64/db_trace.c
index 9676963..73ffac5 100644
--- a/sys/amd64/amd64/db_trace.c
+++ b/sys/amd64/amd64/db_trace.c
@@ -69,12 +69,10 @@ static db_varfcn_t db_ss;
 #define	DB_OFFSET(x)	(db_expr_t *)offsetof(struct trapframe, x)
 struct db_variable db_regs[] = {
 	{ "cs",		DB_OFFSET(tf_cs),	db_frame },
-#if 0
 	{ "ds",		DB_OFFSET(tf_ds),	db_frame },
 	{ "es",		DB_OFFSET(tf_es),	db_frame },
 	{ "fs",		DB_OFFSET(tf_fs),	db_frame },
 	{ "gs",		DB_OFFSET(tf_gs),	db_frame },
-#endif
 	{ "ss",		NULL,			db_ss },
 	{ "rax",	DB_OFFSET(tf_rax),	db_frame },
 	{ "rcx",        DB_OFFSET(tf_rcx),	db_frame },
@@ -94,7 +92,7 @@ struct db_variable db_regs[] = {
 	{ "r15",	DB_OFFSET(tf_r15),	db_frame },
 	{ "rip",	DB_OFFSET(tf_rip),	db_frame },
 	{ "rflags",	DB_OFFSET(tf_rflags),	db_frame },
-#define	DB_N_SHOW_REGS	20	/* Don't show registers after here. */
+#define	DB_N_SHOW_REGS	24	/* Don't show registers after here. */
 	{ "dr0",	NULL,			db_dr0 },
 	{ "dr1",	NULL,			db_dr1 },
 	{ "dr2",	NULL,			db_dr2 },
@@ -316,6 +314,7 @@ db_nextframe(struct amd64_frame **fp, db_addr_t *ip, struct thread *td)
 		    strcmp(name, "Xtimerint") == 0 ||
 		    strcmp(name, "Xipi_intr_bitmap_handler") == 0 ||
 		    strcmp(name, "Xcpustop") == 0 ||
+		    strcmp(name, "Xcpususpend") == 0 ||
 		    strcmp(name, "Xrendezvous") == 0)
 			frame_type = INTERRUPT;
 		else if (strcmp(name, "Xfast_syscall") == 0)
@@ -327,6 +326,7 @@ db_nextframe(struct amd64_frame **fp, db_addr_t *ip, struct thread *td)
 		/* XXX: These are interrupts with trap frames. */
 		else if (strcmp(name, "Xtimerint") == 0 ||
 		    strcmp(name, "Xcpustop") == 0 ||
+		    strcmp(name, "Xcpususpend") == 0 ||
 		    strcmp(name, "Xrendezvous") == 0 ||
 		    strcmp(name, "Xipi_intr_bitmap_handler") == 0)
 			frame_type = TRAP_INTERRUPT;
@@ -355,7 +355,7 @@ db_nextframe(struct amd64_frame **fp, db_addr_t *ip, struct thread *td)
 		rbp = tf->tf_rbp;
 		switch (frame_type) {
 		case TRAP:
-			db_printf("--- trap %#lr", tf->tf_trapno);
+			db_printf("--- trap %#r", tf->tf_trapno);
 			break;
 		case SYSCALL:
 			db_printf("--- syscall");
diff --git a/sys/amd64/amd64/elf_machdep.c b/sys/amd64/amd64/elf_machdep.c
index 4f6d178..c5e19cf 100644
--- a/sys/amd64/amd64/elf_machdep.c
+++ b/sys/amd64/amd64/elf_machdep.c
@@ -84,7 +84,8 @@ static Elf64_Brandinfo freebsd_brand_info = {
 	.interp_path	= "/libexec/ld-elf.so.1",
 	.sysvec		= &elf64_freebsd_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &elf64_freebsd_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 SYSINIT(elf64, SI_SUB_EXEC, SI_ORDER_ANY,
@@ -99,7 +100,8 @@ static Elf64_Brandinfo freebsd_brand_oinfo = {
 	.interp_path	= "/usr/libexec/ld-elf.so.1",
 	.sysvec		= &elf64_freebsd_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &elf64_freebsd_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 SYSINIT(oelf64, SI_SUB_EXEC, SI_ORDER_ANY,
diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S
index 897bfec..daa5c25 100644
--- a/sys/amd64/amd64/exception.S
+++ b/sys/amd64/amd64/exception.S
@@ -42,6 +42,7 @@
 #include <machine/asmacros.h>
 #include <machine/psl.h>
 #include <machine/trap.h>
+#include <machine/specialreg.h>
 
 #include "assym.s"
 
@@ -99,7 +100,7 @@ MCOUNT_LABEL(btrap)
 /* Traps that we leave interrupts disabled for.. */
 #define	TRAP_NOEN(a)	\
 	subq $TF_RIP,%rsp; \
-	movq $(a),TF_TRAPNO(%rsp) ; \
+	movl $(a),TF_TRAPNO(%rsp) ; \
 	movq $0,TF_ADDR(%rsp) ; \
 	movq $0,TF_ERR(%rsp) ; \
 	jmp alltraps_noen
@@ -111,7 +112,7 @@ IDTVEC(bpt)
 /* Regular traps; The cpu does not supply tf_err for these. */
 #define	TRAP(a)	 \
 	subq $TF_RIP,%rsp; \
-	movq $(a),TF_TRAPNO(%rsp) ; \
+	movl $(a),TF_TRAPNO(%rsp) ; \
 	movq $0,TF_ADDR(%rsp) ; \
 	movq $0,TF_ERR(%rsp) ; \
 	jmp alltraps
@@ -139,7 +140,7 @@ IDTVEC(xmm)
 /* This group of traps have tf_err already pushed by the cpu */
 #define	TRAP_ERR(a)	\
 	subq $TF_ERR,%rsp; \
-	movq $(a),TF_TRAPNO(%rsp) ; \
+	movl $(a),TF_TRAPNO(%rsp) ; \
 	movq $0,TF_ADDR(%rsp) ; \
 	jmp alltraps
 IDTVEC(tss)
@@ -164,6 +165,10 @@ alltraps:
 	testb	$SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
 	jz	alltraps_testi		/* already running with kernel GS.base */
 	swapgs
+	movw	%fs,TF_FS(%rsp)
+	movw	%gs,TF_GS(%rsp)
+	movw	%es,TF_ES(%rsp)
+	movw	%ds,TF_DS(%rsp)
 alltraps_testi:
 	testl	$PSL_I,TF_RFLAGS(%rsp)
 	jz	alltraps_pushregs
@@ -185,6 +190,7 @@ alltraps_pushregs_no_rdi:
 	movq	%r13,TF_R13(%rsp)
 	movq	%r14,TF_R14(%rsp)
 	movq	%r15,TF_R15(%rsp)
+	movl	$TF_HASSEGS,TF_FLAGS(%rsp)
 	FAKE_MCOUNT(TF_RIP(%rsp))
 #ifdef KDTRACE_HOOKS
 	/*
@@ -193,7 +199,7 @@ alltraps_pushregs_no_rdi:
 	 * interrupt. For all other trap types, just handle them in
 	 * the usual way.
 	 */
-	cmpq	$T_BPTFLT,TF_TRAPNO(%rsp)
+	cmpl	$T_BPTFLT,TF_TRAPNO(%rsp)
 	jne	calltrap
 
 	/* Check if there is no DTrace hook registered. */
@@ -228,13 +234,17 @@ calltrap:
 	.type	alltraps_noen,@function
 alltraps_noen:
 	testb	$SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
-	jz	alltraps_pushregs	/* already running with kernel GS.base */
+	jz	1f	/* already running with kernel GS.base */
 	swapgs
+1:	movw	%fs,TF_FS(%rsp)
+	movw	%gs,TF_GS(%rsp)
+	movw	%es,TF_ES(%rsp)
+	movw	%ds,TF_DS(%rsp)
 	jmp	alltraps_pushregs
 
 IDTVEC(dblfault)
 	subq	$TF_ERR,%rsp
-	movq	$T_DOUBLEFLT,TF_TRAPNO(%rsp)
+	movl	$T_DOUBLEFLT,TF_TRAPNO(%rsp)
 	movq	$0,TF_ADDR(%rsp)
 	movq	$0,TF_ERR(%rsp)
 	movq	%rdi,TF_RDI(%rsp)
@@ -252,6 +262,11 @@ IDTVEC(dblfault)
 	movq	%r13,TF_R13(%rsp)
 	movq	%r14,TF_R14(%rsp)
 	movq	%r15,TF_R15(%rsp)
+	movw	%fs,TF_FS(%rsp)
+	movw	%gs,TF_GS(%rsp)
+	movw	%es,TF_ES(%rsp)
+	movw	%ds,TF_DS(%rsp)
+	movl	$TF_HASSEGS,TF_FLAGS(%rsp)
 	testb	$SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
 	jz	1f			/* already running with kernel GS.base */
 	swapgs
@@ -262,7 +277,7 @@ IDTVEC(dblfault)
 
 IDTVEC(page)
 	subq	$TF_ERR,%rsp
-	movq	$T_PAGEFLT,TF_TRAPNO(%rsp)
+	movl	$T_PAGEFLT,TF_TRAPNO(%rsp)
 	testb	$SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
 	jz	1f			/* already running with kernel GS.base */
 	swapgs
@@ -270,6 +285,10 @@ IDTVEC(page)
 	movq	%rdi,TF_RDI(%rsp)	/* free up a GP register */
 	movq	%cr2,%rdi		/* preserve %cr2 before ..  */
 	movq	%rdi,TF_ADDR(%rsp)	/* enabling interrupts. */
+	movw	%fs,TF_FS(%rsp)
+	movw	%gs,TF_GS(%rsp)
+	movw	%es,TF_ES(%rsp)
+	movw	%ds,TF_DS(%rsp)
 	testl	$PSL_I,TF_RFLAGS(%rsp)
 	jz	alltraps_pushregs_no_rdi
 	sti
@@ -283,17 +302,19 @@ IDTVEC(page)
 	 */
 IDTVEC(prot)
 	subq	$TF_ERR,%rsp
-	movq	$T_PROTFLT,TF_TRAPNO(%rsp)
+	movl	$T_PROTFLT,TF_TRAPNO(%rsp)
 	movq	$0,TF_ADDR(%rsp)
 	movq	%rdi,TF_RDI(%rsp)	/* free up a GP register */
 	leaq	doreti_iret(%rip),%rdi
 	cmpq	%rdi,TF_RIP(%rsp)
-	je	2f			/* kernel but with user gsbase!! */
+	je	1f			/* kernel but with user gsbase!! */
 	testb	$SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */
-	jz	1f			/* already running with kernel GS.base */
-2:
-	swapgs
-1:
+	jz	2f			/* already running with kernel GS.base */
+1:	swapgs
+2:	movw	%fs,TF_FS(%rsp)
+	movw	%gs,TF_GS(%rsp)
+	movw	%es,TF_ES(%rsp)
+	movw	%ds,TF_DS(%rsp)
 	testl	$PSL_I,TF_RFLAGS(%rsp)
 	jz	alltraps_pushregs_no_rdi
 	sti
@@ -316,6 +337,10 @@ IDTVEC(fast_syscall)
 	movq	%rcx,TF_RIP(%rsp)	/* %rcx original value is in %r10 */
 	movq	PCPU(SCRATCH_RSP),%r11	/* %r11 already saved */
 	movq	%r11,TF_RSP(%rsp)	/* user stack pointer */
+	movw	%fs,TF_FS(%rsp)
+	movw	%gs,TF_GS(%rsp)
+	movw	%es,TF_ES(%rsp)
+	movw	%ds,TF_DS(%rsp)
 	sti
 	movq	$KUDSEL,TF_SS(%rsp)
 	movq	$KUCSEL,TF_CS(%rsp)
@@ -333,40 +358,11 @@ IDTVEC(fast_syscall)
 	movq	%r13,TF_R13(%rsp)	/* C preserved */
 	movq	%r14,TF_R14(%rsp)	/* C preserved */
 	movq	%r15,TF_R15(%rsp)	/* C preserved */
+	movl	$TF_HASSEGS,TF_FLAGS(%rsp)
 	FAKE_MCOUNT(TF_RIP(%rsp))
 	movq	%rsp, %rdi
 	call	syscall
 	movq	PCPU(CURPCB),%rax
-	testq	$PCB_FULLCTX,PCB_FLAGS(%rax)
-	jne	3f
-1:	/* Check for and handle AST's on return to userland */
-	cli
-	movq	PCPU(CURTHREAD),%rax
-	testl	$TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
-	je	2f
-	sti
-	movq	%rsp, %rdi
-	call	ast
-	jmp	1b
-2:	/* restore preserved registers */
-	MEXITCOUNT
-	movq	TF_RDI(%rsp),%rdi	/* bonus; preserve arg 1 */
-	movq	TF_RSI(%rsp),%rsi	/* bonus: preserve arg 2 */
-	movq	TF_RDX(%rsp),%rdx	/* return value 2 */
-	movq	TF_RAX(%rsp),%rax	/* return value 1 */
-	movq	TF_RBX(%rsp),%rbx	/* C preserved */
-	movq	TF_RBP(%rsp),%rbp	/* C preserved */
-	movq	TF_R12(%rsp),%r12	/* C preserved */
-	movq	TF_R13(%rsp),%r13	/* C preserved */
-	movq	TF_R14(%rsp),%r14	/* C preserved */
-	movq	TF_R15(%rsp),%r15	/* C preserved */
-	movq	TF_RFLAGS(%rsp),%r11	/* original %rflags */
-	movq	TF_RIP(%rsp),%rcx	/* original %rip */
-	movq	TF_RSP(%rsp),%r9	/* user stack pointer */
-	movq	%r9,%rsp		/* original %rsp */
-	swapgs
-	sysretq
-3:	/* Requested full context restore, use doreti for that */
 	andq	$~PCB_FULLCTX,PCB_FLAGS(%rax)
 	MEXITCOUNT
 	jmp	doreti
@@ -405,7 +401,7 @@ IDTVEC(fast_syscall32)
 
 IDTVEC(nmi)
 	subq	$TF_RIP,%rsp
-	movq	$(T_NMI),TF_TRAPNO(%rsp)
+	movl	$(T_NMI),TF_TRAPNO(%rsp)
 	movq	$0,TF_ADDR(%rsp)
 	movq	$0,TF_ERR(%rsp)
 	movq	%rdi,TF_RDI(%rsp)
@@ -423,6 +419,11 @@ IDTVEC(nmi)
 	movq	%r13,TF_R13(%rsp)
 	movq	%r14,TF_R14(%rsp)
 	movq	%r15,TF_R15(%rsp)
+	movw	%fs,TF_FS(%rsp)
+	movw	%gs,TF_GS(%rsp)
+	movw	%es,TF_ES(%rsp)
+	movw	%ds,TF_DS(%rsp)
+	movl	$TF_HASSEGS,TF_FLAGS(%rsp)
 	xorl	%ebx,%ebx
 	testb	$SEL_RPL_MASK,TF_CS(%rsp)
 	jnz	nmi_fromuserspace
@@ -515,9 +516,7 @@ outofnmi:
 nocallchain:
 #endif
 	testl	%ebx,%ebx
-	jz	nmi_kernelexit
-	swapgs
-	jmp	nmi_restoreregs
+	jnz	doreti_exit
 nmi_kernelexit:	
 	/*
 	 * Put back the preserved MSR_GSBASE value.
@@ -633,7 +632,55 @@ doreti_ast:
 	 */
 doreti_exit:
 	MEXITCOUNT
-	movq	TF_RDI(%rsp),%rdi
+	movq	PCPU(CURTHREAD),%r8
+	movq	TD_PCB(%r8),%r8
+
+	/*
+	 * Do not reload segment registers for kernel.
+	 * Since we do not reload segments registers with sane
+	 * values on kernel entry, descriptors referenced by
+	 * segments registers may be not valid. This is fatal
+	 * for the usermode, but is innocent for the kernel.
+	 */
+	testb	$SEL_RPL_MASK,TF_CS(%rsp)
+	jz	ld_regs
+
+	testl	$TF_HASSEGS,TF_FLAGS(%rsp)
+	je	set_segs
+
+do_segs:
+	/* Restore %fs and fsbase */
+	movw	TF_FS(%rsp),%ax
+	.globl	ld_fs
+ld_fs:	movw	%ax,%fs
+	cmpw	$KUF32SEL,%ax
+	jne	1f
+	movl	$MSR_FSBASE,%ecx
+	movl	PCB_FSBASE(%r8),%eax
+	movl	PCB_FSBASE+4(%r8),%edx
+	wrmsr
+1:
+	/* Restore %gs and gsbase */
+	movw	TF_GS(%rsp),%si
+	pushfq
+	cli
+	movl	$MSR_GSBASE,%ecx
+	rdmsr
+	.globl	ld_gs
+ld_gs:	movw	%si,%gs
+	wrmsr
+	popfq
+	cmpw	$KUG32SEL,%si
+	jne	1f
+	movl	$MSR_KGSBASE,%ecx
+	movl	PCB_GSBASE(%r8),%eax
+	movl	PCB_GSBASE+4(%r8),%edx
+	wrmsr
+1:	.globl	ld_es
+ld_es:	movw	TF_ES(%rsp),%es
+	.globl	ld_ds
+ld_ds:	movw	TF_DS(%rsp),%ds
+ld_regs:movq	TF_RDI(%rsp),%rdi
 	movq	TF_RSI(%rsp),%rsi
 	movq	TF_RDX(%rsp),%rdx
 	movq	TF_RCX(%rsp),%rcx
@@ -657,6 +704,14 @@ doreti_exit:
 doreti_iret:
 	iretq
 
+set_segs:
+	movw	$KUDSEL,%ax
+	movw	%ax,TF_DS(%rsp)
+	movw	%ax,TF_ES(%rsp)
+	movw	$KUF32SEL,TF_FS(%rsp)
+	movw	$KUG32SEL,TF_GS(%rsp)
+	jmp	do_segs
+
 	/*
 	 * doreti_iret_fault.  Alternative return code for
 	 * the case where we get a fault in the doreti_exit code
@@ -671,7 +726,12 @@ doreti_iret_fault:
 	testl	$PSL_I,TF_RFLAGS(%rsp)
 	jz	1f
 	sti
-1:	movq	%rdi,TF_RDI(%rsp)
+1:	movw	%fs,TF_FS(%rsp)
+	movw	%gs,TF_GS(%rsp)
+	movw	%es,TF_ES(%rsp)
+	movw	%ds,TF_DS(%rsp)
+	movl	$TF_HASSEGS,TF_FLAGS(%rsp)
+	movq	%rdi,TF_RDI(%rsp)
 	movq	%rsi,TF_RSI(%rsp)
 	movq	%rdx,TF_RDX(%rsp)
 	movq	%rcx,TF_RCX(%rsp)
@@ -686,11 +746,48 @@ doreti_iret_fault:
 	movq	%r13,TF_R13(%rsp)
 	movq	%r14,TF_R14(%rsp)
 	movq	%r15,TF_R15(%rsp)
-	movq	$T_PROTFLT,TF_TRAPNO(%rsp)
+	movl	$T_PROTFLT,TF_TRAPNO(%rsp)
 	movq	$0,TF_ERR(%rsp)	/* XXX should be the error code */
 	movq	$0,TF_ADDR(%rsp)
 	FAKE_MCOUNT(TF_RIP(%rsp))
 	jmp	calltrap
+
+	ALIGN_TEXT
+	.globl	ds_load_fault
+ds_load_fault:
+	movl	$T_PROTFLT,TF_TRAPNO(%rsp)
+	movzwl	TF_DS(%rsp),%edx
+	movl	%edx,TF_ERR(%rsp)
+	movw	$KUDSEL,TF_DS(%rsp)
+	jmp	calltrap
+
+	ALIGN_TEXT
+	.globl	es_load_fault
+es_load_fault:
+	movl	$T_PROTFLT,TF_TRAPNO(%rsp)
+	movzwl	TF_ES(%rsp),%edx
+	movl	%edx,TF_ERR(%rsp)
+	movw	$KUDSEL,TF_ES(%rsp)
+	jmp	calltrap
+
+	ALIGN_TEXT
+	.globl	fs_load_fault
+fs_load_fault:
+	movl	$T_PROTFLT,TF_TRAPNO(%rsp)
+	movzwl	TF_FS(%rsp),%edx
+	movl	%edx,TF_ERR(%rsp)
+	movw	$KUF32SEL,TF_FS(%rsp)
+	jmp	calltrap
+
+	ALIGN_TEXT
+	.globl	gs_load_fault
+gs_load_fault:
+	popfq
+	movl	$T_PROTFLT,TF_TRAPNO(%rsp)
+	movzwl	TF_GS(%rsp),%edx
+	movl	%edx,TF_ERR(%rsp)
+	movw	$KUG32SEL,TF_GS(%rsp)
+	jmp	calltrap
 #ifdef HWPMC_HOOKS
 	ENTRY(end_exceptions)
 #endif
diff --git a/sys/amd64/amd64/fpu.c b/sys/amd64/amd64/fpu.c
index 8f16c46..10cb6c2 100644
--- a/sys/amd64/amd64/fpu.c
+++ b/sys/amd64/amd64/fpu.c
@@ -101,7 +101,7 @@ static	void	fpu_clean_state(void);
 SYSCTL_INT(_hw, HW_FLOATINGPT, floatingpoint, CTLFLAG_RD,
     NULL, 1, "Floating point instructions executed in hardware");
 
-static	struct savefpu		fpu_cleanstate;
+static	struct savefpu		fpu_initialstate;
 
 /*
  * Initialize the floating point unit.  On the boot CPU we generate a
@@ -123,13 +123,13 @@ fpuinit(void)
 	mxcsr = __INITIAL_MXCSR__;
 	ldmxcsr(mxcsr);
 	if (PCPU_GET(cpuid) == 0) {
-		fxsave(&fpu_cleanstate);
-		if (fpu_cleanstate.sv_env.en_mxcsr_mask)
-			cpu_mxcsr_mask = fpu_cleanstate.sv_env.en_mxcsr_mask;
+		fxsave(&fpu_initialstate);
+		if (fpu_initialstate.sv_env.en_mxcsr_mask)
+			cpu_mxcsr_mask = fpu_initialstate.sv_env.en_mxcsr_mask;
 		else
 			cpu_mxcsr_mask = 0xFFBF;
-		bzero(fpu_cleanstate.sv_fp, sizeof(fpu_cleanstate.sv_fp));
-		bzero(fpu_cleanstate.sv_xmm, sizeof(fpu_cleanstate.sv_xmm));
+		bzero(fpu_initialstate.sv_fp, sizeof(fpu_initialstate.sv_fp));
+		bzero(fpu_initialstate.sv_xmm, sizeof(fpu_initialstate.sv_xmm));
 	}
 	start_emulating();
 	intr_restore(savecrit);
@@ -416,10 +416,11 @@ fpudna(void)
 
 	if ((pcb->pcb_flags & PCB_FPUINITDONE) == 0) {
 		/*
-		 * This is the first time this thread has used the FPU,
-		 * explicitly load sanitized registers.
+		 * This is the first time this thread has used the FPU or
+		 * the PCB doesn't contain a clean FPU state.  Explicitly
+		 * load an initial state.
 		 */
-		fxrstor(&fpu_cleanstate);
+		fxrstor(&fpu_initialstate);
 		if (pcb->pcb_initial_fpucw != __INITIAL_FPUCW__)
 			fldcw(&pcb->pcb_initial_fpucw);
 		pcb->pcb_flags |= PCB_FPUINITDONE;
@@ -453,7 +454,7 @@ fpugetregs(struct thread *td, struct savefpu *addr)
 	register_t s;
 
 	if ((td->td_pcb->pcb_flags & PCB_FPUINITDONE) == 0) {
-		bcopy(&fpu_cleanstate, addr, sizeof(fpu_cleanstate));
+		bcopy(&fpu_initialstate, addr, sizeof(fpu_initialstate));
 		addr->sv_env.en_cw = td->td_pcb->pcb_initial_fpucw;
 		return (_MC_FPOWNED_NONE);
 	}
@@ -479,7 +480,6 @@ fpusetregs(struct thread *td, struct savefpu *addr)
 
 	s = intr_disable();
 	if (td == PCPU_GET(fpcurthread)) {
-		fpu_clean_state();
 		fxrstor(addr);
 		intr_restore(s);
 	} else {
@@ -498,10 +498,10 @@ fpusetregs(struct thread *td, struct savefpu *addr)
  * In order to avoid leaking this information across processes, we clean
  * these values by performing a dummy load before executing fxrstor().
  */
-static	double	dummy_variable = 0.0;
 static void
 fpu_clean_state(void)
 {
+	static float dummy_variable = 0.0;
 	u_short status;
 
 	/*
diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c
index 1924be7..8bf1162 100644
--- a/sys/amd64/amd64/genassym.c
+++ b/sys/amd64/amd64/genassym.c
@@ -72,13 +72,16 @@ __FBSDID("$FreeBSD$");
 #include <machine/pcb.h>
 #include <machine/sigframe.h>
 #include <machine/proc.h>
-#include <machine/specialreg.h>
 #include <machine/segments.h>
 
 ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace));
 ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap));
 ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));
 
+ASSYM(P_MD, offsetof(struct proc, p_md));
+ASSYM(MD_LDT, offsetof(struct mdproc, md_ldt));
+ASSYM(MD_LDT_SD, offsetof(struct mdproc, md_ldt_sd));
+
 ASSYM(TD_LOCK, offsetof(struct thread, td_lock));
 ASSYM(TD_FLAGS, offsetof(struct thread, td_flags));
 ASSYM(TD_PCB, offsetof(struct thread, td_pcb));
@@ -132,16 +135,13 @@ ASSYM(PCB_RBX, offsetof(struct pcb, pcb_rbx));
 ASSYM(PCB_RIP, offsetof(struct pcb, pcb_rip));
 ASSYM(PCB_FSBASE, offsetof(struct pcb, pcb_fsbase));
 ASSYM(PCB_GSBASE, offsetof(struct pcb, pcb_gsbase));
-ASSYM(PCB_DS, offsetof(struct pcb, pcb_ds));
-ASSYM(PCB_ES, offsetof(struct pcb, pcb_es));
-ASSYM(PCB_FS, offsetof(struct pcb, pcb_fs));
-ASSYM(PCB_GS, offsetof(struct pcb, pcb_gs));
 ASSYM(PCB_DR0, offsetof(struct pcb, pcb_dr0));
 ASSYM(PCB_DR1, offsetof(struct pcb, pcb_dr1));
 ASSYM(PCB_DR2, offsetof(struct pcb, pcb_dr2));
 ASSYM(PCB_DR3, offsetof(struct pcb, pcb_dr3));
 ASSYM(PCB_DR6, offsetof(struct pcb, pcb_dr6));
 ASSYM(PCB_DR7, offsetof(struct pcb, pcb_dr7));
+ASSYM(PCB_TSSP, offsetof(struct pcb, pcb_tssp));
 ASSYM(PCB_DBREGS, PCB_DBREGS);
 ASSYM(PCB_32BIT, PCB_32BIT);
 ASSYM(PCB_GS32BIT, PCB_GS32BIT);
@@ -155,6 +155,18 @@ ASSYM(PCB_GS32SD, offsetof(struct pcb, pcb_gs32sd));
 
 ASSYM(PCB_SIZE, sizeof(struct pcb));
 
+ASSYM(XPCB_PCB, offsetof(struct xpcb, xpcb_pcb));
+ASSYM(XPCB_CR0, offsetof(struct xpcb, xpcb_cr0));
+ASSYM(XPCB_CR2, offsetof(struct xpcb, xpcb_cr2));
+ASSYM(XPCB_CR4, offsetof(struct xpcb, xpcb_cr4));
+ASSYM(XPCB_KGSBASE, offsetof(struct xpcb, xpcb_kgsbase));
+ASSYM(XPCB_GDT, offsetof(struct xpcb, xpcb_gdt));
+ASSYM(XPCB_IDT, offsetof(struct xpcb, xpcb_idt));
+ASSYM(XPCB_LDT, offsetof(struct xpcb, xpcb_ldt));
+ASSYM(XPCB_TR, offsetof(struct xpcb, xpcb_tr));
+
+ASSYM(XPCB_SIZE, sizeof(struct xpcb));
+
 ASSYM(COMMON_TSS_RSP0, offsetof(struct amd64tss, tss_rsp0));
 
 ASSYM(TF_R15, offsetof(struct trapframe, tf_r15));
@@ -180,7 +192,13 @@ ASSYM(TF_CS, offsetof(struct trapframe, tf_cs));
 ASSYM(TF_RFLAGS, offsetof(struct trapframe, tf_rflags));
 ASSYM(TF_RSP, offsetof(struct trapframe, tf_rsp));
 ASSYM(TF_SS, offsetof(struct trapframe, tf_ss));
+ASSYM(TF_DS, offsetof(struct trapframe, tf_ds));
+ASSYM(TF_ES, offsetof(struct trapframe, tf_es));
+ASSYM(TF_FS, offsetof(struct trapframe, tf_fs));
+ASSYM(TF_GS, offsetof(struct trapframe, tf_gs));
+ASSYM(TF_FLAGS, offsetof(struct trapframe, tf_flags));
 ASSYM(TF_SIZE, sizeof(struct trapframe));
+ASSYM(TF_HASSEGS, TF_HASSEGS);
 
 ASSYM(SIGF_HANDLER, offsetof(struct sigframe, sf_ahu.sf_handler));
 ASSYM(SIGF_UC, offsetof(struct sigframe, sf_uc));
@@ -202,7 +220,11 @@ ASSYM(PC_SCRATCH_RSP, offsetof(struct pcpu, pc_scratch_rsp));
 ASSYM(PC_CURPMAP, offsetof(struct pcpu, pc_curpmap));
 ASSYM(PC_TSSP, offsetof(struct pcpu, pc_tssp));
 ASSYM(PC_RSP0, offsetof(struct pcpu, pc_rsp0));
+ASSYM(PC_FS32P, offsetof(struct pcpu, pc_fs32p));
 ASSYM(PC_GS32P, offsetof(struct pcpu, pc_gs32p));
+ASSYM(PC_LDT, offsetof(struct pcpu, pc_ldt));
+ASSYM(PC_COMMONTSSP, offsetof(struct pcpu, pc_commontssp));
+ASSYM(PC_TSS, offsetof(struct pcpu, pc_tss));
  
 ASSYM(LA_VER, offsetof(struct LAPIC, version));
 ASSYM(LA_TPR, offsetof(struct LAPIC, tpr));
@@ -217,10 +239,12 @@ ASSYM(KDSEL, GSEL(GDATA_SEL, SEL_KPL));
 ASSYM(KUCSEL, GSEL(GUCODE_SEL, SEL_UPL));
 ASSYM(KUDSEL, GSEL(GUDATA_SEL, SEL_UPL));
 ASSYM(KUC32SEL, GSEL(GUCODE32_SEL, SEL_UPL));
+ASSYM(KUF32SEL, GSEL(GUFS32_SEL, SEL_UPL));
+ASSYM(KUG32SEL, GSEL(GUGS32_SEL, SEL_UPL));
+ASSYM(TSSSEL, GSEL(GPROC0_SEL, SEL_KPL));
+ASSYM(LDTSEL, GSEL(GUSERLDT_SEL, SEL_KPL));
 ASSYM(SEL_RPL_MASK, SEL_RPL_MASK);
 
-ASSYM(MSR_GSBASE, MSR_GSBASE);
-
 #ifdef	HWPMC_HOOKS
 ASSYM(PMC_FN_USER_CALLCHAIN, PMC_FN_USER_CALLCHAIN);
 #endif
diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index 3a2b170..4a5871a 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -159,7 +159,7 @@ extern vm_offset_t ksym_start, ksym_end;
 #define ICH_PMBASE	0x400
 #define ICH_SMI_EN	ICH_PMBASE + 0x30
 
-int	_udatasel, _ucodesel, _ucode32sel;
+int	_udatasel, _ucodesel, _ucode32sel, _ufssel, _ugssel;
 
 int cold = 1;
 
@@ -192,10 +192,13 @@ struct mtx icu_lock;
 
 struct mem_range_softc mem_range_softc;
 
+struct mtx dt_lock;	/* lock for GDT and LDT */
+
 static void
 cpu_startup(dummy)
 	void *dummy;
 {
+	uintmax_t memsize;
 	char *sysenv;
 
 	/*
@@ -226,8 +229,18 @@ cpu_startup(dummy)
 #ifdef PERFMON
 	perfmon_init();
 #endif
-	printf("usable memory = %ju (%ju MB)\n", ptoa((uintmax_t)physmem),
-	    ptoa((uintmax_t)physmem) / 1048576);
+	sysenv = getenv("smbios.memory.enabled");
+	if (sysenv != NULL) {
+		memsize = (uintmax_t)strtoul(sysenv, (char **)NULL, 10);
+		freeenv(sysenv);
+	} else
+		memsize = 0;
+	if (memsize > 0)
+		printf("real memory  = %ju (%ju MB)\n", memsize << 10,
+		    memsize >> 10);
+	else
+		printf("real memory  = %ju (%ju MB)\n", ptoa((uintmax_t)Maxmem),
+		    ptoa((uintmax_t)Maxmem) / 1048576);
 	realmem = Maxmem;
 	/*
 	 * Display any holes after the first chunk of extended memory.
@@ -250,7 +263,7 @@ cpu_startup(dummy)
 
 	vm_ksubmap_init(&kmi);
 
-	printf("avail memory  = %ju (%ju MB)\n",
+	printf("avail memory = %ju (%ju MB)\n",
 	    ptoa((uintmax_t)cnt.v_free_count),
 	    ptoa((uintmax_t)cnt.v_free_count) / 1048576);
 
@@ -267,7 +280,7 @@ cpu_startup(dummy)
  * Send an interrupt to process.
  *
  * Stack is set up to allow sigcode stored
- * at top to call routine, followed by kcall
+ * at top to call routine, followed by call
  * to sigreturn routine below.  After sigreturn
  * resets the signal mask, the stack, and the
  * frame pointer, it returns to the user
@@ -305,6 +318,8 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	sf.sf_uc.uc_mcontext.mc_len = sizeof(sf.sf_uc.uc_mcontext); /* magic */
 	get_fpcontext(td, &sf.sf_uc.uc_mcontext);
 	fpstate_drop(td);
+	sf.sf_uc.uc_mcontext.mc_fsbase = td->td_pcb->pcb_fsbase;
+	sf.sf_uc.uc_mcontext.mc_gsbase = td->td_pcb->pcb_gsbase;
 
 	/* Allocate space for the signal handler context. */
 	if ((td->td_pflags & TDP_ALTSTACK) != 0 && !oonstack &&
@@ -359,6 +374,11 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	regs->tf_rip = PS_STRINGS - *(p->p_sysent->sv_szsigcode);
 	regs->tf_rflags &= ~(PSL_T | PSL_D);
 	regs->tf_cs = _ucodesel;
+	regs->tf_ds = _udatasel;
+	regs->tf_es = _udatasel;
+	regs->tf_fs = _ufssel;
+	regs->tf_gs = _ugssel;
+	regs->tf_flags = TF_HASSEGS;
 	PROC_LOCK(p);
 	mtx_lock(&psp->ps_mtx);
 }
@@ -390,9 +410,16 @@ sigreturn(td, uap)
 	ksiginfo_t ksi;
 
 	error = copyin(uap->sigcntxp, &uc, sizeof(uc));
-	if (error != 0)
+	if (error != 0) {
+		printf("sigreturn (pid %d): copyin failed\n", p->p_pid);
 		return (error);
+	}
 	ucp = &uc;
+	if ((ucp->uc_mcontext.mc_flags & ~_MC_FLAG_MASK) != 0) {
+		printf("sigreturn (pid %d): mc_flags %x\n", p->p_pid,
+		    ucp->uc_mcontext.mc_flags);
+		return (EINVAL);
+	}
 	regs = td->td_frame;
 	rflags = ucp->uc_mcontext.mc_rflags;
 	/*
@@ -409,7 +436,8 @@ sigreturn(td, uap)
 	 * one less debugger trap, so allowing it is fairly harmless.
 	 */
 	if (!EFL_SECURE(rflags & ~PSL_RF, regs->tf_rflags & ~PSL_RF)) {
-		printf("sigreturn: rflags = 0x%lx\n", rflags);
+		printf("sigreturn (pid %d): rflags = 0x%lx\n", p->p_pid,
+		    rflags);
 		return (EINVAL);
 	}
 
@@ -420,7 +448,7 @@ sigreturn(td, uap)
 	 */
 	cs = ucp->uc_mcontext.mc_cs;
 	if (!CS_SECURE(cs)) {
-		printf("sigreturn: cs = 0x%x\n", cs);
+		printf("sigreturn (pid %d): cs = 0x%x\n", p->p_pid, cs);
 		ksiginfo_init_trap(&ksi);
 		ksi.ksi_signo = SIGBUS;
 		ksi.ksi_code = BUS_OBJERR;
@@ -431,9 +459,13 @@ sigreturn(td, uap)
 	}
 
 	ret = set_fpcontext(td, &ucp->uc_mcontext);
-	if (ret != 0)
+	if (ret != 0) {
+		printf("sigreturn (pid %d): set_fpcontext\n", p->p_pid);
 		return (ret);
+	}
 	bcopy(&ucp->uc_mcontext.mc_rdi, regs, sizeof(*regs));
+	td->td_pcb->pcb_fsbase = ucp->uc_mcontext.mc_fsbase;
+	td->td_pcb->pcb_gsbase = ucp->uc_mcontext.mc_gsbase;
 
 	PROC_LOCK(p);
 #if defined(COMPAT_43)
@@ -727,22 +759,16 @@ exec_setregs(td, entry, stack, ps_strings)
 {
 	struct trapframe *regs = td->td_frame;
 	struct pcb *pcb = td->td_pcb;
+
+	mtx_lock(&dt_lock);
+	if (td->td_proc->p_md.md_ldt != NULL)
+		user_ldt_free(td);
+	else
+		mtx_unlock(&dt_lock);
 	
-	critical_enter();
-	wrmsr(MSR_FSBASE, 0);
-	wrmsr(MSR_KGSBASE, 0);	/* User value while we're in the kernel */
 	pcb->pcb_fsbase = 0;
 	pcb->pcb_gsbase = 0;
-	critical_exit();
 	pcb->pcb_flags &= ~(PCB_32BIT | PCB_GS32BIT);
-	load_ds(_udatasel);
-	load_es(_udatasel);
-	load_fs(_udatasel);
-	load_gs(_udatasel);
-	pcb->pcb_ds = _udatasel;
-	pcb->pcb_es = _udatasel;
-	pcb->pcb_fs = _udatasel;
-	pcb->pcb_gs = _udatasel;
 	pcb->pcb_initial_fpucw = __INITIAL_FPUCW__;
 
 	bzero((char *)regs, sizeof(struct trapframe));
@@ -752,6 +778,11 @@ exec_setregs(td, entry, stack, ps_strings)
 	regs->tf_rflags = PSL_USER | (regs->tf_rflags & PSL_T);
 	regs->tf_ss = _udatasel;
 	regs->tf_cs = _ucodesel;
+	regs->tf_ds = _udatasel;
+	regs->tf_es = _udatasel;
+	regs->tf_fs = _ufssel;
+	regs->tf_gs = _ugssel;
+	regs->tf_flags = TF_HASSEGS;
 
 	/*
 	 * Reset the hardware debug registers if they were in use.
@@ -815,90 +846,130 @@ CTASSERT(sizeof(struct nmi_pcpu) == 16);
 
 struct amd64tss common_tss[MAXCPU];
 
-/* software prototypes -- in more palatable form */
+/*
+ * Software prototypes -- in more palatable form.
+ *
+ * Keep GUFS32, GUGS32, GUCODE32 and GUDATA at the same
+ * slots as corresponding segments for i386 kernel.
+ */
 struct soft_segment_descriptor gdt_segs[] = {
 /* GNULL_SEL	0 Null Descriptor */
-{	0x0,			/* segment base address  */
-	0x0,			/* length */
-	0,			/* segment type */
-	0,			/* segment descriptor priority level */
-	0,			/* segment descriptor present */
-	0,			/* long */
-	0,			/* default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
-/* GCODE_SEL	1 Code Descriptor for kernel */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMERA,		/* segment type */
-	SEL_KPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	1,			/* long */
-	0,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
-/* GDATA_SEL	2 Data Descriptor for kernel */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMRWA,		/* segment type */
-	SEL_KPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	1,			/* long */
-	0,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
-/* GUCODE32_SEL	3 32 bit Code Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMERA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0,			/* long */
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
-/* GUDATA_SEL	4 32/64 bit Data Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMRWA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0,			/* long */
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
-/* GUCODE_SEL	5 64 bit Code Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMERA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	1,			/* long */
-	0,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
-/* GPROC0_SEL	6 Proc 0 Tss Descriptor */
-{
-	0x0,			/* segment base address */
-	sizeof(struct amd64tss)-1,/* length */
-	SDT_SYSTSS,		/* segment type */
-	SEL_KPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0,			/* long */
-	0,			/* unused - default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_long = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
+/* GNULL2_SEL	1 Null Descriptor */
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_long = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
+/* GUFS32_SEL	2 32 bit %gs Descriptor for user */
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_long = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
+/* GUGS32_SEL	3 32 bit %fs Descriptor for user */
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_long = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
+/* GCODE_SEL	4 Code Descriptor for kernel */
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMERA,
+	.ssd_dpl = SEL_KPL,
+	.ssd_p = 1,
+	.ssd_long = 1,
+	.ssd_def32 = 0,
+	.ssd_gran = 1		},
+/* GDATA_SEL	5 Data Descriptor for kernel */
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = SEL_KPL,
+	.ssd_p = 1,
+	.ssd_long = 1,
+	.ssd_def32 = 0,
+	.ssd_gran = 1		},
+/* GUCODE32_SEL	6 32 bit Code Descriptor for user */
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMERA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_long = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
+/* GUDATA_SEL	7 32/64 bit Data Descriptor for user */
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_long = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
+/* GUCODE_SEL	8 64 bit Code Descriptor for user */
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMERA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_long = 1,
+	.ssd_def32 = 0,
+	.ssd_gran = 1		},
+/* GPROC0_SEL	9 Proc 0 Tss Descriptor */
+{	.ssd_base = 0x0,
+	.ssd_limit = sizeof(struct amd64tss) + IOPAGES * PAGE_SIZE - 1,
+	.ssd_type = SDT_SYSTSS,
+	.ssd_dpl = SEL_KPL,
+	.ssd_p = 1,
+	.ssd_long = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 /* Actually, the TSS is a system descriptor which is double size */
-{	0x0,			/* segment base address  */
-	0x0,			/* length */
-	0,			/* segment type */
-	0,			/* segment descriptor priority level */
-	0,			/* segment descriptor present */
-	0,			/* long */
-	0,			/* default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
-/* GUGS32_SEL	8 32 bit GS Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMRWA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0,			/* long */
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_long = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
+/* GUSERLDT_SEL	11 LDT Descriptor */
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_long = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
+/* GUSERLDT_SEL	12 LDT Descriptor, double size */
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_long = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 };
 
 void
@@ -1329,12 +1400,12 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	/*
 	 * make gdt memory segments
 	 */
-	gdt_segs[GPROC0_SEL].ssd_base = (uintptr_t)&common_tss[0];
-
 	for (x = 0; x < NGDT; x++) {
-		if (x != GPROC0_SEL && x != (GPROC0_SEL + 1))
+		if (x != GPROC0_SEL && x != (GPROC0_SEL + 1) &&
+		    x != GUSERLDT_SEL && x != (GUSERLDT_SEL) + 1)
 			ssdtosd(&gdt_segs[x], &gdt[x]);
 	}
+	gdt_segs[GPROC0_SEL].ssd_base = (uintptr_t)&common_tss[0];
 	ssdtosyssd(&gdt_segs[GPROC0_SEL],
 	    (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
 
@@ -1352,6 +1423,10 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	PCPU_SET(curthread, &thread0);
 	PCPU_SET(curpcb, thread0.td_pcb);
 	PCPU_SET(tssp, &common_tss[0]);
+	PCPU_SET(commontssp, &common_tss[0]);
+	PCPU_SET(tss, (struct system_segment_descriptor *)&gdt[GPROC0_SEL]);
+	PCPU_SET(ldt, (struct system_segment_descriptor *)&gdt[GUSERLDT_SEL]);
+	PCPU_SET(fs32p, &gdt[GUFS32_SEL]);
 	PCPU_SET(gs32p, &gdt[GUGS32_SEL]);
 
 	/*
@@ -1364,6 +1439,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	 */
 	mutex_init();
 	mtx_init(&icu_lock, "icu", NULL, MTX_SPIN | MTX_NOWITNESS);
+	mtx_init(&dt_lock, "descriptor tables", NULL, MTX_DEF);
 
 	/* exceptions */
 	for (x = 0; x < NIDT; x++)
@@ -1452,7 +1528,8 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	common_tss[0].tss_ist2 = (long) np;
 
 	/* Set the IO permission bitmap (empty due to tss seg limit) */
-	common_tss[0].tss_iobase = sizeof(struct amd64tss);
+	common_tss[0].tss_iobase = sizeof(struct amd64tss) +
+	    IOPAGES * PAGE_SIZE;
 
 	gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
 	ltr(gsel_tss);
@@ -1480,10 +1557,12 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	_ucodesel = GSEL(GUCODE_SEL, SEL_UPL);
 	_udatasel = GSEL(GUDATA_SEL, SEL_UPL);
 	_ucode32sel = GSEL(GUCODE32_SEL, SEL_UPL);
+	_ufssel = GSEL(GUFS32_SEL, SEL_UPL);
+	_ugssel = GSEL(GUGS32_SEL, SEL_UPL);
 
 	load_ds(_udatasel);
 	load_es(_udatasel);
-	load_fs(_udatasel);
+	load_fs(_ufssel);
 
 	/* setup proc 0's pcb */
 	thread0.td_pcb->pcb_flags = 0;
@@ -1605,6 +1684,17 @@ fill_regs(struct thread *td, struct reg *regs)
 	regs->r_rflags = tp->tf_rflags;
 	regs->r_rsp = tp->tf_rsp;
 	regs->r_ss = tp->tf_ss;
+	if (tp->tf_flags & TF_HASSEGS) {
+		regs->r_ds = tp->tf_ds;
+		regs->r_es = tp->tf_es;
+		regs->r_fs = tp->tf_fs;
+		regs->r_gs = tp->tf_gs;
+	} else {
+		regs->r_ds = 0;
+		regs->r_es = 0;
+		regs->r_fs = 0;
+		regs->r_gs = 0;
+	}
 	return (0);
 }
 
@@ -1638,6 +1728,13 @@ set_regs(struct thread *td, struct reg *regs)
 	tp->tf_rflags = rflags;
 	tp->tf_rsp = regs->r_rsp;
 	tp->tf_ss = regs->r_ss;
+	if (0) {	/* XXXKIB */
+		tp->tf_ds = regs->r_ds;
+		tp->tf_es = regs->r_es;
+		tp->tf_fs = regs->r_fs;
+		tp->tf_gs = regs->r_gs;
+		tp->tf_flags = TF_HASSEGS;
+	}
 	td->td_pcb->pcb_flags |= PCB_FULLCTX;
 	return (0);
 }
@@ -1757,8 +1854,15 @@ get_mcontext(struct thread *td, mcontext_t *mcp, int flags)
 	mcp->mc_cs = tp->tf_cs;
 	mcp->mc_rsp = tp->tf_rsp;
 	mcp->mc_ss = tp->tf_ss;
+	mcp->mc_ds = tp->tf_ds;
+	mcp->mc_es = tp->tf_es;
+	mcp->mc_fs = tp->tf_fs;
+	mcp->mc_gs = tp->tf_gs;
+	mcp->mc_flags = tp->tf_flags;
 	mcp->mc_len = sizeof(*mcp);
 	get_fpcontext(td, mcp);
+	mcp->mc_fsbase = td->td_pcb->pcb_fsbase;
+	mcp->mc_gsbase = td->td_pcb->pcb_gsbase;
 	return (0);
 }
 
@@ -1776,7 +1880,8 @@ set_mcontext(struct thread *td, const mcontext_t *mcp)
 	int ret;
 
 	tp = td->td_frame;
-	if (mcp->mc_len != sizeof(*mcp))
+	if (mcp->mc_len != sizeof(*mcp) ||
+	    (mcp->mc_flags & ~_MC_FLAG_MASK) != 0)
 		return (EINVAL);
 	rflags = (mcp->mc_rflags & PSL_USERCHANGE) |
 	    (tp->tf_rflags & ~PSL_USERCHANGE);
@@ -1802,6 +1907,17 @@ set_mcontext(struct thread *td, const mcontext_t *mcp)
 	tp->tf_rflags = rflags;
 	tp->tf_rsp = mcp->mc_rsp;
 	tp->tf_ss = mcp->mc_ss;
+	tp->tf_flags = mcp->mc_flags;
+	if (tp->tf_flags & TF_HASSEGS) {
+		tp->tf_ds = mcp->mc_ds;
+		tp->tf_es = mcp->mc_es;
+		tp->tf_fs = mcp->mc_fs;
+		tp->tf_gs = mcp->mc_gs;
+	}
+	if (mcp->mc_flags & _MC_HASBASES) {
+		td->td_pcb->pcb_fsbase = mcp->mc_fsbase;
+		td->td_pcb->pcb_gsbase = mcp->mc_gsbase;
+	}
 	td->td_pcb->pcb_flags |= PCB_FULLCTX;
 	return (0);
 }
@@ -2062,45 +2178,24 @@ user_dbreg_trap(void)
 #ifdef KDB
 
 /*
- * Provide inb() and outb() as functions.  They are normally only
- * available as macros calling inlined functions, thus cannot be
- * called from the debugger.
- *
- * The actual code is stolen from <machine/cpufunc.h>, and de-inlined.
+ * Provide inb() and outb() as functions.  They are normally only available as
+ * inline functions, thus cannot be called from the debugger.
  */
 
-#undef inb
-#undef outb
-
 /* silence compiler warnings */
-u_char inb(u_int);
-void outb(u_int, u_char);
+u_char inb_(u_short);
+void outb_(u_short, u_char);
 
 u_char
-inb(u_int port)
+inb_(u_short port)
 {
-	u_char	data;
-	/*
-	 * We use %%dx and not %1 here because i/o is done at %dx and not at
-	 * %edx, while gcc generates inferior code (movw instead of movl)
-	 * if we tell it to load (u_short) port.
-	 */
-	__asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port));
-	return (data);
+	return inb(port);
 }
 
 void
-outb(u_int port, u_char data)
+outb_(u_short port, u_char data)
 {
-	u_char	al;
-	/*
-	 * Use an unnecessary assignment to help gcc's register allocator.
-	 * This make a large difference for gcc-1.40 and a tiny difference
-	 * for gcc-2.6.0.  For gcc-1.40, al had to be ``asm("ax")'' for
-	 * best results.  gcc-2.6.0 can't handle this.
-	 */
-	al = data;
-	__asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port));
+	outb(port, data);
 }
 
 #endif /* KDB */
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index c65f6f8..59e3e9b 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -58,6 +58,7 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/apicreg.h>
 #include <machine/cputypes.h>
+#include <machine/cpufunc.h>
 #include <machine/md_var.h>
 #include <machine/mp_watchdog.h>
 #include <machine/pcb.h>
@@ -100,9 +101,8 @@ extern pt_entry_t *KPTphys;
 /* SMP page table page */
 extern pt_entry_t *SMPpt;
 
-extern int  _udatasel;
-
 struct pcb stoppcbs[MAXCPU];
+struct xpcb *stopxpcbs = NULL;
 
 /* Variables needed for SMP tlb shootdown. */
 vm_offset_t smp_tlb_addr1;
@@ -344,6 +344,9 @@ cpu_mp_start(void)
 	/* Install an inter-CPU IPI for CPU stop/restart */
 	setidt(IPI_STOP, IDTVEC(cpustop), SDT_SYSIGT, SEL_KPL, 0);
 
+	/* Install an inter-CPU IPI for CPU suspend/resume */
+	setidt(IPI_SUSPEND, IDTVEC(cpususpend), SDT_SYSIGT, SEL_KPL, 0);
+
 	/* Set boot_cpu_id if needed. */
 	if (boot_cpu_id == -1) {
 		boot_cpu_id = PCPU_GET(apic_id);
@@ -458,7 +461,8 @@ init_secondary(void)
 	/* Init tss */
 	common_tss[cpu] = common_tss[0];
 	common_tss[cpu].tss_rsp0 = 0;   /* not used until after switch */
-	common_tss[cpu].tss_iobase = sizeof(struct amd64tss);
+	common_tss[cpu].tss_iobase = sizeof(struct amd64tss) +
+	    IOPAGES * PAGE_SIZE;
 	common_tss[cpu].tss_ist1 = (long)&doublefault_stack[PAGE_SIZE];
 
 	/* The NMI stack runs on IST2. */
@@ -467,12 +471,13 @@ init_secondary(void)
 
 	/* Prepare private GDT */
 	gdt_segs[GPROC0_SEL].ssd_base = (long) &common_tss[cpu];
-	ssdtosyssd(&gdt_segs[GPROC0_SEL],
-	   (struct system_segment_descriptor *)&gdt[NGDT * cpu + GPROC0_SEL]);
 	for (x = 0; x < NGDT; x++) {
-		if (x != GPROC0_SEL && x != (GPROC0_SEL + 1))
+		if (x != GPROC0_SEL && x != (GPROC0_SEL + 1) &&
+		    x != GUSERLDT_SEL && x != (GUSERLDT_SEL + 1))
 			ssdtosd(&gdt_segs[x], &gdt[NGDT * cpu + x]);
 	}
+	ssdtosyssd(&gdt_segs[GPROC0_SEL],
+	    (struct system_segment_descriptor *)&gdt[NGDT * cpu + GPROC0_SEL]);
 	ap_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
 	ap_gdt.rd_base =  (long) &gdt[NGDT * cpu];
 	lgdt(&ap_gdt);			/* does magic intra-segment return */
@@ -486,8 +491,14 @@ init_secondary(void)
 	pc->pc_prvspace = pc;
 	pc->pc_curthread = 0;
 	pc->pc_tssp = &common_tss[cpu];
+	pc->pc_commontssp = &common_tss[cpu];
 	pc->pc_rsp0 = 0;
+	pc->pc_tss = (struct system_segment_descriptor *)&gdt[NGDT * cpu +
+	    GPROC0_SEL];
+	pc->pc_fs32p = &gdt[NGDT * cpu + GUFS32_SEL];
 	pc->pc_gs32p = &gdt[NGDT * cpu + GUGS32_SEL];
+	pc->pc_ldt = (struct system_segment_descriptor *)&gdt[NGDT * cpu +
+	    GUSERLDT_SEL];
 
 	/* Save the per-cpu pointer for use by the NMI handler. */
 	np->np_pcpu = (register_t) pc;
@@ -596,7 +607,7 @@ init_secondary(void)
 	load_cr4(rcr4() | CR4_PGE);
 	load_ds(_udatasel);
 	load_es(_udatasel);
-	load_fs(_udatasel);
+	load_fs(_ufssel);
 	mtx_unlock_spin(&ap_boot_mtx);
 
 	/* wait until all the AP's are up */
@@ -1145,6 +1156,41 @@ cpustop_handler(void)
 }
 
 /*
+ * Handle an IPI_SUSPEND by saving our current context and spinning until we
+ * are resumed.
+ */
+void
+cpususpend_handler(void)
+{
+	struct savefpu *stopfpu;
+	register_t cr3, rf;
+	int cpu = PCPU_GET(cpuid);
+	int cpumask = PCPU_GET(cpumask);
+
+	rf = intr_disable();
+	cr3 = rcr3();
+	stopfpu = &stopxpcbs[cpu].xpcb_pcb.pcb_save;
+	if (savectx2(&stopxpcbs[cpu])) {
+		fpugetregs(curthread, stopfpu);
+		wbinvd();
+		atomic_set_int(&stopped_cpus, cpumask);
+	} else
+		fpusetregs(curthread, stopfpu);
+
+	/* Wait for resume */
+	while (!(started_cpus & cpumask))
+		ia32_pause();
+
+	atomic_clear_int(&started_cpus, cpumask);
+	atomic_clear_int(&stopped_cpus, cpumask);
+
+	/* Restore CR3 and enable interrupts */
+	load_cr3(cr3);
+	lapic_setup(0);
+	intr_restore(rf);
+}
+
+/*
  * This is called once the rest of the system is up and running and we're
  * ready to let the AP's out of the pen.
  */
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index f9cbad2..883b7ed 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -542,7 +542,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr)
 	 * Initialize the kernel pmap (which is statically allocated).
 	 */
 	PMAP_LOCK_INIT(kernel_pmap);
-	kernel_pmap->pm_pml4 = (pdp_entry_t *) (KERNBASE + KPML4phys);
+	kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys);
 	kernel_pmap->pm_root = NULL;
 	kernel_pmap->pm_active = -1;	/* don't allow deactivation */
 	TAILQ_INIT(&kernel_pmap->pm_pvchunk);
@@ -1351,7 +1351,7 @@ pmap_pinit0(pmap_t pmap)
 {
 
 	PMAP_LOCK_INIT(pmap);
-	pmap->pm_pml4 = (pml4_entry_t *)(KERNBASE + KPML4phys);
+	pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys);
 	pmap->pm_root = NULL;
 	pmap->pm_active = 0;
 	TAILQ_INIT(&pmap->pm_pvchunk);
@@ -1442,8 +1442,6 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags)
 	 * it isn't already there.
 	 */
 
-	pmap->pm_stats.resident_count++;
-
 	if (ptepindex >= (NUPDE + NUPDPE)) {
 		pml4_entry_t *pml4;
 		vm_pindex_t pml4index;
@@ -1469,7 +1467,8 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags)
 			if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index,
 			    flags) == NULL) {
 				--m->wire_count;
-				vm_page_free(m);
+				atomic_subtract_int(&cnt.v_wire_count, 1);
+				vm_page_free_zero(m);
 				return (NULL);
 			}
 		} else {
@@ -1501,7 +1500,8 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags)
 			if (_pmap_allocpte(pmap, NUPDE + pdpindex,
 			    flags) == NULL) {
 				--m->wire_count;
-				vm_page_free(m);
+				atomic_subtract_int(&cnt.v_wire_count, 1);
+				vm_page_free_zero(m);
 				return (NULL);
 			}
 			pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME);
@@ -1514,7 +1514,9 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags)
 				if (_pmap_allocpte(pmap, NUPDE + pdpindex,
 				    flags) == NULL) {
 					--m->wire_count;
-					vm_page_free(m);
+					atomic_subtract_int(&cnt.v_wire_count,
+					    1);
+					vm_page_free_zero(m);
 					return (NULL);
 				}
 			} else {
@@ -1530,6 +1532,8 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags)
 		*pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M;
 	}
 
+	pmap->pm_stats.resident_count++;
+
 	return m;
 }
 
@@ -2338,6 +2342,7 @@ pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
 		mpte = pmap_lookup_pt_page(pmap, sva);
 		if (mpte != NULL) {
 			pmap_remove_pt_page(pmap, mpte);
+			pmap->pm_stats.resident_count--;
 			KASSERT(mpte->wire_count == NPTEPG,
 			    ("pmap_remove_pde: pte page wire count error"));
 			mpte->wire_count = 0;
@@ -3847,6 +3852,7 @@ pmap_remove_pages(pmap_t pmap)
 					mpte = pmap_lookup_pt_page(pmap, pv->pv_va);
 					if (mpte != NULL) {
 						pmap_remove_pt_page(pmap, mpte);
+						pmap->pm_stats.resident_count--;
 						KASSERT(mpte->wire_count == NPTEPG,
 						    ("pmap_remove_pages: pte page wire count error"));
 						mpte->wire_count = 0;
@@ -4689,7 +4695,7 @@ if (oldpmap)	/* XXX FIXME */
 	oldpmap->pm_active &= ~PCPU_GET(cpumask);
 	pmap->pm_active |= PCPU_GET(cpumask);
 #endif
-	cr3 = vtophys(pmap->pm_pml4);
+	cr3 = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4);
 	td->td_pcb->pcb_cr3 = cr3;
 	load_cr3(cr3);
 	critical_exit();
diff --git a/sys/amd64/amd64/sys_machdep.c b/sys/amd64/amd64/sys_machdep.c
index 7f022d0..834dd2c 100644
--- a/sys/amd64/amd64/sys_machdep.c
+++ b/sys/amd64/amd64/sys_machdep.c
@@ -36,16 +36,39 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/priv.h>
 #include <sys/proc.h>
 #include <sys/sysproto.h>
-#include <machine/specialreg.h>
-#include <machine/sysarch.h>
-#include <machine/pcb.h>
+#include <sys/uio.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
+#include <vm/vm_kern.h>		/* for kernel_map */
+#include <vm/vm_extern.h>
+
+#include <machine/frame.h>
+#include <machine/md_var.h>
+#include <machine/pcb.h>
+#include <machine/specialreg.h>
+#include <machine/sysarch.h>
+#include <machine/tss.h>
 #include <machine/vmparam.h>
 
+#include <security/audit/audit.h>
+
+int max_ldt_segment = 1024;
+#define LD_PER_PAGE 512
+#define	NULL_LDT_BASE	((caddr_t)NULL)
+
+#ifdef notyet
+#ifdef SMP
+static void set_user_ldt_rv(struct vmspace *vmsp);
+#endif
+#endif
+static void user_ldt_derefl(struct proc_ldt *pldt);
+
 #ifndef _SYS_SYSPROTO_H_
 struct sysarch_args {
 	int op;
@@ -54,6 +77,83 @@ struct sysarch_args {
 #endif
 
 int
+sysarch_ldt(struct thread *td, struct sysarch_args *uap, int uap_space)
+{
+	struct i386_ldt_args *largs, la;
+	struct user_segment_descriptor *lp;
+	int error = 0;
+
+	/*
+	 * XXXKIB check that the BSM generation code knows to encode
+	 * the op argument.
+	 */
+	AUDIT_ARG(cmd, uap->op);
+	if (uap_space == UIO_USERSPACE) {
+		error = copyin(uap->parms, &la, sizeof(struct i386_ldt_args));
+		if (error != 0)
+			return (error);
+		largs = &la;
+	} else
+		largs = (struct i386_ldt_args *)uap->parms;
+	if (largs->num > max_ldt_segment || largs->num <= 0)
+		return (EINVAL);
+
+	switch (uap->op) {
+	case I386_GET_LDT:
+		error = amd64_get_ldt(td, largs);
+		break;
+	case I386_SET_LDT:
+		if (largs->descs != NULL) {
+			lp = (struct user_segment_descriptor *)
+			    kmem_alloc(kernel_map, largs->num *
+			    sizeof(struct user_segment_descriptor));
+			if (lp == NULL) {
+				error = ENOMEM;
+				break;
+			}
+			error = copyin(largs->descs, lp, largs->num *
+			    sizeof(struct user_segment_descriptor));
+			if (error == 0)
+				error = amd64_set_ldt(td, largs, lp);
+			kmem_free(kernel_map, (vm_offset_t)lp, largs->num *
+			    sizeof(struct user_segment_descriptor));
+		} else {
+			error = amd64_set_ldt(td, largs, NULL);
+		}
+		break;
+	}
+	return (error);
+}
+
+void
+update_gdt_gsbase(struct thread *td, uint32_t base)
+{
+	struct user_segment_descriptor *sd;
+
+	if (td != curthread)
+		return;
+	critical_enter();
+	sd = PCPU_GET(gs32p);
+	sd->sd_lobase = base & 0xffffff;
+	sd->sd_hibase = (base >> 24) & 0xff;
+	critical_exit();
+}
+
+void
+update_gdt_fsbase(struct thread *td, uint32_t base)
+{
+	struct user_segment_descriptor *sd;
+
+	if (td != curthread)
+		return;
+	critical_enter();
+	sd = PCPU_GET(fs32p);
+	sd->sd_lobase = base & 0xffffff;
+	sd->sd_hibase = (base >> 24) & 0xff;
+	critical_exit();
+}
+
+int
 sysarch(td, uap)
 	struct thread *td;
 	register struct sysarch_args *uap;
@@ -62,8 +162,36 @@ sysarch(td, uap)
 	struct pcb *pcb = curthread->td_pcb;
 	uint32_t i386base;
 	uint64_t a64base;
+	struct i386_ioperm_args iargs;
+
+	if (uap->op == I386_GET_LDT || uap->op == I386_SET_LDT)
+		return (sysarch_ldt(td, uap, UIO_USERSPACE));
+	/*
+	 * XXXKIB check that the BSM generation code knows to encode
+	 * the op argument.
+	 */
+	AUDIT_ARG(cmd, uap->op);
+	switch (uap->op) {
+	case I386_GET_IOPERM:
+	case I386_SET_IOPERM:
+		if ((error = copyin(uap->parms, &iargs,
+		    sizeof(struct i386_ioperm_args))) != 0)
+			return (error);
+		break;
+	default:
+		break;
+	}
 
-	switch(uap->op) {
+	switch (uap->op) {
+	case I386_GET_IOPERM:
+		error = amd64_get_ioperm(td, &iargs);
+		if (error == 0)
+			error = copyout(&iargs, uap->parms,
+			    sizeof(struct i386_ioperm_args));
+		break;
+	case I386_SET_IOPERM:
+		error = amd64_set_ioperm(td, &iargs);
+		break;
 	case I386_GET_FSBASE:
 		i386base = pcb->pcb_fsbase;
 		error = copyout(&i386base, uap->parms, sizeof(i386base));
@@ -71,10 +199,9 @@ sysarch(td, uap)
 	case I386_SET_FSBASE:
 		error = copyin(uap->parms, &i386base, sizeof(i386base));
 		if (!error) {
-			critical_enter();
-			wrmsr(MSR_FSBASE, i386base);
 			pcb->pcb_fsbase = i386base;
-			critical_exit();
+			td->td_frame->tf_fs = _ufssel;
+			update_gdt_fsbase(td, i386base);
 		}
 		break;
 	case I386_GET_GSBASE:
@@ -84,10 +211,9 @@ sysarch(td, uap)
 	case I386_SET_GSBASE:
 		error = copyin(uap->parms, &i386base, sizeof(i386base));
 		if (!error) {
-			critical_enter();
-			wrmsr(MSR_KGSBASE, i386base);
 			pcb->pcb_gsbase = i386base;
-			critical_exit();
+			td->td_frame->tf_gs = _ugssel;
+			update_gdt_gsbase(td, i386base);
 		}
 		break;
 	case AMD64_GET_FSBASE:
@@ -98,13 +224,10 @@ sysarch(td, uap)
 		error = copyin(uap->parms, &a64base, sizeof(a64base));
 		if (!error) {
 			if (a64base < VM_MAXUSER_ADDRESS) {
-				critical_enter();
-				wrmsr(MSR_FSBASE, a64base);
 				pcb->pcb_fsbase = a64base;
-				critical_exit();
-			} else {
+				td->td_frame->tf_fs = _ufssel;
+			} else
 				error = EINVAL;
-			}
 		}
 		break;
 
@@ -116,13 +239,10 @@ sysarch(td, uap)
 		error = copyin(uap->parms, &a64base, sizeof(a64base));
 		if (!error) {
 			if (a64base < VM_MAXUSER_ADDRESS) {
-				critical_enter();
-				wrmsr(MSR_KGSBASE, a64base);
 				pcb->pcb_gsbase = a64base;
-				critical_exit();
-			} else {
+				td->td_frame->tf_gs = _ugssel;
+			} else
 				error = EINVAL;
-			}
 		}
 		break;
 
@@ -132,3 +252,424 @@ sysarch(td, uap)
 	}
 	return (error);
 }
+
+int
+amd64_set_ioperm(td, uap)
+	struct thread *td;
+	struct i386_ioperm_args *uap;
+{
+	int i, error;
+	char *iomap;
+	struct amd64tss *tssp;
+	struct system_segment_descriptor *tss_sd;
+	u_long *addr;
+	struct pcb *pcb;
+
+	if ((error = priv_check(td, PRIV_IO)) != 0)
+		return (error);
+	if ((error = securelevel_gt(td->td_ucred, 0)) != 0)
+		return (error);
+	if (uap->start + uap->length > IOPAGES * PAGE_SIZE * NBBY)
+		return (EINVAL);
+
+	/*
+	 * XXX
+	 * While this is restricted to root, we should probably figure out
+	 * whether any other driver is using this i/o address, as so not to
+	 * cause confusion.  This probably requires a global 'usage registry'.
+	 */
+	pcb = td->td_pcb;
+	if (pcb->pcb_tssp == NULL) {
+		tssp = (struct amd64tss *)kmem_alloc(kernel_map,
+		    ctob(IOPAGES+1));
+		if (tssp == NULL)
+			return (ENOMEM);
+		iomap = (char *)&tssp[1];
+		addr = (u_long *)iomap;
+		for (i = 0; i < (ctob(IOPAGES) + 1) / sizeof(u_long); i++)
+			*addr++ = ~0;
+		critical_enter();
+		/* Takes care of tss_rsp0. */
+		memcpy(tssp, &common_tss[PCPU_GET(cpuid)],
+		    sizeof(struct amd64tss));
+		tssp->tss_iobase = sizeof(*tssp);
+		pcb->pcb_tssp = tssp;
+		tss_sd = PCPU_GET(tss);
+		tss_sd->sd_lobase = (u_long)tssp & 0xffffff;
+		tss_sd->sd_hibase = ((u_long)tssp >> 24) & 0xfffffffffful;
+		tss_sd->sd_type = SDT_SYSTSS;
+		ltr(GSEL(GPROC0_SEL, SEL_KPL));
+		PCPU_SET(tssp, tssp);
+		critical_exit();
+	} else
+		iomap = (char *)&pcb->pcb_tssp[1];
+	for (i = uap->start; i < uap->start + uap->length; i++) {
+		if (uap->enable)
+			iomap[i >> 3] &= ~(1 << (i & 7));
+		else
+			iomap[i >> 3] |= (1 << (i & 7));
+	}
+	return (error);
+}
+
+int
+amd64_get_ioperm(td, uap)
+	struct thread *td;
+	struct i386_ioperm_args *uap;
+{
+	int i, state;
+	char *iomap;
+
+	if (uap->start >= IOPAGES * PAGE_SIZE * NBBY)
+		return (EINVAL);
+	if (td->td_pcb->pcb_tssp == NULL) {
+		uap->length = 0;
+		goto done;
+	}
+
+	iomap = (char *)&td->td_pcb->pcb_tssp[1];
+
+	i = uap->start;
+	state = (iomap[i >> 3] >> (i & 7)) & 1;
+	uap->enable = !state;
+	uap->length = 1;
+
+	for (i = uap->start + 1; i < IOPAGES * PAGE_SIZE * NBBY; i++) {
+		if (state != ((iomap[i >> 3] >> (i & 7)) & 1))
+			break;
+		uap->length++;
+	}
+
+done:
+	return (0);
+}
+
+/*
+ * Update the GDT entry pointing to the LDT to point to the LDT of the
+ * current process.
+ */
+void
+set_user_ldt(struct mdproc *mdp)
+{
+
+	critical_enter();
+	*PCPU_GET(ldt) = mdp->md_ldt_sd;
+	lldt(GSEL(GUSERLDT_SEL, SEL_KPL));
+	critical_exit();
+}
+
+#ifdef notyet
+#ifdef SMP
+static void
+set_user_ldt_rv(struct vmspace *vmsp)
+{
+	struct thread *td;
+
+	td = curthread;
+	if (vmsp != td->td_proc->p_vmspace)
+		return;
+
+	set_user_ldt(&td->td_proc->p_md);
+}
+#endif
+#endif
+
+struct proc_ldt *
+user_ldt_alloc(struct proc *p, int force)
+{
+	struct proc_ldt *pldt, *new_ldt;
+	struct mdproc *mdp;
+	struct soft_segment_descriptor sldt;
+
+	mtx_assert(&dt_lock, MA_OWNED);
+	mdp = &p->p_md;
+	if (!force && mdp->md_ldt != NULL)
+		return (mdp->md_ldt);
+	mtx_unlock(&dt_lock);
+	new_ldt = malloc(sizeof(struct proc_ldt), M_SUBPROC, M_WAITOK);
+	new_ldt->ldt_base = (caddr_t)kmem_alloc(kernel_map,
+	     max_ldt_segment * sizeof(struct user_segment_descriptor));
+	if (new_ldt->ldt_base == NULL) {
+		FREE(new_ldt, M_SUBPROC);
+		mtx_lock(&dt_lock);
+		return (NULL);
+	}
+	new_ldt->ldt_refcnt = 1;
+	sldt.ssd_base = (uint64_t)new_ldt->ldt_base;
+	sldt.ssd_limit = max_ldt_segment *
+	    sizeof(struct user_segment_descriptor) - 1;
+	sldt.ssd_type = SDT_SYSLDT;
+	sldt.ssd_dpl = SEL_KPL;
+	sldt.ssd_p = 1;
+	sldt.ssd_long = 0;
+	sldt.ssd_def32 = 0;
+	sldt.ssd_gran = 0;
+	mtx_lock(&dt_lock);
+	pldt = mdp->md_ldt;
+	if (pldt != NULL && !force) {
+		kmem_free(kernel_map, (vm_offset_t)new_ldt->ldt_base,
+		    max_ldt_segment * sizeof(struct user_segment_descriptor));
+		free(new_ldt, M_SUBPROC);
+		return (pldt);
+	}
+
+	mdp->md_ldt = new_ldt;
+	if (pldt != NULL) {
+		bcopy(pldt->ldt_base, new_ldt->ldt_base, max_ldt_segment *
+		    sizeof(struct user_segment_descriptor));
+		user_ldt_derefl(pldt);
+	}
+	ssdtosyssd(&sldt, &p->p_md.md_ldt_sd);
+	if (p == curproc)
+		set_user_ldt(mdp);
+
+	return (mdp->md_ldt);
+}
+
+void
+user_ldt_free(struct thread *td)
+{
+	struct proc *p = td->td_proc;
+	struct mdproc *mdp = &p->p_md;
+	struct proc_ldt *pldt;
+
+	mtx_assert(&dt_lock, MA_OWNED);
+	if ((pldt = mdp->md_ldt) == NULL) {
+		mtx_unlock(&dt_lock);
+		return;
+	}
+
+	mdp->md_ldt = NULL;
+	bzero(&mdp->md_ldt_sd, sizeof(mdp->md_ldt_sd));
+	if (td == curthread)
+		lldt(GSEL(GNULL_SEL, SEL_KPL));
+	user_ldt_deref(pldt);
+}
+
+static void
+user_ldt_derefl(struct proc_ldt *pldt)
+{
+
+	if (--pldt->ldt_refcnt == 0) {
+		kmem_free(kernel_map, (vm_offset_t)pldt->ldt_base,
+		    max_ldt_segment * sizeof(struct user_segment_descriptor));
+		free(pldt, M_SUBPROC);
+	}
+}
+
+void
+user_ldt_deref(struct proc_ldt *pldt)
+{
+
+	mtx_assert(&dt_lock, MA_OWNED);
+	user_ldt_derefl(pldt);
+	mtx_unlock(&dt_lock);
+}
+
+/*
+ * Note for the authors of compat layers (linux, etc): copyout() in
+ * the function below is not a problem since it presents data in
+ * arch-specific format (i.e. i386-specific in this case), not in
+ * the OS-specific one.
+ */
+int
+amd64_get_ldt(td, uap)
+	struct thread *td;
+	struct i386_ldt_args *uap;
+{
+	int error = 0;
+	struct proc_ldt *pldt;
+	int num;
+	struct user_segment_descriptor *lp;
+
+#ifdef	DEBUG
+	printf("amd64_get_ldt: start=%d num=%d descs=%p\n",
+	    uap->start, uap->num, (void *)uap->descs);
+#endif
+
+	if ((pldt = td->td_proc->p_md.md_ldt) != NULL) {
+		lp = &((struct user_segment_descriptor *)(pldt->ldt_base))
+		    [uap->start];
+		num = min(uap->num, max_ldt_segment);
+	} else
+		return (EINVAL);
+
+	if ((uap->start > (unsigned int)max_ldt_segment) ||
+	    ((unsigned int)num > (unsigned int)max_ldt_segment) ||
+	    ((unsigned int)(uap->start + num) > (unsigned int)max_ldt_segment))
+		return(EINVAL);
+
+	error = copyout(lp, uap->descs, num *
+	    sizeof(struct user_segment_descriptor));
+	if (!error)
+		td->td_retval[0] = num;
+
+	return(error);
+}
+
+int
+amd64_set_ldt(td, uap, descs)
+	struct thread *td;
+	struct i386_ldt_args *uap;
+	struct user_segment_descriptor *descs;
+{
+	int error = 0, i;
+	int largest_ld;
+	struct mdproc *mdp = &td->td_proc->p_md;
+	struct proc_ldt *pldt;
+	struct user_segment_descriptor *dp;
+	struct proc *p;
+
+#ifdef	DEBUG
+	printf("amd64_set_ldt: start=%d num=%d descs=%p\n",
+	    uap->start, uap->num, (void *)uap->descs);
+#endif
+
+	p = td->td_proc;
+	if (descs == NULL) {
+		/* Free descriptors */
+		if (uap->start == 0 && uap->num == 0)
+			uap->num = max_ldt_segment;
+		if (uap->num <= 0)
+			return (EINVAL);
+		if ((pldt = mdp->md_ldt) == NULL ||
+		    uap->start >= max_ldt_segment)
+			return (0);
+		largest_ld = uap->start + uap->num;
+		if (largest_ld > max_ldt_segment)
+			largest_ld = max_ldt_segment;
+		i = largest_ld - uap->start;
+		mtx_lock(&dt_lock);
+		bzero(&((struct user_segment_descriptor *)(pldt->ldt_base))
+		    [uap->start], sizeof(struct user_segment_descriptor) * i);
+		mtx_unlock(&dt_lock);
+		return (0);
+	}
+
+	if (!(uap->start == LDT_AUTO_ALLOC && uap->num == 1)) {
+		/* verify range of descriptors to modify */
+		largest_ld = uap->start + uap->num;
+		if (uap->start >= max_ldt_segment ||
+		    uap->num < 0 || largest_ld > max_ldt_segment)
+			return (EINVAL);
+	}
+
+	/* Check descriptors for access violations */
+	for (i = 0; i < uap->num; i++) {
+		dp = &descs[i];
+
+		switch (dp->sd_type) {
+		case SDT_SYSNULL:	/* system null */
+			dp->sd_p = 0;
+			break;
+		case SDT_SYS286TSS:
+		case SDT_SYSLDT:
+		case SDT_SYS286BSY:
+		case SDT_SYS286CGT:
+		case SDT_SYSTASKGT:
+		case SDT_SYS286IGT:
+		case SDT_SYS286TGT:
+		case SDT_SYSNULL2:
+		case SDT_SYSTSS:
+		case SDT_SYSNULL3:
+		case SDT_SYSBSY:
+		case SDT_SYSCGT:
+		case SDT_SYSNULL4:
+		case SDT_SYSIGT:
+		case SDT_SYSTGT:
+			/* I can't think of any reason to allow a user proc
+			 * to create a segment of these types.  They are
+			 * for OS use only.
+			 */
+			return (EACCES);
+			/*NOTREACHED*/
+
+		/* memory segment types */
+		case SDT_MEMEC:   /* memory execute only conforming */
+		case SDT_MEMEAC:  /* memory execute only accessed conforming */
+		case SDT_MEMERC:  /* memory execute read conforming */
+		case SDT_MEMERAC: /* memory execute read accessed conforming */
+			 /* Must be "present" if executable and conforming. */
+			if (dp->sd_p == 0)
+				return (EACCES);
+			break;
+		case SDT_MEMRO:   /* memory read only */
+		case SDT_MEMROA:  /* memory read only accessed */
+		case SDT_MEMRW:   /* memory read write */
+		case SDT_MEMRWA:  /* memory read write accessed */
+		case SDT_MEMROD:  /* memory read only expand dwn limit */
+		case SDT_MEMRODA: /* memory read only expand dwn lim accessed */
+		case SDT_MEMRWD:  /* memory read write expand dwn limit */
+		case SDT_MEMRWDA: /* memory read write expand dwn lim acessed */
+		case SDT_MEME:    /* memory execute only */
+		case SDT_MEMEA:   /* memory execute only accessed */
+		case SDT_MEMER:   /* memory execute read */
+		case SDT_MEMERA:  /* memory execute read accessed */
+			break;
+		default:
+			return(EINVAL);
+			/*NOTREACHED*/
+		}
+
+		/* Only user (ring-3) descriptors may be present. */
+		if ((dp->sd_p != 0) && (dp->sd_dpl != SEL_UPL))
+			return (EACCES);
+	}
+
+	if (uap->start == LDT_AUTO_ALLOC && uap->num == 1) {
+		/* Allocate a free slot */
+		mtx_lock(&dt_lock);
+		pldt = user_ldt_alloc(p, 0);
+		if (pldt == NULL) {
+			mtx_unlock(&dt_lock);
+			return (ENOMEM);
+		}
+
+		/*
+		 * start scanning a bit up to leave room for NVidia and
+		 * Wine, which still user the "Blat" method of allocation.
+		 */
+		i = 16;
+		dp = &((struct user_segment_descriptor *)(pldt->ldt_base))[i];
+		for (; i < max_ldt_segment; ++i, ++dp) {
+			if (dp->sd_type == SDT_SYSNULL)
+				break;
+		}
+		if (i >= max_ldt_segment) {
+			mtx_unlock(&dt_lock);
+			return (ENOSPC);
+		}
+		uap->start = i;
+		error = amd64_set_ldt_data(td, i, 1, descs);
+		mtx_unlock(&dt_lock);
+	} else {
+		largest_ld = uap->start + uap->num;
+		if (largest_ld > max_ldt_segment)
+			return (EINVAL);
+		mtx_lock(&dt_lock);
+		if (user_ldt_alloc(p, 0) != NULL) {
+			error = amd64_set_ldt_data(td, uap->start, uap->num,
+			    descs);
+		}
+		mtx_unlock(&dt_lock);
+	}
+	if (error == 0)
+		td->td_retval[0] = uap->start;
+	return (error);
+}
+
+int
+amd64_set_ldt_data(struct thread *td, int start, int num,
+    struct user_segment_descriptor *descs)
+{
+	struct mdproc *mdp = &td->td_proc->p_md;
+	struct proc_ldt *pldt = mdp->md_ldt;
+
+	mtx_assert(&dt_lock, MA_OWNED);
+
+	/* Fill in range */
+	bcopy(descs,
+	    &((struct user_segment_descriptor *)(pldt->ldt_base))[start],
+	    num * sizeof(struct user_segment_descriptor));
+	return (0);
+}
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c
index a519414..467feaf 100644
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@@ -171,6 +171,52 @@ SYSCTL_INT(_machdep, OID_AUTO, prot_fault_translation, CTLFLAG_RW,
 
 extern char *syscallnames[];
 
+/* #define DEBUG 1 */
+#ifdef DEBUG
+static void
+report_seg_fault(const char *segn, struct trapframe *frame)
+{
+	struct proc_ldt *pldt;
+	struct trapframe *pf;
+
+	pldt = curproc->p_md.md_ldt;
+	printf("%d: %s load fault %lx %p %d\n",
+	    curproc->p_pid, segn, frame->tf_err,
+	    pldt != NULL ? pldt->ldt_base : NULL,
+	    pldt != NULL ? pldt->ldt_refcnt : 0);
+	kdb_backtrace();
+	pf = (struct trapframe *)frame->tf_rsp;
+	printf("rdi %lx\n", pf->tf_rdi);
+	printf("rsi %lx\n", pf->tf_rsi);
+	printf("rdx %lx\n", pf->tf_rdx);
+	printf("rcx %lx\n", pf->tf_rcx);
+	printf("r8  %lx\n", pf->tf_r8);
+	printf("r9  %lx\n", pf->tf_r9);
+	printf("rax %lx\n", pf->tf_rax);
+	printf("rbx %lx\n", pf->tf_rbx);
+	printf("rbp %lx\n", pf->tf_rbp);
+	printf("r10 %lx\n", pf->tf_r10);
+	printf("r11 %lx\n", pf->tf_r11);
+	printf("r12 %lx\n", pf->tf_r12);
+	printf("r13 %lx\n", pf->tf_r13);
+	printf("r14 %lx\n", pf->tf_r14);
+	printf("r15 %lx\n", pf->tf_r15);
+	printf("fs  %x\n", pf->tf_fs);
+	printf("gs  %x\n", pf->tf_gs);
+	printf("es  %x\n", pf->tf_es);
+	printf("ds  %x\n", pf->tf_ds);
+	printf("tno %x\n", pf->tf_trapno);
+	printf("adr %lx\n", pf->tf_addr);
+	printf("flg %x\n", pf->tf_flags);
+	printf("err %lx\n", pf->tf_err);
+	printf("rip %lx\n", pf->tf_rip);
+	printf("cs  %lx\n", pf->tf_cs);
+	printf("rfl %lx\n", pf->tf_rflags);
+	printf("rsp %lx\n", pf->tf_rsp);
+	printf("ss  %lx\n", pf->tf_ss);
+}
+#endif
+
 /*
  * Exception, fault, and trap interface to the FreeBSD kernel.
  * This common code is called from assembly language IDT gate entry
@@ -258,6 +304,9 @@ trap(struct trapframe *frame)
 			 */
 			printf("kernel trap %d with interrupts disabled\n",
 			    type);
+#ifdef DEBUG
+			report_seg_fault("hlt", frame);
+#endif
 			/*
 			 * We shouldn't enable interrupts while holding a
 			 * spin lock or servicing an NMI.
@@ -470,6 +519,38 @@ trap(struct trapframe *frame)
 				frame->tf_rip = (long)doreti_iret_fault;
 				goto out;
 			}
+			if (frame->tf_rip == (long)ld_ds) {
+#ifdef DEBUG
+				report_seg_fault("ds", frame);
+#endif
+				frame->tf_rip = (long)ds_load_fault;
+				frame->tf_ds = _udatasel;
+				goto out;
+			}
+			if (frame->tf_rip == (long)ld_es) {
+#ifdef DEBUG
+				report_seg_fault("es", frame);
+#endif
+				frame->tf_rip = (long)es_load_fault;
+				frame->tf_es = _udatasel;
+				goto out;
+			}
+			if (frame->tf_rip == (long)ld_fs) {
+#ifdef DEBUG
+				report_seg_fault("fs", frame);
+#endif
+				frame->tf_rip = (long)fs_load_fault;
+				frame->tf_fs = _ufssel;
+				goto out;
+			}
+			if (frame->tf_rip == (long)ld_gs) {
+#ifdef DEBUG
+				report_seg_fault("gs", frame);
+#endif
+				frame->tf_rip = (long)gs_load_fault;
+				frame->tf_gs = _ugssel;
+				goto out;
+			}
 			if (PCPU_GET(curpcb)->pcb_onfault != NULL) {
 				frame->tf_rip =
 				    (long)PCPU_GET(curpcb)->pcb_onfault;
@@ -564,6 +645,9 @@ trap(struct trapframe *frame)
 	trapsignal(td, &ksi);
 
 #ifdef DEBUG
+{
+	register_t rg,rgk, rf;
+
 	if (type <= MAX_TRAP_MSG) {
 		uprintf("fatal process exception: %s",
 			trap_msg[type]);
@@ -571,6 +655,17 @@ trap(struct trapframe *frame)
 			uprintf(", fault VA = 0x%lx", frame->tf_addr);
 		uprintf("\n");
 	}
+	rf = rdmsr(0xc0000100);
+	rg = rdmsr(0xc0000101);
+	rgk = rdmsr(0xc0000102);
+	uprintf("pid %d TRAP %d rip %lx err %lx addr %lx cs %lx ss %lx ds %x "
+		"es %x fs %x fsbase %lx %lx gs %x gsbase %lx %lx %lx\n",
+		curproc->p_pid, type, frame->tf_rip, frame->tf_err,
+		frame->tf_addr,
+		frame->tf_cs, frame->tf_ss, frame->tf_ds, frame->tf_es,
+		frame->tf_fs, td->td_pcb->pcb_fsbase, rf,
+		frame->tf_gs, td->td_pcb->pcb_gsbase, rg, rgk);
+}
 #endif
 
 user:
diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c
index 1e87bdc..928be34 100644
--- a/sys/amd64/amd64/vm_machdep.c
+++ b/sys/amd64/amd64/vm_machdep.c
@@ -71,6 +71,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/md_var.h>
 #include <machine/pcb.h>
 #include <machine/specialreg.h>
+#include <machine/tss.h>
 
 #include <vm/vm.h>
 #include <vm/vm_extern.h>
@@ -102,11 +103,24 @@ cpu_fork(td1, p2, td2, flags)
 {
 	register struct proc *p1;
 	struct pcb *pcb2;
-	struct mdproc *mdp2;
+	struct mdproc *mdp1, *mdp2;
+	struct proc_ldt *pldt;
+	pmap_t pmap2;
 
 	p1 = td1->td_proc;
-	if ((flags & RFPROC) == 0)
+	if ((flags & RFPROC) == 0) {
+		if ((flags & RFMEM) == 0) {
+			/* unshare user LDT */
+			mdp1 = &p1->p_md;
+			mtx_lock(&dt_lock);
+			if ((pldt = mdp1->md_ldt) != NULL &&
+			    pldt->ldt_refcnt > 1 &&
+			    user_ldt_alloc(p1, 1) == NULL)
+				panic("could not copy LDT");
+			mtx_unlock(&dt_lock);
+		}
 		return;
+	}
 
 	/* Ensure that p1's pcb is up to date. */
 	fpuexit(td1);
@@ -150,7 +164,8 @@ cpu_fork(td1, p2, td2, flags)
 	 * Set registers for trampoline to user mode.  Leave space for the
 	 * return address on stack.  These are the kernel mode register values.
 	 */
-	pcb2->pcb_cr3 = vtophys(vmspace_pmap(p2->p_vmspace)->pm_pml4);
+	pmap2 = vmspace_pmap(p2->p_vmspace);
+	pcb2->pcb_cr3 = DMAP_TO_PHYS((vm_offset_t)pmap2->pm_pml4);
 	pcb2->pcb_r12 = (register_t)fork_return;	/* fork_trampoline argument */
 	pcb2->pcb_rbp = 0;
 	pcb2->pcb_rsp = (register_t)td2->td_frame - sizeof(void *);
@@ -168,6 +183,32 @@ cpu_fork(td1, p2, td2, flags)
 	td2->td_md.md_spinlock_count = 1;
 	td2->td_md.md_saved_flags = PSL_KERNEL | PSL_I;
 
+	/* As an i386, do not copy io permission bitmap. */
+	pcb2->pcb_tssp = NULL;
+
+	/* Copy the LDT, if necessary. */
+	mdp1 = &td1->td_proc->p_md;
+	mdp2 = &p2->p_md;
+	mtx_lock(&dt_lock);
+	if (mdp1->md_ldt != NULL) {
+		if (flags & RFMEM) {
+			mdp1->md_ldt->ldt_refcnt++;
+			mdp2->md_ldt = mdp1->md_ldt;
+			bcopy(&mdp1->md_ldt_sd, &mdp2->md_ldt_sd, sizeof(struct
+			    system_segment_descriptor));
+		} else {
+			mdp2->md_ldt = NULL;
+			mdp2->md_ldt = user_ldt_alloc(p2, 0);
+			if (mdp2->md_ldt == NULL)
+				panic("could not copy LDT");
+			amd64_set_ldt_data(td2, 0, max_ldt_segment,
+			    (struct user_segment_descriptor *)
+			    mdp1->md_ldt->ldt_base);
+		}
+	} else
+		mdp2->md_ldt = NULL;
+	mtx_unlock(&dt_lock);
+
 	/*
 	 * Now, cpu_switch() can schedule the new process.
 	 * pcb_rsp is loaded pointing to the cpu_switch() stack frame
@@ -202,25 +243,49 @@ cpu_set_fork_handler(td, func, arg)
 void
 cpu_exit(struct thread *td)
 {
+
+	/*
+	 * If this process has a custom LDT, release it.
+	 */
+	mtx_lock(&dt_lock);
+	if (td->td_proc->p_md.md_ldt != 0)
+		user_ldt_free(td);
+	else
+		mtx_unlock(&dt_lock);
 }
 
 void
 cpu_thread_exit(struct thread *td)
 {
+	struct pcb *pcb;
 
 	if (td == PCPU_GET(fpcurthread))
 		fpudrop();
 
+	pcb = td->td_pcb;
+
 	/* Disable any hardware breakpoints. */
-	if (td->td_pcb->pcb_flags & PCB_DBREGS) {
+	if (pcb->pcb_flags & PCB_DBREGS) {
 		reset_dbregs();
-		td->td_pcb->pcb_flags &= ~PCB_DBREGS;
+		pcb->pcb_flags &= ~PCB_DBREGS;
 	}
 }
 
 void
 cpu_thread_clean(struct thread *td)
 {
+	struct pcb *pcb;
+
+	pcb = td->td_pcb;
+
+	/*
+	 * Clean TSS/iomap
+	 */
+	if (pcb->pcb_tssp != NULL) {
+		kmem_free(kernel_map, (vm_offset_t)pcb->pcb_tssp,
+		    ctob(IOPAGES + 1));
+		pcb->pcb_tssp = NULL;
+	}
 }
 
 void
@@ -245,6 +310,8 @@ cpu_thread_alloc(struct thread *td)
 void
 cpu_thread_free(struct thread *td)
 {
+
+	cpu_thread_clean(td);
 }
 
 /*
@@ -287,7 +354,6 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 	 * Set registers for trampoline to user mode.  Leave space for the
 	 * return address on stack.  These are the kernel mode register values.
 	 */
-	pcb2->pcb_cr3 = vtophys(vmspace_pmap(td->td_proc->p_vmspace)->pm_pml4);
 	pcb2->pcb_r12 = (register_t)fork_return;	    /* trampoline arg */
 	pcb2->pcb_rbp = 0;
 	pcb2->pcb_rsp = (register_t)td->td_frame - sizeof(void *);	/* trampoline arg */
@@ -295,6 +361,7 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 	pcb2->pcb_rip = (register_t)fork_trampoline;
 	/*
 	 * If we didn't copy the pcb, we'd need to do the following registers:
+	 * pcb2->pcb_cr3:	cloned above.
 	 * pcb2->pcb_dr*:	cloned above.
 	 * pcb2->pcb_savefpu:	cloned above.
 	 * pcb2->pcb_onfault:	cloned above (always NULL here?).
@@ -356,6 +423,11 @@ cpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg,
 	    ((register_t)stack->ss_sp + stack->ss_size) & ~0x0f;
 	td->td_frame->tf_rsp -= 8;
 	td->td_frame->tf_rip = (register_t)entry;
+	td->td_frame->tf_ds = _udatasel;
+	td->td_frame->tf_es = _udatasel;
+	td->td_frame->tf_fs = _ufssel;
+	td->td_frame->tf_gs = _ugssel;
+	td->td_frame->tf_flags = TF_HASSEGS;
 
 	/*
 	 * Pass the address of the mailbox for this kse to the uts
@@ -373,25 +445,11 @@ cpu_set_user_tls(struct thread *td, void *tls_base)
 
 #ifdef COMPAT_IA32
 	if (td->td_proc->p_sysent->sv_flags & SV_ILP32) {
-		if (td == curthread) {
-			critical_enter();
-			td->td_pcb->pcb_gsbase = (register_t)tls_base;
-			wrmsr(MSR_KGSBASE, td->td_pcb->pcb_gsbase);
-			critical_exit();
-		} else {
-			td->td_pcb->pcb_gsbase = (register_t)tls_base;
-		}
+		td->td_pcb->pcb_gsbase = (register_t)tls_base;
 		return (0);
 	}
 #endif
-	if (td == curthread) {
-		critical_enter();
-		td->td_pcb->pcb_fsbase = (register_t)tls_base;
-		wrmsr(MSR_FSBASE, td->td_pcb->pcb_fsbase);
-		critical_exit();
-	} else {
-		td->td_pcb->pcb_fsbase = (register_t)tls_base;
-	}
+	td->td_pcb->pcb_fsbase = (register_t)tls_base;
 	return (0);
 }
 
diff --git a/sys/amd64/conf/GENERIC b/sys/amd64/conf/GENERIC
index 197de84..663420f 100644
--- a/sys/amd64/conf/GENERIC
+++ b/sys/amd64/conf/GENERIC
@@ -200,7 +200,7 @@ device		ppi		# Parallel port interface device
 device		de		# DEC/Intel DC21x4x (``Tulip'')
 device		em		# Intel PRO/1000 Gigabit Ethernet Family
 device		igb		# Intel PRO/1000 PCIE Server Gigabit Family
-device		ixgb		# Intel PRO/10GbE Ethernet Card
+device		ixgbe		# Intel PRO/10GbE PCIE Ethernet Family
 device		le		# AMD Am7900 LANCE and Am79C9xx PCnet
 device		ti		# Alteon Networks Tigon I/II gigabit Ethernet
 device		txp		# 3Com 3cR990 (``Typhoon'')
@@ -295,7 +295,6 @@ device		ums		# Mouse
 device		ural		# Ralink Technology RT2500USB wireless NICs
 device		rum		# Ralink Technology RT2501USB wireless NICs
 device		urio		# Diamond Rio 500 MP3 player
-device		uscanner	# Scanners
 # USB Serial devices
 device		uark		# Technologies ARK3116 based serial adapters
 device		ubsa		# Belkin F5U103 and compatible serial adapters
diff --git a/sys/amd64/conf/NOTES b/sys/amd64/conf/NOTES
index 76dca9d..95ae9be 100644
--- a/sys/amd64/conf/NOTES
+++ b/sys/amd64/conf/NOTES
@@ -400,10 +400,10 @@ device		ichwd
 # Temperature sensors:
 #
 # coretemp: on-die sensor on Intel Core and newer CPUs
-# k8temp: on-die sensor on AMD K8 CPUs
+# amdtemp: on-die sensor on AMD K8/K10/K11 CPUs
 #
 device		coretemp
-device		k8temp
+device		amdtemp
 
 #
 # CPU control pseudo-device. Provides access to MSRs, CPUID info and
diff --git a/sys/amd64/ia32/ia32_exception.S b/sys/amd64/ia32/ia32_exception.S
index 4820f53..76c5d5a 100644
--- a/sys/amd64/ia32/ia32_exception.S
+++ b/sys/amd64/ia32/ia32_exception.S
@@ -60,6 +60,11 @@ IDTVEC(int0x80_syscall)
 	movq	%r13,TF_R13(%rsp)
 	movq	%r14,TF_R14(%rsp)
 	movq	%r15,TF_R15(%rsp)
+	movw	%fs,TF_FS(%rsp)
+	movw	%gs,TF_GS(%rsp)
+	movw	%es,TF_ES(%rsp)
+	movw	%ds,TF_DS(%rsp)
+	movl	$TF_HASSEGS,TF_FLAGS(%rsp)
 	FAKE_MCOUNT(TF_RIP(%rsp))
 	movq	%rsp, %rdi
 	call	ia32_syscall
diff --git a/sys/mips/alchemy/uart_cpu_alchemy.c b/sys/amd64/ia32/ia32_misc.c
index 931aed6..2fa1972 100644
--- a/sys/mips/alchemy/uart_cpu_alchemy.c
+++ b/sys/amd64/ia32/ia32_misc.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2006 Wojciech A. Koszek <wkoszek@FreeBSD.org>
+ * Copyright (c) 2009 Konstantin Belousov
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -22,58 +22,50 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
- *
- * $Id$
- */
-/*
- * Skeleton of this file was based on respective code for ARM
- * code written by Olivier Houchard.
  */
 
-#include "opt_uart.h"
-
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_compat.h"
+
 #include <sys/param.h>
+#include <sys/mount.h>
+#include <sys/proc.h>
+#include <sys/socket.h>
+#include <sys/sysent.h>
+#include <sys/sysproto.h>
 #include <sys/systm.h>
-#include <sys/bus.h>
-#include <sys/cons.h>
+#include <sys/uio.h>
 
-#include <machine/bus.h>
+#include <machine/cpu.h>
+#include <machine/sysarch.h>
 
-#include <dev/uart/uart.h>
-#include <dev/uart/uart_cpu.h>
-
-#include <mips/alchemy/aureg.h>
-
-bus_space_tag_t uart_bus_space_io;
-bus_space_tag_t uart_bus_space_mem;
+#include <compat/freebsd32/freebsd32_util.h>
+#include <compat/freebsd32/freebsd32.h>
+#include <compat/freebsd32/freebsd32_proto.h>
 
 int
-uart_cpu_eqres(struct uart_bas *b1, struct uart_bas *b2)
+freebsd32_sysarch(struct thread *td, struct freebsd32_sysarch_args *uap)
 {
+	struct sysarch_args uap1;
+	struct i386_ldt_args uapl;
+	struct i386_ldt_args32 uapl32;
+	int error;
 
-	return ((b1->bsh == b2->bsh && b1->bst == b2->bst) ? 1 : 0);
-}
-
-int
-uart_cpu_getdev(int devtype, struct uart_devinfo *di)
-{
-
-	di->ops = uart_getops(&uart_ns8250_class);
-	di->bas.chan = 0;
-	di->bas.bst = 0;
-	di->bas.regshft = 0;
-	di->bas.rclk = 0;
-	di->baudrate = 115200;
-	di->databits = 8;
-	di->stopbits = 1;
-	di->parity = UART_PARITY_NONE;
-
-	uart_bus_space_io = 0;
-	uart_bus_space_mem = MIPS_PHYS_TO_KSEG1(UART0_BASE);
-	di->bas.bsh = MIPS_PHYS_TO_KSEG1(UART0_BASE);
-
-	return (0);
+	if (uap->op == I386_SET_LDT || uap->op == I386_GET_LDT) {
+		if ((error = copyin(uap->parms, &uapl32, sizeof(uapl32))) != 0)
+			return (error);
+		uap1.op = uap->op;
+		uap1.parms = (char *)&uapl;
+		uapl.start = uapl32.start;
+		uapl.descs = (struct user_segment_descriptor *)(uintptr_t)
+		    uapl32.descs;
+		uapl.num = uapl32.num;
+		return (sysarch_ldt(td, &uap1, UIO_SYSSPACE));
+	} else {
+		uap1.op = uap->op;
+		uap1.parms = uap->parms;
+		return (sysarch(td, &uap1));
+	}
 }
diff --git a/sys/amd64/ia32/ia32_reg.c b/sys/amd64/ia32/ia32_reg.c
index 8abc6fc..49dd4e2 100644
--- a/sys/amd64/ia32/ia32_reg.c
+++ b/sys/amd64/ia32/ia32_reg.c
@@ -85,9 +85,17 @@ fill_regs32(struct thread *td, struct reg32 *regs)
 
 	tp = td->td_frame;
 	pcb = td->td_pcb;
-	regs->r_fs = pcb->pcb_fs;
-	regs->r_es = pcb->pcb_es;
-	regs->r_ds = pcb->pcb_ds;
+	if (tp->tf_flags & TF_HASSEGS) {
+		regs->r_gs = tp->tf_gs;
+		regs->r_fs = tp->tf_fs;
+		regs->r_es = tp->tf_es;
+		regs->r_ds = tp->tf_ds;
+	} else {
+		regs->r_gs = _ugssel;
+		regs->r_fs = _ufssel;
+		regs->r_es = _udatasel;
+		regs->r_ds = _udatasel;
+	}
 	regs->r_edi = tp->tf_rdi;
 	regs->r_esi = tp->tf_rsi;
 	regs->r_ebp = tp->tf_rbp;
@@ -100,7 +108,6 @@ fill_regs32(struct thread *td, struct reg32 *regs)
 	regs->r_eflags = tp->tf_rflags;
 	regs->r_esp = tp->tf_rsp;
 	regs->r_ss = tp->tf_ss;
-	regs->r_gs = pcb->pcb_gs;
 	return (0);
 }
 
@@ -114,14 +121,11 @@ set_regs32(struct thread *td, struct reg32 *regs)
 	if (!EFL_SECURE(regs->r_eflags, tp->tf_rflags) || !CS_SECURE(regs->r_cs))
 		return (EINVAL);
 	pcb = td->td_pcb;
-#if 0
-	load_fs(regs->r_fs);
-	pcb->pcb_fs = regs->r_fs;
-	load_es(regs->r_es);
-	pcb->pcb_es = regs->r_es;
-	load_ds(regs->r_ds);
-	pcb->pcb_ds = regs->r_ds;
-#endif
+	tp->tf_gs = regs->r_gs;
+	tp->tf_fs = regs->r_fs;
+	tp->tf_es = regs->r_es;
+	tp->tf_ds = regs->r_ds;
+	tp->tf_flags = TF_HASSEGS;
 	tp->tf_rdi = regs->r_edi;
 	tp->tf_rsi = regs->r_esi;
 	tp->tf_rbp = regs->r_ebp;
@@ -134,10 +138,6 @@ set_regs32(struct thread *td, struct reg32 *regs)
 	tp->tf_rflags = regs->r_eflags;
 	tp->tf_rsp = regs->r_esp;
 	tp->tf_ss = regs->r_ss;
-#if 0
-	load_gs(regs->r_gs);
-	pcb->pcb_gs = regs->r_gs;
-#endif
 	return (0);
 }
 
@@ -166,7 +166,8 @@ fill_fpregs32(struct thread *td, struct fpreg32 *regs)
 	penv_87->en_fcs = td->td_frame->tf_cs;
 	penv_87->en_opcode = penv_xmm->en_opcode;
 	penv_87->en_foo = penv_xmm->en_rdp;
-	penv_87->en_fos = td->td_pcb->pcb_ds;
+	/* Entry into the kernel always sets TF_HASSEGS */
+	penv_87->en_fos = td->td_frame->tf_ds;
 
 	/* FPU registers */
 	for (i = 0; i < 8; ++i)
diff --git a/sys/amd64/ia32/ia32_signal.c b/sys/amd64/ia32/ia32_signal.c
index 019faba..37e8013 100644
--- a/sys/amd64/ia32/ia32_signal.c
+++ b/sys/amd64/ia32/ia32_signal.c
@@ -85,8 +85,6 @@ static void freebsd4_ia32_sendsig(sig_t, ksiginfo_t *, sigset_t *);
 static void ia32_get_fpcontext(struct thread *td, struct ia32_mcontext *mcp);
 static int ia32_set_fpcontext(struct thread *td, const struct ia32_mcontext *mcp);
 
-extern int _ucode32sel, _udatasel;
-
 #define	CS_SECURE(cs)		(ISPL(cs) == SEL_UPL)
 #define	EFL_SECURE(ef, oef)	((((ef) ^ (oef)) & ~PSL_USERCHANGE) == 0)
 
@@ -134,10 +132,11 @@ ia32_get_mcontext(struct thread *td, struct ia32_mcontext *mcp, int flags)
 	PROC_LOCK(curthread->td_proc);
 	mcp->mc_onstack = sigonstack(tp->tf_rsp);
 	PROC_UNLOCK(curthread->td_proc);
-	mcp->mc_gs = td->td_pcb->pcb_gs;
-	mcp->mc_fs = td->td_pcb->pcb_fs;
-	mcp->mc_es = td->td_pcb->pcb_es;
-	mcp->mc_ds = td->td_pcb->pcb_ds;
+	/* Entry into kernel always sets TF_HASSEGS */
+	mcp->mc_gs = tp->tf_gs;
+	mcp->mc_fs = tp->tf_fs;
+	mcp->mc_es = tp->tf_es;
+	mcp->mc_ds = tp->tf_ds;
 	mcp->mc_edi = tp->tf_rdi;
 	mcp->mc_esi = tp->tf_rsi;
 	mcp->mc_ebp = tp->tf_rbp;
@@ -158,6 +157,8 @@ ia32_get_mcontext(struct thread *td, struct ia32_mcontext *mcp, int flags)
 	mcp->mc_ss = tp->tf_ss;
 	mcp->mc_len = sizeof(*mcp);
 	ia32_get_fpcontext(td, mcp);
+	mcp->mc_fsbase = td->td_pcb->pcb_fsbase;
+	mcp->mc_gsbase = td->td_pcb->pcb_gsbase;
 	return (0);
 }
 
@@ -182,11 +183,11 @@ ia32_set_mcontext(struct thread *td, const struct ia32_mcontext *mcp)
 	ret = ia32_set_fpcontext(td, mcp);
 	if (ret != 0)
 		return (ret);
-#if 0	/* XXX deal with load_fs() and friends */
+	tp->tf_gs = mcp->mc_gs;
 	tp->tf_fs = mcp->mc_fs;
 	tp->tf_es = mcp->mc_es;
 	tp->tf_ds = mcp->mc_ds;
-#endif
+	tp->tf_flags = TF_HASSEGS;
 	tp->tf_rdi = mcp->mc_edi;
 	tp->tf_rsi = mcp->mc_esi;
 	tp->tf_rbp = mcp->mc_ebp;
@@ -199,9 +200,6 @@ ia32_set_mcontext(struct thread *td, const struct ia32_mcontext *mcp)
 	tp->tf_rflags = rflags;
 	tp->tf_rsp = mcp->mc_esp;
 	tp->tf_ss = mcp->mc_ss;
-#if 0	/* XXX deal with load_gs() and friends */
-	td->td_pcb->pcb_gs = mcp->mc_gs;
-#endif
 	td->td_pcb->pcb_flags |= PCB_FULLCTX;
 	return (0);
 }
@@ -326,10 +324,6 @@ freebsd4_ia32_sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	sf.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK)
 	    ? ((oonstack) ? SS_ONSTACK : 0) : SS_DISABLE;
 	sf.sf_uc.uc_mcontext.mc_onstack = (oonstack) ? 1 : 0;
-	sf.sf_uc.uc_mcontext.mc_gs = rgs();
-	sf.sf_uc.uc_mcontext.mc_fs = rfs();
-	__asm __volatile("mov %%es,%0" : "=rm" (sf.sf_uc.uc_mcontext.mc_es));
-	__asm __volatile("mov %%ds,%0" : "=rm" (sf.sf_uc.uc_mcontext.mc_ds));
 	sf.sf_uc.uc_mcontext.mc_edi = regs->tf_rdi;
 	sf.sf_uc.uc_mcontext.mc_esi = regs->tf_rsi;
 	sf.sf_uc.uc_mcontext.mc_ebp = regs->tf_rbp;
@@ -345,6 +339,10 @@ freebsd4_ia32_sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	sf.sf_uc.uc_mcontext.mc_eflags = regs->tf_rflags;
 	sf.sf_uc.uc_mcontext.mc_esp = regs->tf_rsp;
 	sf.sf_uc.uc_mcontext.mc_ss = regs->tf_ss;
+	sf.sf_uc.uc_mcontext.mc_ds = regs->tf_ds;
+	sf.sf_uc.uc_mcontext.mc_es = regs->tf_es;
+	sf.sf_uc.uc_mcontext.mc_fs = regs->tf_fs;
+	sf.sf_uc.uc_mcontext.mc_gs = regs->tf_gs;
 
 	/* Allocate space for the signal handler context. */
 	if ((td->td_pflags & TDP_ALTSTACK) != 0 && !oonstack &&
@@ -394,10 +392,8 @@ freebsd4_ia32_sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	regs->tf_rflags &= ~(PSL_T | PSL_D);
 	regs->tf_cs = _ucode32sel;
 	regs->tf_ss = _udatasel;
-	load_ds(_udatasel);
-	td->td_pcb->pcb_ds = _udatasel;
-	load_es(_udatasel);
-	td->td_pcb->pcb_es = _udatasel;
+	regs->tf_ds = _udatasel;
+	regs->tf_es = _udatasel;
 	/* leave user %fs and %gs untouched */
 	PROC_LOCK(p);
 	mtx_lock(&psp->ps_mtx);
@@ -441,10 +437,6 @@ ia32_sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	sf.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK)
 	    ? ((oonstack) ? SS_ONSTACK : 0) : SS_DISABLE;
 	sf.sf_uc.uc_mcontext.mc_onstack = (oonstack) ? 1 : 0;
-	sf.sf_uc.uc_mcontext.mc_gs = rgs();
-	sf.sf_uc.uc_mcontext.mc_fs = rfs();
-	__asm __volatile("mov %%es,%0" : "=rm" (sf.sf_uc.uc_mcontext.mc_es));
-	__asm __volatile("mov %%ds,%0" : "=rm" (sf.sf_uc.uc_mcontext.mc_ds));
 	sf.sf_uc.uc_mcontext.mc_edi = regs->tf_rdi;
 	sf.sf_uc.uc_mcontext.mc_esi = regs->tf_rsi;
 	sf.sf_uc.uc_mcontext.mc_ebp = regs->tf_rbp;
@@ -460,9 +452,15 @@ ia32_sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	sf.sf_uc.uc_mcontext.mc_eflags = regs->tf_rflags;
 	sf.sf_uc.uc_mcontext.mc_esp = regs->tf_rsp;
 	sf.sf_uc.uc_mcontext.mc_ss = regs->tf_ss;
+	sf.sf_uc.uc_mcontext.mc_ds = regs->tf_ds;
+	sf.sf_uc.uc_mcontext.mc_es = regs->tf_es;
+	sf.sf_uc.uc_mcontext.mc_fs = regs->tf_fs;
+	sf.sf_uc.uc_mcontext.mc_gs = regs->tf_gs;
 	sf.sf_uc.uc_mcontext.mc_len = sizeof(sf.sf_uc.uc_mcontext); /* magic */
 	ia32_get_fpcontext(td, &sf.sf_uc.uc_mcontext);
 	fpstate_drop(td);
+	sf.sf_uc.uc_mcontext.mc_fsbase = td->td_pcb->pcb_fsbase;
+	sf.sf_uc.uc_mcontext.mc_gsbase = td->td_pcb->pcb_gsbase;
 
 	/* Allocate space for the signal handler context. */
 	if ((td->td_pflags & TDP_ALTSTACK) != 0 && !oonstack &&
@@ -514,11 +512,9 @@ ia32_sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	regs->tf_rflags &= ~(PSL_T | PSL_D);
 	regs->tf_cs = _ucode32sel;
 	regs->tf_ss = _udatasel;
-	load_ds(_udatasel);
-	td->td_pcb->pcb_ds = _udatasel;
-	load_es(_udatasel);
-	td->td_pcb->pcb_es = _udatasel;
-	/* leave user %fs and %gs untouched */
+	regs->tf_ds = _udatasel;
+	regs->tf_es = _udatasel;
+	/* XXXKIB leave user %fs and %gs untouched */
 	PROC_LOCK(p);
 	mtx_lock(&psp->ps_mtx);
 }
@@ -591,7 +587,6 @@ freebsd4_freebsd32_sigreturn(td, uap)
 		return (EINVAL);
 	}
 
-	/* Segment selectors restored by sigtramp.S */
 	regs->tf_rdi = ucp->uc_mcontext.mc_edi;
 	regs->tf_rsi = ucp->uc_mcontext.mc_esi;
 	regs->tf_rbp = ucp->uc_mcontext.mc_ebp;
@@ -606,6 +601,10 @@ freebsd4_freebsd32_sigreturn(td, uap)
 	regs->tf_rflags = ucp->uc_mcontext.mc_eflags;
 	regs->tf_rsp = ucp->uc_mcontext.mc_esp;
 	regs->tf_ss = ucp->uc_mcontext.mc_ss;
+	regs->tf_ds = ucp->uc_mcontext.mc_ds;
+	regs->tf_es = ucp->uc_mcontext.mc_es;
+	regs->tf_fs = ucp->uc_mcontext.mc_fs;
+	regs->tf_gs = ucp->uc_mcontext.mc_gs;
 
 	PROC_LOCK(p);
 	td->td_sigmask = ucp->uc_sigmask;
@@ -678,7 +677,6 @@ freebsd32_sigreturn(td, uap)
 	if (ret != 0)
 		return (ret);
 
-	/* Segment selectors restored by sigtramp.S */
 	regs->tf_rdi = ucp->uc_mcontext.mc_edi;
 	regs->tf_rsi = ucp->uc_mcontext.mc_esi;
 	regs->tf_rbp = ucp->uc_mcontext.mc_ebp;
@@ -693,6 +691,11 @@ freebsd32_sigreturn(td, uap)
 	regs->tf_rflags = ucp->uc_mcontext.mc_eflags;
 	regs->tf_rsp = ucp->uc_mcontext.mc_esp;
 	regs->tf_ss = ucp->uc_mcontext.mc_ss;
+	regs->tf_ds = ucp->uc_mcontext.mc_ds;
+	regs->tf_es = ucp->uc_mcontext.mc_es;
+	regs->tf_fs = ucp->uc_mcontext.mc_fs;
+	regs->tf_gs = ucp->uc_mcontext.mc_gs;
+	regs->tf_flags = TF_HASSEGS;
 
 	PROC_LOCK(p);
 	td->td_sigmask = ucp->uc_sigmask;
@@ -715,20 +718,14 @@ ia32_setregs(td, entry, stack, ps_strings)
 	struct trapframe *regs = td->td_frame;
 	struct pcb *pcb = td->td_pcb;
 	
-	critical_enter();
-	wrmsr(MSR_FSBASE, 0);
-	wrmsr(MSR_KGSBASE, 0);	/* User value while we're in the kernel */
+	mtx_lock(&dt_lock);
+	if (td->td_proc->p_md.md_ldt != NULL)
+		user_ldt_free(td);
+	else
+		mtx_unlock(&dt_lock);
+
 	pcb->pcb_fsbase = 0;
 	pcb->pcb_gsbase = 0;
-	critical_exit();
-	load_ds(_udatasel);
-	load_es(_udatasel);
-	load_fs(_udatasel);
-	load_gs(_udatasel);
-	pcb->pcb_ds = _udatasel;
-	pcb->pcb_es = _udatasel;
-	pcb->pcb_fs = _udatasel;
-	pcb->pcb_gs = _udatasel;
 	pcb->pcb_initial_fpucw = __INITIAL_FPUCW_I386__;
 
 	bzero((char *)regs, sizeof(struct trapframe));
@@ -738,6 +735,12 @@ ia32_setregs(td, entry, stack, ps_strings)
 	regs->tf_ss = _udatasel;
 	regs->tf_cs = _ucode32sel;
 	regs->tf_rbx = ps_strings;
+	regs->tf_ds = _udatasel;
+	regs->tf_es = _udatasel;
+	regs->tf_fs = _ufssel;
+	regs->tf_gs = _ugssel;
+	regs->tf_flags = TF_HASSEGS;
+
 	load_cr0(rcr0() | CR0_MP | CR0_TS);
 	fpstate_drop(td);
 
diff --git a/sys/amd64/ia32/ia32_sigtramp.S b/sys/amd64/ia32/ia32_sigtramp.S
index 1cd220a..9455169 100644
--- a/sys/amd64/ia32/ia32_sigtramp.S
+++ b/sys/amd64/ia32/ia32_sigtramp.S
@@ -45,8 +45,6 @@ ia32_sigcode:
 	calll	*IA32_SIGF_HANDLER(%esp)
 	leal	IA32_SIGF_UC(%esp),%eax	/* get ucontext */
 	pushl	%eax
-	mov	IA32_UC_ES(%eax),%es	/* restore %es */
-	mov	IA32_UC_DS(%eax),%ds	/* restore %ds */
 	movl	$SYS_sigreturn,%eax
 	pushl	%eax			/* junk to fake return addr. */
 	int	$0x80			/* enter kernel with args */
@@ -60,8 +58,6 @@ freebsd4_ia32_sigcode:
 	calll	*IA32_SIGF_HANDLER(%esp)
 	leal	IA32_SIGF_UC4(%esp),%eax/* get ucontext */
 	pushl	%eax
-	mov	IA32_UC4_ES(%eax),%es	/* restore %es */
-	mov	IA32_UC4_DS(%eax),%ds	/* restore %ds */
 	movl	$344,%eax		/* 4.x SYS_sigreturn */
 	pushl	%eax			/* junk to fake return addr. */
 	int	$0x80			/* enter kernel with args */
diff --git a/sys/amd64/include/apicvar.h b/sys/amd64/include/apicvar.h
index ea971ec..3cc028d 100644
--- a/sys/amd64/include/apicvar.h
+++ b/sys/amd64/include/apicvar.h
@@ -130,6 +130,7 @@
 #define IPI_IS_BITMAPED(x) ((x) <= IPI_BITMAP_LAST)
 
 #define	IPI_STOP	(APIC_IPI_INTS + 7)	/* Stop CPU until restarted. */
+#define	IPI_SUSPEND	(APIC_IPI_INTS + 8)	/* Suspend CPU until restarted. */
 
 /*
  * The spurious interrupt can share the priority class with the IPIs since
diff --git a/sys/amd64/include/asmacros.h b/sys/amd64/include/asmacros.h
index 788f39f..0bf0029 100644
--- a/sys/amd64/include/asmacros.h
+++ b/sys/amd64/include/asmacros.h
@@ -161,7 +161,12 @@
 	movq	%r12,TF_R12(%rsp) ;					\
 	movq	%r13,TF_R13(%rsp) ;					\
 	movq	%r14,TF_R14(%rsp) ;					\
-	movq	%r15,TF_R15(%rsp)
+	movq	%r15,TF_R15(%rsp) ;					\
+	movw	%fs,TF_FS(%rsp) ;					\
+	movw	%gs,TF_GS(%rsp) ;					\
+	movw	%es,TF_ES(%rsp) ;					\
+	movw	%ds,TF_DS(%rsp) ;					\
+	movl	$TF_HASSEGS,TF_FLAGS(%rsp)
 
 #define POP_FRAME							\
 	movq	TF_RDI(%rsp),%rdi ;					\
diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h
index cec9b4a..15fc0e9 100644
--- a/sys/amd64/include/cpufunc.h
+++ b/sys/amd64/include/cpufunc.h
@@ -164,70 +164,12 @@ halt(void)
 	__asm __volatile("hlt");
 }
 
-#if !defined(__GNUCLIKE_BUILTIN_CONSTANT_P) || __GNUCLIKE_ASM < 3
-
-#define	inb(port)		inbv(port)
-#define	outb(port, data)	outbv(port, data)
-
-#else /* __GNUCLIKE_BUILTIN_CONSTANT_P && __GNUCLIKE_ASM >= 3 */
-
-/*
- * The following complications are to get around gcc not having a
- * constraint letter for the range 0..255.  We still put "d" in the
- * constraint because "i" isn't a valid constraint when the port
- * isn't constant.  This only matters for -O0 because otherwise
- * the non-working version gets optimized away.
- * 
- * Use an expression-statement instead of a conditional expression
- * because gcc-2.6.0 would promote the operands of the conditional
- * and produce poor code for "if ((inb(var) & const1) == const2)".
- *
- * The unnecessary test `(port) < 0x10000' is to generate a warning if
- * the `port' has type u_short or smaller.  Such types are pessimal.
- * This actually only works for signed types.  The range check is
- * careful to avoid generating warnings.
- */
-#define	inb(port) __extension__ ({					\
-	u_char	_data;							\
-	if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100	\
-	    && (port) < 0x10000)					\
-		_data = inbc(port);					\
-	else								\
-		_data = inbv(port);					\
-	_data; })
-
-#define	outb(port, data) (						\
-	__builtin_constant_p(port) && ((port) & 0xffff) < 0x100		\
-	&& (port) < 0x10000						\
-	? outbc(port, data) : outbv(port, data))
-
 static __inline u_char
-inbc(u_int port)
+inb(u_int port)
 {
 	u_char	data;
 
-	__asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port)));
-	return (data);
-}
-
-static __inline void
-outbc(u_int port, u_char data)
-{
-	__asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port)));
-}
-
-#endif /* __GNUCLIKE_BUILTIN_CONSTANT_P  && __GNUCLIKE_ASM >= 3*/
-
-static __inline u_char
-inbv(u_int port)
-{
-	u_char	data;
-	/*
-	 * We use %%dx and not %1 here because i/o is done at %dx and not at
-	 * %edx, while gcc generates inferior code (movw instead of movl)
-	 * if we tell it to load (u_short) port.
-	 */
-	__asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port));
+	__asm volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
 	return (data);
 }
 
@@ -236,7 +178,7 @@ inl(u_int port)
 {
 	u_int	data;
 
-	__asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port));
+	__asm volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
 	return (data);
 }
 
@@ -278,33 +220,20 @@ inw(u_int port)
 {
 	u_short	data;
 
-	__asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port));
+	__asm volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
 	return (data);
 }
 
 static __inline void
-outbv(u_int port, u_char data)
+outb(u_int port, u_char data)
 {
-	u_char	al;
-	/*
-	 * Use an unnecessary assignment to help gcc's register allocator.
-	 * This make a large difference for gcc-1.40 and a tiny difference
-	 * for gcc-2.6.0.  For gcc-1.40, al had to be ``asm("ax")'' for
-	 * best results.  gcc-2.6.0 can't handle this.
-	 */
-	al = data;
-	__asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port));
+	__asm volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
 }
 
 static __inline void
 outl(u_int port, u_int data)
 {
-	/*
-	 * outl() and outw() aren't used much so we haven't looked at
-	 * possible micro-optimizations such as the unnecessary
-	 * assignment for them.
-	 */
-	__asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port));
+	__asm volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
 }
 
 static __inline void
@@ -334,7 +263,7 @@ outsl(u_int port, const void *addr, size_t cnt)
 static __inline void
 outw(u_int port, u_short data)
 {
-	__asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port));
+	__asm volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
 }
 
 static __inline void
@@ -535,12 +464,9 @@ cpu_mwait(int extensions, int hints)
 static __inline void
 load_fs(u_int sel)
 {
-	register u_int32_t fsbase __asm("ecx");
-
 	/* Preserve the fsbase value across the selector load */
-	fsbase = MSR_FSBASE;
-        __asm __volatile("rdmsr; mov %0,%%fs; wrmsr"
-            : : "rm" (sel), "c" (fsbase) : "eax", "edx");
+	__asm __volatile("rdmsr; mov %0,%%fs; wrmsr"
+	    : : "rm" (sel), "c" (MSR_FSBASE) : "eax", "edx");
 }
 
 #ifndef	MSR_GSBASE
@@ -549,16 +475,13 @@ load_fs(u_int sel)
 static __inline void
 load_gs(u_int sel)
 {
-	register u_int32_t gsbase __asm("ecx");
-
 	/*
 	 * Preserve the gsbase value across the selector load.
 	 * Note that we have to disable interrupts because the gsbase
 	 * being trashed happens to be the kernel gsbase at the time.
 	 */
-	gsbase = MSR_GSBASE;
-        __asm __volatile("pushfq; cli; rdmsr; mov %0,%%gs; wrmsr; popfq"
-            : : "rm" (sel), "c" (gsbase) : "eax", "edx");
+	__asm __volatile("pushfq; cli; rdmsr; mov %0,%%gs; wrmsr; popfq"
+	    : : "rm" (sel), "c" (MSR_GSBASE) : "eax", "edx");
 }
 #else
 /* Usable by userland */
diff --git a/sys/amd64/include/elf.h b/sys/amd64/include/elf.h
index a4c7f79..e5c95f7 100644
--- a/sys/amd64/include/elf.h
+++ b/sys/amd64/include/elf.h
@@ -81,16 +81,14 @@ __ElfType(Auxinfo);
 #define	AT_BASE		7	/* Interpreter's base address. */
 #define	AT_FLAGS	8	/* Flags (unused for i386). */
 #define	AT_ENTRY	9	/* Where interpreter should transfer control. */
-/*
- * The following non-standard values are used in Linux ELF binaries.
- */
 #define	AT_NOTELF	10	/* Program is not ELF ?? */
 #define	AT_UID		11	/* Real uid. */
 #define	AT_EUID		12	/* Effective uid. */
 #define	AT_GID		13	/* Real gid. */
 #define	AT_EGID		14	/* Effective gid. */
+#define	AT_EXECPATH	15	/* Path to the executable. */
 
-#define	AT_COUNT	15	/* Count of defined aux entry types. */
+#define	AT_COUNT	16	/* Count of defined aux entry types. */
 
 /*
  * Relocation types.
diff --git a/sys/amd64/include/endian.h b/sys/amd64/include/endian.h
index 24e59e4..712b5a5 100644
--- a/sys/amd64/include/endian.h
+++ b/sys/amd64/include/endian.h
@@ -69,25 +69,6 @@ extern "C" {
 
 #if defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE_BUILTIN_CONSTANT_P)
 
-#define __word_swap_int_var(x) \
-__extension__ ({ register __uint32_t __X = (x); \
-   __asm ("rorl $16, %0" : "+r" (__X)); \
-   __X; })
-
-#ifdef __OPTIMIZE__
-
-#define	__word_swap_int_const(x) \
-	((((x) & 0xffff0000) >> 16) | \
-	 (((x) & 0x0000ffff) << 16))
-#define	__word_swap_int(x) (__builtin_constant_p(x) ? \
-	__word_swap_int_const(x) : __word_swap_int_var(x))
-
-#else	/* __OPTIMIZE__ */
-
-#define	__word_swap_int(x) __word_swap_int_var(x)
-
-#endif	/* __OPTIMIZE__ */
-
 #define __byte_swap_int_var(x) \
 __extension__ ({ register __uint32_t __X = (x); \
    __asm ("bswap %0" : "+r" (__X)); \
@@ -135,26 +116,6 @@ __extension__ ({ register __uint64_t __X = (x); \
 
 #endif	/* __OPTIMIZE__ */
 
-#define __byte_swap_word_var(x) \
-__extension__ ({ register __uint16_t __X = (x); \
-   __asm ("xchgb %h0, %b0" : "+Q" (__X)); \
-   __X; })
-
-#ifdef __OPTIMIZE__
-
-#define	__byte_swap_word_const(x) \
-	((((x) & 0xff00) >> 8) | \
-	 (((x) & 0x00ff) << 8))
-
-#define	__byte_swap_word(x) (__builtin_constant_p(x) ? \
-	__byte_swap_word_const(x) : __byte_swap_word_var(x))
-
-#else	/* __OPTIMIZE__ */
-
-#define	__byte_swap_word(x) __byte_swap_word_var(x)
-
-#endif	/* __OPTIMIZE__ */
-
 static __inline __uint64_t
 __bswap64(__uint64_t _x)
 {
@@ -172,8 +133,7 @@ __bswap32(__uint32_t _x)
 static __inline __uint16_t
 __bswap16(__uint16_t _x)
 {
-
-	return (__byte_swap_word(_x));
+	return (_x << 8 | _x >> 8);
 }
 
 #define	__htonl(x)	__bswap32(x)
diff --git a/sys/amd64/include/frame.h b/sys/amd64/include/frame.h
index 26c9dd0..12722a4 100644
--- a/sys/amd64/include/frame.h
+++ b/sys/amd64/include/frame.h
@@ -64,9 +64,13 @@ struct trapframe {
 	register_t	tf_r13;
 	register_t	tf_r14;
 	register_t	tf_r15;
-	register_t	tf_trapno;
+	uint32_t	tf_trapno;
+	uint16_t	tf_fs;
+	uint16_t	tf_gs;
 	register_t	tf_addr;
-	register_t	tf_flags;
+	uint32_t	tf_flags;
+	uint16_t	tf_es;
+	uint16_t	tf_ds;
 	/* below portion defined in hardware */
 	register_t	tf_err;
 	register_t	tf_rip;
@@ -76,4 +80,7 @@ struct trapframe {
 	register_t	tf_ss;
 };
 
+#define	TF_HASSEGS	0x1
+/* #define	_MC_HASBASES	0x2 */
+
 #endif /* _MACHINE_FRAME_H_ */
diff --git a/sys/amd64/include/md_var.h b/sys/amd64/include/md_var.h
index 2125b9f..892e19d 100644
--- a/sys/amd64/include/md_var.h
+++ b/sys/amd64/include/md_var.h
@@ -60,6 +60,11 @@ extern	char	sigcode[];
 extern	int	szsigcode;
 extern	uint64_t *vm_page_dump;
 extern	int	vm_page_dump_size;
+extern	int	_udatasel;
+extern	int	_ucodesel;
+extern	int	_ucode32sel;
+extern	int	_ufssel;
+extern	int	_ugssel;
 
 typedef void alias_for_inthand_t(u_int cs, u_int ef, u_int esp, u_int ss);
 struct	thread;
@@ -72,6 +77,14 @@ void	busdma_swi(void);
 void	cpu_setregs(void);
 void	doreti_iret(void) __asm(__STRING(doreti_iret));
 void	doreti_iret_fault(void) __asm(__STRING(doreti_iret_fault));
+void	ld_ds(void) __asm(__STRING(ld_ds));
+void	ld_es(void) __asm(__STRING(ld_es));
+void	ld_fs(void) __asm(__STRING(ld_fs));
+void	ld_gs(void) __asm(__STRING(ld_gs));
+void	ds_load_fault(void) __asm(__STRING(ds_load_fault));
+void	es_load_fault(void) __asm(__STRING(es_load_fault));
+void	fs_load_fault(void) __asm(__STRING(fs_load_fault));
+void	gs_load_fault(void) __asm(__STRING(gs_load_fault));
 void	dump_add_page(vm_paddr_t);
 void	dump_drop_page(vm_paddr_t);
 void	initializecpu(void);
diff --git a/sys/amd64/include/pcb.h b/sys/amd64/include/pcb.h
index e6a5add..7361049 100644
--- a/sys/amd64/include/pcb.h
+++ b/sys/amd64/include/pcb.h
@@ -62,10 +62,6 @@ struct pcb {
 #define	PCB_32BIT	0x40	/* process has 32 bit context (segs etc) */
 #define	PCB_FULLCTX	0x80	/* full context restore on sysret */
 
-	u_int32_t	pcb_ds;
-	u_int32_t	pcb_es;
-	u_int32_t	pcb_fs;
-	u_int32_t	pcb_gs;
 	u_int64_t	pcb_dr0;
 	u_int64_t	pcb_dr1;
 	u_int64_t	pcb_dr2;
@@ -80,6 +76,20 @@ struct pcb {
 
 	/* 32-bit segment descriptor */
 	struct user_segment_descriptor	pcb_gs32sd;
+	/* local tss, with i/o bitmap; NULL for common */
+	struct amd64tss *pcb_tssp;
+};
+
+struct xpcb {
+	struct pcb	xpcb_pcb;
+	register_t	xpcb_cr0;
+	register_t	xpcb_cr2;
+	register_t	xpcb_cr4;
+	register_t	xpcb_kgsbase;
+	struct region_descriptor xpcb_gdt;
+	struct region_descriptor xpcb_idt;
+	struct region_descriptor xpcb_ldt;
+	uint16_t	xpcb_tr;
 };
 
 #ifdef _KERNEL
@@ -87,6 +97,7 @@ struct trapframe;
 
 void	makectx(struct trapframe *, struct pcb *);
 void	savectx(struct pcb *);
+int	savectx2(struct xpcb *);
 #endif
 
 #endif /* _AMD64_PCB_H_ */
diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h
index 23818ca..139281a 100644
--- a/sys/amd64/include/pcpu.h
+++ b/sys/amd64/include/pcpu.h
@@ -62,12 +62,20 @@
 	char	pc_monitorbuf[128] __aligned(128); /* cache line */	\
 	struct	pcpu *pc_prvspace;	/* Self-reference */		\
 	struct	pmap *pc_curpmap;					\
-	struct	amd64tss *pc_tssp;					\
+	struct	amd64tss *pc_tssp;	/* TSS segment active on CPU */	\
+	struct	amd64tss *pc_commontssp;/* Common TSS for the CPU */	\
 	register_t pc_rsp0;						\
 	register_t pc_scratch_rsp;	/* User %rsp in syscall */	\
 	u_int	pc_apic_id;						\
 	u_int   pc_acpi_id;		/* ACPI CPU id */		\
-	struct user_segment_descriptor	*pc_gs32p			\
+	/* Pointer to the CPU %fs descriptor */				\
+	struct user_segment_descriptor	*pc_fs32p;			\
+	/* Pointer to the CPU %gs descriptor */				\
+	struct user_segment_descriptor	*pc_gs32p;			\
+	/* Pointer to the CPU LDT descriptor */				\
+	struct system_segment_descriptor *pc_ldt;			\
+	/* Pointer to the CPU TSS descriptor */				\
+	struct system_segment_descriptor *pc_tss
 	PCPU_XEN_FIELDS
 
 #ifdef _KERNEL
diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h
index 36d3667..c95a9f9 100644
--- a/sys/amd64/include/pmap.h
+++ b/sys/amd64/include/pmap.h
@@ -160,13 +160,7 @@ typedef u_int64_t pml4_entry_t;
 #define	PDESHIFT	(3)
 
 /*
- * Address of current and alternate address space page table maps
- * and directories.
- * XXX it might be saner to just direct map all of physical memory
- * into the kernel using 2MB pages.  We have enough space to do
- * it (2^47 bits of KVM, while current max physical addressability
- * is 2^40 physical bits).  Then we can get rid of the evil hole
- * in the page tables and the evil overlapping.
+ * Address of current address space page table maps and directories.
  */
 #ifdef _KERNEL
 #define	addr_PTmap	(KVADDR(PML4PML4I, 0, 0, 0))
@@ -243,6 +237,10 @@ struct md_page {
 	TAILQ_HEAD(,pv_entry)	pv_list;
 };
 
+/*
+ * The kernel virtual address (KVA) of the level 4 page table page is always
+ * within the direct map (DMAP) region.
+ */
 struct pmap {
 	struct mtx		pm_mtx;
 	pml4_entry_t		*pm_pml4;	/* KVA of level 4 page table */
diff --git a/sys/amd64/include/proc.h b/sys/amd64/include/proc.h
index a3ebd79..33d5181 100644
--- a/sys/amd64/include/proc.h
+++ b/sys/amd64/include/proc.h
@@ -33,6 +33,13 @@
 #ifndef _MACHINE_PROC_H_
 #define	_MACHINE_PROC_H_
 
+#include <machine/segments.h>
+
+struct proc_ldt {
+	caddr_t ldt_base;
+	int     ldt_refcnt;
+};
+
 /*
  * Machine-dependent part of the proc structure for AMD64.
  */
@@ -42,6 +49,8 @@ struct mdthread {
 };
 
 struct mdproc {
+	struct proc_ldt *md_ldt;	/* (t) per-process ldt */
+	struct system_segment_descriptor md_ldt_sd;
 };
 
 #ifdef	_KERNEL
@@ -55,6 +64,18 @@ struct mdproc {
 	    (char *)&td;						\
 } while (0)
 
+void set_user_ldt(struct mdproc *);
+struct proc_ldt *user_ldt_alloc(struct proc *, int);
+void user_ldt_free(struct thread *);
+void user_ldt_deref(struct proc_ldt *);
+struct sysarch_args;
+int sysarch_ldt(struct thread *td, struct sysarch_args *uap, int uap_space);
+int amd64_set_ldt_data(struct thread *td, int start, int num,
+    struct user_segment_descriptor *descs);
+
+extern struct mtx dt_lock;
+extern int max_ldt_segment;
+
 #endif  /* _KERNEL */
 
 #endif /* !_MACHINE_PROC_H_ */
diff --git a/sys/amd64/include/reg.h b/sys/amd64/include/reg.h
index 9ca092f..89211a3 100644
--- a/sys/amd64/include/reg.h
+++ b/sys/amd64/include/reg.h
@@ -56,8 +56,12 @@ struct reg {
 	register_t	r_rdx;
 	register_t	r_rcx;
 	register_t	r_rax;
-	register_t	r_trapno;
-	register_t	r_err;
+	uint32_t	r_trapno;
+	uint16_t	r_fs;
+	uint16_t	r_gs;
+	uint32_t	r_err;
+	uint16_t	r_es;
+	uint16_t	r_ds;
 	register_t	r_rip;
 	register_t	r_cs;
 	register_t	r_rflags;
diff --git a/sys/amd64/include/segments.h b/sys/amd64/include/segments.h
index 1c83d1c..3dca80a 100644
--- a/sys/amd64/include/segments.h
+++ b/sys/amd64/include/segments.h
@@ -108,12 +108,29 @@ struct	gate_descriptor {
 	u_int64_t sd_xx1:32;
 } __packed;
 
+/*
+ * Generic descriptor
+ */
+union	descriptor	{
+	struct	user_segment_descriptor sd;
+	struct	gate_descriptor gd;
+};
+
 	/* system segments and gate types */
 #define	SDT_SYSNULL	 0	/* system null */
+#define	SDT_SYS286TSS	 1	/* system 286 TSS available */
 #define	SDT_SYSLDT	 2	/* system 64 bit local descriptor table */
+#define	SDT_SYS286BSY	 3	/* system 286 TSS busy */
+#define	SDT_SYS286CGT	 4	/* system 286 call gate */
+#define	SDT_SYSTASKGT	 5	/* system task gate */
+#define	SDT_SYS286IGT	 6	/* system 286 interrupt gate */
+#define	SDT_SYS286TGT	 7	/* system 286 trap gate */
+#define	SDT_SYSNULL2	 8	/* system null again */
 #define	SDT_SYSTSS	 9	/* system available 64 bit TSS */
+#define	SDT_SYSNULL3	10	/* system null again */
 #define	SDT_SYSBSY	11	/* system busy 64 bit TSS */
 #define	SDT_SYSCGT	12	/* system 64 bit call gate */
+#define	SDT_SYSNULL4	13	/* system null again */
 #define	SDT_SYSIGT	14	/* system 64 bit interrupt gate */
 #define	SDT_SYSTGT	15	/* system 64 bit trap gate */
 
@@ -195,15 +212,19 @@ struct region_descriptor {
  * Entries in the Global Descriptor Table (GDT)
  */
 #define	GNULL_SEL	0	/* Null Descriptor */
-#define	GCODE_SEL	1	/* Kernel Code Descriptor */
-#define	GDATA_SEL	2	/* Kernel Data Descriptor */
-#define	GUCODE32_SEL	3	/* User 32 bit code Descriptor */
-#define	GUDATA_SEL	4	/* User 32/64 bit Data Descriptor */
-#define	GUCODE_SEL	5	/* User 64 bit Code Descriptor */
-#define	GPROC0_SEL	6	/* TSS for entering kernel etc */
-/* slot 7 is second half of GPROC0_SEL */
-#define	GUGS32_SEL	8	/* User 32 bit GS Descriptor */
-#define	NGDT 		9
+#define	GNULL2_SEL	1	/* Null Descriptor */
+#define	GUFS32_SEL	2	/* User 32 bit %fs Descriptor */
+#define	GUGS32_SEL	3	/* User 32 bit %gs Descriptor */
+#define	GCODE_SEL	4	/* Kernel Code Descriptor */
+#define	GDATA_SEL	5	/* Kernel Data Descriptor */
+#define	GUCODE32_SEL	6	/* User 32 bit code Descriptor */
+#define	GUDATA_SEL	7	/* User 32/64 bit Data Descriptor */
+#define	GUCODE_SEL	8	/* User 64 bit Code Descriptor */
+#define	GPROC0_SEL	9	/* TSS for entering kernel etc */
+/* slot 10 is second half of GPROC0_SEL */
+#define	GUSERLDT_SEL	11	/* LDT */
+/* slot 11 is second half of GUSERLDT_SEL */
+#define	NGDT 		13
 
 #ifdef _KERNEL
 extern struct user_segment_descriptor gdt[];
@@ -218,6 +239,9 @@ void	ssdtosd(struct soft_segment_descriptor *ssdp,
 	    struct user_segment_descriptor *sdp);
 void	ssdtosyssd(struct soft_segment_descriptor *ssdp,
 	    struct system_segment_descriptor *sdp);
+void	update_gdt_gsbase(struct thread *td, uint32_t base);
+void	update_gdt_fsbase(struct thread *td, uint32_t base);
+
 #endif /* _KERNEL */
 
 #endif /* !_MACHINE_SEGMENTS_H_ */
diff --git a/sys/amd64/include/signal.h b/sys/amd64/include/signal.h
index 7a4c2a7..228e2d9 100644
--- a/sys/amd64/include/signal.h
+++ b/sys/amd64/include/signal.h
@@ -78,9 +78,13 @@ struct sigcontext {
 	long	sc_r13;
 	long	sc_r14;
 	long	sc_r15;
-	long	sc_trapno;
+	int	sc_trapno;
+	short	sc_fs;
+	short	sc_gs;
 	long	sc_addr;
-	long	sc_flags;
+	int	sc_flags;
+	short	sc_es;
+	short	sc_ds;
 	long	sc_err;
 	long	sc_rip;
 	long	sc_cs;
@@ -95,7 +99,11 @@ struct sigcontext {
 	long	sc_fpformat;
 	long	sc_ownedfp;
 	long	sc_fpstate[64] __aligned(16);
-	long	sc_spare[8];
+
+	long	sc_fsbase;
+	long	sc_gsbase;
+
+	long	sc_spare[6];
 };
 #endif /* __BSD_VISIBLE */
 
diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h
index 06222f8..d2ff189 100644
--- a/sys/amd64/include/smp.h
+++ b/sys/amd64/include/smp.h
@@ -48,11 +48,13 @@ inthand_t
 	IDTVEC(invlcache),	/* Write back and invalidate cache */
 	IDTVEC(ipi_intr_bitmap_handler), /* Bitmap based IPIs */ 
 	IDTVEC(cpustop),	/* CPU stops & waits to be restarted */
+	IDTVEC(cpususpend),	/* CPU suspends & waits to be resumed */
 	IDTVEC(rendezvous);	/* handle CPU rendezvous */
 
 /* functions in mp_machdep.c */
 void	cpu_add(u_int apic_id, char boot_cpu);
 void	cpustop_handler(void);
+void	cpususpend_handler(void);
 void	init_secondary(void);
 void	ipi_selected(u_int cpus, u_int ipi);
 void	ipi_all_but_self(u_int ipi);
diff --git a/sys/amd64/include/sysarch.h b/sys/amd64/include/sysarch.h
index 67c8a4a..6c3e6c9 100644
--- a/sys/amd64/include/sysarch.h
+++ b/sys/amd64/include/sysarch.h
@@ -35,6 +35,15 @@
 #ifndef _MACHINE_SYSARCH_H_
 #define _MACHINE_SYSARCH_H_
 
+#define I386_GET_LDT	0
+#define I386_SET_LDT	1
+#define	LDT_AUTO_ALLOC	0xffffffff
+				/* I386_IOPL */
+#define I386_GET_IOPERM	3
+#define I386_SET_IOPERM	4
+
+/* XXX Not implementable #define I386_VM86	6 */
+
 #define	I386_GET_FSBASE		7
 #define	I386_SET_FSBASE		8
 #define	I386_GET_GSBASE		9
@@ -46,6 +55,18 @@
 #define	AMD64_GET_GSBASE	130
 #define	AMD64_SET_GSBASE	131
 
+struct i386_ldt_args {
+	unsigned int start;
+	struct user_segment_descriptor *descs __packed;
+	unsigned int num;
+};
+
+struct i386_ioperm_args {
+	unsigned int start;
+	unsigned int length;
+	int	enable;
+};
+
 #ifndef _KERNEL
 #include <sys/cdefs.h>
 
@@ -56,6 +77,15 @@ int amd64_set_fsbase(void *);
 int amd64_set_gsbase(void *);
 int sysarch(int, void *);
 __END_DECLS
+#else
+struct thread;
+union descriptor;
+
+int amd64_get_ldt(struct thread *, struct i386_ldt_args *);
+int amd64_set_ldt(struct thread *, struct i386_ldt_args *,
+    struct user_segment_descriptor *);
+int amd64_get_ioperm(struct thread *, struct i386_ioperm_args *);
+int amd64_set_ioperm(struct thread *, struct i386_ioperm_args *);
 #endif
 
 #endif /* !_MACHINE_SYSARCH_H_ */
diff --git a/sys/amd64/include/ucontext.h b/sys/amd64/include/ucontext.h
index 5c13803..c5bbd65 100644
--- a/sys/amd64/include/ucontext.h
+++ b/sys/amd64/include/ucontext.h
@@ -32,9 +32,16 @@
 #ifndef _MACHINE_UCONTEXT_H_
 #define	_MACHINE_UCONTEXT_H_
 
+/*
+ * mc_trapno bits. Shall be in sync with TF_XXX.
+ */
+#define	_MC_HASSEGS	0x1
+#define	_MC_HASBASES	0x2
+#define	_MC_FLAG_MASK	(_MC_HASSEGS | _MC_HASBASES)
+
 typedef struct __mcontext {
 	/*
-	 * The first 20 fields must match the definition of
+	 * The first 24 fields must match the definition of
 	 * sigcontext. So that we can support sigcontext
 	 * and ucontext_t at the same time.
 	 */
@@ -54,9 +61,13 @@ typedef struct __mcontext {
 	__register_t	mc_r13;
 	__register_t	mc_r14;
 	__register_t	mc_r15;
-	__register_t	mc_trapno;
+	__uint32_t	mc_trapno;
+	__uint16_t	mc_fs;
+	__uint16_t	mc_gs;
 	__register_t	mc_addr;
-	__register_t	mc_flags;
+	__uint32_t	mc_flags;
+	__uint16_t	mc_es;
+	__uint16_t	mc_ds;
 	__register_t	mc_err;
 	__register_t	mc_rip;
 	__register_t	mc_cs;
@@ -65,6 +76,7 @@ typedef struct __mcontext {
 	__register_t	mc_ss;
 
 	long	mc_len;			/* sizeof(mcontext_t) */
+
 #define	_MC_FPFMT_NODEV		0x10000	/* device not present or configured */
 #define	_MC_FPFMT_XMM		0x10002
 	long	mc_fpformat;
@@ -76,7 +88,11 @@ typedef struct __mcontext {
 	 * See <machine/fpu.h> for the internals of mc_fpstate[].
 	 */
 	long	mc_fpstate[64] __aligned(16);
-	long	mc_spare[8];
+
+	__register_t	mc_fsbase;
+	__register_t	mc_gsbase;
+
+	long	mc_spare[6];
 } mcontext_t;
 
 #endif /* !_MACHINE_UCONTEXT_H_ */
diff --git a/sys/amd64/linux32/linux.h b/sys/amd64/linux32/linux.h
index 94372d6..0e30382 100644
--- a/sys/amd64/linux32/linux.h
+++ b/sys/amd64/linux32/linux.h
@@ -179,8 +179,8 @@ struct l_mmap_argv {
 	l_int		prot;
 	l_int		flags;
 	l_int		fd;
-	l_off_t		pgoff;
-} __packed;
+	l_ulong		pgoff;
+};
 
 /*
  * stat family of syscalls
diff --git a/sys/amd64/linux32/linux32_locore.s b/sys/amd64/linux32/linux32_locore.s
index 6045925..36e1abf 100644
--- a/sys/amd64/linux32/linux32_locore.s
+++ b/sys/amd64/linux32/linux32_locore.s
@@ -11,8 +11,6 @@
 NON_GPROF_ENTRY(linux_sigcode)
 	call	*LINUX_SIGF_HANDLER(%esp)
 	leal	LINUX_SIGF_SC(%esp),%ebx	/* linux scp */
-	mov	LINUX_SC_ES(%ebx),%es
-	mov	LINUX_SC_DS(%ebx),%ds
 	movl	%esp, %ebx			/* pass sigframe */
 	push	%eax				/* fake ret addr */
 	movl	$LINUX_SYS_linux_sigreturn,%eax	/* linux_sigreturn() */
@@ -24,8 +22,6 @@ linux_rt_sigcode:
 	call	*LINUX_RT_SIGF_HANDLER(%esp)
 	leal	LINUX_RT_SIGF_UC(%esp),%ebx	/* linux ucp */
 	leal	LINUX_RT_SIGF_SC(%ebx),%ecx	/* linux sigcontext */
-	mov	LINUX_SC_ES(%ecx),%es
-	mov	LINUX_SC_DS(%ecx),%ds
 	push	%eax				/* fake ret addr */
 	movl	$LINUX_SYS_linux_rt_sigreturn,%eax   /* linux_rt_sigreturn() */
 	int	$0x80				/* enter kernel with args */
diff --git a/sys/amd64/linux32/linux32_machdep.c b/sys/amd64/linux32/linux32_machdep.c
index eb91623..42ea070 100644
--- a/sys/amd64/linux32/linux32_machdep.c
+++ b/sys/amd64/linux32/linux32_machdep.c
@@ -716,8 +716,8 @@ linux_clone(struct thread *td, struct linux_clone_args *args)
 				    sd.sd_long, sd.sd_def32, sd.sd_gran);
 #endif
 			td2->td_pcb->pcb_gsbase = (register_t)info.base_addr;
-			td2->td_pcb->pcb_gs32sd = sd;
-			td2->td_pcb->pcb_gs = GSEL(GUGS32_SEL, SEL_UPL);
+/* XXXKIB		td2->td_pcb->pcb_gs32sd = sd; */
+			td2->td_frame->tf_gs = GSEL(GUGS32_SEL, SEL_UPL);
 			td2->td_pcb->pcb_flags |= PCB_GS32BIT | PCB_32BIT;
 		}
 	}
@@ -1359,12 +1359,9 @@ linux_set_thread_area(struct thread *td,
 		    sd.sd_gran);
 #endif
 
-	critical_enter();
 	td->td_pcb->pcb_gsbase = (register_t)info.base_addr;
-	td->td_pcb->pcb_gs32sd = *PCPU_GET(gs32p) = sd;
 	td->td_pcb->pcb_flags |= PCB_32BIT | PCB_GS32BIT;
-	wrmsr(MSR_KGSBASE, td->td_pcb->pcb_gsbase);
-	critical_exit();
+	update_gdt_gsbase(td, info.base_addr);
 
 	return (0);
 }
diff --git a/sys/amd64/linux32/linux32_sysvec.c b/sys/amd64/linux32/linux32_sysvec.c
index a834c98..a82c182 100644
--- a/sys/amd64/linux32/linux32_sysvec.c
+++ b/sys/amd64/linux32/linux32_sysvec.c
@@ -290,7 +290,6 @@ elf_linux_fixup(register_t **stack_base, struct image_params *imgp)
 	return 0;
 }
 
-extern int _ucodesel, _ucode32sel, _udatasel;
 extern unsigned long linux_sznonrtsigcode;
 
 static void
@@ -360,13 +359,7 @@ linux_rt_sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 
 	bsd_to_linux_sigset(mask, &frame.sf_sc.uc_sigmask);
 
-	frame.sf_sc.uc_mcontext.sc_mask	= frame.sf_sc.uc_sigmask.__bits[0];
-	frame.sf_sc.uc_mcontext.sc_gs     = rgs();
-	frame.sf_sc.uc_mcontext.sc_fs     = rfs();
-	__asm __volatile("mov %%es,%0" :
-	    "=rm" (frame.sf_sc.uc_mcontext.sc_es));
-	__asm __volatile("mov %%ds,%0" :
-	    "=rm" (frame.sf_sc.uc_mcontext.sc_ds));
+	frame.sf_sc.uc_mcontext.sc_mask   = frame.sf_sc.uc_sigmask.__bits[0];
 	frame.sf_sc.uc_mcontext.sc_edi    = regs->tf_rdi;
 	frame.sf_sc.uc_mcontext.sc_esi    = regs->tf_rsi;
 	frame.sf_sc.uc_mcontext.sc_ebp    = regs->tf_rbp;
@@ -376,6 +369,10 @@ linux_rt_sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	frame.sf_sc.uc_mcontext.sc_eax    = regs->tf_rax;
 	frame.sf_sc.uc_mcontext.sc_eip    = regs->tf_rip;
 	frame.sf_sc.uc_mcontext.sc_cs     = regs->tf_cs;
+	frame.sf_sc.uc_mcontext.sc_gs     = regs->tf_gs;
+	frame.sf_sc.uc_mcontext.sc_fs     = regs->tf_fs;
+	frame.sf_sc.uc_mcontext.sc_es     = regs->tf_es;
+	frame.sf_sc.uc_mcontext.sc_ds     = regs->tf_ds;
 	frame.sf_sc.uc_mcontext.sc_eflags = regs->tf_rflags;
 	frame.sf_sc.uc_mcontext.sc_esp_at_signal = regs->tf_rsp;
 	frame.sf_sc.uc_mcontext.sc_ss     = regs->tf_ss;
@@ -413,11 +410,11 @@ linux_rt_sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	regs->tf_rflags &= ~(PSL_T | PSL_D);
 	regs->tf_cs = _ucode32sel;
 	regs->tf_ss = _udatasel;
-	load_ds(_udatasel);
-	td->td_pcb->pcb_ds = _udatasel;
-	load_es(_udatasel);
-	td->td_pcb->pcb_es = _udatasel;
-	/* leave user %fs and %gs untouched */
+	regs->tf_ds = _udatasel;
+	regs->tf_es = _udatasel;
+	regs->tf_fs = _ufssel;
+	regs->tf_gs = _ugssel;
+	regs->tf_flags = TF_HASSEGS;
 	PROC_LOCK(p);
 	mtx_lock(&psp->ps_mtx);
 }
@@ -495,10 +492,10 @@ linux_sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	 * Build the signal context to be used by sigreturn.
 	 */
 	frame.sf_sc.sc_mask   = lmask.__bits[0];
-	frame.sf_sc.sc_gs     = rgs();
-	frame.sf_sc.sc_fs     = rfs();
-	__asm __volatile("mov %%es,%0" : "=rm" (frame.sf_sc.sc_es));
-	__asm __volatile("mov %%ds,%0" : "=rm" (frame.sf_sc.sc_ds));
+	frame.sf_sc.sc_gs     = regs->tf_gs;
+	frame.sf_sc.sc_fs     = regs->tf_fs;
+	frame.sf_sc.sc_es     = regs->tf_es;
+	frame.sf_sc.sc_ds     = regs->tf_ds;
 	frame.sf_sc.sc_edi    = regs->tf_rdi;
 	frame.sf_sc.sc_esi    = regs->tf_rsi;
 	frame.sf_sc.sc_ebp    = regs->tf_rbp;
@@ -535,11 +532,11 @@ linux_sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	regs->tf_rflags &= ~(PSL_T | PSL_D);
 	regs->tf_cs = _ucode32sel;
 	regs->tf_ss = _udatasel;
-	load_ds(_udatasel);
-	td->td_pcb->pcb_ds = _udatasel;
-	load_es(_udatasel);
-	td->td_pcb->pcb_es = _udatasel;
-	/* leave user %fs and %gs untouched */
+	regs->tf_ds = _udatasel;
+	regs->tf_es = _udatasel;
+	regs->tf_fs = _ufssel;
+	regs->tf_gs = _ugssel;
+	regs->tf_flags = TF_HASSEGS;
 	PROC_LOCK(p);
 	mtx_lock(&psp->ps_mtx);
 }
@@ -624,7 +621,6 @@ linux_sigreturn(struct thread *td, struct linux_sigreturn_args *args)
 	/*
 	 * Restore signal context.
 	 */
-	/* Selectors were restored by the trampoline. */
 	regs->tf_rdi    = frame.sf_sc.sc_edi;
 	regs->tf_rsi    = frame.sf_sc.sc_esi;
 	regs->tf_rbp    = frame.sf_sc.sc_ebp;
@@ -634,6 +630,10 @@ linux_sigreturn(struct thread *td, struct linux_sigreturn_args *args)
 	regs->tf_rax    = frame.sf_sc.sc_eax;
 	regs->tf_rip    = frame.sf_sc.sc_eip;
 	regs->tf_cs     = frame.sf_sc.sc_cs;
+	regs->tf_ds     = frame.sf_sc.sc_ds;
+	regs->tf_es     = frame.sf_sc.sc_es;
+	regs->tf_fs     = frame.sf_sc.sc_fs;
+	regs->tf_gs     = frame.sf_sc.sc_gs;
 	regs->tf_rflags = eflags;
 	regs->tf_rsp    = frame.sf_sc.sc_esp_at_signal;
 	regs->tf_ss     = frame.sf_sc.sc_ss;
@@ -722,7 +722,10 @@ linux_rt_sigreturn(struct thread *td, struct linux_rt_sigreturn_args *args)
 	/*
 	 * Restore signal context
 	 */
-	/* Selectors were restored by the trampoline. */
+	regs->tf_gs	= context->sc_gs;
+	regs->tf_fs	= context->sc_fs;
+	regs->tf_es	= context->sc_es;
+	regs->tf_ds	= context->sc_ds;
 	regs->tf_rdi    = context->sc_edi;
 	regs->tf_rsi    = context->sc_esi;
 	regs->tf_rbp    = context->sc_ebp;
@@ -827,27 +830,30 @@ exec_linux_setregs(td, entry, stack, ps_strings)
 	struct trapframe *regs = td->td_frame;
 	struct pcb *pcb = td->td_pcb;
 
+	mtx_lock(&dt_lock);
+	if (td->td_proc->p_md.md_ldt != NULL)
+		user_ldt_free(td);
+	else
+		mtx_unlock(&dt_lock);
+
 	critical_enter();
 	wrmsr(MSR_FSBASE, 0);
 	wrmsr(MSR_KGSBASE, 0);	/* User value while we're in the kernel */
 	pcb->pcb_fsbase = 0;
 	pcb->pcb_gsbase = 0;
 	critical_exit();
-	load_ds(_udatasel);
-	load_es(_udatasel);
-	load_fs(_udatasel);
-	load_gs(_udatasel);
-	pcb->pcb_ds = _udatasel;
-	pcb->pcb_es = _udatasel;
-	pcb->pcb_fs = _udatasel;
-	pcb->pcb_gs = _udatasel;
 	pcb->pcb_initial_fpucw = __LINUX_NPXCW__;
 
 	bzero((char *)regs, sizeof(struct trapframe));
 	regs->tf_rip = entry;
 	regs->tf_rsp = stack;
 	regs->tf_rflags = PSL_USER | (regs->tf_rflags & PSL_T);
+	regs->tf_gs = _ugssel;
+	regs->tf_fs = _ufssel;
+	regs->tf_es = _udatasel;
+	regs->tf_ds = _udatasel;
 	regs->tf_ss = _udatasel;
+	regs->tf_flags = TF_HASSEGS;
 	regs->tf_cs = _ucode32sel;
 	regs->tf_rbx = ps_strings;
 	load_cr0(rcr0() | CR0_MP | CR0_TS);
@@ -1047,6 +1053,16 @@ struct sysentvec elf_linux_sysvec = {
 	.sv_flags	= SV_ABI_LINUX | SV_ILP32 | SV_IA32
 };
 
+static char GNULINUX_ABI_VENDOR[] = "GNU";
+
+static Elf_Brandnote linux32_brandnote = {
+	.hdr.n_namesz	= sizeof(GNULINUX_ABI_VENDOR),
+	.hdr.n_descsz	= 16,
+	.hdr.n_type	= 1,
+	.vendor		= GNULINUX_ABI_VENDOR,
+	.flags		= 0
+};
+
 static Elf32_Brandinfo linux_brand = {
 	.brand		= ELFOSABI_LINUX,
 	.machine	= EM_386,
@@ -1055,7 +1071,8 @@ static Elf32_Brandinfo linux_brand = {
 	.interp_path	= "/lib/ld-linux.so.1",
 	.sysvec		= &elf_linux_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &linux32_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 static Elf32_Brandinfo linux_glibc2brand = {
@@ -1066,7 +1083,8 @@ static Elf32_Brandinfo linux_glibc2brand = {
 	.interp_path	= "/lib/ld-linux.so.2",
 	.sysvec		= &elf_linux_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &linux32_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 Elf32_Brandinfo *linux_brandlist[] = {
diff --git a/sys/amd64/pci/pci_cfgreg.c b/sys/amd64/pci/pci_cfgreg.c
index 8f757a7..61a2db5 100644
--- a/sys/amd64/pci/pci_cfgreg.c
+++ b/sys/amd64/pci/pci_cfgreg.c
@@ -119,6 +119,7 @@ pci_docfgregread(int bus, int slot, int func, int reg, int bytes)
 {
 
 	if (cfgmech == CFGMECH_PCIE &&
+	    (bus >= pcie_minbus && bus <= pcie_maxbus) &&
 	    (bus != 0 || !(1 << slot & pcie_badslots)))
 		return (pciereg_cfgread(bus, slot, func, reg, bytes));
 	else
@@ -158,6 +159,7 @@ pci_cfgregwrite(int bus, int slot, int func, int reg, u_int32_t data, int bytes)
 {
 
 	if (cfgmech == CFGMECH_PCIE &&
+	    (bus >= pcie_minbus && bus <= pcie_maxbus) &&
 	    (bus != 0 || !(1 << slot & pcie_badslots)))
 		pciereg_cfgwrite(bus, slot, func, reg, data, bytes);
 	else
diff --git a/sys/arm/arm/elf_machdep.c b/sys/arm/arm/elf_machdep.c
index 693eab1..174deeb 100644
--- a/sys/arm/arm/elf_machdep.c
+++ b/sys/arm/arm/elf_machdep.c
@@ -84,7 +84,8 @@ static Elf32_Brandinfo freebsd_brand_info = {
 	.interp_path	= "/libexec/ld-elf.so.1",
 	.sysvec		= &elf32_freebsd_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &elf32_freebsd_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 SYSINIT(elf32, SI_SUB_EXEC, SI_ORDER_ANY,
@@ -99,7 +100,8 @@ static Elf32_Brandinfo freebsd_brand_oinfo = {
 	.interp_path	= "/usr/libexec/ld-elf.so.1",
 	.sysvec		= &elf32_freebsd_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &elf32_freebsd_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 SYSINIT(oelf32, SI_SUB_EXEC, SI_ORDER_ANY,
diff --git a/sys/arm/arm/locore.S b/sys/arm/arm/locore.S
index b465edc..e56a91c 100644
--- a/sys/arm/arm/locore.S
+++ b/sys/arm/arm/locore.S
@@ -110,7 +110,7 @@ from_ram:
 	nop
 #endif
 	adr	r7, Lunmapped
-	bic     r7, r7, #0xff000000
+	bic     r7, r7, #0xf0000000
 	orr     r7, r7, #PHYSADDR
 
 
@@ -144,7 +144,7 @@ Lunmapped:
 	ldmia	r4!, {r1,r2,r3}   /* # of sections, VA, PA|attr */
 	cmp	r1, #0
 	adrne	r5, 2b
-	bicne	r5, r5, #0xff000000
+	bicne	r5, r5, #0xf0000000
 	orrne	r5, r5, #PHYSADDR
 	movne	pc, r5
 
diff --git a/sys/arm/arm/trap.c b/sys/arm/arm/trap.c
index fc03b52..e6150c0 100644
--- a/sys/arm/arm/trap.c
+++ b/sys/arm/arm/trap.c
@@ -520,7 +520,8 @@ dab_fatal(trapframe_t *tf, u_int fsr, u_int far, struct thread *td, struct ksig
 	printf(", pc =%08x\n\n", tf->tf_pc);
 
 #ifdef KDB
-	kdb_trap(fsr, 0, tf);
+	if (debugger_on_panic || kdb_active)
+		kdb_trap(fsr, 0, tf);
 #endif
 	panic("Fatal abort");
 	/*NOTREACHED*/
@@ -530,7 +531,7 @@ dab_fatal(trapframe_t *tf, u_int fsr, u_int far, struct thread *td, struct ksig
  * dab_align() handles the following data aborts:
  *
  *  FAULT_ALIGN_0 - Alignment fault
- *  FAULT_ALIGN_0 - Alignment fault
+ *  FAULT_ALIGN_1 - Alignment fault
  *
  * These faults are fatal if they happen in kernel mode. Otherwise, we
  * deliver a bus error to the process.
diff --git a/sys/arm/at91/files.at91 b/sys/arm/at91/files.at91
index 9142749..c9aafbc 100644
--- a/sys/arm/at91/files.at91
+++ b/sys/arm/at91/files.at91
@@ -13,12 +13,10 @@ arm/at91/at91_spi.c		optional	at91_spi		\
 	dependency	"spibus_if.h"
 arm/at91/at91_tc.c		optional	at91_tc
 arm/at91/at91_twi.c		optional	at91_twi
-arm/at91/at91_udp.c		optional	at91_udp
 arm/at91/if_ate.c		optional	ate
 arm/at91/uart_bus_at91usart.c	optional	uart
 arm/at91/uart_cpu_at91rm9200usart.c	optional	uart
 arm/at91/uart_dev_at91usart.c	optional	uart
-dev/usb/controller/ohci_atmelarm.c	optional	ohci
 #
 # All the boards we support
 #
diff --git a/sys/arm/conf/AVILA b/sys/arm/conf/AVILA
index 510e8af..cd1d338 100644
--- a/sys/arm/conf/AVILA
+++ b/sys/arm/conf/AVILA
@@ -111,7 +111,6 @@ device		wlan_tkip	# 802.11 TKIP support
 device		wlan_xauth
 
 device		ath		# Atheros pci/cardbus NIC's
-options		ATH_SUPPORT_TDMA
 options 	ATH_DEBUG
 options		ATH_DIAGAPI
 #options		ATH_TX99_DIAG
diff --git a/sys/arm/conf/CAMBRIA b/sys/arm/conf/CAMBRIA
index c938b91..70655de 100644
--- a/sys/arm/conf/CAMBRIA
+++ b/sys/arm/conf/CAMBRIA
@@ -114,7 +114,6 @@ device		wlan_tkip	# 802.11 TKIP support
 device		wlan_xauth
 
 device		ath		# Atheros pci/cardbus NIC's
-options		ATH_SUPPORT_TDMA
 options 	ATH_DEBUG
 options		ATH_DIAGAPI
 #options		ATH_TX99_DIAG
diff --git a/sys/arm/conf/HL200 b/sys/arm/conf/HL200
index 5bbce3f..de322ec 100644
--- a/sys/arm/conf/HL200
+++ b/sys/arm/conf/HL200
@@ -103,7 +103,6 @@ device          umass           # Disks/Mass storage - Requires scbus and da
 device          ural            # Ralink Technology RT2500USB wireless NICs
 device          rum             # Ralink Technology RT2501USB wireless NICs
 device          urio            # Diamond Rio 500 MP3 player
-device          uscanner        # Scanners
 # USB Ethernet, requires miibus
 device		miibus
 device          aue             # ADMtek USB Ethernet
diff --git a/sys/arm/conf/KB920X b/sys/arm/conf/KB920X
index 00af54b..1f688f8 100644
--- a/sys/arm/conf/KB920X
+++ b/sys/arm/conf/KB920X
@@ -104,7 +104,6 @@ device          umass           # Disks/Mass storage - Requires scbus and da
 device          ural            # Ralink Technology RT2500USB wireless NICs
 device          rum             # Ralink Technology RT2501USB wireless NICs
 device          urio            # Diamond Rio 500 MP3 player
-device          uscanner        # Scanners
 # USB Ethernet, requires miibus
 device		miibus
 device          aue             # ADMtek USB Ethernet
diff --git a/sys/arm/include/atomic.h b/sys/arm/include/atomic.h
index 71e31c3..9f16220 100644
--- a/sys/arm/include/atomic.h
+++ b/sys/arm/include/atomic.h
@@ -264,22 +264,23 @@ atomic_clear_32(volatile uint32_t *address, uint32_t clearmask)
 static __inline uint32_t
 atomic_fetchadd_32(volatile uint32_t *p, uint32_t v)
 {
-	uint32_t start, ras_start = ARM_RAS_START;
+	uint32_t start, tmp, ras_start = ARM_RAS_START;
 
 	__asm __volatile("1:\n"
 	    "adr	%1, 1b\n"
 	    "str	%1, [%0]\n"
 	    "adr	%1, 2f\n"
 	    "str	%1, [%0, #4]\n"
-	    "ldr	%1, [%2]\n"
-	    "add	%1, %1, %3\n"
-	    "str	%0, [%2]\n"
+	    "ldr	%1, [%3]\n"
+	    "mov	%2, %1\n"
+	    "add	%2, %2, %4\n"
+	    "str	%2, [%3]\n"
 	    "2:\n"
-	    "mov	%3, #0\n"
-	    "str	%3, [%0]\n"
-	    "mov	%3, #0xffffffff\n"
-	    "str	%3, [%0, #4]\n"
-	    : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (v)
+	    "mov	%2, #0\n"
+	    "str	%2, [%0]\n"
+	    "mov	%2, #0xffffffff\n"
+	    "str	%2, [%0, #4]\n"
+	    : "+r" (ras_start), "=r" (start), "=r" (tmp), "+r" (p), "+r" (v)
 	    : : "memory");
 	return (start);
 }
diff --git a/sys/arm/include/elf.h b/sys/arm/include/elf.h
index c516864..ee2843f 100644
--- a/sys/arm/include/elf.h
+++ b/sys/arm/include/elf.h
@@ -75,8 +75,9 @@ __ElfType(Auxinfo);
 #define AT_EUID         12      /* Effective uid. */
 #define AT_GID          13      /* Real gid. */
 #define AT_EGID         14      /* Effective gid. */
+#define	AT_EXECPATH	15	/* Path to the executable. */
 
-#define AT_COUNT        15      /* Count of defined aux entry types. */
+#define AT_COUNT        16      /* Count of defined aux entry types. */
 
 #define	R_ARM_COUNT		33	/* Count of defined relocation types. */
 
diff --git a/sys/arm/include/vmparam.h b/sys/arm/include/vmparam.h
index bc82fd0..d54671d 100644
--- a/sys/arm/include/vmparam.h
+++ b/sys/arm/include/vmparam.h
@@ -141,8 +141,6 @@
 #define SGROWSIZ        (128*1024)
 #define MAXSLP		20
 
-#define VM_PROT_READ_IS_EXEC
-
 #ifdef ARM_USE_SMALL_ALLOC
 #define UMA_MD_SMALL_ALLOC
 #endif /* ARM_USE_SMALL_ALLOC */
diff --git a/sys/boot/forth/loader.conf b/sys/boot/forth/loader.conf
index 7443d68..59a5018 100644
--- a/sys/boot/forth/loader.conf
+++ b/sys/boot/forth/loader.conf
@@ -275,6 +275,7 @@ if_tl_load="NO"			# Texas Instruments TNETE100 ("ThunderLAN")
 if_tx_load="NO"			# SMC 83c17x Fast Ethernet
 if_txp_load="NO"		# 3Com 3XP Typhoon/Sidewinder (3CR990)
 if_vge_load="NO"		# VIA VT6122 PCI Gigabit Ethernet
+if_uath_load="NO"		# Atheros USB wireless for AR5005UG & AR5005UX
 if_udav_load="NO"		# Davicom DM9601 USB Ethernet
 if_upgt_load="NO"		# Conexant/Intersil PrismGT USB wireless
 if_ural_load="NO"		# Ralink Technology USB wireless
@@ -379,7 +380,6 @@ ulpt_load="NO"			# Printer
 ums_load="NO"			# Mouse
 umass_load="NO"			# Mass Storage Devices
 umodem_load="NO"		# Modems
-uscanner_load="NO"		# Scanners
 if_aue_load="NO"		# ADMtek USB ethernet
 if_axe_load="NO"		# ASIX Electronics AX88172 USB ethernet
 if_cue_load="NO"		# CATC USB ethernet
@@ -402,7 +402,7 @@ accf_http_load="NO"		# Wait for full HTTP request accept filter
 random_load="NO"		# Random device
 speaker_load="NO"		# AT speaker module
 coretemp_load="NO"		# Intel Core CPU temperature monitor
-k8temp_load="NO"		# AMD K8 temperature monitor
+amdtemp_load="NO"		# AMD K8/K10/K11 temperature monitor
 
 ##############################################################
 ###  ACPI settings  ##########################################
diff --git a/sys/boot/i386/libi386/Makefile b/sys/boot/i386/libi386/Makefile
index 3430bef..9a2a3fa 100644
--- a/sys/boot/i386/libi386/Makefile
+++ b/sys/boot/i386/libi386/Makefile
@@ -31,6 +31,10 @@ CFLAGS+= -DDISK_DEBUG
 .if !defined(BOOT_HIDE_SERIAL_NUMBERS)
 # Export serial numbers, UUID, and asset tag from loader.
 CFLAGS+= -DSMBIOS_SERIAL_NUMBERS
+.if defined(BOOT_LITTLE_ENDIAN_UUID)
+# Use little-endian UUID format as defined in SMBIOS 2.6.
+CFLAGS+= -DSMBIOS_LITTLE_ENDIAN_UUID
+.endif
 .endif
 
 .if !defined(LOADER_NO_GPT_SUPPORT)
diff --git a/sys/boot/i386/libi386/bioscd.c b/sys/boot/i386/libi386/bioscd.c
index 93de958..a6dba0c 100644
--- a/sys/boot/i386/libi386/bioscd.c
+++ b/sys/boot/i386/libi386/bioscd.c
@@ -173,9 +173,9 @@ bc_add(int biosdev)
 static void
 bc_print(int verbose)
 {
-	int i;
 	char line[80];
-    
+	int i;
+
 	for (i = 0; i < nbcinfo; i++) {
 		sprintf(line, "    cd%d: Device 0x%x\n", i,
 		    bcinfo[i].bc_sp.sp_devicespec);
@@ -235,7 +235,7 @@ bc_strategy(void *devdata, int rw, daddr_t dblk, size_t size, char *buf,
 	if (dblk % (BIOSCD_SECSIZE / DEV_BSIZE) != 0)
 		return (EINVAL);
 	dblk /= (BIOSCD_SECSIZE / DEV_BSIZE);
-	DEBUG("read %d from %d to %p", blks, dblk, buf);
+	DEBUG("read %d from %lld to %p", blks, dblk, buf);
 
 	if (rsize)
 		*rsize = 0;
@@ -244,9 +244,9 @@ bc_strategy(void *devdata, int rw, daddr_t dblk, size_t size, char *buf,
 		return (EIO);
 	}
 #ifdef BD_SUPPORT_FRAGS
-	DEBUG("bc_strategy: frag read %d from %d+%d to %p", 
+	DEBUG("frag read %d from %lld+%d to %p", 
 	    fragsize, dblk, blks, buf + (blks * BIOSCD_SECSIZE));
-	if (fragsize && bc_read(unit, dblk + blks, 1, fragsize)) {
+	if (fragsize && bc_read(unit, dblk + blks, 1, fragbuf)) {
 		DEBUG("frag read error");
 		return(EIO);
 	}
@@ -257,11 +257,15 @@ bc_strategy(void *devdata, int rw, daddr_t dblk, size_t size, char *buf,
 	return (0);
 }
 
+/* Max number of sectors to bounce-buffer at a time. */
+#define	CD_BOUNCEBUF	8
+
 static int
 bc_read(int unit, daddr_t dblk, int blks, caddr_t dest)
 {
-	u_int result, retry;
-	static unsigned short packet[8];
+	u_int maxfer, resid, result, retry, x;
+	caddr_t bbuf, p, xp;
+	static struct edd_packet packet;
 	int biosdev;
 #ifdef DISK_DEBUG
 	int error;
@@ -275,47 +279,77 @@ bc_read(int unit, daddr_t dblk, int blks, caddr_t dest)
 	if (blks == 0)
 		return (0);
 
+	/* Decide whether we have to bounce */
+	if (VTOP(dest) >> 20 != 0) {
+		/* 
+		 * The destination buffer is above first 1MB of
+		 * physical memory so we have to arrange a suitable
+		 * bounce buffer.
+		 */
+		x = min(CD_BOUNCEBUF, (unsigned)blks);
+		bbuf = alloca(x * BIOSCD_SECSIZE);
+		maxfer = x;
+	} else {
+		bbuf = NULL;
+		maxfer = 0;
+	}
+	
 	biosdev = bc_unit2bios(unit);
-	/*
-	 * Loop retrying the operation a couple of times.  The BIOS
-	 * may also retry.
-	 */
-	for (retry = 0; retry < 3; retry++) {
-		/* If retrying, reset the drive */
-		if (retry > 0) {
+	resid = blks;
+	p = dest;
+
+	while (resid > 0) {
+		if (bbuf)
+			xp = bbuf;
+		else
+			xp = p;
+		x = resid;
+		if (maxfer > 0)
+			x = min(x, maxfer);
+
+		/*
+		 * Loop retrying the operation a couple of times.  The BIOS
+		 * may also retry.
+		 */
+		for (retry = 0; retry < 3; retry++) {
+			/* If retrying, reset the drive */
+			if (retry > 0) {
+				v86.ctl = V86_FLAGS;
+				v86.addr = 0x13;
+				v86.eax = 0;
+				v86.edx = biosdev;
+				v86int();
+			}
+
+			packet.len = 0x10;
+			packet.count = x;
+			packet.offset = VTOPOFF(xp);
+			packet.seg = VTOPSEG(xp);
+			packet.lba = dblk;
 			v86.ctl = V86_FLAGS;
 			v86.addr = 0x13;
-			v86.eax = 0;
+			v86.eax = 0x4200;
 			v86.edx = biosdev;
+			v86.ds = VTOPSEG(&packet);
+			v86.esi = VTOPOFF(&packet);
 			v86int();
+			result = (v86.efl & PSL_C);
+			if (result == 0)
+				break;
 		}
-	    
-		packet[0] = 0x10;
-		packet[1] = blks;
-		packet[2] = VTOPOFF(dest);
-		packet[3] = VTOPSEG(dest);
-		packet[4] = dblk & 0xffff;
-		packet[5] = dblk >> 16;
-		packet[6] = 0;
-		packet[7] = 0;
-		v86.ctl = V86_FLAGS;
-		v86.addr = 0x13;
-		v86.eax = 0x4200;
-		v86.edx = biosdev;
-		v86.ds = VTOPSEG(packet);
-		v86.esi = VTOPOFF(packet);
-		v86int();
-		result = (v86.efl & PSL_C);
-		if (result == 0)
-			break;
-	}
 	
 #ifdef DISK_DEBUG
-	error = (v86.eax >> 8) & 0xff;
+		error = (v86.eax >> 8) & 0xff;
 #endif
-	DEBUG("%d sectors from %ld to %p (0x%x) %s", blks, dblk, dest,
-	    VTOP(dest), result ? "failed" : "ok");
-	DEBUG("unit %d  status 0x%x",  unit, error);
+		DEBUG("%d sectors from %lld to %p (0x%x) %s", x, dblk, p,
+		    VTOP(p), result ? "failed" : "ok");
+		DEBUG("unit %d  status 0x%x", unit, error);
+		if (bbuf != NULL)
+			bcopy(bbuf, p, x * BIOSCD_SECSIZE);
+		p += (x * BIOSCD_SECSIZE);
+		dblk += x;
+		resid -= x;
+	}
 	
 /*	hexdump(dest, (blks * BIOSCD_SECSIZE)); */
 	return(0);
diff --git a/sys/boot/i386/libi386/biosdisk.c b/sys/boot/i386/libi386/biosdisk.c
index 9d31325..a54418f 100644
--- a/sys/boot/i386/libi386/biosdisk.c
+++ b/sys/boot/i386/libi386/biosdisk.c
@@ -1125,14 +1125,6 @@ bd_realstrategy(void *devdata, int rw, daddr_t dblk, size_t size, char *buf, siz
 /* Max number of sectors to bounce-buffer if the request crosses a 64k boundary */
 #define FLOPPY_BOUNCEBUF	18
 
-struct edd_packet {
-    uint16_t	len;
-    uint16_t	count;
-    uint16_t	offset;
-    uint16_t	seg;
-    uint64_t	lba;
-};
-
 static int
 bd_edd_io(struct open_disk *od, daddr_t dblk, int blks, caddr_t dest, int write)
 {
diff --git a/sys/boot/i386/libi386/libi386.h b/sys/boot/i386/libi386/libi386.h
index ccae099..4ac5cb5 100644
--- a/sys/boot/i386/libi386/libi386.h
+++ b/sys/boot/i386/libi386/libi386.h
@@ -52,6 +52,14 @@ struct i386_devdesc
     } d_kind;
 };
 
+struct edd_packet {
+    uint16_t	len;
+    uint16_t	count;
+    uint16_t	offset;
+    uint16_t	seg;
+    uint64_t	lba;
+};
+                 
 int	i386_getdev(void **vdev, const char *devspec, const char **path);
 char	*i386_fmtdev(void *vdev);
 int	i386_setcurrdev(struct env_var *ev, int flags, const void *value);
diff --git a/sys/boot/i386/libi386/smbios.c b/sys/boot/i386/libi386/smbios.c
index dd39cc2..51044d6 100644
--- a/sys/boot/i386/libi386/smbios.c
+++ b/sys/boot/i386/libi386/smbios.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2005, 2006 Jung-uk Kim <jkim@FreeBSD.org>
+ * Copyright (c) 2005-2009 Jung-uk Kim <jkim@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -29,6 +29,7 @@ __FBSDID("$FreeBSD$");
 
 #include <stand.h>
 #include <bootstrap.h>
+#include <sys/endian.h>
 
 #include "btxv86.h"
 #include "libi386.h"
@@ -37,17 +38,18 @@ __FBSDID("$FreeBSD$");
  * Detect SMBIOS and export information about the SMBIOS into the
  * environment.
  *
- * System Management BIOS Reference Specification, v2.4 Final
- * http://www.dmtf.org/standards/published_documents/DSP0134.pdf
+ * System Management BIOS Reference Specification, v2.6 Final
+ * http://www.dmtf.org/standards/published_documents/DSP0134_2.6.0.pdf
  */
 
 /*
- * Spec. 2.1.1 SMBIOS Structure Table Entry Point
+ * 2.1.1 SMBIOS Structure Table Entry Point
  *
- * 'The SMBIOS Entry Point structure, described below, can be located by
- * application software by searching for the anchor-string on paragraph
- * (16-byte) boundaries within the physical memory address range
- * 000F0000h to 000FFFFFh.'
+ * "On non-EFI systems, the SMBIOS Entry Point structure, described below, can
+ * be located by application software by searching for the anchor-string on
+ * paragraph (16-byte) boundaries within the physical memory address range
+ * 000F0000h to 000FFFFFh. This entry point encapsulates an intermediate anchor
+ * string that is used by some existing DMI browsers."
  */
 #define	SMBIOS_START		0xf0000
 #define	SMBIOS_LENGTH		0x10000
@@ -55,87 +57,174 @@ __FBSDID("$FreeBSD$");
 #define	SMBIOS_SIG		"_SM_"
 #define	SMBIOS_DMI_SIG		"_DMI_"
 
+#define	SMBIOS_GET8(base, off)	(*(uint8_t *)((base) + (off)))
+#define	SMBIOS_GET16(base, off)	(*(uint16_t *)((base) + (off)))
+#define	SMBIOS_GET32(base, off)	(*(uint32_t *)((base) + (off)))
+
+#define	SMBIOS_GETLEN(base)	SMBIOS_GET8(base, 0x01)
+#define	SMBIOS_GETSTR(base)	((base) + SMBIOS_GETLEN(base))
+
+static uint32_t	smbios_enabled_memory = 0;
+static uint32_t	smbios_old_enabled_memory = 0;
 static uint8_t	smbios_enabled_sockets = 0;
 static uint8_t	smbios_populated_sockets = 0;
 
-static uint8_t	*smbios_parse_table(const uint8_t *dmi);
-static void	smbios_setenv(const char *name, const uint8_t *dmi,
-		    const int offset);
-static uint8_t	smbios_checksum(const caddr_t addr, const uint8_t len);
-static uint8_t	*smbios_sigsearch(const caddr_t addr, const uint32_t len);
+static uint8_t
+smbios_checksum(const caddr_t addr, const uint8_t len)
+{
+	uint8_t		sum;
+	int		i;
 
-#ifdef SMBIOS_SERIAL_NUMBERS
-static void	smbios_setuuid(const char *name, const uint8_t *dmi,
-		    const int offset);
-#endif
+	for (sum = 0, i = 0; i < len; i++)
+		sum += SMBIOS_GET8(addr, i);
+	return (sum);
+}
 
-void
-smbios_detect(void)
+static caddr_t
+smbios_sigsearch(const caddr_t addr, const uint32_t len)
 {
-	uint8_t		*smbios, *dmi, *addr;
-	uint16_t	i, length, count;
-	uint32_t	paddr;
-	char		buf[4];
+	caddr_t		cp;
 
-	/* locate and validate the SMBIOS */
-	smbios = smbios_sigsearch(PTOV(SMBIOS_START), SMBIOS_LENGTH);
-	if (smbios == NULL)
-		return;
+	/* Search on 16-byte boundaries. */
+	for (cp = addr; cp < addr + len; cp += SMBIOS_STEP)
+		if (strncmp(cp, SMBIOS_SIG, 4) == 0 &&
+		    smbios_checksum(cp, SMBIOS_GET8(cp, 0x05)) == 0 &&
+		    strncmp(cp + 0x10, SMBIOS_DMI_SIG, 5) == 0 &&
+		    smbios_checksum(cp + 0x10, 0x0f) == 0)
+			return (cp);
+	return (NULL);
+}
 
-	length = *(uint16_t *)(smbios + 0x16);	/* Structure Table Length */
-	paddr = *(uint32_t *)(smbios + 0x18);	/* Structure Table Address */
-	count = *(uint16_t *)(smbios + 0x1c);	/* No of SMBIOS Structures */
-
-	for (dmi = addr = PTOV(paddr), i = 0;
-	     dmi - addr < length && i < count; i++)
-		dmi = smbios_parse_table(dmi);
-	sprintf(buf, "%d", smbios_enabled_sockets);
-	setenv("smbios.socket.enabled", buf, 1);
-	sprintf(buf, "%d", smbios_populated_sockets);
-	setenv("smbios.socket.populated", buf, 1);
+static void
+smbios_setenv(const char *name, caddr_t addr, const int offset)
+{
+	caddr_t		cp;
+	int		i, idx;
+
+	idx = SMBIOS_GET8(addr, offset);
+	if (idx != 0) {
+		cp = SMBIOS_GETSTR(addr);
+		for (i = 1; i < idx; i++)
+			cp += strlen(cp) + 1;
+		setenv(name, cp, 1);
+	}
 }
 
-static uint8_t *
-smbios_parse_table(const uint8_t *dmi)
+#ifdef SMBIOS_SERIAL_NUMBERS
+
+#define	UUID_SIZE		16
+#define	UUID_TYPE		uint32_t
+#define	UUID_STEP		sizeof(UUID_TYPE)
+#define	UUID_ALL_BITS		(UUID_SIZE / UUID_STEP)
+#define	UUID_GET(base, off)	(*(UUID_TYPE *)((base) + (off)))
+
+static void
+smbios_setuuid(const char *name, const caddr_t addr, const int ver)
 {
-	uint8_t		*dp;
+	char		uuid[37];
+	int		i, ones, zeros;
+	UUID_TYPE	n;
+	uint32_t	f1;
+	uint16_t	f2, f3;
+
+	for (i = 0, ones = 0, zeros = 0; i < UUID_SIZE; i += UUID_STEP) {
+		n = UUID_GET(addr, i) + 1;
+		if (zeros == 0 && n == 0)
+			ones++;
+		else if (ones == 0 && n == 1)
+			zeros++;
+		else
+			break;
+	}
 
-	switch(dmi[0]) {
-	case 0:		/* Type 0: BIOS */
-		smbios_setenv("smbios.bios.vendor", dmi, 0x04);
-		smbios_setenv("smbios.bios.version", dmi, 0x05);
-		smbios_setenv("smbios.bios.reldate", dmi, 0x08);
+	if (ones != UUID_ALL_BITS && zeros != UUID_ALL_BITS) {
+		/*
+		 * 3.3.2.1 System UUID
+		 *
+		 * "Although RFC 4122 recommends network byte order for all
+		 * fields, the PC industry (including the ACPI, UEFI, and
+		 * Microsoft specifications) has consistently used
+		 * little-endian byte encoding for the first three fields:
+		 * time_low, time_mid, time_hi_and_version. The same encoding,
+		 * also known as wire format, should also be used for the
+		 * SMBIOS representation of the UUID."
+		 *
+		 * Note: We use network byte order for backward compatibility
+		 * unless SMBIOS version is 2.6+ or little-endian is forced.
+		 */
+#ifndef SMBIOS_LITTLE_ENDIAN_UUID
+		if (ver < 0x0206) {
+			f1 = ntohl(SMBIOS_GET32(addr, 0));
+			f2 = ntohs(SMBIOS_GET16(addr, 4));
+			f3 = ntohs(SMBIOS_GET16(addr, 6));
+		} else
+#endif
+		{
+			f1 = le32toh(SMBIOS_GET32(addr, 0));
+			f2 = le16toh(SMBIOS_GET16(addr, 4));
+			f3 = le16toh(SMBIOS_GET16(addr, 6));
+		}
+		sprintf(uuid,
+		    "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x",
+		    f1, f2, f3, SMBIOS_GET8(addr, 8), SMBIOS_GET8(addr, 9),
+		    SMBIOS_GET8(addr, 10), SMBIOS_GET8(addr, 11),
+		    SMBIOS_GET8(addr, 12), SMBIOS_GET8(addr, 13),
+		    SMBIOS_GET8(addr, 14), SMBIOS_GET8(addr, 15));
+		setenv(name, uuid, 1);
+	}
+}
+
+#undef UUID_SIZE
+#undef UUID_TYPE
+#undef UUID_STEP
+#undef UUID_ALL_BITS
+#undef UUID_GET
+
+#endif
+
+static caddr_t
+smbios_parse_table(const caddr_t addr, const int ver)
+{
+	caddr_t		cp;
+	int		proc, size, osize, type;
+
+	type = SMBIOS_GET8(addr, 0);	/* 3.1.2 Structure Header Format */
+	switch(type) {
+	case 0:		/* 3.3.1 BIOS Information (Type 0) */
+		smbios_setenv("smbios.bios.vendor", addr, 0x04);
+		smbios_setenv("smbios.bios.version", addr, 0x05);
+		smbios_setenv("smbios.bios.reldate", addr, 0x08);
 		break;
 
-	case 1:		/* Type 1: System */
-		smbios_setenv("smbios.system.maker", dmi, 0x04);
-		smbios_setenv("smbios.system.product", dmi, 0x05);
-		smbios_setenv("smbios.system.version", dmi, 0x06);
+	case 1:		/* 3.3.2 System Information (Type 1) */
+		smbios_setenv("smbios.system.maker", addr, 0x04);
+		smbios_setenv("smbios.system.product", addr, 0x05);
+		smbios_setenv("smbios.system.version", addr, 0x06);
 #ifdef SMBIOS_SERIAL_NUMBERS
-		smbios_setenv("smbios.system.serial", dmi, 0x07);
-		smbios_setuuid("smbios.system.uuid", dmi, 0x08);
+		smbios_setenv("smbios.system.serial", addr, 0x07);
+		smbios_setuuid("smbios.system.uuid", addr + 0x08, ver);
 #endif
 		break;
 
-	case 2:		/* Type 2: Base Board (or Module) */
-		smbios_setenv("smbios.planar.maker", dmi, 0x04);
-		smbios_setenv("smbios.planar.product", dmi, 0x05);
-		smbios_setenv("smbios.planar.version", dmi, 0x06);
+	case 2:		/* 3.3.3 Base Board (or Module) Information (Type 2) */
+		smbios_setenv("smbios.planar.maker", addr, 0x04);
+		smbios_setenv("smbios.planar.product", addr, 0x05);
+		smbios_setenv("smbios.planar.version", addr, 0x06);
 #ifdef SMBIOS_SERIAL_NUMBERS
-		smbios_setenv("smbios.planar.serial", dmi, 0x07);
+		smbios_setenv("smbios.planar.serial", addr, 0x07);
 #endif
 		break;
 
-	case 3:		/* Type 3: System Enclosure or Chassis */
-		smbios_setenv("smbios.chassis.maker", dmi, 0x04);
-		smbios_setenv("smbios.chassis.version", dmi, 0x06);
+	case 3:		/* 3.3.4 System Enclosure or Chassis (Type 3) */
+		smbios_setenv("smbios.chassis.maker", addr, 0x04);
+		smbios_setenv("smbios.chassis.version", addr, 0x06);
 #ifdef SMBIOS_SERIAL_NUMBERS
-		smbios_setenv("smbios.chassis.serial", dmi, 0x07);
-		smbios_setenv("smbios.chassis.tag", dmi, 0x08);
+		smbios_setenv("smbios.chassis.serial", addr, 0x07);
+		smbios_setenv("smbios.chassis.tag", addr, 0x08);
 #endif
 		break;
 
-	case 4:		/* Type 4: Processor Information */
+	case 4:		/* 3.3.5 Processor Information (Type 4) */
 		/*
 		 * Offset 18h: Processor Status
 		 *
@@ -153,99 +242,106 @@ smbios_parse_table(const uint8_t *dmi)
 		 *		5-6h - Reserved
 		 *		7h - Other
 		 */
-		if ((dmi[0x18] & 0x07) == 1)
+		proc = SMBIOS_GET8(addr, 0x18);
+		if ((proc & 0x07) == 1)
 			smbios_enabled_sockets++;
-		if (dmi[0x18] & 0x40)
+		if ((proc & 0x40) != 0)
 			smbios_populated_sockets++;
 		break;
 
-	default: /* skip other types */
+	case 6:		/* 3.3.7 Memory Module Information (Type 6, Obsolete) */
+		/*
+		 * Offset 0Ah: Enabled Size
+		 *
+		 * Bit 7	Bank connection
+		 *		1 - Double-bank connection
+		 *		0 - Single-bank connection
+		 * Bit 6:0	Size (n), where 2**n is the size in MB
+		 *		7Dh - Not determinable (Installed Size only)
+		 *		7Eh - Module is installed, but no memory
+		 *		      has been enabled
+		 *		7Fh - Not installed
+		 */
+		osize = SMBIOS_GET8(addr, 0x0a) & 0x7f;
+		if (osize > 0 && osize < 22)
+			smbios_old_enabled_memory += 1 << (osize + 10);
 		break;
-	}
-	
-	/* find structure terminator */
-	dp = __DECONST(uint8_t *, dmi + dmi[1]);
-	while (dp[0] != 0 || dp[1] != 0)
-		dp++;
 
-	return(dp + 2);
-}
+	case 17:	/* 3.3.18 Memory Device (Type 17) */
+		/*
+		 * Offset 0Ch: Size
+		 *
+		 * Bit 15	Granularity
+		 *		1 - Value is in kilobytes units
+		 *		0 - Value is in megabytes units
+		 * Bit 14:0	Size
+		 */
+		size = SMBIOS_GET16(addr, 0x0c);
+		if (size != 0 && size != 0xffff)
+			smbios_enabled_memory += (size & 0x8000) != 0 ?
+			    (size & 0x7fff) : (size << 10);
+		break;
 
-static void
-smbios_setenv(const char *name, const uint8_t *dmi, const int offset)
-{
-	char		*cp = __DECONST(char *, dmi + dmi[1]);
-	int		i;
+	default:	/* skip other types */
+		break;
+	}
 
-	/* skip undefined string */
-	if (dmi[offset] == 0)
-		return;
+	/* Find structure terminator. */
+	cp = SMBIOS_GETSTR(addr);
+	while (SMBIOS_GET16(cp, 0) != 0)
+		cp++;
 
-	for (i = 0; i < dmi[offset] - 1; i++)
-		cp += strlen(cp) + 1;
-	setenv(name, cp, 1);
+	return (cp + 2);
 }
 
-static uint8_t
-smbios_checksum(const caddr_t addr, const uint8_t len)
+void
+smbios_detect(void)
 {
-	const uint8_t	*cp = addr;
-	uint8_t		sum;
-	int		i;
-
-	for (sum = 0, i = 0; i < len; i++)
-		sum += cp[i];
+	char		buf[16];
+	caddr_t		addr, dmi, smbios;
+	size_t		count, length;
+	uint32_t	paddr;
+	int		i, major, minor, ver;
 
-	return(sum);
-}
+	/* Search signatures and validate checksums. */
+	smbios = smbios_sigsearch(PTOV(SMBIOS_START), SMBIOS_LENGTH);
+	if (smbios == NULL)
+		return;
 
-static uint8_t *
-smbios_sigsearch(const caddr_t addr, const uint32_t len)
-{
-	caddr_t		cp;
+	length = SMBIOS_GET16(smbios, 0x16);	/* Structure Table Length */
+	paddr = SMBIOS_GET32(smbios, 0x18);	/* Structure Table Address */
+	count = SMBIOS_GET16(smbios, 0x1c);	/* No of SMBIOS Structures */
+	ver = SMBIOS_GET8(smbios, 0x1e);	/* SMBIOS BCD Revision */
 
-	/* search on 16-byte boundaries */
-	for (cp = addr; cp < addr + len; cp += SMBIOS_STEP) {
-		/* compare signature, validate checksum */
-		if (!strncmp(cp, SMBIOS_SIG, 4)) {
-			if (smbios_checksum(cp, *(uint8_t *)(cp + 0x05)))
-				continue;
-			if (strncmp(cp + 0x10, SMBIOS_DMI_SIG, 5))
-				continue;
-			if (smbios_checksum(cp + 0x10, 0x0f))
-				continue;
-
-			return(cp);
-		}
+	if (ver != 0) {
+		major = ver >> 4;
+		minor = ver & 0x0f;
+		if (major > 9 || minor > 9)
+			ver = 0;
 	}
-
-	return(NULL);
-}
-
-#ifdef SMBIOS_SERIAL_NUMBERS
-static void
-smbios_setuuid(const char *name, const uint8_t *dmi, const int offset)
-{
-	const uint8_t	*idp = dmi + offset;
-	int		i, f = 0, z = 0;
-	char		uuid[37];
-
-	for (i = 0; i < 16; i++) {
-		if (idp[i] == 0xff)
-			f++;
-		else if (idp[i] == 0x00)
-			z++;
-		else
-			break;
+	if (ver == 0) {
+		major = SMBIOS_GET8(smbios, 0x06); /* SMBIOS Major Version */
+		minor = SMBIOS_GET8(smbios, 0x07); /* SMBIOS Minor Version */
 	}
-	if (f != 16 && z != 16) {
-		sprintf(uuid, "%02x%02x%02x%02x-"
-		    "%02x%02x-%02x%02x-%02x%02x-"
-		    "%02x%02x%02x%02x%02x%02x",
-		    idp[0], idp[1], idp[2], idp[3],
-		    idp[4], idp[5], idp[6], idp[7], idp[8], idp[9],
-		    idp[10], idp[11], idp[12], idp[13], idp[14], idp[15]);
-		setenv(name, uuid, 1);
+	ver = (major << 8) | minor;
+
+	addr = PTOV(paddr);
+	for (dmi = addr, i = 0; dmi < addr + length && i < count; i++)
+		dmi = smbios_parse_table(dmi, ver);
+
+	sprintf(buf, "%d.%d", major, minor);
+	setenv("smbios.version", buf, 1);
+	if (smbios_enabled_memory > 0 || smbios_old_enabled_memory > 0) {
+		sprintf(buf, "%u", smbios_enabled_memory > 0 ?
+		    smbios_enabled_memory : smbios_old_enabled_memory);
+		setenv("smbios.memory.enabled", buf, 1);
+	}
+	if (smbios_enabled_sockets > 0) {
+		sprintf(buf, "%u", smbios_enabled_sockets);
+		setenv("smbios.socket.enabled", buf, 1);
+	}
+	if (smbios_populated_sockets > 0) {
+		sprintf(buf, "%u", smbios_populated_sockets);
+		setenv("smbios.socket.populated", buf, 1);
 	}
 }
-#endif
diff --git a/sys/boot/pc98/libpc98/Makefile b/sys/boot/pc98/libpc98/Makefile
index cc93af0..1a28b6b 100644
--- a/sys/boot/pc98/libpc98/Makefile
+++ b/sys/boot/pc98/libpc98/Makefile
@@ -5,8 +5,9 @@ INTERNALLIB=
 
 .PATH:	${.CURDIR}/../../i386/libi386
 
-SRCS=	bioscd.c biosdisk.c biosmem.c biospnp.c biospci.c biossmap.c \
-	bootinfo.c bootinfo32.c comconsole.c devicename.c elf32_freebsd.c \
+SRCS=	bioscd.c biosdisk.c biosmem.c biospnp.c \
+	biospci.c biossmap.c bootinfo.c bootinfo32.c \
+	comconsole.c devicename.c elf32_freebsd.c \
 	i386_copy.c i386_module.c nullconsole.c pxe.c pxetramp.s \
 	time.c vidconsole.c
 
diff --git a/sys/boot/pc98/libpc98/bioscd.c b/sys/boot/pc98/libpc98/bioscd.c
index 40d455b..3e1ba71 100644
--- a/sys/boot/pc98/libpc98/bioscd.c
+++ b/sys/boot/pc98/libpc98/bioscd.c
@@ -170,9 +170,9 @@ bc_add(int biosdev)
 static void
 bc_print(int verbose)
 {
-	int i;
 	char line[80];
-    
+	int i;
+
 	for (i = 0; i < nbcinfo; i++) {
 		sprintf(line, "    cd%d: Device 0x%x\n", i,
 		    bcinfo[i].bc_sp.sp_devicespec);
@@ -232,7 +232,7 @@ bc_strategy(void *devdata, int rw, daddr_t dblk, size_t size, char *buf,
 	if (dblk % (BIOSCD_SECSIZE / DEV_BSIZE) != 0)
 		return (EINVAL);
 	dblk /= (BIOSCD_SECSIZE / DEV_BSIZE);
-	DEBUG("read %d from %d to %p", blks, dblk, buf);
+	DEBUG("read %d from %lld to %p", blks, dblk, buf);
 
 	if (rsize)
 		*rsize = 0;
@@ -241,9 +241,9 @@ bc_strategy(void *devdata, int rw, daddr_t dblk, size_t size, char *buf,
 		return (EIO);
 	}
 #ifdef BD_SUPPORT_FRAGS
-	DEBUG("bc_strategy: frag read %d from %d+%d to %p", 
+	DEBUG("frag read %d from %lld+%d to %p", 
 	    fragsize, dblk, blks, buf + (blks * BIOSCD_SECSIZE));
-	if (fragsize && bc_read(unit, dblk + blks, 1, fragsize)) {
+	if (fragsize && bc_read(unit, dblk + blks, 1, fragbuf)) {
 		DEBUG("frag read error");
 		return(EIO);
 	}
@@ -254,11 +254,14 @@ bc_strategy(void *devdata, int rw, daddr_t dblk, size_t size, char *buf,
 	return (0);
 }
 
+/* Max number of sectors to bounce-buffer at a time. */
+#define	CD_BOUNCEBUF	8
+
 static int
 bc_read(int unit, daddr_t dblk, int blks, caddr_t dest)
 {
-	u_int result, retry;
-	static unsigned short packet[8];
+	u_int maxfer, resid, result, retry, x;
+	caddr_t bbuf, p, xp;
 	int biosdev;
 #ifdef DISK_DEBUG
 	int error;
@@ -272,40 +275,73 @@ bc_read(int unit, daddr_t dblk, int blks, caddr_t dest)
 	if (blks == 0)
 		return (0);
 
+	/* Decide whether we have to bounce */
+	if (VTOP(dest) >> 20 != 0) {
+		/* 
+		 * The destination buffer is above first 1MB of
+		 * physical memory so we have to arrange a suitable
+		 * bounce buffer.
+		 */
+		x = min(CD_BOUNCEBUF, (unsigned)blks);
+		bbuf = alloca(x * BIOSCD_SECSIZE);
+		maxfer = x;
+	} else {
+		bbuf = NULL;
+		maxfer = 0;
+	}
+	
 	biosdev = bc_unit2bios(unit);
-	/*
-	 * Loop retrying the operation a couple of times.  The BIOS
-	 * may also retry.
-	 */
-	for (retry = 0; retry < 3; retry++) {
-		/* If retrying, reset the drive */
-		if (retry > 0) {
+	resid = blks;
+	p = dest;
+
+	while (resid > 0) {
+		if (bbuf)
+			xp = bbuf;
+		else
+			xp = p;
+		x = resid;
+		if (maxfer > 0)
+			x = min(x, maxfer);
+
+		/*
+		 * Loop retrying the operation a couple of times.  The BIOS
+		 * may also retry.
+		 */
+		for (retry = 0; retry < 3; retry++) {
+			/* If retrying, reset the drive */
+			if (retry > 0) {
+				v86.ctl = V86_FLAGS;
+				v86.addr = 0x1b;
+				v86.eax = 0x0300 | biosdev;
+				v86int();
+			}
+
 			v86.ctl = V86_FLAGS;
 			v86.addr = 0x1b;
-			v86.eax = 0x0300 | biosdev;
+			v86.eax = 0x0600 | (biosdev & 0x7f);
+			v86.ebx = x * BIOSCD_SECSIZE;
+			v86.ecx = dblk & 0xffff;
+			v86.edx = (dblk >> 16) & 0xffff;
+			v86.ebp = VTOPOFF(xp);
+			v86.es = VTOPSEG(xp);
 			v86int();
+			result = (v86.efl & PSL_C);
+			if (result == 0)
+				break;
 		}
-
-		v86.ctl = V86_FLAGS;
-		v86.addr = 0x1b;
-		v86.eax = 0x0600 | (biosdev & 0x7f);
-		v86.ebx = blks * BIOSCD_SECSIZE;
-		v86.ecx = dblk & 0xffff;
-		v86.edx = (dblk >> 16) & 0xffff;
-		v86.ebp = VTOPOFF(dest);
-		v86.es = VTOPSEG(dest);
-		v86int();
-		result = (v86.efl & PSL_C);
-		if (result == 0)
-			break;
-	}
 	
 #ifdef DISK_DEBUG
-	error = (v86.eax >> 8) & 0xff;
+		error = (v86.eax >> 8) & 0xff;
 #endif
-	DEBUG("%d sectors from %ld to %p (0x%x) %s", blks, dblk, dest,
-	    VTOP(dest), result ? "failed" : "ok");
-	DEBUG("unit %d  status 0x%x",  unit, error);
+		DEBUG("%d sectors from %lld to %p (0x%x) %s", x, dblk, p,
+		    VTOP(p), result ? "failed" : "ok");
+		DEBUG("unit %d  status 0x%x", unit, error);
+		if (bbuf != NULL)
+			bcopy(bbuf, p, x * BIOSCD_SECSIZE);
+		p += (x * BIOSCD_SECSIZE);
+		dblk += x;
+		resid -= x;
+	}
 	
 /*	hexdump(dest, (blks * BIOSCD_SECSIZE)); */
 	return(0);
diff --git a/sys/boot/pc98/libpc98/biosdisk.c b/sys/boot/pc98/libpc98/biosdisk.c
index dc5f82d..2f18285 100644
--- a/sys/boot/pc98/libpc98/biosdisk.c
+++ b/sys/boot/pc98/libpc98/biosdisk.c
@@ -269,17 +269,8 @@ bd_print(int verbose)
 
 		/* Check for a "dedicated" disk */
 		for (j = 0; j < od->od_nslices; j++) {
-		    switch(dptr[j].dp_mid) {
-		    case DOSMID_386BSD:
-		        sprintf(line, "      disk%ds%d", i, j + 1);
-			bd_printbsdslice(od,
-			    dptr[j].dp_scyl * od->od_hds * od->od_sec +
-			    dptr[j].dp_shd * od->od_sec + dptr[j].dp_ssect,
-			    line, verbose);
-			break;
-		    default:
-			break;
-		    }
+		    sprintf(line, "      disk%ds%d", i, j + 1);
+		    bd_printslice(od, &dptr[j], line, verbose);
 		}
 	    }
 	    bd_closedisk(od);
@@ -311,6 +302,52 @@ display_size(uint64_t size)
 }
 
 /*
+ * Print information about slices on a disk.  For the size calculations we
+ * assume a 512 byte sector.
+ */
+static void
+bd_printslice(struct open_disk *od, struct pc98_partition *dp, char *prefix,
+	int verbose)
+{
+	int cylsecs, start, size;
+	char stats[80];
+	char line[80];
+
+	cylsecs = od->od_hds * od->od_sec;
+	start = dp->dp_scyl * cylsecs + dp->dp_shd * od->od_sec + dp->dp_ssect;
+	size = (dp->dp_ecyl - dp->dp_scyl + 1) * cylsecs;
+
+	if (verbose)
+		sprintf(stats, " %s (%d - %d)", display_size(size),
+		    start, start + size);
+	else
+		stats[0] = '\0';
+
+	switch(dp->dp_mid & PC98_MID_MASK) {
+	case PC98_MID_386BSD:
+		bd_printbsdslice(od, start, prefix, verbose);
+		return;
+	case 0x00:				/* unused partition */
+		return;
+	case 0x01:
+		sprintf(line, "%s: FAT-12%s\n", prefix, stats);
+		break;
+	case 0x11:
+	case 0x20:
+	case 0x21:
+	case 0x22:
+	case 0x23:
+	case 0x24:
+		sprintf(line, "%s: FAT-16%s\n", prefix, stats);
+		break;
+	default:
+		sprintf(line, "%s: Unknown fs: 0x%x %s\n", prefix, dp->dp_mid,
+		    stats);
+	}
+	pager_output(line);
+}
+
+/*
  * Print out each valid partition in the disklabel of a FreeBSD slice.
  * For size calculations, we assume a 512 byte sector size.
  */
@@ -349,7 +386,7 @@ bd_printbsdslice(struct open_disk *od, daddr_t offset, char *prefix,
 
 	    /* Only print out statistics in verbose mode */
 	    if (verbose)
-		sprintf(line, "  %s%c: %s %s (%d - %d)\n", prefix, 'a' + i,
+	        sprintf(line, "  %s%c: %s %s (%d - %d)\n", prefix, 'a' + i,
 		    (lp->d_partitions[i].p_fstype == FS_SWAP) ? "swap " : 
 		    (lp->d_partitions[i].p_fstype == FS_VINUM) ? "vinum" :
 		    "FFS  ",
@@ -625,9 +662,9 @@ bd_bestslice(struct open_disk *od)
 
 	dp = &od->od_slicetab[0];
 	for (i = 0; i < od->od_nslices; i++, dp++) {
-		switch(dp->dp_mid & 0x7f) {
-		case DOSMID_386BSD & 0x7f:		/* FreeBSD */
-			if ((dp->dp_mid & 0x80) &&
+		switch(dp->dp_mid & PC98_MID_MASK) {
+		case PC98_MID_386BSD:		/* FreeBSD */
+			if ((dp->dp_mid & PC98_MID_BOOTABLE) &&
 			    (preflevel > PREF_FBSD_ACT)) {
 				pref = i;
 				preflevel = PREF_FBSD_ACT;
@@ -643,7 +680,7 @@ bd_bestslice(struct open_disk *od)
 		case 0x22:
 		case 0x23:
 		case 0x63:
-			if ((dp->dp_mid & 0x80) &&
+			if ((dp->dp_mid & PC98_MID_BOOTABLE) &&
 			    (preflevel > PREF_DOS_ACT)) {
 				pref = i;
 				preflevel = PREF_DOS_ACT;
diff --git a/sys/boot/pc98/libpc98/time.c b/sys/boot/pc98/libpc98/time.c
index b8dbe02..5d832bb 100644
--- a/sys/boot/pc98/libpc98/time.c
+++ b/sys/boot/pc98/libpc98/time.c
@@ -33,23 +33,21 @@ __FBSDID("$FreeBSD$");
 #include "bootstrap.h"
 #include "libi386.h"
 
+static int	bios_seconds(void);
+
 /*
- * Return the time in seconds since the beginning of the day.
- *
- * If we pass midnight, don't wrap back to 0.
+ * Return the BIOS time-of-day value.
  *
  * XXX uses undocumented BCD support from libstand.
  */
-
-time_t
-time(time_t *t)
+static int
+bios_seconds(void)
 {
-    static time_t	lasttime, now;
     int			hr, minute, sec;
     unsigned char	bios_time[6];
-
+    
     v86.ctl = 0;
-    v86.addr = 0x1c;            /* int 0x1c, function 0 */
+    v86.addr = 0x1c;		/* int 0x1c, function 0 */
     v86.eax = 0x0000;
     v86.es  = VTOPSEG(bios_time);
     v86.ebx = VTOPOFF(bios_time);
@@ -59,7 +57,20 @@ time(time_t *t)
     minute = bcd2bin(bios_time[4]);
     sec = bcd2bin(bios_time[5]);
     
-    now = hr * 3600 + minute * 60 + sec;
+    return (hr * 3600 + minute * 60 + sec);
+}
+
+/*
+ * Return the time in seconds since the beginning of the day.
+ */
+time_t
+time(time_t *t)
+{
+    static time_t lasttime;
+    time_t now;
+
+    now = bios_seconds();
+
     if (now < lasttime)
 	now += 24 * 3600;
     lasttime = now;
diff --git a/sys/boot/pc98/loader/Makefile b/sys/boot/pc98/loader/Makefile
index 91d5150..9f2e32f 100644
--- a/sys/boot/pc98/loader/Makefile
+++ b/sys/boot/pc98/loader/Makefile
@@ -18,6 +18,7 @@ CFLAGS+=	-DLOADER_TFTP_SUPPORT
 CFLAGS+=	-DLOADER_NFS_SUPPORT
 .endif
 
+# Include bcache code.
 HAVE_BCACHE=	yes
 
 # Enable PnP and ISA-PnP code.
diff --git a/sys/boot/pc98/loader/main.c b/sys/boot/pc98/loader/main.c
index 83e9bdf..6573990 100644
--- a/sys/boot/pc98/loader/main.c
+++ b/sys/boot/pc98/loader/main.c
@@ -145,6 +145,14 @@ main(void)
 	    bc_add(initial_bootdev);
     }
 
+    archsw.arch_autoload = i386_autoload;
+    archsw.arch_getdev = i386_getdev;
+    archsw.arch_copyin = i386_copyin;
+    archsw.arch_copyout = i386_copyout;
+    archsw.arch_readin = i386_readin;
+    archsw.arch_isainb = isa_inb;
+    archsw.arch_isaoutb = isa_outb;
+
     /*
      * March through the device switch probing for things.
      */
@@ -163,14 +171,6 @@ main(void)
 
     extract_currdev();				/* set $currdev and $loaddev */
     setenv("LINES", "24", 1);			/* optional */
-    
-    archsw.arch_autoload = i386_autoload;
-    archsw.arch_getdev = i386_getdev;
-    archsw.arch_copyin = i386_copyin;
-    archsw.arch_copyout = i386_copyout;
-    archsw.arch_readin = i386_readin;
-    archsw.arch_isainb = isa_inb;
-    archsw.arch_isaoutb = isa_outb;
 
     interact();			/* doesn't return */
 
@@ -188,7 +188,8 @@ static void
 extract_currdev(void)
 {
     struct i386_devdesc	new_currdev;
-    int			major, biosdev = -1;
+    int			major;
+    int			biosdev = -1;
 
     /* Assume we are booting from a BIOS disk by default */
     new_currdev.d_dev = &biosdisk;
diff --git a/sys/cddl/compat/opensolaris/sys/vnode.h b/sys/cddl/compat/opensolaris/sys/vnode.h
index 616e11a..68288a7 100644
--- a/sys/cddl/compat/opensolaris/sys/vnode.h
+++ b/sys/cddl/compat/opensolaris/sys/vnode.h
@@ -217,7 +217,6 @@ zfs_vn_rdwr(enum uio_rw rw, vnode_t *vp, caddr_t base, ssize_t len,
 	vfslocked = VFS_LOCK_GIANT(vp->v_mount);
 	if (rw == UIO_WRITE) {
 		ioflag = IO_SYNC;
-		VOP_LEASE(vp, td, td->td_ucred, LEASE_WRITE);
 	} else {
 		ioflag = IO_DIRECT;
 	}
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c
index 45ec88b..3ae43f1 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c
@@ -940,6 +940,8 @@ top:
 		/* NB: we already did dmu_tx_wait() if necessary */
 		goto top;
 	}
+	if (error == 0)
+		VOP_UNLOCK(*xvpp, 0);
 
 	return (error);
 }
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
index f213d10..7792f6e 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
@@ -4577,7 +4577,6 @@ vop_deleteextattr {
 		ZFS_EXIT(zfsvfs);
 		return (error);
 	}
-	VOP_LEASE(nd.ni_dvp, td, ap->a_cred, LEASE_WRITE);
 	error = VOP_REMOVE(nd.ni_dvp, vp, &nd.ni_cnd);
 
 	vput(nd.ni_dvp);
@@ -4639,7 +4638,6 @@ vop_setextattr {
 		return (error);
 	}
 
-	VOP_LEASE(vp, td, ap->a_cred, LEASE_WRITE);
 	VATTR_NULL(&va);
 	va.va_size = 0;
 	error = VOP_SETATTR(vp, &va, ap->a_cred);
diff --git a/sys/compat/freebsd32/freebsd32.h b/sys/compat/freebsd32/freebsd32.h
index 08d6510..84e832c 100644
--- a/sys/compat/freebsd32/freebsd32.h
+++ b/sys/compat/freebsd32/freebsd32.h
@@ -191,4 +191,10 @@ struct thr_param32 {
 	uint32_t spare[3];
 };
 
+struct i386_ldt_args32 {
+	uint32_t start;
+	uint32_t descs;
+	uint32_t num;
+};
+
 #endif /* !_COMPAT_FREEBSD32_FREEBSD32_H_ */
diff --git a/sys/compat/freebsd32/freebsd32_ioctl.c b/sys/compat/freebsd32/freebsd32_ioctl.c
index 5cca29a..122bb4d 100644
--- a/sys/compat/freebsd32/freebsd32_ioctl.c
+++ b/sys/compat/freebsd32/freebsd32_ioctl.c
@@ -35,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/cdio.h>
 #include <sys/fcntl.h>
+#include <sys/filio.h>
 #include <sys/file.h>
 #include <sys/ioccom.h>
 #include <sys/mdioctl.h>
@@ -178,6 +179,22 @@ freebsd32_ioctl_ioc_read_toc(struct thread *td,
 	return error;
 }
 
+static int
+freebsd32_ioctl_fiodgname(struct thread *td,
+    struct freebsd32_ioctl_args *uap, struct file *fp)
+{
+	struct fiodgname_arg fgn;
+	struct fiodgname_arg32 fgn32;
+	int error;
+
+	if ((error = copyin(uap->data, &fgn32, sizeof fgn32)) != 0)
+		return (error);
+	CP(fgn32, fgn, len);
+	PTRIN_CP(fgn32, fgn, buf);
+	error = fo_ioctl(fp, FIODGNAME, (caddr_t)&fgn, td->td_ucred, td);
+	fdrop(fp, td);
+	return (error);
+}
 
 int
 freebsd32_ioctl(struct thread *td, struct freebsd32_ioctl_args *uap)
@@ -210,6 +227,9 @@ freebsd32_ioctl(struct thread *td, struct freebsd32_ioctl_args *uap)
 	case CDIOREADTOCHEADER_32:
 		return freebsd32_ioctl_ioc_toc_header(td, uap, fp);
 
+	case FIODGNAME_32:
+		return freebsd32_ioctl_fiodgname(td, uap, fp);
+
 	default:
 		fdrop(fp, td);
 		ap.fd = uap->fd;
diff --git a/sys/compat/freebsd32/freebsd32_ioctl.h b/sys/compat/freebsd32/freebsd32_ioctl.h
index 7844066..fddf553 100644
--- a/sys/compat/freebsd32/freebsd32_ioctl.h
+++ b/sys/compat/freebsd32/freebsd32_ioctl.h
@@ -62,11 +62,17 @@ struct md_ioctl32 {
 	int		md_pad[MDNPAD32]; /* padding for future ideas */
 };
 
+struct fiodgname_arg32 {
+	int		len;
+	caddr_t32	buf;
+};
+
 #define	CDIOREADTOCENTRYS_32 _IOWR('c', 5, struct ioc_read_toc_entry32)
 #define	CDIOREADTOCHEADER_32 _IOR('c', 4, struct ioc_toc_header32)
 #define	MDIOCATTACH_32	_IOC(IOC_INOUT, 'm', 0, sizeof(struct md_ioctl32) + 4)
 #define	MDIOCDETACH_32	_IOC(IOC_INOUT, 'm', 1, sizeof(struct md_ioctl32) + 4)
 #define	MDIOCQUERY_32	_IOC(IOC_INOUT, 'm', 2, sizeof(struct md_ioctl32) + 4)
 #define	MDIOCLIST_32	_IOC(IOC_INOUT, 'm', 3, sizeof(struct md_ioctl32) + 4)
+#define	FIODGNAME_32	_IOW('f', 120, struct fiodgname_arg32)
 
 #endif	/* _COMPAT_FREEBSD32_IOCTL_H_ */
diff --git a/sys/compat/freebsd32/freebsd32_misc.c b/sys/compat/freebsd32/freebsd32_misc.c
index 59809d8..1acf5e8 100644
--- a/sys/compat/freebsd32/freebsd32_misc.c
+++ b/sys/compat/freebsd32/freebsd32_misc.c
@@ -2043,8 +2043,9 @@ freebsd32_jail(struct thread *td, struct freebsd32_jail_args *uap)
 	error = copyin(uap->jail, &version, sizeof(uint32_t));
 	if (error)
 		return (error);
+
 	switch (version) {
-	case 0:	
+	case 0:
 	{
 		/* FreeBSD single IPv4 jails. */
 		struct jail32_v0 j32_v0;
diff --git a/sys/compat/freebsd32/freebsd32_proto.h b/sys/compat/freebsd32/freebsd32_proto.h
index 625378f..bcca32b 100644
--- a/sys/compat/freebsd32/freebsd32_proto.h
+++ b/sys/compat/freebsd32/freebsd32_proto.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 185878 2008-12-10 20:56:19Z jhb 
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 190621 2009-04-01 13:11:50Z kib 
  */
 
 #ifndef _FREEBSD32_SYSPROTO_H_
@@ -119,6 +119,10 @@ struct freebsd32_adjtime_args {
 	char delta_l_[PADL_(struct timeval32 *)]; struct timeval32 * delta; char delta_r_[PADR_(struct timeval32 *)];
 	char olddelta_l_[PADL_(struct timeval32 *)]; struct timeval32 * olddelta; char olddelta_r_[PADR_(struct timeval32 *)];
 };
+struct freebsd32_sysarch_args {
+	char op_l_[PADL_(int)]; int op; char op_r_[PADR_(int)];
+	char parms_l_[PADL_(char *)]; char * parms; char parms_r_[PADR_(char *)];
+};
 struct freebsd32_semsys_args {
 	char which_l_[PADL_(int)]; int which; char which_r_[PADR_(int)];
 	char a2_l_[PADL_(int)]; int a2; char a2_r_[PADR_(int)];
@@ -456,6 +460,7 @@ int	freebsd32_writev(struct thread *, struct freebsd32_writev_args *);
 int	freebsd32_settimeofday(struct thread *, struct freebsd32_settimeofday_args *);
 int	freebsd32_utimes(struct thread *, struct freebsd32_utimes_args *);
 int	freebsd32_adjtime(struct thread *, struct freebsd32_adjtime_args *);
+int	freebsd32_sysarch(struct thread *, struct freebsd32_sysarch_args *);
 int	freebsd32_semsys(struct thread *, struct freebsd32_semsys_args *);
 int	freebsd32_msgsys(struct thread *, struct freebsd32_msgsys_args *);
 int	freebsd32_shmsys(struct thread *, struct freebsd32_shmsys_args *);
@@ -682,6 +687,7 @@ int	freebsd6_freebsd32_ftruncate(struct thread *, struct freebsd6_freebsd32_ftru
 #define	FREEBSD32_SYS_AUE_freebsd32_settimeofday	AUE_SETTIMEOFDAY
 #define	FREEBSD32_SYS_AUE_freebsd32_utimes	AUE_UTIMES
 #define	FREEBSD32_SYS_AUE_freebsd32_adjtime	AUE_ADJTIME
+#define	FREEBSD32_SYS_AUE_freebsd32_sysarch	AUE_SYSARCH
 #define	FREEBSD32_SYS_AUE_freebsd32_semsys	AUE_SEMSYS
 #define	FREEBSD32_SYS_AUE_freebsd32_msgsys	AUE_MSGSYS
 #define	FREEBSD32_SYS_AUE_freebsd32_shmsys	AUE_SHMSYS
diff --git a/sys/compat/freebsd32/freebsd32_syscall.h b/sys/compat/freebsd32/freebsd32_syscall.h
index d491d83..b6e2b6e 100644
--- a/sys/compat/freebsd32/freebsd32_syscall.h
+++ b/sys/compat/freebsd32/freebsd32_syscall.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 185878 2008-12-10 20:56:19Z jhb 
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 190621 2009-04-01 13:11:50Z kib 
  */
 
 #define	FREEBSD32_SYS_syscall	0
@@ -159,7 +159,7 @@
 				/* 162 is obsolete getdomainname */
 				/* 163 is obsolete setdomainname */
 				/* 164 is obsolete uname */
-#define	FREEBSD32_SYS_sysarch	165
+#define	FREEBSD32_SYS_freebsd32_sysarch	165
 #define	FREEBSD32_SYS_rtprio	166
 #define	FREEBSD32_SYS_freebsd32_semsys	169
 #define	FREEBSD32_SYS_freebsd32_msgsys	170
diff --git a/sys/compat/freebsd32/freebsd32_syscalls.c b/sys/compat/freebsd32/freebsd32_syscalls.c
index 88a5e83..cf3f1cc 100644
--- a/sys/compat/freebsd32/freebsd32_syscalls.c
+++ b/sys/compat/freebsd32/freebsd32_syscalls.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 185878 2008-12-10 20:56:19Z jhb 
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 190621 2009-04-01 13:11:50Z kib 
  */
 
 const char *freebsd32_syscallnames[] = {
@@ -172,7 +172,7 @@ const char *freebsd32_syscallnames[] = {
 	"obs_getdomainname",			/* 162 = obsolete getdomainname */
 	"obs_setdomainname",			/* 163 = obsolete setdomainname */
 	"obs_uname",			/* 164 = obsolete uname */
-	"sysarch",			/* 165 = sysarch */
+	"freebsd32_sysarch",			/* 165 = freebsd32_sysarch */
 	"rtprio",			/* 166 = rtprio */
 	"#167",			/* 167 = nosys */
 	"#168",			/* 168 = nosys */
diff --git a/sys/compat/freebsd32/freebsd32_sysent.c b/sys/compat/freebsd32/freebsd32_sysent.c
index e0177c8..9e75246 100644
--- a/sys/compat/freebsd32/freebsd32_sysent.c
+++ b/sys/compat/freebsd32/freebsd32_sysent.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 185878 2008-12-10 20:56:19Z jhb 
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 190621 2009-04-01 13:11:50Z kib 
  */
 
 #include "opt_compat.h"
@@ -203,7 +203,7 @@ struct sysent freebsd32_sysent[] = {
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0 },			/* 162 = obsolete getdomainname */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0 },			/* 163 = obsolete setdomainname */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0 },			/* 164 = obsolete uname */
-	{ AS(sysarch_args), (sy_call_t *)sysarch, AUE_SYSARCH, NULL, 0, 0 },	/* 165 = sysarch */
+	{ AS(freebsd32_sysarch_args), (sy_call_t *)freebsd32_sysarch, AUE_SYSARCH, NULL, 0, 0 },	/* 165 = freebsd32_sysarch */
 	{ AS(rtprio_args), (sy_call_t *)rtprio, AUE_RTPRIO, NULL, 0, 0 },	/* 166 = rtprio */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0 },			/* 167 = nosys */
 	{ 0, (sy_call_t *)nosys, AUE_NULL, NULL, 0, 0 },			/* 168 = nosys */
diff --git a/sys/compat/freebsd32/syscalls.master b/sys/compat/freebsd32/syscalls.master
index 25cb81c..4d571f2 100644
--- a/sys/compat/freebsd32/syscalls.master
+++ b/sys/compat/freebsd32/syscalls.master
@@ -298,7 +298,7 @@
 162	AUE_NULL	OBSOL	getdomainname
 163	AUE_NULL	OBSOL	setdomainname
 164	AUE_NULL	OBSOL	uname
-165	AUE_SYSARCH	NOPROTO	{ int sysarch(int op, char *parms); }
+165	AUE_SYSARCH	STD	{ int freebsd32_sysarch(int op, char *parms); }
 166	AUE_RTPRIO	NOPROTO	{ int rtprio(int function, pid_t pid, \
 				    struct rtprio *rtp); }
 167	AUE_NULL	UNIMPL	nosys
diff --git a/sys/compat/ia32/ia32_signal.h b/sys/compat/ia32/ia32_signal.h
index f2be96d..6ebb0de 100644
--- a/sys/compat/ia32/ia32_signal.h
+++ b/sys/compat/ia32/ia32_signal.h
@@ -59,7 +59,9 @@ struct ia32_mcontext {
 	 * See <i386/include/npx.h> for the internals of mc_fpstate[].
 	 */
 	u_int32_t	mc_fpstate[128] __aligned(16);
-	u_int32_t	mc_spare2[8];
+	u_int32_t	mc_fsbase;
+	u_int32_t	mc_gsbase;
+	u_int32_t	mc_spare2[6];
 };
 
 struct ia32_ucontext {
diff --git a/sys/compat/ia32/ia32_sysvec.c b/sys/compat/ia32/ia32_sysvec.c
index 0b32b9a..af8168e 100644
--- a/sys/compat/ia32/ia32_sysvec.c
+++ b/sys/compat/ia32/ia32_sysvec.c
@@ -148,7 +148,8 @@ static Elf32_Brandinfo ia32_brand_info = {
 	.interp_path	= "/libexec/ld-elf.so.1",
 	.sysvec		= &ia32_freebsd_sysvec,
 	.interp_newpath	= "/libexec/ld-elf32.so.1",
-	.flags		= BI_CAN_EXEC_DYN
+	.brand_note	= &elf32_freebsd_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 SYSINIT(ia32, SI_SUB_EXEC, SI_ORDER_ANY,
@@ -163,7 +164,8 @@ static Elf32_Brandinfo ia32_brand_oinfo = {
 	.interp_path	= "/usr/libexec/ld-elf.so.1",
 	.sysvec		= &ia32_freebsd_sysvec,
 	.interp_newpath	= "/libexec/ld-elf32.so.1",
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &elf32_freebsd_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 SYSINIT(oia32, SI_SUB_EXEC, SI_ORDER_ANY,
@@ -187,15 +189,21 @@ ia32_copyout_strings(struct image_params *imgp)
 	char *stringp, *destp;
 	u_int32_t *stack_base;
 	struct freebsd32_ps_strings *arginfo;
+	size_t execpath_len;
 	int szsigcode;
 
 	/*
 	 * Calculate string base and vector table pointers.
 	 * Also deal with signal trampoline code for this exec type.
 	 */
+	if (imgp->execpath != NULL && imgp->auxargs != NULL)
+		execpath_len = strlen(imgp->execpath) + 1;
+	else
+		execpath_len = 0;
 	arginfo = (struct freebsd32_ps_strings *)FREEBSD32_PS_STRINGS;
 	szsigcode = *(imgp->proc->p_sysent->sv_szsigcode);
 	destp =	(caddr_t)arginfo - szsigcode - SPARE_USRSPACE -
+		roundup(execpath_len, sizeof(char *)) -
 		roundup((ARG_MAX - imgp->args->stringspace), sizeof(char *));
 
 	/*
@@ -206,6 +214,15 @@ ia32_copyout_strings(struct image_params *imgp)
 			((caddr_t)arginfo - szsigcode), szsigcode);
 
 	/*
+	 * Copy the image path for the rtld.
+	 */
+	if (execpath_len != 0) {
+		imgp->execpathp = (uintptr_t)arginfo - szsigcode - execpath_len;
+		copyout(imgp->execpath, (void *)imgp->execpathp,
+		    execpath_len);
+	}
+
+	/*
 	 * If we have a valid auxargs ptr, prepare some room
 	 * on the stack.
 	 */
@@ -221,9 +238,9 @@ ia32_copyout_strings(struct image_params *imgp)
 		 * the arg and env vector sets,and imgp->auxarg_size is room
 		 * for argument of Runtime loader.
 		 */
-		vectp = (u_int32_t *) (destp - (imgp->args->argc + imgp->args->envc + 2 +
-				       imgp->auxarg_size) * sizeof(u_int32_t));
-
+		vectp = (u_int32_t *) (destp - (imgp->args->argc +
+		    imgp->args->envc + 2 + imgp->auxarg_size + execpath_len) *
+		    sizeof(u_int32_t));
 	} else
 		/*
 		 * The '+ 2' is for the null pointers at the end of each of
diff --git a/sys/compat/linprocfs/linprocfs.c b/sys/compat/linprocfs/linprocfs.c
index 5cc1f7d..52f6e94 100644
--- a/sys/compat/linprocfs/linprocfs.c
+++ b/sys/compat/linprocfs/linprocfs.c
@@ -75,6 +75,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/vmmeter.h>
 #include <sys/vnode.h>
 #include <sys/vimage.h>
+#include <sys/bus.h>
 
 #include <net/if.h>
 #include <net/route.h>
@@ -90,6 +91,9 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/clock.h>
 
+#include <geom/geom.h>
+#include <geom/geom_int.h>
+
 #if defined(__i386__) || defined(__amd64__)
 #include <machine/cputypes.h>
 #include <machine/md_var.h>
@@ -359,6 +363,9 @@ linprocfs_domtab(PFS_FILL_ARGS)
 			sbuf_printf(sb, "/sys %s sysfs %s", mntto,
 			    mp->mnt_stat.f_flags & MNT_RDONLY ? "ro" : "rw");
 		} else {
+			/* For Linux msdosfs is called vfat */
+			if (strcmp(fstype, "msdosfs") == 0)
+				fstype = "vfat";
 			sbuf_printf(sb, "%s %s %s %s", mntfrom, mntto, fstype,
 			    mp->mnt_stat.f_flags & MNT_RDONLY ? "ro" : "rw");
 		}
@@ -383,6 +390,69 @@ linprocfs_domtab(PFS_FILL_ARGS)
 }
 
 /*
+ * Filler function for proc/partitions
+ *
+ */
+static int
+linprocfs_dopartitions(PFS_FILL_ARGS)
+{
+	struct g_class *cp;
+	struct g_geom *gp;
+	struct g_provider *pp;
+	struct nameidata nd;
+	const char *lep;
+	char  *dlep, *flep;
+	size_t lep_len;
+	int error;
+	int major, minor;
+
+	/* resolve symlinks etc. in the emulation tree prefix */
+	NDINIT(&nd, LOOKUP, FOLLOW | MPSAFE, UIO_SYSSPACE, linux_emul_path, td);
+	flep = NULL;
+	error = namei(&nd);
+	lep = linux_emul_path;
+	if (error == 0) {
+		if (vn_fullpath(td, nd.ni_vp, &dlep, &flep) == 0)
+			lep = dlep;
+		vrele(nd.ni_vp);
+		VFS_UNLOCK_GIANT(NDHASGIANT(&nd));
+	}
+	lep_len = strlen(lep);
+
+	g_topology_lock();
+	error = 0;
+	sbuf_printf(sb, "major minor  #blocks  name rio rmerge rsect "
+	    "ruse wio wmerge wsect wuse running use aveq\n");
+
+	LIST_FOREACH(cp, &g_classes, class) {
+		if (strcmp(cp->name, "DISK") == 0 ||
+		    strcmp(cp->name, "PART") == 0)
+			LIST_FOREACH(gp, &cp->geom, geom) {
+				LIST_FOREACH(pp, &gp->provider, provider) {
+					if (linux_driver_get_major_minor(
+					    pp->name, &major, &minor) != 0) {
+						major = 0;
+						minor = 0;
+					}
+					sbuf_printf(sb, "%d %d %lld %s "
+					    "%d %d %d %d %d "
+					     "%d %d %d %d %d %d\n",
+					     major, minor,
+					     (long long)pp->mediasize, pp->name,
+					     0, 0, 0, 0, 0,
+					     0, 0, 0, 0, 0, 0);
+				}
+			}
+	}
+	g_topology_unlock();
+
+	if (flep != NULL)
+		free(flep, M_TEMP);
+	return (error);
+}
+
+
+/*
  * Filler function for proc/stat
  */
 static int
@@ -1206,6 +1276,8 @@ linprocfs_init(PFS_INIT_ARGS)
 	    NULL, NULL, NULL, PFS_RD);
 	pfs_create_file(root, "mtab", &linprocfs_domtab,
 	    NULL, NULL, NULL, PFS_RD);
+	pfs_create_file(root, "partitions", &linprocfs_dopartitions,
+	    NULL, NULL, NULL, PFS_RD);
 	pfs_create_link(root, "self", &procfs_docurproc,
 	    NULL, NULL, NULL, 0);
 	pfs_create_file(root, "stat", &linprocfs_dostat,
diff --git a/sys/compat/linux/linux_emul.h b/sys/compat/linux/linux_emul.h
index 7125c92..2c1b3ca 100644
--- a/sys/compat/linux/linux_emul.h
+++ b/sys/compat/linux/linux_emul.h
@@ -31,8 +31,6 @@
 #ifndef _LINUX_EMUL_H_
 #define	_LINUX_EMUL_H_
 
-#include <compat/linux/linux_futex.h>
-
 struct linux_emuldata_shared {
 	int	refs;
 	pid_t	group_pid;
diff --git a/sys/compat/linux/linux_file.c b/sys/compat/linux/linux_file.c
index b368429..54ab6a2 100644
--- a/sys/compat/linux/linux_file.c
+++ b/sys/compat/linux/linux_file.c
@@ -1109,6 +1109,9 @@ linux_mount(struct thread *td, struct linux_mount_args *args)
 	} else if (strcmp(fstypename, "proc") == 0) {
 		strcpy(fstypename, "linprocfs");
 		fsdata = NULL;
+	} else if (strcmp(fstypename, "vfat") == 0) {
+		strcpy(fstypename, "msdosfs");
+		fsdata = NULL;
 	} else {
 		return (ENODEV);
 	}
@@ -1135,6 +1138,12 @@ linux_mount(struct thread *td, struct linux_mount_args *args)
 			"fstype", fstypename,
 			"fspath", mntonname,
 			NULL);
+	} else if (strcmp(fstypename, "msdosfs") == 0) {
+		error = kernel_vmount(fsflags,
+			"fstype", fstypename,
+			"fspath", mntonname,
+			"from", mntfromname,
+			NULL);
 	} else
 		error = EOPNOTSUPP;
 	return (error);
diff --git a/sys/compat/linux/linux_futex.c b/sys/compat/linux/linux_futex.c
index b19d290..e1d214f 100644
--- a/sys/compat/linux/linux_futex.c
+++ b/sys/compat/linux/linux_futex.c
@@ -40,18 +40,17 @@ __KERNEL_RCSID(1, "$NetBSD: linux_futex.c,v 1.7 2006/07/24 19:01:49 manu Exp $")
 #include "opt_compat.h"
 
 #include <sys/param.h>
-#include <sys/types.h>
-#include <sys/time.h>
 #include <sys/systm.h>
-#include <sys/proc.h>
-#include <sys/queue.h>
 #include <sys/imgact.h>
+#include <sys/kernel.h>
 #include <sys/lock.h>
+#include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/priv.h>
+#include <sys/proc.h>
+#include <sys/queue.h>
 #include <sys/sched.h>
 #include <sys/sx.h>
-#include <sys/malloc.h>
 
 #ifdef COMPAT_LINUX32
 #include <machine/../linux32/linux.h>
@@ -60,8 +59,8 @@ __KERNEL_RCSID(1, "$NetBSD: linux_futex.c,v 1.7 2006/07/24 19:01:49 manu Exp $")
 #include <machine/../linux/linux.h>
 #include <machine/../linux/linux_proto.h>
 #endif
-#include <compat/linux/linux_emul.h>
 #include <compat/linux/linux_futex.h>
+#include <compat/linux/linux_emul.h>
 
 struct futex;
 
@@ -276,13 +275,6 @@ linux_sys_futex(struct thread *td, struct linux_sys_futex_args *args)
 		FUTEX_SYSTEM_UNLOCK;
 		break;
 
-	case LINUX_FUTEX_FD:
-#ifdef DEBUG
-		printf("linux_sys_futex: unimplemented op %d\n",
-		    args->op);
-#endif
-		return (ENOSYS);
-
 	case LINUX_FUTEX_WAKE_OP:
 		FUTEX_SYSTEM_LOCK;
 #ifdef DEBUG
diff --git a/sys/compat/linux/linux_futex.h b/sys/compat/linux/linux_futex.h
index d658925..ca8b768 100644
--- a/sys/compat/linux/linux_futex.h
+++ b/sys/compat/linux/linux_futex.h
@@ -38,7 +38,7 @@
 
 #define LINUX_FUTEX_WAIT	0
 #define LINUX_FUTEX_WAKE	1
-#define LINUX_FUTEX_FD		2
+#define LINUX_FUTEX_FD		2	/* unused */
 #define LINUX_FUTEX_REQUEUE	3
 #define LINUX_FUTEX_CMP_REQUEUE	4
 #define LINUX_FUTEX_WAKE_OP	5
diff --git a/sys/compat/ndis/subr_ntoskrnl.c b/sys/compat/ndis/subr_ntoskrnl.c
index 237da31..69bea9e 100644
--- a/sys/compat/ndis/subr_ntoskrnl.c
+++ b/sys/compat/ndis/subr_ntoskrnl.c
@@ -1197,6 +1197,11 @@ IofCompleteRequest(irp *ip, uint8_t prioboost)
 		sl++;
 	} while (ip->irp_currentstackloc <= (ip->irp_stackcnt + 1));
 
+	if (ip->irp_usriostat != NULL)
+		*ip->irp_usriostat = ip->irp_iostat;
+	if (ip->irp_usrevent != NULL)
+		KeSetEvent(ip->irp_usrevent, prioboost, FALSE);
+
 	/* Handle any associated IRPs. */
 
 	if (ip->irp_flags & IRP_ASSOCIATED_IRP) {
@@ -1220,16 +1225,10 @@ IofCompleteRequest(irp *ip, uint8_t prioboost)
 
 	/* With any luck, these conditions will never arise. */
 
-	if (ip->irp_flags & (IRP_PAGING_IO|IRP_CLOSE_OPERATION)) {
-		if (ip->irp_usriostat != NULL)
-			*ip->irp_usriostat = ip->irp_iostat;
-		if (ip->irp_usrevent != NULL)
-			KeSetEvent(ip->irp_usrevent, prioboost, FALSE);
-		if (ip->irp_flags & IRP_PAGING_IO) {
-			if (ip->irp_mdl != NULL)
-				IoFreeMdl(ip->irp_mdl);
-			IoFreeIrp(ip);
-		}
+	if (ip->irp_flags & IRP_PAGING_IO) {
+		if (ip->irp_mdl != NULL)
+			IoFreeMdl(ip->irp_mdl);
+		IoFreeIrp(ip);
 	}
 
 	return;
diff --git a/sys/compat/ndis/subr_usbd.c b/sys/compat/ndis/subr_usbd.c
index a07e2cc..bb72732 100644
--- a/sys/compat/ndis/subr_usbd.c
+++ b/sys/compat/ndis/subr_usbd.c
@@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb_process.h>
 #include <dev/usb/usb_device.h>
 #include <dev/usb/usb_error.h>
+#include <dev/usb/usb_parse.h>
 #include <dev/usb/usb_request.h>
 
 #include <compat/ndis/pe_var.h>
@@ -76,6 +77,38 @@ __FBSDID("$FreeBSD$");
 
 static driver_object usbd_driver;
 static usb2_callback_t usbd_non_isoc_callback;
+static usb2_callback_t usbd_ctrl_callback;
+
+#define	USBD_CTRL_READ_PIPE		0
+#define	USBD_CTRL_WRITE_PIPE		1
+#define	USBD_CTRL_MAX_PIPE		2
+#define	USBD_CTRL_READ_BUFFER_SP	256
+#define	USBD_CTRL_READ_BUFFER_SIZE	\
+	(sizeof(struct usb2_device_request) + USBD_CTRL_READ_BUFFER_SP)
+#define	USBD_CTRL_WRITE_BUFFER_SIZE	\
+	(sizeof(struct usb2_device_request))
+static struct usb2_config usbd_default_epconfig[USBD_CTRL_MAX_PIPE] = {
+	[USBD_CTRL_READ_PIPE] = {
+		.type =		UE_CONTROL,
+		.endpoint =	0x00,	/* control pipe */
+		.direction =	UE_DIR_ANY,
+		.if_index =	0,
+		.bufsize =	USBD_CTRL_READ_BUFFER_SIZE,
+		.flags =	{ .short_xfer_ok = 1, },
+		.callback =	&usbd_ctrl_callback,
+		.timeout =	5000,	/* 5 seconds */
+	},
+	[USBD_CTRL_WRITE_PIPE] = {
+		.type =		UE_CONTROL,
+		.endpoint =	0x00,	/* control pipe */
+		.direction =	UE_DIR_ANY,
+		.if_index =	0,
+		.bufsize =	USBD_CTRL_WRITE_BUFFER_SIZE,
+		.flags =	{ .proxy_buffer = 1, },
+		.callback =	&usbd_ctrl_callback,
+		.timeout =	5000,	/* 5 seconds */
+	}
+};
 
 static int32_t		 usbd_func_bulkintr(irp *);
 static int32_t		 usbd_func_vendorclass(irp *);
@@ -83,6 +116,9 @@ static int32_t		 usbd_func_selconf(irp *);
 static int32_t		 usbd_func_abort_pipe(irp *);
 static usb2_error_t	 usbd_setup_endpoint(irp *, uint8_t,
 			    struct usb2_endpoint_descriptor	*);
+static usb2_error_t	 usbd_setup_endpoint_default(irp *, uint8_t);
+static usb2_error_t	 usbd_setup_endpoint_one(irp *, uint8_t,
+			    struct ndisusb_ep *, struct usb2_config *);
 static int32_t		 usbd_func_getdesc(irp *);
 static union usbd_urb	*usbd_geturb(irp *);
 static struct ndisusb_ep*usbd_get_ndisep(irp *, usb_endpoint_descriptor_t *);
@@ -557,6 +593,57 @@ usbd_func_selconf(ip)
 }
 
 static usb2_error_t
+usbd_setup_endpoint_one(ip, ifidx, ne, epconf)
+	irp				*ip;
+	uint8_t				ifidx;
+	struct ndisusb_ep		*ne;
+	struct usb2_config		*epconf;
+{
+	device_t dev = IRP_NDIS_DEV(ip);
+	struct ndis_softc *sc = device_get_softc(dev);
+	struct usb2_xfer *xfer;
+	usb2_error_t status;
+
+	InitializeListHead(&ne->ne_active);
+	InitializeListHead(&ne->ne_pending);
+	KeInitializeSpinLock(&ne->ne_lock);
+
+	status = usb2_transfer_setup(sc->ndisusb_dev, &ifidx, ne->ne_xfer,
+	    epconf, 1, sc, &sc->ndisusb_mtx);
+	if (status != USB_ERR_NORMAL_COMPLETION) {
+		device_printf(dev, "couldn't setup xfer: %s\n",
+		    usb2_errstr(status));
+		return (status);
+	}
+	xfer = ne->ne_xfer[0];
+	xfer->priv_fifo = ne;
+
+	return (status);
+}
+
+static usb2_error_t
+usbd_setup_endpoint_default(ip, ifidx)
+	irp				*ip;
+	uint8_t				ifidx;
+{
+	device_t dev = IRP_NDIS_DEV(ip);
+	struct ndis_softc *sc = device_get_softc(dev);
+	usb2_error_t status;
+
+	if (ifidx > 0)
+		device_printf(dev, "warning: ifidx > 0 isn't supported.\n");
+
+	status = usbd_setup_endpoint_one(ip, ifidx, &sc->ndisusb_dread_ep,
+	    &usbd_default_epconfig[USBD_CTRL_READ_PIPE]);
+	if (status != USB_ERR_NORMAL_COMPLETION)
+		return (status);
+
+	status = usbd_setup_endpoint_one(ip, ifidx, &sc->ndisusb_dwrite_ep,
+	    &usbd_default_epconfig[USBD_CTRL_WRITE_PIPE]);
+	return (status);
+}
+
+static usb2_error_t
 usbd_setup_endpoint(ip, ifidx, ep)
 	irp				*ip;
 	uint8_t				ifidx;
@@ -587,11 +674,11 @@ usbd_setup_endpoint(ip, ifidx, ep)
 	cfg.type	= UE_GET_XFERTYPE(ep->bmAttributes);
 	cfg.endpoint	= UE_GET_ADDR(ep->bEndpointAddress);
 	cfg.direction	= UE_GET_DIR(ep->bEndpointAddress);
-	cfg.mh.callback	= &usbd_non_isoc_callback;
-	cfg.mh.bufsize	= UGETW(ep->wMaxPacketSize);
-	cfg.mh.flags.proxy_buffer = 1;
+	cfg.callback	= &usbd_non_isoc_callback;
+	cfg.bufsize	= UGETW(ep->wMaxPacketSize);
+	cfg.flags.proxy_buffer = 1;
 	if (UE_GET_DIR(ep->bEndpointAddress) == UE_DIR_IN)
-		cfg.mh.flags.short_xfer_ok = 1;
+		cfg.flags.short_xfer_ok = 1;
 
 	status = usb2_transfer_setup(sc->ndisusb_dev, &ifidx, ne->ne_xfer,
 	    &cfg, 1, sc, &sc->ndisusb_mtx);
@@ -643,62 +730,54 @@ usbd_func_vendorclass(ip)
 	irp			*ip;
 {
 	device_t dev = IRP_NDIS_DEV(ip);
+	int32_t error;
 	struct ndis_softc *sc = device_get_softc(dev);
+	struct ndisusb_ep *ne;
+	struct ndisusb_xfer *nx;
 	struct usbd_urb_vendor_or_class_request *vcreq;
-	uint8_t type = 0;
 	union usbd_urb *urb;
-	struct usb2_device_request req;
-	usb2_error_t status;
+
+	if (!(sc->ndisusb_status & NDISUSB_STATUS_SETUP_EP)) {
+		/*
+		 * XXX In some cases the interface number isn't 0.  However
+		 * some driver (eg. RTL8187L NDIS driver) calls this function
+		 * before calling URB_FUNCTION_SELECT_CONFIGURATION.
+		 */
+		error = usbd_setup_endpoint_default(ip, 0);
+		if (error != USB_ERR_NORMAL_COMPLETION)
+			return usbd_usb2urb(error);
+		sc->ndisusb_status |= NDISUSB_STATUS_SETUP_EP;
+	}
 
 	urb = usbd_geturb(ip);
 	vcreq = &urb->uu_vcreq;
+	ne = (vcreq->uvc_trans_flags & USBD_TRANSFER_DIRECTION_IN) ?
+	    &sc->ndisusb_dread_ep : &sc->ndisusb_dwrite_ep;
+	IRP_NDISUSB_EP(ip) = ne;
+	ip->irp_cancelfunc = (cancel_func)usbd_irpcancel_wrap;
 
-	switch (urb->uu_hdr.uuh_func) {
-	case URB_FUNCTION_CLASS_DEVICE:
-		type = UT_CLASS | UT_DEVICE;
-		break;
-	case URB_FUNCTION_CLASS_INTERFACE:
-		type = UT_CLASS | UT_INTERFACE;
-		break;
-	case URB_FUNCTION_CLASS_OTHER:
-		type = UT_CLASS | UT_OTHER;
-		break;
-	case URB_FUNCTION_CLASS_ENDPOINT:
-		type = UT_CLASS | UT_ENDPOINT;
-		break;
-	case URB_FUNCTION_VENDOR_DEVICE:
-		type = UT_VENDOR | UT_DEVICE;
-		break;
-	case URB_FUNCTION_VENDOR_INTERFACE:
-		type = UT_VENDOR | UT_INTERFACE;
-		break;
-	case URB_FUNCTION_VENDOR_OTHER:
-		type = UT_VENDOR | UT_OTHER;
-		break;
-	case URB_FUNCTION_VENDOR_ENDPOINT:
-		type = UT_VENDOR | UT_ENDPOINT;
-		break;
-	default:
-		/* never reached.  */
-		break;
+	nx = malloc(sizeof(struct ndisusb_xfer), M_USBDEV, M_NOWAIT | M_ZERO);
+	if (nx == NULL) {
+		device_printf(IRP_NDIS_DEV(ip), "out of memory\n");
+		return (USBD_STATUS_NO_MEMORY);
 	}
+	nx->nx_ep = ne;
+	nx->nx_priv = ip;
+	KeAcquireSpinLockAtDpcLevel(&ne->ne_lock);
+	InsertTailList((&ne->ne_pending), (&nx->nx_next));
+	KeReleaseSpinLockFromDpcLevel(&ne->ne_lock);
 
-	type |= (vcreq->uvc_trans_flags & USBD_TRANSFER_DIRECTION_IN) ?
-	    UT_READ : UT_WRITE;
-	type |= vcreq->uvc_reserved1;
-
-	req.bmRequestType = type;
-	req.bRequest = vcreq->uvc_req;
-	USETW(req.wIndex, vcreq->uvc_idx);
-	USETW(req.wValue, vcreq->uvc_value);
-	USETW(req.wLength, vcreq->uvc_trans_buflen);
+	/* we've done to setup xfer.  Let's transfer it.  */
+	ip->irp_iostat.isb_status = STATUS_PENDING;
+	ip->irp_iostat.isb_info = 0;
+	USBD_URB_STATUS(urb) = USBD_STATUS_PENDING;
+	IoMarkIrpPending(ip);
 
-	NDISUSB_LOCK(sc);
-	status = usb2_do_request(sc->ndisusb_dev, &sc->ndisusb_mtx, &req,
-	    vcreq->uvc_trans_buf);
-	NDISUSB_UNLOCK(sc);
+	error = usbd_taskadd(ip, NDISUSB_TASK_VENDOR);
+	if (error != USBD_STATUS_SUCCESS)
+		return (error);
 
-	return usbd_usb2urb(status);
+	return (USBD_STATUS_PENDING);
 }
 
 static void
@@ -706,8 +785,9 @@ usbd_irpcancel(dobj, ip)
 	device_object		*dobj;
 	irp			*ip;
 {
+	device_t dev = IRP_NDIS_DEV(ip);
+	struct ndis_softc *sc = device_get_softc(dev);
 	struct ndisusb_ep *ne = IRP_NDISUSB_EP(ip);
-	uint8_t irql;
 
 	if (ne == NULL) {
 		ip->irp_cancel = TRUE;
@@ -719,10 +799,10 @@ usbd_irpcancel(dobj, ip)
 	 * Make sure that the current USB transfer proxy is
 	 * cancelled and then restarted.
 	 */
-	KeRaiseIrql(DISPATCH_LEVEL, &irql);
+	NDISUSB_LOCK(sc);
 	usb2_transfer_stop(ne->ne_xfer[0]);
 	usb2_transfer_start(ne->ne_xfer[0]);
-	KeLowerIrql(irql);
+	NDISUSB_UNLOCK(sc);
 
 	ip->irp_cancel = TRUE;
 	IoReleaseCancelSpinLock(ip->irp_cancelirql);
@@ -870,6 +950,146 @@ extra:
 	}
 }
 
+static void
+usbd_ctrl_callback(struct usb2_xfer *xfer)
+{
+	irp *ip;
+	struct ndis_softc *sc = xfer->priv_sc;
+	struct ndisusb_ep *ne = xfer->priv_fifo;
+	struct ndisusb_xfer *nx;
+	uint8_t irql;
+	union usbd_urb *urb;
+	struct usbd_urb_vendor_or_class_request *vcreq;
+	uint8_t type = 0;
+	struct usb2_device_request req;
+
+	switch (USB_GET_STATE(xfer)) {
+	case USB_ST_TRANSFERRED:
+		nx = usbd_aq_getfirst(sc, ne);
+		if (nx == NULL)
+			return;
+
+		ip = nx->nx_priv;
+		urb = usbd_geturb(ip);
+		vcreq = &urb->uu_vcreq;
+
+		if (vcreq->uvc_trans_flags & USBD_TRANSFER_DIRECTION_IN) {
+			usb2_copy_out(xfer->frbuffers + 1, 0,
+			    vcreq->uvc_trans_buf, xfer->frlengths[1]);
+			nx->nx_urbactlen += xfer->frlengths[1];
+		}
+
+		usbd_xfer_complete(sc, ne, nx, USB_ERR_NORMAL_COMPLETION);
+		/* fall through */
+	case USB_ST_SETUP:
+next:
+		/* get next transfer */
+		KeAcquireSpinLock(&ne->ne_lock, &irql);
+		if (IsListEmpty(&ne->ne_pending)) {
+			KeReleaseSpinLock(&ne->ne_lock, irql);
+			return;
+		}
+		nx = CONTAINING_RECORD(ne->ne_pending.nle_flink,
+		    struct ndisusb_xfer, nx_next);
+		RemoveEntryList(&nx->nx_next);
+		/* add a entry to the active queue's tail.  */
+		InsertTailList((&ne->ne_active), (&nx->nx_next));
+		KeReleaseSpinLock(&ne->ne_lock, irql);
+
+		ip = nx->nx_priv;
+		urb = usbd_geturb(ip);
+		vcreq = &urb->uu_vcreq;
+
+		switch (urb->uu_hdr.uuh_func) {
+		case URB_FUNCTION_CLASS_DEVICE:
+			type = UT_CLASS | UT_DEVICE;
+			break;
+		case URB_FUNCTION_CLASS_INTERFACE:
+			type = UT_CLASS | UT_INTERFACE;
+			break;
+		case URB_FUNCTION_CLASS_OTHER:
+			type = UT_CLASS | UT_OTHER;
+			break;
+		case URB_FUNCTION_CLASS_ENDPOINT:
+			type = UT_CLASS | UT_ENDPOINT;
+			break;
+		case URB_FUNCTION_VENDOR_DEVICE:
+			type = UT_VENDOR | UT_DEVICE;
+			break;
+		case URB_FUNCTION_VENDOR_INTERFACE:
+			type = UT_VENDOR | UT_INTERFACE;
+			break;
+		case URB_FUNCTION_VENDOR_OTHER:
+			type = UT_VENDOR | UT_OTHER;
+			break;
+		case URB_FUNCTION_VENDOR_ENDPOINT:
+			type = UT_VENDOR | UT_ENDPOINT;
+			break;
+		default:
+			/* never reached.  */
+			break;
+		}
+
+		type |= (vcreq->uvc_trans_flags & USBD_TRANSFER_DIRECTION_IN) ?
+		    UT_READ : UT_WRITE;
+		type |= vcreq->uvc_reserved1;
+
+		req.bmRequestType = type;
+		req.bRequest = vcreq->uvc_req;
+		USETW(req.wIndex, vcreq->uvc_idx);
+		USETW(req.wValue, vcreq->uvc_value);
+		USETW(req.wLength, vcreq->uvc_trans_buflen);
+
+		nx->nx_urbbuf		= vcreq->uvc_trans_buf;
+		nx->nx_urblen		= vcreq->uvc_trans_buflen;
+		nx->nx_urbactlen	= 0;
+
+		usb2_copy_in(xfer->frbuffers, 0, &req, sizeof(req));
+		xfer->frlengths[0] = sizeof(req);
+		xfer->nframes = 1;
+		if (vcreq->uvc_trans_flags & USBD_TRANSFER_DIRECTION_IN) {
+			if (vcreq->uvc_trans_buflen >= USBD_CTRL_READ_BUFFER_SP)
+				device_printf(sc->ndis_dev,
+				    "warning: not enough buffer space (%d).\n",
+				    vcreq->uvc_trans_buflen);
+			xfer->frlengths[1] = MIN(xfer->max_data_length,
+			    vcreq->uvc_trans_buflen);
+			xfer->nframes = 2;
+		} else {
+			if (nx->nx_urblen > 0)
+				device_printf(sc->ndis_dev,
+				    "warning: not enough write buffer space"
+				    " (%d).\n", nx->nx_urblen);
+			/*
+			 * XXX with my local tests there was no cases to require
+			 * a extra buffer until now but it'd need to update in
+			 * the future if it needs to be.
+			 */
+			if (nx->nx_urblen > 0) {
+				usb2_copy_in(xfer->frbuffers + 1 , 0,
+				    nx->nx_urbbuf, nx->nx_urblen);
+				xfer->frlengths[1] = nx->nx_urblen;
+				xfer->nframes = 2;
+			}
+		}
+		usb2_start_hardware(xfer);
+		break;
+	default:
+		nx = usbd_aq_getfirst(sc, ne);
+		if (nx == NULL)
+			return;
+		if (xfer->error != USB_ERR_CANCELLED) {
+			xfer->flags.stall_pipe = 1;
+			device_printf(sc->ndis_dev, "usb xfer warning (%s)\n",
+			    usb2_errstr(xfer->error));
+		}
+		usbd_xfer_complete(sc, ne, nx, xfer->error);
+		if (xfer->error != USB_ERR_CANCELLED)
+			goto next;
+		break;
+	}
+}
+
 static struct ndisusb_ep *
 usbd_get_ndisep(ip, ep)
 	irp			*ip;
@@ -900,6 +1120,7 @@ usbd_xfertask(dobj, arg)
 	struct ndisusb_xferdone *nd;
 	struct ndisusb_xfer *nq;
 	struct usbd_urb_bulk_or_intr_transfer *ubi;
+	struct usbd_urb_vendor_or_class_request *vcreq;
 	union usbd_urb *urb;
 	usb2_error_t status;
 	void *priv;
@@ -920,18 +1141,19 @@ usbd_xfertask(dobj, arg)
 		ip = priv;
 		urb = usbd_geturb(ip);
 
-		KASSERT(urb->uu_hdr.uuh_func ==
-		    URB_FUNCTION_BULK_OR_INTERRUPT_TRANSFER,
-		    ("function(%d) isn't for bulk or interrupt",
-			urb->uu_hdr.uuh_func));
-
 		ip->irp_cancelfunc = NULL;
 		IRP_NDISUSB_EP(ip) = NULL;
 
 		switch (status) {
 		case USB_ERR_NORMAL_COMPLETION:
-			ubi = &urb->uu_bulkintr;
-			ubi->ubi_trans_buflen = nq->nx_urbactlen;
+			if (urb->uu_hdr.uuh_func ==
+			    URB_FUNCTION_BULK_OR_INTERRUPT_TRANSFER) {
+				ubi = &urb->uu_bulkintr;
+				ubi->ubi_trans_buflen = nq->nx_urbactlen;
+			} else {
+				vcreq = &urb->uu_vcreq;
+				vcreq->uvc_trans_buflen = nq->nx_urbactlen;
+			}
 			ip->irp_iostat.isb_info = nq->nx_urbactlen;
 			ip->irp_iostat.isb_status = STATUS_SUCCESS;
 			USBD_URB_STATUS(urb) = USBD_STATUS_SUCCESS;
@@ -1035,6 +1257,12 @@ usbd_task(dobj, arg)
 			usb2_transfer_stop(ne->ne_xfer[0]);
 			usb2_transfer_start(ne->ne_xfer[0]);
 			break;
+		case NDISUSB_TASK_VENDOR:
+			ne = (urb->uu_vcreq.uvc_trans_flags &
+			    USBD_TRANSFER_DIRECTION_IN) ?
+			    &sc->ndisusb_dread_ep : &sc->ndisusb_dwrite_ep;
+			usb2_transfer_start(ne->ne_xfer[0]);
+			break;
 		default:
 			break;
 		}
@@ -1206,12 +1434,11 @@ USBD_ParseConfigurationDescriptorEx(conf, start, intfnum,
 	int32_t intfsubclass;
 	int32_t intfproto;
 {
-	char *pos;
+	struct usb2_descriptor *next = NULL;
 	usb_interface_descriptor_t *desc;
 
-	for (pos = start; pos < ((char *)conf + UGETW(conf->wTotalLength));
-	     pos += desc->bLength) {
-		desc = (usb_interface_descriptor_t *)pos;
+	while ((next = usb2_desc_foreach(conf, next)) != NULL) {
+		desc = (usb_interface_descriptor_t *)next;
 		if (desc->bDescriptorType != UDESC_INTERFACE)
 			continue;
 		if (!(intfnum == -1 || desc->bInterfaceNumber == intfnum))
diff --git a/sys/compat/svr4/svr4_sysvec.c b/sys/compat/svr4/svr4_sysvec.c
index 0030e3a..54406f5 100644
--- a/sys/compat/svr4/svr4_sysvec.c
+++ b/sys/compat/svr4/svr4_sysvec.c
@@ -204,6 +204,7 @@ Elf32_Brandinfo svr4_brand = {
 	.interp_path	= "/lib/libc.so.1",
 	.sysvec		= &svr4_sysvec,
 	.interp_newpath	= NULL,
+	.brand_note	= NULL,
 	.flags		= 0
 };
 
diff --git a/sys/conf/Makefile.mips b/sys/conf/Makefile.mips
index e03432c..443065a 100644
--- a/sys/conf/Makefile.mips
+++ b/sys/conf/Makefile.mips
@@ -28,73 +28,35 @@ S=	../../..
 .endif
 .include "$S/conf/kern.pre.mk"
 
-SYSTEM_LD:= ${SYSTEM_LD:$S/conf/ldscript.$M=ldscript.$M}
-SYSTEM_DEP:= ${SYSTEM_DEP:$S/conf/ldscript.$M=ldscript.$M}
-
 # XXX: Such sweeping assumptions...
 MACHINE=mips
 MACHINE_ARCH=mips
-KERNLOADADDR?=0x80001000
-# This obscure value is defined by CFE for WR160N
-# To be changed later
-TRAMPLOADADDR?=0x807963c0
 
 MKMODULESENV+=	MACHINE=${MACHINE} MACHINE_ARCH=${MACHINE_ARCH}
 
 # We default to the MIPS32 ISA, if none specified in the
 # kernel configuration file.
 ARCH_FLAGS?=-march=mips32
-EXTRA_FLAGS=-fno-pic -mno-abicalls -mno-dsp -G0
 
 HACK_EXTRA_FLAGS=-shared
 .if defined(TARGET_BIG_ENDIAN)
 CFLAGS+=-EB
 SYSTEM_LD+=-EB
-EXTRA_FLAGS+=-EB 
-TRAMP_LDFLAGS+=-Wl,-EB 
 HACK_EXTRA_FLAGS+=-EB -Wl,-EB
 .else
 CFLAGS+=-EL
 SYSTEM_LD+=-EL
-EXTRA_FLAGS+=-EL
-TRAMP_LDFLAGS+=-Wl,-EL
 HACK_EXTRA_FLAGS+=-EL -Wl,-EL
 .endif
 
 # We add the -fno-pic flag to kernels because otherwise performance
 # is extremely poor, as well as -mno-abicalls to force no ABI usage.
-CFLAGS+=${EXTRA_FLAGS} $(ARCH_FLAGS)
-HACK_EXTRA_FLAGS+=${EXTRA_FLAGS} $(ARCH_FLAGS)
+CFLAGS+=-fno-pic -mno-abicalls -G0 $(ARCH_FLAGS)
+HACK_EXTRA_FLAGS+=-fno-pic -mno-abicalls -G0 $(ARCH_FLAGS)
 
 # XXX hardcoded kernel entry point
 ASM_CFLAGS+=${CFLAGS} -D_LOCORE -DLOCORE
 
-KERNEL_EXTRA=trampoline
-trampoline: ${KERNEL_KO}.tramp.bin
-${KERNEL_KO}.tramp.bin: ${KERNEL_KO} $S/$M/$M/elf_trampoline.c \
-	$S/$M/$M/inckern.S 
-	${OBJCOPY} --strip-symbol '$$d' --strip-symbol '$$a' \
-	-g --strip-symbol '$$t' ${FULLKERNEL} ${KERNEL_KO}.tmp
-	sed s/${KERNLOADADDR}/${TRAMPLOADADDR}/ ldscript.$M | \
-		sed s/" + SIZEOF_HEADERS"//  > ldscript.$M.tramp.noheader
-	# Generate .S file that setups stack and jumps to trampoline
-	echo "#include <machine/asm.h>" >tmphack.S
-	echo "ENTRY(_start)" >>tmphack.S
-	echo "la t0, kernel_end" >>tmphack.S
-	echo "move sp, t0" >>tmphack.S
-	echo "add sp, 0x2000" >>tmphack.S
-	echo "and sp, ~0x7" >>tmphack.S
-	echo "la t0, _startC" >>tmphack.S
-	echo "j t0" >>tmphack.S
-	echo "END(_start)" >>tmphack.S
-	echo "#define KERNNAME \"${KERNEL_KO}.tmp\""  >opt_kernname.h 
-	${CC} -O -nostdlib -I. -I$S ${EXTRA_FLAGS} ${TRAMP_LDFLAGS} -Xlinker \
-		-T -Xlinker ldscript.$M.tramp.noheader tmphack.S \
-		$S/$M/$M/elf_trampoline.c $S/$M/$M/inckern.S \
-		-o ${KERNEL_KO}.tramp.noheader 
-	${OBJCOPY} -S -O binary ${KERNEL_KO}.tramp.noheader \
-		${KERNEL_KO}.tramp.bin \
-
 %BEFORE_DEPEND
 
 %OBJS
@@ -107,12 +69,6 @@ ${KERNEL_KO}.tramp.bin: ${KERNEL_KO} $S/$M/$M/elf_trampoline.c \
 
 %CLEAN
 
-CLEAN+=	ldscript.$M ldscript.$M.tramp.noheader \
-	${KERNEL_KO}.tramp.noheader ${KERNEL_KO}.tramp.bin
-
-ldscript.$M: $S/conf/ldscript.$M
-	cat $S/conf/ldscript.$M|sed s/KERNLOADADDR/${KERNLOADADDR}/g \
-		> ldscript.$M
 %RULES
 
 .include "$S/conf/kern.post.mk"
diff --git a/sys/conf/NOTES b/sys/conf/NOTES
index d48436e..8a172ac 100644
--- a/sys/conf/NOTES
+++ b/sys/conf/NOTES
@@ -765,10 +765,6 @@ device		arcnet
 #  of synchronous PPP links (like `cx', `ar').
 device		sppp
 
-#  The `sl' device implements the Serial Line IP (SLIP) service.
-#  The `ppp' device implements the Point-to-Point Protocol.
-
-
 #  The `bpf' device enables the Berkeley Packet Filter.  Be
 #  aware of the legal and administrative consequences of enabling this
 #  option.  The number of devices determines the maximum number of
@@ -837,15 +833,6 @@ device		enc
 device		lagg
 
 #
-# The PPP_BSDCOMP option enables support for compress(1) style entire
-# packet compression, the PPP_DEFLATE is for zlib/gzip style compression.
-# PPP_FILTER enables code for filtering the ppp data stream and selecting
-# events for resetting the demand dial activity timer - requires bpf.
-# See pppd(8) for more details.
-#
-
-
-#
 # Internet family options:
 #
 # MROUTING enables the kernel multicast packet forwarder, which works
@@ -1942,7 +1929,7 @@ device		xl		# 3Com 3c90x (``Boomerang'', ``Cyclone'')
 device		de		# DEC/Intel DC21x4x (``Tulip'')
 device		em		# Intel Pro/1000 Gigabit Ethernet
 device		igb		# Intel Pro/1000 PCIE Gigabit Ethernet
-#device		ixgbe		# Intel Pro/10Gbe PCIE Ethernet
+device		ixgbe		# Intel Pro/10Gbe PCIE Ethernet
 device		le		# AMD Am7900 LANCE and Am79C9xx PCnet
 device		mxge		# Myricom Myri-10G 10GbE NIC
 device		nxge		# Neterion Xframe 10GbE Server/Storage Adapter
@@ -2426,8 +2413,6 @@ device		umodem
 device		ums
 # Diamond Rio 500 MP3 player
 device		urio
-# USB scanners
-device		uscanner
 #
 # USB serial support
 device		ucom
@@ -2678,7 +2663,6 @@ options 	SC_DEBUG_LEVEL=5	# Syscons debug level
 options 	SC_RENDER_DEBUG	# syscons rendering debugging
 
 options 	SHOW_BUSYBUFS	# List buffers that prevent root unmount
-options 	SLIP_IFF_OPTS
 options 	VFS_BIO_DEBUG	# VFS buffer I/O debugging
 
 options 	KSTACK_MAX_PAGES=32 # Maximum pages to give the kernel stack
diff --git a/sys/conf/files b/sys/conf/files
index 6a14ef9..b7d30e0 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -515,16 +515,19 @@ dev/ath/if_ath_pci.c		optional ath pci \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/ah_osdep.c		optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
+#
 dev/ath/ath_hal/ah.c		optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/ath_hal/ah_eeprom_v1.c	optional ath_hal | ath_ar5210 \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/ath_hal/ah_eeprom_v3.c	optional ath_hal | ath_ar5211 | ath_ar5212 \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
-dev/ath/ath_hal/ah_eeprom_v14.c optional ath_hal | ath_ar5416 | ath_ar9160 \
+dev/ath/ath_hal/ah_eeprom_v14.c \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/ath_hal/ah_regdomain.c	optional ath \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
+# ar5210
 dev/ath/ath_hal/ar5210/ar5210_attach.c		optional ath_hal | ath_ar5210 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5210/ar5210_beacon.c		optional ath_hal | ath_ar5210 \
@@ -545,6 +548,7 @@ dev/ath/ath_hal/ar5210/ar5210_reset.c		optional ath_hal | ath_ar5210 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5210/ar5210_xmit.c		optional ath_hal | ath_ar5210 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
+# ar5211
 dev/ath/ath_hal/ar5211/ar5211_attach.c		optional ath_hal | ath_ar5211 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5211/ar5211_beacon.c		optional ath_hal | ath_ar5211 \
@@ -565,123 +569,134 @@ dev/ath/ath_hal/ar5211/ar5211_reset.c		optional ath_hal | ath_ar5211 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5211/ar5211_xmit.c		optional ath_hal | ath_ar5211 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
+# ar5212
 dev/ath/ath_hal/ar5212/ar5212_ani.c \
-	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_attach.c \
-	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_beacon.c \
-	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_eeprom.c \
-	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_gpio.c \
-	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_interrupts.c \
-	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_keycache.c \
-	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_misc.c \
-	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_phy.c \
-	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_power.c \
-	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_recv.c \
-	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_reset.c \
-	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_rfgain.c \
-	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5212/ar5212_xmit.c \
-	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 \
-	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
-dev/ath/ath_hal/ar5212/ar2316.c	optional ath_rf2316 \
-	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
-dev/ath/ath_hal/ar5212/ar2317.c	optional ath_rf2317 \
-	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
-dev/ath/ath_hal/ar5212/ar2413.c	optional ath_hal | ath_rf2413 \
-	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
-dev/ath/ath_hal/ar5212/ar2425.c	optional ath_hal | ath_rf2425 | ath_rf2417 \
-	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
-dev/ath/ath_hal/ar5212/ar5111.c	optional ath_hal | ath_rf5111 \
-	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
-dev/ath/ath_hal/ar5212/ar5112.c	optional ath_hal | ath_rf5112 \
-	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
-dev/ath/ath_hal/ar5212/ar5413.c	optional ath_hal | ath_rf5413 \
-	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
-dev/ath/ath_hal/ar5416/ar2133.c	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5212 | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
+# ar5416 (depends on ar5212)
 dev/ath/ath_hal/ar5416/ar5416_ani.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_attach.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_beacon.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_cal.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_cal_iq.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_cal_adcgain.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_cal_adcdc.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_eeprom.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_gpio.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_interrupts.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_keycache.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_misc.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_phy.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_power.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_recv.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_reset.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
 dev/ath/ath_hal/ar5416/ar5416_xmit.c \
-	optional ath_hal | ath_ar5416 | ath_ar9160 \
+	optional ath_hal | ath_ar5416 | ath_ar9160 | ath_ar9280 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
+# ar9160 (depends on ar5416)
 dev/ath/ath_hal/ar5416/ar9160_attach.c optional ath_hal | ath_ar9160 \
 	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
+# ar9280 (depends on ar5416)
+dev/ath/ath_hal/ar5416/ar9280_attach.c optional ath_hal | ath_ar9280 \
+	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
+# rf backends
+dev/ath/ath_hal/ar5212/ar2316.c	optional ath_rf2316 \
+	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
+dev/ath/ath_hal/ar5212/ar2317.c	optional ath_rf2317 \
+	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
+dev/ath/ath_hal/ar5212/ar2413.c	optional ath_hal | ath_rf2413 \
+	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
+dev/ath/ath_hal/ar5212/ar2425.c	optional ath_hal | ath_rf2425 | ath_rf2417 \
+	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
+dev/ath/ath_hal/ar5212/ar5111.c	optional ath_hal | ath_rf5111 \
+	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
+dev/ath/ath_hal/ar5212/ar5112.c	optional ath_hal | ath_rf5112 \
+	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
+dev/ath/ath_hal/ar5212/ar5413.c	optional ath_hal | ath_rf5413 \
+	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
+dev/ath/ath_hal/ar5416/ar2133.c optional ath_hal | ath_ar5416 | ath_ar9160 \
+	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
+dev/ath/ath_hal/ar5416/ar9280.c optional ath_hal | ath_ar9280 \
+	compile-with "${NORMAL_C} -I$S/dev/ath -I$S/dev/ath/ath_hal"
+# ath rate control algorithms
 dev/ath/ath_rate/amrr/amrr.c	optional ath_rate_amrr \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/ath_rate/onoe/onoe.c	optional ath_rate_onoe \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
 dev/ath/ath_rate/sample/sample.c	optional ath_rate_sample \
 	compile-with "${NORMAL_C} -I$S/dev/ath"
+#
 dev/bce/if_bce.c		optional bce
 dev/bfe/if_bfe.c		optional bfe
 dev/bge/if_bge.c		optional bge
@@ -1058,6 +1073,8 @@ dev/ixgbe/ixgbe_common.c	optional ixgbe \
 	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
 dev/ixgbe/ixgbe_82598.c		optional ixgbe \
 	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
+dev/ixgbe/ixgbe_82599.c		optional ixgbe \
+	compile-with "${NORMAL_C} -I$S/dev/ixgbe"
 dev/jme/if_jme.c		optional jme pci
 dev/joy/joy.c			optional joy
 dev/joy/joy_isa.c		optional joy isa
@@ -1088,6 +1105,7 @@ dev/mfi/mfi_cam.c		optional mfip scbus
 dev/mii/acphy.c			optional miibus | acphy
 dev/mii/amphy.c			optional miibus | amphy
 dev/mii/atphy.c			optional miibus | atphy
+dev/mii/axphy.c			optional miibus | axphy
 dev/mii/bmtphy.c		optional miibus | bmtphy
 dev/mii/brgphy.c		optional miibus | brgphy
 dev/mii/ciphy.c			optional miibus | ciphy
@@ -1291,7 +1309,6 @@ dev/random/probe.c		optional random
 dev/random/randomdev.c		optional random
 dev/random/randomdev_soft.c	optional random
 dev/random/yarrow.c		optional random
-dev/ray/if_ray.c		optional ray pccard
 dev/rc/rc.c			optional rc
 dev/re/if_re.c			optional re
 dev/rndtest/rndtest.c		optional rndtest
@@ -1409,8 +1426,6 @@ dev/sound/midi/synth_if.m	optional sound
 dev/spibus/spibus.c		optional spibus				\
 	dependency	"spibus_if.h"
 dev/spibus/spibus_if.m		optional spibus
-dev/sr/if_sr.c			optional sr
-dev/sr/if_sr_pci.c		optional sr pci
 dev/ste/if_ste.c		optional ste pci
 dev/stg/tmc18c30.c		optional stg
 dev/stg/tmc18c30_isa.c		optional stg isa
@@ -1487,15 +1502,6 @@ legacy/dev/usb/ehci_ddb.c	optional oehci
 legacy/dev/usb/ehci_pci.c	optional oehci pci
 legacy/dev/usb/hid.c		optional ousb
 legacy/dev/usb/if_aue.c		optional oaue
-legacy/dev/usb/if_axe.c		optional oaxe
-legacy/dev/usb/if_cdce.c	optional ocdce
-legacy/dev/usb/if_cue.c		optional ocue
-legacy/dev/usb/if_kue.c		optional okue
-legacy/dev/usb/if_ural.c	optional oural
-legacy/dev/usb/if_rue.c		optional orue
-legacy/dev/usb/if_rum.c		optional orum
-legacy/dev/usb/if_udav.c	optional oudav
-legacy/dev/usb/if_zyd.c		optional ozyd
 legacy/dev/usb/ohci.c		optional oohci
 legacy/dev/usb/ohci_pci.c	optional oohci pci
 legacy/dev/usb/sl811hs.c	optional oslhci
@@ -1581,7 +1587,6 @@ dev/usb/usb_msctest.c		optional usb
 dev/usb/usb_parse.c		optional usb
 dev/usb/usb_process.c		optional usb
 dev/usb/usb_request.c		optional usb
-dev/usb/usb_sw_transfer.c	optional usb
 dev/usb/usb_transfer.c		optional usb
 dev/usb/usb_util.c		optional usb
 #
@@ -1653,10 +1658,6 @@ dev/usb/template/usb_template_cdce.c	optional usb_template
 dev/usb/template/usb_template_msc.c	optional usb_template
 dev/usb/template/usb_template_mtp.c	optional usb_template
 #
-# USB image drivers
-#
-dev/usb/image/uscanner.c	optional uscanner
-#
 # USB END
 #
 dev/utopia/idtphy.c		optional utopia
@@ -2145,7 +2146,6 @@ net/bpf_jitter.c		optional bpf_jitter
 net/bpf_filter.c		optional bpf | netgraph_bpf
 net/bpf_zerocopy.c		optional bpf
 net/bridgestp.c			optional bridge | if_bridge
-net/bsd_comp.c			optional ppp_bsdcomp
 net/ieee8023ad_lacp.c		optional lagg
 net/if.c			standard
 net/if_arcsubr.c		optional arcnet
@@ -2169,8 +2169,6 @@ net/if_loop.c			optional loop
 net/if_llatbl.c			standard
 net/if_media.c			standard
 net/if_mib.c			standard
-net/if_ppp.c			optional ppp
-net/if_sl.c			optional sl
 net/if_spppfr.c			optional sppp | netgraph_sppp
 net/if_spppsubr.c		optional sppp | netgraph_sppp
 net/if_stf.c			optional stf
@@ -2180,8 +2178,6 @@ net/if_vlan.c			optional vlan
 net/mppcc.c			optional netgraph_mppc_compression
 net/mppcd.c			optional netgraph_mppc_compression
 net/netisr.c			standard
-net/ppp_deflate.c		optional ppp_deflate
-net/ppp_tty.c			optional ppp
 net/pfil.c			optional ether | inet
 net/radix.c			standard
 net/radix_mpath.c		standard
@@ -2189,20 +2185,20 @@ net/raw_cb.c			standard
 net/raw_usrreq.c		standard
 net/route.c			standard
 net/rtsock.c			standard
-net/slcompress.c		optional netgraph_vjc | ppp | sl | sppp | \
+net/slcompress.c		optional netgraph_vjc | sppp | \
 					 netgraph_sppp
 net/zlib.c			optional crypto | geom_uzip | ipsec | \
-					 mxge | ppp_deflate | netgraph_deflate | \
+					 mxge | netgraph_deflate | \
 					 ddb_ctf
 net80211/ieee80211.c		optional wlan
-net80211/ieee80211_acl.c	optional wlan_acl
+net80211/ieee80211_acl.c	optional wlan wlan_acl
 net80211/ieee80211_adhoc.c	optional wlan
-net80211/ieee80211_amrr.c	optional wlan_amrr
+net80211/ieee80211_amrr.c	optional wlan wlan_amrr
 net80211/ieee80211_crypto.c	optional wlan
-net80211/ieee80211_crypto_ccmp.c optional wlan_ccmp
+net80211/ieee80211_crypto_ccmp.c optional wlan wlan_ccmp
 net80211/ieee80211_crypto_none.c optional wlan
-net80211/ieee80211_crypto_tkip.c optional wlan_tkip
-net80211/ieee80211_crypto_wep.c	optional wlan_wep
+net80211/ieee80211_crypto_tkip.c optional wlan wlan_tkip
+net80211/ieee80211_crypto_wep.c	optional wlan wlan_wep
 net80211/ieee80211_ddb.c	optional wlan ddb
 net80211/ieee80211_dfs.c	optional wlan
 net80211/ieee80211_freebsd.c	optional wlan
@@ -2217,13 +2213,14 @@ net80211/ieee80211_phy.c	optional wlan
 net80211/ieee80211_power.c	optional wlan
 net80211/ieee80211_proto.c	optional wlan
 net80211/ieee80211_regdomain.c	optional wlan
-net80211/ieee80211_rssadapt.c	optional wlan_rssadapt
+net80211/ieee80211_rssadapt.c	optional wlan wlan_rssadapt
 net80211/ieee80211_scan.c	optional wlan
 net80211/ieee80211_scan_sta.c	optional wlan
 net80211/ieee80211_sta.c	optional wlan
-net80211/ieee80211_tdma.c	optional wlan
+net80211/ieee80211_superg.c	optional wlan ieee80211_support_superg
+net80211/ieee80211_tdma.c	optional wlan ieee80211_support_tdma
 net80211/ieee80211_wds.c	optional wlan
-net80211/ieee80211_xauth.c	optional wlan_xauth
+net80211/ieee80211_xauth.c	optional wlan wlan_xauth
 netatalk/aarp.c			optional netatalk
 netatalk/at_control.c		optional netatalk
 netatalk/at_proto.c		optional netatalk
@@ -2472,6 +2469,7 @@ nfsserver/nfs_srvsock.c		optional nfsserver
 nfsserver/nfs_srvcache.c	optional nfsserver
 nfsserver/nfs_srvsubs.c		optional nfsserver
 nfsserver/nfs_syscalls.c	optional nfsserver
+nfs/nfs_nfssvc.c		optional nfsserver
 nlm/nlm_advlock.c		optional nfslockd nfsclient
 nlm/nlm_prot_clnt.c		optional nfslockd
 nlm/nlm_prot_impl.c		optional nfslockd
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index 97f5ee5..56e17d1 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -69,6 +69,19 @@ hptrr_lib.o			optional	hptrr			\
 #
 amd64/acpica/OsdEnvironment.c	optional	acpi
 amd64/acpica/acpi_machdep.c	optional	acpi
+amd64/acpica/acpi_switch.S	optional	acpi
+acpi_wakecode.h			optional acpi		\
+	dependency 	"$S/amd64/acpica/acpi_wakecode.S assym.s"	\
+	compile-with	"${MAKE} -f $S/amd64/acpica/Makefile ${.TARGET} MAKESRCPATH=$S/amd64/acpica"	\
+	no-obj no-implicit-rule before-depend				\
+	clean		"acpi_wakecode.h acpi_wakecode.o acpi_wakecode.bin"
+#
+acpi_wakedata.h			optional acpi		\
+	dependency 	"$S/amd64/acpica/acpi_wakecode.S assym.s"	\
+	compile-with	"${MAKE} -f $S/amd64/acpica/Makefile ${.TARGET} MAKESRCPATH=$S/amd64/acpica"	\
+	no-obj no-implicit-rule before-depend				\
+	clean		"acpi_wakedata.h acpi_wakecode.o acpi_wakecode.bin"
+#
 amd64/acpica/acpi_wakeup.c	optional	acpi
 amd64/acpica/madt.c		optional	acpi
 amd64/amd64/amd64_mem.c		optional	mem
@@ -136,6 +149,7 @@ dev/agp/agp_amd64.c		optional	agp
 dev/agp/agp_i810.c		optional	agp
 dev/agp/agp_intel.c		optional	agp
 dev/agp/agp_via.c		optional	agp
+dev/amdtemp/amdtemp.c		optional	amdtemp
 dev/arcmsr/arcmsr.c		optional	arcmsr pci
 dev/asmc/asmc.c			optional	asmc isa
 dev/atkbdc/atkbd.c		optional	atkbd atkbdc
@@ -169,6 +183,7 @@ dev/ipmi/ipmi_smbus.c		optional	ipmi smbus
 dev/ipmi/ipmi_smbios.c		optional	ipmi
 dev/ipmi/ipmi_ssif.c		optional	ipmi smbus
 dev/ipmi/ipmi_pci.c		optional	ipmi pci
+dev/ipmi/ipmi_linux.c		optional	ipmi compat_linux32
 dev/fdc/fdc.c			optional	fdc
 dev/fdc/fdc_acpi.c		optional	fdc
 dev/fdc/fdc_isa.c		optional	fdc isa
@@ -187,7 +202,6 @@ dev/hwpmc/hwpmc_core.c		optional	hwpmc
 dev/hwpmc/hwpmc_piv.c		optional	hwpmc
 dev/hwpmc/hwpmc_tsc.c		optional	hwpmc
 dev/hwpmc/hwpmc_x86.c		optional	hwpmc
-dev/k8temp/k8temp.c		optional	k8temp
 dev/kbd/kbd.c			optional	atkbd | sc | ukbd | usb2_input_kbd
 dev/mem/memutil.c		optional	mem
 dev/nfe/if_nfe.c		optional	nfe pci
@@ -218,6 +232,7 @@ amd64/ia32/ia32_reg.c		optional	compat_ia32
 amd64/ia32/ia32_signal.c	optional	compat_ia32
 amd64/ia32/ia32_sigtramp.S	optional	compat_ia32
 amd64/ia32/ia32_syscall.c	optional	compat_ia32
+amd64/ia32/ia32_misc.c		optional	compat_ia32
 compat/freebsd32/freebsd32_ioctl.c	optional	compat_ia32
 compat/freebsd32/freebsd32_misc.c	optional	compat_ia32
 compat/freebsd32/freebsd32_syscalls.c	optional	compat_ia32
@@ -269,6 +284,7 @@ i386/bios/smbios.c		optional	smbios
 i386/bios/vpd.c			optional	vpd
 i386/cpufreq/powernow.c		optional	cpufreq
 i386/cpufreq/est.c		optional	cpufreq
+i386/cpufreq/hwpstate.c		optional	cpufreq
 i386/cpufreq/p4tcc.c		optional	cpufreq
 #
 libkern/memmove.c		standard
diff --git a/sys/conf/files.i386 b/sys/conf/files.i386
index 069e3b9..13f0417 100644
--- a/sys/conf/files.i386
+++ b/sys/conf/files.i386
@@ -128,10 +128,8 @@ dev/agp/agp_nvidia.c		optional agp
 dev/agp/agp_sis.c		optional agp
 dev/agp/agp_via.c		optional agp
 dev/aic/aic_isa.c		optional aic isa
+dev/amdtemp/amdtemp.c		optional amdtemp
 dev/arcmsr/arcmsr.c		optional arcmsr pci
-dev/ar/if_ar.c			optional ar
-dev/ar/if_ar_isa.c		optional ar isa
-dev/ar/if_ar_pci.c		optional ar pci
 dev/asmc/asmc.c			optional asmc isa
 dev/atkbdc/atkbd.c		optional atkbd atkbdc
 dev/atkbdc/atkbd_atkbdc.c	optional atkbd atkbdc
@@ -198,7 +196,7 @@ dev/ipmi/ipmi_smbus.c		optional ipmi smbus
 dev/ipmi/ipmi_smbios.c		optional ipmi
 dev/ipmi/ipmi_ssif.c		optional ipmi smbus
 dev/ipmi/ipmi_pci.c		optional ipmi pci
-dev/k8temp/k8temp.c		optional k8temp
+dev/ipmi/ipmi_linux.c		optional ipmi compat_linux
 dev/kbd/kbd.c			optional atkbd | sc | ukbd | usb2_input_kbd
 dev/le/if_le_isa.c		optional le isa
 dev/mem/memutil.c		optional mem
@@ -218,7 +216,6 @@ dev/sio/sio_pccard.c		optional sio pccard
 dev/sio/sio_pci.c		optional sio pci
 dev/sio/sio_puc.c		optional sio puc
 dev/speaker/spkr.c		optional speaker
-dev/sr/if_sr_isa.c		optional sr isa
 dev/syscons/apm/apm_saver.c	optional apm_saver apm
 dev/syscons/scterm-teken.c	optional sc
 dev/syscons/scvesactl.c		optional sc vga vesa
@@ -245,6 +242,7 @@ i386/bios/smapi_bios.S		optional smapi
 i386/bios/smbios.c		optional smbios
 i386/bios/vpd.c			optional vpd
 i386/cpufreq/est.c		optional cpufreq
+i386/cpufreq/hwpstate.c		optional cpufreq
 i386/cpufreq/p4tcc.c		optional cpufreq
 i386/cpufreq/powernow.c		optional cpufreq
 i386/cpufreq/smist.c		optional cpufreq
diff --git a/sys/conf/files.ia64 b/sys/conf/files.ia64
index 62e36a5..706d64e 100644
--- a/sys/conf/files.ia64
+++ b/sys/conf/files.ia64
@@ -70,6 +70,7 @@ ia64/acpica/madt.c		optional	acpi
 ia64/disasm/disasm_decode.c	standard
 ia64/disasm/disasm_extract.c	standard
 ia64/disasm/disasm_format.c	standard
+ia64/ia32/ia32_misc.c		optional	compat_ia32
 ia64/ia32/ia32_reg.c		optional	compat_ia32
 ia64/ia32/ia32_signal.c		optional	compat_ia32
 ia64/ia32/ia32_trap.c		optional	compat_ia32
diff --git a/sys/conf/files.mips b/sys/conf/files.mips
index bbedd96..1e233d7 100644
--- a/sys/conf/files.mips
+++ b/sys/conf/files.mips
@@ -96,7 +96,3 @@ dev/cfe/cfe_api.c		optional	cfe
 dev/cfe/cfe_console.c		optional	cfe_console
 #dev/cfe/cfe_resource.c		optional	cfe	# not yet needed
 
-dev/siba/siba.c			optional	siba
-dev/siba/siba_pcib.c		optional	siba pci
-dev/siba/siba_cc.c		optional	siba
-#mips/sentry5/siba_mips.c			optional siba # not yet
diff --git a/sys/conf/files.pc98 b/sys/conf/files.pc98
index f8c6f0a..a6b62b1 100644
--- a/sys/conf/files.pc98
+++ b/sys/conf/files.pc98
@@ -85,8 +85,6 @@ dev/agp/agp_nvidia.c		optional agp
 dev/agp/agp_sis.c		optional agp
 dev/agp/agp_via.c		optional agp
 dev/aic/aic_cbus.c		optional aic isa
-dev/ar/if_ar.c			optional ar
-dev/ar/if_ar_pci.c		optional ar pci
 dev/ce/ceddk.c			optional ce
 dev/ce/if_ce.c			optional ce
 dev/ce/tau32-ddk.c		optional ce
diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc
index 4c40219..00ac74a 100644
--- a/sys/conf/files.powerpc
+++ b/sys/conf/files.powerpc
@@ -76,13 +76,13 @@ powerpc/aim/interrupt.c		optional	aim
 powerpc/aim/locore.S		optional	aim no-obj
 powerpc/aim/machdep.c		optional	aim
 powerpc/aim/mmu_oea.c		optional	aim
+powerpc/aim/mmu_oea64.c		optional	aim
 powerpc/aim/mp_cpudep.c		optional	aim smp
 powerpc/aim/nexus.c		optional	aim
 powerpc/aim/ofw_machdep.c	optional	aim
 powerpc/aim/ofwmagic.S		optional	aim
 powerpc/aim/swtch.S		optional	aim
 powerpc/aim/trap.c		optional	aim
-powerpc/aim/uio_machdep.c	optional	aim
 powerpc/aim/uma_machdep.c	optional	aim
 powerpc/aim/vm_machdep.c	optional	aim
 powerpc/booke/clock.c		optional	e500
@@ -93,7 +93,6 @@ powerpc/booke/machdep.c		optional	e500
 powerpc/booke/pmap.c		optional	e500
 powerpc/booke/swtch.S		optional	e500
 powerpc/booke/trap.c		optional	e500
-powerpc/booke/uio_machdep.c	optional	e500
 powerpc/booke/vm_machdep.c	optional	e500
 powerpc/fpu/fpu_add.c		optional	fpu_emu
 powerpc/fpu/fpu_compare.c	optional	fpu_emu
@@ -112,10 +111,11 @@ powerpc/mpc85xx/opic.c		optional	mpc85xx
 powerpc/mpc85xx/pci_ocp.c	optional	pci mpc85xx
 powerpc/ofw/ofw_pcibus.c	optional	pci aim
 powerpc/ofw/ofw_pcib_pci.c	optional	pci aim
+powerpc/ofw/ofw_real.c		optional	aim
 powerpc/ofw/ofw_syscons.c	optional	sc aim
-powerpc/powermac/ata_kauai.c	optional	powermac ata
-powerpc/powermac/ata_macio.c	optional	powermac ata
-powerpc/powermac/ata_dbdma.c	optional	powermac ata
+powerpc/powermac/ata_kauai.c	optional	powermac ata | powermac atamacio
+powerpc/powermac/ata_macio.c	optional	powermac ata | powermac atamacio
+powerpc/powermac/ata_dbdma.c	optional	powermac ata | powermac atamacio
 powerpc/powermac/dbdma.c	optional	powermac pci
 powerpc/powermac/grackle.c	optional	powermac pci
 powerpc/powermac/hrowpic.c	optional	powermac pci
@@ -127,6 +127,7 @@ powerpc/powermac/uninorth.c	optional	powermac pci
 powerpc/powermac/cuda.c		optional	powermac cuda
 powerpc/powermac/pmu.c		optional	powermac pmu 
 powerpc/powermac/macgpio.c	optional	powermac pci 
+powerpc/powermac/cpcht.c	optional	powermac pci
 powerpc/powerpc/altivec.c	optional	aim
 powerpc/powerpc/atomic.S	standard
 powerpc/powerpc/autoconf.c	standard
@@ -139,6 +140,7 @@ powerpc/powerpc/db_disasm.c	optional	ddb
 powerpc/powerpc/db_hwwatch.c	optional	ddb
 powerpc/powerpc/db_interface.c	optional	ddb
 powerpc/powerpc/db_trace.c	optional	ddb
+powerpc/powerpc/dump_machdep.c	standard
 powerpc/powerpc/elf_machdep.c	standard
 powerpc/powerpc/fpu.c		optional	aim
 powerpc/powerpc/fuswintr.c	standard
@@ -158,6 +160,7 @@ powerpc/powerpc/stack_machdep.c	optional	ddb | stack
 powerpc/powerpc/suswintr.c	standard
 powerpc/powerpc/syncicache.c	standard
 powerpc/powerpc/sys_machdep.c	standard
+powerpc/powerpc/uio_machdep.c	standard
 powerpc/psim/iobus.c 		optional	psim
 powerpc/psim/ata_iobus.c	optional	ata psim
 powerpc/psim/openpic_iobus.c	optional	psim
diff --git a/sys/conf/ldscript.mips b/sys/conf/ldscript.mips
index 3e55dba..ca4a70b 100644
--- a/sys/conf/ldscript.mips
+++ b/sys/conf/ldscript.mips
@@ -43,7 +43,7 @@ PROVIDE (_DYNAMIC = 0);
 SECTIONS
 {
   /* Read-only sections, merged into text segment: */
-  . = KERNLOADADDR + SIZEOF_HEADERS;
+  . = 0x80100000 + SIZEOF_HEADERS;
   .interp     : { *(.interp) 	}
   .hash          : { *(.hash)		}
   .dynsym        : { *(.dynsym)		}
diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh
index f141a8c..134e46e 100644
--- a/sys/conf/newvers.sh
+++ b/sys/conf/newvers.sh
@@ -100,7 +100,13 @@ for dir in /bin /usr/bin /usr/local/bin; do
 done
 
 if [ -n "$svnversion" -a -d "${SRCDIR}/.svn" ] ; then
-	svn=" r`cd $SRCDIR && $svnversion`"
+	# If we are called from the kernel build, limit
+	# the scope of svnversion to sys/ .
+	if [ -e "${SRCDIR}/sys/conf/newvers.sh" ] ; then
+		svn=" r`cd $SRCDIR/sys && $svnversion`"
+	else
+		svn=" r`cd $SRCDIR && $svnversion`"
+	fi
 else
 	svn=""
 fi
diff --git a/sys/conf/options b/sys/conf/options
index bf39e9a..dc5707e 100644
--- a/sys/conf/options
+++ b/sys/conf/options
@@ -407,9 +407,6 @@ MROUTING		opt_mrouting.h
 NCP
 NETATALK		opt_atalk.h
 NFSLOCKD
-PPP_BSDCOMP		opt_ppp.h
-PPP_DEFLATE		opt_ppp.h
-PPP_FILTER		opt_ppp.h
 RADIX_MPATH		opt_mpath.h
 ROUTETABLES		opt_route.h
 COMPAT_ROUTE_FLAGS	opt_route.h
@@ -748,7 +745,6 @@ ATH_TXBUF		opt_ath.h
 ATH_RXBUF		opt_ath.h
 ATH_DIAGAPI		opt_ath.h
 ATH_TX99_DIAG		opt_ath.h
-ATH_SUPPORT_TDMA	opt_ath.h
 
 # options for the Atheros hal
 AH_SUPPORT_AR5416	opt_ah.h
@@ -792,6 +788,7 @@ INTR_FILTER
 IEEE80211_DEBUG		opt_wlan.h
 IEEE80211_DEBUG_REFCNT	opt_wlan.h
 IEEE80211_AMPDU_AGE	opt_wlan.h
+IEEE80211_SUPPORT_SUPERG	opt_wlan.h
 IEEE80211_SUPPORT_TDMA	opt_wlan.h
 
 # 802.11 TDMA support
diff --git a/sys/conf/options.mips b/sys/conf/options.mips
index 7c38616..42e0263 100644
--- a/sys/conf/options.mips
+++ b/sys/conf/options.mips
@@ -45,8 +45,8 @@ YAMON		opt_global.h
 CFE		opt_global.h
 CFE_CONSOLE	opt_global.h
 
-KERNLOADADDR	opt_global.h
-TRAMPLOADADDR	opt_global.h
+KERNPHYSADDR	opt_global.h
+KERNVIRTADDR	opt_global.h
 PHYSADDR	opt_global.h
 SOFTFLOAT	opt_global.h
 
diff --git a/sys/contrib/dev/uath/ar5523.bin.uu b/sys/contrib/dev/uath/ar5523.bin.uu
new file mode 100644
index 0000000..6fa430d
--- /dev/null
+++ b/sys/contrib/dev/uath/ar5523.bin.uu
@@ -0,0 +1,3359 @@
+/*-
+ * Copyright (c) 2002-2008 Atheros Communications, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms are permitted
+ * provided that the following conditions are met:
+ * 1. The materials contained herein are unmodified and are used
+ *    unmodified.
+ * 2. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following NO
+ *    ''WARRANTY'' disclaimer below (''Disclaimer''), without
+ *    modification.
+ * 3. Redistributions in binary form must reproduce at minimum a
+ *    disclaimer similar to the Disclaimer below and any redistribution
+ *    must be conditioned upon including a substantially similar
+ *    Disclaimer requirement for further binary redistribution.
+ * 4. Neither the names of the above-listed copyright holders nor the
+ *    names of any contributors may be used to endorse or promote
+ *    product derived from this software without specific prior written
+ *    permission.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ''AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT,
+ * MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE
+ * FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGES.
+ */
+begin 755 ar5523.bin
+M0!I@`#P;``@#>M@D$V``#0`````#X`@A/!N``"=[!@@\'X``)_\`-`-_V"($
+M$0`!``````-_V"``(/@A`V``"``````\&P`0`WK8)!-@``T``````^`((3P;
+M@``G>P8`/!^``"?_`'0#?]@B!!$``0`````#?]@@`"#X(0-@``@`````)!H`
+M`D":@``#X`@A/`*``"1"!!`\'X``)_\`K`!?$"($$0`!``````!?$"``(/@A
+M`$``"```````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````0!IH```````S6@!_/!N``"=[`@`#
+M>M@@CWL```-@``@``````````````````````````````````````````$`:
+M:```````,UH`?SP;@``G>P(``WK8((][```#8``(````````````````````
+M```````````````````````D&@"`/!N``"=[`@"/>P"``V``"```````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M``````!`@&@``````#P"$``T0O\`0()@`````````````````"0"``)`@H``
+M`````````````````^`((3P:@``G6@MT/!^``"?_!&0#7]`B!!$``0`````#
+M7]`@`"#X(0-`^`D``````^`((3P:@`(G6D:X/!^``"?_!)0#7]`B!!$``0``
+M```#7]`@`"#X(0-`^`D`````0`)@```````\`___-&,`_P!#$"1`@F``````
+M````````````/`2``B2$5_``@.@A/`$?_S0A__\#H>@D/`&@``.AZ"5`!(``
+M`````````````````^`((3P:@`(G6J6$/!^``"?_!1@#7]`B!!$``0`````#
+M7]`@`"#X(3P!'_\T(?__`T'0)#P!H``#0=`E`T#X"0`````\'(`#)YS&L#P$
+M@`(DA%?P`(#H(3P(@``\"8``)2D!P#P+@``E:P'DC2(``"$I``2M`@``(0@`
+M!!4K__L`````-`0`$#P%@``DI09T/`:``"3&`@"LQ0``(,8`!""$__\4@/_\
+M`````#P$@``DA`(`/`6``"2E!X"LA0``/`6``"2E"RBLA0`D/`6``"2E!G2L
+MA0"`K(4`A*R%`(@,``*L``````P`:5H`````#`!J>P`````,`&D:``````@`
+M;<\\'P``$````0````!`&F``/!O__S=[__`#6]`D0)I@````````````````
+M`$`:\`````````````````!`FG``````````````````0!I@`#P;_[\W>___
+M`UO0)$":8``D&@"(/!N``"=[`@"/>P"(`V``"``````#H-@A([W^L*^Z`(BO
+MH```KZ$`!*^B``BOHP`,KZ0`$*^E`!2OI@`8KZ<`'*^H`""OJ0`DKZH`**^K
+M`"ROK``PKZT`-*^N`#BOKP`\K[``0*^Q`$2OL@!(K[,`3*^T`%"OM0!4K[8`
+M6*^W`%RON`!@K[D`9*^\`'"OO@!XK[\`?```(!```"@2KZ0`@*^E`(2ONP!T
+M0`AH`$`)8`!`"C@`0`M``$`,&`!`#7@`0`YP`*^H`)BOJ0",KZH`E*^K`)RO
+MK`"DKZT`H*^N`)`\'(`#)YS&L`.@@"$CO?_@0!I@`#P;__\W>__P`UO0)$":
+M8``````````````````,`&D.`@`@(0@``DT``````Z#8(2.]_K"ON@"(KZ``
+M`*^A``2OH@`(KZ,`#*^D`!"OI0`4KZ8`&*^G`!ROJ``@KZD`)*^J`"BOJP`L
+MKZP`,*^M`#2OK@`XKZ\`/*^P`$"OL0!$K[(`2*^S`$ROM`!0K[4`5*^V`%BO
+MMP!<K[@`8*^Y`&2OO`!PK[X`>*^_`'P``"`0```H$J^D`("OI0"$K[L`=$`)
+M8`!`"C@`0`MP`*^I`(ROJ@"4KZL`D#P<@`,GG,:P/`*``B1"1E2,1```((0`
+M`:Q$```#H(`A)[W_X$`#8```````0`)H````````0Q`D``(2@C!"`'\\`X``
+M)&,*/`!#$""`4@```D"((20!``02(0`#`````!````P`````/!&@P#8Q`!".
+M,0``,C$`?SP#@``D8PH\`'$8((!Q```F,0`&`B"0(0``````$8B`0!I@`#P;
+M__\W>__P`UO0)$":8``````````````````\"H`")4HQ>`%14""-2@``/`6`
+M`B2E,?``L2@@C*4```)`("$!0/@)``````!`D"$\!8`")*4R:`"Q*"",I0``
+M`@`P(0P`=7,`0"`A`D`0(2.]`"!``F``/`/__S1C__X`0Q`D0()@````````
+M`````````(^H`)2/J0"`CZH`A$"(.``!(``1`4``$X^A``2/H@`(CZ,`#(^D
+M`!"/I0`4CZ8`&(^G`!R/J``@CZD`)(^J`"B/JP`LCZP`,(^M`#2/K@`XCZ\`
+M/(^P`$"/L0!$C[(`2(^S`$R/M`!0C[4`5(^V`%B/MP!<C[@`8(^Y`&2/O`!P
+MC[X`>(^_`'R/NP",-!K_``-:T"<#>M@D0!I@````````````,UK_``-ZV"6/
+MN@"0C[T`=#=[``)`FV``````````````````0)IP````````````````````
+M``]"```8`````````0`"``$``P`!``(``0`$``$``@`!``,``0`"``$`!0`!
+M``(``0`#``$``@`!``0``0`"``$``P`!``(``0`#X``(``````.@&"$`H.@A
+M([W_X*^C``BOOP`0`(#X"0````"/OP`0C[T`"`/@``@`````/`2``B2$1Z@\
+M!8`#)*6B'#"&``,4P``2``````"D.",`!SE"``<Y0`"'."&L@```K(``!*R`
+M``BL@``,K(``$*R``!2L@``8K(``'"2$`"`4Y/_V`````!"D``4`````H(``
+M`"2$``$4A?_]``````/@``@`````0!IP`$`;:``'8@`!(UH`!(];```\&@`'
+M-UH`#1=;``@`````-!H`.#P;@``G>P(``WK8((][```#8``(`````#0:`"0(
+M``&=`````#P)H,`U*0"0C24``"0$``0`I"@EK24``#P)H+,U*0`<C24``#"E
+M``$0H``,`````#P)H+,U*0`<C24``#P$__PTA/__`*0H)"0$````I"@EK24`
+M`!````D`````/`F@LS4I`!R-)0``)`3`_P"D*"0D!`L``*0H):TE```#X``(
+M`````".]_UBOL`!`K[$`1*^R`$BOLP!,K[0`4*^U`%2OM@!8K[<`7*^^`'BO
+MOP!\K[\`D".H`*BOJ`!T0`E@`*^I`(RLO0``C)T``(^D`!"/L`!`C[$`1(^R
+M`$B/LP!,C[0`4(^U`%2/M@!8C[<`7(^^`'B/OP!\CZ8`C(^]`'1``F``)`,`
+M!P##,"0`8!@G`$,0)`!&$"4#X``(0()@``````"LGP`LK)X`**R<`"2LEP`@
+MK)8`'*R5`!BLE``4K),`$*R2``RLD0`(K)``!*R=```#X``()`(```````",
+MGP`LC)X`*(R<`"2,EP`@C)8`'(R5`!B,E``4C),`$(R2``R,D0`(C)``!(R=
+M```#X``(`*`0(0`````GO?_PK[\```P``TL``$`AC[\```/@``@GO0`0)[W_
+M\*^R``BOLP`,K[$`!*^P````P$@A`0"0(0#@6"$`H'`A%,`!``"`4"$`AQ`K
+M$$``9#0"__\`1Q`K%$``60#@*"$D`@`(+.,!``!`,"$``S`+/`.``@#%$`8D
+M8Q9\`$,0(9!$```D`@`@`(8@(0!$.",0X``(``LT`@!'$",`3A`&`.H8!`!B
+M4"4`ZU@$`.YP!``+-`(!1@`;,6K__P`.)`)0P``!```!S0``$!(``!@0``,<
+M``!D&"4``"@2``````````!P2D@"`&D0*U!```L`:1@C`&L8(0!K$"L40``&
+M)*7__P!I$"M00``$`&D8(R2E__\`:Q@A`&D8(P!F`!M0P``!```!S3'$__\`
+M`!`2```8$``#'```9!@E```P$@``````````<$I(`@!I$"L00``+``44``!K
+M&"$`:Q`K%$``!B3&__\`:1`K$$``!``%%```:Q@A),;__P`%%```1C`E`&EP
+M(P``F"$20``$`.[(!@``P"&N6```KED`!`)@$"&/L@`(C[,`#(^Q``2/L```
+M`,`8(0/@``@GO0`0/`(`_S1"__\D`P`0`$<0*R0$`!@`8#`A$`#_I0"",`L4
+MX``*`$L0*R0"``$`1@`;4.```0```<T``%@2```````````T`O__`$L0*Q1`
+M`(8\`@#_)`(`""UC`0``0"@A``,H"SP#@`(`JQ`&)&,6?`!#$"&01```)`(`
+M(`"%("$`1#@C%.``.@!'>",!2U`C)!,``0`+9`(Q:/__`4P`&P`.+`)1@``!
+M```!S0``(!(``!@0``,<``!E&"4``&@2``````````!PB$@"`&D0*U!```L`
+M:1@C`&L8(0!K$"L40``&)(W__P!I$"M00``$`&D8(R6M__\`:Q@A`&D8(P!L
+M`!LQQ/__48```0```<T``!`2```8$``#'```9!@E``!0$@``````````<$A(
+M`@!I$"L00``+``T4``!K&"$`:Q`K%$``!B5*__\`:1`K$$``!``-%```:Q@A
+M)4K__P`-%```2C`E$`#_F`!I<",`ZU@$`>H@!@`+9`(`C``;,6C__P'N&`8`
+MZA`$`$-0)0`*+`)1@``!```!S0&`:"$`[G`$```P$@``(!``!"0``(48)0``
+M>!(``````````'#(2`(`:1`K$$``"@$`F"$`:Q@A`&L0*Q1```8DS___`&D0
+M*U!```0`:1@C)>___P!K&"$`:1@C`&T`&S%$__]1H``!```!S0``$!(``!@0
+M``,<``!D&"4``#`2``````````!P4T@"`&D0*Q!```L`#Q0``&L8(0!K$"L4
+M0``&),;__P!I$"L00``$``\4`"3&__\`:Q@A``\4``!&F"40`/^.`&E0(S1"
+M__\D`P`0`$L0*R0$`!@`8"@A$`#_>0""*`L`AA`K$$``!S0"__\``#`A``"8
+M(1$`_U$`H,@A$`#_30%`P"$`1A`K%$``?#P"`/\D`@`(+,,!``!`*"$``R@+
+M/`.``@"I$`8D8Q9\`$,0(9!$```D`@`@`(4@(0!$.",4X``1`$=X(P$J$"L4
+M0``%`<L@(P'+$"L40``'```P(0'+(",!Q!`K`4D8(P!B4",`@'`A)`8``1)`
+M_S```)@A$`#_W@'`R"$!ZQ`&`.D8!`!B2"4`"60"`>H@!@",`!LQ,___`>X0
+M!@#J&`0`8E`E``HL`E&```$```'-`.M8!```,!(``"`0``0D``"%&"4``$`2
+M``````````!PTV@"`&T0*Q!```H`[G`$`&D8(0!I$"L40``&),C__P!M$"M0
+M0``$`&T8(R4(__\`:1@A`&T8(P!L`!LQ1/__48```0```<T``!`2```8$``#
+M'```9"@E```P$@``````````<%-H`@"M$"L00``+``@4``"I*"$`J1`K%$``
+M!B3&__\`K1`K$$``!``(%``DQO__`*DH(0`(%```1C`E`,L`&0"M*",``&`0
+M````````````K!`K``!H$@``````````%$``$`&K(",1A0`,`<T0*Q)`_N(`
+M`)@A`<T8(P"L(",!PQ`K`()0(P'J*`0`XQ@&`*/()1``_M<`ZL`&$$#_]`&K
+M(",!I!`K`8D8(P!B8",`@&@A$`#_[B3&__\T0O__)`,`$`!&$"LD!``8`&`H
+M(1``_X,`@B@+)[W_X*^_`!`,``3L`Z!`(8^B``"/HP`$C[\`$`/@``@GO0`@
+M)[W_\*^R``BOLP`,K[$`!*^P````P$@A`0"0(0#@6"$`H'`A%,`!``"`4"$`
+MAQ`K$$``9#0"__\`1Q`K%$``60#@*"$D`@`(+.,!``!`,"$``S`+/`.``@#%
+M$`8D8Q9\`$,0(9!$```D`@`@`(8@(0!$.",0X``(``LT`@!'$",`3A`&`.H8
+M!`!B4"4`ZU@$`.YP!``+-`(!1@`;,6K__P`.)`)0P``!```!S0``$!(``!@0
+M``,<``!D&"4``"@2``````````!P2D@"`&D0*U!```L`:1@C`&L8(0!K$"L4
+M0``&)*7__P!I$"M00``$`&D8(R2E__\`:Q@A`&D8(P!F`!M0P``!```!S3'$
+M__\``!`2```8$``#'```9!@E```P$@``````````<$I(`@!I$"L00``+``44
+M``!K&"$`:Q`K%$``!B3&__\`:1`K$$``!``%%```:Q@A),;__P`%%```1C`E
+M`&EP(P``F"$20``$`.[(!@``P"&N6```KED`!`)@$"&/L@`(C[,`#(^Q``2/
+ML````,`8(0/@``@GO0`0/`(`_S1"__\D`P`0`$<0*R0$`!@`8#`A$`#_I0""
+M,`L4X``*`$L0*R0"``$`1@`;4.```0```<T``%@2```````````T`O__`$L0
+M*Q1``(8\`@#_)`(`""UC`0``0"@A``,H"SP#@`(`JQ`&)&,6?`!#$"&01```
+M)`(`(`"%("$`1#@C%.``.@!'>",!2U`C)!,``0`+9`(Q:/__`4P`&P`.+`)1
+M@``!```!S0``(!(``!@0``,<``!E&"4``&@2``````````!PB$@"`&D0*U!`
+M``L`:1@C`&L8(0!K$"L40``&)(W__P!I$"M00``$`&D8(R6M__\`:Q@A`&D8
+M(P!L`!LQQ/__48```0```<T``!`2```8$``#'```9!@E``!0$@``````````
+M<$A(`@!I$"L00``+``T4``!K&"$`:Q`K%$``!B5*__\`:1`K$$``!``-%```
+M:Q@A)4K__P`-%```2C`E$`#_F`!I<",`ZU@$`>H@!@`+9`(`C``;,6C__P'N
+M&`8`ZA`$`$-0)0`*+`)1@``!```!S0&`:"$`[G`$```P$@``(!``!"0``(48
+M)0``>!(``````````'#(2`(`:1`K$$``"@$`F"$`:Q@A`&L0*Q1```8DS___
+M`&D0*U!```0`:1@C)>___P!K&"$`:1@C`&T`&S%$__]1H``!```!S0``$!(`
+M`!@0``,<``!D&"4``#`2``````````!P4T@"`&D0*Q!```L`#Q0``&L8(0!K
+M$"L40``&),;__P!I$"L00``$``\4`"3&__\`:Q@A``\4``!&F"40`/^.`&E0
+M(S1"__\D`P`0`$L0*R0$`!@`8"@A$`#_>0""*`L`AA`K$$``!S0"__\``#`A
+M``"8(1$`_U$`H,@A$`#_30%`P"$`1A`K%$``?#P"`/\D`@`(+,,!``!`*"$`
+M`R@+/`.``@"I$`8D8Q9\`$,0(9!$```D`@`@`(4@(0!$.",4X``1`$=X(P$J
+M$"L40``%`<L@(P'+$"L40``'```P(0'+(",!Q!`K`4D8(P!B4",`@'`A)`8`
+M`1)`_S```)@A$`#_W@'`R"$!ZQ`&`.D8!`!B2"4`"60"`>H@!@",`!LQ,___
+M`>X0!@#J&`0`8E`E``HL`E&```$```'-`.M8!```,!(``"`0``0D``"%&"4`
+M`$`2``````````!PTV@"`&T0*Q!```H`[G`$`&D8(0!I$"L40``&),C__P!M
+M$"M00``$`&T8(R4(__\`:1@A`&T8(P!L`!LQ1/__48```0```<T``!`2```8
+M$``#'```9"@E```P$@``````````<%-H`@"M$"L00``+``@4``"I*"$`J1`K
+M%$``!B3&__\`K1`K$$``!``(%``DQO__`*DH(0`(%```1C`E`,L`&0"M*",`
+M`&`0````````````K!`K``!H$@``````````%$``$`&K(",1A0`,`<T0*Q)`
+M_N(``)@A`<T8(P"L(",!PQ`K`()0(P'J*`0`XQ@&`*/()1``_M<`ZL`&$$#_
+M]`&K(",!I!`K`8D8(P!B8",`@&@A$`#_[B3&__\T0O__)`,`$`!&$"LD!``8
+M`&`H(1``_X,`@B@+)[W_X#P$H)"OOP`0#``&C`.@*"&/OP`0`^``"">]`"`G
+MO?_0K[``$`"`@"$D!`!<K[0`(*^S`!ROL0`4K[\`)*^R`!@,`'I7`*"@(0!`
+MF"$00``*)!$``B0&`%P`0"`A#`!PJ@``*"$"`"`A#``=90.@*"$00``+`$"(
+M(5:```&ND0``C[\`)(^T`""/LP`<C[(`&(^Q`!2/L``0`^``"">]`#"/H@``
+M`$`@(0P`!L^N8@`$%$#_\0!`B"$,`!FN`F`@(11`_^T`0(@A#``2D`)@("$4
+M0/_I`$"((0P`&=D"8"`A`$"0(1!`_^0D$0`"C[````!`("$,`!G1)`4'(`!`
+MB"$D`@<@K@(;[`)`*"$"8"`A#``<@JYR```\`X`"K'-'J!``_]6N8@`()[W_
+M\*^P``"OOP`$#``4GP"`@"$"`"`A#``4K```*"$00``&)`,`%H^_``2/L```
+M`&`0(0/@``@GO0`0#``?*P(`("$`0!@A)@4;<`(`("$40/_U)`8``0P`-X4`
+M`````$`8(20%``$40/_O`@`@(0P`%*P`````)`,`%A``_^H``A@*K*0`!`/@
+M``BLX```)[W_H*^S`$P`H)@A```H(:^Q`$2OOP!0K[(`2*^P`$`,`!2L`("(
+M(11``!6OH@`PCB(7W!1``"T#H"`A#`!Z=0````"/I@`@CZ4`'#P$@`(DT,0`
+M`!`7P@("$"$``I!##`!P="2$"(@"("`A#``)M@)`*"&OH@`P$$``"P.@("$2
+M8``"CZ(`,*YB``"/OP!0C[,`3(^R`$B/L0!$C[``0`/@``@GO0!@#`!Z=0``
+M``"/H@`@`B`@(210Q``,``RQ`@`H(3P$@`(DA`BD`@`H(0)`,"$40/_JKZ(`
+M,`P`<'0`````)`(``:XB%]P"("`A#``'5R>E`#"/H@`P4$#_X*X@%^@0`/_>
+M`````">]_^"OL0`4K[``$*^_`!B,@Q?@)`(``0"@B"$`@(`A`Z`H(1!B``NO
+MH```$B``!JX`%^B/OP`8C[$`%(^P`!`#X``()[T`((^B```0`/_YKB(```P`
+M!Y4`````CZ(``!1`__("`"`A#`!1E``````"`"`A#``4K"0%``$0`/_K````
+M`">]_^"OL@`(`*"0(0``*"&OLP`,K[$`!*^P``"OOP`0#``4K`"`@"$D$P`!
+M`$"((1!```L"`"`A$D```JX`%^BN40``C[\`$(^S``R/L@`(C[$`!(^P```#
+MX``()[T`(*X`&QP,``A^KA,;)`P`"A@"`"`A#`!Q7``````D8P#(+&4`R(X$
+M&`@`11`A`$`P(20(```D"0#(#`!Q=`!@."$,`'%<`````"1C`,@L90#(C@08
+M0`!%$"$`0#`A)`@``"0)`,@,`'%T`&`X(8X"``0,`!T8C$0```(`("$,`"D\
+M/`6```P`!_8"`"`AKA,7X!``_]$`````)[W_\*^R``BOL0`$K[```*^_``R,
+M@A?@)!(``0"@B"$`@(`A$%(`"@``*"$2(``"KA(7Z*X@``"/OP`,C[(`"(^Q
+M``2/L````^``"">]`!`,`!2L`````#P%@``,`"EL`@`@(0P`<7N.!!@(#`!Q
+M>XX$&$`,``LT`@`@(0P`$7<"`"`A#``(E0(`("$D!0`!#``*2`(`("$,``@'
+M`@`@(8X"``0,`!T*C$0``*X`%^`0`/_>`````">]__"OL````*"`(:^Q``0`
+MP"@A`("((0(`("&OOP`,K[(`"`P`!S8`P)`A#`!1E`(`("$"("`A)`4`"```
+M,"$,`!$9)`<``59```&N0```C[\`#(^R``B/L0`$C[````/@``@GO0`0)[W_
+M\*^Q``2OL````("((0"@@"&OOP`(#``4;20%``$"("`A$@``"SP%``0,`"D\
+M``````(@("$,`!1M```H(8^_``B/L0`$C[````/@``@GO0`0#``I;#P%``00
+M`/_V`B`@(2>]__`D`@`!K((<S*R"',BOL``````H(0"`@"&OOP`$#`!K-"0$
+M`!&.!!S,#`!K-"0%``&/OP`$C[````/@``@GO0`0)[W_\*^P``"L@!S(`("`
+M(20%``&OOP`$#`!K-"0$`!&.!!S,#`!K-```*"&/OP`$C[````/@``@GO0`0
+M)[W_\*^P````H(`A)`4``:^_``ROL@`(K[$`!`"`D"$,`!1M`,"((28$`!`,
+M`&LT)`4``0(`("$,`&LT)`4``0(`("$,`&LE`B`H(0)`("$,`!1M```H(8^_
+M``R/L@`(C[$`!(^P```#X``()[T`$">]_]"OM0`4`*"H(20%``&OOP`@K[<`
+M'*^V`!BOM``0K[,`#`$`H"$`X)@AK[(`"*^Q``0`P)`A`("((:^P```,`!1M
+M/!:``CP0@`(FI``0/!>``@``*"&N$C.`KM,SA`P`:S2N]#.(`J`@(0P`:S0D
+M!0`!CB4``(X$,X`\`O_QC*-`$#1"__\`!"1``&(8)#P"``X`@B`D`&08):RC
+M0!".)0``CL0SA#P"_X^,HT`0-$+__P`$)0``8A@D/`(`<`""("0`9!@EK*-`
+M$(XE``".Y#.(/`+_?XRC0!`T0O__``0EP`!B&"0\`@"``((@)`!D&"6LHT`0
+M`B`@(0P`%&T``"@AC[\`((^W`!R/M@`8C[4`%(^T`!"/LP`,C[(`"(^Q``2/
+ML````^``"">]`#`GO?_PK[```"0&``0`@(`AK[\`!`P`-@X``"@A#`!1O28$
+M'8P00``'`@`@(0P`-;2,10`D#``UN`(`("$,`#7H`@`@(0P`<=`F!!K(C[\`
+M!(^P```#X``()[T`$">]__```"@A)`8``Z^P``"OOP`$#``V#@"`@"$,`''"
+M)@0:R`P`<+4`````#``U_0(`("$,`#6_`@`@(0P`<+H`````#``(VP(`("$D
+M`@`!K@(;)(^_``2/L````^``"">]`!",I0`H)`/_\#"B``\D0@`G`$,X)`"C
+M&"0`9Q`A`$,0*Q1```<`8#`AO'$``"1C`!``QQ`A`$,0*Q!`__L`````D*(`
+M`"0#`(`P0@#\4$,`!)2C`!0``"`A`^``"`"`$"&4@AQ>5&+__```("&4HP`2
+ME((<7%1B__@``"`AE((<6I2C`!`48O_S)`0``1``__(`````C((:_(Q#```#
+MX``(K(,:_(R"&ORLH@```^``"*R%&OPGO?_@)`(``:^P``"OOP`<K[8`&*^U
+M`!2OM``0K[,`#*^R``BOL0`$`("`(:R"&R0F%AV,#`!1O0+`("$`0(@A$$``
+MD0``("&,0P``C$(`)!!B`(Z/OP`<`@`@(0P`-L\"("@A`$"H(20"``T2H@"&
+M)`0``8XD`"B.`QKT)`7_\#""``\D0@`0EC0`0`!%,"0`A2@D`*80(0!T&"$`
+M11`KK@,:]"8S`$`DDO_\%$``!P"@&"&\L0``)*4`$`!F$"$`11`K$$#_^P``
+M``"F5``"#`!1OP+`("&2`AKX$$``9@(@*"$D`@`=$J(`5B:#`$"2`AKXHD(`
+M`(X"&^PD0O_\`$,0*Q1``$,D`__\DD0``99"``(TA``!)$(``P!#&"0P@@`"
+MHD0``:9#``(``(@A%$``!"9%``0P8O_\`D(0(21%``260@`")$(`0*9"``*.
+M`AKT)`,`%:RC``0D0@!`K*(``(X"&U268P`")*0`#*RB``BL@P`$K(``"(X"
+M&O2LM0`,E@,;<*R"``R28@`&C@8:W*R"`!"28@`*K((`%()G``FL@P`<K(<`
+M&))B``2L@@`HDF(`"ZR"`"R28@`%K((`())B``@L10`@)$/_X`!E$`H0P``#
+MK((`)%*@``&N!Q?`C@(`!`)`*"$,`!K'C$0``!(@``0"("@AC@(`!`P`&L>,
+M1```H@`:^!``_XBN`!KT#``(TP(`("$`0(@AD@(:^"8E``0D0@`!HB(``"0"
+M``.B(@`!)`(`0!``_\2F(@`"D@,:^`)`*"$D8@`!H@(:^))"``&B0P``,$(`
+M_J)"``&.`@`$#``:QXQ$```0`/]P)A8=C`P`"*X"`"`A$`#_F*X"&MR/OP`<
+MC[8`&(^U`!2/M``0C[,`#(^R``B/L0`$C[````"`$"$#X``()[T`(">]__"O
+ML0`$K[````#`B"$`H(`A)`4``:^_``ROL@`(#``4;0"`D"$"`"`A#``W=`(@
+M*"$"`#`A`@`H(0P`4=@F1!V,`D`@(0P`%&T``"@AC[\`#(^R``B/L0`$C[``
+M``/@``@GO0`0)[W_\*^_``"0H@`!,$(``A!```8DI_^D#``(UP````"/OP``
+M`^``"">]`!"@H```H*```8R&&^RDH``"`.`H(0P`"8@DQO_\$`#_]H^_```G
+MO?_0K[\`(*^W`!ROM@`8K[,`#*^Q``2OL```K[4`%*^T`!"OL@`(C((;[`"`
+MF"$DA!V,)%``H`"P`!JN8!M44@```0```<T``+`2#`!1KP``B"$00``1`$"X
+M(1(@``0"X!`A#`!Z:@(@("$"X!`AC[\`((^W`!R/M@`8C[4`%(^T`!"/LP`,
+MC[(`"(^Q``2/L````^``"">]`#!RT!@")`(``:YB';0D<``0#`!Z5P(`("$0
+M0``R`$"((0!`("$``"@A`@`P(0!`D"$,`'"J)C4`8`+V$"JN<1JX`B"@(:YP
+M&KP00``5`J"((0+`@"&.9AOLKE0`)*Y5`"BN40`$`D`H(0)@("$,``F(),;_
+M_(YB&^PF$/__`B(H(0)"&"$"HB`A`H(0(21R`%PDE0!<)%0`7!8`_^XDL0!<
+M)I$`$!K```BN8!K\`L"`(0(@*"$"8"`A#``(UR80__\6`/_[)C$`1"9P&L@,
+M`'&T`@`@(0P`<<D"`"`A$`#_O0+@$"$0`/^V)!<``B>]__"OL````("`(:^_
+M``0,`''0)(0:F`P`<=`F!!J$)`(``:X"&K2/OP`$C[````/@``@GO0`0)[W_
+M\*^R``BOL0`$K[```*^_``P`@)`A`("`(201``..`@`($$``#R8%``@`H!@A
+M)`0``R0"``$DA/__K&(`-*1@`#"D8``R!('_^R1C``@DI0!<K@``6*X``%P,
+M`!#@`D`@(28Q__\&(?_M)A`!'(^_``R/L@`(C[$`!(^P```#X``()[T`$">]
+M__"OL0`$`("((:X@&K0DA!J$K[\`"*^P```,`''"`*"`(0P`<<(F)!J8$@``
+M"P(@("$,``U[`B`@(0P`"B8"("`AKB`;'(^_``B/L0`$C[````/@``@GO0`0
+M#``.V0`````0`/_S`````(RB`#P\`X?_-&/__SP$?_\`0Q`D-(3__SP#_O\T
+M8___`$00)`!#$"2LH``$)[W_\*R@```DI`!<K*``"*RB`#RLH``,K*``(*R@
+M`!"LI0`DK*``%*R@`"BLH``8K*4`,*RE`"RLH``<K[\```P`&RD`````C[\`
+M``/@``@GO0`0)[W_\"2E__ROL0`$K[\`"*^P```0H``'`("((8RP```"("`A
+M#``*8R2E_Z06`/_[`@`H(8^_``B/L0`$C[````/@``@GO0`0)[W_\*^Q``2O
+MOP`(K[````"`B"&,L``@#``*8P(@("$6`/_\`@`H(8^_``B/L0`$C[````/@
+M``@GO0`0)[W_X*^_`!"OLP`,K[$`!*^P``"OL@`(C((`!`"@F"$D!0`4C%(`
+M``#`B"$,`!L*`D`@(0!`@"$"(#`AK%,`"`!`("$,`!L^)`4`&`)`("$,`!J]
+M`@`H(8^_`!"/LP`,C[(`"(^Q``2/L```)`(``0/@``@GO0`@)[W_X*^S``RO
+ML@`(K[```#P"@`*OOP`0K[$`!(Q%1ZPD@P`T)((`+`"`@"&L@@`PK(,`.*R`
+M`"RL@``T``"0(1B@`!0``)@A#`!Z5R0$``PF4@`!`$"((0!`("$``"@A$$``
+M%R0&``P,`'"J`````#P"@`*,0D>LC@,`.*X@```"0A`JKA$`.*QQ```40/_N
+MKB,`!"8$`"0,`'&'```H(0)@$"&/OP`0C[,`#(^R``B/L0`$C[````/@``@G
+MO0`@$`#_]R03``(GO?_PK[$`!*^_``BOL```C)``!`P`<+4`H(@AC@0`-(R%
+M```0H``3C((`!*RB``2.`@`PC(,`!*R1``BL@@`$C@(`,*QE``"N!``PK(``
+M``P`<+JL1```#`!QHR8$`"2/OP`(C[$`!(^P```#X``()[T`$!``_^ZN`@`X
+M)[W_\*^Q``0`@(@A)(0`)*^_``@,`'&.K[````P`<+4`````CB0`+(R%```0
+MH``3C((`!*RB``2.(@`XC(,`!(R0``BL@@`$CB(`.*QE``"N)``XK(```*R`
+M``@,`'"ZK$0```(`$"&/OP`(C[$`!(^P```#X``()[T`$!``_^ZN(@`P)[W_
+MX*^Q`!2OL``0K[\`'*^R`!B,D@`$`("`(0.@*"$F1``D#`!QJJ^@``"/H@``
+M'$``!P``B"&/OP`<C[(`&(^Q`!2/L``0`^``"">]`"`,``L3`D`@(1!`__<`
+M0"@A,$(`#R1"`"LD`__P`$,P)`"C&"0`9A`A`$,0*Q1```<`8"`AO'$``"1C
+M`!``AA`A`$,0*Q!`__L`````C*,`""0"`!`08@`*``````P`"H,"`"`ACZ(`
+M`"8Q``$"(A`J%$#_XX^_`!P0`/_=C[(`&`P`"O0"`"`A$`#_]X^B```GO?_P
+MK[(`"*^P``"OOP`,K[$`!(R1``0D`@`!``"`(0(@("&N(A?8#``*@R8R&JP"
+M0"`A#`!QAP``*"$,``ZZ`B`@(2H#``440``.)A```5!@``TF,!J8#`!Q7```
+M```D8P!D+&0`9`!$$"$`0#`A`&`X(0P`<94"0"`A$`#_[P`````F,!J8#`!Q
+MT`(`("$,`'&C)B07T`P`<8XF)!?(`@`@(:X@%]@,`''"`````(^_``R/L@`(
+MC[$`!(^P```#X``()[T`$">]_]"OL0`DK[\`+*^R`"BOL``@`*"((8R2``0`
+M`%`A``!P(1"@`'X``'@AC*T`#(RD`!`QH@`/``(8P`!B&",``QC``D,8(20"
+M__\0@@`8)'D8>``$$,``1!`A``(0P`!$$",``A"``D(0(8Q$``@D`P`!KZ,`
+M!!2```<D3P`(C*(`%%!```6-[@!8C*4`!!2@`&\"0"`AC>X`6!'``&DD`@`&
+MC<,`!%!B`&<``'@A)BG__(XL`!@!((`A)CC_H!$@`#\D!@`<`29`(24J_Z0E
+M"``$K4@`!(TK``"10P`.)`7P```+$"L``A$`,&,`[P!B&"6A0P`.CB(``)5#
+M``HE9_^D)$+_Y#!"#_\`91@D`&(8):5#``J.0AOLE40`#@`+.`H`1A`C)$+_
+M_`&"&"L!@Q`+`(4@)#!"#_\`@B`EI40`#I5&``ZM)_^DC4(`/"0#_?\`#2E`
+M`$,0)#"E`@``#1E"/`3^_S2$__\`11`E,&,``0!$$"0``QX``$,0):U"`#RM
+M2``HCB,`$##&#_\D`O__)&0`!*U'`""M6``PK4\`4*U.`$P08@`C`89@(ZU$
+M`#@!8$@A%6#_PP``,"$"`$@A$2``#`%`@"$E*O^DC4,`/(TK```\`G__-$+_
+M_P!B&"0!8$@AK4,`/*U9`%@58/_VK5``+`)`("$#("@A`P`P(0P`#TL``#@A
+MCB(`%%!```.N``!4CB(`!*X"`%2/OP`LC[(`*(^Q`"2/L``@`^``"">]`#`D
+M`G__$`#_W:U"`#@``'@A$`#_F0``<"$,``JD`Z`P(:X@``0"0"`A#``*@P(@
+M*"$0`/_NC[\`+">]_]"OL0`4K[\`(*^S`!ROL@`8K[``$`"`B"$,`'#^C)(`
+M!`!`("$,`'$!```H(0(@("$,``L3)E,:F`)@("$,`''"`$"`(0)`("$,`!1M
+M)`4``8X#``@D`@`0`@`H(1!B`!X"("`ACD(7X`(`*"$40``6`D`@(0P`"H,`
+M````)B0`)`P`<:H#H"@ACZ(`````@"$00``$`B`@(0P`"Q,``````$"`(58`
+M_^N.`P`(#`!QT`)@("$"0"`A#``4;0``*"$0`/_;`B`@(0P`"YX"("`A$`#_
+MZR8D`"0,``ML`````!``_^<F)``D)[W_X*^S``ROL@`(K[$`!*^_`!"OL```
+M`("((0P`"L*,D``$/!.``A!```D`0)`A`D`0(8^_`!"/LP`,C[(`"(^Q``2/
+ML````^``"">]`"`,`'&T)@0:F#P%@``\!X`"/`B``CP+@`(D!``!)FID$"2E
+M,.@"(#`A).<(Q"4(6!`E:V08#`!PRR0)#``,`'#DCF1D$!``_^<"0!`A)[W_
+M\*^P``"OOP`$C((;]`"`@"&,0@!$%$``!20$.@"/OP`$C[````/@``@GO0`0
+M)`(`!PP`>E>N`AL`)$("`"0#_@``0Q`DK@(=Q!``__2N`AL$)[W_X*^_`!RO
+MM@`8K[4`%*^T`!"OL@`(K[$`!*^P``"OLP`,C((;[`"`B"$DD!AX)%4`7`"U
+M`!H\`H`"4J```0```<V,E@`$``"@(0``D"$``"@2#``,G*Q%1ZR.(QOT)`(`
+M!JQB`&@,`%&O`@`@(292``$`0)@A)A``.!9@`"(J0@`&%$#_^"8D&JP``"@A
+MKB`7V`P`<8<\$H`")B07R`P`<8<``"@A)B07T`P`<8<``"@ACD)'K'!5@`(,
+M`'I7`@`@(1!``#$`0*`A`@`P(0!`("$,`'"J```H(8Y"1ZRN,!IXKC0:=`)B
+M$"HFD`!<%$``&P``D"$,``QU`L`@(1!``!``0)@A$H``!`)@$"$,`'IJ`H`@
+M(0)@$"&/OP`<C[8`&(^U`!2/M``0C[,`#(^R``B/L0`$C[````/@``@GO0`@
+M)C`:A`P`<;0"`"`A#`!QR0(`("$0`/_P`F`0(28%_Z0,``IC`B`@(3P"@`*,
+M0D>L)E(``0)"$"H40/_X`A6`(1``_]P`````$`#_WB03``(GO?_`K[``,*^_
+M`#BOL0`TC((;](S+```D#``!C$,`"`%L$`0`@(`A`&(8)!1@`"D`H$@A+*(`
+M!A!``"8`!1#`C,H`&(S#`!2,QP`0C,0`!`!%$".,Q0`(C,8`#``"$,`"`A`A
+M,4@`"*^C``"OJP`(KZ0`#*^E`!"OI@`4KZ<`("11&'BOJ@`$KZ``&*^@`!RO
+MH``D$0``!J^@`"@5(``6`````(X"'<0D0AP`KZ(`*`(`("$,`$IH`Z`H(0!`
+M&"&.`AOTC$(`:`!B$"M40``&KB,`)(^_`#B/L0`TC[``,`/@``@GO0!`#`!Q
+MM`(@("$0`/_YC[\`.%$L_^R.`AW$)`+_]P%"$"00`/_IKZ(`!">]__"OL@`(
+MK[$`!*^P``"OOP`,`("`(0``D"$DD1G(C@,:<"0"``$"0A`$`&(8)`)`*"$4
+M8``+`@`@(292``$N0@`&%$#_]B8Q`!R/OP`,C[(`"(^Q``2/L````^``"">]
+M`!`,`$J'``````(`("$"0"@A#``-%@(@,"$0`/_P)E(``2>]__"OL@`(K[$`
+M!*^P``"OOP`,`("`(0``D"$DD1AXC@(:<"0#``$"0Q@$`$,0)%!```DF4@`!
+MC@(;]`)`*"$"`"`AC$(`"`!#$"040``+)`8`!"92``$N0@`&%$#_\"8Q`#B/
+MOP`,C[(`"(^Q``2/L````^``"">]`!`,`$JV)E(``0(`("$,`!#@`B`H(1``
+M__(N0@`&)[W_\*^_``2OL```C*8`.`P`2[4`H(`AC@(`+(^_``2/L```K$``
+M0`/@``@GO0`0)[W_L*^U`#2OM``PK[,`+*^_`$"OMP`\K[8`.*^R`"BOL0`D
+MK[``((RR`!0`H*@A`("8(1)``&XDE`2`CD(`/`1!``0``+`ACD(`(!!``.8`
+M0)`ACD(`4%!```6.40`LC$(``%!```&N0`!0CE$`+(XB`$`40``')C<`0`)@
+M("$,`$VK`B`H(20#``T00P!9C[\`0`)@("$,`!"&`B`H(11``,\D`@`!CB,`
+M0*Y@&QP08@`&CE``4%(```6.H@`<CF(;:!1``+\"8"`ACJ(`'"1"__\2```*
+MKJ(`'(ZB`"2.!`!0``(0P`!0$"&40P`P)(3__ZX$`%`D8___I$,`,(XB`#P\
+M`X``KC$`,`!#$"6N(@`\KC$`+(ZB`!12(@`$CNL```!`L"&NL0`4CNL``"0"
+M``$18@!5CDH`*(Z"`#".@P`T)&,``2QD``$`1!`AKH(`,*Z#`#2.H@`P4$``
+M!H[C``",0P`PCB(`,!!#`#\"8"`ACN,``"0"``%08@`"KZ``!*^B``26X@`(
+MEN,`!(XD`%2OH@`,KZ,`"*^@`!`4@``HKZ``%%+```^.L@`44M$`#8ZR`!2.
+MP@!4`L"`(11``!:.U@`@`@`H(0P`"F,"8"`A4L``!(ZR`!16T?_WCL(`5(ZR
+M`!160/^5CD(`/(^_`$"/MP`\C[8`.(^U`#2/M``PC[,`+(^R`"B/L0`DC[``
+M(`/@``@GO0!0#`!PN@````".!0!4`F`@(0P`"J0#H#`A#`!PM0`````0`/_B
+MK@``5`P`<+H`````CB4`5`)@("$,``JD`Z`P(0P`<+4`````$`#_T*X@`%0,
+M`"EL/`6``*Z@`#`"8"`A#``I/#P%@``0`/^\CN,``))$`#R.A@`0CH<`%```
+M$"$PA``!)(0``0#D."$`Y"`K`,(P(0#$,"&NA@`0KH<`%)9$``J.B``XCHD`
+M/(YE!:@PA`__`21((0$D("L!`D`A`01`(:Z(`#BNB0`\!*(``:Y@!:B.8ANP
+M$$L`(B5"``200P``,&(``51```HD`@#_CH(`&(Z#`!PD8P`!+&0``0!$$"&N
+M@@`8KH,`'!``_XR.H@`P$$,`"@````".@@`HCH,`+"1C``$L9``!`$00(:Z"
+M`"BN@P`L$`#_@8ZB`#".@@`@CH,`)"1C``$L9``!`$00(:Z"`""N@P`D$`#_
+M>(ZB`#"50P`(EF(<7E!B``.50P`&$`#_VB5"`!"68AQ<%&+_UR5"`!"50P`$
+MEF(<6E1B_],E0@`0CH(!+"1"``$0`/_TKH(!+`*@*"$,`!`!`B`P(5!`_X6.
+ML@`4$`#_/8ZB`!P0`/^#KB``0(YB%]A00/^!C[\`0`P`<:,F9!JL$`#_?8^_
+M`$",@P`4$&``""0$``&,8@`\!$,`!0``("&,8@`@$$```@```````"`A`^``
+M"`"`$"$GO?_PK[$`!*^_``@`@(@AK[```(XB&\P00``4)(08>`P`#JX`````
+M)B08L`P`#JX``H`K)B08Z`P`#JX``H`*)B09(`P`#JX``H`*4$```0``@"$"
+M`!`AC[\`"(^Q``2/L````^``"">]`!`,``ZN`````!``__<``H`K)[W_\*^P
+M``"OOP`$#`!PM0"`@"&.`AO,)@48>!!```L"`"`A#``-K0`````F!1BP#``-
+MK0(`("$F!1CH#``-K0(`("$F!1D@`@`@(0P`#:T`````#`!PN@````"/OP`$
+MC[````/@``@GO0`0C*<`*```2"&0X@`!,$(`!!1``!",J``LD.(`%C!#``]4
+M8``-C*(`/)4"`$8``"`A``(1`J#B`!>1`@!',$(`#P`"$0``8A`EH.(`%@/@
+M``@`@!`AC*(`/``"%H(P0@`!5$``#Y#B``&5"0!&D.,`%C$B``\``A$`,&,`
+M#P!B&"4`"1$"H.,`%J#B`!>,H@`\/`,$``!#$"6LH@`\D.(``3!"``000/_I
+M`2`@(8T&`"!0P/_F`2`@(8S(`"PD!/_PC,<`*"4#``@P8@`/)$(`)P!D&"0`
+M1"@D`&40(0!#$"L40``'`&`@(;QQ```D8P`0`(40(0!#$"L00/_[`````(S"
+M`#P``A:",$(``51```Z0X@`!D.,`%C$B``\``A$`,&,`#P!B&"4`"1$"H.,`
+M%J#B`!>,P@`\/`,$``!#$"6LP@`\D.(``3!"``140/_8C08`(!``_[T!("`A
+M)[W_X*^S``P`P)@AK[4`%*^T`!"OL0`$K[```*^_`!BOL@`(`("@(0"@B"$,
+M`'"U``"H(1)@`$("8(`AC@(`/``"%D(P0@`!%$``58X2`%`20`!4`H`@(8Y"
+M`%000`!1`````!*@`$R.`@`LC$,``(Q"`""NHP``KJ(`((X&`"PF1`!<`@`H
+M(:S``""LP```#`!1V`````".0@!X)$(``:Y"`'@2H``"`F"`(8ZP`"!6`/_C
+MC@(`/!)@`"$`````EB,`-%!@`!&.(@`8CF0`*)""`!8P10`/4*``"Z8@`#0`
+M`Q$"H((`%Y(B`#60@P`!,$(`#P`"$0``HA`E-&,`"*"#``&@@@`6CB(`&%1`
+M`!VL4P`@)`(``:XB`"RN,P`4KC4`&`````^.(@`L5$``$(YF`"2.)0`D#`!*
+MG`*`("$,`'"Z`````(^_`!B/M0`4C[0`$(^S``R/L@`(C[$`!(^P`````!`A
+M`^``"">]`"".)0`D#`!*E0*`("$0`/_MKB``+(XC`!B.8@`DK&(``!``_^2N
+M-0`8$`#_N(Q3`"`"@"`A#``-H0(`*"&.@ALX4$``&(XB`!Q2```6CB(`'(X"
+M`#P``A9",$(``11```D"@"`ACB8`&(XG`#".*``D#`!*_@(`*"%40``*CB(`
+M'`*`("$,`"EL/`6``(X"`#`"@"`A/`6```P`*3RN(@`PCB(`'"1"``$20``*
+MKB(`'(XB`"2.1`!0``(0P`!2$"&40P`P)(0``:Y$`%`D8P`!I$,`,!``_Y>.
+M%0`L)[W_\*^R``BOOP`,K[$`!*^P``",H@!4%$``!P"`D"$DL0!<CB(`'!1`
+M``F.)@`4KB``&*X@`!2/OP`,C[(`"(^Q``2/L````^``"">]`!",P@`LC,4`
+M6`)`("&,4``@K$```*Q``"`,``]+```X(8XB`!P"`#`A)$+__Q1`__2N(@`<
+M$`#_ZZX@`!@GO?_@K[0`$*^S``ROL0`$K[\`%*^R``BOL```C-``,(RB`"2,
+MPP!`CA(`4``"$,``H(@A`D(H(20"``(`@*`A$&(`.22S`#".(P`4C&(`/`1"
+M`"V,8@`@C@(`/#P#`@".!P`L`$,0):X"`#R.(@`<EF4``)9D``(D0O__KB(`
+M'(Y"`%".0P!4C.8`("2E__\D0O__)(3__R1C__^.,``4IF0``J9E``"N0@!0
+MKD,`5!3```*N)@`4KB``&*S@```"`#`AK.``(`(@*"$``#@A#``/2P*`("$"
+M@"`A#``/W@)`*"$``!`AC[\`%(^T`!"/LP`,C[(`"(^Q``2/L````^``"">]
+M`""L8````&`H(:XB`!0"@"`A#``*8ZQ@`"`0`/_.C@(`/)2C`#".0@!4)`8`
+M`:YF``0`0Q`AKD(`5*9C``*.`P`\/`6'_S2E__\``Q;",$(`#R1"``$P0@`/
+M``(6P`!E&"0`8A@EK@,`/(R"&V@``Q[",&,`#P!#$"M40``%CD(`5(R"&K14
+M0``1C((%@(Y"`%268P`")`7__R1"__^N0@!4CH(%A"1C__^F8P`")$(``0P`
+M5NZN@@6$`H`@(0P`#]X"0"@A$`#_QB0"__\D0@`!K((%@(X"`#P``A;",$(`
+M#Q!&``8`````C@,`*)!B``$T0@`($`#_DJ!B``$,``[T`@`H(1``__BF(@`T
+M)[W_\*^Q``2OL```K[\`"`"@@"&.`P!`C*4`,"0"``$`@(@A$&(`3HRD`%".
+M(AMH$$``1@``$"%0@`!%C[\`"(RB`"B00@`!,$(`!!!``"6.`P`@$&``/0``
+M$"&,<``LC@(`,(Q"`"B00@`6,$(`#Q!``#LF!``(,((`#R0#__`D0@`G`(,@
+M)`!#*"0`A1`A`$00*Q1```<`@!@AO)$``"2$`!``91`A`$00*Q!`__L`````
+M`B`@(0(`*"$,`$Z')`8``1!``"$D`O__C@(`,(Q"`"B00@`!,$(`!!1`_]V.
+M`P`@$&``&0``$"&,<``L)`3_\"8#``@P8@`/)$(`)P!D&"0`1"@D`&40(0!#
+M$"L40``'`&`@(;QQ```D8P`0`(40(0!#$"L00/_[``````(@("$"`"@A#`!.
+MAR0&``$D`___``(8"P!@$"&/OP`(C[$`!(^P```#X``()[T`$!``__H``!`A
+M)[W_T*^Q`"0`H(@AK[(`**^P`""OOP`L#`!PM0"`D"&.,``4$@``&0````".
+M0@2PCD,$M(X$`%0D8P`!+&4``0!%$"&N0@2PKD,$M!2``!<D`@`!C@(`+(Q0
+M`"`6`/_S`````(XP`!02```(`D`@(:X@`#"N(``4KB``&*X@`!RF(``T#``*
+ME0(`*"$,`'"Z`````(^_`"R/L@`HC[$`)(^P`"`#X``()[T`,*^B``2OH``(
+MKZ``#*^@`!`,`'"ZKZ``%(X%`%0"0"`A#``*I`.@,"$,`'"U`````!``_]VN
+M``!4)[W_\"2"`%2OL@`(K[$`!*^P``"OOP`,`("`(0"@B"$`0"`A$.``(0#`
+MD"$,`'&.``````P`<+4`````C@0`3(R%```0H``7C((`!*RB``2.`@!(C(,`
+M!*R1``BL@@`$C@(`2*QE``"LD@`,K(```*X$`$@,`'"ZK$0```P`<:,F!``\
+M```@(8^_``R/L@`(C[$`!(^P````@!`A`^``"">]`!`0`/_JK@(`4`P`<9P`
+M````)`,``11#__,D!``"$`#_W``````GO?_PK[````"`@"$DA``\K[\`#*^Q
+M``2OL@`(#`!QC@"@B"$,`'"U`````(X%`$2,HP``$&``'(RB``2L8@`$C*(`
+M!"8$`%2,L@`($B```ZQ#``",H@`,KB(``(X"`%`D`___K*```*RB``2.`@!0
+MK*,`"*X%`%"L10``#`!QHZR@``P,`'"Z``````)`$"&/OP`,C[(`"(^Q``2/
+ML````^``"">]`!`0`/_EK@(`2">]_^"OOP`4K[``$(R0``2OH```)@0`/`P`
+M<:H#H"@ACZ(```(`("$40``()Z4`!(^B``!40/_X)@0`/(^_`!2/L``0`^``
+M"">]`"`,`!%+`````!``__>/H@``)[W_\"2E__^OL0`$K[\`#*^R``BOL```
+M+*(`&0"`B"$00``4C)(`!#P#@`(`!1"`)&,(X`!#$"&,0@```$``"`````",
+MA```#``;"B0%`!$`0(`A)D8$@`!`("$D!0$X#``;/@````".)```#``:O0(`
+M*"&/OP`,C[(`"(^Q``2/L````^``"">]`!```#`A`D`@(0P`+H$D!0`"CB0`
+M``P`&PHD!0`2`$"`(29&!(@`0"`A$`#_ZB0%``B,A```#``;"B0%`!..)```
+M`$"`(20"`@`0`/_EK@(```P`$ZB,A``$$`#_Y(^_``P,`!/`C(0`!!``_^"/
+MOP`,)[W_T*^S`!ROOP`@K[(`&*^Q`!2OL``0C)(`!`"`F"$#H"@A`F`@(0P`
+M$4LF41?L`B`@(0P`<<(`0(`A`D`@(0P`%&TD!0`!CZ8```(`*"$,`!&.`F`@
+M(0)`("$,`!1M```H(0P`<=`"("`A$`#_[0.@*"$GO?_@K[0`$*^S``ROOP`<
+MK[8`&*^U`!2OL@`(K[$`!*^P``",D0`$``"@(0``F"&.(A?@5$``"HXB%]PF
+M,!K(#`!QP@(`("$,`''0`@`@(8XB%^!00/_Z)C`:R(XB%]P00``X`````"8V
+M&N`,`''"`L`@(8XC'*0D`@`#$&(`-0`````D!0`!`B`@(0P`%&TF,AK()C4:
+MA`P`<<D"0"`A)!```0*@("$,`'')`@*@"Q*``!H"`I@+$F``$P```````(`A
+M#``(VP(@("&.(@`$)A```0P`'0.,1```#``.V0(@("&.(@`$#``=`XQ$```\
+M`H`"D$)$:!!```,J`P!D%&#_\``````2@``$``````P`<=`"0"`A``"@(5)@
+M``4"("`A#`!QT`*@("$``)@A`B`@(0P`%&T``"@A#`!QT`+`("$,`'#U````
+M`!``_[@`````#`!I/S0$PU`0`/_*)`4``2>]__"OOP``C*(`!`"@("$``#`A
+MC$(7Q"0%`!@40``$```X(8^_```#X``()[T`$`P`$1D`````$`#_^X^_```G
+MO?_P`*`@(0``,"$D!0`9K[\```P`$1D``#@AC[\```/@``@GO0`0)[W_\*^_
+M``BOL0`$K[```(R0``0,`'%9`("((0!`("$,`'%7)@4;7(X$&UP\!8``)@<8
+M""8(&!`"(#`A#`!Q822E20P,`'%<`````"1C`,@L90#(C@08"`!%$"$D"```
+M)`D`R`!`,"$,`'%T`&`X(8X$&UP\!8``)@<80"8(&$@"(#`A#`!Q822E24P,
+M`'%<`````"1C`,B.!!A`+&4`R`!%$"$`0#`A)`@``"0)`,@,`'%T`&`X(8^_
+M``B/L0`$C[````/@``@GO0`0)[W_X"2"`$0D@P!,K[$`!`"`B"&OOP`8K[4`
+M%*^S``ROL@`(K[0`$*^P```DA``\KB(`2*XC`%```"@AKB``1*X@`$PF,P!4
+M#`!QAXXT``0"8"`A#`!QAP``*"$``*@A``"0(0P`>E<D!``0)E(``0!`@"$`
+M0"`A```H(1!``#XD!@`0#`!PJ@````".(@!0)`/__ZXP`%"N````K@(`!*Q0
+M```"8"`AK@,`"`P`<:.N```,*D(``A1`_^H`````#`!QM":$%^P\$(`"/`6`
+M`#P'@`(\"(`"/`N``B8*;+@DI4>H).<)1"4(9+@E:VS``B`P(20)"``,`'#+
+M)`0``8X$;+@,`'#D/!"``CP%@``\!X`"/`B``CP+@`(F"GE@)*5'-"3G"50E
+M"&U@)6MY:"0$``$"(#`A#`!PRR0)#``,`'#DC@1Y8`P`$ET"("`A`J`0(8^_
+M`!B/M0`4C[0`$(^S``R/L@`(C[$`!(^P```#X``()[T`(!``__4D%0`")[W_
+MT*^P`!``H(`AK[\`(*^S`!ROL@`8`("8(:^Q`!0,`'%"KZ``!`P`*.@"`"`A
+M%$``$B02``$,`'%0`F`@(20"``$20@`)``````)`$"&/OP`@C[,`'(^R`!B/
+ML0`4C[``$`/@``@GO0`P#`!Q20)@("$0`/_V`D`0(0(`("$#H"@A#``H[2>F
+M``0`0(@A,$(``11``%`"`"`A,B(`(%1``$:.`@2$,B((`!1``#X"`"`A,B(`
+M0!1``#<"`"`A/`(`!`(B$"040``O`@`@(3(B`(`00``2`````(X"&\Q00``B
+MC@48G(^B``0P0@`(5$``&HX%&42/H@`$,$(`!%1``!*.!1D,CZ(`!#!"``)4
+M0``*C@48U"0"_W\"(H@D$B#_Q:^@``".`AV$)!(``P!1$"40`/_`K@(=A`P`
+M2IP"`"`A$`#_]20"_W\,`$J<`@`@(1``_^V/H@`$#`!*G`(`("$0`/_ECZ(`
+M!(^B``0`LA@$`$,0)!!`_^D`````$`#_[@`````,`"EL/`4`!!``_]`R(@"`
+M#``I;"0%`$`0`/_(/`(`!`P`2@HD!0`!)`+W_Q``_[\"(H@D`@`@(20%`"`D
+M0@`!#``I;*X"!(00`/^V,B((``P`*6PD!0`!$`#_KS(B`"`GO?_@K[0`$*^R
+M``BOL```K[\`%*^S``ROL0`$C,,<I"0"``(`@*`A)`4``0#`@"$`P"`A$&(`
+M%@``D"$,`!1M/!,`!`(`("$,`"EL/`6``(X1'82N`!V$`@`@(0P`*3P\!8``
+M$B``!`(S$"0"`"`A%$``$S(R`"`"0"@A#``I/`(`("$"`"`A#``4;0``*"$,
+M`'%0`H`@(0P`<4D"@"`AC[\`%(^T`!"/LP`,C[(`"(^Q``2/L````^``"">]
+M`"`,`#>``````%1```..!``$$`#_Z0)3D"4``#`A```X(0P`$1DD!0`#4$#_
+MY`)`*"$0`/_A`E.0)2>]__"OL```K[\`!(R"%^`00``/`("`(8R"%]P00``,
+M)`(``HR#'*008@`))`(``1!B``B/OP`$#`!PM0`````,`##@`@`@(0P`<+H`
+M````C[\`!(^P```#X``()[T`$">]_]"OL0`DK[\`+*^R`"BOL``@C((7X!!`
+M`"8`@(@AC((7W!!``",D`@`"C(,<I!!B`"`D$@`!$'(`'X^_`"P,`'"U````
+M`(XB&QQ40``%CC`;0(XB&R140``@CB(;S(XP&T`GH@`0`B`@(0``*"$``#`A
+M```X(0``0"$``$@A``!0(20+``&OH@``#`!7EZ^R``0\`@`$`@*`)!8```L"
+M("`A#``-6@(@("$,`'"ZKB`;)(^_`"R/L@`HC[$`)(^P`"`#X``()[T`,`P`
+M*3P\!0`$$`#_\P````!00``4CB(8E```*"$`!1#``$40(P`"$,``41`AC$,8
+ME"2B``$P10#_$&```RRD``0D`@`!KB(;'!"`_^4`````CB(;'!1`_^(`!1#`
+M$`#_\0!%$",00/_>)`(``1``_]RN(AL<)[W_X"0#``&OLP`,K[(`"*^Q``2O
+ML```K[\`$`"@D"$`P)@A`("`(1"C`$8``(@A4*``,(R"'*0D`@`"$*(`$R0"
+M``,0H@`,`````%(@``&N$ARD5F```:YQ``"/OP`0C[,`#(^R``B/L0`$C[``
+M``/@``@GO0`@#``4K"0%``(D$0`6$`#_\0`"B`H,`"EL/`6```(`("$,`!2L
+M```H(201`!8"`"`A#``(E0`"B`HD!0`!#``*2`(`("&.`@`$#``="HQ$```,
+M`%&4`@`@(0(`("&N`!?@#``I/#P%@``"`"`A#``4K"0%``$0`/_7`````!!#
+M``D``````@`@(0``*"$,`!2L`````%1`_\\D$0`6$`#_S0````",@A?@4$#_
+M]P(`("&,@@`$#``=&(Q$```,`''0)@0:X!``__`"`"`AC((7X%1```2,@@`$
+M`@`@(1``_^LD!0`!#``="HQ$```,`''")@0:X!``__D"`"`A)[W_\*^P```P
+ML`#_K[\`#*^R``BOL0`$``"0(0P`<+4`@(@A#`!Q-``````2```:CB(<K!1`
+M``@D0@`!CB(<J`(@("$``"@A%$``#R0&``&.(ARL)$(``:XB'*P,`'$[````
+M``P`<+H``````D`0(8^_``R/L@`(C[$`!(^P```#X``()[T`$`P`-.4D$@`6
+M$`#_[P`"D`HD0O__%$#_[ZXB'*R.(ARH`B`@(20&``$00/_J`$`H(0P`-.4D
+M$@`6$`#_Y@`"D`HGO?_PK[````"`@"&OOP`$#`!QM"2$&N"N`!RDK@`<K*X`
+M'*B/OP`$C[````/@``@GO0`0)[W_\*^R``BOL0`$K[```*^_``P`@(`A#`!P
+MM0"@B"$D`@`!$B(`(P``D"%2(``7C@(<K"0"``)2(@`*C@(<K`P`<+H`````
+M`D`0(8^_``R/L@`(C[$`!(^P```#X``()[T`$!!```,"`"`A$`#_]*X1'*@D
+M!0`"#``TY20&``$D$@`6$`#_^0`"D`H00``#`@`@(1``_^JN`!RH```H(0P`
+M-.4D!@`!)!(`%A``__D``I`*C@(<K%1`_^&N$1RH`@`@(1``_^PD!0`!)[W_
+M\*^R``BOL0`$K[```*^_``P`H(@A`("`(1"@`#<``)`A)`(``1"B``DD!0`$
+M)!+__P)`$"&/OP`,C[(`"(^Q``2/L````^``"">]`!`,`#3E)`8``8X#%]PD
+M$@`6$&``!``"D`J.`A?@$%$`&3P%@``60/_O)`(``8X#'*A08@`/C@(<K"0"
+M``(48O_J`D`0(8X"'*P40/_G`D`0(0(`("$D!0`"#``TY20&``%40/_@)!(`
+M%A``_]\"0!`A%$#_\B0"``("`"`A$`#_]B0%``$,`"D\`@`@(0P`-;@"`"`A
+M#``UZ`(`("$,`'"Z`````!``_]\`````C((<J!1``!X`````C@(7W!!```4F
+M!1MPC@,7X"0"``$08@`,``````(`("$,`#>%)`8``1!```0`0)`A)!(`%A9`
+M_[X"0!`A`@`@(1``_]<D!0`##`!PM0`````"`"`A#``I;#P%@``,`#7]`@`@
+M(0P`-;\"`"`A$`#_\0`````,`#3E)`8``202`!80`/_?``*0"B>]__`D@@`4
+M)(,`'*^R``BOL```K[\`#*^Q``0`@(`AK((`&*R#`""L@``4K(``'```D"$,
+M`'I7)`0`#`!`B"$2(``1)`(``@(@("$``"@A#`!PJB0&``R.`@`@)E(``2I#
+M``*N$0`@KB```*XB``048/_PK%$``"8$``P,`'&'```H(0``$"&/OP`,C[(`
+M"(^Q``2/L````^``"">]`!`GO?_PK[\``(R&`!R,R```$0``$0"`."&,P@`$
+MK0(`!(SB`!B,PP`$).0`#*S"``2,X@`8K&@``*S%``BLP```K.8`&`P`<:.L
+M1@``C[\```/@``@GO0`0C,(`!!``__"L@@`@)[W_\*^Q``0`@(@A)(0`#*^_
+M``@,`'&.K[````P`<+4`````CB0`%(R%```0H``3C((`!*RB``2.(@`@C(,`
+M!(R0``BL@@`$CB(`(*QE``"N)``@K(```*R```@,`'"ZK$0```(`$"&/OP`(
+MC[$`!(^P```#X``()[T`$!``_^ZN(@`8,*8`#R>]_?`D`O_P),,`3Z^T`@``
+M8A@D`*"@(0"B*"0`HQ`AK[$!]*^_`@ROM@((K[4"!*^S`?ROL@'XK[`!\`!%
+M$"L`@(@AC),`!(R6``"OH`"TKZ``H!1```<`H"`AO+$``"2E`!``@Q`A`$40
+M*Q!`__L`````CH,``"QB`$%40``1CH(`#`!F$"$D0@`/)`/_\`!#*"0"@Q@D
+M`&40(0!#$"L40``'`&`@(;QQ```D8P`0`(40(0!#$"L00/_[`````(Z"``P4
+M0`.[CI``!```J"$F!?__+*(`0Q!``!L\`X`"``40@"1C"80`0Q`AC$(```!`
+M``@`````)Z4`H">F`*0,`!MH`H`@(0*`("$GI0"@#``;:">F`*@"@"`A)Z4`
+MH`P`&V@GI@"L`H`@(2>F`+`,`!MH)Z4`H`(@("$"8"@A#``'PR>F`+02H``'
+M`L`@(8Z"``@"H"@AKJ(`"(^B`+0,`!J]KJ(`#(^_`@R/M@((C[4"!(^T`@"/
+MLP'\C[(!^(^Q`?2/L`'P`^``"">]`A`"@"`A)Z4`H`P`&V@GI@"XCZ8`N`(@
+M("$"8"@A#``&\">G`+00`/_D``````)@("$,`%>0)Z4`M!``_]\``````H`@
+M(2>E`*`,`!MH)Z8`O(^E`+P"8"`A)Z8`P```."$,`%!<)Z@`Q(^E`,`,`!LW
+M`J`@(8^B`,`00``1CZ4`O`*@("$GI0#(#``;5R>F`,R/I0"\CZ<`R`)@("$G
+MI@#0#`!07">H`,2/I0#0`J`@(0P`&V$`````$`#_OP`````"8"`A)Z8`P">G
+M`-0,`%!<)Z@`Q(^E`-0"H"`A#``;-P`````0`/^T`````">E`*`GI@#8#``;
+M:`*`("$"@"`A)Z4`H`P`&V@GI@#<CZ(`W!!```P"@"`A)Z4`H`P`&W(GI@#@
+MCZ4`V(^F`-R/IP#@`F`@(0P`5P0GJ`#D$`#_G@`````GI0"@#``;:">F`.B/
+MI0#8CZ<`Z`)@("$0`/_U```P(0*`("$GI0"@#``;:">F`.R/I0#L`F`@(2>F
+M`/```#@A#`!8(2>H`/2/I0#P#``;-P*@("&/H@#P4$``#H^E`.P"H"`A)Z4`
+M^`P`&U<GI@#\CZ4`[(^G`/@"8"`A)Z8`\`P`6"$GJ`#TCZ4`\!``_[<"H"`A
+M`F`@(2>F`/`GIP$`#`!8(2>H`/2/I0$`$`#_N@*@("$"("`A)`4``0``,"$,
+M`!$9```X(1``_VH``````F`@(0P`!O,GI0"T/`2``@P`<'0DA`ET$J#_:X^_
+M`@P,`"YR`F`@(0*@("$0`/^F`$`H(0)@("$,``<V)Z4`M!``_U@``````F`@
+M(0P`!U<GI0"T$`#_4P`````"8"`A#``'E2>E`+00`/].`````">E`*`GI@$$
+M#``;:`*`("$"@"`A)Z4`H`P`&V@GI@$(`H`@(2>E`*`,`!MH)Z8!#`*`("$G
+MI0"@#``;<B>F`1"/I0$$CZ8!"(^G`1`,`%#?`F`@(1``_S<`````)Z4`H">F
+M`10,`!MH`H`@(0*`("$GI0"@#``;:">F`1@"@"`A)Z4`H`P`&V@GI@$<CZ4!
+M%(^F`1B/J`$<`F`@(0P`42<``#@A$`#_(P`````GI0"@`H`@(0P`&V@GI@$@
+MCZ4!("0"__\0H@`%``````P`46P"8"`A$`#_%P`````,`%&4`F`@(1``_Q,`
+M````#`!QHR9D%\@,`'&.)F07T!``_PT``````H`@(2>E`*`,`!MH)Z8!/(^E
+M`3P"8"`A#``4$">F`+00`/\#``````*`("$GI0"@#``;:">F`42/I0%$#``'
+MW0)@("$0`/[Z`````">E`*`GI@%(#``;:`*`("$"@"`A)Z4`H`P`&V@GI@%,
+MCZ4!2(^F`4P,``@6`F`@(1``_NP`````)Z4`H">F`5`,`!MH`H`@(0*`("$G
+MI0"@#``;:">F`50"@"`A)Z4`H`P`&V@GI@%8`H`@(2>E`*`,`!MH)Z8!7(^E
+M`5"/I@%4CZ<!6(^H`5P,``@R`F`@(1``_M0`````)Z4`H">F`6`,`!MH`H`@
+M(0*`("$GI0"@#``;:">F`60GL``0`H`@(2>E`*`,`!MR)Z8!:`(`("$``"@A
+M#`!PJB0&`!R/HP%DCZ(!:)>E`6("8"`A`@`P(:^B`!2OHP`D#`!/@*^C`!P0
+M`/ZX``````*`("$GI0"@#``;:">F`6R/I@%L`F`@(0P`3\@``"@A$`#^K@``
+M```,`!VR`F`@(1``_JH``````H`@(2>E`*`,`!MH)Z8!<(^E`7`,`"F>`F`@
+M(1``_J$`````)Z0`,```*"$,`'"J)`8`!@*`("$GI0"@#``;:">F`70"@"`A
+M)Z4`H`P`&V@GI@%X`H`@(2>E`*`,`!MH)Z8!?`*`("$GI0"@#``;<B>F`8"/
+MI@&`EZ4`-```@"&4QP`4E,0`!"3#``@XX@`'$*0`,P!B@`L`P(@ACZ,!="QB
+M``040``8``,0P`!#$"$``A#``$,0(P`"$(`"8A`A)$/[F"0"``<0X@`B)`(`
+M!*1B`18D`@`#$.(`&B0"`!`0X@`8)`(`$5#B`!>48@$6).+__BQ"``(40``"
+M)`0`""0$``2D9`$8#``(E0)@("&.8AM4`F`@(0(@,"$D0@`!KF(;5(^E`72/
+MJ`%X#``IO`(`."$,``A^`F`@(1``_ED`````E&(!%B1"``00`/_HI&(!%J1@
+M`1@0`/_KI&`!%I>C`#*4P@`"%&+_S`#`B"&7HP`PE,(``!1B_\<``(@A$`#_
+MQX^C`70GI0"@)Z8!A`P`&V@"@"`A`H`@(2>E`*`,`!MH)Z8!B`*`("$GI0"@
+M#``;:">F`8R/IP&,CZ4!A(^F`8@,`"P_`F`@(8^C`80``A0```(D`RQB``04
+M0/XQ``,0P`!#$"$``A#``$,0(P`"$(`"8A`A$`#^*J1$_*P"@"`A)Z4`H`P`
+M&V@GI@%`CZ4!0`P`5W\"8"`A$`#^(0`````GI0"@)Z8!D`P`&V@"@"`A`H`@
+M(2>E`*`,`!MH)Z8!E`*`("$GI0"@#``;:">F`9@"@"`A)Z4`H`P`&V@GI@&<
+M)Z4`H">F`9P"@"`A#``;:(^P`9P"@"`A)Z4`H`P`&W(GI@&@CZ8!E(^G`9B/
+MJ`&@CZD!D`)@("$"`%`A#`!0$@``*"$0`/W^`````">E`*`GI@&D#``;:`*`
+M("$"@"`A)Z4`H`P`&W(GI@&HCZ4!J`P`+2("8"`A$`#]\0`````GL``X`F`@
+M(0P`+>T"`"@A`J`@(0(`,"$D!0`(#``;/@`````0`/WF``````P`+@0"8"`A
+M$`#]X@`````D`@`"KF(;L`P`+A("8"`A$`#]W``````GI0"@)Z8!K`P`&V@"
+M@"`A`H`@(2>E`*`,`!MH)Z8!L">E`*`GI@&P`H`@(0P`&VB/L`&P`H`@(2>E
+M`*`,`!MR)Z8!M(^C`:PD`O__$&(`)0`#$,``0Q`A``(0P`!#$"./I0&T``(0
+M@`)B$"$D1@`0`*88)3!C``,D1``($&``'R2G`"``X!@AB*(``)BB``.(IP`$
+MF*<`!XBH``B8J``+B*D`#)BI``^HP@``N,(``ZC'``2XQP`'J,@`"+C(``NH
+MR0`,N,D`#R2E`!`4H__N),8`$("B``"@P@``#`!2Q0(`*"$6`/VACZ4!M`P`
+M+BX"8"`A$`#]G0````",H@``C*,`!(RH``B,J0`,K,(``*S#``2LR``(K,D`
+M#"2E`!`4I__V),8`$!``_^H``````H`@(2>E`*`,`!MH)Z8!N(Z"``P00``?
+MCZ,!N"0"``408@`()`(`!!1B_80"8"`A```P(0P`+H$D!0`$$`#^(0*@("$"
+M8"`A)Z8`0`P`+H$D!0`%CZ(`0">F`%@"H"`AKZ(`6(^B`$0D!0`4KZ(`7(^B
+M`$BOH@!@CZ(`3*^B`&27H@!2IZ(`:)>B`%80`/^"IZ(`:H^E`;@"8"`A#``N
+M@0``,"$0`/UE``````P`+PX"8"`A$`#^`P*@("$GI0"@)Z8!O`P`&V@"@"`A
+M`H`@(2>E`*`,`!MH)Z8!P(^E`;R/I@'`#``RH0)@("$0`/U3``````P`,XT"
+M8"`A$`#]3P`````"@"`A)Z4`H`P`&V@GI@'DCZ4!Y`P`,\$"8"`A$`#]1@``
+M```"@"`A)Z4`H`P`&V@GI@'HCF8<_(^E`>@,`#,5`F`@(1``_3RN8AS\)Z4`
+MH">F`30,`!MH`H`@(0*`("$GI0"@#``;:">F`3B/I0$TCZ8!.`P`-@4"8"`A
+M$`#]+@`````GI0"@)Z8!+`P`&V@"@"`A`H`@(2>E`*`,`!MH)Z8!,(^E`2R/
+MI@$P#``V#@)@("$0`/T@``````P`2'\"8"`AKF(%L"9E&W`,`#KG`F`@(1``
+M_1@`````)[`!Q`*`("$GI0"@#``;:`(`,"$"@"`A)Z4`H`(`,"$,`!MHE[(!
+MQB>E`*`GI@'(`H`@(0P`&VB7L0'&`H`@(2>E`*`,`!MH)Z8!S`*`("$GI0"@
+M#``;:">F`=`"@"`A)Z4`H`P`&V@"`#`A`@`P(0*`("$GI0"@#``;:(^P`<2/
+MIP'(CZ@!S(^I`="/JP'$)Z(`M`)@("$"0"@A`B`P(0(`4"&OH@``#`!7EZ^@
+M``00`/SJ``````P`.F<"8"`A$`#\Y@`````"@"`A)Z4`H`P`&V@GI@'4CZ4!
+MU`P`0=<"8"`A$`#\W0`````"@"`A)Z4`H`P`&V@GI@'8CZ(!V`*`("$GI0"@
+M``*`P`("@",,`!MH)Z8!W``0B("/H@'<`G&((28Q&<BOH@!\`H`@(2>E`*"N
+M(@`$#``;:">F`=R/H@'<`H`@(2>E`*"OH@"`)Z8!W`P`&VBN(@`(CZ(!W`*`
+M("$GI0"@KZ(`A">F`=P,`!MHKB(`#(^B`=P`$(#``H`@(:^B`)`GI0"@KB(`
+M$">F`=P,`!MH`G"`(280&'B/H@'<C@4`)`)@("&N(@`8)Z8`<`P`2C.OH@!T
+M$`#\IP`````"@"`A)Z4`H`P`&V@GI@$D)Z4`H">F`20"@"`A#``;:(^P`20"
+M@"`A)Z4`H`P`&W(GI@$HCZ8!*``0$,``4!`CC,,````"$(``4Q`AK$,9R(S#
+M``0D0AG()`<``:Q#``2,PP`(`@<X!`)@("&L0P`(C,,`#`(`*"&L0P`,C,,`
+M$*Q#`!",PP`4K$,`%(S#`!BL0P`8CF(:<(^F`2@`1Q`E#``-%JYB&G`0`/Q[
+M``````*`("$GI0"@#``;:">F`>"/HP'@`F`@(0`#$,``0Q`C``(0P`!3$"$,
+M`$J'C$48G!``_&T``````H`@(2>E`*`,`!MH)Z8![)>B`>X0`/QFIF(;\`+`
+M("$,`!L*`@`H(11`_$,`0*@A)`(``A``_&:OH@"T)[W_\*^Q``2OOP`(`("(
+M(0P`</ZOL````$`@(0P`<0$``"@A#``5A`(@("$`0(`A`B`@(0P`%:4`0"@A
+M#``;'@(`("$0`/_W`````">]__"OL@`(K[$`!*^P``"OOP`,`("`(0P`%4,\
+M$H`#/`6``#P'@`(\"(`"/`N``P!`B"$F2HH()*5F;`(`,"$DYPJ0)0AZ""5K
+MBA`D!``!$$``""0)$``"(!`AC[\`#(^R``B/L0`$C[````/@``@GO0`0#`!P
+MRP`````,`'#DCD2*"!``__4"(!`A)`(``:R"`#`\`H`"K$5$1#P"@`*L140\
+M`^``"```$"$GO?_0K[0`(*^R`!BOL0`4K[``$*^_`"0\$(`"K[,`'(X%,N`\
+M!H``/`>```"`B"$DQFI0```@(23G:K@GJ@`!)Z@`!`P`66L#H$@A/`*``HQ4
+M1$`\`H`"C$9$.(X#,N"3H@``CZ4`!*QQ``"@8@`(K&4`!).B``&.!#+@``"0
+M(:""``F.`C+@K$8`$*Q4``P\$X`"CG`RX`P`>E<"@"`ACF,RX``2B(`",(`A
+MK@(`%`(CB"&.)``4#``;'B92``$J0@`"5$#_\SP3@`*.9#+@#``:VB02``\\
+M$(`"C@(RX"92__\,`'I7C$0`#(X$,N`,`!KW`$`H(09#__@\$(`"C@(RX(^_
+M`"2/M``@C[,`'(^R`!B/L0`4C[``$`/@``@GO0`P)[W_T#"B``\D0@!-)`/_
+M\*^R`!@`H)`A`$,H)`)#&"0`91`A`$,0*Z^Q`!2OOP`@`("((:^S`!ROL``0
+M%$``!P!@("&\<0``)&,`$`"%$"$`0Q`K$$#_^P````"20@`!DD0``)(C`$`0
+M@P`2,%,``8XC`#B.(@`\K&```(XD`#0D0@`!$(``!:XB`#P,`!LIC)```!8`
+M__T"`"`AKB``1*X@`#2B(`!`DD(``!1``$(`````DD(``%1```6.(@!$KC(`
+M-*XR`#BN(`!$CB(`1)9#``*.)``X`$,0(:R2``"N(@!$%F``"ZXR`#B2(@!`
+M)$(``:(B`$"/OP`@C[,`'(^R`!B/L0`4C[``$`/@``@GO0`PKD```(XC``".
+M(@`THB``0"1D`"0D4@`$KB``-`.@*"&,<``$#`!QJJ^@``".0P`()`(`$!!B
+M``J/H@``5$``"8XB``".`A?8)`,``5!#``6.(@``C@(:M!!#``<F$!J8CB(`
+M``)`*"$,``KTC$0`!!``_]V/OP`@#`!QR0(`("%00/_XCB(``(XD```,``N>
+M`D`H(0P`<=`"`"`A$`#_TH^_`"`,`!LI`D`@(1``_\Z/OP`@)[W_\*^_```0
+MX``-,*(`_Q!```8`P"@A#``:(@````"/OP```^``"">]`!",A```#``5:P#`
+M*"$0`/_ZC[\``!!```4`P"`A#``;*0`````0`/_TC[\```P`&QX`P"`A$`#_
+M\(^_```GO?_P,*(`_Z^_```40``&`,`H(0P`&Q8`````C[\```/@``@GO0`0
+MC((```P`":*,1``$$`#_^H^_```GO?_PK[\``(RG``",A``$`*`P(0P`6@0`
+M`"@AC[\```/@``@GO0`0)[W_\*^_```\`H`"C$(RY`"@,"$00``+D(4`,Y3"
+M``(D1P`$#`!:!(R$``2/OP``)`0``3P#@`*L9#+D`^``"">]`!`0`/_W)`<'
+M("2"`!PGO?_PK(``)```*"&OOP``#`!QAP!`("&/OP```^``"">]`!`GO?_P
+MK[````"`@"$DA``<K[\`"`P`<8ZOL0`$#`!PM0````".$0`DCB(```P`<+JN
+M`@`D`B`0(8^_``B/L0`$C[````/@``@GO0`0)[W_\*^P````@(`AK[\`"*^Q
+M``0,`'"U`*"((8X"`"2N(@``KA$`)`P`<+HF$``<#`!QHP(`("&/OP`(C[$`
+M!(^P```#X``()[T`$">]__"OOP`$K[````P`&N0`H(`AK%``!(^_``2/L```
+M)`,`(*Q#```#X``()[T`$">]__"LH``$K[\```P`&O<`````C[\```/@``@G
+MO0`0)[W_\*^_```\`H`"C$(RX`"`,"$``"@A#`!9[HQ$``2/OP```^``"">]
+M`!`GO?_PK[\``#P"@`*,0S+@`(`P(3P"B(B090`SC&0`!#1"B(@,`%GNK,(`
+M`(^_```#X``()[T`$(R"````@A`AK$4``(R"```D0@`$`^``"*R"```GO?_P
+MK[\`"*^Q``2OL````("((8R$````H(`A)`+__`(D("$`P"@AK)````(`,"$D
+MA``$)A```PP`<)T"`H`DCB(``(^_``@`4!`A)$(`!*XB``"/L```C[$`!`/@
+M``@GO0`0C(,``"0"`@``0Q`C$*```@"#("&LI```5,```:S"```#X``(````
+M`(R#```DI0`#)`+__`"B*"0`91@A`^``"*R#```0H``"```8(8RC````@Q`A
+MC$(`(!"@``(D8P`$K*,```/@``BLP@``C*(```""("$0P``")(0`(*S$```#
+MX``(`````">]__"OOP`$K[````P`:_(`@(`A)`,``:X#``RN`@``K@``"*X`
+M``2/OP`$C[````/@``@GO0`0)[W_\*^P``"OOP`$C((`.`"`@"$40``F)`0`
+M`0P`:LDD!``!#`!K\@````"N`@`L#`!K]20$``*.`@`````H(0P`%-^,1``$
+M%$``$P````".`@`P%$``#```("&.`@`X%$``!0``("&/OP`$C[````/@``@G
+MO0`0#`!K#P`````0`/_ZC[\`!`P`:P$`````$`#_\XX"`#@,`&OUC@0`+`P`
+M:LD``"`A$`#_[8X"`#@,`&L/`````!``_]@`````)[W_\*^P``"OOP`$C((`
+M.`"`@"$40``=)`0``8X"```D!0`!#``4WXQ$``000``%`````(^_``2/L```
+M`^``"">]`!`,`&OUC@0`+`P`:LD``"`AC@(`,!1```@D!``!C@(`.!!`__,`
+M`"`A#`!K#P`````0`/_PC[\`!`P`:P$`````$`#_]XX"`#@,`&L/`````!``
+M_^*.`@``)[W_\*^P``"OOP`(K[$`!(S#`"@D`@`!$&(`!@#`@"&/OP`(C[$`
+M!(^P```#X``()[T`$*S``#P,`'$T``````P`&[<"`"`AC@(`*!1``"```"`A
+M$(``$`````".`@`\/`,``C1C2?`D0@`!K@(`/(X"`#P`8A@K$&``!P````".
+M`@``C$(`!(Q1%^@D`@`!$B(`"@`````,`!S\C@0`!"0$``4,`'%)``````P`
+M<3L`````$`#_VX^_``@,`!N'`@`@(20$``L0`/_VKA$`*`P`'0..!``$#``=
+M(XX$``0\`P`"`$`@(1!```XT8TGPC@(`/"1"``&N`@`\C@(`/`!B$"M00``(
+MC@(`*(X"``",0@`$C$,7Z"0"``$08O_-`````(X"`"@40/_I`````!``_\@`
+M````)[W_\*^Q``2OL````*"((:^_``@,`'%"`("`(0(`("$"(#`A#``;W@``
+M*"&/OP`(C[$`!(^P```D`@`!`^``"">]`!`GO?_PK[```#P'@```@(`A/`B`
+M`"2)`"0DB@`().=PL"0$``LE"&]X```H(:^_``0,`'$(`@`P(8X"`"040``%
+M`$`@(8^_``2/L````^``"">]`!`,`'$F`````!``__J/OP`$)[W_\"0#``&O
+ML```K[\`"*^Q``00HP`1`("`(5"@`!*,@@`H)`(``A"B``PD`@`#4*(`!HR"
+M`"B/OP`(C[$`!(^P```#X``()[T`$%1`__JL@``H$`#_^8^_``BN```\$`#_
+M]H^_``@40/_TC[\`"(R"``",0@`$C%$7Z%8C__"/L0`$#``<]8R$``0,`'%"
+M)`0`!0P`&X<"`"`AKA$`*`P`<4DD!``+$`#_Y(^_``@GO?_PK[(`"`"`D"$D
+M!`!`K[$`!*^P``"OOP`,#`!Z5P"@B"$`0(`A$@``%```$"$``"@A)`8`0`P`
+M<*H"`"`A)@0`+*X2``"N$0`$#``;>:X``"@,`'%))`0`!`P`'#T"`"`AC@0`
+M!#P%@``DI7%8#``<UP(`,"$"`!`AC[\`#(^R``B/L0`$C[````/@``@GO0`0
+M)[W_\*^_``",@@`H%$``!```*"&/OP```^``"">]`!``0/@)C(0`+!``__N/
+MOP``)[W_\*^_``",@@`H%$``!"0%``&/OP```^``"">]`!``0/@)C(0`+!``
+M__N/OP``)[W_\*^_``",@@`H%$``!"0%``*/OP```^``"">]`!``0/@)C(0`
+M+!``__N/OP``)[W_\*^_``",@@`H%$``!"0%``./OP```^``"">]`!``0/@)
+MC(0`+!``__N/OP``)[W_\*^_``2OL````("`(8R$``2N!@`L/`:``*X%`"@D
+MQG*<#`!9U```*"&.!``$/`:``"3&<LP,`%G4)`4``8X$``0\!H``),9R_`P`
+M6=0D!0`"C@0`!#P&@``DQG,L#`!9U"0%``./OP`$C[````/@``@GO0`0)[W_
+M\*^_```,`%K'C(0`!(^_```#X``()[T`$">]__"OOP``#`!:SXR$``2/OP``
+M`^``"">]`!`GO?_PK[\```P`6N",A``$C[\```/@``@GO0`0)[W_\*^_``2O
+ML````("`(0P`6M>,A``$#``<_`(`("$,`!T#`@`@(8^_``2/L````^``"">]
+M`!`GO?_PK[```*^_``0,`!SU`("`(0P`6MN.!``$C[\`!(^P```#X``()[T`
+M$">]__"OOP``#`!:](R$``2/OP```^``"">]`!",@@`````H(8Q"0"`P1@#_
+M/`.``@`%$(`D8S,P`$,@(92"```DI0`!$,(`!2RC``A48/_X/`.``@/@``@`
+M`!`A`^``")2"``(D`A0`I((`"B0""9ZD@@`()`(`!:2"`!0D`@`$K((`8"0"
+M`!ZD@@`<)`()*J2"`'0D`@`'K((`>"0"``\D`P`")`4``20&``,D!P`\H((`
+M4R0"$`"LA0!(H(8`?*"#`'VL@@!4I(<`6JR```2@@P`6H(``%ZR%`'"LA@``
+MK(``:*R``&RL@`!,K(``9*R``#2D@`!<I(<`6*2``%X#X``(I(``?B>]_^"O
+ML````("`(20$'<BOLP`,K[$`!*^_`!"OL@`(#`!Z5P"@F"$00`!``$"((0!`
+M("$``"@A#`!PJB0&'<@\`J#`-$(`H(Q#```"("`AKC```#!B`/```A$",&,`
+M#ZXC!'P,`!TJKB($>`!`&"$``(`A/`*``B12,N@`$!#``%(0(91"```00P`&
+M)`(`"280``$J`@`)%$#_]SP"@`(D`@`)$@(`%20#``$,`'I7)`0`O*XB&_00
+M0``0)`/__P``*"$D!@"\#`!PJ@!`("$,`!T\)B0;:``0$,``4A`AC$(`!`!`
+M^`D"("`A%$``"P!`@"&N<0```@`8(8^_`!"/LP`,C[(`"(^Q``2/L````&`0
+M(0/@``@GO0`@#`!Z:HXD&_00`/_SKB`;]!``__(D$``")[W_\"0%``X``#`A
+MK[(`"*^Q``2OL```K[\`#`P`'<X`@)`A`$"((11```<``(`AC[\`#(^R``B/
+ML0`$C[````/@``@GO0`0`@`H(0P`*9X"0"`A)@(``3!0__\"$1`K%$#_^@(`
+M*"$0`/_RC[\`#(R"&_0LHP`4`*!`(21'`#008``*```H(3P"@`(`"!B`)$(*
+M[`!B$"&,0@```$``"```````9Q`AC$4```/@``@`H!`A,,(@`%!`__R4Y0!$
+M$`#_^I3E`$`PPB``4$#_]Y3E`$80`/_UE.4`0B0"`!`0P@`/+,(`$1!```8D
+M`@`1)`(``U#"_^V,Y0!($`#_ZR0%``$0PO_I```H(20"`"!4PO_F)`4``1``
+M_^2,Y0!,C((;S!!`_^&,Y0!0C.(`5!``_]X`HB@D)`(`$%#"_]N,Y0!@+,(`
+M$1!```8D`@`1)`(``U3"_]4D!0`!$`#_TXSE`%@0PO_1```H(20"`"!4PO_.
+M)`4``1``_\R,Y0!<)[W_L#P%@`(\!H`"K[``,*^_`$BOM0!$K[0`0*^S`#RO
+ML@`XK[$`-"3""XPDHPN$C*<+A(QH``2,20`,C$,`!(S&"XR,0@`(```H(0"`
+M@"&OIP`(KZ@`#*^F`!BOHP`<KZ(`(`P`.FZOJ0`D%$``:@!`("&.!0``C*)`
+M(#!#`/\P1``/``,9`B0"``6F`QSP$&(`[Z8$'/(T`H```*(0(8Q"&!@``#@A
+MI@(<]``'2(`!/1`AC@,``(Q1``@``"@A`'$8(8QC``"L0P`0C@(````%'```
+M91@E`%$0(:Q#``".`@``)*4``2RD`0``41`AC$(``%1#`$@D!``6%(#_](X"
+M`````#`A)Z4`&(RC```DQ@`!`%$0(:Q#``"."```+,0`!"2E``0!$1`AC$(`
+M`%1B`#DD!``65(#_](X"```!/1`AC$,`"(Q$`!`DX@`!,$?__P$#&"$LX@`"
+MK&0``!1`_],`!TB`#`!I/R0$`&2.`@``-`2``"0#``<`1!`AK$,8`(X"```D
+M`QP6```X(0!$$"&L0QC0C@,``"3B``$P1___-`:```!F&"$\`@`!+.0`"*QB
+M&(!4@/_XC@,``(X"```D!0`(`$80(8Q$'```!"8",((`#P`"$0``!"$"#`!8
+ME`""("4`0"`A,$,`\*8"'/8D`@`P$&(`D#""``\"`"`A)`4`P0P`+*\#H#`A
+M$$``"R0$`!:/OP!(C[4`1(^T`$"/LP`\C[(`.(^Q`#2/L``P`(`0(0/@``@G
+MO0!0)[4``@(`("$D!0`<#``LKP*@,"$40/_P)`0`%I>B``(40`!K)!0$```4
+M$$`D4?Z`#`!Z5P(@("$`0)@A$$#_YB0$``("(#`A`$`@(0P`<*H``"@AC@(;
+M]"01`,`"-!@KK%,`M!!@``P"8)`A`B`H(0(`("$"H#`A#``LKR8Q``$40``*
+M`C08*Y>B``*F0@``%&#_]B92``("@"@A#``@YP)@("$00``*`````!)@``8D
+M!0`6#`!Z:@)@("&.`AOTK$``M"0%`!80`/_#`*`@(0P`>E<D!``X$$#_]`!`
+MB"$`0"`A```H(0P`<*HD!@`XC@(;]`(@("$,`"#UK%$``!1`_^J7H@```@`@
+M(0)@*"$,`"7=IB(``!1`_^0F)@`"`@`@(0P`+*\D!0`_%$#_WP(`("&.!1OT
+M#``@MB2E`*040/_:``````P`'],"`"`A%$#_UB8%&Z(,`"QX`@`@(8H#&Z*:
+M`QNE%&``!20"__^6`ANF$$``$#P"``,D`O__4&(`"98#&Z:6`ANBE@0;I)8#
+M&Z8``"@AI@(;J*8#&ZP0`/_)I@0;JC0"__]48O_WE@(;HCP"``,T0G^_J@(;
+MHKH"&Z4D`JJJ$`#_[Z8"&Z8P0O_P`@`@(20%`!L"H#`A#``LKP`"HP`40/][
+M)`0`%I>B``(0`/^,`H*@)2Q"``440/]U)`0`%A``_VX"`"`A+((``A1`_W`D
+M!``6$`#_#S0"@``GO?_PK[$`!*^P``"OOP`,K[(`"(R"&_0`@(@A)`0`(`P`
+M>E>,4@"T$$``%P!`@"$`0"`A```H(0P`<*HD!@`@CB(;]`(`*"$"("`A#`!(
+M#JQ0``0``(`A`@`H(0(@("$,`"F>)A```2H"`(`40/_[`@`H(8XE&_0,`#\Y
+M`B`@(11```\`````$D```P`````,`'IJ`D`@(0P`'WP"("`A)`0`%H^_``R/
+ML@`(C[$`!(^P````@!`A`^``"">]`!`,`#3``B`@(0P`+V0"("`ACB0;]#P#
+M@`(D8RRT/!"``CP'@``\"(``/`J``ZR#`!BN(A?$)@E'L"3G2\`E"$VH)4JA
+M8```*"$"(#`A#`!Q"```("$,`'$FC@1'L`P`<4D``"`A#`!Z:@)`("&.(QOT
+M)`(`"@``("&L8@!H$`#_V:Q@`+0GO?_@K[0`$*^_`!2OLP`,K[(`"*^Q``2O
+ML```C)(;](Y0```2```=`("@(8X$``04@`!!`````(X$``@4@``Z`````(X$
+M``P4@``S`````(X$`!`4@``L````````F"$"`(@ACB(`%"9S``$40``B`$`@
+M(8XB`!PF,0`,%$``&@!`("$N8@`#5$#_]XXB`!0,`'IJ`@`@(:Y```".1``$
+M%(``#0`````"@"`A#``_1@)`*"&/OP`4C[0`$(^S``R/L@`(C[$`!(^P````
+M`!`A`^``"">]`"`,`'IJ`````!``__&N0``$#`!Z:@`````0`/_E+F(``PP`
+M>FH`````$`#_W8XB`!P,`'IJ``"8(1``_]0"`(@A#`!Z:@`````0`/_,C@0`
+M$`P`>FH`````$`#_Q8X$``P,`'IJ`````!``_[Z.!``(C(,;]"0"``XD8P`T
+M`^``"``#$`LGO?_0K[0`(*^R`!BOL``0K[\`)*^S`!ROL0`4C)$;]"0%`+\#
+MH#`ACB(```"`D"&GH``$#``LKXQ3``0F,``T%$``I@!`H"&.0AOTEZ,``(Q"
+M``"N(P`TE$(``"Q"4`,40`"_)Z8`!`)`("$,`"RO)`4`RA1``)D`0*`AEZ(`
+M!#!"#\```A%"-$(``:X"`#R.0AOTC$(``)1"```L0E`!$$``IR>F``*GH``"
+MEF(`$I9D`!"68P`4``(0@`""("4``QC`EF4```"#("4T@@`"`$4@"I9C``(`
+M@"@A-*(`$)>D````0R@*,*(``11```.N!```)`+__0"B*"0PH@`(%$``!"0"
+M"0@D`O_O`*(H)"0""0BF`@!$ED(<]C!"`/`L0@`P$$``A*X%``260QSP)`(`
+M!Q!B``4D`@G$)`(`"A!B``(D`@G$)`(*K*8"`$:7J``")`(3.*8"`$`D`A?4
+MI@(`0C$#``(D`@`!K@(`8*X``%P48``(```P(99#'/`L8@`&4$``!"0&``$D
+M`@`%4&(`8I9#'/*61!SP)`(``:X&`%@L@P`&K@(`4*X``$P08`!8```P(20"
+M``50@@!2ED(<\C$$__\`!!!".$(``20'``$P0@`!,0,``:X&`%2N`@!(K@<`
+M"*X'``P48``#```P(3"B``D``C`K``00PCA"``$P0@`!,0,`!*X&`!"N`@`4
+M%&```P``,"$PH@`)``(P*Y9B``:.9``8``49`C!C``&N!@`8K@,`(*X'`"2N
+M`@`HK@<`'!"```0``"@ACF(`'%1```,D`@`!)`4``20"``$Q`P'PK@(`,*X%
+M`"P48``"``,1`B0"``HQ`_``$&``'ZX"`#0``QL")`(``0!B$`2N`@`XCD(;
+M](Q#``"7H@`$,$(/P``"(4*48@``+$)0`Q1```.N!``\-((``:X"`#R"8@`@
+M`D`@(:X"`&2"8@`A#``?SJX"`&@"@!`AC[\`)(^T`""/LP`<C[(`&(^Q`!2/
+ML``0`^``"">]`#`0`/_D)`(`@"Q"``@40/^N,03__Q``_ZLD!@`!)`(``A!B
+M``,L8@`$5$#_G)9$'/`0`/^9)`8``1``_X0D`@G$`D`@(0P`+*\D!0#)%$#_
+MXP!`H"&7H@`"$`#_5*9B`#@"0"`A#``LKR0%`,H40/_;`$"@(9>B``0P0@P`
+M``(90JX#`#P0`/]"IZ(`!">]_]"OLP`<K[(`&*^Q`!2OL``0K[\`(`"@D"$`
+M@)@A``"((0"@@"$F)0"P`F`@(0P`+*\#H#`A)C$``0!`."$40``(*B0`!I>B
+M``"3HP`!``(2`J(#``"B`@`!%(#_\B80``*20P`+)`(`_Q!B``H``(@AHD``
+M#(^_`""/LP`<C[(`&(^Q`!2/L``0`.`0(0/@``@GO0`P`E$@(0)1*".`A@``
+MD*(`"B8Q``$J(P`%H((``!1@__B@I@`*$`#_[:)```LD!P#``.40*Q!```<`
+M`#`AE((``"3G``$`Y1@K`,(P)A1@__LDA``".,/__R0"`!8#X``(``,0"B>]
+M__"OL````("`(:^_``0,`'I7)`0!6*X"``000`!")`,``@!`("$``"@A#`!P
+MJB0&`5@,`'I7)`0#9*X"``@00``Y)`,``@!`("$``"@A#`!PJB0&`V2."0`(
+M)`(`"B0(``FE(@`4)`(``Z4B`AXE(P`<)`(`"R4(__^D8@``!0'__21C`#0`
+M`$`A`2`X(3P#@`(\`H`"``@@0"1C%X0D0A=\`((0(0"#&"&41@``E&,``"4(
+M``$`B2`A)`(`"RD%``.D@P(@I(8")J3B`C*DX@+.%*#_[B3G`#0,`'I7)`0`
+MB*X"``P00``/)`,``@!`("$``"@A#`!PJB0&`(@,`'I7)`0(`*X"`!`00``&
+M)`,``@!`("$``"@A#`!PJB0&"````!@AC[\`!(^P````8!`A`^``"">]`!`G
+MO?_@K[\`$*^S``ROL@`(K[$`!*^P``",@AOT`*!((3##__^,4P``EF(``"Q"
+M,`,40``ICG``")8&`!0``)`A`D80*U!``!"6`@`4`@`X(0`#$$``21`AE$@`
+M`"92``(D8P`!,00`_P`($@("1B@KI.(``*3D``(P8___%*#_]"3G``26`@`4
+M$$``#```D"$"`(@AEB4``)9D`````#`A#``HGB92``&6`P`4IB(```)#&"L4
+M8/_W)C$``H^_`!"/LP`,C[(`"(^Q``2/L````^``"">]`"```Q!``$40(91(
+M```D8P`!,&/__P!@*"$`"#%```@8@@`($D(PQ@!_,&,`?P`%($"F!@`$I@(`
+M`*8#``(`B2`AE(@``"2E``$PH___`&`@(0`($L(`""C```@9`@#","4PI0!_
+M,&,`?P`$$$"F!0`(I@8`!*8#``8`21`AE$@``"2$``$P@___`&`@(0`($T(`
+M"#A```@9@@"B*"4PYP!_,&,`?P`$$$"F!P`,I@4`"*8#``H`21`AE$@``"2$
+M``$PA/__``@I@``(,\(`"!("``@80C"E`'\P0@!_`.8X)3!C`'\`!"!`I@4`
+M$J8"``ZF!P`,I@,`$`")("&4B`````@2@@"B*"40`/^HI@4`$B>]_\"OMP`L
+MK[4`)*^Q`!2OOP`TK[X`,*^V`"BOM``@K[,`'*^R`!BOL``0C((;]##&__^O
+MI0``C%0``*^F``0``+@ACH(`!(Z2``@``(@AE%X`$@``J"$D`@`!```@(1*B
+M`'>/HP`$4J``<997`!0D`@`"$J(`9P`````2X`!5``"P(0`#$$"/HP```("8
+M(0!#@"&68@``)B<`"*8B``"6"@``)A```@`*((``"BD"``HR@C"E`#\PA``_
+MIB0`'J8F``2F)0`"E@H``"80``(`"DN"``I!```*$@(`"AB"`(D@)3$(`#\P
+M0@`_,&,`/Z8H`"2F)``>IB(`(*8C`"*6"@``)A```@`*&P(`"A&"`0-`)3!"
+M`#\Q0P`_IB@`)*8B`":F(P`HE@H``"80``(`"D"```H9`@`*$H(Q"``_,&,`
+M/Z8H`"ZF(@`JIB,`+)8*```F$``"``HC@@`*&((`"A("`01`)3!"`#\P8P`_
+MIB@`+J8B`#"F(P`R#``H>I:$``"6)P`&$.``#0``,"$"("@AA*(`'B3&``$`
+MQR`K``(80`!B&"$``QC``&(8(0`#&$"DHP`>%(#_]B2E``(FU@`!`M<0*R8Q
+M`#040/^Q)G,``B:U``$NH@`#5$#_GR0"``&/OP`TC[X`,(^W`"R/M@`HC[4`
+M)(^T`""/LP`<C[(`&(^Q`!2/L``0`^``"">]`$!3P/_P)K4``997`AXD8P!&
+M)E$"+!``_Y0F1`(@)&,`!291`!80`/^0`D`@(5/`_^4FM0`!EE<"'B1C`#<F
+M40+($`#_B29$`B8GO?^PK[X`0*^W`#ROM@`XK[4`-*^T`#"OL@`HK[``(*^_
+M`$2OLP`LK[$`)(R"&_0Y(P`!`2#P(8Q$```PUO__`0"H(8R"``24A````*"X
+M(21)`1"OH@`<)$(!"@!#2`LP\O__``"@(:^D`!@"P(`A%\``HP``0"$#TA`K
+M$$``(0`2$$(`$!!``%6((9(B``$`%!A`CZ0`&```,"$`?9@A$$``%P!`*"$,
+M`"B>`````":#``$P=/__EB,````4*$``O8@A``,:`H^D`!@``#`AIF(``!!@
+M``H`8"@A#``HG@`````F`P`!,'#__R:#``$P=/__`I(8*Q1@_^*F(@```!(0
+M0@+"$"$P4/__IO0`%!*``&$``$`A``@@0`"($"$`G1@A``(0@)1C````2!`A
+M``(00`!7,"$`$"A``)<@(:2#````M2@AI,,`&)2C```F`@`!,%#__P`#$$`P
+M0@$`,&,`_P!B&".DPP`<E*,````0($``E2`A``,3P@`"$@```QH"`&(8(Z3#
+M`""4@P``)@(``3!0__\``Q!`,$(!`#!C`/\`8A@CI,,`))2#````$"!``)4@
+M(0`#$\(``A(```,:`@!B2".DR0`HE((``"8#``$P</__,$(`'Z3"`!Z4@@``
+M`!`H0`"U*"$``A%",$(`'Z3"`"*4@@``)@,``3!P__\``A*",$(`'Z3"`":4
+MHP```!`00`!5."$``Q!`,$(!`#!C`/\`8A@CI,,`+)2C``"/I0`8``,3P@`"
+M$@```QH"`&(8(Z3#`"Z4X@``+*1``P`"&$`P8P$`,$(`_P!#$",4@``=I,(`
+M,*3)`"J4X@````(2`C!"`#^DP@`:)0(``3!(__\!%!@K)@(``11@_Z$P4/__
+MCZ,`'``>$$"/OP!$`$,0(91"`'Z/O@!`C[8`.*;B`!:/M0`TC[<`/(^T`#"/
+MLP`LC[(`*(^Q`"2/L``@`^``"">]`%"4X@``)`,``:3#`!H``AO"``,:```"
+M$@(`0Q`C$`#_X:3"`"H`"!!``$D0(91%```E`P`!,&C__R0"`/\`%!A`+08`
+M`R:$``$010`#`'T8(3"4__^D90``%,#_\P`($$`0`/]TIO0`%">]__"OL```
+M,+#__P`0$$"OL0`$`("((0!`("&OL@`(K[\`#`P`>E<`P)`AKB(``!!``#,D
+M`P`"`!`@@`"0("$`!""``)`@(0P`>E<`!""`$$``,JXB``BF,``$$@``)P``
+M,"$`!D"```880`$&$"$`<B`ACB4````"$(".)P`(E(0```!&$"$``A"``&48
+M(0!'$"&D9``````H(:1$```!!A`A``(0@``%&(``1A`A`&48(0`"$(``1R`A
+M``,8@`!D&"$DH@`!I&4`!#!%__\LH@`$%$#_\J1@``8DP@`!,$;__R0"``.D
+M@@!"`-`8*R0"``048/_;I((`!@``&"&/OP`,C[(`"(^Q``2/L````&`0(0/@
+M``@GO0`0#`!Z:HXD```0`/_V)`,``I2#`!8GO?_P)`+>K0"`8"&GH@`"I*,`
+M!J>B````8#@A```P(0``("$`AQ`',$(``1A```@LPP`"$&``@```$"$`!AA`
+M),(``0!]&"$P1O__I&0``"2"``$P1/__+((`!!1`__(`AQ`'E8(`%```2"$0
+M0`!QI*(`!``)&$``:1@A``,8@`!I&"$`"4"```,80`$)$"$!@Q@A``(0@(RJ
+M``B49``8`$D0(0`"$(``2A`AI$0``"1F`!B4PP`2I$,``I>C``(T`MZM$&(`
+M7@``("$!"1@A``,8@`!I&"$`!!"``$00(0`#6(`D@P`!``(0@#!D__\!:C@A
+M`$<0(2R#``048/_SI$``!I>C``"4Q``"``,0@`!#$"$``A"``$<0(:1$``B7
+MHP`"``,0@`!#$"$``A"``$<0(20#`!2D0P`(EZ,``@`#$(``0Q`A``(0@`!'
+M$"$D`P`CI$,`"I>C``(``Q"``$,0(0`"$(``1Q`A)`,`/Z1#``R7I`````00
+M@`!$$"$``A"``6(0(0!*$"$D1``$)`(`!*2"``*4PP`&E((`!`!#$"&D@@`&
+ME,,`"@!#$"&D@@`(E,,`#@!#$"&D@@`*E,(`!*2"``R4P@`(I((`#I3"``RD
+M@@`0E,(`$*2"`!*7I``"``00@`!$$"$``A"``6(0(0!*$"$D1``$)`(``Z2"
+M``*4P@`4I((`#)3"`!:D@@`.E,(`&*2"`!"4H@`$)2,``3!I__\!(A`K%$#_
+MD@`)&$`D`@`!`^``"">]`!`!"1@A``,8@`!I&"$`!!"``$00(0`#.(`D@P`!
+M,&3__P`"$(``ZA@A`$,0(2R#``048/_SI$``!I>D````!!"``$00(0`"$(``
+MXA`A`$H0(21$``0D`@`$I((``I3"``*D@@`$E,,`!@!#$"&D@@`&E,,`"@!#
+M$"&D@@`(E,,`#@!#$"&D@@`*E,(`!*2"``R4P@`(I((`#I3"``RD@@`0E,(`
+M$!``_\VD@@`2)[W^L*^W`3ROM@$XK[4!-*^S`2ROL@$HK[$!)*^P`2"OOP%`
+MK[0!,(R"&_0`@+`A`*"X(8Q"```PT?__`."0(8Q4``0``)@A``"H(0``@"$D
+M`@`!4@(`-I:"`!)2```XEH(`$"0"``)2`@`QEH(`%`.@("$``"@A#`!PJB0&
+M`1P"P"`A`Z`H(0(@,"$"H#@A`N!`(0P`(E4"`$@AEZ4`%`)`("$,`",F`Z`P
+M(0!`F"$40``3`$`8(0.@("$,`"-Q`D`H(1!```XD`___EZ,`%``#$(``0Q`A
+M`B(0(18```,P4?__)B(`!3!1__\F$``!+@(``Q1`_]8F4@`,`F`8(8^_`4"/
+MMP$\C[8!.(^U`32/M`$PC[,!+(^R`2B/L0$DC[`!(`!@$"$#X``()[T!4%!`
+M_^\F$``!$`#_S205``-00/_K)A```1``_\DD%0`*)[W_P*^W`"ROM``@K[\`
+M-*^^`#"OM@`HK[4`)*^S`!ROL@`8K[$`%*^P`!",@AOT,,;__Z^E``",4P``
+MKZ<`"*^F``2.8@`$``"X(0``H"&47@`2)`(``0``D"$``*@A$H(`<H^Q``12
+M@`!JCF,`#"0"``(2@@!=)B(`:::@```2X``X``"P(99B``"/I0```!$@0"8C
+M``$`A2`A+$(P`S!Q__\40`!/E)`````0$@*F0@`(ED4`"!"@`$@F(@`!%H``
+M1)9D`````#`A#``HG@````"F0@`(EF(``"Q",`,40``Z`!`8P``0&0``$!""
+M,$(`/S!C`#^F0P`$ID(`!I9B``"/I0```!$@0"8C``$`A2`A+$(P`S!Q__\4
+M0``CE)```)9#``0`$","`!`1@@!D&"4P0@`_,@0`/Z9#``2F0@`"ID0``):B
+M```D0@`!IJ(``";6``$"UQ`K%$#_RB92``HFE``!+H(``Q1`_[HD`@`!CZ(`
+M"(^_`#2/O@`PI%$``(^W`"R/M@`HC[4`)(^T`""/LP`<C[(`&(^Q`!2/L``0
+M`^``"">]`$"60P`$`!`@0@`0*T(`$!'"`&48)3!"`#\0`/_=,(0`/Q``_\@`
+M$!#"$`#_O20&``$0`/_=,%'__Q``_[(`$!)"%\``!#!1__\F(@`&$`#_VC!1
+M__^.8@`,)!<``R15`%(0`/^;)%(`5"8B`%4P4?__`&"H(207``@0`/^5)'(`
+M`B8B`&4P4?__$\#_\"8B``2.8@`,)!<``B15`'(0`/^,)%(`=">]_]"OM@`8
+MK[0`$*^Q``2OOP`@K[<`'*^U`!2OLP`,K[(`"*^P``",@AOT`*"@(3#1__^,
+M50````"P(8ZC``248@$6$$``$8ZR`!``%KA``N,0(91"`1@40``7EJ(``"8C
+M``@F)``'+$(P`P!@B"$`@H@+CJ,`!";6``&48@$6`L(0*Q1`__$F4@!`C[\`
+M((^W`!R/M@`8C[4`%(^T`!"/LP`,C[(`"(^Q``2/L````^``"">]`#`L0C`#
+M%$``0``1$$``5#`A``"8(0)`*"&4R```)G,``BYD``@Q`P#_``@2`J2B``"D
+MHP`(),8``B8Q``$4@/_V)*4`$``1$$``5$@A``"8(0)`."&5*```)G,``BYF
+M``@`""&"``@2`@`(&X(P0@`_,&,``3"$``$Q!0`_I.(``JSC``2DY0`*K.0`
+M#"4I``(F,0`!%,#_[R3G`!```)@A`D"`(8X"``!00``/)G,``8ZB``0"XA`A
+ME$(!&#!#``<08``.```P(20"``,08@`,EJ0``)8%```D!@`!#``HGB9S``&F
+M`@``+F(`"!1`_^TF$``($`#_LHZC``26I```$`#_]I8%````5!`AE$@``"8Q
+M``$`$2!```@I0``(&((`"!)",*4`?S!C`'^F10`0ID(``*9#``@`E"`AE(@`
+M`"8Q``$`$3!```@:P@`((,``"!$"`*,H)3"$`'\P0@!_ID0`(*9%`!"F0@`8
+M`-0P(93(```F,0`!`!$H0``(&T(`"#!```@1@@"#("4PQ@!_,$(`?Z9&`#"F
+M1``@ID(`*`"T*"&4J```)C$``0`1.$``""/"``@I```($@(`"!B"`,0P)3"E
+M`#\P0@!_,&,`/Z9%``JF0@`XID,``J9&`#``]#@AE.@``"8Q``$`$2!```@;
+M`@`($8(`HR@E,$(`/S$#`#^F10`*ID(`$J9#`!H`E"`AE(@``"8Q``$`$2A`
+M``@@@``(&0(`"!*",(0`/S!C`#^F1``RID(`(J9#`"H`M"@AE*@``"8Q``$`
+M"!H"``@3@@""("4P8P`_ID0`,A``_XZF0P`Z)[W_T*^U`"2OM``@K[,`'*^_
+M`"BOL@`8K[$`%*^P`!",@AOT`*`P(0"@H"&,4@```("8(0``J"$,`"8TCD4`
+M!)9B'/8P0@#P+$(`,%1```:60@``CD(`!)1"`"@00``I)`,`%I9"```D$`"0
+M)!$`0"Q",`,"`#`A`B(P"P)@("$,`"%$`H`H(99"```"`#`A+$-``"Q",`,4
+M8``&`B(P"XY"``240P`H)`(``1!B`",F1P`4`F`@(0P`(;X"@"@AED,``"QB
+M0``40``8+&,P`XY"``240@`L)$+^ZS!&__\"8"`A`H`H(0P`)($#H#@AEZ8`
+M``)@("$,`"45`H`H(0*@&"&/OP`HC[4`)(^T`""/LP`<C[(`&(^Q`!2/L``0
+M`&`0(0/@``@GO0`P)`8`D"0"`$`0`/_J`$,P"P)@("$,`"0M`H`H(0!`J"$4
+M0/_M`$`8(1``_]N60P``)[W_T*^V`!BOLP`,K[(`"*^_`""OMP`<K[4`%*^T
+M`!"OL0`$K[```(R"&_0`H)`A`,"8(8Q"``"45```)`(`#Z"B`"(NEC`#)`(`
+M#Q;``BVDH@`F/`*``B17,V@D`@`@I*(!%H[B```D4?]``!$00`!3$"&44```
+M`!`3@@`0&L(`$"D"`!`PP@`0.((`$$!",&,`!P`02\(P0@`!,*4`?S#&``$P
+MYP`!,0@``:9#``:F0@`$ID4`"*9)``"F1@`"ID<`%`P`*+RF2``2CN0`!#(#
+M``$`0Q`E))'_0``1&$"F0@`0`',8(91P```N@T``))'_00`0$@*B0@`@%&``
+M%:)0`"$`$1!``%,0(910```DD?]"`!$80``0$X*F0@`H`',8(0`0*P(`$"-"
+ME'```#"$``$PI0`!`!`3@C!"``$R`P__KD0`&*Y%`!RF0P`LKD(`-```J"$`
+M%1"``%<0(8Q)``@`%6!``9(X(24Q_T``$1!``%,0(910```E,?]!).8`B``0
+M*0``$!("`!`8@@`1($`PI0`_,$(`?S!C`#^DQ0`"I.(`/*3C`$8`DR`AE)``
+M`"4Q_T(`$2!``!`;`@`0$8(`HR@E,$(`/S(#`#^DQ0`"`),@(:3B`)"DXP"6
+ME)```"4Q_T,`$2!``!`H@``0$0(`$!J",*4`/S!"`#\DZP"FI.,`G*3B`*(`
+MDR`AI64``I20```E,?]$`!%`0``0(X(`$#$``!`2`@`0&((`I"@E,,8`/S!"
+M`#\P8P`_).H`N*5E``(!$T`AI.(`KJ3C`+2E1@`"E1```"4Q_T4`$2!``!`;
+M`@`0$8(`PS`E,$(`/S(#`#^E1@`"`),@(:3B`,"DXP#&E)````*R0"$`$!("
+MH0(`:"0"``$2H@&3)3'_1BJB``(40`&$)`(``A*B`7T`$!$"%J``%P`1$$``
+M4Q`AE%```)9&`.HF,0`!`!`[P@`0$P(`$!I"`!`A@@`0*,(`QS`E,$(`!S!C
+M``<PA``',*4`!S('``>F1@#JID(`[*9#`.:F1`#HID4`XJ9'`.0`$1!``%,0
+M(910```F,0`!`9)0(3($`/\`$!H"`!$00*5#`%"E1`!6`%,0(910```F,0`!
+M`!$80#($`/\`$!("I4(`7*5$`&(`<Q@AE'```"5%`'`F,0`!`!`2`C($`/\R
+M`P"`H0(`;1!@``2DI``".((`_P`"$">DH@`"`!$00`!3$"&44```+H5``"8Q
+M``$`$!E"`!`@0C!C`/\PA``/,@(``:5"`(2E0P!X%*``!J5$`'X2H`$T`!`3
+M0B0"``(2H@$N`!`30A;``!$N@C`$`!$00`!3$"&44````9(8(0`0$8(P0@!_
+MI&(`S"0"``$2H@$<)C$``2JB``(40`$4)`(``A*B`0T`$!#"+H(P!!1``00`
+M%1!``!480`!R&"$`$"-")&,`T``1$$"D9``"`%,0(910```F,0`!`!`0P#!"
+M`#@`@B`E)`(``A*B`.VD9``"%J``"2Z"0``4H``'`!`8P@`0$@(P0@`_,&,`
+M'Z9"`0*F0P$&+H)``!1``#`FH@`!)`(``1*B`,$D`@`"$J(`:P`1$$`6H``I
+M)J(``2Z"0`$40``F)J(``0`1$$``4Q`AE%```"Z#4``F,0`!,@(`/Q1@`!VF
+M0@#8`!`1@@`0,T(P0@!_`!$80*9&`$RF0@!"`',8(91P```F,0`!`!$H0#("
+M``<``A#``!`8P@`0(D(`PC`E,&,`/Z)$`&NF1@!,ID,`W@"S*"&4L```)C$`
+M`3("``$``A'``((@)0`0$$*B1`!KHD(`<":B``$P5?__+J(``Q1`_O\`%1"`
+M4L``$X[B`!0`$1!``%,0(910```F,0`!`!$@0``0$,(R`P`',$(`!Z9#`/JF
+M0@#^`),@(920````$!#",$(`!S(#``>F0P#\ID(!`([B`!261P$6``"H(1#@
+M``\D4?]``!$00`!3,"&4T```)J,``@`5$$`P=?__`%(0(3(#`/\`$"("`J<H
+M*Z1#`1JD1`$8%*#_]23&``(2P``+C[\`("0"_\HD`___)`0`'*9"`'(D`@`/
+MID0`6J9#`':F0@!6ID,`=*9$`%B/MP`<C[8`&(^U`!2/M``0C[,`#(^R``B/
+ML0`$C[````/@``@GO0`P`%,0(910```"@"`A)`8``0P`*)XR!0#_)C$``0`0
+M*@("@"`A)`8``0P`*)ZF0@$*`!$80*9"`0P`<Q@AE'```"8Q``$`$2!``!`1
+MPC(#`'\P0@`_ID,`"J9"``X`DR`AE)```"0&``$"@"`A#``HGC(%`/\N@T`!
+MID(!#A1@``0F,0`!`!`2`C!"`#^F0@#<`!$00`!3$"&44```+H1``B8Q``$`
+M$!%",$(`/S(#`!^F0@$$%(``!J9#`0@`$1!``%,0(910```F,0`!HE``(BZ"
+M4``40/^*)J(``0`0$@(`$#/",$(`?P`1&$"F1@!.ID(`1`!S&"&4<```)C$`
+M`0`1*$`R`@`?``(00``0&4(`$"+"`,(P)3!C`#^B1`!LID8`3J9#`.``LR@A
+ME+```"8Q``$R`@`'``(10`""("4`$!#"HD0`;!``_VRB0@!Q`!$00`!3$"&4
+M4````H`@(20&``$,`"B>,@4`_R8Q``$`$"H"`H`@(20&``$,`"B>ID(!$``1
+M&$``<Q@AID(!$I1P```"@"`A)`8``0P`*)XR!0#_+H-``:9"`1048/]3)C$`
+M`0`0$@(P0@`_$`#_3Z9"`-H`$!#",$(`_RZ#0`848/\0ID(`)``0$L(0`/\-
+MID(`)@!2$"$D`P`*I$,`TB0"``\0`/\0ID(`)#!"``<R`P`'ID,`_!``_O"F
+M0@$`%J#^[RZ",`0R`@`_$`#^ZZ9"``P`$!#",$(`!S(#``>F0P#Z$`#^Y:9"
+M`/XP0@`!$`#^T:9"`#`P0@`!$`#^SJ9"`"XP0@`',@,`!Z9"`/80`/Z`ID,`
+M^!:@_I8`$1!``!`00``0&4(`$""",&,`!S"$``<P0@`'ID,`[J9$`/`0`/YT
+MID(`Z@`0$0(P0@`',@,`!Z9"`/(0`/YNID,`]"0"`!"DH@$6/`*``A``_=0D
+M5S-0,(3__RR$,`(PI?__%(``'3#&__\\`H`")$D7I```0"$`"!!``$D0(91$
+M```D`@!D/`-1ZP!$$",`10`8-&.%'R4"``$P2/__+0(`"W"&`````"`2````
+M````````@P`8``0GPP``&!```QE#`&08(Z3C```40/_J).<``@/@``@`````
+M/`*``A``_^0D21>,,*7__R0"`/\P@___$*(`"20'`/\0P``)``40@"QB,`-0
+M0``")*((_"2B"6`P0O__`$`X(0/@``@`X!`A`$40(21"$L`L8S`#$&#_^3!"
+M__\LH@`_%$``!``%$(``11`A$`#_\B1"%2(`11`A``(00!``_^XD0A/L)[W_
+MX"0%`+\#H#`AK[``$*^_`!0,`"RO`("`(1!```8``!@AC[\`%(^P`!``8!`A
+M`^``"">]`"".`AOT)`,`SR0%`,J,0@```@`@(0.@,"&40@``+$)```P`+*\`
+M8B@+C@(;])>C``",1```,&:``#!C"`"4@@```$`@(2Q"0``X0@```,(8"S!B
+M__\LA$````(KPCB$`````A+"`*00"Z>C``"GH@``$`#_WC!#__^,@@``C$)`
+M"#A"``$#X``(+$(``2>]_^"OLP`,K[$`!*^P``"OM``0K[(`"*S```",BP``
+M)`+__P"@F"&-:0"``,"`(0"`4"$D!?__$2(`.```B"&,@QM`/`0!!32$V#`!
+M(Q@D/`(`@`$D0"0Q)0`%`&)H)#4"``$`14`+/!(`(#!G``4P9@%`-0(`0`!R
+MR"0Q)0%`,&P`@`!D&"0`14`+-&(``0!'&`LT8@!`/!0#_SP8(``Q+P`@,2X`
+M@!&@``(`1A@+C7$`C`&#&"4"."`D$:```@!X$"4`1!@+`'(0)0!9&`L!XQ@E
+M$<``#`!@*"&-8P"$)`+_?P$B2"0`=!@D``,<`JX#`````Q0`C4,```!4$"0U
+M"`"`K&(`A(U"``"N:```K$D`@(U"``",0@"`C[0`$(^S``R/L@`(C[$`!(^P
+M````H!`A`^``"">]`"`\`@$%-$+8,#"H`$`PJ0`!,*H`@`2A``\`@#`AC(,;
+M](QB`!000``#)$3__ZQD`!2,8@`4%$```R0"``&,PP``K&(`)`/@``@`````
+M`^``"`````",@P```*(H)(QB`"0X0@`!+$<``5#@``6,PAM`K&``)(R"``",
+M0@`DC,(;0(S$```T0P%``&@0"S1#``4`:1`+`4(0)0!%$"6L@@"@C,0``(R#
+M`*`0X/_FK,(;0"0"``&L@@`D$`#_Y``````$HP`+C(,``(R#&_2,8@`4)$(`
+M`:QB`!2,@@``K$``)(R"``",0@`D`^``"`````",8@`D.$(``2Q'``$0X``%
+M,*(`0*Q@`"2,@@``C$(`)#"B`$`00``#C(8;0"0"_K\`PC`D,*(``1!```0P
+MH@"`)`+_^@#","0PH@"`$$``!#P"`04D`O]_`,(P)#P"`04T0M@PC(,```"B
+M*"0`!1`G`,(P)*QF`*",@P``C&(`H!#@_]VLAAM`)`(``:QB`"0#X``(````
+M`(R#```T`H@```4I0`"B*"$`91@AK&```(R"```D`P`'`$40(:Q```2,@@``
+M`$40(:Q```B,@@```$40(:Q```R,@@```$40(:Q``!",@@```$40(:Q#`!2,
+M@@```$40(:Q``!B,@@```$40(:Q``!P#X``(`````">]_X"OM0!D/!6JJC:U
+MJJHD`@`'K[X`<*^W`&ROM`!@K[(`6*^Q`%0`P)`AK[\`=*^V`&BOLP!<K[``
+M4`#@B"&OH@`T`("@(0"@N"$`"*@*KZ``!*^@`````#`A$.``00``\"&4XP`,
+M+&(`(1!``!,``Q"`/`.``B1C"^0`0Q`AC$(```!```@`````E.4`-"3P`#8D
+M`@`%KZ(`-"0"`&@0H@(#**(`:1!``?\D`@"`)`(`*%"B``Z2`@`%C[\`=(^^
+M`'"/MP!LC[8`:(^U`&2/M`!@C[,`7(^R`%B/L0!4C[``4`/@``@GO0"`D@(`
+M!9(#``22!``&D@4`!P`"$@``8A@E``0D``!D&"4`!2X``&48)0!U&"8P8___
+M``,0)S!"__^OH@`4KZ,`!)(#``&2`@``D@0``I(%``,``QH``$,0)0`$)```
+M1!`E``4N``!%$"4`51`F``(8)Z^C``ROH@``$D``%#0%@`"20@`#DD,``I)$
+M``&210`%``(6```#'`"21@`$DD<```!#$"4`!"(``$00)0`%*@``IC`E`$?P
+M)0`>\$(`!A?``\+P)0`&,$(T!8``-,:``*^E`$2OI@!($B``!:^@`#R6(P`,
+M)`(`$!!B`"H"@"`A```P(0.@."$`%Q#`CH,```!&$"&,Y0`,``(0@#0(B```
+M2!`A),8``0!B&"$LQ``(K.(`"*QE```4@/_S).<`"(Z"````%R%`CZ,```""
+M$"$`2!`AK$,``(Z"```T`X@$`((0(0!#$"&/HP`$K$,``(Z"```T`X@8`((0
+M(0!#$"&L7@``CH(```""("$T`H@<`((@(8^B`$BL@@``$`#_D8^_`'0D!0`2
+M#``=SB0&`!`00/_4```P(1)``0,GI``(```H(0P`<*HD!@!`)`(`!"8P`!2O
+MH@`TD@,`#9("``R2!``.D@4`#P`#&@``0Q`E``0D``!$$"4`!2X``$40)0!5
+M$":OH@`LD@,`"Y("``J2!``,D@4`#0`#&@``0Q`E``0D``!$$"4`!2X``$40
+M)0!5$"8P0O__KZ(`))(#``>2`@`&D@0`"``#&@"2!0`)`$,0)0`$)```1!`E
+MCZ0`!(^C````!2X``$40)0`$("<`51`F,(3__P`#&"<T!8```@"0(:^B`!RO
+MI``4KZ,`#*^E`$2OH``\,O/__P``,"$#H#@A`!,0P(Z$````1A`AC.4`##0#
+MB````A"``$,0(23&``$`@B`A+,,`"*SB``BLA0``%&#_\R3G``@F,`!&)N(`
+M0">D``@``"@A)`8`0`P`<*HP4___D@,``9($``*2(@!&D@4``P`#&@``0Q`E
+M``0D``!$$"4`!2X``$40):^B``R2`P`%D@(`!)($``:2!0`'``,:``!#$"4`
+M!"0``$00)0`%+@``11`EKZ(`'```,"$#H#@A`!,0P(Z#````1A`AC.4`#``"
+M$(`T"(@``$@0(23&``$`8A@A+,0`"*SB``BL90``%(#_\R3G``B.@@``CZ,`
+M```7L4`"PA`A`$@0(:Q#``".@@``-`.(!">D``@"PA`A`$,0(8^C``0``"@A
+M)`8`0*Q#```FX@`@#`!PJC!3__\D`@`$KZ(`-))#``V20@`,DD0`#I)%``\`
+M`QH``$,0)0`$)```1!`E``4N``!%$"4`51`FKZ(`+))#``N20@`*DD0`#))%
+M``T``QH``$,0)0`$)```1!`E``4N``!%$"4`51`F,$+__Z^B`"220P`'DD(`
+M!I)$``B210`)``,:``!#$"4`!"0``$00)0`%+@``11`ECZ0`!`!5$":/HP``
+MKZ(`'(^B`$@`!"`G,(3__P`#&">OI``4KZ,`#*^^`#ROH@!$```P(0.@."$`
+M$Q#`CH0```!&$"&,Y0`,-`.(```"$(``0Q`A),8``0""("$LPP`(K.(`"*R%
+M```48/_S).<`""8P`$XFX@!@)Z0`"```*"$D!@!`#`!PJC!3__^2`P`!DB(`
+M3I($``*2!0`#``,:``!#$"4`!"0``$00)0`%+@``11`EKZ(`#)(#``62`@`$
+MD@0`!I(%``<``QH``$,0)0`$)```1!`E``4N``!%$"6OH@`<```P(0.@."$`
+M$Q#`CH0```!&$"&,Y0`,-`.(```"$(``0Q`A),8``0""("$LPP`(K.(`"*R%
+M```48/_S).<`"(Z"```T`XP``L(0(0!#$"&/HP``K$,``(Z"```T`XP$`L(0
+M(0!#$"&/HP`$K$,``!``_HB/OP!T)[(`"`)`("$``"@A#`!PJB0&`$`D`@`$
+MKZ(`-"8P`!22`P`-D@(`#)($``Z2!0`/``,:``!#$"4`!"0``$00)0`%+@``
+M11`E`%40)J^B`"R2`P`+D@(`"I($``R2!0`-``,:``!#$"4`!"0``$00)0`%
+M+@``11`E`%40)C!"__^OH@`DD@,`!Y("``:2!``(D@4`"0`#&@``0Q`E``0D
+M``!$$"4`!2X``$40)8^D``0`51`FCZ,``*^B`!R/H@!(``0@)S"$__\``Q@G
+MKZ0`%*^C``ROO@`\KZ(`1#+S__\``#`A`Z`X(0`3$,".A````$80(8SE``PT
+M`X@```(0@`!#$"$DQ@`!`((@(2S#``BLX@`(K(4``!1@__,DYP`()C``3@)`
+M("$FX@!````H(20&`$`,`'"J,%/__Y(#``&2(@!.D@0``I(%``,``QH``$,0
+M)0`$)```1!`E``4N``!%$"6OH@`,D@,`!9("``22!``&D@4`!P`#&@``0Q`E
+M``0D``!$$"4`!2X``$40):^B`!P``#`A`Z`X(0`3$,".@P```$80(8SE``P`
+M`A"`-`B(``!($"$DQ@`!`&(8(2S$``BLX@`(K&4``!2`__,DYP`(CH(````7
+M(4"/HP```((0(0!($"&L0P``CH(```""("$T`H@$`((@(1``_G2/H@`$%*+^
+M!8^_`'0D`@!H$*(`)I(#``V2`@`,D@0`#I(%``\``QH``$,0)0`$)```1!`E
+M``4N``!%$"4`51`FKZ(`+)(#``N2`@`*D@0`#)(%``T``QH``$,0)0`$)```
+M1!`E``4N``!%$"4`51`F,$+__Z^B`"22`P`'D@(`!I($``B2!0`)``,:``!#
+M$"4`!"0``$00)0`%+@``11`E`%40)A``_>BOH@`<D@(`#)($``Z2!0`/``,:
+M``!#$"4`!"0``$00)0`%+@``11`E`%40)A``_]HP0@#_E.4`"B0"`&@0H@`-
+M)/``%"BB`&D00``&)`(`@"0"`"A0HOV^KZ``-!``_;TD`@!H5*+]NR0"`&@0
+M`/VW)`(``Q``_;4D`@`!C((;])3E`#0D`P`&C$0`C"0"``<D\``V`$08"A``
+M_:VOHP`TE.4`"B3P`!00`/VH)`(`!)3E``HD\``4$`#]I"0"``<D`W__$*,`
+M%0`'4"N,@AOT,*7__P!@2"&,0@!$`*=("Q!```X`H$`A)`(`$!#"``DDI0`@
+MC((```$$&"&L2`0`C((``*Q*!`2@9QT!`^``"`$@$"$0`/_W,*C__Q``__LD
+M"7__,,;__RS&``T4P``1```8(8R"&_0D`P`!B$0`I)A$`*>(1@"HF$8`JXA'
+M`*R81P"O@$@`L*BD``"XI``#J*8`!+BF``>HIP`(N*<`"Z"H``P#X``(`&`0
+M(8R$&_0PI?__C(,`-!!E``(``!`AK(4`-`/@``@``````^``"```$"$GO?_@
+MK[$`%`.@,"$`H(@A)`4`'Z^R`!BOOP`<`("0(0P`+*^OL``0`D`@(0.@,"$0
+M0``))`4`'B0$`!:/OP`<C[(`&(^Q`!2/L``0`(`0(0/@``@GO0`@E[```).C
+M``$`$!("HB,``0P`+*^B(@```D`@(20%`!T40/_O`Z`P(9>B``"3HP`!`@*`
+M(0`"$@*B(P`##``LKZ(B``(40/_G)`0`%I>B``"3HP`!/`4``@("@"$``A("
+M```@(32E__VB(@`$$@#_W*(C``46!?_<C[\`'!``_]HD!``6)[W_\`"@("&O
+MOP``#`!K4B0%``&/OP`````0(0/@``@GO0`0`^``"`````",AP``)`(``20#
+M``*,YD`0`$48"B0"_Y\`PC`D``,90`##,"6LYD`0C(8``#P"@`*,0S.`C,5`
+M$#P"__$T0O__`*(H)``#'$`\`@`.`&(8)`"C*"6LQ4`0C(8``#P"@`*,0S.$
+MC,5`$#P"_X\T0O__`*(H)``#'0`\`@!P`&(8)`"C*"6LQ4`0C(4``#P"@`*,
+M0S.(C*1`$#P"_W\T0O__`((@)``#'<`\`@"``&(8)`"#("6LI$`0`^``"```
+M````@$`A$*``,P#`2"&(HP``F*,``XBB``"8H@`#B*0``)BD``.4I@`$,&/_
+M`(BJ``"8J@`#``,:```"%@``!"("`$,0)3"$_P`PPP#_`$00)0`*+@(``QH`
+M``8R`@!%*"4`9A@EC0(``#0$@``\!O^``$00(:Q%``B-`@``-,;__P!$$"&L
+M0P`,C0,```!D&"&,8@`,,$+__ZQB``P1(``+).(`!(T#`````A0``&0@(3P#
+M`'\`0Q@DC((`(!#@``(`1A`D`$,0):R"`"`#X``(````````&"$0`/_B```H
+M(2>]_]"OOP`@K[,`'*^R`!BOL0`4K[``$(RC`!",I@`4`*"`(5!@``$```'-
+M`,,`&XX"``P#H"@A`("((0``F!(```````````!B`!L``)`2#``M[0````"/
+MHP`$CZ(``(X&``2.!0`(``,=@``"$H(`8D@EC@,``(X(``PE(@`"),;__P!H
+M("$F9___`((8*P3``*&N!```5&#_]XX#``!PR#@"C@(`$(X#`!@`Y#`A<*(@
+M`JX&``0`AA`A$&``C*X"``B.)```-`6``#P#`!``A2`AC((`!`!#$"6L@@`$
+MCB(``(X#`!0`11`AK$,`)(XB``".`P`8`$40(:Q#`#B.(P``C@(`"`!E&"$`
+M`A#`K&(`,(XB``".`P``-!*```!2$"&L0P`HCB,``)8&``X\$___`'(8(8QB
+M`"`"("`A/`6```!3$"0`1A`EK&(`(`P`*6P`````CB(;0(XD```\`__[-&/_
+M_P!#$"2N(AM`K((`H(XB'82.)```/`7__P!#$"2N(AV$C@(`(`"2("&,@P`8
+M-*4`_P`"$@``91@D,$+_``!B&"6L@P`8CB(;0(XD```\`P`$`$,0):XB&T`\
+M!8``K((`H`P`*3P"("`AC@,`'(X"``P`0Q`K5$```:X#``R.`@`0`$,0*U1`
+M``&N`P`0C@,`#(X"`!!08@`ZC@(`!(X"``2.(P``/`8`!P`"$,`TQO__)$+_
+MZ`!R&"$`1A`DK&(`U(XC```\!0#_-*7__P!R&"&,8@#4/`<*`#P$`!@`11`D
+M`$<0):QB`-2.(P``C[\`(`!R&"&,8@#4`$00):QB`-2.`@``CB,````"$,`D
+M0O_H`$80)`!R&"&L8@#8CB,```!R&"&,8@#8`$40)`!'$"6L8@#8CB(``)8#
+M``X`4A`AK$,`W(XD``".`P`0C[$`%`"2("&,@@#<``,<``!S&"2/L@`8C[,`
+M'(^P`!`P0O__`$,0):R"`-P#X``()[T`,!``_\:N`@``CB,``#0"@``\!/_O
+M`&(8(8QB``0TA/__`$00)*QB``00`/^!CB(``"2E__\HH@``)D;__Q``_UP`
+MXB@+C((``#0&@```1A`AC$(`3*BB``"XH@`#C((```!&$"&,0P!0C$(`4*BB
+M``2XH@`'B*(`!)BB``<00P`&`````(R"````1A`AC$(`3*BB``"XH@`#`^``
+M"`````",@P``-`6``#P&`0``91@AC&(`(`!&$"6L8@`@C(,```!E&"&,8@`@
+M`$80):QB`"`#X``(`````(R#```T!X```(!`(0!G$"&,1``$C&(`%#P%_Y^5
+M!AMR-$(`(*QB`!2-`P``-*7__SP"`((`A2`D`((@)0!G&"$PQC<`)`(4`*QD
+M``10P@`#C0,```/@``@`````)`(#<*T`'/RL8A!P$`#_^@````"4@AMR```8
+M(3!"`@`00``UC(D<")"K```18``.``!`(0"H$"&00@`!)0@``0$+4"HP1@!_
+M,$(`@!!```0`PS@K5.```0!@,"$PPP#_%4#_]0"H$"&-)@``&,``"0``."$E
+M)0`RD*(``!!#``4DI0`8).<``0#F$"I40/_[D*(``!C```L``$`A)24`,P#H
+M$"I40``"H*<``*"H``"-(@``)0@``0$"$"H40/_X)*4`&%!@``Z,@P````,0
+M0BQ"``-00``*C(,``(R$```T`H``/`,"``""("&,@@`$`$,0):R"``0#X``(
+M`````#0"@``\!/W_`&(8(8QB``0TA/__`$00)*QB``0#X``(`````(R#```T
+M`H```&(H(8RB&&0``A3",$,!`#!$`?\08``".((!_P`"(">,H@!0C*,`3`!#
+M$"8#X``(`$00)B2C__\L8@`%`*#`(1!``#L`@'`A``,0@#P#@`(D8PRL`$,0
+M(8Q"````0``(`````(R"```T#X``)(T$@`!/$"&,1`"0C:(`3(VC`%B-QP2`
+M`$00(:VB`$R-P@``C:@`8(VD`%"-I0!4`$\0(8Q"`(R-J0!DC:L`:`!B&"&M
+MHP!8C<P``(VJ`&P!CS`AC,(`E`#B."&MQP2`C,,`B(V&!A```!`A`((@(0$&
+M0"&MJ`!@C<(```"C*"$`HS@KC$(&%`"'("&MI`!0K:4`5`$B2"&MJ0!DC<(`
+M`(Q"!A@!8E@AK:L`:(W"``",0@8<`4)0(20"``(3`@`$K:H`;```("$#X``(
+M`(`0(8W&``"-I``(C:4`#```$"$`SS`AC,,`F`""("$`HR@A`*,P*P"&("&M
+MI``(K:4`#!``__$``"`AC(0``(R#`$`T8@`$K((`0(W$```D`O_[`&(8)*R#
+M`$`0`/_G```@(8R"```T`X`````@(0!#$"&,10#XC$,`]"RB`&040/_>````
+M`#P"4>LT0H4?`*(`&0``$!```BE")`(`9`!E`!M0H``!```!S0``&!(0`/_2
+M`$,@(U#`_]```"`AC((``#0#@```0Q`AC$4`[*S%``",@@```$,0(8Q$`/"L
+MQ``$C<(```!#$"&,0@#TK,(`"(W"````0Q`AC$(`^*S"``R-P@```$,0(8Q#
+M`.R,0@#P`&48)@!$$"8``Q@K``(0*ZS"`!`0`/^SK,,`%(R"```T`X```$,0
+M(8Q"&&0``A,"`^``"#!"`'\PI?__)(0%O```&"&4@@``$$4`"22$`$@00``(
+M``,4`"1C``$L8@!`5$#_^92"```#X``(```0(0`#%``#X``(``(4`R>]__"O
+ML```)`4``@"`@"&OOP`$#``N@0``,"&.`@``-`2``#P#``(`1!`AK$`!)(X"
+M``"/OP`$`$00(:Q``2B.`@``K$``0(X"````1!`AK$,!,(X"``"/L```/`,"
+M``!$$"&L0P$X`^``"">]`!`GO?_P```H(20&`$BOOP`$K[````P`<*H`@(`A
+M/`(`/S1"_SBN`@!$)`(!]*X"`!`D`@!DK@(`'"0"`"BN`@`D/`,`/R0"``<D
+M!`#(K@(`(#1C_@PD`@`"K@0`&*("``2N`P!`K@0`%*(```:B```'H@``!:(`
+M``B/OP`$C[````/@``@GO0`0)[W_\*^R``BOL0`$K[```*^_``P`@(`A``"0
+M(221!;P"("`A#``O0B92``$N0@!`%$#_^R8Q`$B.`@``C@,%_#0%@```11`A
+MK$,!+(X"``".`P8``@`@(0!%$"&L0P$T#``O)@`````D`@'TK@(7O(^_``R/
+ML@`(C[$`!(^P```D`@`!`^``"">]`!`GO?_PK[\`!*^P``",D`6X/`(`0`"`
+M0"&.`P`0`$,@(P!#$"H00``EK@``#*X``$".`P`8/`(`0`!#(",`0Q`J4$``
+M`JX$`$2N``!$C0(``(X#`$`T!X```$<0(:Q#`2R-`@``C@,`1`$`("$`1Q`A
+MK$,!-(T"```\`P`")`4``@!'$"&L0P$PC0(``#P#`@```#`A`$<0(:Q#`3@,
+M`"Z!`````*X``#RN```XC[\`!(^P```#X``()[T`$!``_]RN!`!`)[W_\*^Q
+M``2OOP`(K[```(R#%\``@(@AC)`%N"AB```D9`!_`&`H(0""*`L`!2G#``41
+MP`!B$",H0@!`5$``!0"@("$D8P#^*((```!B(`L`!"'#CB(7Q!!``!2/OP`(
+MD@(``Q!``$TD0@`!D@,`!"QB``(40`!$)&(``8X"`"0`1!`J4$``'XX"`""2
+M`@`&$$``$B0"``&2`P`%+&(``A1```8D8@`!C[\`"(^Q``2/L````^``"">]
+M`!`P1@#_`B`@(20%``2B`@`%#``Q5@`````0`/_UC[\`"`(@("$D!0`"H@(`
+M!@P`,58``#`A`B`@(20%``6B```$$`#_\P``,"$`1!`J4$``"Y8B&W*2`@`&
+M4$#_X9(#``6B```&`B`@(20%``(,`#%6)`8``1``_]J2`P`%,$(0`!!`_]N/
+MOP`(D@(`!A!```@D`@`!D@(`!5!`_]6/OP`(H@``!0(@("$0`/_E)`4`!*("
+M``8"("`A)`4``@P`,58``#`A$`#_])("``4P1@#_`B`@(20%``40`/_-H@(`
+M!#!&`/\"("`A)`4``1``_\BB`@`#C(,7P">]__"OOP``*&(``"1E`'\`8#`A
+M`*(P"P`&,<,`!A'``&(0(RA"`$`40``JC(<%N"1C`/XHH@```&(H"P`%&<.,
+M@A?$$$``#8^_``"0X@`#$$``'"1"``&,X@`@`$,0*E!```^4@AMRD.,`!2QB
+M``(40``$)&(``8^_```#X``()[T`$#!&`/\D!0`$H.(`!0P`,58`````$`#_
+M^(^_```P0A``$$#_]8^_``"0X@`%$$#_\B0%``2@X``%$`#_]```,"$P1@#_
+M)`4``1``__"@X@`#$`#_V@#`&"$GO?_PK[(`"*^Q``2OL```K[\`#)2%&W`,
+M`"\5`("`(0`"&,``8A@A``,8P`(#D"&60P6\E@(;<"91!;P"("`A$&(`+ZX1
+M!;@,`"]"`````)8"&W"F0@6\E@(;<@`"$@(P0@`/HB(``I(F``,"`"`A#``Q
+M5B0%``&2)@`$`@`@(0P`,58D!0`%DB8`!@(`("$D!0`"#``Q5BS&``&2)@`'
+M`@`@(0P`,58D!0`#DB8`!20%``0,`#%6`@`@(0P`+X<"`"`AC@(``#0$@``\
+M`P`"`$00(:Q#`3".`@``C[\`#(^R``B/L0`$C[````!$$"$\`P(`K$,!.`/@
+M``@GO0`0E@(;<I(C``(``A(",$(`#Q1B_\T`````$`#_U)(F``.,@Q?`)[W_
+M\*^_```H8@``)&8`?P!@."$`PC@+``<YPP`'$<``8A`C*$(`0!1``"Z,A06X
+M)&,`_BC"````8C`+``89PXRB`"0`0Q`J5$``#)"B``2,H@`@`$,0*E!```60
+MH@`%D*(`!E1``!R@H``&D*(`!11``!4D0O__D*(`!!1```XD0O__D*(``U1`
+M``0D0O__C[\```/@``@GO0`0H*(``S!&`/\D!0`!#``Q5@`````0`/_XC[\`
+M`*"B``0P1@#_$`#_^20%``6@H@`%,$8`_Q``__4D!0`$)`8``1``__(D!0`"
+M$`#_U@#@&"$GO?_PK[$`!*^_``BOL```C((7Q`"`B"$40``&C)`%N(^_``B/
+ML0`$C[````/@``@GO0`0#``R>P````".`P`,`B`@(20%``(`8A@AK@,`#`P`
+M+H$``#`ACB(``#0$@`".!0!``$08(8QG`2R,9@$T`.40*Q1``%$`````C@,`
+M1`##$"M00``.`,,P(XX#`$0`PQ`K4$#_XX^_``B.(@```$00(:Q#`32.(@``
+M/`,"``!$$"&L0P$X$`#_VH^_``@`Y2@CK@4`.*X&`#R.(Q>\C@0`#``#$(``
+M0Q`A`$00*Q!``!L`9!`KC@(`%#P#$&(T8TW3<((0`@!#`!D``!`0``(1@@!%
+M$"L40``)`````(X"`!QP@A`"`$,`&0``$!```A&"`$80*Q!```4`````#``O
+MAP(@("$0`/^ZC[\`"`P`,*0"("`A$`#_^0`````00/^S/`,08HX"`!`T8TW3
+M<((0`@!#`!D``!`0``(1@@!%$"L40``-`````(X"`!AP@A`"`$,`&0``$!``
+M`A&"`$80*U!`_Z./OP`(#``P)`(@("$0`/_B``````P`+[@"("`A$`#_W@``
+M``"L90$LCB(``#P#``(`1!`AK$,!,!``_Z^.`P!$)*7__RRB``4GO?]P`(!H
+M(1!``%(`P&`A/`.``@`%$(`D8PVL`$,0(8Q"````0``(`````#P"@`(\!H`"
+M)$,-*(VE``",1`THC&(`!(S##3`T"H````Q(@`$]2"&OH@`$`*HH(:^C``BO
+MI```C2,``(RD&%`\`O`/-$+__P""("0``QT`/`(/\`!B&"0`@R`E/`B``B3&
+M#3"LI!A0C,(`!(T##3B-I@``KZ(`#*^C`!``RC`AC2(`"(S%&%P\`__`/`0`
+M/S1C?_\TA(````(3P`!$$"0`HR@D`*(H)3P'@`(E"`TXK,487(SC#4"-`@`$
+MC:4``*^C`!BOH@`4`*HH(8TB`!",I!A<)`.`?P`"$<``@R`D,$)_@`""("6L
+MI!A<).<-0(SC``2-I0``/`+\`Z^C`!P`JB@AC*086(TC`!@T0O__`((@)``#
+M'(`\`@/\`&(8)`"#("6LI!A8`^``"">]`)`\`H`"/`B``B1##4B,1`U(C&(`
+M!(T##5``!F`KC:8````,6(`!?5@A-`Z``*^C`"BOH@`DKZ0`(`#.,"&-8@`@
+MC,48;#P#_^`\!``?-&,__S2$P````A.``$00)`"C*"0`HB@EK,48;#P'@`(E
+M"`U0C0(`!(SC#5B-I0``KZ(`+*^C`#``KB@AC*08;(UC`"@\`O`?-$+__P""
+M("0``QU`/`(/X`!B&"0`@R`EK*08;#P&@`(DYPU8C.(`!(S##6"-J```KZ(`
+M-*^C`#@!#D`AC6,`,(T$&&@\!X`"/`+_`3P)@`(DY0UH-$+__R3&#6",Q@`$
+MC*4`!"4J#7",YPUH`((@)(TI#7`\`@#^``,<0`!B&"0`@R`EC4H`!*^F`#RO
+MI0!$KZ<`0*^I`$BM!!AHC:4``(UD`#@\`H#_`*XH(8RC&&@T0O__``0F``!B
+M&"0\`G\``((@)`!D&"6LHQAHC:4``(UD`$`D`__@`*XH(8RB&&@PA``?KZH`
+M3`!#$"0`1!`EK*(8:(VD``"-9@!()`/`_P".("&,@AAL``8R`##&/P``0Q`D
+M`$80):R"&&Q1@``(C:(``(VC````;A@AC&(8;#1"``&L8AAL$`#_B@`````D
+M!/_^`$X0(8Q#&&P`9!@DK$,8;!``_X,`````/`*``B1##7B,A0``C&8`!(Q$
+M#7@`#!@K-`*````#&(``HB@A`'T8(:^D`%"OI@!4C&0`4(RB(@@D`__`,(0`
+M/P!#$"0`1!`EK*(B"!``_VX`````/`*``B1##8",1`V`C&4`"(QB``2-I@``
+M``P8@*^D`%BOH@!<KZ4`8`!]&"$T`H```,(P(8QB`%B,PQA8/`3__#P%``,`
+M`A,`-(0/_S2E\```9!@D`$40)`!B&"6LPQA8$`#_5``````\`X`")&(-C(Q$
+M``2,:@V,C$L`'(Q&``B,1P`,C$@`$(Q)`!2,0@`8C:4````,&("OI`!LKZ(`
+M@`!]&"$T`H``KZH`:*^F`'"OIP!TKZ@`>*^I`'ROJP"$`*(H(8QB`&B,HQDD
+M)`3_`0`"$$``9!@D,$(`_@!B&"6LHQDD$`#_,P````",A06XC(,``#0"@`",
+MI@`T`&(8(8QJ`.PD!/__C&D`\(QG`/A0P``0K*D`,#P(OJ8`YA`K`.H8(Q1`
+M``HU"')1C*(`+(RD`#``:1@C`&88(P!B&"$`9!@A`&@`&0``&!```R/"K*D`
+M,*RG`#2LJ@`L`^``"`"`$"&,@P``-`*``#"E``<`8A@AK&4`6*R%&T@#X``(
+MK(`;3(R"&_0`@&`A,,8W`(Q"````H&@A)8X;:(Q$``0D`A4`$,(`-0``&"$L
+MPA4!$$``!P`#$$`D`A(`$,(`,20"%`!0P@`!)`,``@`#$$``1!`AE$4`EI1&
+M`+241`"0E$@`G)1#`*Z41P"ZE$D`HI1*`,``!2F`E$L`QI1"`*@`A2`E``8Q
+M@``(0P``9A@E``<[``"(("4`"4R```(6``!G&"4`"E2``(D@)0!J&"4`"UX`
+M`((@)20"``$1H@`-`&LH)3FB``(`HB`*C8(``#0#@`"MS0`T`$,0(:Q$&6"-
+M@@```$,0(:Q%&60#X``(`````!``__4`@"@A$`#_TR0#``(0`/_1)`,``2>]
+M__"OOP``C*H!$``'/@`Q"`#_``<^`Q%(`!DDJ0"4K*@!$(R"!9@D0@`!K((%
+MF"0"``&A(@`_C((;4!1```,D`@`"4,(`!($B`":/OP```^``"">]`!`D`P`!
+M`&@P"B1"``(`XA`J%$#_^`#`*"$,`#*9`````!``__6/OP``C((;4!1`__*/
+MOP``D2,`/X$B`#\D9@`!``8>`"A"``,00/_K``,>`R0"``$D!0`"`$HH"B0"
+M``,48O_EH28`/Q``_^L`````E(,;<B0"%``P8S<`$&(``P`````#X``(`*`0
+M(1"F__T`````$*``!8R#```D`@&,K&(0<!``__<`````$`#__"0"`W`0H``U
+M`(`X(1#``"^,@@``-`.```!#$"$D`P`?K$,8<(SB```T!8``)`,``P!%$"&L
+M0QGPC.0``#P#_/\T8___C()`$#1"``*L@D`0C.0``(R"0!``0Q`D/`,"``!#
+M$"6L@D`0C.0``"0#P'\`A2`AC((`'`!#$"0T0@"`K((`'(SB```D`P`]`$40
+M(:Q#`02,X@``)`,`-`!%$"&L0QATC.(``"0#``T`11`AK$,8>(SB```D`P`_
+M`$40(:Q#&?0#X``(`````#0#@```0Q`A$`#_TB0#``^,A```/`/\_S1C__^,
+M@D`0-`6```!#$"2L@D`0C.0``"0#__V,@D`0`$,0)*R"0!",X@``)`,`#`!%
+M$"&L0QGPC.(``"0#``$`11`AK$,!!(SD```D`\!_`(4@(8R"`!P`0Q`D-$(3
+M@*R"`!R,X@``)`,`?P!%$"&L0QATC.(``"0#``X`11`AK$,8>(SB```D`P#_
+M`$40(:Q#&?00P``&C.(``"0#`!\`11`AK$,8<`/@``@`````)`,`#Q``__L`
+M11`A)[W_\*^R``BOL0`$K[\`#*^P``",@@``-`.```"`B"$`0Q`AC$(`S#!"
+M``$00``'``"0(8^_``R/L@`(C[$`!(^P```#X``()[T`$(XC```T$(``)`0`
+M/`!P&"&,8@#,-$(``:QB`,P,`&D_`````(XB````4!`AC$(`S#!"``$00``-
+M)!``"0P`:3\D!``"CB(``#0#@``F$/__`$,0(8Q"`,PP0@`!$$```R0"__\6
+M`O_U`````"92``$J0@`"5$#_XXXC```0`/_;C[\`##P+_]\\#=`^/`S0/CP*
+M&2\\"1DK-`B``#0'@``U:___-:U&2#6,9X@U2K44-2FU%!"@`!0`@#`AC(,`
+M``!H&"&,8@#`-$(`@*QB`,",@P``/`0`(`!H&"&,8@`$`$00):QB``2,P@``
+M`$@0(:Q-(@B,P@```$@0(:Q*&7`#X``(K,`;4(R#```D!/]_`&<8(8QB`,``
+M1!`DK&(`P(S#````9Q@AC&(`!`!+$"2L8@`$C,(```!'$"&L3"((C,(```!'
+M$"&L21EP)`(``0/@``BLPAM0C((;])2#&^8D!@`!`(`X(1!F``>,0@``5&``
+M/(SB``",0@`$C$(`-%!``#B,X@``C.(``#0%@``D`P`/`$40(:Q#&'",X@``
+M)`,`#93D'/8`11`AK$,8=(SB```D`P`,,(0`\`!%$"&L0QAXC.(``"0#``4L
+MA``P`$40(:Q&&?",X@```$40(:Q#&?04@``<C.(``"0#`!0`11`AK$,9^(SD
+M```T!H``)`/`?P"&("&,@@`</`7\_S2E__\`0Q`D-$(`@*R"`!R,X@``)`,`
+M/0!&$"&L0P$$C.,``(QB0!`T0@`"K&)`$(SC``",8D`0`$40)*QB0!`#X``(
+M`````"0#`!@0`/_E`$40(30%@``D`P`/`$40(:Q#&'",X@``)`,`?Y3D'/8`
+M11`AK$,8=(SB```D`P`.,(0`\`!%$"&L0QAXC.,``"0"``PLA``P`&48(:QB
+M&?",X@``)`,`_P!%$"&L0QGT%(``!HSB```D`P`4`$40(:Q#&?@#X``(````
+M`"0#`!@0`/_[`$40(8R"&_24@QOF/`C\_R0)``$U"/__-`>```"`,"$0:0`'
+MC$(``!1@`#D`````C$(`!(Q"`#000``U`````(S#``"4Q1SVC&)`$#"E`/`L
+MI0`P`$@0)*QB0!",Q```)`/__8R"0!``0Q`DK()`$(S"```D`\!_`$<0(:Q)
+M`02,Q````(<@(8R"`!P`0Q`D-$(/@*R"`!R,P@``)`,`#P!'$"&L0QAPC,(`
+M`"0#`'\`1Q`AK$,8=(S"```D`P`.`$<0(:Q#&'B,P@``)`,`#`!'$"&L0QGP
+MC,(``"0#`/\`1Q`AK$,9]!2@``:,P@``)`,`%`!'$"&L0QGX`^``"``````D
+M`P`8`$<0(:Q#&?@#X``(``````"`."$``#`A),(``0#F("$P1@#_+,,`("0"
+M__\48/_ZH((`!)#B``,D0O__,$8`_R0"`/\0P@`1``800`!&$"$``A#``$<0
+M(21"`"B00P`(),7__P#C("&@A@`$D$(`"0!B&"4`XQ@AH&8`!"0"`/\PI@#_
+M%,+_\0`&$$`#X``(`````">]__"OL```/!"``B80-LBOL0`$K)`<!`"`B"&O
+MOP`(#``TGP(`("$\`H`")$(]0`!`("$,`#2?KB(<"#P"@`(D0C.,`$`@(0P`
+M-)^N(AP`/`*``B1".@0`0"`A#``TGZXB'`P"`"`A#``TGZXP'!`\!(`")(1`
+MP`P`-)^N)!P4C[\`"(^Q``2/L````^``"">]`!`GO?_@+*(`!:^S``ROL```
+MK[\`$*^R``BOL0`$`("`(1!``,0D$P`6/`.``@`%$(`D8PX$`$,0(8Q"````
+M0``(`````(R$```T`H``/`,`!`""("&,@@`$`$,0):R"``00P``/``"8(8X%
+M```D`]__/`3__(RB0!`TA/__`$,0)*RB0!".!0``/`,``HRB0`0`1!`D`$,0
+M):RB0`0``)@A`F`0(8^_`!"/LP`,C[(`"(^Q``2/L````^``"">]`"!0P``:
+M``"8(8R$```\`__\-&/__XR"0`0``(@A`$,0)*R"0`2.`@``/`,``8Q"0!``
+M0Q`D4$``#0``F"$,`&D_)`0`R(X%```\`__\-&/__XRB0`0F,0`!*B0`%`!#
+M$"2LHD`$5(#_\(X"```68/_<-`*``(X#```\!/_[`&(8(8QB``0TA/__`$00
+M)*QB``00`/_4`F`0(8R$```T`H``/`,`!`""("&,@@`$`$,0):R"``10P/_*
+M``"8(8X$```\`__\-&/__XR"0!`T0B``K()`$(X$``",@D`$`$,0)#P#``$`
+M0Q`EK()`!!``_[P``)@A$,#_NP)@$"&,AP``)`+__20%``&,XT`,)`8``0``
+MB"$`8A@DK.-`#(X#``",8D`0-$(@`*QB0!`,`#,F`````(X$```\`__\-&/_
+M_XR"0`0`0Q`D/`,``0!#$"6L@D`$C@(``#P2``&,0D`0`%(0)%1`_YX``)@A
+M#`!I/R0$``J.!```/`/__#1C__^,@D`$)C$``2HE`9``0Q`D`%(0):R"0`2.
+M`P``C&)`##1"``&L8D`,5*#_ZXX"```0`/^,`F`0(1#`_XDD`O_^C(4``#P$
+M__R,HT`,-(3__P``B"$`8A@DK*-`#(X#``",8D`$`$00)*QB0`2.`@``/`,`
+M`8Q"0!``0Q`D4$``$0``F"$,`&D_)`0`"HX$```\`__\-&/__XR"0`0F,0`!
+M*B4!D`!#$"2L@D`$C@,``(QB0`PT0@`"K&)`#%2@_^R.`@``C@8``#P#__PT
+M8___C,)`!`(`("$``"@A`$,0)*S"0`0,`#,F```P(1``_UT"8!`A$`#_6B03
+M`!&,@@``K$4`#`/@``@`````C(,``"0"``2L8@`()`(``:R"%^0#X``(````
+M`">]__"OL@`(K[```*^_``ROL0`$C(,``"0"`"``@(`AK&(`"(R"&[0P0@!`
+M$$``!202`^B,@AP<)`,G$#A"``4`8I`*$D``#0``B"&.`@``)`0`"B8Q``&,
+M0@`(,$(`!!!```T``!`A#`!I/P`````",A`K5$#_]HX"```D`@`6C[\`#(^R
+M``B/L0`$C[````/@``@GO0`0K@`7Y!``__F/OP`,)[W_\*^P``"OOP`$C((7
+MQ!1```P`@(`AC@0``#0"@`"/OP`$`((@(8R"`$B/L```)`/_WP!#$"2L@@!(
+M`^``"">]`!`,`#!<`````!``__..!```C(,``#0"@```8A@AC&(`2#1"`""L
+M8@!(`^``"`````",@@``-`.```!#$"&L10!`C((```!#$"&L1@!$`^``"```
+M```LP@`%`(!((1!``#0`H$`A/`.``@`&$(`D8PY@`$,0(8Q"````0``(````
+M`#"E`#\Q`@0`-*,`0(R$````HA@*-`:``#$%"``T8@"``$48"P"&("&M*!M$
+MK(,`/(TB&T2-(P``)`<`(#!%`$``X"`A,$(`@`!F&"$0H``#``(@"CP"`@(`
+M@B`E-((`"`!%(`NL9`$,C2(;1`#@("$P0P!`,$(`@!1@``\``B`*5(``#HTC
+M``"-(@``)`3_[XQ#`#0`9!@DK$,`-(TB```D!/_WC$,`-`!D&"2L0P`T`^``
+M"`````"-(P``C&(`-#1"`!"L8@`T$`#_]8TB``",A!M$C28``#0'@```A2`E
+M,((`/S"%!``T0P!``$48"C1B`(``QS`A,(4(``!%&`N,P@`\C,(!#*TD&T2L
+MPP`\C2(;1(TC```D!@`@,$4`0`#`("$P0@"``&<8(1"@``,``B`*/`("`@""
+M("4T@@`(`$4@"ZQD`0R-(AM$$`#_R`#`("&,A1M$``@0)XR&````HB@D,*(`
+M/S"D!``T0P!``$08"C0'@``T8@"``,<P(3"D"```1!@+C,(`/(S"`0P0`/_>
+MK24;1#"D`#\Q`@0`C24``#2#`$``@A@*-`:``#$$"``T8@"``$08"P"F*"&L
+MHP`\C2,````($((P10`@,00`0`!F&"$0@``#`*`P(3P"`@(`HC`E-,(`"`!$
+M,`NL9@$,5(#_L8TC``!0H/^CC2(``!``_ZV-(P``C(0;1(TF```T!X``,(,`
+M/S"%!``T8@!``&40"C"$"``T0P"``&00"P#',"&LP@`\C2(;1(TC```D!@`@
+M,$4`0`#`("$P0@"``&<8(1"@``,``B`*/`("`@""("4T@@`(`$4@"ZQD`0R-
+M(AM$`,`@(3!#`$`P0@"`%&```P`"(`I0@``,C2(``(TC``",8@`T-$(`$*QB
+M`#2-(@``)`3_]XQ#`#0`9!@DK$,`-`/@``@`````)`3_[XQ#`#0`9!@DK$,`
+M-!``__6-(@```*`P(22E``@PH@`/)`/_\"1"`"<`HR@D`$,X)`"G$"$GO?_P
+M`$40*Z^_``2OL````(!`(11```<`H!@AO+$``"2E`!``9Q`A`$40*Q!`__L`
+M````)-``$)("``<P0@`!$$``2R0$``V,QP`@)`3_\"3C``@P8@`/)$(`)P!D
+M&"0`1"@D`&40(0!#$"L40``'`&`@(;QQ```D8P`0`(40(0!#$"L00/_[````
+M`"3B`!"00@`',$(``51```B@P`!$C0(``(S#`"0D!``-C$(`#!!#`#"/OP`$
+MH,``1)8"``0P0G__I,(`0I8#``"6`@`"``,9`C!"#_^DP@!`D@(`!J##`$D`
+M`A!"H,(`2)("`````A$"H,(`2HX"`````A/",$(`'Z#"`$>2`@`$``(1PJ#"
+M`$N2`@`'``(00C!"``$40``8)0(<&)("``<``A"",$(``11```XD!``;D@(`
+M!P`"$0(P0@`!$$``#B4"'!B2`@`'``(1`C!"``$00``"```8(9(#``8D!``<
+MH,,`18^_``2/L````(`0(0/@``@GO0`0C$,`/!!@`#BLP@!,C@(````"$\(P
+M0@`?`&(0(9!"``2@P@!&D@(`!P`"$$(P0@`!5$``#XT"&U"2`@`'``(0PC!"
+M``$00``@`````)#"`$0T0@`!H,(`1)("``(``A%",$(``11`_^$D!``:C0(;
+M4!!```H`````D,,`2HT"&TA00P`&K0`;3(T"&TPD0@`!+$,``Q!@``>M`AM,
+MD@(``B0$`!T``A$",$(``1``_\\``B`*D,4`2@P`,ID!`"`A$`#_]@````"2
+M`@`'``(10C!"``$00/_@`````)#"`$00`/_<-$(``A``_\T``!`AK(``"*R`
+M``RL@``0K(``%"2$``B,@@`$)`/P`#"E#_\`0Q`D`$40)0/@``BL@@`$C((`
+M`(Q"``@P0@`$`^``"``"$"LGO?^@K[X`4*^W`$ROM@!(K[4`1*^T`$"OL0`T
+MK[\`5*^S`#ROL@`XK[``,(R"&_0`H*`A)(4<&(Q#`!B,0@```("((:^C`!RO
+MI@`(``#P(0``L"&OH``4KZ(`&"27&V@``*@AKZ4`)!*``)(D`___)`8`$`P`
+M'<XD!0`2/`8(```",`J/H@`($$``!*^F`"".(@``C$(10*^B`!2.(@``-`.`
+M``(@("$`0Q`AC$,`6"00``&OHP`0C$,`!#P"`B``8A@D#``T6J^C``P,`$A_
+M`B`@(0(@("$,`#IN`H`H(8^D`!``0!@A`@0@"A1``'&OI``0EH(``HXC&_0P
+M1#<`)`(5`!""`F*L<``4*((5`1!``E8D`B0`)`(2`!""`E$D`A0`4(("320>
+M``26(QSVCB0``#0"@``P8P#P`((@(2QC`#`D`@`'K((8`!!@`>X"("`A#`!)
+M;@*`*"&6)QST+.(`0A1``#`PY/__CB0``):%```T`H``/`,``@""("$T8Z`"
+M)`()M*R#&"P0H@'/CZ,`&(QB``0\`V9F-&-F9Y1"`"0`0P`8``(7PP``&!``
+M`QA#`&(@(Y:"``(D`Q0`,$(W`!!#`;`T`X``CB(```!#$"&L0!E,CB0``#0&
+M@``D`OW_`(8@(8R#(B@\!?_^-*4#_P!B&"2L@R(HCB,```!F&"&,8B(H`$40
+M)#1""`"L8B(HCB(``"0#``\`1A`AK$,`8##D__\L@@!#%$``!BR"`$2.(@``
+M-`.```!#$"&L0")4+((`1%1```Z.(@``CB4``"0#@!\\!/__C*(2,#2$?_\`
+M0Q`D-$(!X*RB$C".(P``C&(`,`!$$"2L8@`PCB(``#00@``D`P`.`%`0(:Q#
+M&'@"("`A`H`H(0.@,"$D!P`_#`!`MB0(``$00``.)`/__X^_`%2/O@!0C[<`
+M3(^V`$B/M0!$C[0`0(^S`#R/L@`XC[$`-(^P`#``8!`A`^``"">]`&"/I``<
+MCZ8`&`/`."&,@@`$C,4`!`(@("$"@#`A`$#X"0.@0"&6@@`",$($`!1``5@"
+M("`ACZ(`&`(@("$"@#`A#``_58Q%``2/HP`($&``!`````".(@``CZ0`%*Q$
+M$4"*XP`ZFN,`/8KB`#J:X@`]BN0`.IKD`#V*Y@`ZFN8`/3!C_P```QH`CB4`
+M```"%@``!"("`$,0)3"$_P"6XP`^`$00)0`&-@(`1A`E`+`H(:RB``"/I@`,
+M,&(`_XXD`````QH"``(2`([E`$@`0Q`E`,(8)20"``(0H@$I`)`@(8^E`"`\
+M`A"``&48)8^F`"0`8A`EK((`!(C#`$*8PP!%B,(`0IC"`$6(Q`!"F,0`10#`
+M*"&(I@!"F*8`13!C_P```QH```(6```$(@(`0Q`E,(3_``!$$"4`!C8"CB4`
+M``!&$"6/I@`D-!"```"P*"&4PP!&K*(`"(XD```P8@#_``(2```#&@(`0Q`E
+M`)`@(:R"``R.(@``EH,``H^D`!``4!`A,&,W`*Q$`%@D`A0`4&(`]HXE'/R.
+M(P``)`+__X^E`!RL8@"`CB0``"0#``J,H@``K(,`%(XF```D`P<``B`@(0#0
+M,"&LPP`8`$#X"0*`*"$40/]]`$`8(8XB&_Q00``/CB4```!`("$``)`AC(4`
+M`"92``$0H``(+D8`@8XB``",@P`$)(0`"`!%$"&L0P``5,#_]XR%``".)0``
+M-`.```"C&"&,8@!8KB`;3#!"`/^N(AM(C&(B"#!"(`!00``#KB`;4"0"``&N
+M(AM0-`*```"B$"&6@P`"C$(9%#!C`@`08`"[,$0__SP#NBX`!!"`-&.+HP!#
+M`!D``!`0``(1`C00@```L!@A)$0`9"0"``&L8A@<#`!I/R02``&.(@``)`,X
+M``!0$"&,4Q@(K$,8"(XB````4!`AC$(<)#!"`!`40`"5`````(XB```T!H``
+MEN0`'`!&$"&L4Q@(CB,```!F&"&,8AA@-$(``:QB&&".(P```&88(8QB&&`T
+M0@`"K&(88%"``(*.(AOTEH(``B0#$@`P0C<`$$,`?"0"``*.)1OTC*,`+!!B
+M`'@D`P`!CB0``*RC`"P`AB`AC((9(#1"\`"L@AD@CB,``#P$``$`9A@AC&(9
+M(`!$$"6L8AD@#``].0(@("$``)`ACB0``"0#``$"0Q@$`!(0@"92``$`1!`A
+M+D0`"JQ#$`!4@/_XCB0``(XC```\`@`(-$((H*XB&T"L8@"@CB4``#P#``<\
+M!@"`C*(`K`(@("$`0Q`EK*(`K(XB&T".(P```$80):XB&T",8@"@`$80):QB
+M`*".)0``/`,@`(RB`*P`0Q`EK*(`K`P`+'8`````%$``/P```````)`ACB(`
+M`#00@``D!``*`%`0(8Q"&&`P0@`!$$``!B92``$,`&D_`````"Y"`&140/_V
+MCB(```P`,_8"("`ACB,``#0"@``\!/Y_`&(8(8QB`"`TA/__`$00)*QB`"".
+M(ANT,$(`0!1``!\"@"@A`B`@(0P`,\$``"@AEB,<\"QB``900``.CB(``"0"
+M``508@`(EB(<\HXB```T`X```$,0(20#`%*L0P$($`#^Q0``&"$L0@`(%$#_
+M^(XB```\`P`!-&,`J@!0$"&L0P$8CB(``"0#,A``4!`AK$,!'!``_^Z.(@``
+M#``Z*P(@("$0`/_@`B`@(0P`++@"("`A$`#_P```D"&.(AOT$`#_DZQ``"P,
+M`&D_)`0`R"Y"`!000/]H)E(``8XB```T`X```$,0(8Q"'"0P0@`0%$#_]0``
+M```0`/]@CB(``#P"S,PT0LS-`((`&0``$!`0`/]'``(0P@(@("$,`#,5+*8`
+M`1``_PB.(P``CZ(`(`!B&"40`/[8/`(0@@P`0&X"@"@A$`#^IX^B`!B/HP`8
+MCB4````$(".,8@`$``0D@#P#`/R`0@`B`(,@)#0#@````A`C``(1@#!"#\``
+MHR@A`$00):RB&4P0`/Y%CB0``(^F`!@\`F9F-$)F9XS#``2$9``FE&,`)`!D
+M&",`8@`8``,?PP``$!```A!#`$,0(P`"%``0`/XM``(D`PP`:3\D!``!`!Z`
+M@```F"$D$@`H/`.``B1C%[P`$Q#``$,0(8Q$```"`Q@ACB(``(QE```FM0`!
+M`$00(3*C`#\F4O__)A``&"9S``,D!``!K$4``!!@`#L`````!D'_[CP#@`(,
+M`&D_)`0``0``@"$\`H`"`!`8P"1"&Y0`8A@AC&0``(XB``",90`$)K4``0!$
+M$"$RHP`_)A```20$``&L10``$&``(P`````J`@$>%$#_\#P"@`(,`&D_)`0`
+M`0`6@(```)@A)!(`/SP#@`(D8R2$`!,0@`!#$"&,1````@,8(8XB``",90``
+M)K4``0!$$"$RHP`_)E+__R80``PF<P`#)`0``:Q%```08``%``````9!_^X\
+M`X`"$`#]RP(@("$,`&D_`````!``__D`````#`!I/P`````0`/_<*@(!'@P`
+M:3\`````$`#_PP`````0`/VS)!8``A``__TD'@`#$((`!B0")0!4@OVNEB,<
+M]B0>``(0`/VJ)!8``1``__TD'@`!$`#_\B0>``4`@#@AC(0``#0&@``D`__\
+M`(8@(8R"`,"4J``"`$,0)#1"``*L@@#`C.0``"0#_\,Q!2```(8@(8R"`,``
+M0Q`D-$(`&*R"`,",X@```$8P(8S"`,@P0___/`(G$!2@``,`8B`E/`(J^`!B
+M("4Q`B``,$7__S$""``T"8``K,0`R!!``!DD`\``C.0```")("&,@@`4`$,0
+M)#1#$3`T0@^@`&40"JR"`!2,X@```$D@(8R"`,0P0___/`(%>!2@``,`8A`E
+M/`(&!`!B$"6L@@#$)`,$L"0"!2@Q!"```$08"HSB``"L0Q!P`^``"```$"$G
+MO?_PK[\```P`.\\D!0`"C[\```/@``@GO0`0)[W_X*^V`!BOM0`4K[0`$*^S
+M``ROL@`(K[```*^_`!ROL0`$`*"8(0"`@"$D%@`!``"0(0``J"$0H``F``"@
+M(92E``*4@ASV,*,W`#!"`/`X8Q0`-*0"`"Q"`#`40``&`(,H"C"C`@`D%`#K
+M)`(`Z@!#H`HD$@`(,*,&`"0"!@`08@!6,*,$`#9"``$`0Y`*,*(@`#9#``(P
+MI`@`$(``30!BD`J.`ANT,$(`0%!```2N`!V()`(``392`!"N`AV(,*,!`"0"
+M``,``*@A`$.H"P``L"$"`"`A#``[SR0%`!,00``,`$"((0(@$"&/OP`<C[8`
+M&(^U`!2/M``0C[,`#(^R``B/L0`$C[````/@``@GO0`@`@`@(0``*"$,`#3E
+M)`8``11`__``0(@A`@`@(0P`.\\``"@A%L#_ZP!`B"&690`"C@(``#03@``P
+MHP(``%,@(8R&&'P48``1,*,W`"0"%``08@`.`````%#4``6.`@``K)08?`P`
+M:3\D!`$LC@(```!3$"&L4B(`C@(```!3$"&L51@$$`#_U`(@$"&LE1@$C@(`
+M``!3$"&L4B(`$-3_S20$`2R.`@```%,0(:Q4&'P,`&D_`````!``_\<"(!`A
+M$`#_NJX`'8@0`/^L-E(`!">]__"OL0`$K[```*^_``B,BQOT)`(``0"`@"&-
+M8P`L`*"((1!B`#\DA!MHEB(``B0#$@`P0C<`$$,`!"0"``*-8P`L4&(`(HX%
+M```"("@A#``]!`(`("&6)0`",*(``11``!4D`P`8C@,``#0"@``PI"```&(8
+M(8QB&&`T0@`"K&(88!2```D`````E@(<]C!"`/`L0@`P%$``!P``&"$PH@0`
+M$$``!8^_``@,`$AQ`@`@(0``&"&/OP`(C[$`!(^P````8!`A`^``"">]`!`T
+M!H``D(0`4P"F*"&,HQD@/`+__S1"#_\`!",``&(8)#"$\```9!@EK*,9((X$
+M```D`@`!K6(`+`"&("&,@QD@/`(``0!B&"6L@QD@$`#_S`(@*"&.`@``-`.`
+M``!#$"&,0AD@/`,``0!#$"140/^\EB(``JU@`"P``"`AC@,``#0"@``DA``!
+M`&(H(8RJ'!`LAP`*C*8<%(RH'!@10``#`````!3```@`!AA"C*(9(#P#``$`
+M0Q`EK*(9(%3@__".`P````880@`*$$(`0Q`A``(9PA!@_ZD`!B'"*((``E1`
+M_Z<"("@A``@0(P!#`!I08``!```!S8X(```T"8``)`/X'P$)0"&-!QD@`.,X
+M)"0#``*M8P`L)`,`'P``$!(``A8```(6`P%$`!LH1?_@*$8`(`!F$`HXI0``
+M)`/_X`!E$`L``A%`,$('X`#B."6M!QD@C@<``"0"_^``Z3@AC.89(`#","0D
+M`@`/```@$B2$_X`H@__P*(4`$`!%(`HX8P``)`+_\`!#(`LPA``?`,0P):SF
+M&2".`P```&D8(8QB&2`T0@@`K&(9(!``_W0"("@A)[W_X*^S``ROL@`(K[$`
+M!*^P``"OOP`0C(,```"`B"$D$___C&(`""00``8``)`A-$(`(*QB``B,@P``
+MC&((@#1"`_^L8@B`,@(`!!!```DR`@`"CB(``(Q"``@P0@`$%$``!#("``(D
+M`O_[`@*`)#("``(00``*,@(``8XB``",0@A`,$(#_Q1```4R`@`!)`+__0("
+M@"0V$``!,@(``1!```T``#`ACB0`````*"&,@@H`)*4``2RC``HP0@`#`,(P
+M(11@__HDA``$%,```B0"__X"`H`D$@``#B0$`#(,`&D_)E(``2Y"``H40/_7
+M,@(`!`)@$"&/OP`0C[,`#(^R``B/L0`$C[````/@``@GO0`@$`#_]P``F"$G
+MO?_@K[,`##"B`!$D$___K[$`!*^P``"OOP`0K[(`"`"@@"$`@(@A$$``)P"E
+MF`L``"@A#``TY20&``$00``)`$`8(8^_`!"/LP`,C[(`"(^Q``2/L````&`0
+M(0/@``@GO0`@CB(```(@("&L0``DCB(``(Q"`"0,`#N(`````!!``$HD`O_O
+M#`!I/R0$``^.(@``)`0``380`!.,0@`,#`!JO``````,`&G3``````P`::L`
+M````#`!I/R0$``H,`&J\```@(0P`:3\D!``/CB(``"0$``\R<P`#C$,`#*Q0
+M0``,`&D_,A```P``D"&.(@``)E(``0(@("&,0D`````H(0!3$"004``2)`8`
+M`1(```@`````#`!I/R0$`#(N0B<05$#_\XXB```0`/_%)`,`%@P`-.4`````
+M%$#_]@````".(@``K$!``!``__(`````#``TY0````!40``$CB(;](XB``",
+M0@#`CB(;](Q#`%PD`@`#4&(``XXD```0`/^P```8(3P#__$T8___C()`$`!#
+M$"0\`P`(`$,0):R"0!`0`/^G```8(20$``\,`&D_`@*`)(XB``",0@`,$`#_
+MP0`````GO?_@K[0`$*^S``ROL@`(K[$`!*^_`!2OL```E*,``I2E````@)`A
+M,&(@````H"$``)@A$$``LC"Q__^,@AOTE$0`=#!B(``00`"KCD(;])1#`'8P
+MI?__`*00*Q1``#LD`@`.`&40*Q1``#@D`@`.+B(2P!!``%4\`\S,/`5F9B8D
+M]W`TI69G`(4`&``$%\,``!@0``,80P!B&",``Q"``$,0(12"`#DF)/=0`!$0
+M0"1"[N``10`8``"@(0`"%\,``!@0``,8@P!B@",`$!"`,%``_P(`("$,`%B4
+M)`4`"`!`@"$D`@FT$B(`(#0"@`".1```)`/_[P""("&,@B($`$,0)*R"(@0`
+M$QB``!`1``!#$"4`%"!`CD,```!$$"4T11`!-`2``#"B`/\`9!@AK&(8G(Y"
+M````!1H",&4`?P!$$"&L11C8```0(8^_`!2/M``0C[,`#(^R``B/L0`$C[``
+M``/@``@GO0`@CD(``#0#@```0Q`AC$,B!#1C`!"L0R($$`#_X0`3&(``A0`8
+M``07PP``&!```QA#`&(8(P`#$(``0Q`A5(+_Z"0"``X`$1!`)$+NH`!%`!@D
+M%``!```8$!``_[\``A?#-&/,S0(C`!D``!@0``,9`@`#$(``0Q`A``(0@`(B
+M$",P0O__%$``!#P"S,PN(A0`$$``,#P"S,PT0LS-`B(`&0``(!``!!C"``,0
+M@`!#$"$``A!``B(0(S!"__\00``7``08@@`#$(``0Q`A`B(0(S!"__\40/_"
+M)`(`#CP$9F8F(NU`-(1F9P!$`!@``A?#)`4`"```(!``!"!##`!8E`""(",`
+M0(`A)`0``0P`6)0D!0`"$`#_H0!`F"$\!&9F)B+M0#2$9F<`1``8``(7PR0%
+M``@``"`0``0@@P""(",,`%B4``0@0`!`@"$0`/_O)`0``CP$9F8F(NU`-(1F
+M9P!$`!@``A?#)`4`"```(!``!"##`((@(PP`6)0`!""``$"`(1``_^$D!``#
+M$`#_5I1#`'J,@AOT$`#_3Y1$`'B4IP`")`(5````,"$PXS<`$&(`)0``0"$H
+M8A4!$$``)B0")``D`A(`$&(`(20"%`!08@`!)`8``HR#```T`H```&(8(8QB
+M&&`P0@`"%$``%`````",A!OTC&(89``&&$",A`````(4PC!&`?^,@@`$,,0!
+M``!B&"$0@``#A&,`<CC"`?\``C`G`&80*E!```2DI@`*`0<0):2B``*DI@`*
+M`^``"``````0`/_C)`8``A``_^$D!@`!$&(``R0")0!48O_>C(,`````,"$0
+M`/_:)`@``8R"&[2,@P```(`H(0`"$((P0@`!K&($((RB&[R,HP``/`0`$``"
+M$@(P0@`'K&(&`(RC```TA`(!```X(8QB!@`DIAT!-$(`"*QB!@",H@``K$0&
+M!(RB``"L0`8@C*(``*Q'!`"0Q```C*(``"3G``$LXP"`K$0$!!1@__@DQ@`!
+M`^``"``````GO?_`K[X`,*^W`"ROM@`HK[4`)*^T`""OL@`8K[\`-*^S`!RO
+ML0`4K[``$)3"``*,@QOT,.?__S!*-P`D`A4``*!((0#`D"$`@*@A`0#P(:^G
+M``",<0`$``"@(0``L"$``+@AKZ``"*^@``P``"@A$4(!-HQS`"`I0A4!$$`!
+M/R0")``D`A(`$4(!-B0"%`!10@$OE2(`]H^B```\!(`"/`.``@`",(``!2A`
+M)(0GC"1C)Z0`J2@A`,0@(0##&"$\`H`"C(0``(QC``"4IP#2C$(GB)2E`(2N
+M9``$KF,`"*^G``2N8@``)F,`#"0$`"<\`H`")$(GO`#"$"&,0@``)(3__R3&
+M`!BL8@``!('_^"1C``0F<``,`@`@(20&``$D!P$N#``^V```0"&7Q0```@`@
+M(20&``(D!P$.#``^V```0"&7Q0`"`@`@(20&``(D!P$!#``^V```0"&60@`"
+M,$($`%1``,Z.(@`<ED(``C!"$``00`#``L`H(8^E``@"`"`A)`8``R0'`1\,
+M`#[8``!`(8^E``P"`"`A)`8``R0'`2(,`#[8``!`(9:C'/8D`@`U$&(`ER0%
+M``(D`@!%$&(`E20&``*/H@``)F,`K"0$``P``C"`/`*``B1"*WP`PA`AC$(`
+M`"2$__\DQ@`8K&(```2!__@D8P`$ED(``C!"!`!40`![CB(`'(^E``0F9`"L
+M)`8`!B0'``X,`#[8``!`(0P`:3\D!``!/`*``HQ#)X2.H@``CF0``":4``$`
+M0Q`A,H,`/ZQ$```08`!E``````P`:3\D!``!/`*``HQ#)XR.H@``CF0`!":4
+M``$`0Q`A,H,`/ZQ$```08`!5``````P`:3\D!``!/`*``HQ#)Z2.H@``CF0`
+M"":4``$`0Q`A,H,`/ZQ$```08`!%`````"0$``$,`&D_)G``#```D"$D$0`G
+M/`*``B1")[P"0A`AC$,``(ZB``".!0``)I0``0!#$"$R@P`_)C'__R80``0F
+M4@`8)`0``:Q%```08``L``````8A__`\`H`"#`!I/R0$``$F<`"L``"0(201
+M``P\`H`")$(K?`)"$"&,0P``CJ(``(X%```FE``!`$,0(3*#`#\F,?__)A``
+M!"92`!@D!``!K$4``!!@`!$`````!B'_\#P"@`*.HAOTC[\`-(^^`#"/MP`L
+MC[8`*(^U`"2/M``@C[,`'(^R`!B/L0`4C[``$*Q``#`#X``()[T`0`P`:3\`
+M````$`#_[0`````,`&D_`````!``_](`````#`!I/R0$``$0`/^Z)`0``0P`
+M:3\D!``!$`#_J0`````,`&D_)`0``1``_YD`````)F0`K"0&``*$10``)`<`
+M)0P`/M@``$`A$`#_@(^E``0D!@`")`<`6B0(``(,`#[8`@`@(0(`("$``"@A
+M)`8``B0'`%P,`#[8)`@``@(`("$D!0`")`8``B0'`%X,`#[8)`@``@(`("$`
+M`"@A)`8``20'`/X,`#[8)`@``A``_U:/H@``)`8``R0'`1<"`"`A#``^V```
+M0"$"`"`A`N`H(20&``,0`/]")`<!&@(`("$D!@`!A$4``B0'`*@,`#[8)`@`
+M`XXB`!P"`"`A)`8``81%``0D!P"I#``^V"0(``..(@`<`@`@(20&``&$10`&
+M)`<`J@P`/M@D"``#CB(`'`(`("$D!@`!A$4`""0'`*X,`#[8)`@``XXB`!P"
+M`"`A)`8``81%``HD!P"O#``^V"0(``..(@`<`@`@(20&``&$10`,)`<`L`P`
+M/M@D"``#$`#_"I9"``*5(@#VE2<`^"0%``*OH@`($`#^SJ^G``R5(@#RE2<`
+M]"0%``&OH@`($`#^R*^G``P10@`#)`(E`!5"_L6/H@``ED,``"1B\%\L0@3K
+M$$``!21BZW25-@#BE3<`Y!``_KL``"@A+$(`\!!```0D8NJ$E38`YA``__F5
+M-P#H+$(`X1!```0L8A9=E38`ZA``__.5-P#L5$#^K0``*"&5-@#N$`#_[I4W
+M`/`GO?_@K[(`"*^P````@)`A`."`(0"@("$`P"@AK[,`#*^Q``0!`)@A)A#_
+M_Z^_`!`,`%B4`,"((0!`0"$R!P`'&B``'@`0$,(``A"``%*`(0#Q("$D!@`(
+M*((`"0#"(`HD`@`!`.(8!`""$`0D8___)$+__XX%````0Q`F`!,8P`!B$`0`
+MZ"`$`&0@!``"&"<`@B`D`*,H)`(G$"$`QS`C`*0H)211__BN!0```,A`!@``
+M."$>(/_F)A``!(^_`!"/LP`,C[(`"(^Q``2/L````^``"">]`"`DQO__)[W_
+M\*^_````H%`A,,@`!P``8"$`!A#"`*!((1B@`!P``%@A``(0@`!$,"$!"2@A
+M**,`"20"``@`0R@*)`(``0$"(`0`HA`$)(3__XS#```D0O__`$00)@`'(,``
+M@A`$`&(8)`"#&`8!*!`A`0,8!@%C&`0`J"@C)$G_^`&#8"4!95@A``!`(1T@
+M_^@DQ@`$`8`@(0P`6)0!0"@AC[\```/@``@GO0`0)[W_\"0$`."OOP`$K[``
+M``P`>E<`H(`AK@(`((^_``2/L`````(8*P!@$"$#X``()[T`$">]__"OL```
+MK[\`!(RB`"``H(`A%$``!0!`("&/OP`$C[````/@``@GO0`0#`!Z:@`````0
+M`/_YK@``(">]_]"OO@`@K[8`&*^T`!"OLP`,K[(`"*^Q``2OOP`DK[<`'*^U
+M`!2OL```C((;]`#`\"&4Q@`"C$,```"`B"$PPO__``(B`@`"$P(P4P`!,,@W
+M`"0"%0``H)`AE'<``#"4``$1`@#U``"P(2D"%0$00`#V)`(D`"0"$@`1`@#Q
+M)`(4`%$"``$D%@`"CB<``#0"@```%JA``.(X(0*R@"&,XAD0E@,`BB0$_`8`
+M1!`D``,9``!#$"6.)1N<-$(``0(@("$PQO__K.(9$`P`,J$`````*NI0`!5`
+M``:6!``\4H``!8XE````$Q!``%(0(91$`$*.)0``-`*````$(<``HB@AC*(8
+M1"0#P'\PA#^``$,0)`!$$"6LHAA$`M(P(15```6`Q0!H4H``!(XD```"<A`A
+M@$4`:XXD```T`H``)`/_``""("&,@AA0,*4`_P!#$"0`11`EK((84!5```6`
+MQ@!M4H``!(XE```"<A`A@$8`<(XE```T`H``/`3__P"B*"&,HQA0``82`#2$
+M`/\P0O\``&08)`!B&"4"LA`AK*,84!5```:40@!&4H``!8XG````$Q!``%(0
+M(91"`$R.)P``-`B```*R2"$`Z#@AC.082#P%__P\!@`#E2,`7#2E#_\TQO``
+M``(3``!&$"0`A2`DE28`8@""("6LY!A(/`(`_P`#)`".)0```((@)``#'@``
+M!A(``&08)3!"_P``8A@E,,8`_P!F&"4`J"@AK*,8-(XE``"5)`!0/`/__P"H
+M*"&,HA@H-&,`_P`$(@``0Q`D`$00):RB&"B.)@``E2,`5CP%__@`R#`AC,(8
+M9#P$``<TA/````,;`#2E#_\`9!@D`$40)`!#$"4JXS`#K,(89!1@``LD!``"
+ME\(``#!"`!]00``(CB4``"1"__8L0@`-5$``!(XE``"5(@#,)$0``HXE```T
+M!H``)`/_`0"F*"&,HADD``0@0#"$`/<`0Q`D`$00):RB&22.)0```!,00`!2
+M0"$E!P$``*8H(83B``*,HQD@)`3X'P`"$4``9!@D,$('X`!B&"6LHQD@CB4`
+M`(3D``8D`__@`*8H(8RB&2`PA``?*N=``0!#$"0`1!`EK*(9((XC````9A@A
+MC&(9(#1""`"L8AD@%.``&"KB4`$"LA`A%4```Y1$`-A6@``!E00`WI?"``(D
+M`Q4`,$(W`!!#`"D`````CB8``#0"@``\!?\#`,(P(8S#(@P`!"2`-*7__SP"
+M`/P`91@D`((@)`!D&"6LPR(,*N)0`11```J/OP`DED(`.#!"`@`00``1CB(`
+M`#0#@```0Q`A)`,!_ZQ#&>"/OP`DC[X`((^W`!R/M@`8C[4`%(^T`!"/LP`,
+MC[(`"(^Q``2/L````^``"">]`#`T`X```$,0(:Q`&>`0`/_QC[\`)!5`_^0J
+MXE`!$`#_UHXF```0`/\3)!8``A``_Q$D%@`!$0(``R0")0!5`O\.CB<``!``
+M_PL``+`AE*8``I2E```\`F0`,,8!`#P#R```9A`+`$4`&U"@``$```'-`(!0
+M(20(`!\``$@2`0D0!C!"``%40``%C48``"4(__]5`/_[`0D0!HU&```D`@`F
+M`$A`(S0'@``D`P`7`,<P(0!H&",D`@`!C,48%`!B$`0D`P`8`&@8(P$B$"$\
+M!``!`&(0!C2$__\`I"@D``(40`"B*"6LQ1@4C48``"4#__`\!?_^`,<P(8S"
+M&!0\!``!``,;0#2E'_\TA.```$40)`!D&"0`0Q`EK,(8%`/@``@`````/`*`
+M`B>]_\`D0@]\`Z`8(21$`$",10``C$8`!(Q'``B,2``,K&4``*QF``2L9P`(
+MK&@`#"1"`!`41/_V)&,`$`/@``@GO0!`)[W_$*^^`.`GO@`0``<\`*^W`-RO
+MM@#8K[(`R*^Q`,0`H)`A`("((0``*"$#P"`A`,"X(2>V`)`D!@"`KZ@`M*^_
+M`.2OM`#0K[``P*^U`-0`!X0##`!PJJ^S`,P"P"`A```H(0P`<*HD!@`@ED(`
+M`HXC&_0``$`A,$(@`(QT```00`#\KZ``N(8B&\"6(QO``@`@(0!0$"I40``!
+M`&`@(0`$%````H0#*@(`0!1```("`#@A)`<`/P`'%````H0#*@(`/U!```26
+M(QO$%@``"">U`+*6(QO$/`*``B1"+,```QA``&(8(81H```GM0"RK[4``(Z&
+M``PGLP"P`@!((0(@("$"P"@A`D`X(0*`4"$,`$4$`F!8(3P"@`*,0D/\6$``
+M!(XB&_0,`$"D`L`@(8XB&_0"8#`A`J`X(8Q"`!@"X$@A`B`@(8Q"``@GI0`0
+M`$#X"0)`0"&6(QSVEZ(`KC!C`/`D0@`$+&,`,!1@`+^GH@"NEZ,`LB0"`#\#
+MP"`A`$,0(S!%__\``#`AE(,`@"3&``$HQP`0`*,8(3!B__\L0@!`%$```Z2#
+M`(`D`@`_I((`@!3@__4DA``"CB(;](Q#``"D10`DC&(`!)1"`"@L0@`"$$``
+M)SP"@`*6(AST+$(`0A!```>/H@"TED(``B0#%``P0C<`4$,`?)9#``"/H@"T
+M%$``!30%H8"/H@"X$$``&#P"@`(T!:&````P(0`&$(`GHP`0`$,0(91#``"4
+M0@`"CB0````#&@```A(`-$(`_S1C`/\``A0`,&/__R3&``$`0S@E`(4@(22B
+M``0HPP`@K(<``!1@_^TP1?__/`*``HQ"0_PH0@`"$$``5P````"7HP"6EZ0`
+ME)>B`)(P8P`_,(0`/P`$)````QX`,$(`/P!D&"4``A(``&(8)9>B`)"7I`"<
+MCB4``#!"`#\`8C@EEZ,`GI>B`)HPA``_,&,`/P`$)````QX`,$(`/P!D&"4`
+M`A(``&(8)9>B`)@T!H```*8H(3!"`#^LIQDTEZ0`H@!B."67HP"DEZ(`KC"$
+M`#\P8P`_``0D```#'@`P0@`_`&08)0`"$@".)0```&(8)9>B`*``IB@AK*<9
+M.#!"`#\`8C@EEZ0`JI>C`*R7H@"HCB4``#!C`#\PA``_``0D```#'@`P0@`_
+M`*8H(0!D&"4``A(`K*<B-`!B&"67H@"FCB4``(^_`.0P0@`_`&(X)0"F*"&L
+MIR(XCB,``(^^`."/MP#<C[8`V(^U`-2/M`#0C[,`S(^R`,B/L0#$C[``P```
+M$"$`9A@A)`0`/ZQD&3P#X``()[T`\`P`0*0GI`"0$`#_J)>C`)8D`@FT$&(`
+M$#P#9F:.@@`$-&-F9Y1"`"0`0P`8``(7PP``&!```QA#`&(X(P+`*"$"("`A
+M#`!&;2>F`!`D`@`!$`#_=*^B`+B.@P`$/`)F9C1"9F>$9``FE&,`)`!D&",`
+M8@`8``,?PP``$!```A!#`$,0(P`"%``0`/_L``(\`Q``_U,``"@AAB(;PI8C
+M&\(0`/\%`@`@(2RC`$``H#@A)`(`/P!#.`HGO?_@``<\`"2%&W``!SP#`Z`P
+M(:^_`!`,`$"V``!`(8^_`!`#X``()[T`(#"$__\PI?__,,;__P`'/```"$0`
+M`,1((P"%4",`Q1@C``<4`Q#%``P`""0#`2(`&%!@``$```'-<40`````$!(`
+M``````````!#`!H``!@2``,4```"%`,#X``(`````)2C````!"0`,,;__P`$
+M)`,`!A!``*(0(0!D&"H48``$A$+__JS@```#X``(K0````""$"H40``$),+_
+M_ZSB```#X``(K0(``!#```X``!@A``,00`!%2"&5(@``$$0`#21J``&5(@`"
+M`((0*A1```<D8@`!,4/__P!F$"L40/_U``,00`/@``@`````$`#_[*SC``"L
+MXP```^``"*T#```GO?_0K[<`'#"7__\`!SP`+N(``J^V`!BOM0`4K[0`$*^S
+M``ROL@`(K[$`!*^_`""OL````*"((0#`L"$!`*@A``>D`P``F"$D$@`!%$``
+M-```&"$``(`A`!(00`!1$"&40@```%`0*Q!```H`$Q!`)N+__P)"$"H00``&
+M`!,00"9B``$F0P`!,%/__S!R__\`$Q!``!(80`!6("$`=D`A`%$0(0!Q&"&4
+M9@``A(<``)1%``"%"```#`!!Y@(`("$``B0```0D`P`0&$``=1@A`)0@*A2`
+M``RD8@``+@(`0!!```HF`@`!`@`0(28#``$P</__``(00`!5$"$N`P!`%&#_
+M^:14```F`@`!,%#__RX"`$`40/_1`!(00"0#``&/OP`@C[<`'(^V`!B/M0`4
+MC[0`$(^S``R/L@`(C[$`!(^P````8!`A`^``"">]`#"$@@!^A(,```"`2"$`
+M0Q`C*$(`?Q1``$24A`!^)(+_@@`"%````E0#``04```")`,DA___)`(`/R4F
+M`'XDQO_^A,,``"1"__\``A0``.,8*A!@``,``A0#!$'_^23&__X`0#`A)`<`
+M/R3#__\P8O__+$(`/Q!``!(`!A!``$D0(81"````1!`J%$``#0`#%````C0#
+M``800"3#__\`24`A,&+__RQ"`#\00``%`````(4"````1!`J$$#_]0`#%``0
+MP``,).+__P`'&$`DA/_^``(4``!E&"$`!"0```(\`Z1F```$X?_@``0D`P/@
+M``@!0!`A!.#__0``````!Q!``$4H(23B__^4HP`"``(4```"/`.DHP``!.'_
+M^B2E__X0`/_R`````!``_[\`8%`AA((`?H2H````@%`A)`P`0`!($",H0@!_
+M%$``392#`'XD8O^"``(4```";`,``Q0```(D`R0)`#\DB___)4@`?B4(__Z%
+M`P``)2+__P`"%``!8Q@J$&```P`"3`,%(?_Y)0C__I2B`'X!($`A)`D`/Z3B
+M``"$X@```$00*E!``#$``&`A%0```P````!5@``M``!@(1D```X`"!!``$H8
+M(81B````1!`J%$``"24"__\``A0```)$`QD```4D8__^A&(```!$$"H00/_Y
+M)0+__Q4```,E(O__$8``#0```````A0```D80"2$__X`9A@A``0D```"3`,!
+M#!`EI&(```4A_]P`!"0#`^``"`&@$"$%(/_]```````)$$``1C`A)2+__Y3#
+M``(``A0```),`Z3#```%(?_Z),;__A``__(``````*!0(1``_](D"``_$`#_
+MM@$`:"$GO?O0K[X$(*^_!"2OMP0<K[8$&*^U!!2OM`00K[,$#*^R!`BOL00$
+MK[`$`)4"``*,@QOTKZ0#R#!$-P`D`A4`KZ@#V*^E`\ROI@/0KZ<#U`$@\"&,
+M8P``$((!QJ^@`]PH@A4!$$`!R"0")``D`A(`$((!PR0"%`!0@@&_)&,`+(^D
+M`]2/I0/0CZ8#W(2#``"$I```)`+>K:>B`[JGH@.X``,<0``$)$"4Q0`&``0D
+M`P`#'`.OI`/@KZ,#Y```J"$``(@A)Z0#N`(E$`<P0@`!&$``!2ZC``(08`#A
+M)K4``:21```DA``")C$``2XB``040/_V`B40!X^H`]R/IP/8E08`!(3D``"-
+M!0``)Z<#P`P`0?XGJ`/$C[8#P(^B`\0`5A`K%$``@``6$(``5A`A``(0@`!6
+M$"$``A"`KZ(#[*^@`_"/HP/<CZ0#[(QB``B7HP.Z`$2H(30"WJT08@%"E[$#
+MN">W`0@"X"`A`!&`@`(1@"$`$("``A6`(984``8F!0`()[,!(``4B$`"(#`A
+M#`!PG280`!`"`"@A`B`P(0P`<)T"8"`AAJ<``B>R`S@"@"`A`N`H(0)@,"$,
+M`$(F`D!`(1!``*:/I@/P`D`@(2>E`[@`W1`A)$,!.#!B``,00`$4KZ8#Z`"@
+M$"&(AP``F(<``XB(``28B``'B(4`")B%``N(A@`,F(8`#ZAG``"X9P`#J&@`
+M!+AH``>H90`(N&4`"ZAF``RX9@`/)(0`$!2"_^XD8P`0E[$#N@+@("$`$8"`
+M`A&`(0`0@(`"%8`AEA0`!B8%``@F$``0`!2(0`P`<)T"(#`A`@`H(0(@,"$,
+M`'"=`F`@(8:G``("@"`A`N`H(0)@,"$,`$(F`D!`(1!``'./IP/H`D!`(0#]
+M$"$D0P(X,&(``Q!``-4GI`.X`(`0(8D$``"9!``#B04`!)D%``>)!@`(F08`
+M"XD'``R9!P`/J&0``+AD``.H90`$N&4`!ZAF``BX9@`+J&<`#+AG``\E"``0
+M%0+_[B1C`!"/J`/PCZ,#[(^B`\0FU@`!)0@`@"1C`%0`5A`KKZ@#\!!`_XBO
+MHP/LCZ<#W(^F`\"/I0/$C.0````&$$"7IP.Z`$00(0`%&$``9!@AE%0``#0"
+MWJT`IJ@C$.(`@91S`````(@A`Z"0(0`5@8"/J`/8`A&`(0`0@$`"'8`AAD<!
+M.)4$``"&"`$X`H`H(0P`0>8"8#`ACZ,#V(9'`CB&"`(XE&0```*`*"&F0@`(
+M`F`P(0P`0>8F,0`!+B,`0*9"`(@48/_I)E(``B0"``(40`!%AZ,!!H^G`^``
+M9Q`J%$``!(^H`^0`:!`J$$``*8^E`\R/I0/,#`!">B>D``@`0$@A``(?PI>B
+M`(:7I0.XCZ8#T``"%````B?"CZ<#U``"%`,!(Q@A`$00(0`"$$,``QA#I\4`
+M`J>B``"GQ0``I,,``*3B``"/J`/4CZ,#R)4"``",9!OT)`,`/P!B&".D@P`D
+MC[\$)(^^!""/MP0<C[8$&(^U!!2/M`00C[,$#(^R!`B/L00$C[`$``/@``@G
+MO00P#`!">B>D`(@`0$@A``(?PI>B`0:7I0.Z`2,8(0`"%````B?"``(4`P!$
+M$"$``A!#I\4``H^D`]"GH@``I\4``(^E`]0``QA#I(,``!``_]FDH@``CZ8#
+MS">D``@GI0"(#`!"QB>G``(`0$@AEZ,``@`")\*7H@"&``,<```#+\(``A0`
+M``(WP@`#'`,``A0#EZ@#N)>G`[H`91@A`$80(8^E`]"/I@/4`20@(0`$($,`
+M`QA#``(00Z2D``"GHP``I,(``*?'``(0`/^ZI\@`````B"$#H(`A`!41@`!1
+M$"$``A!``%T0(81(`3B/H@/8A@<!.`*`*"&41````F`P(0P`0>8F,0`!+B,`
+M0*8"``@48/_Q)A```H^E`\P,`$)Z)Z0`"`!`2"$``A_"EZ(`AI>E`[B/I@/4
+M``(4```")\(``A0#`$00(8^D`]`!(Q@A``(00P`#&$.D@P``IZ(``*3"``"G
+MQ0`"$`#_DJ?%``"-`@``C04`!(T&``B-!P`,K&(``*QE``2L9@`(K&<`#"4(
+M`!`5!/_V)&,`$!``_S6/J`/PC(<``(R(``2,@@`(C(8`#*QG``"L:``$K&(`
+M"*QF``PDA``0%(7_]B1C`!`0`/[VE[$#NB>S`0@"8"`A`!&`@`(1@"$`$("`
+M`A6`(984``8F!0`()[(!(``4B$`"(#`A#`!PG280`!`"`"@A`B`P(0P`<)T"
+M0"`AAJ<``B>P`S@"@"`A`F`H(0)`,"$,`$(F`@!`(1!`_V6/I0/P`@!`(0"]
+M$"$D0P$X,&(``Q!``!<GI`.X`(`0(8D&``"9!@`#B0<`!)D'``>)!``(F00`
+M"XD%``R9!0`/J&8``+AF``.H9P`$N&<`!ZAD``BX9``+J&4`#+AE``\E"``0
+M%0+_[B1C`!`0`/[RCZ@#\(T&``"-!P`$C0(`"(T%``RL9@``K&<`!*QB``BL
+M90`,)0@`$!4$__8D8P`0$`#^Y8^H`_`D8P`L$`#^0:^C`]P0`/_])&,`(!""
+M``,D`B4`%(+^/(^D`]00`/_W)&,`%">]_Y"OJP`HCZL`<"0"`#^OO@!@K[<`
+M7*^V`%BOM0!4K[0`4*^R`$BOL0!$K[``0*^_`&2OLP!,`."@(:^B`"PD`O_!
+ME.<``I:#``RE8@``CZ(`*`%`D"&-2@`$)`L`/Z1+````"$0```E,```#&$"5
+M0@$6``A$`P`)3`,P8___KZ@`(*^I`"2OHP`PDI,`#@"`J"$`H(`A`,"X(3#^
+M-P`D%@`_``"((1!``!2OH``T`!$00`!*$"&40@$8,$/__Q!@``<R9/__,$(`
+M!P`1,8`09`$E-$(`X%""`22.10`0CDH`!"8C``$P<?__E4(!%@(B$"L40/_O
+M`!$00#/"-P"OH@`XCZ,`."0"%`!08@#JCD(`!*^V`"PSP@0`$$``:H*3``\S
+MT1``4B``X9;G``"6YP!2)N8`5`*@("$"@"@A#`!'&@.@0"&/HP`PEH0``@!S
+M$","PA@J`L,0"P`"%``PA`$`$(``%``"3`.60P``+&(P`11``!&7H@`&CD4`
+M!"QB0`"4HP`(`2,@*@$D&`L``QP`%$``"``#3`-2(``'EZ(`!I2B``H!(A@J
+M`2,0"P`"%````DP#EZ(`!H^K`"`!(A@J`2,0"P`"%````A0#``L@0`!$(",`
+M!"0```1,`Z^B`#0%(`"P`2`0(8^C`"0``A0```),`P$C$"I40``!`2`8(9:D
+M&\8``Q0`$(```P`"3`,`!!1```),`Y>B``27HP`"EZ0```$B*"H!(S`J`20X
+M*@$E$`L!)A@+`2<@"Z8"``JF`P`,I@0`#J8)``BF"0`&I@D`!*8)``*F"0``
+MED(``"Q"0`!40``,I@D`'C/#$``08`"*CD(`!)1"``X!(A`J5$``!`$@$"$0
+M8`""CD(`!)1"``ZF`@`>E@(`#H^K`"B.I!OTI6(``)8#`````Q0```(4`ZZB
+M!9RL@@`HCZ(`<*1#```SP@(`%$``%8^C`#@D`A0`4&(`$Y;G`'*/HP!PCZ0`
+M-(^_`&2$8@``KJ0%H(^^`&"NH@6<C[<`7(^V`%B/M0!4C[0`4(^S`$R/L@!(
+MC[$`1(^P`$`#X``()[T`<);G`'("H"`A)N8`=`*`*"$,`$<:)Z@`$(^D`#"/
+MJP`L`),8(P%C$"H!8A@+EZ(`%@`#'````TP#`2(8*@!`4"$!(Q`+CZ,`-`!B
+M$"H00``(CZL`(`$J$"H!("`A`4(@"@`$%````A0#KZ(`-(^K`"`!*A`J`4)(
+M"@`+$$`!(A`C``(4```"3`,%(``X`2`0(8^C`"0``A0```),`P$C$"H00``"
+M`&`P(0$@,"&7I0`0EZ,`%)>D`!(`!A0```),`P$E$"H!(B@+`2HP*@$C."H!
+M)$`J`2`0(0%&$`H!)Q@+`2@@"Z8"`!"F`P`4I@0`&*8%`!RF`P`2I@0`%J8%
+M`!J6HAO&$$``#(^D`"@``A!`I@(`$*8"`!:F`@`4I@(`$I:B&\8``A!`I@(`
+M&*8"`!RF`@`:CZ0`*)8#`!J$@@```&(0*E1```&D@P``CZL`<)8#`!"%8@``
+M`$,0*E1`_Y>E8P``$`#_EH^C`'`0`/_(```0(1``_W^40@`,$`#_=Y1"``P0
+M`/]0```0(1``_R$FY@`",F,`^#1S``&40@$6$$#_%```B"&.0P`$`!$00`!#
+M$"&40@$8,$/__Q!@``<R9/__,$(`!P`1,8`09``,-$(`X%""``N.10`0CD(`
+M!"8C``$P<?__E$(!%@(B$"M40/_NCD,`!!``_O\SP@0`CD4`$):$```,`$;<
+M`*8H(8^K`"P`0#`A`$`@(0%B*"LR8P#X)`(`X%1B_O*OI@`L5*```0%@("$P
+MA/__$`#_YZ^D`"R.10`0EH0```P`1MP`IB@A`$`P(0!`("$"PB@K,F,`^"0"
+M`.!48O[;`,"P(52@``$"P"`A$`#^T#"6__^,@AOT`*!@(2>]_^",0@``)`4`
+M"(Q"``2`3@`BB8(``)F"``.)@P`$F8,`!XF$``B9A``+B8@`#)F(``^KH@``
+MNZ(``ZNC``2[HP`'JZ0`"+ND``NKJ``,NZ@`#XF"`!"9@@`3B8,`%)F#`!>)
+MA``8F80`&XF(`!R9B``?JZ(`$+NB`!.KHP`4NZ,`%ZND`!B[I``;JZ@`'+NH
+M`!\`!5A``7T8(91B`````$@A``!H(0!.$".D8@```7U`(84"```$0@`!I0``
+M`"0"``@1(@`Y``D00`!,$"&40P``A0H``"4B``$P2?__$4,`)Y4$```M(@`0
+M5$#_\0%]0"$!?2`AA((``"A"`$`40``$`6P0(20"`#^D@@```6P0(91"``"$
+MA```)*4``0!'$",H0P````,0"P`"$$``1A`AE$,````$($`PI?__`(8@(2RB
+M``\40/_2I(,``"0%``@`!1A``'T0(91$```DH@`!,$7__P!L&"$LH@`/%$#_
+M^*1D```#X``()[T`(!%```4D@@`!+:(`,Q1```(D@O__)((``:4"```EH@`!
+M,$W__Q``_]```$@A$`#_SB0)``\GO?_`K[,`+*^R`"BOL0`D,)+__Z^_`#"O
+ML``@`*"((203`#\``"`A``00P`!1$"&40P````000"2&``$08``%`%TH(3#$
+M__\L@@`(%$#_]J2C````@(`A)Z<`$`)`("$GJ``2`Z`H(0P`1[L"`#`A$@``
+M"P``("&7I0`0``000`!=$"&40@``$$4`!22#``$P9/__`)`0*Q1`__D`!!!`
+MEZ,`$)>B`!(08@`/``00P`!1$"&,0@`$$$```P`$$,``41`AE%,``@)@$"&/
+MOP`PC[,`+(^R`"B/L0`DC[``(`/@``@GO0!`%'+_\0`````0`/_T`%$0(2>]
+M_\"OL``@,/#__Z^V`#BOM0`TK[0`,*^S`"ROL@`H`*"8(:^_`#ROL0`D`,"0
+M(0$`J"$``*`A``"P(1(```T``"@A``40@`!%$"$``A!``%(0(91$``@DHP`!
+M``400#!E__\`71`A`+`8*Q1@__6D1```EF0```.@*"$GIP`0)Z@`$@P`1[L"
+M`#`A$@``#@``*"&7IP`0EZ8`$@`%$$``71`AE$,``"2D``$`9Q`F$&8`-P"B
+MH`HPA?__`+`0*Q1`__<`!1!``!2`@``6B(`"%(`A`C:((0`0@$``$8A``C*(
+M(0(2@"&6!P`&EB@`!I9D``"7I0`0EZ8`$@P`1X```$@AIJ(`!I8'``26*``$
+MEF0``)>E`!"7I@`2#`!'@```2"&FH@`$E@<``I8H``*69```EZ4`$)>F`!(,
+M`$>```!((::B``*6!P``EB@``)9D``"7I0`0EZ8`$@P`1X```$@AIJ(``(^_
+M`#R/M@`XC[4`-(^T`#"/LP`LC[(`*(^Q`"2/L``@`^``"">]`$`0`/_-`*"P
+M(3#G__\Q"/__<.@0`C"*__\PI?__,,;__R0$`&000``>```8(20"``$0Q0`I
+M`$D@"P%%&",``Q!``$,0(0`"$,``0Q`A``(0@`#%&",`0P`:4&```0```<T`
+M`!`2``(4```"'`,$8``:*&(`951```PD`@!D42``!S$"__\`"!!``$@0(0`"
+M$,``2!`A``(0@#!"__\`0!@A`^``"`!@$"$`0Q`C`$<`&%"```$```'-<&@`
+M````$!(```````````!$`!H``!@2$`#_\C!B__]1(/_P,.+__P`'$$``1Q`A
+M``(0P!``_^D`1Q`AE*L```"@4"$PA/__,67__P`%&$``!!!``&48(0!$$"$`
+M`QC``&48(0`"$,``1!`A,,;__P`#&(```DB`)&/_]@`&$$`!0A`A`2,8*Q!@
+M``641/_^I.L``)5"```#X``(I0(````$$$``1!`A``(0P`!$$"$``A"`)$(`
+M"@!)$"L00``$`````*3D```#X``(I00``!#``!\``%@A``LH0`"J$"&40P``
+M``,00`!#$"$``A#``$,0(0`"$(`!(A@C`$D0(RAC``H$0``<*$0`"A2``!8`
+MJA`A`*H@(92#``(E8@`!,$O__P`#$$``0Q`A``(0P`!#$"$``A"`)$+_]@$B
+M$"L40``%`68H*Q2@_^0`"RA``^``"`````"4@@``I.(``!``_\V4@@`"E$,`
+M`*3C```0`/_)E$(``%!@_^<`JB`A$`#_^0"J$"$\`H`")$8N/#P"@`(D0RS,
+ME((<]B3(``@P0@#P+$(`,!1```\D9P`(C,0`!"0"`%6LH@`0)`(``:RB`!@`
+M!!#``$00(0`"$(`D`P`4`$@0(:RC``RLH@`<`^``"*RD``",9``$)`(`(ZRB
+M`!`D`@`!K*(`&``$$,``1!`A``(0@"0#`!2LHP`,$`#_\@!'$"$GO?_PK[``
+M`*^_``24@ASV`*"`(3!"`/`L0@`P%$``(20%``<,`$GM``````!`("$D!@`D
+M```X(0P`/PXD!0`!)`,``20'`&L``#`A$$,`$"0$`&N.!0`$``!`(0#E$"L0
+M0``&`*88*Q1@``,`A2@K$*```X^_``0D"``!C[\`!(^P```!`!`A`^``"">]
+M`!`D`@`W)`<`4R0$`%,0`/_MK@(`$`P`2>T``````$`@(20&`"4``#@A#``_
+M#B0%``8X1``_)`8`0``$,`HDPP`W),4`(P!D*`LD`P`4``08"R1"``0D!P`R
+M`$0X"ZX#``RN!0`0$`#_UR3$`#*,A1OTC(0``#0#@`",H@`HA*8`)`"#("$`
+M1A`A``(20#!"?@`T0H``K((9,"0"``$#X``(K*(`,">]__"OL0`$K[\`"*^P
+M``",A1OT`("((8RP``2.`@`8$$``!R0$__^,HP`P)`(``5!B``F.(@``CB(;
+M](Q$`#"/OP`(C[$`!(^P````@!`A`^``"">]`!`T`X```$,0(8Q$&3`P@H``
+M5$#_](XB&_0`!!W",&,``P`$)D(D`@`"$&(`):X$``26(ASV,$(`\"Q"`#`0
+M0``6`B`@(8XB&_26)AMR`B`@(:Q``#`,`$@R`@`H(51`_^&.(AOT#`!(SP(`
+M("%00/_=CB(;]`(`*"$,`$C;`B`@(1A`_]<D`@`"CB,;]*QB`#`0`/_3K&``
+M+`P`23L"`"@AC@0`!(X#`!0`@Q`K5$#_Y:X```0`@Q`C$`#_XJX"``26(AST
+M+$(`0A1```4D@@`%C*(``(Q"``2`0@`B`((0(1``_].N`@`$C(,`!(R"``P`
+M0Q`K$$``!0``*"&,@@`0`&(0*Q1```(`````)`4``0/@``@`H!`AE((<]C!"
+M`/`L0@`P%$``60"@,"$\`H`")$(N/(S%```\`X`#K&**L#P"@`.,1XJP``40
+MP`!%$"&,Q``$C,@`$``"$(``XA`A)$(`"`"(&"L48``>K,(`'!"@`!HD`___
+MK,0`"(S#```\`H`#C$2*L"1G__\`!QC`C,(`'`!G&"$``QB``&08(:S'``",
+M10`0C&(`&(S$``@D8P`(`$40(P`"$$``@B`C`(@0*ZS#`!P40``#K,0`"%3@
+M_^N,PP``)`,``0/@``@`8!`AC,D`#`$D$"L40/_[```8(8SB```D0O__$*+_
+M]R0#__ZLQ``(C.(``"1"__\`HA`K$$#_\20#``*,PP``/`*``XQ(BK`D9P`!
+M``<8P(S"`!P`9Q@A``,8@*S'````:!@AC$4`$(QB`!B,Q``()&,`"`!%$",`
+M`A!``((@(P$D$"NLPP`<%$``!JS$``B-`@``)$+__P#B$"M40/_HC,,``!``
+M_]4D`P`"/`*``A``_Z@D0BS,)[W_\*^Q``2LH``4`*"((20%``>OOP`,K[(`
+M"*^P```,`$GM`("`(0!`("$D!0`!)`8`)`P`/PX``#@A`$"0(20"``%20@`'
+MCB(`'(^_``R/L@`(C[$`!(^P```#X``()[T`$`(`("$D!0`'#`!)[810````
+M0"`A)`4`!"0&`"`,`#\.```X(1(2``X`0!@A4@#_[JX@`!0D`@`"$@(`!P`#
+M$$`D`@`#%@+_Z8^_``P``Q!`$`#_Y:XB`!00`/_])$+_^Q``_^&N(@`4)[W_
+MX*^U`!2OM``0K[\`&*^S``ROL@`(K[$`!*^P``"4H@`"`*"H(3!"`0`00`!Q
+M`("@(8R0'`2.`@``&$``'@``F"$F$0`HDB8`"Y(H``@"`"`A``800`!&$"$`
+M`A#``%`0(8Q#`"@\`H`")$)$```#&(``8A@AC&(````(0(`T`X<`)`4`#@``
+M."$`0/@)`0.0(0!`,"&.`P``CH(``"9S``$"8Q@J`%(0(:Q&```48/_E)C$`
+M&):B``(P0@$`%$``"H^_`!B.D!P,C@(``!A```8``)@AC@,`*"0"``(08@`*
+M)A$`*(^_`!B/M0`4C[0`$(^S``R/L@`(C[$`!(^P```#X``()[T`()(F``L\
+M`H`")%5$```&$$``1A`A``(0P`!0$"&,0@`HDB@`"#0#AP```A"``%40(8Q"
+M````"$"``@`@(20%``X``#@A`$#X"0$#D"$`0#`ACH(```(`("$D!0`.`%(0
+M(:Q&``"2(P`))`<``29S``$``Q"`%&``"P)"D"&.`@```F(0*A!`_]4F,0`8
+MCB,``"0"``)08O_;DB8`"Q``_]"/OP`8DB8`"P`&$$``1A`A``(0P`!0$"&,
+M0@`H``(0@`!5$"&,0@```$#X"0``````0#`ACH(```!2$"&L1@``$`#_YXX"
+M```0`/^0C)`<%)2"'/8P0@#P+$(`,!1``!,PI?__C(,;]"2E__\LH@`'$$``
+M#HQD`"`\`X`"``40@"1C#[P`0Q`AC$(```!```@``````^``"`"`$"$#X``(
+M)((`!`/@``@D@@`(`^``"```$"$#X``()((`#`/@``@D@@"L)[W_\*^Q``2O
+ML````("((0"@@"&OOP`(#``I;#P%@`".)P``)`/\#P(@("&,X@`P/`6```!#
+M0"0P0@/P``(1`BQ&`"42```3+$,``A#```TD0@`!``(1`#!"`_`!`A`EK.(`
+M,`P`*3P`````)`(``8^_``B/L0`$C[````/@``@GO0`0#``I/#P%@``0`/_X
+M```0(1!@_^\D0O__$`#_^0(@("&,P@`4C,<`$"0#``$`0Q`$`(!((0#`0"$`
+MXQ@$C,8`#"1"__\\!``/-(3\`"1C__\``A*``$00)(TG```P8P/_`&(8)0`&
+M-0`\`@_P`,(P)``%*(``9A@E`.4X(:SC$$"-)```C0,`(#P"``\T0O__`&(X
+M)!!@``,`A2`A/`(`$`#B."6-`@`$K(<0P#!&`"`P0@`$$$``!CP$`""-(@``
+M`$40(8Q#$0``9!@EK$,1`!#```8`````C2,```!E&"&,8A$`-$("`*QB$0`#
+MX``(`````">]__"OOP`$K[```(R(&_2,L``(C*,`!(T'``@D`@`!`@)(!`#I
+M."4P:@`",&,``0"@,"&M!P`($&``!`(`*"&-`@`,`$D0):T"``P10``$````
+M`(T"`!``21`EK0(`$`P`3HL``````@`0(8^_``2/L````^``"">]`!",A!OT
+M)`(``0"B$`2,A@`(C(<`#(R#`!```A`G`,(P)`!B&"0`XC@DK(,`$*R&``@#
+MX``(K(<`#(R"````!2B`)*4(``!%$"&L1@```^``"`````",@P``)`(``0"B
+M$`2L8@A``^``"`````",@AOTC$,`""0"``$`HC`$`&88)``%*(`DI0H`$&``
+M"@``$"&,A````(40(8Q"```P0@`#%$``!`````",@@A``$80)``"$"L#X``(
+M`````">]__``!A!`K[(`"*^Q``2OL```K[\`#`!&$"$`@(@A``(0P(R$````
+M1A`A)`,``0"C&`0`H)`A``*`@*R#"(`"("`A#`!*H@)`*"$F$/__$$``!R0$
+M``H&`@`&CB(```P`:3\`````$`#_]@(@("&.(@``C[\`#(^R``B/L0`$C[``
+M`*Q`"(`#X``()[T`$!"@`!4`H#@AD*(`#@`"$0(P0@`!$$``$8SB``@\`]__
+M``8@*S1C__\`0Q`D``0G0`!$$"6LX@`(C.<`(%#@``>,X@`(D.(`#@`"$0(P
+M0@`!5$#_\HSB``B,X@`(/`/?_P`&("LT8___`$,0)``$+T`\`_]_`$40)31C
+M__\`0Q`D``0EP`!$$"4#X``(K.(`"``($,``2!`C)[W_P``"$("OO@`PK[<`
+M+*^V`"BOM``@K[,`'*^R`!BOL0`4K[``$*^_`#2OM0`D`$00(8Q"&=@`@)`A
+M`*#P(:^B``2/HP`$)`(``:^F````X)@A``"@(0``L"$``+@AKZ(`"*^@``P`
+M`(@A%&``#B2P``B/H@`,C[\`-(^^`#"/MP`LC[8`*(^U`"2/M``@C[,`'(^R
+M`!B/L0`4C[``$`/@``@GO0!`#``I;#P%@``28``:)`/_\"9T``@R@@`/)$(`
+M)P!#*"0"@Q@D`&40(0!#$"L40``'`&`@(;QQ```D8P`0`(40(0!#$"L00/_[
+M``````)`("$"8"@A#`!.AR0&``&OH@`(CH(``(Z#``@``HV",'9__S(Q``$"
+M0"`A#``I/#P%@`".`P``/`*`0(X%``P`8A@DC@0`"(^B``@`!34",)5__P`#
+M&"L40``+,,8`'U!@``H#P"@A5B``2H_"`$R/HP`$`M40(0!7$",`8A`K$$``
+M$`*W$",#P"@A`D`@(0P`2MHD!@`!CZ,`#!!@_[B/H@``4$#_MX^B``R,10`P
+M`D`@(0P`2MH``#`A$`#_L8^B``P"0"`A/`6```P`*6P"PK`A$F``'"9E``@P
+MH@`/CH0`""0#__`D0@`G`*,H)`!#,"0D`H```((@)#+#?_\`IA`A`(,@)0!%
+M$"NNA``(%$``!P"@&"&\L0``)*4`$`!F$"$`11`K$$#_^P`````"8"@A`D`@
+M(0P`3H<D!@`!+$(``:^B``P"0"`A#``I/#P%@`"/H@`,$$#_S`/`*"&.!```
+MC@4`"#P"_[\T0O__/`-__P""("0T8___)`*```"#("0`HB@DK@4`"!``_[ZN
+M!```)`4`#HQ#`#P`9A`AD$8`!`!@("$`!A!``$80(0`"$,``0Q`AC$,`*#P"
+M@`(D0D0```,8@`!B&"&,8@```$#X"20'``$0`/^E`$"X(2>]_\"OM``@K[(`
+M&*^Q`!2OL``0K[\`-*^^`#"OMP`LK[8`**^U`"2OLP`<C((;-`"`D"&,O@`H
+M`*"`(0#`H"$DL0`(C+,`4(RD`$P40``1KZ```%)@``63PP``CF(`!!1```P`
+M````D\,``#!B``P40``')`(``3!C`/`D`@!`$&(`!"0"`%`08@`")`(``:^B
+M```0@`'*`````(R6`#R.`@`L`D`@(0(`*"&L0``8K$``'`P`4Z```````$"X
+M(0`"$$".)0```%<0(98D``H``A#``L(0(3P#`#\`HR@E)%4`))*F``^N)0``
+MKB0`"(Y"&\@\`^'_-&/__R1"``$P0@`/``(4``""("4`HR@DKB0`"*XE``"3
+MPP``CB0`!#P%_P\P8P#\.&,`4"0"``,``Q`+-*7__P`"%0``A2`D`((@):XD
+M``0`!A!``$80(9/#`````A#``L(0(21&`"0P8P`,)`(`"%!B`8R3P@`!)`)_
+M_Q*"``P\`O_PCB0`!(XE```T0A__,H,`?P""("0``QM`/`)```"B*"4`@R`E
+MKB0`!*XE```28``9/`2?_Y9B`1B.(P`$``(0@C!"``,TA/__``(70`!D&"0`
+M8A@EKB,`!(X"`"B00@`!,$(`0!!```,``"`AEF(!%@`"((*.(@`$/`/G_S"$
+M``,T8___`$,0)``$)L``1!`EKB(`!)/"``23Q```,$<``3"#``PD`@`($&(!
+M1S"#`/`D`P`#CB0`!#P"^?\T0O__,&,``P""("0``QY``(,@):XD``2.0AMH
+M4$`!,9/"`!828`$N/`+^_XX#`%B.)```C&,`)#1"__\`@B`D``,8P`)C&"$D
+M8P`PC&(`!``"$"L``A8``((@):XD``"L8``$$.``\XXD``".)0`$/`+_OS1"
+M__\\`W__`((@)#1C__\\`@$``*(H)0"#("2N)0`$KB0``(ZC``0D`@`#4&(`
+MW(XB```28``&```@(99"&W(D`Q0`,$(W`%!#`-*.8@`L$(``-X^C``".0ALX
+M$$``-`````".H@`$%$``,0`````4X``O`````(Y#&SP``Q!``$,0(0`"$,"/
+MHP```%80(21"`"@08``#D$0`")!"``D`@B`ECB,`##P4_@\P@@`?-I3__P`"
+M%0``=!@D`&(8):XC``R3P@`!,$(`!%1```:.8P`LD\(`%C!"``]00``&CB0`
+M`(YC`"PD`@`!$&(`CP(`("&.)`````05@C!"``$40``+CZ,``(Y#&]0D`@`!
+M$&(`@#P#@``\`O^_-$+__P""$"0`0Q`EKB(``(^C```08``#DJ0`#)*B``T`
+M@B`ECB(`#"0#_^`PA``?`$,0)`!$$"6N(@`,CZ(``%!```*6LP`2EK,`%(XC
+M```\`H!``&(0)%!``"F.1`6,CB(`"``#'8(D%(```%00)#!C``$48`!'KB(`
+M"``7$$``5Q`A``(0P`!6$"&,0P`H/`*``B1"1````QB``&(8(8XE``",8@``
+MCZ<``(XP``@PI0__`L`@(0!`^`D"X#`ACB,`"#(0?_\"`H`A,A!__P!T&"22
+M)0`$`'`8)3!B?_\PI0`!`%,@(0"%$`H`=!@D,$)__P!B&"6N(P`(CD0%C!"`
+M`"*6H@`(``08P`!D&"$``A'`/`3,S`!B&"$TA,S-`&0`&0``&!```QC"DB(`
+M!*Y#!8P``!@A,$(``0)B&`HP8@#_``(2```#&@(`0Q`E``(:`J/"``.CPP`"
+MC[\`-(^^`#"/MP`LC[8`*(^U`"2/M``@C[,`'(^R`!B/L0`4C[``$`/@``@G
+MO0!`$`#_YP`"&<".(@`,CZ<```+`("$``A4",$(`'P+"$"&01@`$``800`!&
+M$"$``A#``%80(8Q#`"@\`H`")$)$```#&(``8A@AC&(```!`^`DD!0`.CB,`
+M"#!"?_\`=!@D`&(8)1``_Z*N(P`(/`-__SP"`$``@A`E-&/__Q``_X``0Q`D
+M#`!6R0+@*"$`0+@A``(00`!7$"$``A#``L(0(215`"22I@`/``800`!&$"$`
+M`A#``L(0(21&`"2/H@``$$```Y#$``R0P@`-`((@)8XC``PP@@`?``(5``!T
+M&"0`8A@E$`#_9ZXC``PX0@`!$`#_+2Q$``$\`_^_-&/__SP$?_\`0Q`D-(3_
+M_P!$$"00`/\>KB(``)9#&]P\!?^_,((/_P!B&"HTI?__``,=@`"%("0`@R`E
+MKB0``)("`#PP0@`!4$``!(X"`#P`A1`DKB(``(X"`#P``A;",$(`#U!```:.
+M(P``CB(``#P#`$``0Q`EKB(``(XC```\`G__-$+__P!B&"2N(P``CZ,``!!@
+M``.0Q``,D,(`#0""("6.(@`,/`/^#S"$`!\T8___`$,0)``$)0``1!`E$`#^
+M[ZXB``R3P@`6CB,``#P$_O\P0@`/+$(``32$__\`9!@D``(6``!B&"40`/[7
+MKB,``"0"`$`08OZY)`,``Q)@_K<D`G__AF,!%%!B_K0D`P`#5.#^LB0#``..
+M(@``,$(/_RA"`0$40/ZM)`,``HX"`%B,0@`D+$(``E!`_JF.)``$$`#^IHY#
+M&]`P0@!`$$#^="0"?_\6@OYR`````)9"&_"N`@`X$`#^;0!`H"$,`%)>ED0;
+M<@`"$(``4A`A$`#^,XQ6'``GO?_0K[(`"`"@D"$DI0`(,*(`#Z^U`!2OLP`,
+MK[$`!*^P``"OOP`@K[<`'*^V`!BOM``0)`/_\"1"`">.5``P`*,H)`!#,"0`
+MIA`A`$40*P"`J"$F4``8)I,`"(Z6`%".EP`H))$$@!1```<`H!@AO+$``"2E
+M`!``9A`A`$40*Q!`__L``````J`@(0)`*"$,`$Z')`8``1!``&8D`P`-D@(`
+M`XYC``PP0@`!$$``J3!I`!\D`@`!KD(`0)8"``8``A!",$(/_Z9"`$:6`@``
+MID(`1)("``*2`P`#HDD`2C!"``\``QD"`$,0(:9"`$B28@`$,$(``1!``)(`
+M````$L``!20*`!Z"P@"Z%$```@!`4"$D"@`>D@(``S!"``$00`!K*4+_[!1`
+M``Z.(P$()`(!)Q!B`&0``Q#``$,0(0`*(<`\`V9F`$00(31C9F<`0P`8``(7
+MPP``&!```QB#`&(8(Y("``(P0@`/%$``!:XC`0B2`@`#``(1`A!``!8`````
+MCB0`>(XE`'R.8@``)*4``0`"%8(LHP`!`(,@(3!"``&N)`!XKB4`?!1``$$`
+M`#`AD@(``C!"``\00``X``88@``"&(``<1@AC&(`P"1"``&L8@#`D@(`!0`"
+M$<(00``$`````(XB`%PD0@`!KB(`7)("``,``A#",$(``11```P``!@ACJ(;
+M1#!"``$00``)C[\`()+B``0P0@`!%$``!H^W`!R.@@!,5$``#8Q"`#R/OP`@
+MC[<`'(^V`!B/M0`4C[0`$(^S``R/L@`(C[$`!(^P````8!`A`^``"">]`#"2
+M"``#CF<```!)$"&2"P`$D$8`!)9)`$@`"$!",.</_S$(``$Q:P`!`J`@(0P`
+M5/,"P"@A$`#_Y@``&"$`<1@AC&(`@"1"``$0`/_*K&(`@)("``.2`P`"``(1
+M`A``_[XP9@`/$`#_I@`*&<"2`@`#CB,`0``"$$(D8P`!,$(``1!```6N(P!`
+MCB(`1"1"``$0`/^_KB(`1)("``,``A#",$(``1!```4`````CB(`<"1"``$0
+M`/^VKB(`<)("``,``A"",$(``1!`_[$`````CB(`2"1"``$0`/^MKB(`2(X"
+M``0``A+`$`#_<@`"5@.2`@`#``(00C!"``%00/]5)`(``Q``_U,D`@`")*4`
+M&)"B``<#X``(,$(``8S*````P#@A`*!8(0"`0"$10`"V``5(@(S#`!2,Q``0
+M)`(``0!B&`2,Q0`,`((0!"1C__\\!``/)$+__P`#&H`TA/P`C08```!D&"0P
+M0@/_`$,0)0`%+0`\`P_P`*,H)`!%$"4`R3`AK,(00(T"&V@\!0`($$``O32E
+M(`"-!!O()`(`#RR#`!``0R`*``01`#!"`/`P@P`/`$,0)0!%*"6-`@``)`,(
+M``!)$"&L11"`C0(```!)$"&L0PG`C.4`&%"@`!.,Y``DC.,`'(T$```\`@#_
+M-$+__P"B$"0``QX``$,0)0")("&L@@C`C0,``(SE`!P`:1@AC&()P#1$`0$T
+M0@`!`(40"ZQB"<",Y``D4(``"HT$``"-`@``/`,`_S1C__\`@Q@D/`0!``!D
+M&"4`21`AK$,)`(T$``",XP`@/`(`#S1"__\`B2`A$&```P!B*"0\`@`0`*(H
+M):R%$,",X@`@4$``"XSE``2,X@`$,$(`$%!```>,Y0`$C0,```!I&"&,8@G`
+M-$("`*QB"<",Y0`$,*(`!!!```@PH@`@C0(``#P$`"``21`AC$,1``!D&"6L
+M0Q$`,*(`(!!```<D`@`"C0,```!I&"&,8A$`-$("`*QB$0`D`@`"44(`5XT#
+M```D`@`#44(`2(T#```D`@`$44(`/XT#```PH@`(4$``%(T$``",X@`H4$``
+M$8T$``"-`P````L00"1"`%"L8@L`C.,`*(T"``"L0PL$C0,``(T"&P"L8@L(
+MC0,```!I&"&,8@G`-$(0`*QB"<"-!```C04;]"0&_`",@@"DC*,`##P'_```
+M1A`D,&,#_P!#$"6L@@"DC00``(RC``PTY___C((`J#!C`_\`1A`D`$,0):R"
+M`*B-!```C*,`#(R"`*PP8P/_`$80)`!#$"6L@@"LC08``(RB`!`\!`/_C,,`
+MI``"%```1!`D`&<8)`!B&"6LPP"DC0,```!I&"&,8A$`-$(!`*QB$0`#X``(
+M``````!I&"&,8@G`-$(`(*QB"<`0`/^^,*(`"#P$``0`:1@AC&()P#1"`&*L
+M8@G`C0,```!I&"&,8A$``$00):QB$0`0`/^O)`(`!#P$`"4`:1@AC&()P#1"
+M`**L8@G`C0,```!I&"&,8A$``$00):QB$0`0`/^@)`(``SP%``@0`/]+-*4@
+MJB>]__"OL0`$K[```*^_``B,H@"$)`8`!`"@@"$40``&`("((8^_``B/L0`$
+MC[````/@``@GO0`0#`!*MHRE`&R.!0!L#`!*AP(@("$,`'IJC@0`A!``__.N
+M``"$)[W_X*^S``ROL@`(K[$`!*^_`!"OL````("8(0#`B"$0P``I))(<&(Y0
+M`(02```M`````"8$``@``"@A#`!PJB0&`!B6`@`.C@0`*(XE``2.)@`,,$(/
+M_PP`<)T`@B`AE@(`#HXD``R6`P`.,$(/_P!$$"$D!/``,$(/_P!D&"0`8A@E
+MI@,`#HXQ```6(/_M`````)8"``Z6`P`*KA(`3#!"#_\D0@`$`&08)#!"#_\`
+M8A@EI@,`"@)@("$"`"@A#`!8H0``,"&/OP`0C[,`#(^R``B/L0`$C[````/@
+M``@GO0`@#`!Z5R0$`-P`0(`A`$`@(0``*"$,`'"J)`8`7"8"`%RN4`"$K@``
+M`*X``""N`@`$K@(`*!``_\:N$``D)[W_L*^P`#``@(`AK[,`/*^Q`#2OOP!`
+MK[(`.`.@("$``"@A)A$<&`#`F"$D!@`L#`!PJHXR`(2.(@`$C@8;]``"&$``
+M8A@A/`*``B1"+ZP``QB``&(8(8QD``2,90`(C&,``"2$``$D`@`%KZ(`"*^E
+M`!2OI``0KZ,`#(S#``@D`@`")`8`!#!C`"`"`"`A)`4`!11@`!VOH@``#`!1
+MKR8$'&`00``))`/__X^_`$"/LP`\C[(`.(^Q`#2/L``P`&`0(0/@``@GO0!0
+M`@`@(0P`2F@#H"@ACD8`)`!`*"&N(@!L#`!*E0(`("&.)0!L#`!*G`(`("$"
+M`"`A#`!9$@)@*"$0`/_I```8(0P`2K8`````CZ4`"`P`2H<"`"`A$`#_W@``
+M```GO?_@K[$`!"0"__\DD1P8K[8`&*^U`!2OM``0K[,`#*^R``BOL```K[\`
+M'#P%``0`@(`A`,"8(0#@H"$!`)`A`4"H(0$@L"$0X@`FIB8`@@P`4EZ4A!MR
+M.$D``0(`("$"0"@A`F`P(0*`."&N(@`$`$`8(1*@``0X2``$)`(``P!(&`H`
+M"1@*``,0@`!0$"&,0AP`KA48`*XV`!RN`AQ4ED(``*8B`$*60@`"IB(`1)9"
+M``0,`"SJIB(`1H^_`!R/M@`8C[4`%(^T`!"/LP`,C[(`"(^Q``2/L````^``
+M"">]`"`D`@`!K((;L"0"``:L@`6,K(`8``P`*6RN(@`$`@`@(0)`*"$``#`A
+M#``LZ@``."$"`"`A#`!/:0(@*"$0`/_GC[\`'"2E__XLH@`F``!((1!``"0`
+M`%`A/`.``@`%$(`D8P_8`$,0(8Q"````0``(`````(R)!'@50``%`````%3@
+M``&LZ0``5,```:S```!5```!K0H```/@``@`````$`#_]8R)!'P0`/_SE(D<
+M\!``__&4B1SR$`#_[Y2)'/00`/_ME(D<]A``_^N4B1SX$`#_Z0``2"&,@AOT
+M$`#_YHQ)`#00`/_D)`H`#HR"&_00`/_AC$D`<(R"&_00`/_>C$D`.(R"&_00
+M`/_;C$D`/(R"&_00`/_8C$D`0(R"&_00`/_5C$D`1(R"&_00`/_2C$D`2(R"
+M&_00`/_/C$D`3(R"&_00`/_,C$D`4(R"&_00`/_)C$D`5(R"&_00`/_&C$D`
+M6(R"&_00`/_#C$D`7(R"&_00`/_`C$D`8(R"&_00`/^]C$D`9(R"&_00`/^Z
+MC$D`:(R"&_2,0@!L)$G__RTB``500/^T)`D`!!``_[(`````C((;]!``_Z^4
+M20!TC((;]!``_ZR420!VC((;]!``_ZF420!XC((;]!``_Z:420!ZC((;]!``
+M_Z.,20!\C((;]!``_Z",20"`C((;]!``_YV,20"$C((;]!``_YJ,20"(C((;
+M]!``_Y>,20",C((;]!``_Y2,20"0C((;]!``_Y&,20"4C((;]!``_XZ,20"8
+MC((;]!``_XN,20"<``40P`!%$"$``A#``$40(R>]_^```A"`K[,`#*^R``BO
+ML0`$K[```*^_`!``@H`AC@(`"`"`D"$`X)@A$$``""81``B/OP`0C[,`#(^R
+M``B/L0`$C[````/@``@GO0`@`B`@(0``*"$,`'"J)`8!'"9$'!@0@/_S)`(`
+M!HR#``008O_P)`(``:X"``@F`P`\)`4``R0"``$DI?__K&(```2A__TD8P`(
+M`F`X(28B``0F8P`@C.4``(SF``2,Z``(C.D`#*Q%``"L1@`$K$@`"*Q)``PD
+MYP`0%./_]B1"`!",XP``C.4`!(SF``BL0P``K$4`!*Q&``BN)`!8#`!39P(@
+M("$"("`A#`!2Q0``*"$,`'&T)B0`7!``_\N/OP`0``40P`!%$"$``A#``$40
+M(P`"$(``@B`A)[W_\"0"``:OOP``$,(`#B2%``@LP@`'$$``!R0"``<D`@`"
+M4,(``:RH``2/OP```^``"">]`!!0PO_\K*@`+!``__N/OP``)(0`$`$$$"4P
+M0@`#$$``'"4#`"``8!`AB0,``)D#``.)!@`$F08`!XD'``B9!P`+B0D`#)D)
+M``^H@P``N(,``ZB&``2XA@`'J(<`"+B'``NHB0`,N(D`#R4(`!`5`O_N)(0`
+M$($"``"@@@```*`@(0P`4L4D!0`!$`#_VX^_``"-`@``C08`!(T'``B-"0`,
+MK((``*R&``2LAP`(K(D`#"4(`!`5`__V)(0`$!``_^T```````40P`!%$"$`
+M`A#``$40(R>]__```A"`K[````""@"&OL@`(K[$`!*^_``PF$0`(CB(`>`"`
+MD"$\!8`"/`2``B2$$)PDI1!P%$``#B0&`0<,`'&[)@0`9`(@("$``"@A#`!P
+MJB0&`1RN```(C[\`#(^R``B/L0`$C[````/@``@GO0`0#`!P=``````F!0!D
+M#``0X`)`("$0`/_M`````">]__"OL@`(K[$`!*^P``"OOP`,`("0(0``B"$D
+MD``(C@(```(@*"$F$`$<)C$``11```H"0"`A*B(`!%1`__F.`@``C[\`#(^R
+M``B/L0`$C[````/@``@GO0`0#`!1;``````0`/_U*B(`!">]__"OL``````H
+M(20&`#BOOP`$#`!PJ@"`@"$,`'&T`@`@(8^_``2/L``````0(0/@``@GO0`0
+M`^``"(R"`!0GO?_PK[$`!*^P``"OOP`(#`!PM0"`B"&.,``4$@``"0````".
+M`@`@`%`8)@`#$`H40``"KB(`%*X@`!BN```@K@````P`<+H``````@`0(8^_
+M``B/L0`$C[````/@``@GO0`0)[W_\*^Q``0`@(@AK[(`"*^P``"OOP`,`,"`
+M(0P`<+4`H)`ACB(`*%!``!6N```@C@(`)*X0`""N`@``CB(`&%1```JL4@`@
+MKC(`%`P`<+JN,``8C[\`#(^R``B/L0`$C[````/@``@GO0`0CD,`)(XB`!BL
+M0P``$`#_]`````"N````$`#_[HXB`!@PQO__``800`!&$"$``A#``$00(91#
+M`"P\`A!B-$)-TP`#&(``8@`9``40P```&!```QF"`$,0(21"`!4`0P`;4&``
+M`0```<T``!`2``(0@"1"`"0#X``(,$+__S#&__\`!A!``$80(0`"$,``1!`A
+ME$,`+#P"$&(T0DW3``,8P`!B`!D`!1#````8$``#&8(`0Q`A)$(`%0!#`!M0
+M8``!```!S0``$!(``A"`)$(`%@/@``@P0O__,,;__P`&&$``9A@A``,8P`!D
+M&"$`!1(```4HP`!%$",D8P`HE&8`!``"$(``11`A``(0P`!&`!L``"@2$.``
+M!"0$`,"08P`))`(`8`!#(`L`A1`A)$(`"C!"__]0P``!```!S0/@``@`````
+M,,;__P`&$$``1A`A``(0P`!$$"&41``L/`(08C1"3=,`!!B``&(`&0`%$,`L
+MA`/H```8$``#&8(`0Q`A)$(`%0!#`!M08``!```!S0``$!(``A"`)$,`7"1"
+M`+T`9!`*`^``"#!"__\P@___,&((`!1```TD!``%,&("`!1```HD!``",&4W
+M`"0"%``0H@`&)`0``R0"%0`0H@`#)`0`!#!B`0```B`K`^``"`"`$"$#X``(
+MK(``.#"E`/\D`@`!`*(0!`!`&"$0P``%``(H)XR"`#@`0Q`E`^``"*R"`#B,
+M@@`X$`#__`!%$"0PI0#_%*``!(R#`#@``!`A`^``"*#````DI?__!*``"@``
+M$"$D`@`!`*(0!`!B$"040``')`(``22E__\$H__[`*(0!```$"$#X``(H,``
+M``/@``B@Q0``C*H`.*#@``"0H@`\)`@``0$"$"L00``C,,8`_P`($$``2!`A
+M``(0P``&&$``1$@A)`(``0!F&"$!`A`$``,8P`%"$"000``1`&08(94I`"R4
+M8@`L`2(0*U!```V0H@`\D.,````#$$``0Q`A``(0P!!@``4`1!`AE$(`+`!)
+M$"M00``#D*(`/*#H``"0H@`\)0,``3!H`/\!`A`K%$#_X``($$"0X@``%$``
+M`R0#``&@Y@`````8(0/@``@`8!`A)[W_T*^W`!ROM@`8K[$`!*^P``"OOP`D
+MK[X`(*^U`!2OM``0K[,`#*^R``B,@@!8))$`E"26``B,4P`\)`(`$Z(B`#P`
+M`+@A%*``$0``@"$`$!A``'`0(0`"$,``4Q`AD$(`-`!Q&"&@8``!H&(``)(B
+M`#PF`P`!,'``_P("$"L00``#`````!"@__(`$!A`#`!2<`(@("&2P@``CG0`
+M`!1``#,``*@A&H``$@``@"$`$!!``%`0(0`"$,``4Q`AC$,`)"0"``$"L4`A
+M)J<``0(`*"$"("`A$&(`'"0&``$F`@`!,%``_P(4$"H40/_Q`!`00*(U`&&2
+M8@,PHB(`8B;B``&B(@`\DF,#+P+C$"M40``!`N`8(:(C`#V/OP`DC[X`((^W
+M`!R/M@`8C[4`%(^T`!"/LP`,C[(`"(^Q``2/L````^``"">]`#"A$`!C#`!2
+M<C#U`/\"%Q`K$$```@(`&"$"X!@A$`#_W3!W`/\``(`A$$``&0``\"$:@``1
+M``"0(0`2$$``4A`A``(0P`!3("$"T!`AD$,``9""`#(P8P!_,$(`?U!B`"J,
+M@P`D)D(``3!2`/\"5!`J%$#_\@`2$$"2P@``)@,``3!P`/\"`A`K%$#_Z0``
+M``"2*``\``"`(0(($"L00``5CBD`.``0$$``4!`A``(0P"0&``$`4R@A`@80
+M!`$B$"0"L2`A)@<``1!```8FHP`!H)``8XRB`"@GQ``!$$8`"#!U`/\P\`#_
+M`@@0*Q1`_^X`$!!`HC4`81``_ZZB/@!B$`#_^#">`/\D`@`!`D`H(0(@("$4
+M8O_3)`8``0P`4G(``````E<0*Q!```("0!@A`N`8(1``_\LP=P#_)[W_\*^P
+M```DD`"4`@`@(0``*"&OOP`$#`!PJB0&`'P,`'%<```````#*(``HR@A``4H
+M0*X%`%2/OP`$C[````/@``@GO0`0``0F```%+@``!"8#``4N`P`&-@``!C8#
+M`(40*@"F0"H40``(`(8X*A$```0`H!@A`,`8(0#$$"H`@A@*`^``"`!@$"$0
+MX/_]`(`8(0#`&"$`IA`J$`#_^0"B&`N0IP``,(3__Q#@``L``!@A``,00`!%
+M$"&40@`"`((0*Q1```4D9@`!,,/__P!G$"M40/_X``,00`/@``@`8!`A)[W_
+MP*^^`#"OM0`DK[,`'*^Q`!2OOP`TK[<`+*^V`"BOM``@K[(`&*^P`!"4@AMR
+ME*,`"HRW`%```A("`("8(0"@J"$P<0__,%X``8RV`$R,HP`H```0(9"%&`,2
+MX``:D(08!XZB`#P``A)",$(``11```V.U``\CF(;V%!```N.H@`\D&(`!#!"
+M``%40``'CJ(`/)!B```D`P`(,$(`#!!#`!T`````CJ(`/"0#_?\`0Q`DKJ(`
+M/(YB'!12@@`.DL(`@);"`!Z/OP`TC[X`,(^W`"R/M@`HC[4`)(^T`""/LP`<
+MC[(`&(^Q`!2/L``0`^``"">]`$!00/_SEL(`'I:D``J.A0,T#`!3D#"$#_\0
+M`/_NC[\`-!"@``\F\`"4)`(``5""`0"2`@`]5(``"X($`":2`P`]``,00`!#
+M$"$``A#``%00(8Q#`"@D`@`!4&(`\)*"``."!``F@@4`*`P`4WF"!@`I#`!Q
+M7`!`D"$``RB`C@8`4`"C*"$`!2A``*8@(RR"`+D40`#;```8(20#``HN(@,A
+M%$``!)(&`&$D8@`!``(6```"'@.2!P!B`D,0(P`"%@`"0Q@J``(6`SAC```D
+MQ/__``"0(3#H`/\`0Y`*,(0`_P``8"$1```'``!((1?``+\!!A`K`(<0(S!$
+M`/\0@`!H``"((0$P6"$`B1`J`(`P(11``!N1:`!C`)`0(21'`&.0\0``),;_
+M_P#)4"H`$1!``%$8(0`#&,``="`A`%`8(8!B```"0A`J%$``"R3G__^08P`!
+ME(0`+B0"`&0`0Q`C<((8`@!L$"L40``#``````!@8"$"($`A44#_ZI#Q``"B
+M$@`GCJ(`/``"%L(P0P`/$&``!@$`B"&.8AMH``(00@!#$"M40``!D7$`8Y(#
+M`#T`<1`K4$``,)+C`-"2!P`\)&(``0!@B"$P0P#_`&<0*P.@4"$OR0`!)`L`
+M`1!``!.."``X``,00`!#$"$``A#``%0@(20"``$`8A`$`0(0)!!```8D9@`!
+M$2``=B0"``&,@@`H%$L`<R0"``$PPP#_`&<0*Q1`__```Q!`H4``````$"%0
+M0``0DN,`T(X"`%B.@P,D`*(0(P!B&"M08``*DN,`T)("`%TL0@`$5$``!I+C
+M`-"3L0``K@4`6*(``%VB$0`^DN,`T"1B__\`41`J4$``!(YC'!PD8O__,%$`
+M_XYC'!PD`@`%4&(`0XYC&^"2P@"`$$#_2@(@$"&6H@`*/`1KRC2$&O,P0@__
+M``(8@`!B&"$``QD``&0`&``1$$``41`A``(0P`!4$"&40@`L```8$``#*0(`
+M11`K$$#_.`(@$"&2`P`\)B(``3!$`/\`@Q@K`Z!((2_(``$D"@`!$&``%(X'
+M`#@`!!!``$00(0`"$,``5!@A)`(``0""$`0`XA`D$$``!B2&``$1```8)`(`
+M`8QB`"@42@`5)`(``9("`#PPQ`#_`((0*Q1`_^\`!!!`H2``````$"$00/\8
+M`B`0(9.Q````$1!``%$0(0`"$,``5!`AE$(`+`!%$"M40/_9D@,`/!``_PT"
+M(!`A$`#_\J$D```"(Q`K$$``"0)@("&.8@6H4$#_NI+"`(``0Q`K5$#_MJY@
+M!:@0`/^UDL(`@`P`5NXD!0`!$`#_L9+"`(`0`/^3H4,``!!`_T8!,%@A`,<0
+M(Q``_T(P20#_+((`&11`_R8N(@,A)(+_YP`"%0`0`/\A``(>`Y*#`S``0Q`C
+M)$+__Q``_PVB`@`]``(80`!B&"$``QC``'08(8QB`"@01/[[`````!``__62
+M@@`#)[W_L``*5@`Q:P#_K[X`0*^W`#ROM0`T`("X(:^T`#"OL0`D`*"H(:^_
+M`$2OM@`XK[,`+*^R`"BOL``@`,"((:^H```!(*`A``KV`PP`<5ROJP`$``,H
+M@`"C*"$`!2!`KZ0`"*^@`!"2XQ@#$J`!"9+D&`>.H@!8)K``E!!@``2,4P`\
+M)`(``5""`8B28@,O`!&P0`+0D"&21``!CZ,``#""`/\08`$=KZ(`#"0"``$0
+M8@$3)(,`(Y("`#PF)0`!`*(0*A!``!D```````480`!E$"$``A#``%,0(21&
+M`"@`<"`A`M$0(0`"$,``4Q`AC$,`*(S"```"T#@A)*4``11#``HDQ@`8D.,`
+M`9""``$`0Q`K5$```:"#``&2`@`\`*(0*A1`_^\DA``"6B``%9("`#Z"`@`F
+M@@0`)P`1&$``<"@A)$+__@!$&"J`I@```(,0"P`"%@```AX#),(``@!#$"H0
+M0`#@D*0``"2"``2@H@``)`0``:(1`%ROI``0D@(`/E!```,`$1!`$%$`#@`1
+M$$``4!`A@$,````#$(``0Q`A``(00`!`&"$D1``/*$(```""&`L``QD#H@,`
+M*:(#`":B`P`HH@``/@`1D$`"4!`AD$(``2Q"`#T40``0CZ(`$!H@``X"41@A
+MD@0`/0`#&,``!!!``$00(0`"$,``<Q@A`%,0(91C`"R40@`L`$,0*Q!``*T"
+M`"@ACZ(`$!!``$@`$3A`D@(`/"1"__\"(A`J$$``'0(@*"$`$1A``'$0(0`"
+M$,``4S@A`'`P(0)1$"$``A#``%,0(8Q#`"B,X@!`%$,`$22E``&`X@`U@,,`
+M`(#$``*0Z``U`&(8(0"#("HDYP`8$(```Y#"````2!`AH,(``I("`#PD0O__
+M`*(0*A1`_^HDQ@`")B7__P2@`",`!1A``&40(0`"$,``4S@A`'`P(0`1$$``
+M41`A``(0P`!3$"&,0P`HC.(`*%1#`!@`$3A`@.0`-8##``"`P@`"D.@`-0!D
+M&"$`1"`J`$,0*A!```J0PP`"$(``;0!H$".@P```@,(``(#C`#0`0Q`J$$``
+M`I#D`#2@Q```)*7__R3&__X$H?_C).?_Z``1.$``\!`AD$(``8^C``P`0Q`K
+M4$``&8X"`%0F)?__!*``%0`%&$``91`A``(0P`!3$"$D1@`H`'`@(0#Q$"$`
+M`A#``%,0(8Q#`"B,P@``)*7__Q1#``@DQO_HD(,``Y""``$`8A`K5$```:"#
+M``$$H?_R)(3__HX"`%2/I``(CF,#*`""$",`0Q`K5$``&(X"`'B2`@`\$$``
+M$```*"$F9@`T`@`@(8"#``"`P@``)*4``23&`!@`0Q`J$$```Y"#```D8O__
+MH((``)("`#P`HA`J%$#_]"2$``*/I0`(K@4`5(YC`RB.`@!XCZ0`"``#&$(`
+M@A`C`$,0*Q1``!:/OP!$D@(`/!!``!```"@A`@`P(9##``$DI0`!``,0P`!#
+M$",`0!@A)$0`!RA"````@A@+``,8PZ##``&2`@`\`*(0*A1`__,DQ@`"CZ4`
+M"*X%`'B/OP!$C[X`0(^W`#R/M@`XC[4`-(^T`#"/LP`LC[(`*(^Q`"2/L``@
+M`^``"">]`%`0`/^4H,(``#(F`/\F!P`]#`!2E0)@("&/I0`($`#_3JX%`%@D
+M8P`"`,,0*A!`_Q\D@@`"$`#_'J"C```P8@#_+$(`911`_NNB0P`!)`(`9!``
+M_NBB0@`!*H,`"B0"``D`0Z`*/`*``B1"1!``%!B`CZ4`#`!B&"&,8P````40
+MP@""$",``QC"`$,0(:)"``&/J``$`N`@(0*@*"$"@#`A#``RXP/`."&2`@`H
+MD@,`)I(%`#ZB`@`IH@,`**(>`":/H@`(,*0`_Q"```.N`@!0$)$`-"J"``)>
+M@/\,`!&00)(#`%TD`@#_$$,``B1B``&B`@!=`M$0(0`"$,``4Q`A)$<`,(#B
+M``0#PA`J5$#^_P`1D$".`@`P4B(`!8X"`"RN$0`PK@``+*X``#2.`@`LC@,`
+M-"1"``$`?B`A*$,`!*X$`#048/[PK@(`+*X``"RN```T`M`H(22"``*`I@``
+M*$,``"2$``4`@Q`+``(0@R3#``(`0Q`J$$``!Y"C``"`X@`$`$80*A!```0D
+M!``!)&+__Z"B```D!``!$`#^VJ^D`!!40``#H@4`/1``_\JB```^DD(``2Q"
+M`"Y40``$H@``/B0"`!2B0@`!H@``/HYB`R2/HP`(``(00@!B$",0`/^]K@(`
+M6"1"``$"(A`J$$#^=P`1L$"28@,P`!%`0`$1&"$"(B@A``400`!%$"$``QC`
+M``(0P`!S&"$`4Q`AD&,`,)!"`#!08OYH`*"((8YG```DI0`!`*<0*A!`_F0`
+MH#`A``400`!%$"$``A#``%,0(21$`#`!$1`A``(0P`!3$"&00P`PD((``!!B
+M``<DA``8),8``0#'$"H40/_W`1$0(1``_E(`$;!`$`#^3P#`B"$GO?_@K[``
+M$*^_`!ROL@`8K[$`%(R"`$R,A`!0,+#__XQ2`#P0@``3`@`0(221`)2CL```
+MDZ<```(@("$#H#`A``<00`!'$"$``A#``%(0(8Q#`"@D`@`"$&(`#`#@*"$,
+M`%)_`````!1`__.3IP```@`0(8^_`!R/L@`8C[$`%(^P`!`#X``()[T`(!``
+M__D`X!`A%*```P`````#X``(K(`%J`2B``R,@QP<&*``!B0"``6,@QP<4&(`
+M!(R"!:@#X``(`````(R"!:@`11`A`^``"*R"!:@D`@`%%&+_\P`````0`/_V
+M`````"2E__\GO?_@+*(`(J^U`!2OM``0K[(`"*^Q``2OL```K[\`&*^S``P`
+M@(@A`,"0(0#@@"$!`*@A$$``"P``H"$\`X`"``40@"1C$+P`0Q`AC$(```!`
+M``@`````#``L;@#@*"$`0*`A5J```:ZT``"/OP`8C[4`%(^T`!"/LP`,C[(`
+M"(^Q``2/L````^``"">]`"`0`/_TKB<;V*XG'#00`/_QK(`;V!``_^^N)QO(
+M$`#_[:XG&V@0`/_KIB<;W!``_^FN)QO@$`#_YZR`!:@0`/_EKB<;U!``_^.N
+M)QNT$`#_X:(G&[L0`/_?HB<;Y1``_]VN)QO,)(0;H@P`<)T`X"@A$`#_V```
+M```\$X`#CF2*M!"```8\`H`#C$**N!!&``H`````#`!Z:@`````,`'I7`D`@
+M(:YBBK0\`H`#K%**N#P3@`..9(JT5(```P(`*"$0`/_$)!0``@P`<)T"0#`A
+MCF**M!``_[^N(AO\$`#_O:XG&VP0`/^[KB<;O!``_[FN)QN<$`#_MZ8G&\00
+M`/^UIB<;P!``_[.F)QO"$`#_L:8G&\8PX@#_$`#_KJR"&S@PX@#_$`#_JZR"
+M&SP0`/^IKB<;-(R"``0`X"@A#``9T8Q$```40``"`$"@(:XP&^P0`/^@KC`;
+MZ">]__"OL```K(4;"`"`@"&OOP`$#``4;20%``&.!1L(#``LN@(`("$"`"`A
+M#``4;0``*"&/OP`$C[````/@``@GO0`0)[W_\*^_```,`&M,`````(^_```#
+MX``()[T`$">]_]``H!`AK[```#P%@```@(`AK[\`)*^^`""OMP`<K[8`&*^U
+M`!2OM``0K[,`#*^R``@`X)@A,-+__Z^Q``0!`*`A`2"H(0%`\"$!8+`A,%'_
+M_PP`*6R/MP`T#`!Q>XX$&`@"`"`A```H(0``,"$,`"SJ```X(1;@`!,`%B@K
+M,B,W`"0")0"F$AMPIA,;?*(4&WZB%1M_IA$;<A!B`&```"@A)`(5`%!B``$D
+M!0`!C@080`P`<7NN!1@$#`!QPB8$%^P`%B@K#``*2`(`("$,``B5`@`@(0P`
+M<30`````#`!PM0`````F!1MP`\`P(0P`-X4"`"`ACZ,`,!+``#FL8@``#`!P
+MN@`````,`'$[``````(`("$,`"D\/`6``!;@``0`````C@(;5"1"``&N`AM4
+M#``(?@(`("$,``H8`@`@(1+@`!@`````#`!Q7``````D8P#(C@08""QE`,@`
+M11`A`$`P(20(```D"0#(#`!Q=`!@."&/OP`DC[X`((^W`!R/M@`8C[4`%(^T
+M`!"/LP`,C[(`"(^Q``2/L````^``"">]`#`,`''0)@07[`P`<5P`````)&,`
+MR"QE`,B.!!A``$40(0!`,"$D"```)`D`R`P`<70`8#@A$`#_VP`````,`%)>
+M`B`@(0`"&(``<!@AC&,<`(X$&^BN`AP<K@,<5!"```0``!@A,B(@`%1```$D
+M`P`!$`#_NJ(#')@0`/^C)`4``22E__\GO?_@+*(`#Z^U`!2OM``0K[,`#*^R
+M``BOL0`$K[```*^_`!@`P*`A`."`(0$`J"$``(@A``"0(1!``%@``)@A/`.`
+M`@`%$(`D8Q%$`$,0(8Q"````0``(`````!#@`",D$@#,`.`8(22"&QPDA1M<
+MC$8``(Q'``2,2``(C$D`#*QF``"L9P`$K&@`"*QI``PD0@`0%$7_]B1C`!`D
+M@QOH)@(`0"2$&VB,A0``C(8`!(R'``B,B``,K$4``*Q&``2L1P`(K$@`#"2$
+M`!`4@__V)$(`$(R)``",@P`$C(4`"*Q)``"L0P`$K$4`"!9@``<`````%D``
+M`P````!6```!KA$``%:```&ND@``5J```:ZS``"/OP`8C[4`%(^T`!"/LP`,
+MC[(`"(^Q``2/L````^``"">]`""4@AMR$`#_ZP`"BP(0`/_IE)$;<!``_^>,
+MD1LL$`#_Y8R1&S`0`/_CC)$;2!``_^&,D1M,$`#_WXR1&U`0X/_=)!(`!B2%
+M&Z@D!@`&#`!PG0#@("$0`/_7`````!``_]6,D1M4$`#_TR03``X0X/_1)!(`
+M$`#@*"$,`"Q8)`8`$!``_\P`````$`#_RHR1&^P`@#@A```P(1"@``@``!@A
+M`,<0!B3&``$P0@`!``,80`#%("L4@/_Z`&(8)0/@``@`8!`A)[W_\*^R``BO
+ML0`$K[```*^_``R,H@!,)+``"`"@B"&020!`C$H`/*R@`!R,HP`<``D00`!)
+M$"$``A#`K*,`&`!*0"&1!0`PC@,`#(X'``0D`O_@`&(8)#"E`!\D`N__`.(X
+M)`!E&"6N!P`$K@,`#(T#`"@D`@`#`(`X(1!B`"L`P)`AC@,`"(X$``0\`O_P
+M-$+__P!B&"0\`@$``((@)3P"``$`8A@EK@0`!*X#``B.(@`(/`,`/SP$X?\`
+M0Q`E-(3__S)#``\`1!`D``,>0`!#$"6N(@`(C@(`!#P#_P\T8___`$,0)#P#
+M`#``0Q`E/`,&``!#$"6N`@`$CB(`"(XD`"0\`P"``$,0):XB``BN)```C[\`
+M#(^R``B/L0`$C[````/@``@GO0`0CB(`"#P#@``\!?X/`$,0):XB``B0A!T`
+MC.8<5(X#``P`!!!``$00(0`"$,``1A`AD$(`,#2E__\`91@D,$(`'P`"%0``
+M8A@EK@,`#(T#`"@\`H`")$)$```#&(``8A@ACB4`"(QB```!0"`A,*4/_P$@
+M,"$`0/@)```X(8X#``@D!(``,$)__P!D&"0`8A@E$`#_L:X#``@GO?_@K[``
+M$`"@@"$#H"@AK[$`%*^_`!@,`"WM`("((5(```$```'-CZ,`!(^B```T"8``
+M``,=@``"$H(`8A@E`'`X(0#P$"$D0O__`%``&XXC``"6)1N"-`B```!I&"$R
+M"O__```0$@``````````<%`X`JQG`"B.(@````<8P"1F__P`21`AK$8`+(XB
+M```D9O^P`.4@(0!)$"&L1@`PCB(``"3C``$`92`*`$@0(:Q$`#2.(@``/`,`
+M@`(#,"4`2!`AK$8`((XB&X@00``AC[\`&(XB``".)1N0`$@0(:Q*`0".)```
+MCB,;B`"(("&,@@$```,<`#!"__\`0Q`EK((!`(XB&XR.(P```.(0(0!H&"$P
+M0O__K&(`_!"@``>.(@``/`0``@!($"&,0P#\`&08):Q#`/R.(@``/`0``0!)
+M$"&,0P#\`&08):Q#`/R/OP`8C[$`%(^P`!`#X``()[T`(">]_]"OL@`(,)(`
+M_Z^W`!ROM@`8K[4`%*^T`!"OLP`,K[```*^_`""OL0`$`*"@(0#`J"$`X+`A
+M`0"X(0$@@"$!0)@A$D``#"0"``&/OP`@C[<`'(^V`!B/M0`4C[0`$(^S``R/
+ML@`(C[$`!(^P```#X``()[T`,`P`>E<D!``L`$"((1(@__$D`@`"`B`@(0``
+M*"$,`'"J)`8`+"0"``*N\0``/`:``:("```D`@`*)B@`&"3&:<P``"`A```H
+M(0(@."&B8@``KC0`!*XU``BN-@`,#`!@6:(@```,`'I7)`0%$`!`@"$00``J
+MKB(`%`!`("$``"@A#`!PJB0&!1`F!P`(K@<`#*X0``2,Y@`$K@``"*X````"
+M`"`A```H(22E``$D@@`0+*,`0*R``!"LP@``)(0`%!1@__D`0#`A`B`@(0)`
+M*"$,`%\JK.(`!!1```,`````$`#_O```$"$,`&!(`B`@(8XE`!@,`&"'`D`@
+M(8XD`!0,`'IJ`$"`(0P`>FH"("`A`@`0(1``_Z^NX```#`!Z:@(@("$0`/^K
+M)`(``B>]__`LH@`$K[(`"*^Q``2OL```K[\`#`"@B"$`@(`A%$``"`#`D"$D
+M`O__C[\`#(^R``B/L0`$C[````/@``@GO0`0#`!A$9"$``"2!````!$0@`!0
+M$"$,`&$BK%(`'!``__(``!`A,*4`_R>]__`LH@`*K[\``!!```TD!___/`.`
+M`@`%$(`D8T1`)*4``0!#$"$0@``&,*4`_XQ'```,`&(NC(0`$"0'__\``C@*
+MC[\```#@$"$#X``()[T`$#"E`/\GO?_P+*(``J^_```00``()`/__R2E``$0
+M@``%,*4`_PP`8E^,A``0)`/__P`"&`J/OP```&`0(0/@``@GO0`0)[W_\*^_
+M```\`H`"D$)$:`"`2"$`X%`A,*,`_Q!``!<PQ`#_)&7__P#@,"$PI0#_$(``
+M#P$`."$D9?__)`(``3"E`/\0@@`$`4`P(8^_```#X``()[T`$(TD``2-(@`,
+M`$#X"0`````0`/_YC[\``(TD``00`/_ZC2(`"(TG`!0D:___.((``8SC``@L
+M1@`!)`4``1!@_^TDY``(C&(``!1```*LX@`(K(0`!*!K``B@9@`)K&H`#*QH
+M`!"@8``$C.(`!(TD`!BL8```K.,`!`P`8(.L0P``$`#_W8^_```GO?_P/`*`
+M`J^_``"00T1H``40@`"@0"$08``+`$00(8Q"`!P40``$`````(^_```#X``(
+M)[T`$`!`^`F,A``$$`#_^X^_``",A@`4)`4``8S#``@08/_U),<`"(QB```4
+M0``"K,(`"*SG``0D`@`!H&(`!*QH``B,P@`$C(0`&*Q@``"LPP`$#`!@@ZQ#
+M```0`/_GC[\``">]_^"OL0`$K[\`%*^T`!"OLP`,K[(`"*^P```\`H`"D$)$
+M:!!```D`@(@AC[\`%(^T`!"/LP`,C[(`"(^Q``2/L````^``"">]`"".,@`4
+MD(0```P`83,F4P`(CE````P`84R2)```$@#_\8^_`!0,`&$SDB0``(Y"``".
+M%```C$(``!1```*N0@``KE(`!`P`84R2)```D@(`!"0#``%00P`BC@(`"%1`
+M``<"`"`AD@(`"5!``!N2!0`(4$,`$I(%``@"`"`A```H(0P`<*HD!@`,#`!A
+M,Y(D``".8@`$K@```*YP``2L4```DB0```P`84P"@(`A%H#_W8^_`!00`/_,
+MC[0`$(XB``R.)``$C@8`#`!`^`F.!P`0$`#_Z@(`("$0`/_YCB(`"``"$(``
+M41`AC$(`'%!`_^,"`"`A`$#X"8XD``00`/_?`@`@(2>]__"OOP``#`!A,Y"$
+M``"/OP`````0(0/@``@GO0`0)[W_\*^_```,`&%,D(0``(^_`````!`A`^``
+M"">]`!`\`H`"H$!$:`/@``@``!`A)`,``3P"@`*@0T1H`^``"```$"$GO?_P
+MK[```*^_``0,`'"U`("`(0P`6L<"`"`A#`!F*8X$`!`,`%IS`@`@(0P`6L\"
+M`"`A#`!PN@````"/OP`$C[``````$"$#X``()[T`$">]__"OOP``#`!H%HR$
+M`!"/OP```^``"">]`!`GO?_PK[$`!*^P```PT0#_,+``_P(`*"$"(#`AK[(`
+M"*^_``P,`&'W`("0(0)`("$"`"@A%$``"0(@,"$``!@AC[\`#(^R``B/L0`$
+MC[````!@$"$#X``()[T`$`P`8A8`````%$#_]@!`&"$"0"`A`@`H(0P`8?<"
+M(#`A%$#_\"0#``<0`/_N```8(2>]__"OOP``/`*``I!&10,PI0#_`*`X(0``
+M0"$``"@A#`!AYP``2"&/OP```^``"">]`!`PI0#_)[W_\`"@,"&OOP``#`!A
+M8P``*"&/OP```^``"">]`!`GO?_PK[```#"P`/^OL0`$`@`H(:^_``@,`%LK
+M`("((0(`,"$"("`A#`!:^P``*"$"("`A#`!;'@(`*"&/OP`(C[$`!(^P```#
+MX``()[T`$">]__"OOP``/`*``I!#1&D`P#@A)`(`0#AC``(D!@(``$,P"S#G
+M`/\PI0#_)`@``@P`8><``$@AC[\```/@``@GO0`0)[W_\##&`/^OOP``#`!A
+M8S"E`/^/OP```^``"">]`!`GO?_PK[$`!*^P```PT0#_,+#__Z^R``@"`"@A
+M`("0(:^_``P,`%M9`B`P(0)`("$"`"@A#`!:^P(@,"$"0"`A`@`H(0P`6T@"
+M(#`AC[\`#(^R``B/L0`$C[````/@``@GO0`0)[W_\*^R``BOOP`,K[$`!*^P
+M```0H``.`("0(9##``,D`@"!$&(`/2AB`((00``E)`(`@B0"`(`08@`+/!&`
+M`@)`("$``"@A#`!C$@``,"&/OP`,C[(`"(^Q``2/L````^``"">]`!`F,$?"
+M`@`P(0P`8J4D!0`&EB-'P@(`,"$"0"`A,&(`_P`"$@```QH"`$,0)0``*"$D
+M!P`"IB)'P@P`8E\``````D`@(0``*"$``#`A#`!B+@``."$0`/_EC[\`#%1B
+M_]\"0"`AD,,`!SP0@`(F$4?",&,`#S1E`!`"(#`A/`*``@P`8J6@0T?&E@-'
+MP@(@,"$"0"`A,&(`_P`"$@```QH"`$,0)0``*"$D!P`"$`#_XJ8"1\(\`H`"
+M)$9'P0``*"$D!P`!$`#_W*!`1\$GO?_@K[(`&*^P`!``H)`A`,"`(20%``$#
+MH#`AK[$`%*^_`!P,`&*E`("((9>B```L0@`"$$``"@(@("$20``,C[\`')(#
+M``-08``DE@,``"0"``)08@`+E@(```(@("$``"@A#`!C$@``,"&/OP`<C[(`
+M&(^Q`!2/L``0`^``"">]`"`40/_V`B`@(9("``<\$(`#,$(`#S1%`!`#H#`A
+M#`!BI:("BKR2!8J\`B`@(0``,"$TI0`0#`!BUP`````"("`A```H(0``,"$,
+M`&)?```X(1``_^:/OP`<)`(``11B_]\"("`A`Z`P(0P`8J4D!0`&EZ(```(@
+M("$D!0`&,$+__0!`,"$0`/_KIZ(``">]_^"OL0`4K[``$*^_`!ROL@`8`,"`
+M(1"@`$@`@(@AD,,``U!@`"*4PP``)`(``E!B``N4P@```B`@(0``*"$,`&,2
+M```P(8^_`!R/L@`8C[$`%(^P`!`#X``()[T`(!1`__<``"@AD-``!P.@,"$R
+M$``/-A``$`P`8J4"`"@A`B`@(0(`*"$D!@`!#`!BUP`````"("`A```H(0``
+M,"$,`&)?```X(1``_^B/OP`<)!(``1!R`!<D`@`"5&+_WP(@("&0P@`'%$#_
+MW#P"@`*00D1I%&+_V@``*"$D!0`!#`!BI0.@,"&7H@``+$(``Q!`_]$\`X`"
+ME@(`!JQR1&PP0O\`/`.``J1B1'`0`/_B`B`@(0.@,"$,`&*E)`4`!I>B```"
+M("`A)`4`!C1"``(`0#`A$`#_U:>B```\`H`"C$)$;%!`_\&/OP`</`*``I1&
+M1'`D!0`'#`!BUS#&__\0`/^ZC[\`'">]__"OL0`$K[\`"*^P```0H``-`("(
+M(9##``$``"@A```P(0``."$\`H`##`!B7Z!#BKV/OP`(C[$`!(^P```#X``(
+M)[T`$#P"@`.01HJ])`4``PP`8M<D$``!`B`@(20%``$,`&+7)`8``0(`*"$"
+M("`A#`!;820&``$F`@`!,%``_RX"``,40/_Y`@`H(200``$"`"@A`B`@(0P`
+M6V$``#`A)@(``3!0`/\N`@`#%$#_^0(`*"$0`/_?C[\`"">]__"OL```K[\`
+M!`#`."$`@(`A$*``$0``0"&4PP``)`(#`#!E_P`0H@!5**(#`1!``#0D`@8`
+M)`(!`!"B`"HD`@(`$*(`"3P"@`("`"`A```H(0P`8Q(``#`AC[\`!(^P```#
+MX``()[T`$)!"1&DD`P!`.$(``@!B*`LE!``!``@8P#P"@`(`:!@C)$9%'#"(
+M`/\`9A@A``42`BT$``2@8@`7%(#_]:!E`!:4XP`$)`<`+@(`("$L8@`N`&(X
+M"P``*"$,`&)?``````(`("$``"@A```P(0P`8BX``#@A$`#_WX^_``24PP`$
+M)`<`$CP&@`(L8@`2`&(X"Q``__`DQD3\$*(`%R0"!P`4HO_0`@`@(3P"@`*0
+M0D1I)`4`0"0#`@`X0@`"`&(H"R4$``$`"!C`/`*``@!H&",D1D6$,(@`_P!F
+M&"$`!1("+00`!*!B`!<4@/_UH&4`%A``_]24XP`$E,,`!"0'``H\!H`"+&(`
+M"@!B.`L0`/_1),9%$#!C`/\L8@`$%$``"#P"@`(\!H`"D,-$?)3G``0DQD1\
+M`.,0*Q``_\8`8C@*)$(P```#&(``8A@AC&8``)3G``0``"@AD,,```#C$"L0
+M`/^]`&(X"B>]__```"@AK[\```P`8Q(``#`AC[\```/@``@GO0`0)[W_X`"@
+M$"&OL``0K[\`%`.@,"$D!0`$%$``!0"`@"&/OP`4C[``$`/@``@GO0`@#`!B
+MI0````"3HP`!/`*``B1&1\```"@A)`<``0(`("$,`&)?H$-'P`(`("$``"@A
+M```P(0P`8BX``#@A$`#_[8^_`!0GO?_@K[$`%*^P`!"OOP`8`,"((1"@`!L`
+M@(`AD,,``2QB``(00``=```H(11@`!\#H#`A)`4``0P`8J4#H#`AEZ(``"Q"
+M``(00``3`@`@(20%``0,`&+7```P(0(`("$D!0`!)`8``0P`8M<``````@`@
+M(0``*"$``#`A#`!B7P``."&/OP`8C[$`%(^P`!`#X``()[T`(```*"$,`&,2
+M```P(1``__B/OP`8#`!BI20%``22)@`!EZ(``%!&``@D!0`!`@`@(0P`8M<D
+M!0`$`@`@(20%``$0`/_C```P(0``,"$0`/_@`@`@(2>]_^"OL@`8K[$`%`"@
+MD"$`P(@A)`4``0.@,"&OL``0K[\`'`P`8J4`@(`AEZ(```(`("$``"@A%$``
+M%P``,"$60``'`````(^_`!R/L@`8C[$`%(^P`!`#X``()[T`()8G``22)@`'
+M/`*``B1"1'0`PC`A#`!B7P`'."L"`"`A```H(0``,"$,`&(N```X(1``_^Z/
+MOP`<#`!C$@`````0`/_JC[\`'">]__"OOP``$*``"0#`2"&0PP`#)`(``0``
+M*"$``#`A$&(`!@``."$,`&,2`````(^_```#X``()[T`$)$C``<\`H`")$)$
+M=`!B&"&1*``!D&(`````*"$02``#```P(9$B``&@8@``#`!B7P`````0`/_P
+MC[\``">]_^"OLP`,K[```*^_`!2OM``0K[(`"*^Q``0`P(`A$*``"`"`F"&0
+MT0`#)`(``@``*"$2(@`+```P(0P`8Q(`````C[\`%(^T`!"/LP`,C[(`"(^Q
+M``2/L````^``"">]`"`\`H`"D$-%'9("``<\$H`")E1'Q`!#$"LD!0`%%$``
+M!`*`,"$``"@A$`#_ZP``,"$,`&*E`````)8&``261$?$```H(2S"``(P@P#_
+M`,*("P`$(@(``QH``&08)0*`,"$"(#@A`F`@(0P`8E^F0T?$`F`@(0``*"$`
+M`#`A#`!B+@``."$0`/_7C[\`%">]_^"OLP`,K[(`"*^Q``2OL```K[\`$#P0
+M@`,`P)@AC@:*P`"`D"$`H(@A/`2``CP%@`(DA!&`#`!P="2E$9".`HK`)$(`
+M`1(@``>N`HK`EF,``"0"``$08@`A)`(``E!B``B69@`&C[\`$(^S``R/L@`(
+MC[$`!(^P```#X``()[T`(##%``\DH___,&+__RQ"``(00``$,&+__S#"`(`4
+M0``,,&+__RQ"``(00/_N,,(`@%1`_^V/OP`0`D`@(0``,"$,`%MA`````!``
+M_^>/OP`0`D`@(1``__HD!@`!)!```0(`*"$"0"`A#`!;820&``$F`@`!,%``
+M_RX"``,40/_Y`@`H(200``$"`"@A`D`@(0P`6V$``#`A)@(``3!0`/\N`@`#
+M5$#_^0(`*"$0`/_.C[\`$">]__"OL```K[\`!!"@``<`@(`AD,(``A!```@`
+M```````H(0P`8Q(``#`AC[\`!(^P```#X``()[T`$`P`7>X``````@`@(0``
+M*"$``#`A#`!B7P``."$0`/_UC[\`!#P"@`*00D7L$$```CP"@`*@2$1I`^``
+M"``````GO?_PK[```*^_``0\`H`"D$)%[```*"$40``%`("`(8^_``2/L```
+M`^``"">]`!`,`%LTC(0`$(X$`!`,`%LT)`4``0(`("$,`%I-)`4``A``__2/
+MOP`$)[W_\*^_```\`H`"D$)%[!1```0``"@AC[\```/@``@GO0`0#`!:30``
+M```0`/_[C[\```/@``@``````^``"``````GO?_@K[$`%*^P`!"OOP`8`*"(
+M(12@`($`@(`A/`*``B1&1[B0P@`#,$,`8"0"`"`08@`)*&(`(1!``'(D`@!`
+M4&``"I#&``*.!``0```H(0P`8Q(``#`AC[\`&(^Q`!2/L``0`^``"">]`"`L
+MP@`-$$#_]3P#@`(`!A"`)&,1G`!#$"&,0@```$``"`````".!``0/`:``@(@
+M*"$,`%MZ),9'N!``_^V/OP`8C@0`$#P&@`("("@A#`!;QR3&1[@0`/_FC[\`
+M&(X$`!`\!H`"`B`H(0P`7`LDQD>X$`#_WX^_`!B.!``0/`:``@(@*"$,`%QE
+M),9'N!``_]B/OP`8C@0`$#P&@`("("@A#`!<E23&1[@0`/_1C[\`&(X$`!`\
+M!H`"`B`H(0P`70HDQD>X$`#_RH^_`!B.!``0/`:``@(@*"$,`%T2),9'N!``
+M_\./OP`8C@0`$"0%``$,`&*E`Z`P(8X$`!`\!H`"`B`H(0P`72\DQD>XC@0`
+M$">F``(,`&*E)`4``9>B```00/^REZ(``E1`_[&/OP`8`@`@(0P`6DTD!0`#
+M$`#_K(^_`!B.!``0/`:``@(@*"$,`%UJ),9'N!``_Z6/OP`8C@0`$#P&@`("
+M("@A#`!=E"3&1[@0`/^>C[\`&(X$`!`\!H`"`B`H(0P`7;(DQD>X$`#_EX^_
+M`!@48O^1C@0`$`P`7CP"("@A$`#_D8^_`!B,A``0/`:``B3&1[@,`&(/```H
+M(1``_WL\`H`")[W_\*^_```\`H`"D$)%[#"C`/\PZ@#_$$``"`#`*"$!`#@A
+M`4`P(0$@0"$08``*+&(``Q1```0`8"@AC[\```/@``@GO0`0#`!:%0`````0
+M`/_[C[\```P`7H(`````$`#_]X^_```GO?_@K[$`!#"Q`/^OLP`,K[(`"`"`
+MF"$`$9#``B`@(:^_`!`,`&$SK[````)1$"$``A"``%$0(SP#@`,D8Z&```(0
+M@`!#$"$D0P`8E&0`!!"```H"41`A-`+__Q""``8`!"("D&(`!3P#@`(D8T3\
+MH&0`":!B``@"41`A``(0@`!1$",\`X`#)&.A@``"$(``0Q`A)$,`&)1E``8P
+MI/__4(``\B0#``,T`O__$((`[B2B``*D8@`&D&0`!SP#@`(D8T3\``(2`J!B
+M``N@9``*/`*``@`1&$`D0D?@`&*`(9(%``&2!@``/`2``@P`<'0DA!'0D@(`
+M`11``,X\!H`"`E$0(0`"$(``41`C/`.``R1CH8```A"``$,0(91$`"`0@``)
+M)$,`(#0"__\0@@`&``0B`I!B``$\`X`")&-$_*!D``V@8@`,`E$0(0`"$(``
+M41`C/`.``R1DH8```C"``,00(21#`""48@`$)$+__RQ"`$`00``B`E$0(3P%
+M@`(D@@`JD&,`!22D1(``PC`A)(0``@#$$"4D8P`",$(``Z"C1(`00`"6),,`
+M0`!@$"&(PP``F,,``XC%``28Q0`'B,<`")C'``N(R``,F,@`#ZB#``"X@P`#
+MJ(4`!+B%``>HAP`(N(<`"ZB(``RXB``/),8`$!3"_^XDA``0`E$0(0`"$(``
+M41`C/`.``R1HH8```CB``.@P(23#`""48@`&)$+__RQ"`"!00``IE,(`*)!C
+M``<\!8`")0(`:B1C``(`XA`A)*1$Q*"C1,2(20``F$D``XA*``282@`'B$,`
+M")A#``N(10`,F$4`#ZB)``*XB0`%J(H`!KB*``FH@P`*N(,`#:B%``ZXA0`1
+MB$D`$)A)`!.(2@`4F$H`%XA#`!B80P`;B$4`')A%`!^HB0`2N(D`%:B*`!:X
+MB@`9J(,`&KB#`!VHA0`>N(4`(93"`"@D0O__+$(`$!!``!@DPP`HD&(``3P%
+M@`(E`P"*)$(``@#C&"$DI$3HH*)$Z(AF``"89@`#B&<`!)AG``>(:``(F&@`
+M"XAI``R8:0`/J(8``KB&``6HAP`&N(<`":B(``JXB``-J(D`#KB)`!$F9@`0
+M`B`@(0)@*"$,`&&()`<``Q!```DD`___C[\`$(^S``R/L@`(C[$`!(^P````
+M8!`A`^``"">]`"`D`P`!/`*``J!#1>R.9``0```H(0P`6QXD$``!CF0`$`P`
+M6QXD!0`!CF0`$#(%__\,`%M()`8``28"``$``A8```*&`RH"``,40/_XCF0`
+M$"00``$R!?__#`!;2```,"$F`@`!``(6```"A@,J`@`#5$#_^(YD`!`,`&%,
+M`B`@(1``_]<``!@AC,D``(S*``2,P@`(C,4`#*R)``"LB@`$K((`"*R%``PD
+MQ@`0%,/_]B2$`!`0`/]T`E$0(23&11R0P@`'D@0``#!"`-\T0P`@`$08"CP$
+M@`(`8"@A)(01Z`P`<'2@PP`'$`#_*`)1$"$D`P`#/`*``A``_Q>@0T4&)[W_
+M\*^_``2OL````("`(0P`83.0A```/`*``J!`1>P,`&*2C@0`$`P`84R2!```
+MC[\`!(^P`````!`A`^``"">]`!`GO?_@`(`0(20$`#2OM0`4K[0`$*^S``RO
+ML@`(K[$`!*^P``"OOP`8`*"8(0#`H"$`X*@A`0"0(0P`>E<P40#_`$"`(1(`
+M``\D`@`"`@`@(0``*"$,`'"J)`8`-#P"@`(D0D?0`!$8@`!B&"&N4```KA,`
+M(*X4`"RN%0`PK'``````$"&/OP`8C[4`%(^T`!"/LP`,C[(`"(^Q``2/L```
+M`^``"">]`""0@@`H`*(H)0/@``B@A0`H)[W_\`"`$"&OL```K[\`!`"@("$0
+MH``(,%``_PP`>FH`````/`*``@`0&(`D0D?0`&(8(:Q@``"/OP`$C[``````
+M$"$#X``()[T`$#"$`/\`!!C``&08(0`#&(`\`H`"`&08(R1"1]``!""`)[W_
+M\`""("$\`H`#K[$`!*^_``BOL```)$*A@``#&(",D````&(8(9!D`!L`X(@A
+M/`B``CP'@`*N!0`@K@8`)"8)`!PDYX-@)0B#P```*"$"`#`A#`!Q"`(`4"&.
+M`@`<%$``!P!`("&N,```C[\`"(^Q``2/L````^``"">]`!`,`'$F`````!``
+M__BN,```)[W_\*^P``"OOP`$$*``!`"@@"&,H@`<%$``!0!`("&/OP`$C[``
+M``/@``@GO0`0#`!Q+0`````,`'$?C@0`'!``__>N```<)[W_\*^P````H(`A
+MK[\`"*^Q``0,`'%"`("((8X"`"``0/@)C@0`)`P`<5`"("`A#`!Q20(@("&2
+M!``HC[\`"*(``"B/L0`$C[```"0#``$D`@`"`&00"@/@``@GO0`0)[W_\*^_
+M``",P@`L`$#X"8S$`#"/OP```^``"">]`!`PA`#_%(``!``````\`H`#`^``
+M")!"H9L#X``(```0(3"$`/\4@``$`````#P"@`,#X``(C$*AE`/@``@``!`A
+M,(0`_Q2```4`````/`*``XQ"H90#X``()$(!``/@``@``!`A)[W_\#"$`/\4
+M@``%K[\``#P"@`*,0D?4$$``!`````"/OP```^``"">]`!`,`'"U``````P`
+M83,``"`A$`#_^8^_```GO?_P,(0`_Q2```6OOP``/`*``HQ"1]000``$````
+M`(^_```#X``()[T`$`P`<+H`````#`!A3```("$0`/_YC[\``">]__"OL```
+M,)``_Z^Q``2OOP`(#`!PM3P1@`,6```&DB**Q!!```L\`H`#DB**Q"1"``&B
+M(HK$#`!PN@````"/OP`(C[$`!(^P```#X``()[T`$`P`<4*,1*&8$`#_])(B
+MBL0GO?_PK[```*^_``0,`'"U,)``_Q8```8\!(`#D(**Q"1"__\P0P#_$&``
+M!Z""BL0,`'"Z`````(^_``2/L````^``"">]`!`\`H`##`!Q28Q$H9@0`/_V
+M`````">]__`PI0#_K[\```P`8QHPQ@#_C[\```/@``@GO0`0)[W_\*^Q``2O
+MOP`(K[```(R"`!@`@(@AC%````P`81&2!`"0C@(`'%1```^.`@`@KA$`'(XB
+M`!B.`P`HKA$`(*Q```"2!`"0)&,``0P`82*N`P`HC[\`"(^Q``2/L````^``
+M"">]`!",0@`8$`#_\:Q1```GO?_@K[0`$#"4`/\N@@`"K[8`&*^U`!2OL0`$
+MK[\`'*^S``ROL@`(K[````"@L"$`P*@A,/$`_Q1```PD`P`/C[\`'(^V`!B/
+MM0`4C[0`$(^S``R/L@`(C[$`!(^P````8!`A`^``"">]`"`,`'I7)`0`H`!`
+M@"$00/_Q)`,`"@``*"$D!@"@#`!PJ@!`("$"@"`A#`!A`*(4`)`"@"`A#`!@
+M^*X"``BB`@"1)`0'`"0"`/^B$0"3#`!Z5Z8"`(0`0(@A$$#_WB0#`*D`0"`A
+M```H(20&!P`,`'"JK@(`&`P`>E<D!`,`K@(`=!!`_]0D`P`!`$"0(:X``!RN
+M```@K@``*```F"$\`H`")$*%K*Y"``2N,@`8`B`@(0P`86NN4```)F(``3!3
+M`/\N8@!`)C$`'!1`__0F4@`,#`!Z5R0$`!P`0"`A)`8`'```*"$,`'"JK@(`
+M%`*`("$,`&-=`@`H(11`_[8D`P`OKK``````&"$0`/^RKA8`F">]__"OOP``
+MC(,`%#$I`/^@90``C(,`%*!G``&,@P`4H&@``HR%`!2D9@`0#`!G(*RI`!2/
+MOP```^``"">]`!`GO?_PK[\`#*^R``BOL0`$K[````"`D"&0A`"0,+``_PP`
+M81$PT0#_`@`H(0(@,"$,`&>%`D`@(9)$`)`,`&$B`$"`(0(`$"&/OP`,C[(`
+M"(^Q``2/L````^``"">]`!`GO?_PK[\```P`9OXPI0#_C[\```/@``@GO0`0
+M)[W_\*^_``ROL@`(K[$`!*^P````@)`AD(0`D#"P`/\,`&$1,-$`_P(`*"$"
+M(#`A#`!GDP)`("&21`"0#`!A(@!`@"$"`!`AC[\`#(^R``B/L0`$C[````/@
+M``@GO0`0)[W_X*^S``ROL@`(K[$`!*^P``"OOP`0`("((9"$`)``P)`A`."8
+M(0P`81$PL`#_CB(`*!!``!\"("`ACB,`'!!@``<`8"@AC&(`&(Q"```40``"
+MKB(`'*X@`"".(@`HK'(`!"1"__^N(@`H)`(``:!P``"L<P`(H&(``PP`95F@
+M8``!DB0`D`P`82(`0(`A)`(`H@`0$`J/OP`0C[,`#(^R``B/L0`$C[````/@
+M``@GO0`@#`!A(I(D`)`0`/_V)`(``R>]_^"OLP`,K[(`"*^Q``2OL```K[\`
+M$`"`B"&0A`"0`,"`(0#@F"$,`&$1,+(`_XXB`"@00``A`````(XF`!P0P``7
+M)`(`H8S"`!B,0@``%$```JXB`!RN(``@CB(`*"0#``&LT``$)$+__ZXB`"@`
+MP"@A`B`@(:#2``"LTP`(H,,``PP`95*@PP`!DB0`D`P`82(`0(`A)`(`H0`0
+M$`J/OP`0C[,`#(^R``B/L0`$C[````/@``@GO0`@#`!A(I(D`)`0`/_V)`(`
+M`R>]__"OL```K[\`!`P`:"0`@(`A#`!Z:HX$`!`,`'IJC@0`&`P`>FJ.!`!T
+M#`!Z:HX$`!0,`'IJ`@`@(8^_``2/L````^``"">]`!`GO?_PK[(`"*^Q``2O
+ML```K[\`#`"`B"&0A`"0,+``_PP`81$`P)`A)@/__RQB``800``;,@(`$``#
+M$(`\`X`")&,1^`!#$"&,0@```$``"`````"6(@"$ID(```P`82*2)`"0```0
+M(8^_``R/L@`(C[$`!(^P```#X``()[T`$!``__62(@"5$`#_\Y(B`)00`/_Q
+MEB(`B!``_^^6(@"&$$``!3(%``\,`&A<`B`@(1``_^JF0@``#`!A(I(D`)`0
+M`/_I)`(``B>]__"OL@`(K[$`!*^P``"OOP`,`("0(9"$`)`PL0#_#`!A$3#0
+M__\F(___+&(`!Q!``",R(@`0``,0@#P#@`(D8Q(0`$,0(8Q"````0``(````
+M`*90`(0,`&$BDD0`D```$"&/OP`,C[(`"(^Q``2/L````^``"">]`!`R!0#_
+MHE``E0P`9O@"0"`A$`#_\@`````0`/_PHE``E!``_^ZF4`"($`#_[*90`(8"
+M`"@A#`!H9@)`("$0`/_G`````!!```8R)0`/,@8`_PP`:(4"0"`A$`#_X```
+M```,`&$BDD0`D!``_]\D`@`")[W_\#"E`/^OOP``#`!H0##&`/^/OP```^``
+M"">]`!`GO?_@K[,`#*^R``BOL0`$K[\`%*^T`!"OL```C)0`"#"S`/\PQ@#_
+M`!,00(Z#`!@`1A`A``(1@`!BB"$2(``0`("0(3(B``\D0@!/)`/_\`!#*"0"
+M(Q@D`&40(0!#$"L40``'`&`@(;QQ```D8P`0`(40(0!#$"L00/_[`````(XE
+M`#PD`@`!$*(`"@``("&/OP`4C[0`$(^S``R/L@`(C[$`!(^P````@!`A`^``
+M"">]`"``!A$``%,0(0!%@`0"0"`A#`!D-P(`*"$40``(`!,0@`!4$"&N(```
+MC$,`@```("&L0P"`$`#_Z:X@`#P"0"`A#`!D*P(`*"$40/_D)`0`I1``__,`
+M$Q"`)[W_\#"$`/^OL```K[\`!`P`80@`H(`AC$,``#!C`/\`0Q@AK@,`"(Q"
+M`"0P0@`?+$,`!`!`*"$48``"H@(`DR0%``,PI`#_``0AP"2$"``,`'I7H@4`
+MDP!`&"&N`@`P$$``(B0$`*@``A`C,$('_P!B$"$D!`@@#`!Z5ZX"`"P`0!@A
+MK@(`-!!``!@D!`"J``(0(S!"`!\`8A`A)`0#``P`>E>N`@`XK@(`>!!```\D
+M!``!D@0`D#P%@`(F!P"<`@`P(0P`8)HDI9BD)`(`_Z8"`(0"`"`A#`!CG:8`
+M`(8\`H`"K%!'V```("&/OP`$C[````"`$"$#X``()[T`$">]_^"OL@`(K[$`
+M!*^_`!BOM0`4K[0`$*^S``ROL````("((9"$`)`,`&$(CC,`"`!`D"&.8@``
+M)`/__@!#$"2N8@``)`(``JYB``".8@``,$(``A1`__TD`@`&KF(`:)(B`)..
+M)``P```H(0`",<`DQ@@`#`!PJ@`"@$".)``T```H(20&"""N(``\#`!PJJX@
+M`$"2)@"3)B0`1```*"$,`'"J``8PP)(F`),``"@A)B0`7`P`<*H`!C#`KF``
+M;(XE`"P2```,``"@(0"@("$\`@!`-$*``":4``&L@@```I`8*R0"``&L@@`(
+MK(``/!1@__<DA`!`KF4`&(Y"``0P0@`04$``!(XP`#B.8@!$KF(`1(XP`#B.
+M,@!X``"@(8X#``0D`H#_)!4``0!B&"0\`H`")$*0.*Y"``2N$@`<K@,`!*X5
+M``"N40```@`@(0P`9`XFE``!+H(`0"80`"`40/_O)E(`##P"`$`T0@!`KF(`
+M@(YC`(`\`O_^-$+__@!B&"0D`@%'KF,`@*YB``@D`@#_IB(`A*8@`(:/OP`8
+MKG4``(^T`!"/M0`4C[,`#(^R``B/L0`$C[````/@``@GO0`@)[W_\*^Q``2O
+MOP`(K[```(R"`!P`@(@AC%````P`81&2!`"0C@(`/%1```^.`@!`KA$`/(XB
+M`!R6`P"*KA$`0*Q```"2!`"0)&,``0P`82*F`P"*C[\`"(^Q``2/L````^``
+M"">]`!",0@`<$`#_\:Q1``",A@`(```@(:S%`'2,PP!T+(($``!E&"008``#
+M)(0``11`__H``````^``"```$"&,A@`(```X(8S"```LZ`0`).<``31"0`"L
+MP@``C,0`>(S"`'",PP```$00)3!C0``48``#`$40)!4`__,`````C,,``"0$
+MO_\``A`K`&08)`/@``BLPP``)[W_\*^E``"OI@`$CZ4``(^B``2/HP``C(8`
+M"*RB``BL8``,C,(`````("$T0D``K,(``*S'`'",P@!X`$<0)!1```<L@P0`
+M$&``!22$``&,P@``,$)``!1`__<`````C,,``"R"!``D!+__`&08)*S#```#
+MX``()[T`$">]_^"OL0`4K[``$*^_`!B0Z0``C((`")#H``&OI0``KZ8`!(Q#
+M`!@`"8!``@B`(0`0$8``8A@AKZ,`"(^E``@`"$$`CZ8```$)0"$D!P`!`("(
+M(0$'.`0,`&1-`!"`@`(1&"&,8@!$5$``#(QC`%R/H@``K&(`1`(1&"&/I``$
+MC[\`&(^Q`!2/L``0```0(:QD`%P#X``()[T`((^B```0`/_UK&(``">]_^"O
+MOP`0D.@```#@8"&0YP`!C(D`"``($$``1U@AKZ4````+:("OI@`$KZ``"(TC
+M`!@!I!`A`(!0(8Q$`%P`!SD```L1@`!B&"$`Z#@A)`(``:^C``P`XG`$KZ0`
+M"```,"&-(@``+,@$``&J*"$T0D``K2(``(TB`'@`3C@D$.``!B3&``&/H@`(
+MCZ,``(^D``2L0P``K*0`7(TB```P0D``%$```P`````5`/_M`````(TB```D
+M`[__`$,0)!#@``6M(@``C[\`$```$"$#X``()[T`((^C``@`"Q"`CZ4``(^F
+M``0`2A`AK$,`7`%`("$,`&1L`8`X(1``__./OP`0)[W_T*^S`!ROL@`8K[$`
+M%*^_`""OL``0`*"0(8R#``B0I0```("((9)$``&,8P`8``400`!$$"$``A&`
+M`&(8(:^C``"/HP````0A``"%("$D`@`!`()@!```6"$``)@ACD@`"!!@`!,`
+M`%`ACZ0``(^E``"/H@``)`/_\`"#("0P0@`/)$(`3P!#$"0`@B`A`*,H)`"%
+M$"L40``&`````+RQ```DI0`0`(40*Q!`__P`````CZ(``(Q#`#PD`@`!$&(`
+M"20$`*&/OP`@C[,`'(^R`!B/L0`4C[``$`"`$"$#X``()[T`,(XP`#PM`D`!
+M)`E```$"2`L2```Z`0E`(XX&`!R,P@``%$```JXB`#RN(`!`EB,`B@`)%``T
+M18"`)&/__S1$`(`D`@`!K-$```(*F`JN`@``%0``**8C`(JN!0`$CD,`!(X$
+M``0D`H#_`&H8(0""("2LT@`()&=``"1B$``D92``)&8P`*X$``2N`@`,K@4`
+M$*X&`!2N!P`8K@,`"!%@``(!25`AK7```!4`_]4"`%@A`8`H(0P`9#<"("`A
+M%$``!P(@("$"8"@A`@`P(0P`9&P"0#@A$`#_P@``("$"8"@A`@`P(0P`9)8"
+M0#@A$`#_O```("$0`/_9K@0`!!``_[@D!`"F)[W_\*^_```,`&37`````(^_
+M```#X``()[T`$">]__"OOP``#`!DUP````"/OP```^``"">]`!`GO?^PK[,`
+M+*^_`$2OO@!`K[<`/*^V`#BOM0`TK[0`,*^R`"BOL0`DK[``((R"``BOH``(
+MKZ``#*^B``",50!L`("8(:^@`!`2H``2KZ``%)""`),00``/``"@(20"``$"
+M@A`$`J(0)#*%`/\D!@`!)I0``0``."$``$`A%$``I20)``B28@"3`H(0*Q1`
+M__0D`@`!CZ,``(QU`'RL=0!\$J``C(^_`$2OH``$CZ0`!))B`),``*`A$$``
+M?P`$N"L`%Q"``%,0(21"`$2OH@`8`N#P(1+@`(TD`@`!)H,`$`!B$`0"HA`D
+M4$``;))B`)./H@``CZ0`&(Q#`!@`'A&``&*P(1+``!",A0``,L(`#R1"`$\D
+M`__P`$,P)`+#&"0`9A`A`$,0*Q1```<`8"`AO'$``"1C`!``AA`A`$,0*Q!`
+M__L`````4*``5))B`),PH@`/)$(`+R0#__``0S`D`*,8)`!F$"$`0Q`K%$``
+M!P!@("&\<0``)&,`$`"&$"$`0Q`K$$#_^P````",H@`$,$(`@!!``!$`````
+M5H``/Y)B`).,H@`$)`,`0``"%`(P0@!_5$,`.9)B`).,H@`$,$*``%!``#62
+M8@"3C*(`!"0#_W\`0Q`DK*(`!(RB`!R,0@`($$``!@````",0P`$C$0`"*^B
+M`!"OHP`4KZ0`#(RB``",HP`$)`3_X#!C`&@48``[`$20)(RB``2/HP`(CZ0`
+M#``"%`(P0G__`&(8(0"#(".OHP`(KZ0`##*0`/\R\0#_`F`@(0(`*"$,`&B7
+M`B`P(5)```>.9`"8CD(`'(^C`!",0@`($$,`"0````".9`"8CZ@`%(^I``P"
+M`"@A`B`X(0P`7Q```#`AKZ``"!9`_Z\"0"@ADF(`DX^D`!@FE``!`H(0*R2$
+M``BOI``8%$#_B"?>``*/H@`$)$(``:^B``0L0@`"%$#_>8^D``2/OP!$C[X`
+M0(^W`#R/M@`XC[4`-(^T`#"/LP`LC[(`*(^Q`"2/L``@`^``"">]`%"NP``,
+M$`#_SC*0`/\0`/]U`H(0!`P`7Q".9`"8$`#_6I)B`),GO?_PK[$`!*^_``RO
+ML@`(K[```)""`)`\`X`")&-'U``"$(",D@`(`$,0(20#``$`@(@AK$,``(Y0
+M``2.0@`(`@(0)!!``"@R`@!`KE``!!1``"$`````,@(`!!1``!H`````,@(`
+M`A1``!,`````,@(``11```P`````,@(!`!!`_^T`````CD(`1#!"#``00/_I
+M``````P`9IH"("`A$`#_Y0`````,`&5@`B`@(1``__,R`@$`#`!FZ@(@("$0
+M`/_L,@(``0P`9JH"("`A$`#_Y3("``(,`&9O`B`@(1``_]XR`@`$DB,`D#P"
+M@`*/OP`,C[(`"(^Q``2/L```)$)'U``#&(``8A@AK&````/@``@GO0`0)[W_
+M\*^Q``2OOP`(K[```(R0``@`@(@AC(0`F```*"$``#`A```X(0P`7EL``$`A
+MC@,`%#P"`?\T0O__`&(8)*X#`!2.`@!LK@(`;(X"`'RN`@!\C@(`<!1`__XD
+M`O__K@(`=(X"`$0P0@$`$$``!"0"``&N(@``)`(``Z8B`(2.)`"8```H(0``
+M,"$``#@A#`!>6P``0"&/OP`(C[$`!(^P```#X``()[T`$">]__"OOP``E(,`
+MA`"`$"&,A`"8I$,`CB0#`("D0P"$```H(0``,"$``#@A#`!><0``0"&/OP``
+M`^``"">]`!`GO?_PK[```*^_``BOL0`$C((```"`@"$00``"C)$`"*R```".
+M(@!$,$(!`!1```T`````CB(`1#!"`@!00``#H@``DB0"``*B`@"2D@@`DHX$
+M`)@``"@A```P(0P`7E0``#@ACB(`1#!"`(!40``9E@(`A(XB`$0P0@"`%$``
+M!20"``*6`P"$)`(`@!!B``<D`@`"I@(`A(^_``B/L0`$C[````/@``@GO0`0
+ME@(`CHX$`)@``"@AI@(`A```,"$``#@A#`!>?@``0"$0`/_SC[\`"(X$`)@`
+M`"@AI@(`CB0"`("F`@"$```P(0``."$,`%Z```!`(1``_]X`````)[W_\*^P
+M````@(`A/`2``J^_``0,`'!T)(02+)8"`(R/OP`$)$(``:8"`(R/L````^``
+M"">]`!",@@`(``4N0*Q%`!0D`@`!`^``"*2"`(2,B0`(,*4`_P`%$<"-(P`8
+M`&(@(1"``!$D`@`!,((`#R1"`$\D`__P`$-`)`"#&"0`:!`A`$,0*Q1```<`
+M8#@AO'$``"1C`!``Z!`A`$,0*Q!`__L`````)`(``0"B$`2(@P`HF(,`*XB%
+M`"R8A0`OJ,,``+C#``.HQ0`$N,4`!P/@``BM(@!L)[W_X*^S``ROL0`$K[``
+M`*^_`!2OM``0K[(`"(R4``B0IP``D*8``8Z#`!@`!Q!``$80(0`"$8``0X@A
+M`*"`(1(@`!``@)@A,B(`#R1"`$\D`__P`$,H)`(C&"0`91`A`$,0*Q1```<`
+M8"`AO'$``"1C`!``A1`A`$,0*Q!`__L`````CB(`/``&&0``9Q@A)`0``0!D
+MD`000``*```8(8^_`!2/M``0C[,`#(^R``B/L0`$C[````!@$"$#X``()[T`
+M(`)@("$,`&0W`D`H(11``"H"8"`AD@,``B0"``$08@`BE@(`$!!@`!P`````
+MC@,`%!!@``,``B0`/`(@``""("6N)```D@(``)($``$D`P#```(0@`!4$"$\
+M!@#``&0P"HQ%`(`D0P"`$(``"9("``(``A2``,(0)0"B$"6L8@``)`(``:XB
+M`#P0`/_4```8(1``__@``A"```(4`#1"@``0`/_HKB(``#P#0````A0`$`#_
+M^P!#$"4,`&0K`D`H(11`_\8D`P"D$`#_TY(#``(PI0#_,,8`_P`%*$``IB@A
+M``4H@`"D*"&,HP!$$&``!```$"&,8@`<C$(`")!"``,#X``(`````">]_]"O
+MLP`<,-,`_Z^R`!@`$Q$`,+(`_Z^U`"2OM``@K[$`%*^_`"BOL``0`("((0!2
+M$"$`$AA`)`0``0!$(`0`<Q@ACB4`"*^@``BOH``,KZ,``*^D``2/H@``C*,`
+M&```J"$``A&``&(@(1"``!```*`A,((`#R1"`$\D`__P`$,P)`"#&"0`9A`A
+M`$,0*Q1```<`8"@AO'$``"1C`!``IA`A`$,0*Q!`__L`````C((`/!1``$,D
+M`P"CCZ(````"$(``41`AC$4`1!"@`#T``!@A,*(`#R1"`"\D`__P`$,P)`"C
+M&"0`9A`A`$,0*Q1```<`8"`AO'$``"1C`!``AA`A`$,0*Q!`__L`````C*(`
+M'(Q#``@08``%`````(QT``2,8@`(`&"H(:^B``R,H@``C*,`!"0$_^`P8P!H
+M%&``"P!$@"2,H@`$CZ,`"``"%`(P0G__`&(8(:^C``B/H@`,CZ,`"`!#$".O
+MH@`,`B`@(0)`*"$,`&B7`F`P(5(``!J.)`"8C@(`'(Q"``@050`)`````(XD
+M`)B/J0`,`D`H(0``,"$"8#@A#`!?$`*`0"&OH``(%@#_Q@(`*"$``!@AC[\`
+M*(^U`"2/M``@C[,`'(^R`!B/L0`4C[``$`!@$"$#X``()[T`,(^I``P"0"@A
+M```P(0)@."$,`%\0`H!`(1``_^P`````C((`""0%``&,0@!$,$,`P"0"`(!0
+M8@`$E((`A```*"$#X``(`*`0(51#__T``"@A$`#_^P`````GO?_PK[\`!*^P
+M````@(`AC(0`""0#_[B,@@`(`$,0)*R"``B,@@``)`/__@!#$"2L@@``)`(`
+M`JR"```,`'IJC@0`,`P`>FJ.!``T#`!Z:HX$`'B2!`"0#`!@QHX%`)R/OP`$
+MC[````/@``@GO0`0C(0`"#"E`/\PQP#_``400(R&`!@`1Q`A``(1@`!&$"&,
+M0@````48@`!D&"$\!@`!,$*``#3&``$00``%`&`H(8QB`(``1A`E`^``"*QB
+M`(",8P"`/`(``22E`(`4X``"`&(0)31B``$#X``(K*(``(R"``@PI0#_``4H
+M@`"B*"&,H@"`/`,``31C``$`0Q`D`^``"``"$"LGO?_PK[$`!*^_``BOL```
+MC)``"#P"`$`PL?__C@8`@```*"$D!P`U`,(0):X"`(`\!H`")`($`!(B``PD
+MQD7PC@8`1"0#_]4`$1(``,,P)`#"$"6N`@!$C[\`"(^Q``2/L````^``"">]
+M`!`,`&)?`````!``__..!@!$C(,`"#"E`/\\"``!/`?__@`%$(`PQ@#_-0@`
+M`33G__X0P``%`$,@(8R"`(``2!`E`^``"*R"`(",@@"``$<0)`/@``BL@@"`
+M)[W_X*^R``BOOP`0K[,`#*^Q``2OL```C((`"#"E`/\PQ@#_C$,`&``%*$``
+MIB@A``41@`"`D"$`8B`A$(``#S""``\D0@!/)`/_\`!#,"0`@Q@D`&80(0!#
+M$"L40``'`&`@(;QQ```D8P`0`(80(0!#$"L00/_[```````%F(`"<A`AC$0`
+M1!"``#$P@@`/)$(`+R0#__``0S`D`(,8)`!F$"$`0Q`K%$``!P!@*"&\<0``
+M)&,`$`"F$"$`0Q`K$$#_^P````",@P``)`+_X`!BB"02(``+)B(`(`!1$"L"
+M(!@A%$``!P(@*"&\<0``)&,`$"2B`"``0Q`K$$#_^P````",@P`<)`(``:R"
+M``"L<@``K(``!`P`9`Z,<``($B``$P)R$"$2```)K%$`1(XB`!R,0@`($%``
+M!H^_`!".`@`8`@`@(0P`86NL4@``C[\`$(^S``R/L@`(C[$`!(^P`````!`A
+M`^``"">]`""L0`!<$@#_]JQ``$00`/_QC@(`&">]__`T`K]H$*(`!*^_``"/
+MOP```^``"">]`!`D`@`!5(+__(^_```,`&VM`````!``__B/OP``)[W_\"0$
+M``&OOP``#`!H^#0%OVB/OP```^``"">]`!`GO?_PK[\```"`*"&,A`"(#`!T
+M[P`$((*/OP`````0(0/@``@GO0`0`^``"```$"$GO?_P/`*``J^P```D4$:<
+M/`*``B1"1H`"`A`K$$``!:^_``2/OP`$C[````/@``@GO0`0C@(```!`^`DF
+M$/_\/`*``B1"1H`"`A`K4$#_^HX"```0`/_TC[\`!``$$",`@B`D``04``!$
+M(",`!!&``((@(0`$$0``@B`A/`*``B1",!0`!":"`((@(0/@``B`@@``&(``
+M%"0(`!0H@R<1)`(G$`"#$`MP2#`"`((@(QC```M`!4@`0`)(`!"B__X`11@C
+M/`<``P!`*"$$8``(-.<-0`##,",<P/_W`````!R`_^\H@R<1`^``"``````0
+M`/_X`&<8(0/@``@``````^``"``````\`J#`-$(`H(Q"```#X``(,$(`#R>]
+M_^"OOP`0#`!I7``````00`!#C[\`$$"`X`````````````````!`@.@`````
+M````````````/`.``+QI```\`H``)&,`$#1"#_\`0Q`K$$#_^@````!``H``
+M````````````````/`/__S1C__@`0Q`D-$(``T""@`````````````````!`
+M@.``````````````````0(#H`````````````````#P#@`"\:```/`*``"1C
+M`!`T0@__`$,0*Q!`__H`````0`*``````````````````#P#__\T8__X`$,0
+M)#1"``-`@H``````````````````0`*``*^B``"/H@``CZ(``"0#__P`0Q`D
+MKZ(``(^B``!`@H``C[\`$`/@``@GO0`@/`2@P#2$`"",A0``)`+^_SP#H,``
+MHB@DK(4``(R%```D`OW_-&,`D`"B*"2LA0``C&4``#2E``$#X``(K&4``#P#
+MH)`T8T``C&4``"0"__P\!*"0`*(H)*QE```TA$`$C(4``#P#__PT8___`*,H
+M)#P"H)"LA0``-$*8<(Q%```TI0`/K$4``(R%```\`@`!`*,8)`!B*"4#X``(
+MK(4``#P#H,`T8P`@C&(``#1"`P`#X``(K&(``#P%H,`TI0!@C*8``#P#__X\
+M`@`!``0A@#1C__\`@B`D`,,8)`!D,"6LI@``C*8``#P"``(`PC`E`^``"*RF
+M```PA`#_``01@#P&H,`\!:#`,$<`0"0"``$TQ@!@$((`"C2E`&2,PP``)`+_
+MOS1C`#"LPP``C,,```!B$"0`1Q@E`^``"*S#``",HP``)`+__@!B&"00`/_S
+MK*,``">]__"OL```K[\`!`P`:5PPD`#_`$`P(3P#H,`D`@`!$@(`%S1C`&``
+M$!`G/`.@P``0(<`T8P!@``(1P#!%`(`PA`"`C&,``!#```HD`O]_`&(0)`!%
+M&"6/OP`$C[```#P"H,`T0@!@K$,```/@``@GO0`0`&(0)!``__<`1!@EC&,`
+M`%#```,T8P$`)`+^_P!B&"0\`J#`-$(`8*Q#```0`/_B`!`0)R>]__"OOP`$
+M#`!I7*^P````0(`A/`*``@P`:>F01$8J/`*``@P`:@"01$8I/`*@P#P#[_\T
+M0@!@-&/__XQ%```2```4/`(0``"C*"0\`J#`-$(`8#P#H,"L10``-&,`((QE
+M```D`N__/`2@P`"B*"0TA`"0K&4``(R%``"/OP`$C[```#2E``*LA0```^``
+M"">]`!`0`/_M`*(H)4`(8```````-0@``4"(8`````````````/@``@`````
+M/`.@`!"```XT8P0`**(`$!!```LD`O__)*7__Q"B``@`````D&(``"2E__\D
+M8P`!H((``"0"__\4HO_Z)(0``0/@``@`````)[W_X`.@("&OOP`0#`!J620%
+M``^3HP``C[\`$#P$@`(``Q!")(5'X#!"``$P8P`!H(-'X*"B``$#X``()[T`
+M(">]_^`\`Z#`K[\`$#1C`)",8@``/`6@P#2E`&"OH@``CZ(``#P$_O\TA/__
+M-$(`!*^B``"/H@``K&(``(RB``"OH@``CZ(```!$$"2OH@``CZ(```P`:QVL
+MH@``#`!IV0``("$,`&U)``````P`::L`````)`(#Z*^B``"/H@``)$+__Z^B
+M``"/HP``)`+__Q1B__H\!*#`-(0`8(R"```\`__O-&/__Z^B``"/H@```$,0
+M)*^B``"/H@``#`!INZR"```,`&HJ``````P`:T0`````#`!K<0`````,`&IK
+M`````#P"@`(,`&OU@$1&*`P`:E$`````C[\`$`/@``@GO0`@/`*@P#P%_^\T
+M0@!@-*7__XQ#``!0@``#`&48)#P"`!``8A@E/`*@P#1"`&`#X``(K$,``">]
+M__"OL```K[\`!`P`:5P`@(`A`$`P(3P#H,`D`@`5)`0`"0!0(`HT8P!H/`*@
+MP(QE```T0@!@K&0``(Q%```2```%/`*@P!#``",D`O[_-*4!`#P"H,`T0@!@
+MK$4``!(``!F,10``%,``%3P"$``\`N__-$+__P"B*"0\`J#`-$(`8*Q%``",
+M10``%@``!H^_``10P``$-*4!`"0"_O\`HB@DC[\`!(^P```\`J#`-$(`8*Q%
+M```#X``()[T`$!``_^X`HB@E4,#__3P"$``0`/_H/`+O_Q``_]X`HB@D/`:@
+MP#P"__TTQ@!@$(``!C1"__^,PP``/`(``@!B&"4#X``(K,,``(S#````8A@D
+M`^``"*S#```\!J#`/`+^_S3&`&`0@``&-$+__XS#```\`@$``&(8)0/@``BL
+MPP``C,,```!B&"0#X``(K,,``#P$H,`TA``@C(,``#P"__LT0O__`&(8)`/@
+M``BL@P``*((`!!!```L`````)`(``0"",`0`!A`G/`.@L8QC``!0H``"`&(8
+M)`!F&"4\`:"QK",```/@``@`````/`*@L31"``B,0P``)`(``0""$`0``C`G
+M)`(``1"@``8`@A`$`&88)#P"H+$T0@`(`^``"*Q#```0`/_[`&(8)3P$H,`T
+MA``@C(,``#P"__<T0O__`&(8)`/@``BL@P``/`*@P#1"`"`D`P`!K$,``!``
+M__P\`J#`)[W_X*^S``ROL@`(K[$`!*^P```\`H`#K[\`$(Q#H80`!!!``*"0
+M(0!B@"$`P)@A%*``"```B"&/OP`0C[,`#(^R``B/L0`$C[````/@``@GO0`@
+M`!$@0`)D*"$F,0`!#`!L1P($("$",A`K5$#_^@`1($`0`/_QC[\`$">]_^"O
+ML0`4K[``$*^_`!@,`&P^``````!`@"$\`H`#K%"A@"11H8`D`@`!$@(`40``
+M```6```"-`+P`*XB``0\$(`#)A"A@#P"H*"N`@`4)`(``JX"`!@F!@`<)`0`
+M`20%``$,`&M2K@``$"8&`!XD!`!`#`!K4B0%``$F!@`@)`0`00P`:U(D!0`!
+M)@8`(B0$`$(,`&M2)`4``28&`"0D!`!##`!K4B0%``&6`@`D)`4`("8&`"H`
+M`A!"+$,`(`!#*`L,`&M2)`0`1"8&`"8D!`!D#`!K4B0%``&6`@`F)`4`$"8&
+M`&H``A!"+$,`$`!#*`L,`&M2)`0`928&`"@D!`!U#`!K4B0%``&6`@`D)`4`
+M""8&`(H``A!"+$,`"`!#*`L,`&M2)`0`=B8&``PD!``=#`!K4B0%``$F!@`*
+M)`0`'B0%``$,`&M2)A``"`(`,"$D!``?#`!K4B0%``&/OP`8C[$`%(^P`!`#
+MX``()[T`(`P`:]D#H"`AEZ(````"&@(0</^M/`(``20"``)48O^L/!"``Q``
+M_Z@\`@`$)[W_\`"`*"&OOP``#`!L1R0$"'B/OP```^``"">]`!`\`H`"`^``
+M"*Q$1BP\`H`"`^``"(Q"1BP\`E'K-$*%'P""`!D\`H`"```@$``$(8(#X``(
+MK$1&,#P"@`(#X``(C$)&,#P"@`(#X``(C$)&-#P"@`(D0C!4``08@`!B&"$`
+M!!!`)[W_\#P(H,`P1P`.+((`"*^_``2OL```-0@`8!1```4`@#`AC[\`!(^P
+M```#X``()[T`$(T%```\`H`"K$9&-"0"__$`HA`D`$<H)8QD```TI0`!#`!K
+MX:T%```\`H``)$(#`(Q$`!0``"@A$(``!"0&)8",@@`4`$#X"8R$```,`&OD
+M``````P`:^<`0"`A0`A@```````!`(`A/`'__S0A__X!`4`D0(A@````````
+M``````````P`:^\R$``!/`.``JQB1]Q`@$@`````````````````0()8````
+M`````````````$`(8```````,A```0$00"5`B&``````````````````$`#_
+MQX^_``0\`J"S-$(`'(Q"```GO?_PKZ(``(^B```P0@`!`^``"">]`!`\`J$`
+M.(0``@""("&4@@```^``"*2B```#@#@A/!R``R><QK``!!!``$00(0`"&8``
+M0Q`A``(0@`!$$"$``A"``$00(3P#:-L``B&`-&.+K0"#`!@`!!?#```8$``#
+M&P,`8B`C/`(4^#1"M8D`@@`8``0?PP``$!```A-#`$,H(P`%$$``11`A``(9
+M@`!#$"$``A"``$40(0`"$(``11`A``(10`""*"-`!D@`0`-(``!F&",\`@`#
+M!&``"C1"#4``91`J%$#_^0``````@R`C7(#_Y3P"%/@`X.`A`^``"``````0
+M`/_V`&(8(2>]__"OOP```X`0(3P<@`,,`&M,)YS&L#P"O\``0``(`````">]
+M__"OOP`$K[````.`@"$\'(`##`!L@B><QK`"`.`AC[\`!(^P`````!`A`^``
+M"">]`!`#X``(```0(2>]_^"OL@`(K[```*^_`!"OLP`,K[$`!`"`@"$``"@A
+M)!(``0``&"$#@)@A/!R``R><QK`D`O_^$((`-RB"__\00``L)`+__R0"__U0
+M@@`"``"`(280``$D`P`!$&``&#P"@``D40,`CB0`&!"```<\`X`#C((`%(R$
+M````0/@))`4`!`!`*"$\`X`#`!`10"1CBL@`0Q`AKB(`&#P"@`*L4$8X$*``
+M#XXD`!B,@@`4C(0``"0%``4`0/@)``````)@X"$"0!`AC[\`$(^S``R/L@`(
+MC[$`!(^P```#X``()[T`((R"`!2,A```$`#_\B0%``14@O_8)A```3P"@`*,
+M0D8X'$#_U212__\D$O_]$`#_T@!"D`L\`H``)$(#`*Q``!@\`H`"$`#_S*Q$
+M1C@D!0`!```8(0.`,"$\'(`#)YS&L"0"__X0@@`>*(+__Q!``!,D`O__)`+_
+M_5""``(``"`A)(0``20#``$08``)``090#P"@`,D0HK(`&(8(3P"@``D0@,`
+MK$,`%#P"@`*L1$8\`,#@(0/@``@`H!`A5(+_\22$``$\`H`"C$)&/!Q`_^XD
+M1?__)`7__1``_^L`0B@+/`*``"1"`P"L0``4/`*``A``_^6L1$8\`X`0(3P<
+M@`,GG,:P`^``"`!`X"$#@!`A/!R``R><QK`#X``(`$#@(2>]__"OOP``/`6`
+M`I"B1D`00``$```@(8^_```#X``()[T`$#P"@``D0@,`C$(`-"0#``$`0/@)
+MH*-&0!``__>/OP``)[W_\#P"@`"OOP``)$(#`(Q#`!0`!"8```0F`Q!@`!(P
+MA0#_C&0``(QB``P`0/@)`````#P"@``D0P,`C&(`1!1```0``"`AC[\```/@
+M``@GO0`0K&``1`P`;:,`````$`#_^H^_``",0P`8$`#_[0`````GO?_PK[\`
+M!*^P`````#`A/`>````&$(`DY0,`),8``3P#@`(`11`A)&.R8"C$`$"L0P``
+M5(#_]SP'@``\`@`(-$(`%:SB`P`\`H`")$*R"*RB`$`\`H`")$*R+*RB`!`\
+M`H`")$*Q-*RB`$@\`H`")$*T1*RB`"`\`H`")$*T6*RB`!P``#`AK*``3*R@
+M`#@``"@A``88P`!E&"$\`H`#)$**R``#&(``8A@A)*4``3P"@`(D0K)@**0`
+M"*QB```4@/_U``88P"3&``$HP@`"%$#_\```*"$\$(``/`*``B80`P`D0K)H
+MK@(`,#P"@`(D0K.8)`3__@P`;)JN`@`T#`!LYB0$__X,`'RO``````P`;)H`
+M`"`A#`!LY@``("&N``!$C[\`!(^P```#X``()[T`$!"@``H``#`A)*7__P"%
+M$"&`0P``)`(``Q!B``0D!@`!5*#_^B2E__\``#`A`^``"`#`$"$```%-`^``
+M"``````GO?_PK[\```P`:1@`````C[\```/@``@GO0`0)[W_\*^_``2OL```
+M/`*``HQ"1D040``5/`*``B101K@\`H`")$)&N!("``XD`P`!C@(`#`!`^`D"
+M`"`A$$``#XX"`!@T0@`!K@(`&#P"@`(F$``@)$)&N%8"__:.`@`,)`,``3P"
+M@`*L0T9$C[\`!(^P```#X``()[T`$"0#__X0`/_Q`$,0)">]__"OOP``#`!M
+MV0`````,`&W;``````P`!H0`````#`!WP@`````#X``(``````/@``@`````
+M)[W_\``$)@"OL```)`(`"@`$A@.OOP`$$@(`!R0$``T,`&TL`@`@(8^_``2/
+ML````^``"">]`!`,`&TL`````!``__<```````0F`">]__"OOP``#`!MW0`$
+M)@./OP```^``"">]`!`GO?_PK[```*^_``0`@(`AD(0````$)@`4@``$C[\`
+M!(^P```#X``()[T`$``$)@,,`&W=)A```9($````!"8`5(#_^P`$)@,0`/_U
+MC[\`!">]__`D!0`0)`8`*P``."&OOP``#`!N;P``0"&/OP```^``"">]`!`G
+MO?^PK[4`-"0"`"`D%0`PK[X`0*^W`#ROM@`XK[0`,*^S`"ROL@`HK[\`1*^Q
+M`"2OL``@`$BH"@"`D"$`H)@A`,#P(0#@N"$!(+`A``"@(0.T&"$FE``!*H(`
+M(!1`__R@=0```E,0)11``"4``*`A)`(`,"04``&CH@```I80*@+"H`LD`@`M
+M$N(`%@*=$"$FE/__!H``"(^_`$0"G1`A@$0```P`;=TFE/__!H'__`*=$"&/
+MOP!$C[X`0(^W`#R/M@`XC[4`-(^T`#"/LP`LC[(`*(^Q`"2/L``@`^``"">]
+M`%"`0O__)H/__P!5$"8`8J`*`[00(1``_^:@5P``$$#_WP*6$"H``(`A`D`@
+M(0)@*"$"`#`A#``$XP/`."$\!(`")(02,`"#$"&00P```[00(0)`("$"8"@A
+M`@`P(0/`."$,``-$H$,```!`D"$`0Q`E`&"8(11`_^LFE``!$`#_QP*6$"H`
+MH%`A`,!8(0#@8"$`@!@A```0(2>]__`!`$@A`$`@(0!@*"$!0#`A`6`X(:^_
+M```,`&X5`8!`(8^_```#X``()[T`$"0'``$0@``9```0(9""`````AX`$&``
+M%`"`,"$`Q!`C*$((`0`".`H``QX#)`(`"A!B``<D`@`-$&(`!21B_^`L10!?
+M)`(`"%1B``$`!3@*$.``!23&``&0P@````(>`%1@_^\`Q!`C`.`0(0/@``@`
+M````)[W_@`"%$"6OO@!PK[<`;*^V`&BOM0!DK[0`8*^S`%ROL@!8K[\`=*^Q
+M`%2OL`!0`("0(0"@F"$`P*@A`."P(0$`\"$#H*`A%$``&@``N"$D`@`P)[0`
+M`:.B```2G0`(`N`0(2:4__^2@@``)O<``:*B```6G?_[)K4``0+@$"&BH```
+MC[\`=(^^`'"/MP!LC[8`:(^U`&2/M`!@C[,`7(^R`%B/L0!4C[``4`/@``@G
+MO0"`$$#_Z0``````%H?#`D`@(0)@*"$"`#`A#``$XP+`."$#PQ`AD$(```)`
+M("$"8"@AHH(```(`,"$,``-$`L`X(0!`D"$`0Q`E`&"8(11`_^XFE``!$`#_
+MU``````GO?]PK[X`@`"`\"$`P"`AK[<`?*^V`'BOM0!TK[0`<*^S`&ROL@!H
+MK[\`A*^Q`&2OL`!@`,"@(0"@N"$,`&Z``."H(0``D"$``)@AK[T`1*^@`%00
+M0`%D``"P(8*"```FE``!$$``#@!`B"$D`@`E$B(`&`(@("$"X"@A`\#X"0``
+M``"/H@!4)$(``:^B`%2"@@``)I0``11`__0`0(@ACZ(`5(^_`(2/O@"`C[<`
+M?(^V`'B/M0!TC[0`<(^S`&R/L@!HC[$`9(^P`&`#X``()[T`D(*1```D`@`M
+M)I0``0``*"&OH`!0$B(!.J^@`$@D`@`P4B(!,X*1``"OH`!,)B+_T"Q"``H0
+M0``.)`(`+H^C`$@``Q"``$,0(0`"$$``41`A@I$``"1"_]"OH@!()B+_T"Q"
+M``H40/_U)I0``20"`"X2(@$1CZ(`3"0"`&P2(@$'KZ``0"8B_[XP0P#_+&(`
+M-Q!``!HF(O_;``,0@#P#@`(D8Q*D`$,0(8Q"````0``(`````!"@`/(D`O_\
+M)J(`!R0#__@`0Z@DCK(``(ZS``0FM0`()`(`9!(B`.`D`@!$$B(`W@`````D
+M`@``)`/__P)"D"0"8Y@D)B+_VS!#`/\L8@!4$$``RP`#$(`\`X`")&,3@`!#
+M$"&,0@```$``"``````D!``P`\#X"0+@*"$D!`!X`\#X"0+@*"$D`@`!)`,`
+M"*^B`$ROHP!()B+_O#!$`/\L@@`U4$``$J^]`$0\`X`"``00@"1C%-``0Q`A
+MC$(```!```@`````/`B``B4($D0"0"`A`F`H(0.@,"$D!P`*#`!NG0``````
+M0+`AK[T`1(^B`$B/HP!``%:`(R8"__\`0X`+CZ(`3!!```,D$0`@%&``.B01
+M`#"/H@!0%$``#H^B`$`"`!`A&$``"B80__\"("`A`\#X"0+@*"&/HP!4`@`0
+M(280__\D8P`!'$#_^*^C`%2/H@!`%$``(@!`("$"P!`A&$``#B;6__^/H@!$
+M`N`H(8!1```D0@`!KZ(`1`/`^`D"("`ACZ,`5`+`$"$FUO__)&,``1Q`__2O
+MHP!4CZ(`4%!`_UF"@@```@`0(1A`_U4F$/__)`0`(`/`^`D"X"@ACZ,`5`(`
+M$"$F$/__)&,``1Q`__BOHP!4$`#_2X*"```#P/@)`N`H(8^C`%0D8P`!$`#_
+MVJ^C`%0`8"`A`\#X"0+@*"&/HP!4KZ``0"1C``$0`/_`KZ,`5#P(@`(E"!(P
+M`D`@(0)@*"$#H#`A$`#_JR0'`!`\"(`"$`#_^24($E`D!``E`\#X"0+@*"$0
+M`/^HCZ(`2(^V`$@6P``%)L;__R06`$`D`@`@`$6P"B;&__\8P/^=``"`(0`0
+M%H`$00`$``````(2*`<0```'`!(GPQ!```0"$R@&`!`0(P!2$`0`HB@E`A(@
+M!S"B``$#L!@A%$```B0$`#$D!``N)A```0(&$"H40/_LH&0``!``_X>OO0!$
+M)J(``R0#__P`0Z@D@K$``P+@*"$FM0`$$`#_``(@("$FH@`#)`/__`!#J"2.
+MH@``)K4`!!!``!FOH@!$#`!N@(^D`$100``-/`2``H^C`$2`8@``$$#_<```
+ML"&/HP!$)M8``0!V$"&`0@``5$#__2;6``$0`/]ICZ(`2`P`;?<DA!)D#`!N
+M"X^D`$0\`H`")$(2>!``_^VOH@!$/`*``A``__PD0A)\)`0`)0/`^`D"X"@A
+M`B`@(0/`^`D"X"@ACZ,`5"1C``(0`/]3KZ,`5`9!_R<F(O_;)`,`+0`3F",`
+M$I`C`!,0*P)"D",0`/\?KZ,`0":C``,`8J@DCJ(``":U``0`0)@A$`#_#P`"
+ME\."D0``%B+^^":4``&"D0``)`4``1``_O0FE``!@I$``"1"``&OH@!,)B+_
+MT"Q"``H00/[J)I0``8*1```F(O_0+$(`"A1`__PFE``!$`#^Y"0"`&PD`P`!
+M)I0``1``_LROHP!,)`(``8*1``"OH@!0$`#^PR:4``$\!(`"#`!M]R2$$H0,
+M`&X+`H`@(3P$@`(,`&WW)(02G"00``<,`&W=)`0`(":B``,D`__\`$.H)(ZS
+M```F$/__)K4`!`P`;@L"8"`A!@'_]3P$@`(,`&WW)(02H!``_I<``!`A)[W_
+MT*^_```\`H`"`(`8(8Q$1DBOI0`4KZ8`&*^G`!P`8#`A```H(2>G`!2OJ``@
+MKZD`)*^J`"@,`&[AKZL`+(^_```#X``()[T`,">]__`D`B=T$*(`!*^_``"/
+MOP```^``"">]`!`D`@`!5(+__(^_```,`&T;`````!``__B/OP``)[W_\"0$
+M``&OOP``#`!PAR0%)W2/OP```^``"">]`!`DQO__)`+__Q#"``@`@!@AD*(`
+M`"3&__\DI0`!H&(``"0"__\4PO_Z)&,``0/@``@`@!`A),;__R0"__\0P@`&
+M`(`8(23&__\D`O__H&4``!3"__PD8P`!`^``"`"`$"$\`X`"C&)&5"1"``$#
+MX``(K&)&5#P#@`*,8D94)[W_\!!```:OOP``C&)&5"1"__\00``%```@(:QB
+M1E2/OP```^``"">]`!`,`'>$`````!``__H`````)[W_\`"@$"$`P!@A`0!@
+M(0$@:"&OL0`$`6"((0#@6"&OL````(`H(0%`@"$"("`A`$`P(0!@."$!8$`A
+M`8!((:^_``@,`'(H`:!0(:X1``"/OP`(C[$`!(^P```#X``()[T`$">]__"O
+ML```K[\`!(R#`#0D`@`0$&(`!P"`@"$,`',.`@`@(8^_``2/L````^``"">]
+M`!`,`')E`````!``__<`````/`*``XQ$G#0GO?_PK[\```P`=Q\DA``8C[\`
+M``/@``@GO0`0/`*``P/@``B,0IPT)[W_\*^_```,`'/B`````(^_```#X``(
+M)[T`$">]__``H!`A`,`8(0$`6"&OL0`$`4"((0#@4"&OL````(`H(0$@@"$"
+M("`A`$`P(0!@."$!0$`AK[\`"`P`=1(!8$@AKA$``(^_``B/L0`$C[````/@
+M``@GO0`0)[W_\*^_```,`'4:`````(^_```#X``()[T`$">]__"OOP``#`!U
+MB`````"/OP```^``"">]`!`GO?_PK[\```P`=:$`````C[\```/@``@GO0`0
+M)[W_\*^_```,`'6W`````(^_```#X``()[T`$">]__"OOP``#`!USP````"/
+MOP```^``"">]`!`GO?_PK[\```P`=>,`````C[\```/@``@GO0`0)[W_\*^_
+M```,`'8,`````(^_```#X``()[T`$">]__"OOP``#`!V,P````"/OP```^``
+M"">]`!`#X``(K*0``#P"@`(#X``(C$)&=#P"@`*,0D9TC$,`#`/@``B,0@`(
+M)[W_\`"@$"$`P!@AK[$`!*^P```!`(@A`."`(0"`*"$`0#`A`&`X(:^_``@,
+M`'X3`0`@(:X1``"/OP`(C[$`!(^P```#X``()[T`$">]__"OOP``#`!^-P``
+M``"/OP```^``"">]`!`GO?_PK[\``(R"`"@40``$`(`H(8^_```#X``()[T`
+M$`P`?<2,A``($`#_^X^_```GO?_PK[\```P`>'$`````C[\```/@``@GO0`0
+M)[W_\*^_```,`'AT`````(^_```#X``()[T`$">]__"OOP``#`!XN@````"/
+MOP```^``"">]`!`GO?_PK[\```P`>1H`````C[\```/@``@GO0`0)[W_\*^_
+M```,`'DW`````(^_```#X``()[T`$">]__"OOP`$K[````P`>5L`H(`AK@(`
+M`(^_``2/L````^``"">]`!`GO?_PK[\```P`>5T`````C[\```/@``@GO0`0
+M)[W_\*^_```,`'EC`````(^_```#X``()[T`$">]__"OOP``#`!Y:@````"/
+MOP```^``"">]`!`GO?_PK[\```P`>=\`````C[\```/@``@GO0`0)[W_\*^_
+M```,`'H-`````(^_```#X``()[T`$"0#``4\`H`#K$.<,#P"@`,GO?_PK$"<
+M.#P"@`.OL```K[\`!`"`@"&L1)PT/`*``JQ`1E2.`@`0`$#X"8X$`!0,`'-Z
+M`````!``__N.`@`0)[W_\`$J$"&OL0`$`("((:XF`!`DA``8`*`P(:XG`!2N
+M(@`,KBD``*XJ``2N*0`(`B`H(:^_``ROL@`(K[````P`=]$!`)`A/`*``HQ%
+M1G0F,`!(/`:``@(`("$DQM$D#`!^!@(`."$\`X`"E&1&4"0"``2N$0`L)(4`
+M`:XB`#0D`@`!KB(`.*X@`#RN(`!XKB``?"8B`("D9490IB0`1"0#``4D8___
+MK$````1A__TD0@`$KC(`F`P`@40"("`A/`2``P(@*"$,`';M)(2</`(@("$,
+M`($H`B`H(8^_``R/L@`(C[$`!(^P```#X``()[T`$">]__`!*A`AK[$`!`"`
+MB"&N)@`0)(0`&`"@,"&N)P`4KB(`#*XI``"N*@`$KBD`"`(@*"&OOP`,K[(`
+M"*^P```,`'?1`0"0(3P"@`*,149T)C``2#P&@`("`"`A),;1)`P`?@8"`#@A
+M/`.``I1D1E`D`@`$KA$`+"2%``&N(@`T)`(``:XB`#BN(``\KB``>*X@`'PF
+M(@"`I&5&4*8D`$0D`P`%)&/__ZQ````$8?_])$(`!*XR`)@,`(%$`B`@(3P$
+M@`,"("@A#`!V[22$G#P"("`A#`"!*`(@*"&/OP`,C[(`"(^Q``2/L````^``
+M"">]`!`GO?_PK[```"2#`$BOOP`$C&(`*`"`@"$40``4`&`H(3P$@`,"`"@A
+M#`!V[R2$G#R.`@`P4$```HX%`"".!0`LC@@`F(X)``"."@`$C@8`$(X'`!0,
+M`'(H`@`@(8^_``2/L````^``"">]`!`,`'W$C&0`"!``_^L\!(`#)[W_\*^P
+M````@(`A/`2``R2$G#ROOP`$#`!V[P(`*"$\`X`"C&)&5"1"``&L8D94/`*`
+M`HQ$1DR,@P"<$'``'CP"@`*,0D9,%&+_^P!@("$\`X`"C&)&5"1"__\00``2
+M`````*QB1E0F!`!(#`!^(*8``$2.!``8)@,`&!"#``>/OP`$C&(`!*R"``2,
+M8@`$K&,`!*Q$``"N`P`8C[````/@``@GO0`0#`!WA```("$0`/_M`````#P%
+M@`*.`@"<C*-&3!1P_^*L@@"<C@(`G!``_]^LHD9,)[W_\*^_``BOL0`$K[``
+M`#P"@`.,4)PT/!&``HXB1E0D0@`!KB)&5(X"`#0\!(`#)(2</!!``!("`"@A
+MC@(`-#1"``&N`@`TCB)&5"1"__\00``'```@(:XB1E2/OP`(C[$`!(^P```#
+MX``()[T`$`P`=X0`````$`#_^``````,`':J`````!``_^V.`@`T)[W_\*^P
+M``"OOP`(K[$`!`"`@"$\`X`"C&)&5"1"``&L8D94C((`-"0#__P`0Q@D,$(`
+M`Q!```LDD0`8K(,`-(XB``P`@"@A%$``&0!`("&.`@`T/`2``P(`*"$00``0
+M)(2</#P#@`*,8D94)$+__Q!```<``"`AK&)&5(^_``B/L0`$C[````/@``@G
+MO0`0#`!WA``````0`/_X``````P`=GL`````$`#_[@`````,`'=F`````!``
+M_^6N(``,)[W_\*^_````@#`A/`*``HQ#1E0D8P`!K$-&5(R#`#@D`@`!4&(`
+M$8R"`#008``")&+__ZR"`#@\`X`"C&)&5"1"__\00``%```@(:QB1E2/OP``
+M`^``"">]`!`,`'>$`````!``__H`````)`/_^P"`*"&L@``X`$,0)#P$@`,D
+MA)P\%$#_[*S"`#0,`'9[`````!``_^@`````)[W_\*^_````@#`A/`*``HQ#
+M1E0D8P`!K$-&5(R"`#@00``))`/_^XR"`#0`@"@AK(``.`!#$"0\!(`#)(2<
+M/!!```ZLP@`T/`.``HQB1E0D0O__$$``!0``("&L8D94C[\```/@``@GO0`0
+M#`!WA``````0`/_Z``````P`=GL`````$`#_\``````GO?_PK[\``#P"@`*,
+M0T94)&,``:Q#1E2,@P!X+&(`"!!```H``Q"`/`.``B1C%?``0Q`AC$(```!`
+M``@`````)`(`!*R"`'RL@`!X#`!RW@`````\`X`"C&)&5"1"__\00``%````
+M`*QB1E2/OP```^``"">]`!`,`'>$```@(1``__H`````)[W_\*^_``BOL0`$
+MK[```#P#@`.,<)PT/!&``HXB1E0D0@`!KB)&5(QBG#0D0P!(C&(`*!1``!$`
+M8"@AC@(`-#P$@`,D`P`0`@`H(1!#``,DA)P\#`!VJJX#`#2.)$94#`!WA```
+M``"/OP`(C[$`!(^P```#X``()[T`$`P`?<2,9``($`#_[HX"`#0GO?_PK[``
+M`*^_``0\`H`#C$*<-!""`#L`@(`A/`*``HQ#1E0D8P`!K$-&5`P`<S4"`"`A
+M)@4`2(RB`"@40``M`````(X#`'PD`@`&$&(`$`````".`P!X+&(`"!!```P`
+M`Q"`/`.``B1C%A``0Q`AC$(```!```@`````C@(`-!!``!,\!(`#)`(`$*X"
+M`#0,`'+>`@`@(3P#@`*,8D94)$+__Q!```8`````K&)&5(^_``2/L````^``
+M"">]`!`,`'>$```@(1``__D`````)(2</`P`=JH"`"@A$`#_ZR0"`!`D`@`&
+MK@(`?!``_^BN``!X#`!]Q(RD``@0`/_2C@,`?`P`<WH`````$`#_PP`````G
+MO?_@K[(`"*^P``"OOP`0K[,`#*^Q``0`@(`A`*"0(3P#@`*,8D94)$(``:QB
+M1E2,@@`T$$``1```F"$P0@`!4$``"3P$@`,DD0`8CC,`#!)@``0"8"`A#`!W
+M9@(`*"&N(``,/`2``R2$G#P,`';O`@`H(8X"`#!00``&KA(`((X"`"`"0A`J
+M$$```JX2`"RN$@`@/`*``R11G#P"("`A#`!V[0(`*"&.`@`T$$``(@(@("$P
+M0@`!$$``!CP"@`,28``$`F`@(0P`=T0"`"@A/`*``XQ"G#02`@`4)`,``0P`
+M=M$"`"`A/`.``HQB1E0D0O__$$``"0````"L8D94C[\`$(^S``R/L@`(C[$`
+M!(^P```#X``()[T`(`P`=X0``"`A$`#_]@`````\`H`#$`#_[:Q#G#@,`'9[
+M`@`H(1``_^0\`H`#/`2``R2$G#P,`':J`@`H(1``_\,\!(`#)[W_\#P"@`.O
+MOP``C$.<-`"`$"$\!(`#$$,`!"2$G!B/OP```^``"">]`!`,`'3F`````!``
+M__N/OP``)[W_\*^_``",I``L/`.``HQB1E0D0@`!K&)&5(R#`'@L8@`($$``
+M"@`#$(`\`X`")&,6,`!#$"&,0@```$``"``````D`@`'K((`?*R``'@,`'+>
+M`````#P#@`*,8D94)$+__Q!```4`````K&)&5(^_```#X``()[T`$`P`=X0`
+M`"`A$`#_^@`````0`/_N)`(``R>]__"OOP``/`.``XQBBW`D0@`!`$`@(0P`
+M:5BL8HMP$`#_^CP#@`,\`H`#)$*+>`""$",GO?_P``(10Z^P`````H!``@*`
+M(0`0$0`"`H`A`!`2``("@"$`$!0``@*`(0`0@",\`H`#)$*,&``02P`\!H`"
+M/`B``@$B2"$E"!90```X(20*$``DQM&X)`4`'Z^_``BOL0`$#`!QZP"`B"$\
+M!(`#`B`H(0(`,"$,`';@)(2</(^_``B/L0`$C[````/@``@GO0`0)[W_X"0"
+M.IBOLP`,K[(`"*^_`!"OL0`$K[````"@F"$0H@`8`("0(20".I@28@`(````
+M`(^_`!"/LP`,C[(`"(^Q``2/L````^``"">]`"`60/_X/`*``R1"BW@00/_U
+M)%``H#P"@`,D0HMX$@+_\280_V`,`'*$`@`@(1``__H\`H`#)`(``12"_^@D
+M`CJ8/`*``R11BW@``(`A#`!T>`(@("$F$/__)`+__Q8"__LF,0"@$`#_W20"
+M.I@GO?_P)`0``:^_```,`'2@)`4ZF(^_```#X``()[T`$">]__```"`AK[\`
+M``P`=*`D!3J8C[\```/@``@GO0`0`^``"``````\`H`")$+3?*R"```#X``(
+MK(``!">]__"OOP``C((`!(R#````8/@)`$`@(8^_```#X``()[T`$">]__"O
+MOP``/`*``XQ"G#0`@!@A`*`P(0!`("$,`'0Z`&`H(8^_```#X``()[T`$">]
+M__`D`B[@$*(`!*^_``"/OP```^``"">]`!`D`@`!5(+__(^_```\!(`##`!T
+MX22$G!@0`/_WC[\``">]__`D!``!K[\```P`=/LD!2[@C[\```/@``@GO0`0
+MK(``%*R%``"LA@`$K(@`"*R)``RLAP`0`^``"*R``!@GO?_PK[\```P`=:$`
+M````C[\```/@``@GO0`0)[W_\*^P```\$(`#C@*<+!1```6OOP`$C[\`!(^P
+M```#X``()[T`$$`(8````````0`@(3P!__\T(?_^`0%`)$"(8```````````
+M```````PA``!C@.<+(QB`!BN`IPLC&4`%*Q@`!1`"&```````#"$``$!!$`E
+M0(A@`````````````````(QB``R,9@`0`$#X"8QD``".`IPL%$#_XH^_``00
+M`/_>C[```">]__"OOP``#`!U(0````"/OP```^``"">]`!!`"&````````$`
+M*"$\`?__-"'__@$!0"1`B&``````````````````,*4``8R"`!0\!H`#)$(`
+M`:R"`!2,@P`4)`(``1!B``H`````0`A@```````PI0`!`05`)4"(8```````
+M``````/@``@`````C,*<+*R"`!@0`/_TK,2<+">]__`PA``"$(``!:^_```0
+MH``#`*`@(0P`=5,`````/`.``HQB1E0D0O__$$``!0``("&L8D94C[\```/@
+M``@GO0`0#`!WA``````0`/_Z`````(R#```\`H`")$(Q>``#&(``8B@AC*(`
+M`#P"@`(D0C)H`&(X(3P"@`(D0C'P`(`P(8RD````8A@A/`*``B1"MIA0@@`#
+MC,(`"`/@``@`````K*(``(S"`!"L8@``$`#_^JSF``",@P``/`*``B1",7@`
+M`QB``&(X(3P"@`(D0C)HC(4`"(SD````8C`A/`*``B1",?`0A0`#`&(8(0/@
+M``@`````/`*``B1"MIBLX@``K&```!``__FLP```/`2``XR"G"`40``1````
+M`$`(8````````0`8(3P!__\T(?_^`0%`)$"(8``````````````````P8P`!
+M/`*``ZQ#G"@D`P`!/`*``ZQ#G"2,@IP@)$(``0/@``BL@IP@/`*``XQ#G"`D
+M8___K$.<((Q"G"`40``,/`*``ZQ`G"0\`H`#C$*<*$`(8```````,$(``0$"
+M0"5`B&```````````````````^``"`````!`"&````````$`,"$\`?__-"'_
+M_@$!0"1`B&``````````````````,,8``22"__HD`P`!/`6@P`!#&`0L@@`&
+M$$``$S2E`!1``V``)`($``""$`0`0!`G`&(8)$"#8`````````````````!`
+M"&```````##&``$!!D`E0(A@`````````````^``"`````",H@``,$(``0`#
+M$`L0`/_SK*(``$`(8````````0`P(3P!__\T(?_^`0%`)$"(8```````````
+M```````PQ@`!)(+_^B0#``$\!:#``$,8!"R"``800``2-*4`%$`#8``D`@0`
+M`((0!`!B&"5`@V``````````````````0`A@```````PQ@`!`09`)4"(8```
+M``````````/@``@`````C*(```!#$"40`/_TK*(``"R"``8D!0`$`((H"T`#
+M:``D`@0``*(0!`!`$"<`8A@D0(-H`````````````^``"``````D`B[@$*(`
+M`R0"``$#X``(`````!2"__T\`H`#K$"<)!``__H`````)[W_\"0$``&OOP``
+M#`!V0"0%+N"/OP```^``"">]`!`GO?_P```@(:^_```,`'9`)`4NX(^_```#
+MX``()[T`$"0#``$\`H`"K$-&5"0%`!\D@P`$)*7__R0"__^L8```%*+__"1C
+M``0D`P`%/`*``ZR```"L0YPP)`,``3P"@`,#X``(K$.<.">]__"OOP`$K[``
+M``"`@"$,`&DQC(0````"$(`"`H`AC@,`!(^_``2/L```)&+_Z``#$`H#X``(
+M)[T`$">]_^"OL@`(K[$`!*^P``"OOP`0K[,`#(RS`"",HP`D`("`(0`3$(``
+MH)`A`&`@(11@`!X"`H@ACB(`!"9#`!@``"`A`'(@"R0#``$40``$`F,8!(X"
+M````0Q`EK@(``(XC``148``+C&(`!*XD``0,`';1`D`@(8^_`!"/LP`,C[(`
+M"(^Q``2/L````^``"">]`""L@P``K((`!(QB``2L9``$$`#_\JQ$```,`'=F
+M`````!``_^&.(@`$C*<`("2C`!@`!1@*``<0@`"","&,P@`$`&`H(1!B`!(D
+MR``$C&(``(QC``2L0P`$C*,`!*RE``2L8@``K*4``(T"```40``&)`(``8R#
+M````XA`$``(0)P!B&"2L@P```^``"`````!08/_VC0(``(QC``!09?_RK,``
+M!(RB``2L8@`$C*(`!*RE``2L0P``K*4``!``_^JLPP`$/`6``XRCG#2,@@`@
+MC&,`(`!#$"H40``%)`,``8RBG#2,0@`T$$```P`````\`H`#K$.<.`/@``@`
+M````/`*``R1"G#0`!C"``,(P(2>]__``H"`AK,4``*^_```,`',.`````(^_
+M```#X``()[T`$`/@``@``````^``"``````\`X`#C&*<,">]__"OOP``)$+_
+M_Q!```2L8IPPC[\```/@``@GO0`0#`!V_P`````0`/_[C[\``#P%@`.,HYPP
+M/`*``Q1@`!:,1)PTC((`-!1``!,\`H`#C(,`("1"G#P``QB``&(8(8QB``10
+M0``$C&,`!(Q"``"L8@`$C&,`!"1B_^@``Q`*$$0``R0#``$\`H`#K$.<."0"
+M``6LHIPP`^``"`````"LA0`(K(0`!`/@``BLA```)[W_\*^_```\`X`"C&)&
+M5"1"``&L8D94)(3_Z(R"`#040``1/`*``XR#`"`D0IP\``,8@`!B&"&,8@`$
+M4$``!(QC``2,0@``K&(`!(QC``0D8O_H``,0"A!$``HD`P`!/`*``ZQ#G#@\
+M`H`"C$1&5`P`=X0DA/__C[\```/@``@GO0`0)`,`!3P"@`,0`/_VK$.<,(R&
+M```DH@`8```8(13```0`11@+K(,```/@``BLI``DC,(`!*QF``"L8@`$C,(`
+M!*S#``00`/_XK$,``(R#```08``,)&+_Z(QE``!0HP`)K(```(QB``2LH@`$
+MC&(`!*QC``2L10``K&,``*R%```D8O_H``,0"E1```&L0``D`^``"`````"L
+MH``DC(,``"2B`!@`!1`*$$,`"@!`*"&,0@``C*,`!*Q#``2,HP`$K*4`!*QB
+M``"LI0```^``"``````00/_]`````(Q#```08@`)`````(Q"``2L8@`$C*(`
+M!*RE``2L0P``K*4```/@``BL@P```^``"*R````GO?_PK[$`!*^_``ROL@`(
+MK[````"`B"$6(``%/!*``SP"@`.,0IPL%$``+P````".4)PTC@(`-!1```0\
+M`H`#C$*<.!!``!,`````/`2``PP`=FPDA)P\$@(`"0!`("$\`X`#C&*<P"2$
+M``PF!0`,)$(``0P``OZL8IS`KE"<-"0"``4\`X`#K&*<,#P"@`.L0)PX%B``
+M$``````\`X`"K&!&5#P"@`.,0IPL$$``!8^_``R,8D94)$(``1``_]:L8D94
+MC[(`"(^Q``2/L````^``"">]`!`\`H`"K%%&5!``__B/OP`,#`!U3#P2@`,0
+M`/_/`````">]__"OOP``#`!WQ@`````\!(`#)[W_\*^_```,`'9L)(2</#P#
+M@`.L8)PX)$0`##P#@`,,``,/K&*<-">]__"OL````,`H(:^_``0,`'<;`("`
+M(:X``!BN```,K@``$(^_``2/L````^``"">]`!`GO?_PK[$`!*^_``ROL@`(
+MK[```(R2``@`@(@A`+(0*A1```<DA/_HC[\`#(^R``B/L0`$C[````/@``@G
+MO0`0CC``&`P`<^*N(``84@```:XR`!0D`@`!$`#_\ZXB`!@GO?_PK[\``(S"
+M```40``$`*`P(8^_```#X``()[T`$`P`=]Z,I0`@$`#_^X^_```GO?_PK[\`
+M`(R"`!``@#`A%$``"B2$_^B,P@`8$$``"(^_``",PP`4C,(`"*S``!@`0Q`J
+M%$``!`!@*"&/OP```^``"">]`!`,`'/B`````!``__N/OP``)[W_\*^_````
+MH#`A#`!WWHRE`""/OP```^``"">]`!`GO?_PK[\```P`=_8`````C[\```/@
+M``@GO0`0)[W_\*^_```,`'@"`````(^_```#X``()[T`$">]__"OOP``#`!W
+MW@``,"&/OP```^``"">]`!`GO?_PK[\```P`>`(`````C[\```/@``@GO0`0
+M)[W_\"0"*OBOL0`$K[```*^_``@`H(@A$*(`%@"`@"$D`BKX$B(`!@````"/
+MOP`(C[$`!(^P```#X``()[T`$!8`__H\`H`#)%&</"0"__P2(O_V)C``A"8B
+M``02`O_S)A#__`P`@6H"`"`A$`#_^R8B``0D`@`!5(+_ZB0"*O@\!(`##`!V
+M6B2$G#P0`/_E)`(J^">]__`D!``!K[\```P`>#PD!2KXC[\```/@``@GO0`0
+M)[W_\```("&OOP``#`!X/"0%*OB/OP```^``"">]`!"LA0```^``"*R```0G
+MO?_PK[$`!*^P``"OOP`,K[(`"#P"@`,`@(`AC%*<-"01``$\`X`"C&)&5"1"
+M``&L8D94C((``!!``!<\!(`#$B``!`````".`@``)$+__ZX"```\`X`"C&)&
+M5"1"__\00``)`````*QB1E0"(!`AC[\`#(^R``B/L0`$C[````/@``@GO0`0
+M#`!WA```("$0`/_V`````(R#G#0D`@`!K&(`>(R"G#0,`'*ZK$``?`)`*"$,
+M`'=$)@0`!#P"@`(,`'>$C$1&5(Y#`'PH8@`$%$``!BAD``94@``$``"((20"
+M``808@`(`````(X"```40/_3`````!8@_^<\!(`#$`#_U``````,`'-Z````
+M`!``__>.`@``)[W_X*^Q``2OOP`0K[,`#*^R``BOL```/`6``P"`B"&,LIPT
+M/`.``HQB1E0D0@`!K&)&5(RCG#0D`@`#``!`(0``2"&L8@!XC**<-*Q``'R,
+MI)PT#`!^-R2$`$B.0@!\CB,``!1@``,L4``!%@``(CP#@`,\`H`#C$*<-"1%
+M`$B,H@`H%$``&``````2```$`````(XB```D0O__KB(``#P#@`*,8D94)$+_
+M_Q!```H`````K&)&5`(`$"&/OP`0C[,`#(^R``B/L0`$C[````/@``@GO0`@
+M#`!WA```("$0`/_U``````P`?<2,I``($`#_Y@````",8IPT)!,``ZQ3`'B,
+M8IPT#`!RNJQ``'PF)``$#`!W1`)`*"$\`H`"#`!WA(Q$1E2.0P!\*&(`!A!`
+M``TD`@`&*&(`!%!```,``(`A4',``0``@"&.(@``%$#_R3P"@`,6`/_G/`.`
+M`Q``_\4`````5&+_^8XB```,`'-Z`````!``__6.(@``)[W_\*^P``"OOP`$
+M)!```3P"@`*,0T94)&,``:Q#1E2,@@``&$``$21#__^L@P``/`.``HQB1E0D
+M0O__$$``!P``("&L8D94`@`0(8^_``2/L````^``"">]`!`,`'>$`````!``
+M__@`````$`#_\```@"$GO?_PK[\`!*^P````@!@A/!"``HX"1E0D0@`!K@)&
+M5(R"``",A0`$)(0`!"1"``$4H``.K&(``(X"1E0D0O__$$``!@``("&N`D94
+MC[\`!(^P```#X``()[T`$`P`=X0`````$`#_^0`````,`'=3`````"0#``>L
+M0`!XK$,`?`P`<MX`0"`A$`#_ZP`````#X``(C((``"0"``&L@@`,K(``"*"`
+M```#X``(K(``!">]__"OOP``#`"!:B2$``B/OP```^``"">]`!`GO?_@K[,`
+M#*^Q``2OOP`0K[(`"*^P```\`H`#`("((8Q2G#0D$P`!/`.``HQB1E0D0@`!
+MK&)&5(R"``Q00``&CB,`#"9#`!B,8@`0)$(``:QB`!".(P`,)`(``E!B`%F.
+M)0`0DB(``!1``"X\`X`#4F``%HXB``PD`@`!KC(`!*(B```\`X`"C&)&5"1"
+M__\00``*`````*QB1E0"8!`AC[\`$(^S``R/L@`(C[$`!(^P```#X``()[T`
+M(`P`=X0``"`A$`#_]0````!00``&CB,`#"9#`!B,8@`0)$+__ZQB`!".(P`,
+M)`(``1!B``D`````CB,`#"0"``(48O_A``````P`>#4F1``8$`#_W0`````,
+M`'@G)D0`&!``__:.(P`,C&*<-"00``&L4`!XC&*<-`P`<KJL0`!\`D`H(0P`
+M=T0F)``(CB(`#!!0`!@"0"@A/`*``@P`=X2,1$94CD,`?"AB``040``&*&0`
+M!E2```0``)@A)`(`!A!B``@`````DB(``!!`_[D`````%F#_Y#P#@`,0`/_,
+MCB(`#`P`<WH`````$`#_]@````".)``$#`!X&"2$`!@0`/_F/`*``@P`>"XF
+M1``8$`#_I0`````GO?_PK[```*^_``0`@"@A)!```3P"@`*,0T94)&,``:Q#
+M1E20@@``5$``$```@"$\`H`#C$*<-(R#``R@L```)$8`&`#`("$08``$K*(`
+M!(S"`!`D0@`!K,(`$(RC``PD`@`"$&(`$``````\`X`"C&)&5"1"__\00``'
+M```@(:QB1E0"`!`AC[\`!(^P```#X``()[T`$`P`=X0`````$`#_^``````,
+M`'@NC*4`$!``_^X`````)[W_\*^P``"OOP`,K[(`"*^Q``0`@(`A/`*``HQ#
+M1E0D8P`!K$-&5(R"``@40``K))(`"(X"``Q00``'C@,`#(X#``0D8P`8C&(`
+M$"1"__^L8@`0C@,`#"0"``%08@`;C@0`!(X#``PD`@`"4&(`$XX$``2N```$
+MH@```#P#@`*,8D94)$+__Q!```@`````K&)&5(^_``R/L@`(C[$`!(^P```#
+MX``()[T`$`P`=X0``"`A$`#_]P`````,`'@U)(0`&!``_^RN```$#`!X)R2$
+M`!@0`/_DC@,`#`P`=U,"0"`AC@,`#`!`B"$D`@`!4&(`"(X%``0D`@`'KB(`
+M?*X@`'@,`'+>`B`@(1``_\J.`@`,)B0`&`P`>"`"0#`A$`#_]B0"``<GO?_P
+MK[\`!*^P```\`H`"C$9&6`"`*"$\`X`"C&)&5"1"``&L8D94#`!_-`#`("$,
+M`(',`$"`(0(`$"&/OP`$C[````/@``@GO0`0)[W_\*^_`````#`A$(``!`"`
+M*"$\`H`"#`"!?XQ$1EB/OP```^``"">]`!`GO?_````H(20&`"ROL``PK[\`
+M-`P`<*H`@(`A)`+__Z.B`"BOH@``KZ(`!*^B``BOH@`,KZ(`$*^B`!2OH@`8
+MKZ(`(*^B`"0\`H`"C$1&6*.@`!P\`X`"C&)&5"1"``&L8D94`Z`P(0P`@/4D
+M!0!>#`"!S`````"/H@`$7$```:X"``"/H@`(7$```:X"``2/H@`,7$```:X"
+M`!R/H@`07$```:X"`""/H@`87$```:X"`"@"`!`AC[\`-(^P`#`#X``()[T`
+M0">]__`T`K>8K[$`!*^P``"OOP`(`*"((1"B``\`@(`A-`*WF!(B``6/OP`(
+MC[$`!(^P```#X``()[T`$%8`__N/OP`(/`2``PP`@>4DA)S$$`#_]H^_``@\
+M!8`#)*6B(#P&@`0\!(`#)`(``0#%,",DA)S$%@+_Z@``."$,`(';`````!``
+M_^<T`K>8)[W_\"0$``&OOP``#`!ZIS0%MYB/OP```^``"">]`!`GO?_P```@
+M(:^_```,`'JG-`6WF(^_```#X``()[T`$)""``PPI?__``4:`C1"`("@@@`,
+M,$(`?Z"#``2@A0```^``"*""``PGO?_P/`6@P#P&__ROOP`$K[```#2E`"`0
+M@``;-,;__XR#```D`@`#-&,``P!@@"&L@P``C*,```!F&"2LHP``H@``!`P`
+M:^2B`@`,/`,;3C1C@;4`0P`9`@`@(0``$!```A."#`!ZV@!`*"$D`@`'H@(`
+M"(^_``2B```$C[````/@``@GO0`0/`*``A``_^0D1$9@$(``$#"E`/^,A```
+M`X`8(3P<@`,GG,:PD((`%#!"`"`00/_]`````*"%``"0@@`4,$(`(!!`__T`
+M`````^``"`!@X"$\`H`"$`#_[R1$1F`0@``,/`*``HR$`````!@AD((`%#!"
+M``$00``$`````)""```D`P`!H*(```/@``@`8!`A$`#_]"1$1F`GO?_@K[``
+M$*^_`!BOL0`4`("`(0.`B"$\'(`#)YS&L!"```T\`H`"`@`@(0P`>QT#H"@A
+M$$#__0(`("$"(.`ADZ(``(^_`!B/L0`4C[``$`/@``@GO0`@$`#_\R101F`G
+MO?_@K[(`"*^Q``2OL```K[\`$*^S``P`@(@A`*"0(0#`@"$#@)@A/!R``R><
+MQK`0@``5/`*``@(`$"$40``))A#__P)@X"&/OP`0C[,`#(^R``B/L0`$C[``
+M``/@``@GO0`@DD4```(@("$,`'L))E(``0(`$"$40/_Z)A#__Q``__``````
+M$`#_ZR111F`GO?_@K[(`"*^Q``2OL```K[\`$*^S``P`@(@A`*"0(0#`@"$#
+M@)@A/!R``R><QK`0@``5/`*``@(`$"$40``))A#__P)@X"&/OP`0C[,`#(^R
+M``B/L0`$C[````/@``@GO0`@#`![+`(@("$"`!@AHD(``"80__\48/_Z)E(`
+M`1``__``````$`#_ZR111F`GO?_@K[,`#*^Q``2OOP`0K[(`"*^P````@(@A
+M`*"8(0.`D"$\'(`#)YS&L!"``!X\`H`"CB,`!``#$(``0Q`A``*`0`(@("$,
+M`'L=`F`H(0!`&"$D!`!D%$``"B80__\D`O__$@(`!SP"@``D0@,`C$(`2`!`
+M^`D`````$`#_\@(@("$"0.`AC[\`$(^S``R/L@`(C[$`!(^P````8!`A`^``
+M"">]`"`0`/_B)%%&8">]_\"OL@`(K[\`$*^S``ROL0`$K[```*^F`"BOIP`L
+MKZ@`,*^I`#2OJ@`XKZL`/```D"$#@)@A/!R``R><QK`0@`!D/`*``BRB``@0
+M0``:/`.``@`%$(`D8Q9<`$,0(8Q"````0``(`````(R#```\`H`")`4``:Q%
+M1FR08@`$-%```:!P``2,D0`(*B(`!A!``!(\!*#`0`-@`"0"!``"(A`$`&(8
+M)4"#8``````````````````"8.`A`D`0(8^_`!"/LP`,C[(`"(^Q``2/L```
+M`^``"">]`$`TA``4C(,``"8B__H`11`$`&(8)1``__&L@P``C[``*`P`:^2,
+MD0````(1`@!0`!M2```!```!S9(C``0"("`AHB``!#!P`/\``!`2,$+__PP`
+M>MH`0"@AHC``!!``_]\`````C(,``#P"@`*,4D9LK$!&;)!B``0P4`#^H'``
+M!(R1``@J(@`&$$``"SP$H,!``V``)`($``(B$`0`0!`G`&(8)$"#8```````
+M`````!``_\H`````-(0`%(R%```F(__Z)`(``0!B$`0PHP`!```H(0!B*`H0
+M`/_`K(4``!``_[Z,D@`(C)(`!(^B`"@0`/^ZK((`!!``_YPD1$9@)[W_X*^P
+M`!"OOP`8K[$`%`"`,"$`H(`A``!`(0.`B"$\'(`#)YS&L!"``"8D!P`$C,0`
+M""B"``8`XB`*0`-H`"0"!```@A`$`$`0)P!B&"1`@V@`````````````````
+MC,8```.@("$D!0`!D,(`"#!#``X09P`+K@```"0"``P08@`(``````(@X"&/
+MOP`8C[$`%(^P`!`!`!`A`^``"">]`""0P@``#`!MEJ.B```00/_U)`@``20"
+M``$0`/_RK@(``#P"@`(0`/_9)$9&8">]__`\`H``K[(`"*^_``ROL0`$K[``
+M`"1"`P",0@`T`$#X"20$__\`0)`A/`*``HQ$1F@D`@`!/`6@P"2#__H`8A@$
+M*((`!A!``#,TI0`40`-@`"0"!```@A`$`$`0)P!B&"1`@V``````````````
+M````/!"``B801F`\$8``)C$#``P`>N0"`"`ACB(`-`!`^`D``"`ACB,`%#P"
+M@`(D0NT0K'```*QB``0\`H`")$+MH*QB``@\`H`")$+L)*QB``P\`H`")$+L
+ML*QB`!`\`H`")$+NX*QB`!0\`H`")$+PO*QB`!@\`H`")$+N,*QB`!R.(@`T
+M`$#X"0)`("&/OP`,C[(`"(^Q``2/L````^``"">]`!",H@``,$(``0`#$`NL
+MH@``$`#_TSP0@`(\`X`"C&)&<">]__`00``$K[\``(^_```#X``()[T`$"0"
+M``$,`'QCK&)&<!``__J/OP`````0(0``&"&LA0`0K(```*R"``@#X``(K(,`
+M#(R"```00``1`````(R#``!08``,C((``(QE``!0HP`(K(```(QB``2LH@`$
+MC&(`!*QC``2L10``K&,``*R%``",@@``5$#_\HR#```#X``(`````">]_]"O
+ML@`8)`+__R2R__^OL0`4K[\`(*^S`!ROL``0$D(`:P"`B"$\`X`"C&)&5"1"
+M``&L8D94CB0`"(XE``R.(P`0```0(0""("$`HR@A`*,P*P"&("$GLP`(KB0`
+M"*XE``P"8"`A`B`H(:^@```,`((1KZ``"(^P``@2```B`B`@(5(```R.`P`8
+MC@,``%!P``BOH``(C@(`!*QB``2.`@`$KA``!*Q#``"N$```KZ,`"(X#`!B.
+M(@`(`$,0*U1``&./HP``CB(`"%!B`%F.`P`<C@(`((X#`"0`0Q`E%$``1@(@
+M("&N```HC@(`#(X%`!``0/@)`@`@(8^P``@6`/_@`B`@(0P`@A$#H"@A/`.`
+M`HQB1E0D0O__$$``,P````"L8D94CZ(`"!!``!*/H@``CF,``%!@``R.8@``
+MC&0``%"#``BN8```C&(`!*R"``2,8@`$K&,`!*Q$``"L8P``KF0``(YB``!4
+M0/_RCF,``(^B```00``1`Z`H(8RC``!08``,C*(``(QD``!0@P`(K*```(QB
+M``2L@@`$C&(`!*QC``2L1```K&,``*RD``",H@``5$#_\HRC```F4O__)`+_
+M_Q9"_Y<`````C[\`((^S`!R/L@`8C[$`%(^P`!`#X``()[T`,`P`=X0``"`A
+M$`#_S`````".`@`8C@,`'(X&`"".!P`D`&<8(0!G0"L`1A`A`$@0(:X"`!BN
+M`P`<#`!]=@(`*"$0`/^PC@(`#(XB``P`0Q`K%$``!(^C``".(@`,$`#_HXX"
+M`"!48``#C&(`!!``_ZBOL```K@,``*X"``2,8@`$K'``!!``_Z*L4```)[W_
+M\*^Q``2OL```K[\`"`"`B"$`H(`A/`.``HQB1E0D0@`!K&)&5"0"``&LH@`H
+MC*,`&(R"``@`0Q`K5$``+8XC``",@@`(4&(`-(RC`!R.`@`,C@4`$`!`^`D"
+M`"`AC@(`((X#`"0`0Q`E4$``!:X``"B.`@`H%$``$0````"N```H/`.``HQB
+M1E0D0O__$$``!P````"L8D94C[\`"(^Q``2/L````^``"">]`!`,`'>$```@
+M(1``__@`````C@(`&(X#`!R.!@`@C@<`)`!G&"$`9R@K`$80(0!%$"&N`@`8
+MK@,`'`P`?ED"`"`ACB,``%1@``.,8@`$$`#_X:XP``"N`P``K@(`!(QB``2L
+M<``$$`#_VZQ0``",@@`,`$,0*U1`__..(P``C((`#!``_\B.`@`,)[W_\*^_
+M```\`H`"C$-&5"1C``&L0T94C((``!"B`!8`````C*(``(RC``2L0P`$C*,`
+M!*RE``2L8@``K*4``*R@`"@\`X`"C&)&5"1"__\00``%```@(:QB1E2/OP``
+M`^``"">]`!`,`'>$`````!``__H`````4*#_\JR@`"B,HP``4&7_[JR```",
+MH@`$K&(`!(RB``2LI0`$K$,``*RE```0`/_FK(,``">]_^"OI0``)`4``:^_
+M`!2OL``0KZ8`!`P`?+P`@(`ACZ(``(^_`!2N`@`4CZ(`!*X"`!B/L``0`^``
+M"">]`"`GO?_PK[\```P`?,,`````C[\```/@``@GO0`0```0(0``&"&LA0`(
+MK(8`#*R'`!"L@``HK(0`!*R$``"L@@`8K(,`'*R"`"`#X``(K(,`)```$"$`
+M`!@AK(4`"*R&``RLAP`0K(``**R$``2LA```K((`&*R#`!RL@@`@`^``"*R#
+M`"0GO?_PK[```*^_``2,@@`H`("`(11```T`@"@AC@,``!!P``>/OP`$C@(`
+M!*QB``2.`@`$KA``!*Q#``"N$```C[````/@``@GO0`0#`!]Q(R$``@0`/_R
+MC@,``">]_^"OM0`0K[0`#*^S``BOL@`$K[```*^_`!2,@@`H`("`(0"`*"$`
+MP)`A`."8(0$`H"$40``0`2"H(8X$``BN$@`8KA,`'*X4`""N%0`D#`!]=@(`
+M*"&/OP`4C[4`$(^T``R/LP`(C[(`!(^P```#X``()[T`(`P`?<2,A``($`#_
+M[XX$``@GO?_@K[(`"*^_`!2OM0`0K[0`#*^Q``2OL```C((`((R*`"0`0%@A
+M`$H0)1!``!H`@)`AC((`"(R0`"",D0`DC)0`&(R5`!R,0P`,C$(`"`(`,"$`
+M<1@A`'$@*P!0$"$`1!`A`%1`(P!U("L!!$`C`'5((R4E__\LHO__)03__P""
+M("$!:!`K%$``#0(@."$1:``)`4D0*X^_`!2/M0`0C[0`#(^R``B/L0`$C[``
+M``/@``@GO0`@4$#_^(^_`!0,``-$``````(C`!D``"`0```H$@*E."$`````
+M`(``$W(B````Y1`K`.`H(7!P`````"`2`H0P(0#","$`P"`AKD0`&*Y%`!P0
+M`/_DC[\`%">]_^`\`H`"K[\`%*^P`!`D0T9XC&,`!(Q"1G@`@(`A`&`P(0!`
+M*"&OHP`$#`!][Z^B```\"(`"/`F``B8$`!PE"/LD)2G[="0%``4D!@`!#`!U
+M$@(`."$,`&OO`````#P#@`*L8D?<0(!(`````````````````$""6```````
+M```````````,`'6()@0`'`P`=@PD!``%/`*``JQ01G2/OP`4C[``$`/@``@G
+MO0`@)[W_\*^_```,`&OO`````#P#@`*L8D?<0(!(`````````````````$""
+M6``````````````````,`'8S)`0`!8^_```D`@`#`^``"">]`!`GO?_PK[\`
+M``P`?-@`P"`A/`2``PP`=O$DA)P\C[\```/@``@GO0`0)[W_\"0"-K"OL0`$
+MK[```*^_``@`H(@A$*(`$0"`@"$D`C:P$B(`!CP"@`./OP`(C[$`!(^P```#
+MX``()[T`$!8`__HD4:#8#`!U&B8D`!P,`'W_`B`@(1``__6/OP`()`(``52"
+M_^\D`C:P/`2``PP`?ITDA*#8$`#_ZB0"-K`GO?_P)`0``:^_```,`'[G)`4V
+ML(^_```#X``()[T`$">]__```"`AK[\```P`?N<D!3:PC[\```/@``@GO0`0
+MK(4``*R&``2L@``,K(``"```0"$`@#@A)0@``23B``@M`P"`K.(`$*SB`!04
+M8/_Z`$`X(3"C``=08``#```8(20"``@`0Q@C)&(`$`#"$"M00``#C((`!`/@
+M``BL@````*,8(:R#`!`T0@`!`^``"*QB``2,@@```(!@(1!``!T``%@A!*``
+M&P`````DI0`+**(`%U1```,D!0`0)`+_^`"B*"0LH@'X$$``\``%&D(`K!`A
+M)$D`"(TG``P0Z0`/``58PHSC``0D`O_^C.H`#`!B0"0`Z!@AC&(`!(SI``@D
+MZP`(-$(``:U)``BL8@`$K2H`#`/@``@!8!`A)$D`$(TG``Q4Z?_PC.,`!"5K
+M``*-AP`8)8X`$!#N`"XD`__^C.(`!`!#0"0!!3`C*,(`$%1```ZMC@`<`.5`
+M(32B``$!!A@A-,0``23K``BLX@`$K8@`&*QF``"M!``$K0X`"*V(`!P0`/_C
+MK0X`#`3```>MC@`8`.@8(8QB``0DZP`(-$(``1``_]NL8@`$+0("`!!``((`
+M"!I"``@@P@`$$,`!@A`A)$H`"(V#``R-20`(``0@@B0"``$`@A`$`&(8):V#
+M``RLZ@`,K.D`"*TG``RM1P`(*6,``"5B``,!8Q`*C88`#``"$(,D`P`!`$,@
+M!`#$$"M40``[C80`$`"&$"040``,``L0P``$($`D`O_\`6(0)`"&&"048``%
+M)$L`!``$($``AA`D$$#__25K``0`"Q#``8(0(21)``@!8&@A`2!0(8U'``Q0
+MZ@`.)6L``8SB``0D`__^`$-`)`$%,",HP@`04$``/XSI``@$PP`U`.@8(8SG
+M``Q4ZO_WC.(`!"5K``$Q8@`#%$#_[B5*``@QH@`#)2G_^!!``"4EK?__C2(`
+M"!!)__LQH@`#C8,`#``$($``9!`K5$``"HV$`!`0@``'`(,0)%1`_]H`"Q#`
+M``0@0`"#$"00`/_[)6L`!(V$`!`D`__^C((`!`!#$"0`11@K%&``!`!%,",H
+MP@`04$```P"`."$0`/]Z``!8(33#``$`A2`A-*(``:SB``2MA``0K(,`!!``
+M_W(DZP`(C8(`#``$&"<`0Q`D$`#_VZV"``R,8@`$C.H`#(SI``@T0@`!).L`
+M"*QB``00`/]DK4D`"(SJ``P`Y4`A-*(``0$&&"$TQ``!K4D`"*SB``2MB``8
+M).L`"*QF``"M!``$K0X`"!``_W*M*@`,$&``!@`((,(L8@`%$$``'BQB`!4`
+M"!&")$0`.``$$,`!@A`A)$H`"(U)``@1*@`1)(,``XTB``0D`__^`$,0)`$"
+M$"M00/]XC2H`#(TI``@1*@`&)`/__HTB``0`0Q`D`0(0*U1`__J-*0`($`#_
+M;HTJ``PH@@```&(@"XV#``P0`/]E``0@@Q1`_^0D9`!;+&(`51!```0L8@%5
+M``@3`A``_]XD1`!N$$``!"QB!54`"!/"$`#_V21$`'=00/_7)`0`?@`(%((0
+M`/_4)$0`?!!@``8`!5C"+&(`!1!``!TL8@`5``41@B1+`#@`"Q#``8(0(21*
+M``B-1P`,4.K_'25K``&,X@`$)`/__@!#0"0!!3`C*,(`$%!```8E:___!,,`
+M!@#H&"&,YP`,5.K_]XSB``00`/\/)6L``8QB``2,Z@`,C.D`"#1"``$0`/[_
+M).L`"!1`_^4D:P!;+&(`51!```0L8@%5``43`A``_]\D2P!N$$``!"QB!54`
+M!1/"$`#_VB1+`'=00/_8)`L`?@`%%((0`/_5)$L`?`"`6"$0H`!````8(22I
+M__B-)@`$)`+__HR$`!``PD`D`2A0(8U#``011`!U`&(8)#C"``$P0@`!K4,`
+M!!!```H``&`AC*7_^"5B`!`!)4@CC20`"!""`&@!!4`AC2<`#*SD``BLAP`,
+M`4,0(9!"``<X0@`!,$(``1!```PU`@`!%8``!0$#0"&-0P`()6(`$%!B`%0D
+M#``!C48`"(U'``RLY@`(K,<`##4"``$!*!@AK2(`!!6``!.L:```+0("`%!`
+M`!,`"!I"``@@P@`$$,`!8A`A)$<`"(UC``R,Y@`(``0@@B0"``$`@A`$`&(8
+M):UC``RM)P`,K28`"*S)``RLZ0`()`,``0/@``@`8!`A$&``!@`((,(L8@`%
+M$$``'RQB`!4`"!&")$0`.``$$,`!8A`A)$<`"(SF``@0QP`2)(,``XS"``0D
+M`__^`$,0)`$"$"M00/_GC,<`#(S&``A0Q__DC,<`#(S"``0D`__^`$,0)`$"
+M$"M40/_YC,8`"!``_]R,QP`,*((```!B(`N-8P`,$`#_TP`$((,40/_C)&0`
+M6RQB`%400``$+&(!50`($P(0`/_=)$0`;A!```0L8@55``@3PA``_]@D1`!W
+M4$#_UB0$`'X`"!2"$`#_TR1$`'RM:0`<K6D`&*TC``P0`/^MK2,`"!``_YLD
+M#``!.,(``3!"``$00``(`0-`(8RE__@!)4@CC2<`#(TF``@!!4`AK.8`"*S'
+M``PU`@`!)`,``:UI`!`0`/^SK2(`!#"E__\PH@!<4$``*(R"``",@@`0)(@`
+M$"0,`'Z,0P`$)`+__@!B2"0I(@`0.$L``0$@4"&-!P`,4.@`"R6,__^,X@`$
+M)`/__HSG``P`0Q`D`4(8*@!#4`L!(D@A%.C_^"5K``$EC/__!8'_\B4(``@P
+MH@`(4$``!20"__B,@@`$`$D0(ZS"``PD`O_X`4(8)`$B$"0D0O_L)&/_[*S"
+M`!"LRP`(K,,`&(R"``"LP@``K,(`((R#``0D`@`8H,(`**S#``0#X``(K,,`
+M)(R"``PD`__P/`H1$21)_\@!(T@D)2+_6`!#,"0``$`A`,`X(0%($"4E"``!
+M*0,`(*SB```48/_[).<`!#P#@`(D8\=<)`(``:S%`!"LR0!XK,,`D*S"`(RL
+MR0!TK,``A*S``("LPP!\`^``"*R&``PGO?_PK[\``#P#@`*,8D94)$(``:QB
+M1E0\`H`"C$)&3!1``!$`0!@AK(0`G#P"@`*L1$9,/`.``HQB1E0D0O__$$``
+M!0````"L8D94C[\```/@``@GO0`0#`!WA```("$0`/_Z`````!"#__`\`H`"
+MC$)&3(QC`)P48O_[`````!"#_^L\`H`"C&(`G*R"`)P0`/_FK&0`G(R"```0
+M0``1`````(R#``!08``,C((``(QE``!0HP`(K(```(QB``2LH@`$C&(`!*QC
+M``2L10``K&,``*R%``",@@``5$#_\HR#```#X``(`````">]_]"OL@`8K[\`
+M**^U`"2OM``@K[,`'*^Q`!2OL``0`("0(3P"@`*,0T94)&,``0P`@&6L0T94
+M`$"H(8Y"!!`00``:)E0$$*^@```,`'=3`H`@(8Q3`$`"0"`A`$"`(0P`?S2.
+M90```$"((0(`("$00``L`@`H(20"``>N`@!\#`!RWJX``'BN<0`$CD($$%1`
+M_^XF5`00CZ(``!1``!<`````#`"!:@.@("$\`X`"C&)&5"1"__\00``,````
+M`*QB1E0"H!`AC[\`*(^U`"2/M``@C[,`'(^R`!B/L0`4C[``$`/@``@GO0`P
+M#`!WA```("$0`/_S``````P`=U,#H"`A`$`H(0P`=T0"@"`ACZ(``!1`__D`
+M````$`#_X0`````,`'=$`Z`@(1``_]B.0@00)[W_\*^_```\`X`"C&)&5"1"
+M__\00``%```@(:QB1E2/OP```^``"">]`!`,`'>$`````!``__H`````)[W_
+M\*^P``"OOP`$#`!_%P"`@"&N``00C[\`!(^P```#X``()[T`$">]__"OL```
+MK[\`#*^R``BOL0`$`("`(3P"@`*,0T94)&,``:Q#1E2,@@00))($$!1``!,"
+M0(@A/`.``HQB1E0D0O__$$``"@````"L8D94#`"!:@)`("&/OP`,C[(`"(^Q
+M``2/L````^``"">]`!`,`'>$```@(1``__4`````#`!W4P(@("$D`P`%K$,`
+M?*Q``'@,`'+>`$`@(8X"!!`40/_W`````!``_^,`````C*<``!#@``8`@!@A
+MC(8``%3```6,X@`$K(<``*R@```#X``(`````(S$``2LP@`$C&,``*SD``2L
+M0P``$`#_]ZR'``!-96UO<GD@=7-E9#H@)60@1G)E93H@)60@"@H`6T)U9F9E
+M<B!S:7IE(%1X.B`E9"`@4G@Z("5D70H*``!405)'7W=D8U1X37-G1&ES<&%T
+M8VA,;V]P````@`!&?(``1JR``$;$@`!&K(``1JR``$:L@`!&K(``1O2``$:L
+M@`!&K(``1JR``$:L@`!&K(``1JR``$:L@`!&K(``1JR``$:L@`!&K(``1JR`
+M`$:L@`!&K(``1JR``$<4@`!')&%R0V]M<&QE=&5,;V]P``!405)'7W=D8T%S
+M>6YC179T1&ES<&%T8VA,;V]P`````%1A<F=E="!S=&%R=&5D"@"``%>0@`!8
+M*(``6%2``%AH@`!9%(``68R``%H@@`!:/(``6G"``%J$@`!:F(``6JR``%L(
+M@`!;6(``5^"``%N8@`!7X(``5^"``%?@@`!7X(``5^"``%NP@`!;V(``6_R`
+M`%PT@`!<E(``702``%TL@`!=/(``76"``%[$@`!7X(``7SR``%]@@`!?[(``
+M8""``&!,@`!@7(``8'2``&&D@`!B4(``5^"``&)@@`!BF(``8JB``%?@@`!B
+MS(``5^"``%?@@`!B](``8RR``&-D@`!CA(``5^"``&0\@`!D3(``9'"``&5(
+M@`!E^(``5^"``%?@@`!7X(``5^"``%?@@`!7X(``5^"``&8P5$%21U]W9&-#
+M=')L37-G1&ES<&%T8VA,;V]P`$%#22`D260Z("\O9&5P;W0O<W<O8G)A;F-H
+M97,O,2XU55-"1&5V+W-R8R]H86PO:&%L36ES8RYC(S$U("0```"``'=L@`!W
+M;(``=VR``'=L@`!W;(``=VR``'=L@`!W;(``=VR``'=L@`!W;(``=VR``'=L
+M@`!W;(``=VR``'=L@`!W?(``=Y"``'>D@`!X`"1)9#H@+R]D97!O="]S=R]B
+M<F%N8VAE<R\Q+C554T)$978O<W)C+VAA;"]A<C4R,3(O87(U,C$R071T86-H
+M+F,C,3<@)```````@````)@@55555:JJJJIF9F9FF9F9F21)9#H@+R]D97!O
+M="]S=R]B<F%N8VAE<R\Q+C554T)$978O<W)C+VAA;"]A<C4R,3(O87(U,C$R
+M2V5Y0V%C:&4N8R,W("0``(``L'"``*>L@`"G?(``L+B``*>L@`"GK(``IZR`
+M`*>L@`"GK(``IZR``*>L@`"GK(``IZR``*>L@`"GK(``IZR``+#<@`"P[(``
+MIZR``*>L@`"GK(``IZR``*>L@`"GK(``IZR``*>L@`"GK(``IZR``*>L@`"G
+MK(``IZR``*>L@`"P["1)9#H@+R]D97!O="]S=R]B<F%N8VAE<R\Q+C554T)$
+M978O<W)C+VAA;"]A<C4R,3(O87(U,C$R36ES8RYC(S(P("0`@`"Z-(``NC2`
+M`+M$@`"[;(``N\`N+B\N+B\N+B\N+B\N+B]H86PO87(U,C$R+V%R-3(Q,DUI
+M<V,N8P``(2`E+3,P<SH@)2TR,',Z)2TU9"!%0T]37TY/5$1/3D4*````87(U
+M,C$R27-(=T-I<&AE<E-U<'!O<G1E9````/___\G____"____\O____3____`
+M____NO___[+___^P````?P```#(```!_````*````'\```!-````?P```$``
+M```?````$````#\````P````"`````8`````````!`````@````"````!```
+M``8````(````"@````P````.````$(``Q8R``,:\@`#(K(``R0"``,EH)$ED
+M.B`O+V1E<&]T+W-W+V)R86YC:&5S+S$N-5530D1E=B]S<F,O:&%L+V%R-3(Q
+M,B]A<C4R,3)0;W=E<BYC(S4@)`"``-14@`#4[(``T]B``-5(@`#6#"1)9#H@
+M+R]D97!O="]S=R]B<F%N8VAE<R\Q+C554T)$978O<W)C+VAA;"]A<C4R,3(O
+M87(U,C$R4F5C96EV92YC(S@@)````(``V&2``-DT@`#9N(``V?R``-IP)$ED
+M.B`O+V1E<&]T+W-W+V)R86YC:&5S+S$N-5530D1E=B]S<F,O:&%L+V%R-3(Q
+M,B]A<C4R,3)297-E="YC(S$U("0`````(#9M8B!/1D1-````(#EM8B!/1D1-
+M````,3)M8B!/1D1-````,3AM8B!/1D1-````,C1M8B!/1D1-````,S9M8B!/
+M1D1-````-#AM8B!/1D1-````-31M8B!/1D1-````,4P@("!#0TL@````,DP@
+M("!#0TL@````,E,@("!#0TL@````-2XU3"!#0TL@````-2XU4R!#0TL@````
+M,3%,("!#0TL@````,3%3("!#0TL@````6%(@("`@("`@````@`(.O(`"#LB`
+M`@[4@`(.X(`"#NR``@[X@`(/!(`"#Q"``@\<@`(/*(`"#S2``@]`@`(/3(`"
+M#UB``@]D@`(/<(`!)_B``2@`@`$H"(`!*!"``2@0@`$H&(`!*""``4&@@`%!
+MS(`!0=2``4'<@`%!Y(`!0>R``4'T@`%!_(`!0@2``4(0@`%"&(`!0B2``4(P
+M@`%"/(`!0DB``4)4@`%"8(`!0FR``4)X@`%"A(`!0I"``4*<@`%"J(`!0K2`
+M`4+`@`%"X(`!0NR``4+X@`%#!(`!0Q"``4,<@`%#*(`!0S2``4-`@`%#3(`!
+M0UB``4-D@`%#<"XN+RXN+RXN+RXN+RXN+VAA;"]W9&,O=&%R9V5T+W-R8R]A
+M<D-O;FXN8P``)7,Z)60@4F5S971T:6YG(%)E=')Y(%%U975E(0H```"``5QL
+M@`%<I(`!7*R``5RX@`%<P(`!7'B``5QX@`%<R(`!7-"``5S8@`%<X(`!7'B`
+M`5SH@`%<\(`!7/B``5T`@`%<>(`!7'B``5T(@`%<>(`!71R``5V`@`%=B(`!
+M79"``5V8@`%=H(`!7:B``5VP@`%<>(`!7;B``5W$@`%=T(`!7=B``5WT@`%@
+MY(`!8B2``6'`@`%AS(`!8B2``6'4@`%AW(`!8>2``6'L@`%A](`!8?R``6(<
+M@`%B)(`!8BR``6)(55-".B5S(%LE9%T*`````'9E;F1O<E)E<V5T`(`!>IB`
+M`7JT@`%Z4(`!>M"``7I0@`%Z[(`!>PB``7LD@`%[0(`!>UR``7NX@`%[U(`!
+M>_!/=F5R:61E(#H@)60@96X@.B`E9`H```!#;VYF:6<@.B`E>`H`````@`&*
+MZ(`!BO"``8L4@`&+'(`!BR2``8LL@`&+L(`!B[2``8O8@`&+\(`!B_B``8P`
+M@`&,"$4*```P,3(S-#4V-S@Y04)#1$5&`````#`Q,C,T-38W.#D``#`Q,C,T
+M-38W.#EA8F-D968`````/$YO="!A('-T<FEN9SH@,'@````^````/&YU;&P^
+M```\0F%D(&9O<FUA="!S=')I;F<Z(``````@.@``/@H``(`!O0R``;U0@`&]
+M#(`!O5"``;U0@`&]4(`!O5"``;U0@`&]4(`!O5"``;U0@`&]4(`!O5"``;U0
+M@`&]4(`!O5"``;U0@`&]4(`!O5"``;T,@`&]4(`!O5"``;T,@`&]4(`!O5"`
+M`;U0@`&]4(`!O5"``;U0@`&]4(`!O5"``;U0@`&]#(`!O5"``;T,@`&]4(`!
+MO5"``;U0@`&]4(`!O5"``;U0@`&]4(`!O5"``;U0@`&]4(`!O5"``;T,@`&]
+M4(`!O5"``;U0@`&]4(`!O0R``;U0@`&]4(`!O0R``;]0@`'`C(`!P(R``<",
+M@`'`C(`!P(R``<",@`'`C(`!P(R``<",@`'`C(`!P(R``<",@`'`C(`!P(R`
+M`<",@`'`C(`!P(R``<",@`'`C(`!P(R``<",@`'`C(`!P(R``<",@`'`C(`!
+MP(R``<",@`'`C(`!OV2``;_D@`&]I(`!P(R``<",@`'`C(`!P(R``<",@`'`
+MC(`!P(R``<",@`'`C(`!P(R``<",@`'`C(`!P(R``<",@`'`!(`!P(R``;VD
+M@`'`C(`!P(R``;VD@`'`C(`!P(R``<",@`'`C(`!P(R``<",@`'`C(`!P(R`
+M`<",@`&_9(`!O^2``;VD@`'`C(`!P(R``<",@`'`C(`!P(R``<",@`'`C(`!
+MP(R``<",@`'`C(`!P(R``;U\@`'`C(`!P(R``<`$@`'`C(`!O:2``<",@`'`
+MC(`!O:2``;W4@`&]^(`!O?B``;WX@`&]^(`!O?B``;WX@`&]^(`!O?B``;WX
+M@`&]^(`!O?B``;WX@`&]^(`!O?B``;WX@`&]^(`!O=2``;WX@`&]^(`!ORB`
+M`;WX@`&]^(`!O?B``;WX@`&]^(`!O?B``;WX@`&]^(`!O?B``;WX@`&]^(`!
+MO=2``;WX@`&]^(`!O?B``;WX@`&]^(`!O?B``;WX@`&]^(`!O?B``;WX@`&]
+M^(`!OT2``;WX@`&]^(`!O?B``;WX@`&]U(`!O?B``;WX@`&_1"4P.%@Z(```
+M)3`R6"`````@````('P``"5C``!\"@``("`@`"4P.%@@````"@```"`@("`@
+M("`@(````"4P-%@@````("`@("````"``<VT@`'-H(`!S:"``<V@@`'-M(`!
+MS;2``<VT@`'-M(`!SO"``<]8@`'/6(`!SUB``<\,@`'/#(`!SPR``<\,@`'1
+M?(`!T7R``=%H@`'1L(`!T7R``=%\@`'1?(`!T7Q)9&QE(%1H<F5A9`"``>_8
+M@`'OF(`![YB``>^8@`'P((`![TR``?"<@`'PI``!`@(#`P,#!`0$!`0$!`0%
+M!04%!04%!04%!04%!04%!@8&!@8&!@8&!@8&!@8&!@8&!@8&!@8&!@8&!@8&
+M!@8'!P<'!P<'!P<'!P<'!P<'!P<'!P<'!P<'!P<'!P<'!P<'!P<'!P<'!P<'
+M!P<'!P<'!P<'!P<'!P<'!P<'!P<'"`@("`@("`@("`@("`@("`@("`@("`@(
+M"`@("`@("`@("`@("`@("`@("`@("`@("`@("`@("`@("`@("`@("`@("`@(
+M"`@("`@("`@("`@("`@("`@("`@("`@("`@("`@("`@("`@("`@("`@("`@(
+M"`@("`@("`@("`@("`@);`F/";0```D("6P)M```````!0`*`!0`'@`R`$8`
+M50!:`%\`9```````"@`4`!X`*``R`#P`1@!0`%H`9```````,```@!4``(`5
+M``"`%0``@!4``(`5```00``O_`\`+_P/`"_\'P`O_`\`+_P/```01``O_`\`
+M+_P/`"_\'P`O_`\`+_P/```02``O_`\`+_P/`"_\'P`O_`\`+_P/```03``O
+M_`\`+_P/`"_\'P`O_`\`+_P/```04``O_`\`+_P/`"_\'P`O_`\`+_P/```0
+M5``O_`\`+_P/`"_\'P`O_`\`+_P/```06``O_`\`+_P/`"_\'P`O_`\`+_P/
+M```07``O_`\`+_P/`"_\'P`O_`\`+_P/```08``O_`\`+_P/`"_\'P`O_`\`
+M+_P/```09``O_`\`+_P/`"_\'P`O_`\`+_P/```0,````C````'@````L```
+M`6````'@```0<````6@```'@```!N````8P```'@```0L```#F```!&````?
+M'```/C@``!&````0\```H.```4!H``!8@```L.```4!H``"`%`/H`^@&X`;@
+M!"`$(`A`"$`&X`;@``"`'!*-DZ<)B!//!.`3E1+@$ZL)B!//``"8!```````
+M```#```````````````#``"8(`("`@`"`@(``@$"``("`@`"`@(```"8)```
+M#@X```X.```'!P``#@X```X.``"8*`H"``$*`@`!!0(!``H"``$*`@`!``"8
+M-```#@X```X.```.#@``#@X```X.``"8.`````<````'````"P````L````+
+M``"81!-R%AP3<APE$W(7*!-R%J(3<APE``"82``8VFT`&-IM`!C*=0`8RG4`
+M&,IU``"84`WHM.`-Z+3@#>BTX`WHM.`-Z+3@``"86'Z`#2Y^@`TN?N@-+G[H
+M#2Y^@`TN``"87#$W9EXQ-V9>,3=F7C$W9EXQ-V9>``"88```G1```)T0``"=
+M&```G1@``)T0``"89``!S@```<X```'.```!S@```<X```"8:$":09!`FD&0
+M0)I!D$":09!`FD&0``"8;`4,L($%#+"!!0RP@04,L($%#+"!``"9%```!]``
+M``?0```$3```")@```?0``"9&````;@```&X````A````0@```&X``"9)!`%
+MB@40!8H%$`6*!1`%B@40!8H%``"91/>X$"#WN!`@][@-$/>X$!#WN!`0``"B
+M!```````````````"`````@````(``"B"-:^1XC6OD>(T#Y'B-`^1XC0/D>(
+M``"B#&0L`4!D+`%`9$+!8&1"P6!D0L%@``"B'!B#@`H8@X`*&&.`"AB#@`H8
+M@X`*``"B,````````````````````0@`````````#``````````T````!0``
+M`$``````````1`````@```!(````"````$P````0````4`````````!4````
+M'P``"``````````(!`````````@(````````"`P````````($`````````@4
+M````````"!@````````('`````````@@````````""0````````2,```````
+M`!)P````````$#@````````0>````````!"X````````$/@````````1.```
+M`````!%X````````$;@````````1^````````!(X````````$G@````````2
+MN````````!+X````````$S@````````3>````````!.X````````$_@`````
+M```4.````````!1X````````%+@````````4^````````!4X````````%7@`
+M```````5N````````!7X````````%C@````````6>````````!:X````````
+M%O@````````7.````````!=X````````%[@````````7^````````!`\````
+M````$'P````````0O````````!#\````````$3P````````1?````````!&\
+M````````$?P````````2/````````!)\````````$KP````````2_```````
+M`!,\````````$WP````````3O````````!/\````````%#P````````4?```
+M`````(`$````````@`@```````"`#````````(`8````````@"````````"`
+M)````````(`H````,```@"P`!___``"`,`'___\``(`T````,0``@#@`````
+M``"`/````````(!(````````@%0```````"`6````````(!<___'_P``@&``
+M```/``"`@````````("$````````@(@```````"`C````````("0````````
+M@)0```````"`F````````(#`*H(P&@``@,0%W`'@``"`R!]`)Q```(#,`?0`
+M````@-```!X<``"`U``"JJH``(#8`@!550``@-P```````"`X/____\``(#D
+M``#__P``@.@```````"`[````````(#P````````@/0```````"`^```````
+M`(#\````B```AP````````"'!````(P``(<(````Y```APP```+5``"'$```
+M`````(<4````````AQ@```"@``"''````<D``(<@````+```AR0````L``"'
+M*````#```(<L````/```AS`````L``"'-````"P``(<X````,```ASP````\
+M``"'0````````(=$````````AT@```````"'3````````(=0````````AU0`
+M``````"'6````````(=<````````AV````#5``"'9````-\``(=H```!`@``
+MAVP```$Z``"'<````'4``(=T````?P``AW@```"B``"'?````````($```$`
+M`@``@00````!``"!"````,```($,````````@1````%H``"!%````````(%`
+M@``"J```@40```````"'P`,"`0```(?$!P8%!```A\@+"@D(``"'S`\.#0P`
+M`(?0$Q(1$```A]07%A44``"'V!L:&1@``(?<'QX='```A^`#`@$```"'Y`<&
+M!00``(?H"PH)"```A^P/#@T,``"'\!,2$1```(?T%Q85%```A_@;&AD8``"'
+M_!\>'1P``)@(````````F`RMA(X9``"8$'THX````)@4G`J?:P``F!P`````
+M``"8+``"H`(``)@\``(!````F$`@:@%Z``"83!*$(SP``)A4```(60``F0``
+M``````"9!````````)D(````````F0P`@`````"9$`````$``)D<```,@```
+MF2`%$`````"9*`````$``)DL````!```F3```$B"``"9-!X?("(``)DX"@L,
+M#0``F3P````_``"90`````0``)E(DH"R$@``F51=4.&(``"96````/\``)E<
+M`$MJC@``F6@```/.``"9<!DOM14``)ET````````F7@````!``"9?```````
+M`)F`````````F80"@`````"9H````````)G@````````F>2JJJJJ``"9Z#Q&
+M9'@``)GL````J@``HA``@&,S``"B%``0;!```*(8`)Q`8```HB`!B##&``"B
+M)```!````*(H```)M0``HBP```````"B-"`@("```*(X("`@(```HCP3R(FO
+M``"B0#A)"B```*)$``![M@``HD@/_S_\``"B3````````*)0````````HE0`
+M``````";`````````)L$`````0``FP@````"``";#`````,``)L0````!```
+MFQ0````%``";&`````@``)L<````"0``FR`````*``";)`````L``)LH````
+M#```FRP````-``";,````!```)LT````$0``FS@````2``";/````!,``)M`
+M````%```FT0````5``";2````!@``)M,````&0``FU`````:``";5````!L`
+M`)M8````'```FUP````=``";8````"```)MD````(0``FV@````B``";;```
+M`",``)MP````)```FW0````E``";>````"@``)M\````*0``FX`````J``";
+MA````"L``)N(````+```FXP````M``";D````#```)N4````,0``FY@````R
+M``";G````#,``)N@````-```FZ0````U``";J````#4``)NL````-0``F[``
+M```U``";M````#4``)NX````-0``F[P````U``";P````#4``)O$````-0``
+MF\@````U``";S````#4``)O0````-0``F]0````U``";V````#4``)O<````
+M-0``F^`````U``";Y````#4``)OH````-0``F^P````U``";\````#4``)OT
+M````-0``F_@````0``";_````!H``)H`````!P````<``)H$````1P```$<`
+M`)H(````AP```(<``)H,```!H````:```)H0```!X````>```)H4````(```
+M`"```)H8````8````&```)H<```!H0```:$``)H@```!X0```>$``)HD````
+M(0```"$``)HH````80```&$``)HL```!8@```6(``)HP```!H@```:(``)HT
+M```!X@```>(``)HX````(@```"(``)H\````8@```&(``)I````!8P```6,`
+M`)I$```!HP```:,``)I(```!XP```>,``)I,````(P```",``)I0````8P``
+M`&,``)I4```!A````80``)I8```!Q````<0``)I<````!`````0``)I@```!
+MZ@````L``)ID````*@```$L``)IH````:@```(L``)IL````J@```:P``)IP
+M```!JP```>P``)IT```!ZP```"P``)IX````*P```!(``)I\````:P```%(`
+M`)J`````JP```)(``)J$```!K````9,``)J(```![````=,``)J,````+```
+M`!,``)J0````.@```%,``)J4````>@```),``)J8````N@```90``)J<```!
+MNP```=0``)J@```!^P```!0``)JD````.P```#H``)JH````>P```'H``)JL
+M````NP```+H``)JP```!O````;L``)JT```!_````?L``)JX````/````#L`
+M`)J\````?````'L``)K`````O````+L``)K$````_````;P``)K(````_```
+M`?P``)K,````_````#P``)K0````_````'P``)K4````_````+P``)K8````
+M_````/P``)K<````_````/P``)K@````_````/P``)KD````_````/P``)KH
+M````_````/P``)KL````_````/P``)KP````_````/P``)KT````_````/P`
+M`)KX````_````/P``)K\````_````/P``)C4````(```F-`#!@0(`P<$"`,&
+M!`@#!@0(`P<$"```F-P`H,#``*#`P`#@P,``X,#``.#`P```F)P*````"@``
+M``H````*````"@``````F)P`````````````````````````````F)P`@```
+M`(````"`````@````(``````F)P`*@```"H````J````*@```"H`````F)P`
+M`0````$````!`````0````$`````F)P`````````````````````````````
+MF)P`&````!@````8````&````!@`````F)P`8````&````!N````;@```&X`
+M````F)P`QP```,<```#'````QP```,<`````F)P`2P```$L```!+````2P``
+M`$L`````F)P$2```!$@```1(```$2```!$@`````F)P`3````$P```!,````
+M3````$P`````F)P`Y````.0```#D````Y````.0`````F)P`````````````
+M````````````````F)P`_````/P```#\````_````/P`````F)P`_P```/\`
+M``#_````_P```/\`````F)P$/P``!#\```0_```$/P``!#\`````F)P,#```
+M#`P```P,```,#```#`P`````F)P"&0```AD```(9```"&0```AD`````F)P`
+M)````"0````D````)````"0`````F)P`M````+0```"T````M````+0`````
+MF)P`F0```)D```"9````F0```)D`````F)P`4````%````!0````4````%``
+M````F)P`*@```"H````J````*@```"H`````F)P`$@```!(````2````$@``
+M`!(`````F)S`,@``P#(``,`R``#`,@``P#(`````F)P!=````70```%T```!
+M=````70`````F)P`$0```!$````1````$0```!$`````F)R&*```AB@``(8H
+M``"&*```AB@`````F)PQA```,80``#&$```QA```,80`````F)P`\@"``/(`
+M@`#R`(``\@"``/(`@```F)P`)P`9`"<`&0`G`!D`)P`9`"<`&0``F)P````#
+M`````P````,````#`````P``F)P`````````````````````````````F)P`
+M``"R````L@```+(```"R````L@``F)P`L"&$`+`AA`"P(80`L"&$`+`AA```
+MF)P`026D`$$EI`!!):0`026D`$$EI```F)P`$9(@`!&2(``1DB``$9(@`!&2
+M(```F)P`&D@``!I(```:2```&D@``!I(````F-@`"P(P``L",``+`C``"P(P
+M``L",```F)P```"4````E````)0```"4````E```F)P```"1````D0```)$`
+M``"1````D0``F)P````2````$@```!(````2````$@``F)P```"`````@```
+M`(````"`````@```F)P```#9````V0```-D```#9````V0``F)P```!@````
+M8````&````!@````8```F)P```#P````\````/````#P````\```F)P```"B
+M````H@```*(```"B````H@``F)P```!2````4@```%(```!2````4@``F)P`
+M``#4````U````-0```#4````U```F)P``!3,```4S```%,P``!3,```4S```
+MF)P```2,```$C```!(P```2,```$C```F,0````#`````P````,````#````
+M`X``\02``/5L@`$,<`````,`!@`)`#\```````D````$``0``0`!``$`````
+M``````````9&1S@```````````````````0````!``$```````````````1&
+M1S<```````````````````,``0`!``$```````````````-&1S8`````````
+M``````````0```````$```````````````%&1S4```````````````````0`
+M`0`!``````````````````!&1S0```````````````````0````!````````
+M`````/____Y&1S,```````````````````,``0`!`````````````/____U&
+M1S(```````````````````0``````````````````/____Q&1S$`````````
+M``````````(``0`!`````````````/____I&1S``````````````````````
+M````````````````````````````````````````````````"`````$``P``
+M````````````````````!D9'-P```````````````````@``````````````
+M`````````$9'-@```````````````````0``````````````````_____49'
+M-0``````````````````````````````````````____^D9'-```````````
+M```````````!``$`````````````____^$9',P`````````````````````!
+M``$````!``$`````____]D9',@`````````````````````!`````0`!``$`
+M````____\T9',0`````````````````````!`````0`!`````0``____\$9'
+M,```````````````````````````````````````````````````````````
+M```````````````````````````````````````````````````````````"
+M````!`````H````!`````P````H````"````!0````H````"````!`````H`
+M```"`````P````H``````````@````,```````````````"``D1X@`)$@(`"
+M1,2``D3H@`)$?/\``0P"!@`-`P`'``````X*!```"```&0``````%1L/'PL%
+M```````)```8```4&AX`````%P`3'0`6$AP1$``"8EH``/#<*``$```!;C8`
+M`<G#@`.3AP`')PX``+<;``````"```>`@``&=(``!G2```9T@``&=(``!G2`
+M``9T@``&=(``!G2```9T@``&=(``!G2```9T@``&=(``!G2```9T@``&=(``
+M!G2```9T@``&=(``!G2```9T@``&=(``!G2```9T@``&=(``!G2```9T@``&
+M=(``!G2```9T@``&=(``!G2```9T@``&=(``!G2```9T@``&=(``!G2```9T
+M@``&=(``!G2```9T@``&=(``!G2```9T@``&=(``!G2```9T@``&=(``!G2`
+M``9T@``&=(``!G2```9T@``&=(``!G2```9T@``&=(``!G2```9T@``&=(``
+M!G2```9T@`&VF(`!MIB``;:8@`&VF(`!MIB``;:8@`&VF(`!MIB``;:8@`&V
+MF(`!MIB``;:8@`&VF(`!MIB``;:8@`&VF(`!MIB``;:8@`&VF(`!MIB``;:8
+M@`&VF(`!MIB``;:8@`&VF(`!MIB``;:8@`&VF(`!MIB``;:8````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````@`*A$```````$P``@`!X2!$3``"``'A(`#```(``>$@`%```@`!X
+M2``5``"``'A(`!8``(``>$@`%P``@`!X2``8``"``'A(`!D``(``>$@`4``3
+M`%$`$P!3`!,`5@`3`%4`$P!5`!,`@``3`($`$P```,(```#$````Q0```-``
+M``#:````Y````,(```##````U````/(```$-```!*```````````````````
+M`!#_``````````````````````````````````````````````$`````%W`5
+M&`L`C``"`0`\`#P```````$`````(R@>>`\`$@`#`0`\`#P```````$`````
+M+N`G$`H`F`($`@`P`#````````$`````1E`V3`X`)`(&`@`P`#````````$`
+M````7<!#E`D`L`0*`P`L`"P```````$`````C*!9V`T`2`0.`P`L`"P`````
+M``$`````NX!K"`@`8`03`P`L`"P```````$`````TO!R=`P`;`07`P`L`"P`
+M``````$````!%W`I:`L`C`@"`0`B`"(```````$````!(R@\\`\`$@@#`0`B
+M`"(```````$````!+N!+R`H`F`H$`@`>`!X```````$````!1E!I%`X`)`H&
+M`@`>`!X```````$````!7<"!L`D`L`P*`P`:`!H```````$````!C*"J4`T`
+M2`P.`P`:`!H```````$````!NX#(9`@`8`P3`P`:`!H```````$````!TO#7
+M/`P`;`P7`P`:`!H`````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M`````````````````````````````````````````````````````&0````R
+M!0LR!P@```````````````````C_````````````````````````````````
+M``````````````$````!%W`5&`L`C``"`0`B`"(```````$````!(R@>>`\`
+M$@`$`0`B`"(```````$````!+N`G$`H`F`('`@`>`!X```````$````!1E`V
+M3`X`)`()`@`>`!X```````$````!7<!#E`D`L`0.`P`:`!H```````$````!
+MC*!9V`T`2`01`P`:`!H```````$````!NX!K"`@`8`06`P`:`!H```````$`
+M```!TO!R=`P`;`0:`P`:`!H`````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M`````````````````&0````R````!P@``````````````````!/_````````
+M``````````````````````````````````````$````"`^@#A!L``@```0$Z
+M`3H```````$````"!]`';!H$!`$!`0$"`*(```````$````"%7P3)!D$"P("
+M`@#?`'\```````$````"*O@?I!@$%@,#`@#5`'4`````````````%W`5&`L`
+M#`0"`0`\`#P`````````````(R@>>`\`$@0#`0`\`#P```````$`````+N`G
+M=`H`&`8$`0`P`#````````$`````1E`W%`X`)`8&`@`P`#````````$`````
+M7<!%)`D`,`@*`P`L`"P```````$`````C*!<E`T`2`@.`P`L`"P```````$`
+M````NX!K"`@`8`@3`P`L`"P```````$`````TO!R=`P`;`@7`P`L`"P`````
+M``$````!%W`I:`L`C`P"`0`B`"(```````$````!+N!+R`H`&`T$`0`>`!X`
+M``````$````!1E!I%`X`)`T&`@`>`!X```````$````!7<"!L`D`L`\*`P`:
+M`!H```````$````!C*"J4`T`2`\.`P`:`!H```````$````!NX#(9`@`8`\3
+M`P`:`!H```````$````!TO#7/`P`;`\7`P`:`!H`````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M`````````````````````````````````````````&0````R"0XR"P<`````
+M``````````````3_````````````````````````````````````````````
+M``$````"`^@#A!L`@@```0$Z`3H```````$````"!]`'"!H$A`$!`0$"`*(`
+M``````$````"%7P0S!D$BP$"`@$"`*(```````$````"*O@;O!@$E@$$9`$"
+M`*(`````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M`````,@```!D`````P````````````````8``&P`V`&\`XP%5`JP````````
+M````````````````````````````````````````````````````````````
+M````````#?\``````````````````````````````````````````````0``
+M``,`^@#(`P`!`/<!`F0"9````````0````,!]`&0!P`!`?@$`<D!R0``````
+M`0````,#Z`.$`@`"`OP"`.0`Y````````0````,'T`<(!@`$`_X!`*``H```
+M`````0````,+N`J,`0`&!/\!`(P`C````````0`````7<!48"P",!0(!`#P`
+M/````````0`````C*!YX#P`2!0,!`#P`/````````0`````NX"<0"@"8!P0"
+M`#``,````````0````!&4#9,#@`D!P8"`#``,````````0````!=P$.4"0"P
+M"0H#`"P`+````````0````",H%G8#0!("0X#`"P`+````````0````"[@&L(
+M"`!@"1,#`"P`+````````0````#2\')T#`!L"1<#`"P`+```````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+M````````````````````````````````````````````````````````````
+MR````&0````,`````(`"0'P``````````(`!1^R``4A(@`%(I(`!21``````
+M````&0```#(```!+````4````%,```!5````5P```%@```!:```"````!R``
+M``(````'(````@````(````"`````@````(````"`````@````(`````````
+M``````````````0#"00$`R``-@-!`'0`:`!E`'(`;P!S`"``0P!O`&T`;0!U
+M`&X`:0!C`&$`=`!I`&\`;@!S`"``20!N`&,````````````````````.`T$`
+M4@`U`#4`,@`S```````````````````````````````(`S$`+@`P````````
+M`````````!(!``+_``!`\PP!``$``0(#`0``"@8``O\``$`!````"0(N``$!
+M`(#Z"00```3_````!P6!`D````<%`0)````'!8("0```!P4"`D````<%`P)`
+M```'!00"0```!P4%`D````<%!@)````'!0<"0```!P4(`D````<%"0)````'
+M!0H"0``````)!RX``0$`H/H)!```!/\````'!8$"0```!P4!`D````<%@@)`
+M```'!0("0```!P4#`D````<%!`)````'!04"0```!P4&`D````<%!P)````'
+M!0@"0```!P4)`D````<%"@)```````````````````````"JJJJJJJJJJN[N
+M[N[N[N[N_O______________?[_?[_?[_?Q^O]_O]_O]?@`````!`0`"8EH`
+M``8:@`````#____^_____@``````````@`&WO````````0```````8`"G,0`
+M````H+```````^@````(````````````````.YK*`````&2``:08@`'K*(`!
+MTSR``?P<@`'4*(`!V2B``>&$@`'"5(`!ZTB``=-<@`'\/(`!V4B``>&D````
+M`#P$@``DA````^`((3P%@``DI0``/!^``B?_1N``OR@B!!$``0``````OR@@
+M`"#X(0/@,"$#X`@A/!J``B=:1W@\'X`")_]'#`-?T"($$0`!``````-?T"``
+M(/@A`T#X"0``````P$`A`^`((3P)@`(E*4=D/!^``B?_1T`!/T@B!!$``0``
+M```!/T@@`"#X(0$H0"(\`H`")$)'7`!```@`````!!$``0`````#Z$`B`0#X
+M(0/@``@```````````"`0"$`H$@A/`N``B5K1W2-(@``K0(``"4I``0E"``$
+0%0O_^P`````#X``(````````
+`
+end
diff --git a/sys/contrib/pf/net/pf.c b/sys/contrib/pf/net/pf.c
index d69372b..71b7bda 100644
--- a/sys/contrib/pf/net/pf.c
+++ b/sys/contrib/pf/net/pf.c
@@ -6153,7 +6153,7 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp,
 	if (r->rt == PF_FASTROUTE) {
 		in_rtalloc(ro, 0);
 		if (ro->ro_rt == 0) {
-			V_ipstat.ips_noroute++;
+			IPSTAT_INC(ips_noroute);
 			goto bad;
 		}
 
@@ -6284,16 +6284,16 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp,
 		if ((ifp->if_capabilities & IFCAP_CSUM_IPv4) &&
 		    ifp->if_bridge == NULL) {
 			m0->m_pkthdr.csum_flags |= M_IPV4_CSUM_OUT;
-			V_ipstat.ips_outhwcsum++;
+			IPSTAT_INC(ips_outhwcsum);
 		} else {
 			ip->ip_sum = 0;
 			ip->ip_sum = in_cksum(m0, ip->ip_hl << 2);
 		}
 		/* Update relevant hardware checksum stats for TCP/UDP */
 		if (m0->m_pkthdr.csum_flags & M_TCPV4_CSUM_OUT)
-			V_tcpstat.tcps_outhwcsum++;
+			TCPSTAT_INC(tcpstat.tcps_outhwcsum);
 		else if (m0->m_pkthdr.csum_flags & M_UDPV4_CSUM_OUT)
-			V_udpstat.udps_outhwcsum++;
+			UDPSTAT_INC(udps_outhwcsum);
 		error = (*ifp->if_output)(ifp, m0, sintosa(dst), NULL);
 		goto done;
 	}
@@ -6303,7 +6303,7 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp,
 	 * Must be able to put at least 8 bytes per fragment.
 	 */
 	if (ip->ip_off & htons(IP_DF)) {
-		V_ipstat.ips_cantfrag++;
+		IPSTAT_INC(ips_cantfrag);
 		if (r->rt != PF_DUPTO) {
 #ifdef __FreeBSD__
 			/* icmp_error() expects host byte ordering */
@@ -6360,7 +6360,7 @@ pf_route(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp,
 	}
 
 	if (error == 0)
-		V_ipstat.ips_fragmented++;
+		IPSTAT_INC(ips_fragmented);
 
 done:
 	if (r->rt != PF_DUPTO)
@@ -6635,26 +6635,26 @@ pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p, sa_family_t a
 		case IPPROTO_TCP:
 		    {
 			INIT_VNET_INET(curvnet);
-			V_tcpstat.tcps_rcvbadsum++;
+			TCPSTAT_INC(tcps_rcvbadsum);
 			break;
 		    }
 		case IPPROTO_UDP:
 		    {
 			INIT_VNET_INET(curvnet);
-			V_udpstat.udps_badsum++;
+			UDPSTAT_INC(udps_badsum);
 			break;
 		    }
 		case IPPROTO_ICMP:
 		    {
 			INIT_VNET_INET(curvnet);
-			V_icmpstat.icps_checksum++;
+			ICMPSTAT_INC(icps_checksum);
 			break;
 		    }
 #ifdef INET6
 		case IPPROTO_ICMPV6:
 		    {
 			INIT_VNET_INET6(curvnet);
-			V_icmp6stat.icp6s_checksum++;
+			ICMP6STAT_INC(icp6s_checksum);
 			break;
 		    }
 #endif /* INET6 */
@@ -6741,17 +6741,17 @@ pf_check_proto_cksum(struct mbuf *m, int off, int len, u_int8_t p,
 		m->m_pkthdr.csum_flags |= flag_bad;
 		switch (p) {
 		case IPPROTO_TCP:
-			V_tcpstat.tcps_rcvbadsum++;
+			TCPSTAT_INC(tcps_rcvbadsum);
 			break;
 		case IPPROTO_UDP:
-			V_udpstat.udps_badsum++;
+			UDPSTAT_INC(udps_badsum);
 			break;
 		case IPPROTO_ICMP:
-			V_icmpstat.icps_checksum++;
+			ICMPSTAT_INC(icps_checksum);
 			break;
 #ifdef INET6
 		case IPPROTO_ICMPV6:
-			V_icmp6stat.icp6s_checksum++;
+			ICMP6STAT_INC(icp6s_checksum);
 			break;
 #endif /* INET6 */
 		}
diff --git a/sys/dev/acpi_support/acpi_asus.c b/sys/dev/acpi_support/acpi_asus.c
index 36cc4f9..28f9e7c 100644
--- a/sys/dev/acpi_support/acpi_asus.c
+++ b/sys/dev/acpi_support/acpi_asus.c
@@ -176,16 +176,39 @@ static struct acpi_asus_model acpi_asus_models[] = {
 		.disp_set	= "SDSP"
 	},
 	{
+		.name           = "A3E",
+		.mled_set       = "MLED",
+		.wled_set       = "WLED",
+		.lcd_get        = "\\_SB.PCI0.SBRG.EC0.RPIN(0x67)",
+		.lcd_set        = "\\_SB.PCI0.SBRG.EC0._Q10",
+		.brn_get        = "GPLV",
+		.brn_set        = "SPLV",
+		.disp_get       = "\\_SB.PCI0.P0P2.VGA.GETD",
+		.disp_set       = "SDSP"
+	},
+	{
+		.name           = "A3F",
+		.mled_set       = "MLED",
+		.wled_set       = "WLED",
+		.bled_set       = "BLED",
+		.lcd_get        = "\\_SB.PCI0.SBRG.EC0.RPIN(0x11)",
+		.lcd_set        = "\\_SB.PCI0.SBRG.EC0._Q10",
+		.brn_get        = "GPLV",
+		.brn_set        = "SPLV",
+		.disp_get       = "\\SSTE",
+		.disp_set       = "SDSP"
+	},
+	{
 		.name           = "A3N",
 		.mled_set       = "MLED",
 		.bled_set       = "BLED",
 		.wled_set       = "WLED",
-		.lcd_get        = NULL,
+		.lcd_get        = "\\BKLT",
 		.lcd_set        = "\\_SB.PCI0.SBRG.EC0._Q10",
+		.brn_get        = "GPLV",
 		.brn_set        = "SPLV",
-		.brn_get        = "SDSP",
-		.disp_set       = "SDSP",
-		.disp_get       = "\\_SB.PCI0.P0P3.VGA.GETD"
+		.disp_get       = "\\_SB.PCI0.P0P3.VGA.GETD",
+		.disp_set       = "SDSP"
 	},
 	{
 		.name		= "A4D",
@@ -577,7 +600,7 @@ acpi_asus_probe(device_t dev)
 			return (0);
 		}
 
-		/* if EeePC */
+		/* EeePC */
 		if (strncmp("ASUS010", rstr, 7) == 0) {
 			sc->model = &acpi_eeepc_models[0];
 			device_set_desc(dev, "ASUS EeePC");
@@ -627,6 +650,9 @@ good:
 		else if (strncmp(model->name, "A2x", 3) == 0 &&
 		    strncmp(Obj->String.Pointer, "A2", 2) == 0)
 			goto good;
+		else if (strncmp(model->name, "A3F", 3) == 0 &&
+		    strncmp(Obj->String.Pointer, "A6F", 3) == 0)
+			goto good;
 		else if (strncmp(model->name, "D1x", 3) == 0 &&
 		    strncmp(Obj->String.Pointer, "D1", 2) == 0)
 			goto good;
diff --git a/sys/dev/acpica/acpi.c b/sys/dev/acpica/acpi.c
index 8271917..50b84a5 100644
--- a/sys/dev/acpica/acpi.c
+++ b/sys/dev/acpica/acpi.c
@@ -46,7 +46,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/linker.h>
 #include <sys/power.h>
 #include <sys/sbuf.h>
+#ifdef SMP
+#include <sys/sched.h>
+#endif
 #include <sys/smp.h>
+#include <sys/timetc.h>
 
 #if defined(__i386__) || defined(__amd64__)
 #include <machine/pci_cfgreg.h>
@@ -250,6 +254,12 @@ TUNABLE_INT("debug.acpi.do_powerstate", &acpi_do_powerstate);
 SYSCTL_INT(_debug_acpi, OID_AUTO, do_powerstate, CTLFLAG_RW,
     &acpi_do_powerstate, 1, "Turn off devices when suspending.");
 
+/* Reset system clock while resuming.  XXX Remove once tested. */
+static int acpi_reset_clock = 1;
+TUNABLE_INT("debug.acpi.reset_clock", &acpi_reset_clock);
+SYSCTL_INT(_debug_acpi, OID_AUTO, reset_clock, CTLFLAG_RW,
+    &acpi_reset_clock, 1, "Reset system clock while resuming.");
+
 /* Allow users to override quirks. */
 TUNABLE_INT("debug.acpi.quirks", &acpi_quirks);
 
@@ -2274,6 +2284,7 @@ acpi_SetSleepState(struct acpi_softc *sc, int state)
     return (acpi_EnterSleepState(sc, state));
 }
 
+#if defined(__amd64__) || defined(__i386__)
 static void
 acpi_sleep_force(void *arg)
 {
@@ -2284,6 +2295,7 @@ acpi_sleep_force(void *arg)
     if (ACPI_FAILURE(acpi_EnterSleepState(sc, sc->acpi_next_sstate)))
 	printf("acpi: force sleep state S%d failed\n", sc->acpi_next_sstate);
 }
+#endif
 
 /*
  * Request that the system enter the given suspend state.  All /dev/apm
@@ -2294,7 +2306,9 @@ acpi_sleep_force(void *arg)
 int
 acpi_ReqSleepState(struct acpi_softc *sc, int state)
 {
+#if defined(__i386__)
     struct apm_clone_data *clone;
+#endif
 
     if (state < ACPI_STATE_S1 || state > ACPI_STATE_S5)
 	return (EINVAL);
@@ -2307,11 +2321,7 @@ acpi_ReqSleepState(struct acpi_softc *sc, int state)
 	    return (ENXIO);
     }
 
-#if !defined(__i386__)
-    /* This platform does not support acpi suspend/resume. */
-    return (EOPNOTSUPP);
-#endif
-
+#if defined(__amd64__) || defined(__i386__)
     /* If a suspend request is already in progress, just return. */
     ACPI_LOCK(acpi);
     if (sc->acpi_next_sstate != 0) {
@@ -2321,6 +2331,7 @@ acpi_ReqSleepState(struct acpi_softc *sc, int state)
 
     /* Record the pending state and notify all apm devices. */
     sc->acpi_next_sstate = state;
+#if defined(__i386__)
     STAILQ_FOREACH(clone, &sc->apm_cdevs, entries) {
 	clone->notify_status = APM_EV_NONE;
 	if ((clone->flags & ACPI_EVF_DEVD) == 0) {
@@ -2328,9 +2339,10 @@ acpi_ReqSleepState(struct acpi_softc *sc, int state)
 	    KNOTE_UNLOCKED(&clone->sel_read.si_note, 0);
 	}
     }
+#endif
 
     /* If devd(8) is not running, immediately enter the sleep state. */
-    if (devctl_process_running() == FALSE) {
+    if (!devctl_process_running()) {
 	ACPI_UNLOCK(acpi);
 	if (ACPI_SUCCESS(acpi_EnterSleepState(sc, sc->acpi_next_sstate))) {
 	    return (0);
@@ -2339,9 +2351,6 @@ acpi_ReqSleepState(struct acpi_softc *sc, int state)
 	}
     }
 
-    /* Now notify devd(8) also. */
-    acpi_UserNotify("Suspend", ACPI_ROOT_OBJECT, state);
-
     /*
      * Set a timeout to fire if userland doesn't ack the suspend request
      * in time.  This way we still eventually go to sleep if we were
@@ -2351,7 +2360,15 @@ acpi_ReqSleepState(struct acpi_softc *sc, int state)
      */
     callout_reset(&sc->susp_force_to, 10 * hz, acpi_sleep_force, sc);
     ACPI_UNLOCK(acpi);
+
+    /* Now notify devd(8) also. */
+    acpi_UserNotify("Suspend", ACPI_ROOT_OBJECT, state);
+
     return (0);
+#else
+    /* This platform does not support acpi suspend/resume. */
+    return (EOPNOTSUPP);
+#endif
 }
 
 /*
@@ -2364,14 +2381,10 @@ acpi_ReqSleepState(struct acpi_softc *sc, int state)
 int
 acpi_AckSleepState(struct apm_clone_data *clone, int error)
 {
+#if defined(__amd64__) || defined(__i386__)
     struct acpi_softc *sc;
     int ret, sleeping;
 
-#if !defined(__i386__)
-    /* This platform does not support acpi suspend/resume. */
-    return (EOPNOTSUPP);
-#endif
-
     /* If no pending sleep state, return an error. */
     ACPI_LOCK(acpi);
     sc = clone->acpi_sc;
@@ -2395,8 +2408,9 @@ acpi_AckSleepState(struct apm_clone_data *clone, int error)
      * all devices, seeing if they agree yet.  We only count devices that
      * are writable since read-only devices couldn't ack the request.
      */
-    clone->notify_status = APM_EV_ACKED;
     sleeping = TRUE;
+#if defined(__i386__)
+    clone->notify_status = APM_EV_ACKED;
     STAILQ_FOREACH(clone, &sc->apm_cdevs, entries) {
 	if ((clone->flags & ACPI_EVF_WRITE) != 0 &&
 	    clone->notify_status != APM_EV_ACKED) {
@@ -2404,6 +2418,7 @@ acpi_AckSleepState(struct apm_clone_data *clone, int error)
 	    break;
 	}
     }
+#endif
 
     /* If all devices have voted "yes", we will suspend now. */
     if (sleeping)
@@ -2414,15 +2429,34 @@ acpi_AckSleepState(struct apm_clone_data *clone, int error)
 	if (ACPI_FAILURE(acpi_EnterSleepState(sc, sc->acpi_next_sstate)))
 		ret = ENODEV;
     }
-
     return (ret);
+#else
+    /* This platform does not support acpi suspend/resume. */
+    return (EOPNOTSUPP);
+#endif
 }
 
 static void
 acpi_sleep_enable(void *arg)
 {
+    struct acpi_softc	*sc = (struct acpi_softc *)arg;
 
-    ((struct acpi_softc *)arg)->acpi_sleep_disabled = 0;
+    ACPI_LOCK(acpi);
+    sc->acpi_sleep_disabled = 0;
+    ACPI_UNLOCK(acpi);
+}
+
+static ACPI_STATUS
+acpi_sleep_disable(struct acpi_softc *sc)
+{
+    ACPI_STATUS		status;
+
+    ACPI_LOCK(acpi);
+    status = sc->acpi_sleep_disabled ? AE_ERROR : AE_OK;
+    sc->acpi_sleep_disabled = 1;
+    ACPI_UNLOCK(acpi);
+
+    return (status);
 }
 
 enum acpi_sleep_state {
@@ -2449,21 +2483,24 @@ acpi_EnterSleepState(struct acpi_softc *sc, int state)
     ACPI_FUNCTION_TRACE_U32((char *)(uintptr_t)__func__, state);
 
     /* Re-entry once we're suspending is not allowed. */
-    status = AE_OK;
-    ACPI_LOCK(acpi);
-    if (sc->acpi_sleep_disabled) {
-	ACPI_UNLOCK(acpi);
+    status = acpi_sleep_disable(sc);
+    if (ACPI_FAILURE(status)) {
 	printf("acpi: suspend request ignored (not ready yet)\n");
-	return (AE_ERROR);
+	return (status);
     }
-    sc->acpi_sleep_disabled = 1;
-    ACPI_UNLOCK(acpi);
+
+#ifdef SMP
+    thread_lock(curthread);
+    sched_bind(curthread, 0);
+    thread_unlock(curthread);
+#endif
 
     /*
      * Be sure to hold Giant across DEVICE_SUSPEND/RESUME since non-MPSAFE
      * drivers need this.
      */
     mtx_lock(&Giant);
+
     slp_state = ACPI_SS_NONE;
     switch (state) {
     case ACPI_STATE_S1:
@@ -2539,6 +2576,7 @@ acpi_EnterSleepState(struct acpi_softc *sc, int state)
 	 * shutdown handlers.
 	 */
 	shutdown_nice(RB_POWEROFF);
+	status = AE_OK;
 	break;
     case ACPI_STATE_S0:
     default:
@@ -2562,17 +2600,40 @@ acpi_EnterSleepState(struct acpi_softc *sc, int state)
     if (slp_state >= ACPI_SS_SLEPT)
 	acpi_enable_fixed_events(sc);
 
+    mtx_unlock(&Giant);
+
+#ifdef SMP
+    thread_lock(curthread);
+    sched_unbind(curthread);
+    thread_unlock(curthread);
+#endif
+
     /* Allow another sleep request after a while. */
     if (state != ACPI_STATE_S5)
 	timeout(acpi_sleep_enable, sc, hz * ACPI_MINIMUM_AWAKETIME);
 
     /* Run /etc/rc.resume after we are back. */
-    acpi_UserNotify("Resume", ACPI_ROOT_OBJECT, state);
+    if (devctl_process_running())
+	acpi_UserNotify("Resume", ACPI_ROOT_OBJECT, state);
 
-    mtx_unlock(&Giant);
     return_ACPI_STATUS (status);
 }
 
+void
+acpi_resync_clock(struct acpi_softc *sc)
+{
+
+    if (!acpi_reset_clock)
+	return;
+
+    /*
+     * Warm up timecounter again and reset system clock.
+     */
+    (void)timecounter->tc_get_timecount(timecounter);
+    (void)timecounter->tc_get_timecount(timecounter);
+    inittodr(time_second + sc->acpi_sleep_delay);
+}
+
 /* Initialize a device's wake GPE. */
 int
 acpi_wake_init(device_t dev, int type)
diff --git a/sys/dev/acpica/acpi_cpu.c b/sys/dev/acpica/acpi_cpu.c
index 2de059a..5044c4a 100644
--- a/sys/dev/acpica/acpi_cpu.c
+++ b/sys/dev/acpica/acpi_cpu.c
@@ -609,10 +609,6 @@ acpi_cpu_generic_cx_probe(struct acpi_cpu_softc *sc)
 	    sc->cpu_cx_count++;
 	}
     }
-
-    /* Update the largest cx_count seen so far */
-    if (sc->cpu_cx_count > cpu_cx_count)
-	cpu_cx_count = sc->cpu_cx_count;
 }
 
 /*
@@ -752,6 +748,8 @@ acpi_cpu_startup(void *arg)
 	for (i = 0; i < cpu_ndevices; i++) {
 	    sc = device_get_softc(cpu_devices[i]);
 	    acpi_cpu_generic_cx_probe(sc);
+	    if (sc->cpu_cx_count > cpu_cx_count)
+		    cpu_cx_count = sc->cpu_cx_count;
 	}
 
 	/*
diff --git a/sys/dev/acpica/acpi_ec.c b/sys/dev/acpica/acpi_ec.c
index bb75223..696542c 100644
--- a/sys/dev/acpica/acpi_ec.c
+++ b/sys/dev/acpica/acpi_ec.c
@@ -747,7 +747,7 @@ EcSpaceHandler(UINT32 Function, ACPI_PHYSICAL_ADDRESS Address, UINT32 width,
      * If booting, check if we need to run the query handler.  If so, we
      * we call it directly here since our thread taskq is not active yet.
      */
-    if (cold || rebooting) {
+    if (cold || rebooting || sc->ec_suspending) {
 	if ((EC_GET_CSR(sc) & EC_EVENT_SCI)) {
 	    CTR0(KTR_ACPI, "ec running gpe handler directly");
 	    EcGpeQueryHandler(sc);
diff --git a/sys/dev/acpica/acpivar.h b/sys/dev/acpica/acpivar.h
index a5ac2de..ce6a242 100644
--- a/sys/dev/acpica/acpivar.h
+++ b/sys/dev/acpica/acpivar.h
@@ -330,6 +330,7 @@ ACPI_STATUS	acpi_SetIntrModel(int model);
 int		acpi_ReqSleepState(struct acpi_softc *sc, int state);
 int		acpi_AckSleepState(struct apm_clone_data *clone, int error);
 ACPI_STATUS	acpi_SetSleepState(struct acpi_softc *sc, int state);
+void		acpi_resync_clock(struct acpi_softc *sc);
 int		acpi_wake_init(device_t dev, int type);
 int		acpi_wake_set_enable(device_t dev, int enable);
 int		acpi_parse_prw(ACPI_HANDLE h, struct acpi_prw_data *prw);
diff --git a/sys/dev/age/if_age.c b/sys/dev/age/if_age.c
index f8d8a2e..74afa2e 100644
--- a/sys/dev/age/if_age.c
+++ b/sys/dev/age/if_age.c
@@ -106,8 +106,6 @@ static int age_miibus_writereg(device_t, int, int, int);
 static void age_miibus_statchg(device_t);
 static void age_mediastatus(struct ifnet *, struct ifmediareq *);
 static int age_mediachange(struct ifnet *);
-static int age_read_vpd_word(struct age_softc *, uint32_t, uint32_t,
-    uint32_t *);
 static int age_probe(device_t);
 static void age_get_macaddr(struct age_softc *);
 static void age_phy_reset(struct age_softc *);
@@ -321,29 +319,6 @@ age_mediachange(struct ifnet *ifp)
 }
 
 static int
-age_read_vpd_word(struct age_softc *sc, uint32_t vpdc, uint32_t offset,
-    uint32_t *word)
-{
-	int i;
-
-	pci_write_config(sc->age_dev, vpdc + PCIR_VPD_ADDR, offset, 2);
-	for (i = AGE_TIMEOUT; i > 0; i--) {
-		DELAY(10);
-		if ((pci_read_config(sc->age_dev, vpdc + PCIR_VPD_ADDR, 2) &
-		    0x8000) == 0x8000)
-			break;
-	}
-	if (i == 0) {
-		device_printf(sc->age_dev, "VPD read timeout!\n");
-		*word = 0;
-		return (ETIMEDOUT);
-	}
-
-	*word = pci_read_config(sc->age_dev, vpdc + PCIR_VPD_DATA, 4);
-	return (0);
-}
-
-static int
 age_probe(device_t dev)
 {
 	struct age_dev *sp;
@@ -368,8 +343,8 @@ age_probe(device_t dev)
 static void
 age_get_macaddr(struct age_softc *sc)
 {
-	uint32_t ea[2], off, reg, word;
-	int vpd_error, match, vpdc;
+	uint32_t ea[2], reg;
+	int i, vpdc;
 
 	reg = CSR_READ_4(sc, AGE_SPI_CTRL);
 	if ((reg & SPI_VPD_ENB) != 0) {
@@ -378,123 +353,114 @@ age_get_macaddr(struct age_softc *sc)
 		CSR_WRITE_4(sc, AGE_SPI_CTRL, reg);
 	}
 
-	vpd_error = 0;
-	ea[0] = ea[1] = 0;
-	if ((vpd_error = pci_find_extcap(sc->age_dev, PCIY_VPD, &vpdc)) == 0) {
+	if (pci_find_extcap(sc->age_dev, PCIY_VPD, &vpdc) == 0) {
 		/*
-		 * PCI VPD capability exists, but it seems that it's
-		 * not in the standard form as stated in PCI VPD
-		 * specification such that driver could not use
-		 * pci_get_vpd_readonly(9) with keyword 'NA'.
-		 * Search VPD data starting at address 0x0100. The data
-		 * should be used as initializers to set AGE_PAR0,
-		 * AGE_PAR1 register including other PCI configuration
-		 * registers.
+		 * PCI VPD capability found, let TWSI reload EEPROM.
+		 * This will set ethernet address of controller.
 		 */
-		word = 0;
-		match = 0;
-		reg = 0;
-		for (off = AGE_VPD_REG_CONF_START; off < AGE_VPD_REG_CONF_END;
-		    off += sizeof(uint32_t)) {
-			vpd_error = age_read_vpd_word(sc, vpdc, off, &word);
-			if (vpd_error != 0)
-				break;
-			if (match != 0) {
-				switch (reg) {
-				case AGE_PAR0:
-					ea[0] = word;
-					break;
-				case AGE_PAR1:
-					ea[1] = word;
-					break;
-				default:
-					break;
-				}
-				match = 0;
-			} else if ((word & 0xFF) == AGE_VPD_REG_CONF_SIG) {
-				match = 1;
-				reg = word >> 16;
-			} else
+		CSR_WRITE_4(sc, AGE_TWSI_CTRL, CSR_READ_4(sc, AGE_TWSI_CTRL) |
+		    TWSI_CTRL_SW_LD_START);
+		for (i = 100; i > 0; i--) {
+			DELAY(1000);
+			reg = CSR_READ_4(sc, AGE_TWSI_CTRL);
+			if ((reg & TWSI_CTRL_SW_LD_START) == 0)
 				break;
 		}
-		if (off >= AGE_VPD_REG_CONF_END)
-			vpd_error = ENOENT;
-		if (vpd_error == 0) {
-			/*
-			 * Don't blindly trust ethernet address obtained
-			 * from VPD. Check whether ethernet address is
-			 * valid one. Otherwise fall-back to reading
-			 * PAR register.
-			 */
-			ea[1] &= 0xFFFF;
-			if ((ea[0] == 0 && ea[1] == 0) ||
-			    (ea[0] == 0xFFFFFFFF && ea[1] == 0xFFFF)) {
-				if (bootverbose)
-					device_printf(sc->age_dev,
-					    "invalid ethernet address "
-					    "returned from VPD.\n");
-				vpd_error = EINVAL;
-			}
-		}
-		if (vpd_error != 0 && (bootverbose))
-			device_printf(sc->age_dev, "VPD access failure!\n");
+		if (i == 0)
+			device_printf(sc->age_dev,
+			    "reloading EEPROM timeout!\n");
 	} else {
 		if (bootverbose)
 			device_printf(sc->age_dev,
 			    "PCI VPD capability not found!\n");
 	}
 
-	/*
-	 * It seems that L1 also provides a way to extract ethernet
-	 * address via SPI flash interface. Because SPI flash memory
-	 * device of different vendors vary in their instruction
-	 * codes for read ID instruction, it's very hard to get
-	 * instructions codes without detailed information for the
-	 * flash memory device used on ethernet controller. To simplify
-	 * code, just read AGE_PAR0/AGE_PAR1 register to get ethernet
-	 * address which is supposed to be set by hardware during
-	 * power on reset.
-	 */
-	if (vpd_error != 0) {
-		/*
-		 * VPD is mapped to SPI flash memory or BIOS set it.
-		 */
-		ea[0] = CSR_READ_4(sc, AGE_PAR0);
-		ea[1] = CSR_READ_4(sc, AGE_PAR1);
-	}
-
-	ea[1] &= 0xFFFF;
-	if ((ea[0] == 0 && ea[1]  == 0) ||
-	    (ea[0] == 0xFFFFFFFF && ea[1] == 0xFFFF)) {
-		device_printf(sc->age_dev,
-		    "generating fake ethernet address.\n");
-		ea[0] = arc4random();
-		/* Set OUI to ASUSTek COMPUTER INC. */
-		sc->age_eaddr[0] = 0x00;
-		sc->age_eaddr[1] = 0x1B;
-		sc->age_eaddr[2] = 0xFC;
-		sc->age_eaddr[3] = (ea[0] >> 16) & 0xFF;
-		sc->age_eaddr[4] = (ea[0] >> 8) & 0xFF;
-		sc->age_eaddr[5] = (ea[0] >> 0) & 0xFF;
-	} else {
-		sc->age_eaddr[0] = (ea[1] >> 8) & 0xFF;
-		sc->age_eaddr[1] = (ea[1] >> 0) & 0xFF;
-		sc->age_eaddr[2] = (ea[0] >> 24) & 0xFF;
-		sc->age_eaddr[3] = (ea[0] >> 16) & 0xFF;
-		sc->age_eaddr[4] = (ea[0] >> 8) & 0xFF;
-		sc->age_eaddr[5] = (ea[0] >> 0) & 0xFF;
-	}
+	ea[0] = CSR_READ_4(sc, AGE_PAR0);
+	ea[1] = CSR_READ_4(sc, AGE_PAR1);
+	sc->age_eaddr[0] = (ea[1] >> 8) & 0xFF;
+	sc->age_eaddr[1] = (ea[1] >> 0) & 0xFF;
+	sc->age_eaddr[2] = (ea[0] >> 24) & 0xFF;
+	sc->age_eaddr[3] = (ea[0] >> 16) & 0xFF;
+	sc->age_eaddr[4] = (ea[0] >> 8) & 0xFF;
+	sc->age_eaddr[5] = (ea[0] >> 0) & 0xFF;
 }
 
 static void
 age_phy_reset(struct age_softc *sc)
 {
+	uint16_t reg, pn;
+	int i, linkup;
 
 	/* Reset PHY. */
 	CSR_WRITE_4(sc, AGE_GPHY_CTRL, GPHY_CTRL_RST);
-	DELAY(1000);
+	DELAY(2000);
 	CSR_WRITE_4(sc, AGE_GPHY_CTRL, GPHY_CTRL_CLR);
-	DELAY(1000);
+	DELAY(2000);
+
+#define	ATPHY_DBG_ADDR		0x1D
+#define	ATPHY_DBG_DATA		0x1E
+#define	ATPHY_CDTC		0x16
+#define	PHY_CDTC_ENB		0x0001
+#define	PHY_CDTC_POFF		8
+#define	ATPHY_CDTS		0x1C
+#define	PHY_CDTS_STAT_OK	0x0000
+#define	PHY_CDTS_STAT_SHORT	0x0100
+#define	PHY_CDTS_STAT_OPEN	0x0200
+#define	PHY_CDTS_STAT_INVAL	0x0300
+#define	PHY_CDTS_STAT_MASK	0x0300
+
+	/* Check power saving mode. Magic from Linux. */
+	age_miibus_writereg(sc->age_dev, sc->age_phyaddr, MII_BMCR, BMCR_RESET);
+	for (linkup = 0, pn = 0; pn < 4; pn++) {
+		age_miibus_writereg(sc->age_dev, sc->age_phyaddr, ATPHY_CDTC,
+		    (pn << PHY_CDTC_POFF) | PHY_CDTC_ENB);
+		for (i = 200; i > 0; i--) {
+			DELAY(1000);
+			reg = age_miibus_readreg(sc->age_dev, sc->age_phyaddr,
+			    ATPHY_CDTC);
+			if ((reg & PHY_CDTC_ENB) == 0)
+				break;
+		}
+		DELAY(1000);
+		reg = age_miibus_readreg(sc->age_dev, sc->age_phyaddr,
+		    ATPHY_CDTS);
+		if ((reg & PHY_CDTS_STAT_MASK) != PHY_CDTS_STAT_OPEN) {
+			linkup++;
+			break;
+		}
+	}
+	age_miibus_writereg(sc->age_dev, sc->age_phyaddr, MII_BMCR,
+	    BMCR_RESET | BMCR_AUTOEN | BMCR_STARTNEG);
+	if (linkup == 0) {
+		age_miibus_writereg(sc->age_dev, sc->age_phyaddr,
+		    ATPHY_DBG_ADDR, 0);
+		age_miibus_writereg(sc->age_dev, sc->age_phyaddr,
+		    ATPHY_DBG_DATA, 0x124E);
+		age_miibus_writereg(sc->age_dev, sc->age_phyaddr,
+		    ATPHY_DBG_ADDR, 1);
+		reg = age_miibus_readreg(sc->age_dev, sc->age_phyaddr,
+		    ATPHY_DBG_DATA);
+		age_miibus_writereg(sc->age_dev, sc->age_phyaddr,
+		    ATPHY_DBG_DATA, reg | 0x03);
+		/* XXX */
+		DELAY(1500 * 1000);
+		age_miibus_writereg(sc->age_dev, sc->age_phyaddr,
+		    ATPHY_DBG_ADDR, 0);
+		age_miibus_writereg(sc->age_dev, sc->age_phyaddr,
+		    ATPHY_DBG_DATA, 0x024E);
+    }
+
+#undef	ATPHY_DBG_ADDR
+#undef	ATPHY_DBG_DATA
+#undef	ATPHY_CDTC
+#undef	PHY_CDTC_ENB
+#undef	PHY_CDTC_POFF
+#undef	ATPHY_CDTS
+#undef	PHY_CDTS_STAT_OK
+#undef	PHY_CDTS_STAT_SHORT
+#undef	PHY_CDTS_STAT_OPEN
+#undef	PHY_CDTS_STAT_INVAL
+#undef	PHY_CDTS_STAT_MASK
 }
 
 static int
@@ -539,7 +505,8 @@ age_attach(device_t dev)
 	sc->age_chip_rev = CSR_READ_4(sc, AGE_MASTER_CFG) >>
 	    MASTER_CHIP_REV_SHIFT;
 	if (bootverbose) {
-		device_printf(dev, "PCI device revision : 0x%04x\n", sc->age_rev);
+		device_printf(dev, "PCI device revision : 0x%04x\n",
+		    sc->age_rev);
 		device_printf(dev, "Chip id/revision : 0x%04x\n",
 		    sc->age_chip_rev);
 	}
@@ -1369,7 +1336,7 @@ age_setwol(struct age_softc *sc)
 
 	AGE_LOCK_ASSERT(sc);
 
-	if (pci_find_extcap(sc->age_dev, PCIY_PMG, &pmc) == 0) {
+	if (pci_find_extcap(sc->age_dev, PCIY_PMG, &pmc) != 0) {
 		CSR_WRITE_4(sc, AGE_WOL_CFG, 0);
 		/*
 		 * No PME capability, PHY power down.
@@ -1524,6 +1491,9 @@ age_resume(device_t dev)
 		cmd &= ~0x0400;
 		pci_write_config(sc->age_dev, PCIR_COMMAND, cmd, 2);
 	}
+	AGE_UNLOCK(sc);
+	age_phy_reset(sc);
+	AGE_LOCK(sc);
 	ifp = sc->age_ifp;
 	if ((ifp->if_flags & IFF_UP) != 0)
 		age_init_locked(sc);
@@ -2544,14 +2514,8 @@ age_reset(struct age_softc *sc)
 	int i;
 
 	CSR_WRITE_4(sc, AGE_MASTER_CFG, MASTER_RESET);
-	for (i = AGE_RESET_TIMEOUT; i > 0; i--) {
-		DELAY(1);
-		if ((CSR_READ_4(sc, AGE_MASTER_CFG) & MASTER_RESET) == 0)
-			break;
-	}
-	if (i == 0)
-		device_printf(sc->age_dev, "master reset timeout!\n");
-
+	CSR_READ_4(sc, AGE_MASTER_CFG);
+	DELAY(1000);
 	for (i = AGE_RESET_TIMEOUT; i > 0; i--) {
 		if ((reg = CSR_READ_4(sc, AGE_IDLE_STATUS)) == 0)
 			break;
diff --git a/sys/dev/age/if_agereg.h b/sys/dev/age/if_agereg.h
index 091eff1..284e5a4 100644
--- a/sys/dev/age/if_agereg.h
+++ b/sys/dev/age/if_agereg.h
@@ -91,6 +91,9 @@
 #define	AGE_SPI_OP_READ			0x217	/* 8bits */
 
 #define	AGE_TWSI_CTRL			0x218
+#define	TWSI_CTRL_SW_LD_START		0x00000800
+#define	TWSI_CTRL_HW_LD_START		0x00001000
+#define	TWSI_CTRL_LD_EXIST		0x00400000
 
 #define AGE_DEV_MISC_CTRL		0x21C
 
diff --git a/sys/dev/agp/agp.c b/sys/dev/agp/agp.c
index af34d14..560d3d8 100644
--- a/sys/dev/agp/agp.c
+++ b/sys/dev/agp/agp.c
@@ -532,7 +532,7 @@ agp_generic_bind_memory(device_t dev, struct agp_memory *mem,
 	int error;
 
 	/* Do some sanity checks first. */
-	if (offset < 0 || (offset & (AGP_PAGE_SIZE - 1)) != 0 ||
+	if ((offset & (AGP_PAGE_SIZE - 1)) != 0 ||
 	    offset + mem->am_size > AGP_GET_APERTURE(dev)) {
 		device_printf(dev, "binding memory at bad offset %#x\n",
 		    (int)offset);
diff --git a/sys/dev/agp/agp_amd64.c b/sys/dev/agp/agp_amd64.c
index c68847a..e2bfa67 100644
--- a/sys/dev/agp/agp_amd64.c
+++ b/sys/dev/agp/agp_amd64.c
@@ -337,7 +337,7 @@ agp_amd64_bind_page(device_t dev, vm_offset_t offset, vm_offset_t physical)
 {
 	struct agp_amd64_softc *sc = device_get_softc(dev);
 
-	if (offset < 0 || offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
+	if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
 		return (EINVAL);
 
 	sc->gatt->ag_virtual[offset >> AGP_PAGE_SHIFT] =
@@ -351,7 +351,7 @@ agp_amd64_unbind_page(device_t dev, vm_offset_t offset)
 {
 	struct agp_amd64_softc *sc = device_get_softc(dev);
 
-	if (offset < 0 || offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
+	if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
 		return (EINVAL);
 
 	sc->gatt->ag_virtual[offset >> AGP_PAGE_SHIFT] = 0;
diff --git a/sys/dev/agp/agp_i810.c b/sys/dev/agp/agp_i810.c
index 48cf171..891bfff 100644
--- a/sys/dev/agp/agp_i810.c
+++ b/sys/dev/agp/agp_i810.c
@@ -840,7 +840,7 @@ agp_i810_bind_page(device_t dev, vm_offset_t offset, vm_offset_t physical)
 {
 	struct agp_i810_softc *sc = device_get_softc(dev);
 
-	if (offset < 0 || offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT)) {
+	if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT)) {
 		device_printf(dev, "failed: offset is 0x%08jx, shift is %d, entries is %d\n", (intmax_t)offset, AGP_PAGE_SHIFT, sc->gatt->ag_entries);
 		return EINVAL;
 	}
@@ -862,7 +862,7 @@ agp_i810_unbind_page(device_t dev, vm_offset_t offset)
 {
 	struct agp_i810_softc *sc = device_get_softc(dev);
 
-	if (offset < 0 || offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
+	if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
 		return EINVAL;
 
 	if ( sc->chiptype != CHIP_I810 ) {
@@ -1016,7 +1016,7 @@ agp_i810_bind_memory(device_t dev, struct agp_memory *mem,
 	vm_offset_t i;
 
 	/* Do some sanity checks first. */
-	if (offset < 0 || (offset & (AGP_PAGE_SIZE - 1)) != 0 ||
+	if ((offset & (AGP_PAGE_SIZE - 1)) != 0 ||
 	    offset + mem->am_size > AGP_GET_APERTURE(dev)) {
 		device_printf(dev, "binding memory at bad offset %#x\n",
 		    (int)offset);
diff --git a/sys/dev/agp/agp_intel.c b/sys/dev/agp/agp_intel.c
index 3463d82..ac10c8e 100644
--- a/sys/dev/agp/agp_intel.c
+++ b/sys/dev/agp/agp_intel.c
@@ -371,7 +371,7 @@ agp_intel_bind_page(device_t dev, vm_offset_t offset, vm_offset_t physical)
 
 	sc = device_get_softc(dev);
 
-	if (offset < 0 || offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
+	if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
 		return (EINVAL);
 
 	sc->gatt->ag_virtual[offset >> AGP_PAGE_SHIFT] = physical | 0x17;
@@ -385,7 +385,7 @@ agp_intel_unbind_page(device_t dev, vm_offset_t offset)
 
 	sc = device_get_softc(dev);
 
-	if (offset < 0 || offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
+	if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
 		return (EINVAL);
 
 	sc->gatt->ag_virtual[offset >> AGP_PAGE_SHIFT] = 0;
diff --git a/sys/dev/agp/agp_via.c b/sys/dev/agp/agp_via.c
index 2873692..76ce382 100644
--- a/sys/dev/agp/agp_via.c
+++ b/sys/dev/agp/agp_via.c
@@ -362,7 +362,7 @@ agp_via_bind_page(device_t dev, vm_offset_t offset, vm_offset_t physical)
 {
 	struct agp_via_softc *sc = device_get_softc(dev);
 
-	if (offset < 0 || offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
+	if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
 		return EINVAL;
 
 	sc->gatt->ag_virtual[offset >> AGP_PAGE_SHIFT] = physical;
@@ -374,7 +374,7 @@ agp_via_unbind_page(device_t dev, vm_offset_t offset)
 {
 	struct agp_via_softc *sc = device_get_softc(dev);
 
-	if (offset < 0 || offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
+	if (offset >= (sc->gatt->ag_entries << AGP_PAGE_SHIFT))
 		return EINVAL;
 
 	sc->gatt->ag_virtual[offset >> AGP_PAGE_SHIFT] = 0;
diff --git a/sys/dev/k8temp/k8temp.c b/sys/dev/amdtemp/amdtemp.c
index 35cca98..fdf0875 100644
--- a/sys/dev/k8temp/k8temp.c
+++ b/sys/dev/amdtemp/amdtemp.c
@@ -1,5 +1,6 @@
 /*-
- * Copyright (c) 2008 Rui Paulo <rpaulo@FreeBSD.org>
+ * Copyright (c) 2008, 2009 Rui Paulo <rpaulo@FreeBSD.org>
+ * Copyright (c) 2009 Norikatsu Shigemura <nork@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -25,8 +26,8 @@
  */
 
 /*
- * Driver for the AMD K8 thermal sensors. Based on a Linux driver by the
- * same name.
+ * Driver for the AMD K8/K10/K11 thermal sensors. Initially based on the
+ * k8temp Linux driver.
  */
 
 #include <sys/cdefs.h>
@@ -48,77 +49,92 @@ __FBSDID("$FreeBSD$");
 #include <dev/pci/pcireg.h>
 #include <dev/pci/pcivar.h>
 
-struct k8temp_softc {
+typedef enum {
+	SENSOR0_CORE0,
+	SENSOR0_CORE1,
+	SENSOR1_CORE0,
+	SENSOR1_CORE1,
+	CORE0,
+	CORE1
+} amdsensor_t;
+
+struct amdtemp_softc {
 	device_t	sc_dev;
 	int		sc_temps[4];
 	int		sc_ntemps;
 	struct sysctl_oid *sc_oid;
 	struct sysctl_oid *sc_sysctl_cpu[2];
 	struct intr_config_hook sc_ich;
+	int32_t (*sc_gettemp)(device_t, amdsensor_t);
 };
 
 #define VENDORID_AMD		0x1022
-#define DEVICEID_AMD_MISC	0x1103
-
-static struct k8temp_product {
-	uint16_t	k8temp_vendorid;
-	uint16_t	k8temp_deviceid;
-} k8temp_products[] = {
-	{ VENDORID_AMD,	DEVICEID_AMD_MISC },
+#define DEVICEID_AMD_MISC0F	0x1103
+#define DEVICEID_AMD_MISC10	0x1203
+#define DEVICEID_AMD_MISC11	0x1303
+
+static struct amdtemp_product {
+	uint16_t	amdtemp_vendorid;
+	uint16_t	amdtemp_deviceid;
+} amdtemp_products[] = {
+	{ VENDORID_AMD,	DEVICEID_AMD_MISC0F },
+	{ VENDORID_AMD,	DEVICEID_AMD_MISC10 },
+	{ VENDORID_AMD,	DEVICEID_AMD_MISC11 },
 	{ 0, 0 }
 };
 
 /*
- * Register control
+ * Register control (K8 family)
  */
-#define	K8TEMP_REG		0xe4
-#define	K8TEMP_REG_SELSENSOR	0x40
-#define	K8TEMP_REG_SELCORE	0x04
+#define	AMDTEMP_REG0F		0xe4
+#define	AMDTEMP_REG_SELSENSOR	0x40
+#define	AMDTEMP_REG_SELCORE	0x04
 
-#define K8TEMP_MINTEMP		49	/* -49 C is the mininum temperature */
+/*
+ * Register control (K10 & K11) family
+ */
+#define	AMDTEMP_REG		0xa4
 
-typedef enum {
-	SENSOR0_CORE0,
-	SENSOR0_CORE1,
-	SENSOR1_CORE0,
-	SENSOR1_CORE1,
-	CORE0,
-	CORE1
-} k8sensor_t;
+#define	TZ_ZEROC		2732
+
+					/* -49 C is the mininum temperature */
+#define	AMDTEMP_OFFSET0F	(TZ_ZEROC-490)
+#define	AMDTEMP_OFFSET		(TZ_ZEROC)
 
 /*
  * Device methods.
  */
-static void 	k8temp_identify(driver_t *driver, device_t parent);
-static int	k8temp_probe(device_t dev);
-static int	k8temp_attach(device_t dev);
-static void	k8temp_intrhook(void *arg);
-static int	k8temp_detach(device_t dev);
-static int 	k8temp_match(device_t dev);
-static int32_t	k8temp_gettemp(device_t dev, k8sensor_t sensor);
-static int	k8temp_sysctl(SYSCTL_HANDLER_ARGS);
-
-static device_method_t k8temp_methods[] = {
+static void 	amdtemp_identify(driver_t *driver, device_t parent);
+static int	amdtemp_probe(device_t dev);
+static int	amdtemp_attach(device_t dev);
+static void	amdtemp_intrhook(void *arg);
+static int	amdtemp_detach(device_t dev);
+static int 	amdtemp_match(device_t dev);
+static int32_t	amdtemp_gettemp0f(device_t dev, amdsensor_t sensor);
+static int32_t	amdtemp_gettemp(device_t dev, amdsensor_t sensor);
+static int	amdtemp_sysctl(SYSCTL_HANDLER_ARGS);
+
+static device_method_t amdtemp_methods[] = {
 	/* Device interface */
-	DEVMETHOD(device_identify,	k8temp_identify),
-	DEVMETHOD(device_probe,		k8temp_probe),
-	DEVMETHOD(device_attach,	k8temp_attach),
-	DEVMETHOD(device_detach,	k8temp_detach),
+	DEVMETHOD(device_identify,	amdtemp_identify),
+	DEVMETHOD(device_probe,		amdtemp_probe),
+	DEVMETHOD(device_attach,	amdtemp_attach),
+	DEVMETHOD(device_detach,	amdtemp_detach),
 
 	{0, 0}
 };
 
-static driver_t k8temp_driver = {
-	"k8temp",
-	k8temp_methods,
-	sizeof(struct k8temp_softc),
+static driver_t amdtemp_driver = {
+	"amdtemp",
+	amdtemp_methods,
+	sizeof(struct amdtemp_softc),
 };
 
-static devclass_t k8temp_devclass;
-DRIVER_MODULE(k8temp, hostb, k8temp_driver, k8temp_devclass, NULL, NULL);
+static devclass_t amdtemp_devclass;
+DRIVER_MODULE(amdtemp, hostb, amdtemp_driver, amdtemp_devclass, NULL, NULL);
 
 static int
-k8temp_match(device_t dev)
+amdtemp_match(device_t dev)
 {
 	int i;
 	uint16_t vendor, devid;
@@ -126,9 +142,9 @@ k8temp_match(device_t dev)
         vendor = pci_get_vendor(dev);
 	devid = pci_get_device(dev);
 
-	for (i = 0; k8temp_products[i].k8temp_vendorid != 0; i++) {
-		if (vendor == k8temp_products[i].k8temp_vendorid &&
-		    devid == k8temp_products[i].k8temp_deviceid)
+	for (i = 0; amdtemp_products[i].amdtemp_vendorid != 0; i++) {
+		if (vendor == amdtemp_products[i].amdtemp_vendorid &&
+		    devid == amdtemp_products[i].amdtemp_deviceid)
 			return (1);
 	}
 
@@ -136,28 +152,28 @@ k8temp_match(device_t dev)
 }
 
 static void
-k8temp_identify(driver_t *driver, device_t parent)
+amdtemp_identify(driver_t *driver, device_t parent)
 {
 	device_t child;
 
 	/* Make sure we're not being doubly invoked. */
-	if (device_find_child(parent, "k8temp", -1) != NULL)
+	if (device_find_child(parent, "amdtemp", -1) != NULL)
 		return;
 	
-	if (k8temp_match(parent)) {
-		child = device_add_child(parent, "k8temp", -1);
+	if (amdtemp_match(parent)) {
+		child = device_add_child(parent, "amdtemp", -1);
 		if (child == NULL)
-			device_printf(parent, "add k8temp child failed\n");
+			device_printf(parent, "add amdtemp child failed\n");
 	}
     
 }
 
 static int
-k8temp_probe(device_t dev)
+amdtemp_probe(device_t dev)
 {
 	uint32_t regs[4];
 	
-	if (resource_disabled("k8temp", 0))
+	if (resource_disabled("amdtemp", 0))
 		return (ENXIO);
 
 	do_cpuid(1, regs);
@@ -173,9 +189,9 @@ k8temp_probe(device_t dev)
 }
 
 static int
-k8temp_attach(device_t dev)
+amdtemp_attach(device_t dev)
 {
-	struct k8temp_softc *sc = device_get_softc(dev);
+	struct amdtemp_softc *sc = device_get_softc(dev);
 	struct sysctl_ctx_list *sysctlctx;
 	struct sysctl_oid *sysctlnode;
 
@@ -185,7 +201,7 @@ k8temp_attach(device_t dev)
 	 * needed because the cpu driver may be loaded late on boot, after
 	 * us.
 	 */
-	sc->sc_ich.ich_func = k8temp_intrhook;
+	sc->sc_ich.ich_func = amdtemp_intrhook;
 	sc->sc_ich.ich_arg = dev;
 	if (config_intrhook_establish(&sc->sc_ich) != 0) {
 		device_printf(dev, "config_intrhook_establish "
@@ -193,8 +209,15 @@ k8temp_attach(device_t dev)
 		return (ENXIO);
 	}
 	
+	if (pci_get_device(dev) == DEVICEID_AMD_MISC0F)
+		sc->sc_gettemp = amdtemp_gettemp0f;
+	else {
+		sc->sc_gettemp = amdtemp_gettemp;
+		return (0);
+	}
+
 	/*
-	 * dev.k8temp.N tree.
+	 * dev.amdtemp.N tree.
 	 */
 	sysctlctx = device_get_sysctl_ctx(dev);
 	sysctlnode = SYSCTL_ADD_NODE(sysctlctx,
@@ -204,13 +227,13 @@ k8temp_attach(device_t dev)
 	SYSCTL_ADD_PROC(sysctlctx,
 	    SYSCTL_CHILDREN(sysctlnode),
 	    OID_AUTO, "core0", CTLTYPE_INT | CTLFLAG_RD,
-	    dev, SENSOR0_CORE0, k8temp_sysctl, "I",
+	    dev, SENSOR0_CORE0, amdtemp_sysctl, "IK",
 	    "Sensor 0 / Core 0 temperature");
 	
 	SYSCTL_ADD_PROC(sysctlctx,
 	    SYSCTL_CHILDREN(sysctlnode),
 	    OID_AUTO, "core1", CTLTYPE_INT | CTLFLAG_RD,
-	    dev, SENSOR0_CORE1, k8temp_sysctl, "I",
+	    dev, SENSOR0_CORE1, amdtemp_sysctl, "IK",
 	    "Sensor 0 / Core 1 temperature");
 	
 	sysctlnode = SYSCTL_ADD_NODE(sysctlctx,
@@ -220,25 +243,25 @@ k8temp_attach(device_t dev)
 	SYSCTL_ADD_PROC(sysctlctx,
 	    SYSCTL_CHILDREN(sysctlnode),
 	    OID_AUTO, "core0", CTLTYPE_INT | CTLFLAG_RD,
-	    dev, SENSOR1_CORE0, k8temp_sysctl, "I",
+	    dev, SENSOR1_CORE0, amdtemp_sysctl, "IK",
 	    "Sensor 1 / Core 0 temperature");
 	
 	SYSCTL_ADD_PROC(sysctlctx,
 	    SYSCTL_CHILDREN(sysctlnode),
 	    OID_AUTO, "core1", CTLTYPE_INT | CTLFLAG_RD,
-	    dev, SENSOR1_CORE1, k8temp_sysctl, "I",
+	    dev, SENSOR1_CORE1, amdtemp_sysctl, "IK",
 	    "Sensor 1 / Core 1 temperature");
 
 	return (0);
 }
 
 void
-k8temp_intrhook(void *arg)
+amdtemp_intrhook(void *arg)
 {
 	int i;
 	device_t nexus, acpi, cpu;
 	device_t dev = (device_t) arg;
-	struct k8temp_softc *sc;
+	struct amdtemp_softc *sc;
 	struct sysctl_ctx_list *sysctlctx;
 
 	sc = device_get_softc(dev);
@@ -258,7 +281,7 @@ k8temp_intrhook(void *arg)
 			sc->sc_sysctl_cpu[i] = SYSCTL_ADD_PROC(sysctlctx,
 			    SYSCTL_CHILDREN(device_get_sysctl_tree(cpu)),
 			    OID_AUTO, "temperature", CTLTYPE_INT | CTLFLAG_RD,
-			    dev, CORE0, k8temp_sysctl, "I",
+			    dev, CORE0, amdtemp_sysctl, "IK",
 			    "Max of sensor 0 / 1");
 		}
 	}
@@ -266,41 +289,42 @@ k8temp_intrhook(void *arg)
 }
 
 int
-k8temp_detach(device_t dev)
+amdtemp_detach(device_t dev)
 {
 	int i;
-	struct k8temp_softc *sc = device_get_softc(dev);
+	struct amdtemp_softc *sc = device_get_softc(dev);
 	
 	for (i = 0; i < 2; i++) {
 		if (sc->sc_sysctl_cpu[i])
 			sysctl_remove_oid(sc->sc_sysctl_cpu[i], 1, 0);
 	}
 
-	/* NewBus removes the dev.k8temp.N tree by itself. */
+	/* NewBus removes the dev.amdtemp.N tree by itself. */
 	
 	return (0);
 }
 
 static int
-k8temp_sysctl(SYSCTL_HANDLER_ARGS)
+amdtemp_sysctl(SYSCTL_HANDLER_ARGS)
 {
 	device_t dev = (device_t) arg1;
+	struct amdtemp_softc *sc = device_get_softc(dev);
 	int error;
 	int32_t temp, auxtemp[2];
 
 	switch (arg2) {
 	case CORE0:
-		auxtemp[0] = k8temp_gettemp(dev, SENSOR0_CORE0);
-		auxtemp[1] = k8temp_gettemp(dev, SENSOR1_CORE0);
+		auxtemp[0] = sc->sc_gettemp(dev, SENSOR0_CORE0);
+		auxtemp[1] = sc->sc_gettemp(dev, SENSOR1_CORE0);
 		temp = imax(auxtemp[0], auxtemp[1]);
 		break;
 	case CORE1:
-		auxtemp[0] = k8temp_gettemp(dev, SENSOR0_CORE1);
-		auxtemp[1] = k8temp_gettemp(dev, SENSOR1_CORE1);
+		auxtemp[0] = sc->sc_gettemp(dev, SENSOR0_CORE1);
+		auxtemp[1] = sc->sc_gettemp(dev, SENSOR1_CORE1);
 		temp = imax(auxtemp[0], auxtemp[1]);
 		break;
 	default:
-		temp = k8temp_gettemp(dev, arg2);
+		temp = sc->sc_gettemp(dev, arg2);
 		break;
 	}
 	error = sysctl_handle_int(oidp, &temp, 0, req);
@@ -309,34 +333,45 @@ k8temp_sysctl(SYSCTL_HANDLER_ARGS)
 }
 
 static int32_t
-k8temp_gettemp(device_t dev, k8sensor_t sensor)
+amdtemp_gettemp0f(device_t dev, amdsensor_t sensor)
 {
 	uint8_t cfg;
 	uint32_t temp;
 	
-	cfg = pci_read_config(dev, K8TEMP_REG, 1);
+	cfg = pci_read_config(dev, AMDTEMP_REG0F, 1);
 	switch (sensor) {
 	case SENSOR0_CORE0:
-		cfg &= ~(K8TEMP_REG_SELSENSOR | K8TEMP_REG_SELCORE);
+		cfg &= ~(AMDTEMP_REG_SELSENSOR | AMDTEMP_REG_SELCORE);
 		break;
 	case SENSOR0_CORE1:
-		cfg &= ~K8TEMP_REG_SELSENSOR;
-		cfg |= K8TEMP_REG_SELCORE;
+		cfg &= ~AMDTEMP_REG_SELSENSOR;
+		cfg |= AMDTEMP_REG_SELCORE;
 		break;
 	case SENSOR1_CORE0:
-		cfg &= ~K8TEMP_REG_SELCORE;
-		cfg |= K8TEMP_REG_SELSENSOR;
+		cfg &= ~AMDTEMP_REG_SELCORE;
+		cfg |= AMDTEMP_REG_SELSENSOR;
 		break;
 	case SENSOR1_CORE1:
-		cfg |= (K8TEMP_REG_SELSENSOR | K8TEMP_REG_SELCORE);
+		cfg |= (AMDTEMP_REG_SELSENSOR | AMDTEMP_REG_SELCORE);
 		break;
 	default:
 		cfg = 0;
 		break;
 	}
-	pci_write_config(dev, K8TEMP_REG, cfg, 1);
-	temp = pci_read_config(dev, K8TEMP_REG, 4);
-	temp = ((temp >> 16) & 0xff) - K8TEMP_MINTEMP;
+	pci_write_config(dev, AMDTEMP_REG0F, cfg, 1);
+	temp = pci_read_config(dev, AMDTEMP_REG0F, 4);
+	temp = ((temp >> 16) & 0xff) * 10 + AMDTEMP_OFFSET0F;
 	
 	return (temp);
 }
+
+static int32_t
+amdtemp_gettemp(device_t dev, amdsensor_t sensor)
+{
+	uint32_t temp;
+
+	temp = pci_read_config(dev, AMDTEMP_REG, 4);
+	temp = ((temp >> 21) & 0x3ff) * 10 / 8 + AMDTEMP_OFFSET;
+
+	return (temp);
+}
diff --git a/sys/dev/ata/ata-pci.c b/sys/dev/ata/ata-pci.c
index 97261d2..5fe26a8 100644
--- a/sys/dev/ata/ata-pci.c
+++ b/sys/dev/ata/ata-pci.c
@@ -584,22 +584,35 @@ ata_pcichannel_detach(device_t dev)
 static int
 ata_pcichannel_suspend(device_t dev)
 {
+    struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev));
     struct ata_channel *ch = device_get_softc(dev);
+    int error;
 
     if (!ch->attached)
 	return (0);
 
-    return ata_suspend(dev);
+    if ((error = ata_suspend(dev)))
+	return (error);
+
+    if (ctlr->ch_suspend != NULL && (error = ctlr->ch_suspend(dev)))
+	return (error);
+
+    return (0);
 }
 
 static int
 ata_pcichannel_resume(device_t dev)
 {
+    struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev));
     struct ata_channel *ch = device_get_softc(dev);
+    int error;
 
     if (!ch->attached)
 	return (0);
 
+    if (ctlr->ch_resume != NULL && (error = ctlr->ch_resume(dev)))
+	return (error);
+
     return ata_resume(dev);
 }
 
diff --git a/sys/dev/ata/ata-pci.h b/sys/dev/ata/ata-pci.h
index 27a341e..9e31bc7 100644
--- a/sys/dev/ata/ata-pci.h
+++ b/sys/dev/ata/ata-pci.h
@@ -57,6 +57,8 @@ struct ata_pci_controller {
     int                 (*resume)(device_t);
     int                 (*ch_attach)(device_t);
     int                 (*ch_detach)(device_t);
+    int                 (*ch_suspend)(device_t);
+    int                 (*ch_resume)(device_t);
     int                 (*locking)(device_t, int);
     void                (*reset)(device_t);
     void                (*setmode)(device_t, int);
@@ -443,7 +445,9 @@ int ata_mode2idx(int mode);
 
 /* global prototypes ata-sata.c */
 void ata_sata_phy_check_events(device_t dev);
-int ata_sata_phy_reset(device_t dev);
+int ata_sata_scr_read(struct ata_channel *ch, int port, int reg, uint32_t *val);
+int ata_sata_scr_write(struct ata_channel *ch, int port, int reg, uint32_t val);
+int ata_sata_phy_reset(device_t dev, int port, int quick);
 void ata_sata_setmode(device_t dev, int mode);
 int ata_request2fis_h2d(struct ata_request *request, u_int8_t *fis);
 void ata_pm_identify(device_t dev);
@@ -452,6 +456,8 @@ void ata_pm_identify(device_t dev);
 int ata_ahci_chipinit(device_t);
 int ata_ahci_ch_attach(device_t dev);
 int ata_ahci_ch_detach(device_t dev);
+int ata_ahci_ch_suspend(device_t dev);
+int ata_ahci_ch_resume(device_t dev);
 void ata_ahci_reset(device_t dev);
 int ata_marvell_edma_chipinit(device_t);
 int ata_sii_chipinit(device_t);
diff --git a/sys/dev/ata/ata-queue.c b/sys/dev/ata/ata-queue.c
index 707dfe9..4a4bb19 100644
--- a/sys/dev/ata/ata-queue.c
+++ b/sys/dev/ata/ata-queue.c
@@ -444,8 +444,7 @@ ata_completed(void *context, int dummy)
 	}
 
 	if (!request->result &&
-	     (request->u.atapi.sense.key & ATA_SENSE_KEY_MASK ?
-	     request->u.atapi.sense.key & ATA_SENSE_KEY_MASK : 
+	     (request->u.atapi.sense.key & ATA_SENSE_KEY_MASK ||
 	     request->error))
 	    request->result = EIO;
     }
diff --git a/sys/dev/ata/ata-sata.c b/sys/dev/ata/ata-sata.c
index 642a94e..1151ca1 100644
--- a/sys/dev/ata/ata-sata.c
+++ b/sys/dev/ata/ata-sata.c
@@ -73,59 +73,141 @@ ata_sata_phy_check_events(device_t dev)
     }
 }
 
+int
+ata_sata_scr_read(struct ata_channel *ch, int port, int reg, uint32_t *val)
+{
+    int r;
+
+    if (port < 0) {
+	*val = ATA_IDX_INL(ch, reg);
+	return (0);
+    } else {
+	switch (reg) {
+	    case ATA_SSTATUS:
+		r = 0;
+		break;
+	    case ATA_SERROR:
+		r = 1;
+		break;
+	    case ATA_SCONTROL:
+		r = 2;
+		break;
+	    default:
+		return (EINVAL);
+	}
+	return (ch->hw.pm_read(ch->dev, port, r, val));
+    }
+}
+
+int
+ata_sata_scr_write(struct ata_channel *ch, int port, int reg, uint32_t val)
+{
+    int r;
+
+    if (port < 0) {
+	ATA_IDX_OUTL(ch, reg, val);
+	return (0);
+    } else {
+	switch (reg) {
+	    case ATA_SERROR:
+		r = 1;
+		break;
+	    case ATA_SCONTROL:
+		r = 2;
+		break;
+	    default:
+		return (EINVAL);
+	}
+	return (ch->hw.pm_write(ch->dev, port, r, val));
+    }
+}
+
 static int
-ata_sata_connect(struct ata_channel *ch)
+ata_sata_connect(struct ata_channel *ch, int port)
 {
     u_int32_t status;
     int timeout;
 
     /* wait up to 1 second for "connect well" */
     for (timeout = 0; timeout < 100 ; timeout++) {
-	status = ATA_IDX_INL(ch, ATA_SSTATUS);
+	if (ata_sata_scr_read(ch, port, ATA_SSTATUS, &status))
+	    return (0);
 	if ((status & ATA_SS_CONWELL_MASK) == ATA_SS_CONWELL_GEN1 ||
 	    (status & ATA_SS_CONWELL_MASK) == ATA_SS_CONWELL_GEN2)
 	    break;
 	ata_udelay(10000);
     }
     if (timeout >= 100) {
-	if (bootverbose)
-	    device_printf(ch->dev, "SATA connect status=%08x\n", status);
+	if (bootverbose) {
+	    if (port < 0) {
+		device_printf(ch->dev, "SATA connect timeout status=%08x\n",
+		    status);
+	    } else {
+		device_printf(ch->dev, "p%d: SATA connect timeout status=%08x\n",
+		    port, status);
+	    }
+	}
 	return 0;
     }
-    if (bootverbose)
-	device_printf(ch->dev, "SATA connect time=%dms\n", timeout * 10);
+    if (bootverbose) {
+	if (port < 0) {
+	    device_printf(ch->dev, "SATA connect time=%dms status=%08x\n",
+		timeout * 10, status);
+	} else {
+	    device_printf(ch->dev, "p%d: SATA connect time=%dms status=%08x\n",
+		port, timeout * 10, status);
+	}
+    }
 
     /* clear SATA error register */
-    ATA_IDX_OUTL(ch, ATA_SERROR, ATA_IDX_INL(ch, ATA_SERROR));
+    ata_sata_scr_write(ch, port, ATA_SERROR, 0xffffffff);
 
     return 1;
 }
 
 int
-ata_sata_phy_reset(device_t dev)
+ata_sata_phy_reset(device_t dev, int port, int quick)
 {
     struct ata_channel *ch = device_get_softc(dev);
     int loop, retry;
+    uint32_t val;
 
-    if ((ATA_IDX_INL(ch, ATA_SCONTROL) & ATA_SC_DET_MASK) == ATA_SC_DET_IDLE)
-	return ata_sata_connect(ch);
+    if (quick) {
+	if (ata_sata_scr_read(ch, port, ATA_SCONTROL, &val))
+	    return (0);
+	if ((val & ATA_SC_DET_MASK) == ATA_SC_DET_IDLE)
+	    return ata_sata_connect(ch, port);
+    }
 
+    if (bootverbose) {
+	if (port < 0) {
+	    device_printf(dev, "hardware reset ...\n");
+	} else {
+	    device_printf(dev, "p%d: hardware reset ...\n", port);
+	}
+    }
     for (retry = 0; retry < 10; retry++) {
 	for (loop = 0; loop < 10; loop++) {
-	    ATA_IDX_OUTL(ch, ATA_SCONTROL, ATA_SC_DET_RESET);
+	    if (ata_sata_scr_write(ch, port, ATA_SCONTROL, ATA_SC_DET_RESET))
+		return (0);
 	    ata_udelay(100);
-	    if ((ATA_IDX_INL(ch, ATA_SCONTROL) & ATA_SC_DET_MASK) == 
-		ATA_SC_DET_RESET)
+	    if (ata_sata_scr_read(ch, port, ATA_SCONTROL, &val))
+		return (0);
+	    if ((val & ATA_SC_DET_MASK) == ATA_SC_DET_RESET)
 		break;
 	}
 	ata_udelay(5000);
 	for (loop = 0; loop < 10; loop++) {
-	    ATA_IDX_OUTL(ch, ATA_SCONTROL, ATA_SC_DET_IDLE |
-					   ATA_SC_IPM_DIS_PARTIAL |
-					   ATA_SC_IPM_DIS_SLUMBER);
+	    if (ata_sata_scr_write(ch, port, ATA_SCONTROL,
+					ATA_SC_DET_IDLE |
+					ATA_SC_IPM_DIS_PARTIAL |
+					ATA_SC_IPM_DIS_SLUMBER))
+		return (0);
 	    ata_udelay(100);
-	    if ((ATA_IDX_INL(ch, ATA_SCONTROL) & ATA_SC_DET_MASK) == 0)
-		return ata_sata_connect(ch);
+	    if (ata_sata_scr_read(ch, port, ATA_SCONTROL, &val))
+		return (0);
+	    if ((val & ATA_SC_DET_MASK) == 0)
+		return ata_sata_connect(ch, port);
 	}
     }
     return 0;
@@ -237,14 +319,27 @@ ata_pm_identify(device_t dev)
     /* chip specific quirks */
     switch (pm_chipid) {
     case 0x37261095:
-	/* Some of these bogusly reports 6 ports */
+	/* This PM declares 6 ports, while only 5 of them are real.
+	 * Port 5 is enclosure management bridge port, which has implementation
+	 * problems, causing probe faults. Hide it for now. */
+	device_printf(dev, "SiI 3726 (rev=%x) Port Multiplier with %d (5) ports\n",
+		      pm_revision, pm_ports);
 	pm_ports = 5;
-	device_printf(dev, "SiI 3726 r%x Portmultiplier with %d ports\n",
+	break;
+
+    case 0x47261095:
+	/* This PM declares 7 ports, while only 5 of them are real.
+	 * Port 5 is some fake "Config  Disk" with 640 sectors size,
+	 * port 6 is enclosure management bridge port.
+	 * Both fake ports has implementation problems, causing
+	 * probe faults. Hide them for now. */
+	device_printf(dev, "SiI 4726 (rev=%x) Port Multiplier with %d (5) ports\n",
 		      pm_revision, pm_ports);
+	pm_ports = 5;
 	break;
 
     default:
-	device_printf(dev, "Portmultiplier (id=%08x rev=%x) with %d ports\n",
+	device_printf(dev, "Port Multiplier (id=%08x rev=%x) with %d ports\n",
 		      pm_chipid, pm_revision, pm_ports);
     }
 
@@ -253,41 +348,17 @@ ata_pm_identify(device_t dev)
 
     /* reset all ports and register if anything connected */
     for (port=0; port < pm_ports; port++) {
-	u_int32_t signature, status;
-	int timeout;
+	u_int32_t signature;
 
-	if (ch->hw.pm_write(dev, port, 2, ATA_SC_DET_RESET)) {
-	    device_printf(dev, "p%d: writing ATA_SC_DET_RESET failed\n", port);
+	if (!ata_sata_phy_reset(dev, port, 1))
 	    continue;
-	}
-
-	ata_udelay(5000);
-
-	if (ch->hw.pm_write(dev, port, 2, ATA_SC_DET_IDLE)) {
-	    device_printf(dev, "p%d: writing ATA_SC_DET_idle failed\n", port);
-	    continue;
-	}
-
-	ata_udelay(5000);
-
-	/* wait up to 1 second for "connect well" */
-	for (timeout = 0; timeout < 100 ; timeout++) {
-	    ch->hw.pm_read(dev, port, 0, &status);
-	    if ((status & ATA_SS_CONWELL_MASK) == ATA_SS_CONWELL_GEN1 ||
-		(status & ATA_SS_CONWELL_MASK) == ATA_SS_CONWELL_GEN2)
-		break;
-	    ata_udelay(10000);
-	}
-	if (timeout >= 100) {
-	    if (bootverbose)
-		device_printf(dev, "p%d: connect status=%08x\n", port, status);
-	    continue;
-	}
-	if (bootverbose)
-	    device_printf(dev, "p%d: connect time %dms\n", port, timeout * 10);
 
-	/* clear SERROR register */
-	ch->hw.pm_write(dev, port, 1, 0xffffffff);
+	/*
+	 * XXX: I have no idea how to properly wait for PMP port hardreset
+	 * completion. Without this delay soft reset does not completes
+	 * successfully.
+	 */
+	DELAY(1000000);
 
 	signature = ch->hw.softreset(dev, port);
 
diff --git a/sys/dev/ata/chipsets/ata-ahci.c b/sys/dev/ata/chipsets/ata-ahci.c
index 991d2ad..9e78132 100644
--- a/sys/dev/ata/chipsets/ata-ahci.c
+++ b/sys/dev/ata/chipsets/ata-ahci.c
@@ -59,10 +59,16 @@ static int ata_ahci_begin_transaction(struct ata_request *request);
 static int ata_ahci_end_transaction(struct ata_request *request);
 static int ata_ahci_pm_read(device_t dev, int port, int reg, u_int32_t *result);
 static int ata_ahci_pm_write(device_t dev, int port, int reg, u_int32_t result);
+static int ata_ahci_hardreset(device_t dev, int port, uint32_t *signature);
 static u_int32_t ata_ahci_softreset(device_t dev, int port);
 static void ata_ahci_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error);
 static int ata_ahci_setup_fis(struct ata_ahci_cmd_tab *ctp, struct ata_request *equest);
 static void ata_ahci_dmainit(device_t dev);
+static void ata_ahci_start(device_t dev);
+static void ata_ahci_stop(device_t dev);
+static void ata_ahci_clo(device_t dev);
+static void ata_ahci_start_fr(device_t dev);
+static void ata_ahci_stop_fr(device_t dev);
 
 /*
  * AHCI v1.x compliant SATA chipset support functions
@@ -131,6 +137,8 @@ ata_ahci_chipinit(device_t dev)
     ctlr->reset = ata_ahci_reset;
     ctlr->ch_attach = ata_ahci_ch_attach;
     ctlr->ch_detach = ata_ahci_ch_detach;
+    ctlr->ch_suspend = ata_ahci_ch_suspend;
+    ctlr->ch_resume = ata_ahci_ch_resume;
     ctlr->setmode = ata_sata_setmode;
     ctlr->suspend = ata_ahci_suspend;
     ctlr->resume = ata_ahci_ctlr_reset;
@@ -192,7 +200,6 @@ ata_ahci_suspend(device_t dev)
     return 0;
 }
 
-
 int
 ata_ahci_ch_attach(device_t dev)
 {
@@ -220,12 +227,22 @@ ata_ahci_ch_attach(device_t dev)
     ch->hw.pm_read = ata_ahci_pm_read;
     ch->hw.pm_write = ata_ahci_pm_write;
 
+    ata_ahci_ch_resume(dev);
     return 0;
 }
 
 int
 ata_ahci_ch_detach(device_t dev)
 {
+
+    ata_ahci_ch_suspend(dev);
+    ata_dmafini(dev);
+    return (0);
+}
+
+int
+ata_ahci_ch_suspend(device_t dev)
+{
     struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev));
     struct ata_channel *ch = device_get_softc(dev);
     int offset = ch->unit << 7;
@@ -233,6 +250,8 @@ ata_ahci_ch_detach(device_t dev)
     /* Disable port interrupts. */
     ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IE + offset, 0);
     /* Reset command register. */
+    ata_ahci_stop(dev);
+    ata_ahci_stop_fr(dev);
     ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, 0);
 
     /* Allow everything including partial and slumber modes. */
@@ -243,7 +262,35 @@ ata_ahci_ch_detach(device_t dev)
     /* Disable PHY. */
     ATA_IDX_OUTL(ch, ATA_SCONTROL, ATA_SC_DET_DISABLE);
 
-    ata_dmafini(dev);
+    return (0);
+}
+
+int
+ata_ahci_ch_resume(device_t dev)
+{
+    struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev));
+    struct ata_channel *ch = device_get_softc(dev);
+    uint64_t work;
+    int offset = ch->unit << 7;
+
+    /* Disable port interrupts */
+    ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IE + offset, 0);
+
+    /* setup work areas */
+    work = ch->dma.work_bus + ATA_AHCI_CL_OFFSET;
+    ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CLB + offset, work & 0xffffffff);
+    ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CLBU + offset, work >> 32);
+
+    work = ch->dma.work_bus + ATA_AHCI_FB_OFFSET;
+    ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_FB + offset, work & 0xffffffff); 
+    ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_FBU + offset, work >> 32);
+
+    /* activate the channel and power/spin up device */
+    ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset,
+	     (ATA_AHCI_P_CMD_ACTIVE | ATA_AHCI_P_CMD_POD | ATA_AHCI_P_CMD_SUD));
+    ata_ahci_start_fr(dev);
+    ata_ahci_start(dev);
+
     return (0);
 }
 
@@ -366,9 +413,6 @@ ata_ahci_begin_transaction(struct ata_request *request)
 		 ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset) &
 		 ~ATA_AHCI_P_CMD_ATAPI);
 
-    /* set PM port to address */
-    //ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_FBS + offset, (port << 8) | 0x00000001);
-
     /* issue command to controller */
     ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CI + offset, (1 << request->tag));
     
@@ -465,9 +509,6 @@ ata_ahci_issue_cmd(device_t dev, u_int16_t flags, int timeout)
     clp->bytecount = 0;
     clp->cmd_table_phys = htole64(ch->dma.work_bus + ATA_AHCI_CT_OFFSET);
 
-    /* set PM port */
-    //ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_FBS + offset, (port << 8) | 0x00000001);
-
     /* issue command to controller */
     ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CI + offset, 1);
 
@@ -480,7 +521,7 @@ ata_ahci_issue_cmd(device_t dev, u_int16_t flags, int timeout)
 
     /* clear interrupts */
     ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IS + offset,
-	     ATA_INL(ctlr->r_res2, ATA_AHCI_P_IS + offset));
+	    ATA_INL(ctlr->r_res2, ATA_AHCI_P_IS + offset));
 
     if (timeout && (count >= timeout)) {
 	if (bootverbose) {
@@ -559,7 +600,7 @@ ata_ahci_stop(device_t dev)
     /* kill off all activity on this channel */
     cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset);
     ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset,
-	     cmd & ~(ATA_AHCI_P_CMD_FRE | ATA_AHCI_P_CMD_ST));
+	     cmd & ~ATA_AHCI_P_CMD_ST);
 
     /* XXX SOS this is not entirely wrong */
     timeout = 0;
@@ -617,10 +658,47 @@ ata_ahci_start(device_t dev)
     /* start operations on this channel */
     cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset);
     ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset,
-	     cmd | (ATA_AHCI_P_CMD_FRE | ATA_AHCI_P_CMD_ST) |
+	     cmd | ATA_AHCI_P_CMD_ST |
 	     (ch->devices & ATA_PORTMULTIPLIER ? ATA_AHCI_P_CMD_PMA : 0));
 }
 
+static void
+ata_ahci_stop_fr(device_t dev)
+{
+    struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev));
+    struct ata_channel *ch = device_get_softc(dev);
+    u_int32_t cmd;
+    int offset = ch->unit << 7;
+    int timeout;
+
+    /* kill off all activity on this channel */
+    cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset);
+    ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, cmd & ~ATA_AHCI_P_CMD_FRE);
+
+    timeout = 0;
+    do {
+	DELAY(1000);
+	if (timeout++ > 1000) {
+	    device_printf(dev, "stopping AHCI FR engine failed\n");
+	    break;
+	}
+    }
+    while (ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset) & ATA_AHCI_P_CMD_FR);
+}
+
+static void
+ata_ahci_start_fr(device_t dev)
+{
+    struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev));
+    struct ata_channel *ch = device_get_softc(dev);
+    u_int32_t cmd;
+    int offset = ch->unit << 7;
+
+    /* start FIS reception on this channel */
+    cmd = ATA_INL(ctlr->r_res2, ATA_AHCI_P_CMD + offset);
+    ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset, cmd | ATA_AHCI_P_CMD_FRE);
+}
+
 static int
 ata_ahci_wait_ready(device_t dev, int t)
 {
@@ -628,13 +706,14 @@ ata_ahci_wait_ready(device_t dev, int t)
     struct ata_channel *ch = device_get_softc(dev);
     int offset = ch->unit << 7;
     int timeout = 0;
+    uint32_t val;
 
-    while (ATA_INL(ctlr->r_res2, ATA_AHCI_P_TFD + offset) &
+    while ((val = ATA_INL(ctlr->r_res2, ATA_AHCI_P_TFD + offset)) &
 	(ATA_S_BUSY | ATA_S_DRQ)) {
 	    DELAY(1000);
 	    if (timeout++ > t) {
-		device_printf(dev, "port is not ready (timeout %dms)\n", t);
-		return (-1);
+		device_printf(dev, "port is not ready (timeout %dms) tfd = %08x\n", t, val);
+		return (EBUSY);
 	    }
     } 
     if (bootverbose)
@@ -642,6 +721,28 @@ ata_ahci_wait_ready(device_t dev, int t)
     return (0);
 }
 
+static int
+ata_ahci_hardreset(device_t dev, int port, uint32_t *signature)
+{
+    struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev));
+    struct ata_channel *ch = device_get_softc(dev);
+    int offset = ch->unit << 7;
+
+    *signature = 0xffffffff;
+    ata_ahci_stop(dev);
+    /* Reset port */
+    if (!ata_sata_phy_reset(dev, port, 0))
+	return (ENOENT);
+    /* Wait for clearing busy status. */
+    if (ata_ahci_wait_ready(dev, 10000)) {
+	device_printf(dev, "hardware reset timeout\n");
+	return (EBUSY);
+    }
+    *signature = ATA_INL(ctlr->r_res2, ATA_AHCI_P_SIG + offset);
+    ata_ahci_start(dev);
+    return (0);
+}
+
 static u_int32_t
 ata_ahci_softreset(device_t dev, int port)
 {
@@ -679,9 +780,9 @@ ata_ahci_softreset(device_t dev, int port)
     ctp->cfis[1] = port & 0x0f;
     //ctp->cfis[7] = ATA_D_LBA | ATA_D_IBM;
     ctp->cfis[15] = ATA_A_4BIT;
-    ata_ahci_issue_cmd(dev, 0, 1000);
+    ata_ahci_issue_cmd(dev, 0, 3000);
 
-    if (ata_ahci_wait_ready(dev, 1000)) {
+    if (ata_ahci_wait_ready(dev, 0)) {
 	device_printf(dev, "software reset clear timeout\n");
 	return (-1);
     }
@@ -694,7 +795,6 @@ ata_ahci_reset(device_t dev)
 {
     struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev));
     struct ata_channel *ch = device_get_softc(dev);
-    u_int64_t work;
     u_int32_t signature;
     int offset = ch->unit << 7;
 
@@ -704,25 +804,7 @@ ata_ahci_reset(device_t dev)
     /* Disable port interrupts */
     ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IE + offset, 0);
 
-    /* setup work areas */
-    work = ch->dma.work_bus + ATA_AHCI_CL_OFFSET;
-    ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CLB + offset, work & 0xffffffff);
-    ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CLBU + offset, work >> 32);
-
-    work = ch->dma.work_bus + ATA_AHCI_FB_OFFSET;
-    ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_FB + offset, work & 0xffffffff); 
-    ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_FBU + offset, work >> 32);
-
-    /* activate the channel and power/spin up device */
-    ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_CMD + offset,
-	     (ATA_AHCI_P_CMD_ACTIVE | ATA_AHCI_P_CMD_POD | ATA_AHCI_P_CMD_SUD));
-
-    ata_ahci_stop(dev);
-
-    /* enable FIS based switching */
-    //ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_FBS + offset, 0x00000003);
-
-    if (!ata_sata_phy_reset(dev)) {
+    if (ata_ahci_hardreset(dev, -1, &signature)) {
 	if (bootverbose)
 	    device_printf(dev, "AHCI reset done: phy reset found no device\n");
 	ch->devices = 0;
@@ -733,8 +815,6 @@ ata_ahci_reset(device_t dev)
 	return;
     }
 
-    ata_ahci_start(dev);
-
     /* enable wanted port interrupts */
     ATA_OUTL(ctlr->r_res2, ATA_AHCI_P_IE + offset,
 	     (ATA_AHCI_P_IX_CPD | ATA_AHCI_P_IX_TFE | ATA_AHCI_P_IX_HBF |
@@ -743,9 +823,6 @@ ata_ahci_reset(device_t dev)
 	      ATA_AHCI_P_IX_UF | ATA_AHCI_P_IX_SDB | ATA_AHCI_P_IX_DS |
 	      ATA_AHCI_P_IX_PS | ATA_AHCI_P_IX_DHR));
 
-    /* Wait for initial TFD from device. */
-    ata_ahci_wait_ready(dev, 10000);
-
     /* only probe for PortMultiplier if HW has support */
     if (ATA_INL(ctlr->r_res2, ATA_AHCI_CAP) & ATA_AHCI_CAP_SPM) {
 	signature = ata_ahci_softreset(dev, ATA_PM);
diff --git a/sys/dev/ata/chipsets/ata-intel.c b/sys/dev/ata/chipsets/ata-intel.c
index 8bd2f03..cae1778 100644
--- a/sys/dev/ata/chipsets/ata-intel.c
+++ b/sys/dev/ata/chipsets/ata-intel.c
@@ -517,7 +517,7 @@ ata_intel_31244_tf_write(struct ata_request *request)
 static void
 ata_intel_31244_reset(device_t dev)
 {
-    if (ata_sata_phy_reset(dev))
+    if (ata_sata_phy_reset(dev, -1, 1))
 	ata_generic_reset(dev);
 }
 
diff --git a/sys/dev/ata/chipsets/ata-jmicron.c b/sys/dev/ata/chipsets/ata-jmicron.c
index 301b99b..b1e7aef 100644
--- a/sys/dev/ata/chipsets/ata-jmicron.c
+++ b/sys/dev/ata/chipsets/ata-jmicron.c
@@ -55,6 +55,8 @@ __FBSDID("$FreeBSD$");
 static int ata_jmicron_chipinit(device_t dev);
 static int ata_jmicron_ch_attach(device_t dev);
 static int ata_jmicron_ch_detach(device_t dev);
+static int ata_jmicron_ch_suspend(device_t dev);
+static int ata_jmicron_ch_resume(device_t dev);
 static void ata_jmicron_reset(device_t dev);
 static void ata_jmicron_setmode(device_t dev, int mode);
 
@@ -127,6 +129,8 @@ ata_jmicron_chipinit(device_t dev)
 
 	ctlr->ch_attach = ata_jmicron_ch_attach;
 	ctlr->ch_detach = ata_jmicron_ch_detach;
+	ctlr->ch_suspend = ata_jmicron_ch_suspend;
+	ctlr->ch_resume = ata_jmicron_ch_resume;
 	ctlr->reset = ata_jmicron_reset;
 	ctlr->setmode = ata_jmicron_setmode;
 
@@ -173,6 +177,30 @@ ata_jmicron_ch_detach(device_t dev)
     return (error);
 }
 
+static int
+ata_jmicron_ch_suspend(device_t dev)
+{
+    struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev));
+    struct ata_channel *ch = device_get_softc(dev);
+    int error = 0;
+
+    if (ch->unit < ctlr->chip->cfg1)
+	error = ata_ahci_ch_suspend(dev);
+    return error;
+}
+
+static int
+ata_jmicron_ch_resume(device_t dev)
+{
+    struct ata_pci_controller *ctlr = device_get_softc(device_get_parent(dev));
+    struct ata_channel *ch = device_get_softc(dev);
+    int error = 0;
+
+    if (ch->unit < ctlr->chip->cfg1)
+	error = ata_ahci_ch_resume(dev);
+    return (error);
+}
+
 static void
 ata_jmicron_reset(device_t dev)
 {
diff --git a/sys/dev/ata/chipsets/ata-marvell.c b/sys/dev/ata/chipsets/ata-marvell.c
index a0fa485..361bc5b 100644
--- a/sys/dev/ata/chipsets/ata-marvell.c
+++ b/sys/dev/ata/chipsets/ata-marvell.c
@@ -503,7 +503,7 @@ ata_marvell_edma_reset(device_t dev)
     ATA_OUTL(ctlr->r_res1, 0x0200c + ATA_MV_EDMA_BASE(ch), ~0x0);
 
     /* enable channel and test for devices */
-    if (ata_sata_phy_reset(dev))
+    if (ata_sata_phy_reset(dev, -1, 1))
 	ata_generic_reset(dev);
 
     /* enable EDMA machinery */
diff --git a/sys/dev/ata/chipsets/ata-nvidia.c b/sys/dev/ata/chipsets/ata-nvidia.c
index 7e03afa..9e1de81 100644
--- a/sys/dev/ata/chipsets/ata-nvidia.c
+++ b/sys/dev/ata/chipsets/ata-nvidia.c
@@ -249,7 +249,7 @@ ata_nvidia_status(device_t dev)
 static void
 ata_nvidia_reset(device_t dev)
 {
-    if (ata_sata_phy_reset(dev))
+    if (ata_sata_phy_reset(dev, -1, 1))
 	ata_generic_reset(dev);
 }
 
diff --git a/sys/dev/ata/chipsets/ata-promise.c b/sys/dev/ata/chipsets/ata-promise.c
index 3856535..6b34596 100644
--- a/sys/dev/ata/chipsets/ata-promise.c
+++ b/sys/dev/ata/chipsets/ata-promise.c
@@ -769,7 +769,7 @@ ata_promise_mio_reset(device_t dev)
 	if ((ctlr->chip->cfg2 == PR_SATA) ||
 	    ((ctlr->chip->cfg2 == PR_CMBO) && (ch->unit < 2))) {
 
-	    if (ata_sata_phy_reset(dev))
+	    if (ata_sata_phy_reset(dev, -1, 1))
 		ata_generic_reset(dev);
 
 	    /* reset and enable plug/unplug intr */
@@ -805,7 +805,7 @@ ata_promise_mio_reset(device_t dev)
 		     (ATA_INL(ctlr->r_res2, 0x414 + (ch->unit << 8)) &
 		     ~0x00000003) | 0x00000001);
 
-	    if (ata_sata_phy_reset(dev)) {
+	    if (ata_sata_phy_reset(dev, -1, 1)) {
 		u_int32_t signature = ch->hw.softreset(dev, ATA_PM);
 
 		if (1 | bootverbose)
diff --git a/sys/dev/ata/chipsets/ata-serverworks.c b/sys/dev/ata/chipsets/ata-serverworks.c
index 366f9a1..c591d3d 100644
--- a/sys/dev/ata/chipsets/ata-serverworks.c
+++ b/sys/dev/ata/chipsets/ata-serverworks.c
@@ -58,6 +58,9 @@ static int ata_serverworks_ch_detach(device_t dev);
 static void ata_serverworks_tf_read(struct ata_request *request);
 static void ata_serverworks_tf_write(struct ata_request *request);
 static void ata_serverworks_setmode(device_t dev, int mode);
+#ifdef __powerpc__
+static int ata_serverworks_status(device_t dev);
+#endif
 
 /* misc defines */
 #define SWKS_33		0
@@ -98,6 +101,23 @@ ata_serverworks_probe(device_t dev)
     return 0;
 }
 
+#ifdef __powerpc__
+static int
+ata_serverworks_status(device_t dev)
+{
+    struct ata_channel *ch = device_get_softc(dev);
+
+    /*
+     * We need to do a 4-byte read on the status reg before the values
+     * will report correctly
+     */
+
+    ATA_IDX_INL(ch,ATA_STATUS);
+
+    return ata_pci_status(dev);
+}
+#endif
+
 static int
 ata_serverworks_chipinit(device_t dev)
 {
@@ -186,6 +206,9 @@ ata_serverworks_ch_attach(device_t dev)
     ata_pci_hw(dev);
     ch->hw.tf_read = ata_serverworks_tf_read;
     ch->hw.tf_write = ata_serverworks_tf_write;
+#ifdef __powerpc__
+    ch->hw.status = ata_serverworks_status;
+#endif
 
     /* chip does not reliably do 64K DMA transfers */
     ch->dma.max_iosize = 64 * DEV_BSIZE;
diff --git a/sys/dev/ata/chipsets/ata-siliconimage.c b/sys/dev/ata/chipsets/ata-siliconimage.c
index f5093c7..b163276 100644
--- a/sys/dev/ata/chipsets/ata-siliconimage.c
+++ b/sys/dev/ata/chipsets/ata-siliconimage.c
@@ -380,7 +380,7 @@ ata_sii_status(device_t dev)
 static void
 ata_sii_reset(device_t dev)
 {
-    if (ata_sata_phy_reset(dev))
+    if (ata_sata_phy_reset(dev, -1, 1))
 	ata_generic_reset(dev);
 }
 
@@ -832,7 +832,7 @@ ata_siiprb_reset(device_t dev)
     }
 
     /* reset phy */
-    if (!ata_sata_phy_reset(dev)) {
+    if (!ata_sata_phy_reset(dev, -1, 1)) {
 	if (bootverbose)
 	    device_printf(dev, "phy reset found no device\n");
 	ch->devices = 0;
diff --git a/sys/dev/ata/chipsets/ata-sis.c b/sys/dev/ata/chipsets/ata-sis.c
index 1d9dac7..8e02ee9 100644
--- a/sys/dev/ata/chipsets/ata-sis.c
+++ b/sys/dev/ata/chipsets/ata-sis.c
@@ -226,7 +226,7 @@ ata_sis_ch_attach(device_t dev)
 static void
 ata_sis_reset(device_t dev)
 {
-    if (ata_sata_phy_reset(dev))
+    if (ata_sata_phy_reset(dev, -1, 1))
 	ata_generic_reset(dev);
 }
 
diff --git a/sys/dev/ata/chipsets/ata-via.c b/sys/dev/ata/chipsets/ata-via.c
index 4530adf..525ca2f 100644
--- a/sys/dev/ata/chipsets/ata-via.c
+++ b/sys/dev/ata/chipsets/ata-via.c
@@ -269,7 +269,7 @@ ata_via_reset(device_t dev)
     if ((ctlr->chip->cfg2 & VIABAR) && (ch->unit > 1))
         ata_generic_reset(dev);
     else
-	if (ata_sata_phy_reset(dev))
+	if (ata_sata_phy_reset(dev, -1, 1))
 	    ata_generic_reset(dev);
 }
 
diff --git a/sys/dev/ath/ah_osdep.c b/sys/dev/ath/ah_osdep.c
index c124772..b5880ee 100644
--- a/sys/dev/ath/ah_osdep.c
+++ b/sys/dev/ath/ah_osdep.c
@@ -71,12 +71,7 @@ extern	void ath_hal_assert_failed(const char* filename,
 		int lineno, const char* msg);
 #endif
 #ifdef AH_DEBUG
-#if HAL_ABI_VERSION >= 0x08090101
 extern	void HALDEBUG(struct ath_hal *ah, u_int mask, const char* fmt, ...);
-#else
-extern	void HALDEBUG(struct ath_hal *ah, const char* fmt, ...);
-extern	void HALDEBUGn(struct ath_hal *ah, u_int level, const char* fmt, ...);
-#endif
 #endif /* AH_DEBUG */
 
 /* NB: put this here instead of the driver to avoid circular references */
@@ -140,7 +135,6 @@ ath_hal_ether_sprintf(const u_int8_t *mac)
 }
 
 #ifdef AH_DEBUG
-#if HAL_ABI_VERSION >= 0x08090101
 void
 HALDEBUG(struct ath_hal *ah, u_int mask, const char* fmt, ...)
 {
@@ -151,29 +145,6 @@ HALDEBUG(struct ath_hal *ah, u_int mask, const char* fmt, ...)
 		va_end(ap);
 	}
 }
-#else
-void
-HALDEBUG(struct ath_hal *ah, const char* fmt, ...)
-{
-	if (ath_hal_debug) {
-		__va_list ap;
-		va_start(ap, fmt);
-		ath_hal_vprintf(ah, fmt, ap);
-		va_end(ap);
-	}
-}
-
-void
-HALDEBUGn(struct ath_hal *ah, u_int level, const char* fmt, ...)
-{
-	if (ath_hal_debug >= level) {
-		__va_list ap;
-		va_start(ap, fmt);
-		ath_hal_vprintf(ah, fmt, ap);
-		va_end(ap);
-	}
-}
-#endif
 #endif /* AH_DEBUG */
 
 #ifdef AH_DEBUG_ALQ
diff --git a/sys/dev/ath/ath_hal/ah.h b/sys/dev/ath/ath_hal/ah.h
index da9060a..c1bff9c 100644
--- a/sys/dev/ath/ath_hal/ah.h
+++ b/sys/dev/ath/ath_hal/ah.h
@@ -598,8 +598,6 @@ struct ieee80211_channel;
  */
 struct ath_hal {
 	uint32_t	ah_magic;	/* consistency check magic number */
-	uint32_t	ah_abi;		/* HAL ABI version */
-#define	HAL_ABI_VERSION	0x08112800	/* YYMMDDnn */
 	uint16_t	ah_devid;	/* PCI device ID */
 	uint16_t	ah_subvendorid;	/* PCI subvendor ID */
 	HAL_SOFTC	ah_sc;		/* back pointer to driver/os state */
diff --git a/sys/dev/ath/ath_hal/ah_internal.h b/sys/dev/ath/ath_hal/ah_internal.h
index 8b00e30..ef5aba5 100644
--- a/sys/dev/ath/ath_hal/ah_internal.h
+++ b/sys/dev/ath/ath_hal/ah_internal.h
@@ -333,8 +333,10 @@ struct ath_hal_private {
 #define	ath_hal_disablePCIE(_ah) \
 	(_ah)->ah_disablePCIE(_ah)
 
-#define	ath_hal_eepromDetach(_ah) \
-	AH_PRIVATE(_ah)->ah_eepromDetach(_ah)
+#define	ath_hal_eepromDetach(_ah) do {				\
+	if (AH_PRIVATE(_ah)->ah_eepromDetach != AH_NULL)	\
+		AH_PRIVATE(_ah)->ah_eepromDetach(_ah);		\
+} while (0)
 #define	ath_hal_eepromGet(_ah, _param, _val) \
 	AH_PRIVATE(_ah)->ah_eepromGet(_ah, _param, _val)
 #define	ath_hal_eepromSet(_ah, _param, _val) \
diff --git a/sys/dev/ath/ath_hal/ar5210/ar5210_attach.c b/sys/dev/ath/ath_hal/ar5210/ar5210_attach.c
index 3df3924..13ad711 100644
--- a/sys/dev/ath/ath_hal/ar5210/ar5210_attach.c
+++ b/sys/dev/ath/ath_hal/ar5210/ar5210_attach.c
@@ -38,7 +38,6 @@ static void ar5210DisablePCIE(struct ath_hal *ah);
 
 static const struct ath_hal_private ar5210hal = {{
 	.ah_magic			= AR5210_MAGIC,
-	.ah_abi				= HAL_ABI_VERSION,
 
 	.ah_getRateTable		= ar5210GetRateTable,
 	.ah_detach			= ar5210Detach,
diff --git a/sys/dev/ath/ath_hal/ar5211/ar5211_attach.c b/sys/dev/ath/ath_hal/ar5211/ar5211_attach.c
index 81af95f..e55c67a 100644
--- a/sys/dev/ath/ath_hal/ar5211/ar5211_attach.c
+++ b/sys/dev/ath/ath_hal/ar5211/ar5211_attach.c
@@ -38,7 +38,6 @@ static void ar5211DisablePCIE(struct ath_hal *ah);
 
 static const struct ath_hal_private ar5211hal = {{
 	.ah_magic			= AR5211_MAGIC,
-	.ah_abi				= HAL_ABI_VERSION,
 
 	.ah_getRateTable		= ar5211GetRateTable,
 	.ah_detach			= ar5211Detach,
diff --git a/sys/dev/ath/ath_hal/ar5212/ar5212_attach.c b/sys/dev/ath/ath_hal/ar5212/ar5212_attach.c
index 742523f..300c0a3 100644
--- a/sys/dev/ath/ath_hal/ar5212/ar5212_attach.c
+++ b/sys/dev/ath/ath_hal/ar5212/ar5212_attach.c
@@ -34,7 +34,6 @@ static void ar5212DisablePCIE(struct ath_hal *ah);
 
 static const struct ath_hal_private ar5212hal = {{
 	.ah_magic			= AR5212_MAGIC,
-	.ah_abi				= HAL_ABI_VERSION,
 
 	.ah_getRateTable		= ar5212GetRateTable,
 	.ah_detach			= ar5212Detach,
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416.h b/sys/dev/ath/ath_hal/ar5416/ar5416.h
index 27aa9e8..a02f732 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416.h
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416.h
@@ -45,6 +45,8 @@ typedef struct {
 #define	AR5416_CCA_MAX_HIGH_VALUE	-62
 #define	AR5416_CCA_MIN_BAD_VALUE	-140
 
+#define AR5416_SPUR_RSSI_THRESH		40
+
 struct ath_hal_5416 {
 	struct ath_hal_5212 ah_5212;
 
@@ -59,6 +61,11 @@ struct ath_hal_5416 {
 	HAL_INI_ARRAY	ah_ini_addac;
 	HAL_INI_ARRAY	ah_ini_pcieserdes;
 
+	void		(*ah_writeIni)(struct ath_hal *,
+			    const struct ieee80211_channel *);
+	void		(*ah_spurMitigate)(struct ath_hal *,
+			    const struct ieee80211_channel *);
+
 	u_int       	ah_globaltxtimeout;	/* global tx timeout */
 	u_int		ah_gpioMask;
 	int		ah_hangs;		/* h/w hangs state */
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c b/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c
index 8328a4e..1130529 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_attach.c
@@ -22,6 +22,8 @@
 #include "ah_internal.h"
 #include "ah_devid.h"
 
+#include "ah_eeprom_v14.h"
+
 #include "ar5416/ar5416.h"
 #include "ar5416/ar5416reg.h"
 #include "ar5416/ar5416phy.h"
@@ -29,6 +31,10 @@
 #include "ar5416/ar5416.ini"
 
 static void ar5416ConfigPCIE(struct ath_hal *ah, HAL_BOOL restore);
+static void ar5416WriteIni(struct ath_hal *ah,
+	    const struct ieee80211_channel *chan);
+static void ar5416SpurMitigate(struct ath_hal *ah,
+	    const struct ieee80211_channel *chan);
 
 static void
 ar5416AniSetup(struct ath_hal *ah)
@@ -152,6 +158,8 @@ ar5416InitState(struct ath_hal_5416 *ahp5416, uint16_t devid, HAL_SOFTC sc,
 #endif
 	ahp->ah_priv.ah_getChipPowerLimits = ar5416GetChipPowerLimits;
 
+	AH5416(ah)->ah_writeIni		= ar5416WriteIni;
+	AH5416(ah)->ah_spurMitigate	= ar5416SpurMitigate;
 	/*
 	 * Start by setting all Owl devices to 2x2
 	 */
@@ -393,6 +401,301 @@ ar5416ConfigPCIE(struct ath_hal *ah, HAL_BOOL restore)
 	}
 }
 
+static void
+ar5416WriteIni(struct ath_hal *ah, const struct ieee80211_channel *chan)
+{
+	u_int modesIndex, freqIndex;
+	int regWrites = 0;
+
+	/* Setup the indices for the next set of register array writes */
+	/* XXX Ignore 11n dynamic mode on the AR5416 for the moment */
+	if (IEEE80211_IS_CHAN_2GHZ(chan)) {
+		freqIndex = 2;
+		if (IEEE80211_IS_CHAN_HT40(chan))
+			modesIndex = 3;
+		else if (IEEE80211_IS_CHAN_108G(chan))
+			modesIndex = 5;
+		else
+			modesIndex = 4;
+	} else {
+		freqIndex = 1;
+		if (IEEE80211_IS_CHAN_HT40(chan) ||
+		    IEEE80211_IS_CHAN_TURBO(chan))
+			modesIndex = 2;
+		else
+			modesIndex = 1;
+	}
+
+	/* Set correct Baseband to analog shift setting to access analog chips. */
+	OS_REG_WRITE(ah, AR_PHY(0), 0x00000007);
+
+	/*
+	 * Write addac shifts
+	 */
+	OS_REG_WRITE(ah, AR_PHY_ADC_SERIAL_CTL, AR_PHY_SEL_EXTERNAL_RADIO);
+#if 0
+	/* NB: only required for Sowl */
+	ar5416EepromSetAddac(ah, chan);
+#endif
+	regWrites = ath_hal_ini_write(ah, &AH5416(ah)->ah_ini_addac, 1,
+	    regWrites);
+	OS_REG_WRITE(ah, AR_PHY_ADC_SERIAL_CTL, AR_PHY_SEL_INTERNAL_ADDAC);
+
+	regWrites = ath_hal_ini_write(ah, &AH5212(ah)->ah_ini_modes,
+	    modesIndex, regWrites);
+	regWrites = ath_hal_ini_write(ah, &AH5212(ah)->ah_ini_common,
+	    1, regWrites);
+
+	/* XXX updated regWrites? */
+	AH5212(ah)->ah_rfHal->writeRegs(ah, modesIndex, freqIndex, regWrites);
+}
+
+/*
+ * Convert to baseband spur frequency given input channel frequency 
+ * and compute register settings below.
+ */
+
+static void
+ar5416SpurMitigate(struct ath_hal *ah, const struct ieee80211_channel *chan)
+{
+    uint16_t freq = ath_hal_gethwchannel(ah, chan);
+    static const int pilot_mask_reg[4] = { AR_PHY_TIMING7, AR_PHY_TIMING8,
+                AR_PHY_PILOT_MASK_01_30, AR_PHY_PILOT_MASK_31_60 };
+    static const int chan_mask_reg[4] = { AR_PHY_TIMING9, AR_PHY_TIMING10,
+                AR_PHY_CHANNEL_MASK_01_30, AR_PHY_CHANNEL_MASK_31_60 };
+    static const int inc[4] = { 0, 100, 0, 0 };
+
+    int bb_spur = AR_NO_SPUR;
+    int bin, cur_bin;
+    int spur_freq_sd;
+    int spur_delta_phase;
+    int denominator;
+    int upper, lower, cur_vit_mask;
+    int tmp, new;
+    int i;
+
+    int8_t mask_m[123];
+    int8_t mask_p[123];
+    int8_t mask_amt;
+    int tmp_mask;
+    int cur_bb_spur;
+    HAL_BOOL is2GHz = IEEE80211_IS_CHAN_2GHZ(chan);
+
+    OS_MEMZERO(mask_m, sizeof(mask_m));
+    OS_MEMZERO(mask_p, sizeof(mask_p));
+
+    /*
+     * Need to verify range +/- 9.5 for static ht20, otherwise spur
+     * is out-of-band and can be ignored.
+     */
+    /* XXX ath9k changes */
+    for (i = 0; i < AR5416_EEPROM_MODAL_SPURS; i++) {
+        cur_bb_spur = ath_hal_getSpurChan(ah, i, is2GHz);
+        if (AR_NO_SPUR == cur_bb_spur)
+            break;
+        cur_bb_spur = cur_bb_spur - (freq * 10);
+        if ((cur_bb_spur > -95) && (cur_bb_spur < 95)) {
+            bb_spur = cur_bb_spur;
+            break;
+        }
+    }
+    if (AR_NO_SPUR == bb_spur)
+        return;
+
+    bin = bb_spur * 32;
+
+    tmp = OS_REG_READ(ah, AR_PHY_TIMING_CTRL4_CHAIN(0));
+    new = tmp | (AR_PHY_TIMING_CTRL4_ENABLE_SPUR_RSSI |
+        AR_PHY_TIMING_CTRL4_ENABLE_SPUR_FILTER |
+        AR_PHY_TIMING_CTRL4_ENABLE_CHAN_MASK |
+        AR_PHY_TIMING_CTRL4_ENABLE_PILOT_MASK);
+
+    OS_REG_WRITE(ah, AR_PHY_TIMING_CTRL4_CHAIN(0), new);
+
+    new = (AR_PHY_SPUR_REG_MASK_RATE_CNTL |
+        AR_PHY_SPUR_REG_ENABLE_MASK_PPM |
+        AR_PHY_SPUR_REG_MASK_RATE_SELECT |
+        AR_PHY_SPUR_REG_ENABLE_VIT_SPUR_RSSI |
+        SM(AR5416_SPUR_RSSI_THRESH, AR_PHY_SPUR_REG_SPUR_RSSI_THRESH));
+    OS_REG_WRITE(ah, AR_PHY_SPUR_REG, new);
+    /*
+     * Should offset bb_spur by +/- 10 MHz for dynamic 2040 MHz
+     * config, no offset for HT20.
+     * spur_delta_phase = bb_spur/40 * 2**21 for static ht20,
+     * /80 for dyn2040.
+     */
+    spur_delta_phase = ((bb_spur * 524288) / 100) &
+        AR_PHY_TIMING11_SPUR_DELTA_PHASE;
+    /*
+     * in 11A mode the denominator of spur_freq_sd should be 40 and
+     * it should be 44 in 11G
+     */
+    denominator = IEEE80211_IS_CHAN_2GHZ(chan) ? 440 : 400;
+    spur_freq_sd = ((bb_spur * 2048) / denominator) & 0x3ff;
+
+    new = (AR_PHY_TIMING11_USE_SPUR_IN_AGC |
+        SM(spur_freq_sd, AR_PHY_TIMING11_SPUR_FREQ_SD) |
+        SM(spur_delta_phase, AR_PHY_TIMING11_SPUR_DELTA_PHASE));
+    OS_REG_WRITE(ah, AR_PHY_TIMING11, new);
+
+
+    /*
+     * ============================================
+     * pilot mask 1 [31:0] = +6..-26, no 0 bin
+     * pilot mask 2 [19:0] = +26..+7
+     *
+     * channel mask 1 [31:0] = +6..-26, no 0 bin
+     * channel mask 2 [19:0] = +26..+7
+     */
+    //cur_bin = -26;
+    cur_bin = -6000;
+    upper = bin + 100;
+    lower = bin - 100;
+
+    for (i = 0; i < 4; i++) {
+        int pilot_mask = 0;
+        int chan_mask  = 0;
+        int bp         = 0;
+        for (bp = 0; bp < 30; bp++) {
+            if ((cur_bin > lower) && (cur_bin < upper)) {
+                pilot_mask = pilot_mask | 0x1 << bp;
+                chan_mask  = chan_mask | 0x1 << bp;
+            }
+            cur_bin += 100;
+        }
+        cur_bin += inc[i];
+        OS_REG_WRITE(ah, pilot_mask_reg[i], pilot_mask);
+        OS_REG_WRITE(ah, chan_mask_reg[i], chan_mask);
+    }
+
+    /* =================================================
+     * viterbi mask 1 based on channel magnitude
+     * four levels 0-3
+     *  - mask (-27 to 27) (reg 64,0x9900 to 67,0x990c)
+     *      [1 2 2 1] for -9.6 or [1 2 1] for +16
+     *  - enable_mask_ppm, all bins move with freq
+     *
+     *  - mask_select,    8 bits for rates (reg 67,0x990c)
+     *  - mask_rate_cntl, 8 bits for rates (reg 67,0x990c)
+     *      choose which mask to use mask or mask2
+     */
+
+    /*
+     * viterbi mask 2  2nd set for per data rate puncturing
+     * four levels 0-3
+     *  - mask_select, 8 bits for rates (reg 67)
+     *  - mask (-27 to 27) (reg 98,0x9988 to 101,0x9994)
+     *      [1 2 2 1] for -9.6 or [1 2 1] for +16
+     */
+    cur_vit_mask = 6100;
+    upper        = bin + 120;
+    lower        = bin - 120;
+
+    for (i = 0; i < 123; i++) {
+        if ((cur_vit_mask > lower) && (cur_vit_mask < upper)) {
+            if ((abs(cur_vit_mask - bin)) < 75) {
+                mask_amt = 1;
+            } else {
+                mask_amt = 0;
+            }
+            if (cur_vit_mask < 0) {
+                mask_m[abs(cur_vit_mask / 100)] = mask_amt;
+            } else {
+                mask_p[cur_vit_mask / 100] = mask_amt;
+            }
+        }
+        cur_vit_mask -= 100;
+    }
+
+    tmp_mask = (mask_m[46] << 30) | (mask_m[47] << 28)
+          | (mask_m[48] << 26) | (mask_m[49] << 24)
+          | (mask_m[50] << 22) | (mask_m[51] << 20)
+          | (mask_m[52] << 18) | (mask_m[53] << 16)
+          | (mask_m[54] << 14) | (mask_m[55] << 12)
+          | (mask_m[56] << 10) | (mask_m[57] <<  8)
+          | (mask_m[58] <<  6) | (mask_m[59] <<  4)
+          | (mask_m[60] <<  2) | (mask_m[61] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_BIN_MASK_1, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_VIT_MASK2_M_46_61, tmp_mask);
+
+    tmp_mask =             (mask_m[31] << 28)
+          | (mask_m[32] << 26) | (mask_m[33] << 24)
+          | (mask_m[34] << 22) | (mask_m[35] << 20)
+          | (mask_m[36] << 18) | (mask_m[37] << 16)
+          | (mask_m[48] << 14) | (mask_m[39] << 12)
+          | (mask_m[40] << 10) | (mask_m[41] <<  8)
+          | (mask_m[42] <<  6) | (mask_m[43] <<  4)
+          | (mask_m[44] <<  2) | (mask_m[45] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_BIN_MASK_2, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_MASK2_M_31_45, tmp_mask);
+
+    tmp_mask = (mask_m[16] << 30) | (mask_m[16] << 28)
+          | (mask_m[18] << 26) | (mask_m[18] << 24)
+          | (mask_m[20] << 22) | (mask_m[20] << 20)
+          | (mask_m[22] << 18) | (mask_m[22] << 16)
+          | (mask_m[24] << 14) | (mask_m[24] << 12)
+          | (mask_m[25] << 10) | (mask_m[26] <<  8)
+          | (mask_m[27] <<  6) | (mask_m[28] <<  4)
+          | (mask_m[29] <<  2) | (mask_m[30] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_BIN_MASK_3, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_MASK2_M_16_30, tmp_mask);
+
+    tmp_mask = (mask_m[ 0] << 30) | (mask_m[ 1] << 28)
+          | (mask_m[ 2] << 26) | (mask_m[ 3] << 24)
+          | (mask_m[ 4] << 22) | (mask_m[ 5] << 20)
+          | (mask_m[ 6] << 18) | (mask_m[ 7] << 16)
+          | (mask_m[ 8] << 14) | (mask_m[ 9] << 12)
+          | (mask_m[10] << 10) | (mask_m[11] <<  8)
+          | (mask_m[12] <<  6) | (mask_m[13] <<  4)
+          | (mask_m[14] <<  2) | (mask_m[15] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_MASK_CTL, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_MASK2_M_00_15, tmp_mask);
+
+    tmp_mask =             (mask_p[15] << 28)
+          | (mask_p[14] << 26) | (mask_p[13] << 24)
+          | (mask_p[12] << 22) | (mask_p[11] << 20)
+          | (mask_p[10] << 18) | (mask_p[ 9] << 16)
+          | (mask_p[ 8] << 14) | (mask_p[ 7] << 12)
+          | (mask_p[ 6] << 10) | (mask_p[ 5] <<  8)
+          | (mask_p[ 4] <<  6) | (mask_p[ 3] <<  4)
+          | (mask_p[ 2] <<  2) | (mask_p[ 1] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_1, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_MASK2_P_15_01, tmp_mask);
+
+    tmp_mask =             (mask_p[30] << 28)
+          | (mask_p[29] << 26) | (mask_p[28] << 24)
+          | (mask_p[27] << 22) | (mask_p[26] << 20)
+          | (mask_p[25] << 18) | (mask_p[24] << 16)
+          | (mask_p[23] << 14) | (mask_p[22] << 12)
+          | (mask_p[21] << 10) | (mask_p[20] <<  8)
+          | (mask_p[19] <<  6) | (mask_p[18] <<  4)
+          | (mask_p[17] <<  2) | (mask_p[16] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_2, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_MASK2_P_30_16, tmp_mask);
+
+    tmp_mask =             (mask_p[45] << 28)
+          | (mask_p[44] << 26) | (mask_p[43] << 24)
+          | (mask_p[42] << 22) | (mask_p[41] << 20)
+          | (mask_p[40] << 18) | (mask_p[39] << 16)
+          | (mask_p[38] << 14) | (mask_p[37] << 12)
+          | (mask_p[36] << 10) | (mask_p[35] <<  8)
+          | (mask_p[34] <<  6) | (mask_p[33] <<  4)
+          | (mask_p[32] <<  2) | (mask_p[31] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_3, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_MASK2_P_45_31, tmp_mask);
+
+    tmp_mask = (mask_p[61] << 30) | (mask_p[60] << 28)
+          | (mask_p[59] << 26) | (mask_p[58] << 24)
+          | (mask_p[57] << 22) | (mask_p[56] << 20)
+          | (mask_p[55] << 18) | (mask_p[54] << 16)
+          | (mask_p[53] << 14) | (mask_p[52] << 12)
+          | (mask_p[51] << 10) | (mask_p[50] <<  8)
+          | (mask_p[49] <<  6) | (mask_p[48] <<  4)
+          | (mask_p[47] <<  2) | (mask_p[46] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_4, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_MASK2_P_61_45, tmp_mask);
+}
+
 /*
  * Fill all software cached or static hardware state information.
  * Return failure if capabilities are to come from EEPROM and
diff --git a/sys/dev/ath/ath_hal/ar5416/ar5416_reset.c b/sys/dev/ath/ath_hal/ar5416/ar5416_reset.c
index 49b29c7..ac3d84a 100644
--- a/sys/dev/ath/ath_hal/ar5416/ar5416_reset.c
+++ b/sys/dev/ath/ath_hal/ar5416/ar5416_reset.c
@@ -27,9 +27,6 @@
 #include "ar5416/ar5416.h"
 #include "ar5416/ar5416reg.h"
 #include "ar5416/ar5416phy.h"
-#ifdef AH_SUPPORT_AR9280
-#include "ar5416/ar9280.h"
-#endif
 
 /* Eeprom versioning macros. Returns true if the version is equal or newer than the ver specified */ 
 #define	EEP_MINOR(_ah) \
@@ -55,10 +52,6 @@ static HAL_BOOL ar5416SetTransmitPower(struct ath_hal *ah,
 static HAL_BOOL	ar5416ChannelChange(struct ath_hal *, const struct ieee80211_channel *);
 #endif
 static void ar5416SetDeltaSlope(struct ath_hal *, const struct ieee80211_channel *);
-static void ar5416SpurMitigate(struct ath_hal *ah, const struct ieee80211_channel *chan);
-#ifdef AH_SUPPORT_AR9280
-static void ar9280SpurMitigate(struct ath_hal *ah, const struct ieee80211_channel *chan);
-#endif
 
 static HAL_BOOL ar5416SetResetPowerOn(struct ath_hal *ah);
 static HAL_BOOL ar5416SetReset(struct ath_hal *ah, int type);
@@ -120,11 +113,10 @@ ar5416Reset(struct ath_hal *ah, HAL_OPMODE opmode,
 	uint32_t saveDefAntenna, saveLedState;
 	uint32_t macStaId1;
 	uint16_t rfXpdGain[2];
-	u_int modesIndex, freqIndex;
 	HAL_STATUS ecode;
-	int i, regWrites = 0;
 	uint32_t powerVal, rssiThrReg;
 	uint32_t ackTpcPow, ctsTpcPow, chirpTpcPow;
+	int i;
 
 	OS_MARK(ah, AH_MARK_RESET, bChannelChange);
 
@@ -181,12 +173,6 @@ ar5416Reset(struct ath_hal *ah, HAL_OPMODE opmode,
 		(AR_MAC_LED_ASSOC | AR_MAC_LED_MODE |
 		 AR_MAC_LED_BLINK_THRESH_SEL | AR_MAC_LED_BLINK_SLOW);
 
-	/*
-	 * Adjust gain parameters before reset if
-	 * there's an outstanding gain updated.
-	 */
-	(void) ar5416GetRfgain(ah);
-
 	if (!ar5416ChipReset(ah, chan)) {
 		HALDEBUG(ah, HAL_DEBUG_ANY, "%s: chip reset failed\n", __func__);
 		FAIL(HAL_EIO);
@@ -195,67 +181,12 @@ ar5416Reset(struct ath_hal *ah, HAL_OPMODE opmode,
 	/* Restore bmiss rssi & count thresholds */
 	OS_REG_WRITE(ah, AR_RSSI_THR, rssiThrReg);
 
-	/* Setup the indices for the next set of register array writes */
-	/* XXX Ignore 11n dynamic mode on the AR5416 for the moment */
-	if (IEEE80211_IS_CHAN_2GHZ(chan)) {
-		freqIndex = 2;
-		if (IEEE80211_IS_CHAN_HT40(chan))
-			modesIndex = 3;
-		else if (IEEE80211_IS_CHAN_108G(chan))
-			modesIndex = 5;
-		else
-			modesIndex = 4;
-	} else {
-		freqIndex = 1;
-		if (IEEE80211_IS_CHAN_HT40(chan) ||
-		    IEEE80211_IS_CHAN_TURBO(chan))
-			modesIndex = 2;
-		else
-			modesIndex = 1;
-	}
-
 	OS_MARK(ah, AH_MARK_RESET_LINE, __LINE__);
 
-	/* Set correct Baseband to analog shift setting to access analog chips. */
-	OS_REG_WRITE(ah, AR_PHY(0), 0x00000007);
+	AH5416(ah)->ah_writeIni(ah, chan);
 
-	 /*
-	 * Write addac shifts
-	 */
-	OS_REG_WRITE(ah, AR_PHY_ADC_SERIAL_CTL, AR_PHY_SEL_EXTERNAL_RADIO);
-#if 0
-	/* NB: only required for Sowl */
-	ar5416EepromSetAddac(ah, chan);
-#endif
-	regWrites = ath_hal_ini_write(ah, &AH5416(ah)->ah_ini_addac, 1,
-	    regWrites);
-	OS_REG_WRITE(ah, AR_PHY_ADC_SERIAL_CTL, AR_PHY_SEL_INTERNAL_ADDAC);
-
-	/* XXX Merlin ini fixups */
-	/* XXX Merlin 100us delay for shift registers */
-	regWrites = ath_hal_ini_write(ah, &ahp->ah_ini_modes, modesIndex,
-	    regWrites);
-#ifdef AH_SUPPORT_AR9280
-	if (AR_SREV_MERLIN_20_OR_LATER(ah)) {
-		regWrites = ath_hal_ini_write(ah, &AH9280(ah)->ah_ini_rxgain,
-		    modesIndex, regWrites);
-		regWrites = ath_hal_ini_write(ah, &AH9280(ah)->ah_ini_txgain,
-		    modesIndex, regWrites);
-	}
-#endif
-	/* XXX Merlin 100us delay for shift registers */
-	regWrites = ath_hal_ini_write(ah, &ahp->ah_ini_common, 1, regWrites);
 	/* Setup 11n MAC/Phy mode registers */
 	ar5416Set11nRegs(ah, chan);	
-	/* XXX updated regWrites? */
-	ahp->ah_rfHal->writeRegs(ah, modesIndex, freqIndex, regWrites);
-#ifdef AH_SUPPORT_AR9280
-	if (AR_SREV_MERLIN_20(ah) && IS_5GHZ_FAST_CLOCK_EN(ah, chan)) {
-		/* 5GHz channels w/ Fast Clock use different modal values */
-		regWrites = ath_hal_ini_write(ah, &AH9280(ah)->ah_ini_xmodes,
-		    modesIndex, regWrites);
-	}
-#endif
 
 	OS_MARK(ah, AH_MARK_RESET_LINE, __LINE__);
 
@@ -300,7 +231,8 @@ ar5416Reset(struct ath_hal *ah, HAL_OPMODE opmode,
 	}
 
 	/* Write the analog registers */
-	if (!ahp->ah_rfHal->setRfRegs(ah, chan, freqIndex, rfXpdGain)) {
+	if (!ahp->ah_rfHal->setRfRegs(ah, chan,
+	    IEEE80211_IS_CHAN_2GHZ(chan) ? 2: 1, rfXpdGain)) {
 		HALDEBUG(ah, HAL_DEBUG_ANY,
 		    "%s: ar5212SetRfRegs failed\n", __func__);
 		FAIL(HAL_EIO);
@@ -310,12 +242,7 @@ ar5416Reset(struct ath_hal *ah, HAL_OPMODE opmode,
 	if (IEEE80211_IS_CHAN_OFDM(chan)|| IEEE80211_IS_CHAN_HT(chan))
 		ar5416SetDeltaSlope(ah, chan);
 
-#ifdef AH_SUPPORT_AR9280
-	if (AR_SREV_MERLIN_10_OR_LATER(ah))
-		ar9280SpurMitigate(ah, chan);
-	else
-#endif
-		ar5416SpurMitigate(ah, chan);
+	AH5416(ah)->ah_spurMitigate(ah, chan);
 
 	/* Setup board specific options for EEPROM version 3 */
 	if (!ar5416SetBoardValues(ah, chan)) {
@@ -677,8 +604,6 @@ ar5416InitUserSettings(struct ath_hal *ah)
 HAL_BOOL
 ar5416ChipReset(struct ath_hal *ah, const struct ieee80211_channel *chan)
 {
-	uint32_t rfMode = 0;
-
 	OS_MARK(ah, AH_MARK_CHIPRESET, chan ? chan->ic_freq : 0);
 	/*
 	 * Warm reset is optimistic.
@@ -705,9 +630,11 @@ ar5416ChipReset(struct ath_hal *ah, const struct ieee80211_channel *chan)
 	 * radio device.
 	 */
 	if (chan != AH_NULL) { 
+		uint32_t rfMode;
+
 		/* treat channel B as channel G , no  B mode suport in owl */
-		rfMode |= IEEE80211_IS_CHAN_CCK(chan) ?
-			AR_PHY_MODE_DYNAMIC : AR_PHY_MODE_OFDM;
+		rfMode = IEEE80211_IS_CHAN_CCK(chan) ?
+		    AR_PHY_MODE_DYNAMIC : AR_PHY_MODE_OFDM;
 		if (AR_SREV_MERLIN_20(ah) && IS_5GHZ_FAST_CLOCK_EN(ah, chan)) {
 			/* phy mode bits for 5GHz channels require Fast Clock */
 			rfMode |= AR_PHY_MODE_DYNAMIC
@@ -804,558 +731,6 @@ ar5416SetDeltaSlope(struct ath_hal *ah, const struct ieee80211_channel *chan)
 }
 
 /*
- * Convert to baseband spur frequency given input channel frequency 
- * and compute register settings below.
- */
-#define SPUR_RSSI_THRESH 40
-
-static void
-ar5416SpurMitigate(struct ath_hal *ah, const struct ieee80211_channel *chan)
-{
-    uint16_t freq = ath_hal_gethwchannel(ah, chan);
-    static const int pilot_mask_reg[4] = { AR_PHY_TIMING7, AR_PHY_TIMING8,
-                AR_PHY_PILOT_MASK_01_30, AR_PHY_PILOT_MASK_31_60 };
-    static const int chan_mask_reg[4] = { AR_PHY_TIMING9, AR_PHY_TIMING10,
-                AR_PHY_CHANNEL_MASK_01_30, AR_PHY_CHANNEL_MASK_31_60 };
-    static const int inc[4] = { 0, 100, 0, 0 };
-
-    int bb_spur = AR_NO_SPUR;
-    int bin, cur_bin;
-    int spur_freq_sd;
-    int spur_delta_phase;
-    int denominator;
-    int upper, lower, cur_vit_mask;
-    int tmp, new;
-    int i;
-
-    int8_t mask_m[123];
-    int8_t mask_p[123];
-    int8_t mask_amt;
-    int tmp_mask;
-    int cur_bb_spur;
-    HAL_BOOL is2GHz = IEEE80211_IS_CHAN_2GHZ(chan);
-
-    OS_MEMZERO(mask_m, sizeof(mask_m));
-    OS_MEMZERO(mask_p, sizeof(mask_p));
-
-    /*
-     * Need to verify range +/- 9.5 for static ht20, otherwise spur
-     * is out-of-band and can be ignored.
-     */
-    /* XXX ath9k changes */
-    for (i = 0; i < AR5416_EEPROM_MODAL_SPURS; i++) {
-        cur_bb_spur = ath_hal_getSpurChan(ah, i, is2GHz);
-        if (AR_NO_SPUR == cur_bb_spur)
-            break;
-        cur_bb_spur = cur_bb_spur - (freq * 10);
-        if ((cur_bb_spur > -95) && (cur_bb_spur < 95)) {
-            bb_spur = cur_bb_spur;
-            break;
-        }
-    }
-    if (AR_NO_SPUR == bb_spur)
-        return;
-
-    bin = bb_spur * 32;
-
-    tmp = OS_REG_READ(ah, AR_PHY_TIMING_CTRL4_CHAIN(0));
-    new = tmp | (AR_PHY_TIMING_CTRL4_ENABLE_SPUR_RSSI |
-        AR_PHY_TIMING_CTRL4_ENABLE_SPUR_FILTER |
-        AR_PHY_TIMING_CTRL4_ENABLE_CHAN_MASK |
-        AR_PHY_TIMING_CTRL4_ENABLE_PILOT_MASK);
-
-    OS_REG_WRITE(ah, AR_PHY_TIMING_CTRL4_CHAIN(0), new);
-
-    new = (AR_PHY_SPUR_REG_MASK_RATE_CNTL |
-        AR_PHY_SPUR_REG_ENABLE_MASK_PPM |
-        AR_PHY_SPUR_REG_MASK_RATE_SELECT |
-        AR_PHY_SPUR_REG_ENABLE_VIT_SPUR_RSSI |
-        SM(SPUR_RSSI_THRESH, AR_PHY_SPUR_REG_SPUR_RSSI_THRESH));
-    OS_REG_WRITE(ah, AR_PHY_SPUR_REG, new);
-    /*
-     * Should offset bb_spur by +/- 10 MHz for dynamic 2040 MHz
-     * config, no offset for HT20.
-     * spur_delta_phase = bb_spur/40 * 2**21 for static ht20,
-     * /80 for dyn2040.
-     */
-    spur_delta_phase = ((bb_spur * 524288) / 100) &
-        AR_PHY_TIMING11_SPUR_DELTA_PHASE;
-    /*
-     * in 11A mode the denominator of spur_freq_sd should be 40 and
-     * it should be 44 in 11G
-     */
-    denominator = IEEE80211_IS_CHAN_2GHZ(chan) ? 440 : 400;
-    spur_freq_sd = ((bb_spur * 2048) / denominator) & 0x3ff;
-
-    new = (AR_PHY_TIMING11_USE_SPUR_IN_AGC |
-        SM(spur_freq_sd, AR_PHY_TIMING11_SPUR_FREQ_SD) |
-        SM(spur_delta_phase, AR_PHY_TIMING11_SPUR_DELTA_PHASE));
-    OS_REG_WRITE(ah, AR_PHY_TIMING11, new);
-
-
-    /*
-     * ============================================
-     * pilot mask 1 [31:0] = +6..-26, no 0 bin
-     * pilot mask 2 [19:0] = +26..+7
-     *
-     * channel mask 1 [31:0] = +6..-26, no 0 bin
-     * channel mask 2 [19:0] = +26..+7
-     */
-    //cur_bin = -26;
-    cur_bin = -6000;
-    upper = bin + 100;
-    lower = bin - 100;
-
-    for (i = 0; i < 4; i++) {
-        int pilot_mask = 0;
-        int chan_mask  = 0;
-        int bp         = 0;
-        for (bp = 0; bp < 30; bp++) {
-            if ((cur_bin > lower) && (cur_bin < upper)) {
-                pilot_mask = pilot_mask | 0x1 << bp;
-                chan_mask  = chan_mask | 0x1 << bp;
-            }
-            cur_bin += 100;
-        }
-        cur_bin += inc[i];
-        OS_REG_WRITE(ah, pilot_mask_reg[i], pilot_mask);
-        OS_REG_WRITE(ah, chan_mask_reg[i], chan_mask);
-    }
-
-    /* =================================================
-     * viterbi mask 1 based on channel magnitude
-     * four levels 0-3
-     *  - mask (-27 to 27) (reg 64,0x9900 to 67,0x990c)
-     *      [1 2 2 1] for -9.6 or [1 2 1] for +16
-     *  - enable_mask_ppm, all bins move with freq
-     *
-     *  - mask_select,    8 bits for rates (reg 67,0x990c)
-     *  - mask_rate_cntl, 8 bits for rates (reg 67,0x990c)
-     *      choose which mask to use mask or mask2
-     */
-
-    /*
-     * viterbi mask 2  2nd set for per data rate puncturing
-     * four levels 0-3
-     *  - mask_select, 8 bits for rates (reg 67)
-     *  - mask (-27 to 27) (reg 98,0x9988 to 101,0x9994)
-     *      [1 2 2 1] for -9.6 or [1 2 1] for +16
-     */
-    cur_vit_mask = 6100;
-    upper        = bin + 120;
-    lower        = bin - 120;
-
-    for (i = 0; i < 123; i++) {
-        if ((cur_vit_mask > lower) && (cur_vit_mask < upper)) {
-            if ((abs(cur_vit_mask - bin)) < 75) {
-                mask_amt = 1;
-            } else {
-                mask_amt = 0;
-            }
-            if (cur_vit_mask < 0) {
-                mask_m[abs(cur_vit_mask / 100)] = mask_amt;
-            } else {
-                mask_p[cur_vit_mask / 100] = mask_amt;
-            }
-        }
-        cur_vit_mask -= 100;
-    }
-
-    tmp_mask = (mask_m[46] << 30) | (mask_m[47] << 28)
-          | (mask_m[48] << 26) | (mask_m[49] << 24)
-          | (mask_m[50] << 22) | (mask_m[51] << 20)
-          | (mask_m[52] << 18) | (mask_m[53] << 16)
-          | (mask_m[54] << 14) | (mask_m[55] << 12)
-          | (mask_m[56] << 10) | (mask_m[57] <<  8)
-          | (mask_m[58] <<  6) | (mask_m[59] <<  4)
-          | (mask_m[60] <<  2) | (mask_m[61] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_BIN_MASK_1, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_VIT_MASK2_M_46_61, tmp_mask);
-
-    tmp_mask =             (mask_m[31] << 28)
-          | (mask_m[32] << 26) | (mask_m[33] << 24)
-          | (mask_m[34] << 22) | (mask_m[35] << 20)
-          | (mask_m[36] << 18) | (mask_m[37] << 16)
-          | (mask_m[48] << 14) | (mask_m[39] << 12)
-          | (mask_m[40] << 10) | (mask_m[41] <<  8)
-          | (mask_m[42] <<  6) | (mask_m[43] <<  4)
-          | (mask_m[44] <<  2) | (mask_m[45] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_BIN_MASK_2, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_MASK2_M_31_45, tmp_mask);
-
-    tmp_mask = (mask_m[16] << 30) | (mask_m[16] << 28)
-          | (mask_m[18] << 26) | (mask_m[18] << 24)
-          | (mask_m[20] << 22) | (mask_m[20] << 20)
-          | (mask_m[22] << 18) | (mask_m[22] << 16)
-          | (mask_m[24] << 14) | (mask_m[24] << 12)
-          | (mask_m[25] << 10) | (mask_m[26] <<  8)
-          | (mask_m[27] <<  6) | (mask_m[28] <<  4)
-          | (mask_m[29] <<  2) | (mask_m[30] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_BIN_MASK_3, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_MASK2_M_16_30, tmp_mask);
-
-    tmp_mask = (mask_m[ 0] << 30) | (mask_m[ 1] << 28)
-          | (mask_m[ 2] << 26) | (mask_m[ 3] << 24)
-          | (mask_m[ 4] << 22) | (mask_m[ 5] << 20)
-          | (mask_m[ 6] << 18) | (mask_m[ 7] << 16)
-          | (mask_m[ 8] << 14) | (mask_m[ 9] << 12)
-          | (mask_m[10] << 10) | (mask_m[11] <<  8)
-          | (mask_m[12] <<  6) | (mask_m[13] <<  4)
-          | (mask_m[14] <<  2) | (mask_m[15] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_MASK_CTL, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_MASK2_M_00_15, tmp_mask);
-
-    tmp_mask =             (mask_p[15] << 28)
-          | (mask_p[14] << 26) | (mask_p[13] << 24)
-          | (mask_p[12] << 22) | (mask_p[11] << 20)
-          | (mask_p[10] << 18) | (mask_p[ 9] << 16)
-          | (mask_p[ 8] << 14) | (mask_p[ 7] << 12)
-          | (mask_p[ 6] << 10) | (mask_p[ 5] <<  8)
-          | (mask_p[ 4] <<  6) | (mask_p[ 3] <<  4)
-          | (mask_p[ 2] <<  2) | (mask_p[ 1] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_1, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_MASK2_P_15_01, tmp_mask);
-
-    tmp_mask =             (mask_p[30] << 28)
-          | (mask_p[29] << 26) | (mask_p[28] << 24)
-          | (mask_p[27] << 22) | (mask_p[26] << 20)
-          | (mask_p[25] << 18) | (mask_p[24] << 16)
-          | (mask_p[23] << 14) | (mask_p[22] << 12)
-          | (mask_p[21] << 10) | (mask_p[20] <<  8)
-          | (mask_p[19] <<  6) | (mask_p[18] <<  4)
-          | (mask_p[17] <<  2) | (mask_p[16] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_2, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_MASK2_P_30_16, tmp_mask);
-
-    tmp_mask =             (mask_p[45] << 28)
-          | (mask_p[44] << 26) | (mask_p[43] << 24)
-          | (mask_p[42] << 22) | (mask_p[41] << 20)
-          | (mask_p[40] << 18) | (mask_p[39] << 16)
-          | (mask_p[38] << 14) | (mask_p[37] << 12)
-          | (mask_p[36] << 10) | (mask_p[35] <<  8)
-          | (mask_p[34] <<  6) | (mask_p[33] <<  4)
-          | (mask_p[32] <<  2) | (mask_p[31] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_3, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_MASK2_P_45_31, tmp_mask);
-
-    tmp_mask = (mask_p[61] << 30) | (mask_p[60] << 28)
-          | (mask_p[59] << 26) | (mask_p[58] << 24)
-          | (mask_p[57] << 22) | (mask_p[56] << 20)
-          | (mask_p[55] << 18) | (mask_p[54] << 16)
-          | (mask_p[53] << 14) | (mask_p[52] << 12)
-          | (mask_p[51] << 10) | (mask_p[50] <<  8)
-          | (mask_p[49] <<  6) | (mask_p[48] <<  4)
-          | (mask_p[47] <<  2) | (mask_p[46] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_4, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_MASK2_P_61_45, tmp_mask);
-}
-
-#ifdef AH_SUPPORT_AR9280
-#define	AR_BASE_FREQ_2GHZ	2300
-#define	AR_BASE_FREQ_5GHZ	4900
-#define	AR_SPUR_FEEQ_BOUND_HT40	19
-#define	AR_SPUR_FEEQ_BOUND_HT20	10
-
-static void
-ar9280SpurMitigate(struct ath_hal *ah, const struct ieee80211_channel *chan)
-{
-    static const int pilot_mask_reg[4] = { AR_PHY_TIMING7, AR_PHY_TIMING8,
-                AR_PHY_PILOT_MASK_01_30, AR_PHY_PILOT_MASK_31_60 };
-    static const int chan_mask_reg[4] = { AR_PHY_TIMING9, AR_PHY_TIMING10,
-                AR_PHY_CHANNEL_MASK_01_30, AR_PHY_CHANNEL_MASK_31_60 };
-    static int inc[4] = { 0, 100, 0, 0 };
-
-    int bb_spur = AR_NO_SPUR;
-    int freq;
-    int bin, cur_bin;
-    int bb_spur_off, spur_subchannel_sd;
-    int spur_freq_sd;
-    int spur_delta_phase;
-    int denominator;
-    int upper, lower, cur_vit_mask;
-    int tmp, newVal;
-    int i;
-    CHAN_CENTERS centers;
-
-    int8_t mask_m[123];
-    int8_t mask_p[123];
-    int8_t mask_amt;
-    int tmp_mask;
-    int cur_bb_spur;
-    HAL_BOOL is2GHz = IEEE80211_IS_CHAN_2GHZ(chan);
-
-    OS_MEMZERO(&mask_m, sizeof(int8_t) * 123);
-    OS_MEMZERO(&mask_p, sizeof(int8_t) * 123);
-
-    ar5416GetChannelCenters(ah, chan, &centers);
-    freq = centers.synth_center;
-
-    /*
-     * Need to verify range +/- 9.38 for static ht20 and +/- 18.75 for ht40,
-     * otherwise spur is out-of-band and can be ignored.
-     */
-    for (i = 0; i < AR5416_EEPROM_MODAL_SPURS; i++) {
-        cur_bb_spur = ath_hal_getSpurChan(ah, i, is2GHz);
-        /* Get actual spur freq in MHz from EEPROM read value */ 
-        if (is2GHz) {
-            cur_bb_spur =  (cur_bb_spur / 10) + AR_BASE_FREQ_2GHZ;
-        } else {
-            cur_bb_spur =  (cur_bb_spur / 10) + AR_BASE_FREQ_5GHZ;
-        }
-
-        if (AR_NO_SPUR == cur_bb_spur)
-            break;
-        cur_bb_spur = cur_bb_spur - freq;
-
-        if (IEEE80211_IS_CHAN_HT40(chan)) {
-            if ((cur_bb_spur > -AR_SPUR_FEEQ_BOUND_HT40) && 
-                (cur_bb_spur < AR_SPUR_FEEQ_BOUND_HT40)) {
-                bb_spur = cur_bb_spur;
-                break;
-            }
-        } else if ((cur_bb_spur > -AR_SPUR_FEEQ_BOUND_HT20) &&
-                   (cur_bb_spur < AR_SPUR_FEEQ_BOUND_HT20)) {
-            bb_spur = cur_bb_spur;
-            break;
-        }
-    }
-
-    if (AR_NO_SPUR == bb_spur) {
-#if 1
-        /*
-         * MRC CCK can interfere with beacon detection and cause deaf/mute.
-         * Disable MRC CCK for now.
-         */
-        OS_REG_CLR_BIT(ah, AR_PHY_FORCE_CLKEN_CCK, AR_PHY_FORCE_CLKEN_CCK_MRC_MUX);
-#else
-        /* Enable MRC CCK if no spur is found in this channel. */
-        OS_REG_SET_BIT(ah, AR_PHY_FORCE_CLKEN_CCK, AR_PHY_FORCE_CLKEN_CCK_MRC_MUX);
-#endif
-        return;
-    } else {
-        /* 
-         * For Merlin, spur can break CCK MRC algorithm. Disable CCK MRC if spur
-         * is found in this channel.
-         */
-        OS_REG_CLR_BIT(ah, AR_PHY_FORCE_CLKEN_CCK, AR_PHY_FORCE_CLKEN_CCK_MRC_MUX);
-    }
-
-    bin = bb_spur * 320;
-
-    tmp = OS_REG_READ(ah, AR_PHY_TIMING_CTRL4_CHAIN(0));
-
-    newVal = tmp | (AR_PHY_TIMING_CTRL4_ENABLE_SPUR_RSSI |
-        AR_PHY_TIMING_CTRL4_ENABLE_SPUR_FILTER |
-        AR_PHY_TIMING_CTRL4_ENABLE_CHAN_MASK |
-        AR_PHY_TIMING_CTRL4_ENABLE_PILOT_MASK);
-    OS_REG_WRITE(ah, AR_PHY_TIMING_CTRL4_CHAIN(0), newVal);
-
-    newVal = (AR_PHY_SPUR_REG_MASK_RATE_CNTL |
-        AR_PHY_SPUR_REG_ENABLE_MASK_PPM |
-        AR_PHY_SPUR_REG_MASK_RATE_SELECT |
-        AR_PHY_SPUR_REG_ENABLE_VIT_SPUR_RSSI |
-        SM(SPUR_RSSI_THRESH, AR_PHY_SPUR_REG_SPUR_RSSI_THRESH));
-    OS_REG_WRITE(ah, AR_PHY_SPUR_REG, newVal);
-
-    /* Pick control or extn channel to cancel the spur */
-    if (IEEE80211_IS_CHAN_HT40(chan)) {
-        if (bb_spur < 0) {
-            spur_subchannel_sd = 1;
-            bb_spur_off = bb_spur + 10;
-        } else {
-            spur_subchannel_sd = 0;
-            bb_spur_off = bb_spur - 10;
-        }
-    } else {
-        spur_subchannel_sd = 0;
-        bb_spur_off = bb_spur;
-    }
-
-    /*
-     * spur_delta_phase = bb_spur/40 * 2**21 for static ht20,
-     * /80 for dyn2040.
-     */
-    if (IEEE80211_IS_CHAN_HT40(chan))
-        spur_delta_phase = ((bb_spur * 262144) / 10) & AR_PHY_TIMING11_SPUR_DELTA_PHASE;    
-    else
-        spur_delta_phase = ((bb_spur * 524288) / 10) & AR_PHY_TIMING11_SPUR_DELTA_PHASE;
-
-    /*
-     * in 11A mode the denominator of spur_freq_sd should be 40 and
-     * it should be 44 in 11G
-     */
-    denominator = IEEE80211_IS_CHAN_2GHZ(chan) ? 44 : 40;
-    spur_freq_sd = ((bb_spur_off * 2048) / denominator) & 0x3ff;
-
-    newVal = (AR_PHY_TIMING11_USE_SPUR_IN_AGC |
-        SM(spur_freq_sd, AR_PHY_TIMING11_SPUR_FREQ_SD) |
-        SM(spur_delta_phase, AR_PHY_TIMING11_SPUR_DELTA_PHASE));
-    OS_REG_WRITE(ah, AR_PHY_TIMING11, newVal);
-
-    /* Choose to cancel between control and extension channels */
-    newVal = spur_subchannel_sd << AR_PHY_SFCORR_SPUR_SUBCHNL_SD_S;
-    OS_REG_WRITE(ah, AR_PHY_SFCORR_EXT, newVal);
-
-    /*
-     * ============================================
-     * Set Pilot and Channel Masks
-     *
-     * pilot mask 1 [31:0] = +6..-26, no 0 bin
-     * pilot mask 2 [19:0] = +26..+7
-     *
-     * channel mask 1 [31:0] = +6..-26, no 0 bin
-     * channel mask 2 [19:0] = +26..+7
-     */
-    cur_bin = -6000;
-    upper = bin + 100;
-    lower = bin - 100;
-
-    for (i = 0; i < 4; i++) {
-        int pilot_mask = 0;
-        int chan_mask  = 0;
-        int bp         = 0;
-        for (bp = 0; bp < 30; bp++) {
-            if ((cur_bin > lower) && (cur_bin < upper)) {
-                pilot_mask = pilot_mask | 0x1 << bp;
-                chan_mask  = chan_mask | 0x1 << bp;
-            }
-            cur_bin += 100;
-        }
-        cur_bin += inc[i];
-        OS_REG_WRITE(ah, pilot_mask_reg[i], pilot_mask);
-        OS_REG_WRITE(ah, chan_mask_reg[i], chan_mask);
-    }
-
-    /* =================================================
-     * viterbi mask 1 based on channel magnitude
-     * four levels 0-3
-     *  - mask (-27 to 27) (reg 64,0x9900 to 67,0x990c)
-     *      [1 2 2 1] for -9.6 or [1 2 1] for +16
-     *  - enable_mask_ppm, all bins move with freq
-     *
-     *  - mask_select,    8 bits for rates (reg 67,0x990c)
-     *  - mask_rate_cntl, 8 bits for rates (reg 67,0x990c)
-     *      choose which mask to use mask or mask2
-     */
-
-    /*
-     * viterbi mask 2  2nd set for per data rate puncturing
-     * four levels 0-3
-     *  - mask_select, 8 bits for rates (reg 67)
-     *  - mask (-27 to 27) (reg 98,0x9988 to 101,0x9994)
-     *      [1 2 2 1] for -9.6 or [1 2 1] for +16
-     */
-    cur_vit_mask = 6100;
-    upper        = bin + 120;
-    lower        = bin - 120;
-
-    for (i = 0; i < 123; i++) {
-        if ((cur_vit_mask > lower) && (cur_vit_mask < upper)) {
-            if ((abs(cur_vit_mask - bin)) < 75) {
-                mask_amt = 1;
-            } else {
-                mask_amt = 0;
-            }
-            if (cur_vit_mask < 0) {
-                mask_m[abs(cur_vit_mask / 100)] = mask_amt;
-            } else {
-                mask_p[cur_vit_mask / 100] = mask_amt;
-            }
-        }
-        cur_vit_mask -= 100;
-    }
-
-    tmp_mask = (mask_m[46] << 30) | (mask_m[47] << 28)
-          | (mask_m[48] << 26) | (mask_m[49] << 24)
-          | (mask_m[50] << 22) | (mask_m[51] << 20)
-          | (mask_m[52] << 18) | (mask_m[53] << 16)
-          | (mask_m[54] << 14) | (mask_m[55] << 12)
-          | (mask_m[56] << 10) | (mask_m[57] <<  8)
-          | (mask_m[58] <<  6) | (mask_m[59] <<  4)
-          | (mask_m[60] <<  2) | (mask_m[61] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_BIN_MASK_1, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_VIT_MASK2_M_46_61, tmp_mask);
-
-    tmp_mask =             (mask_m[31] << 28)
-          | (mask_m[32] << 26) | (mask_m[33] << 24)
-          | (mask_m[34] << 22) | (mask_m[35] << 20)
-          | (mask_m[36] << 18) | (mask_m[37] << 16)
-          | (mask_m[48] << 14) | (mask_m[39] << 12)
-          | (mask_m[40] << 10) | (mask_m[41] <<  8)
-          | (mask_m[42] <<  6) | (mask_m[43] <<  4)
-          | (mask_m[44] <<  2) | (mask_m[45] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_BIN_MASK_2, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_MASK2_M_31_45, tmp_mask);
-
-    tmp_mask = (mask_m[16] << 30) | (mask_m[16] << 28)
-          | (mask_m[18] << 26) | (mask_m[18] << 24)
-          | (mask_m[20] << 22) | (mask_m[20] << 20)
-          | (mask_m[22] << 18) | (mask_m[22] << 16)
-          | (mask_m[24] << 14) | (mask_m[24] << 12)
-          | (mask_m[25] << 10) | (mask_m[26] <<  8)
-          | (mask_m[27] <<  6) | (mask_m[28] <<  4)
-          | (mask_m[29] <<  2) | (mask_m[30] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_BIN_MASK_3, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_MASK2_M_16_30, tmp_mask);
-
-    tmp_mask = (mask_m[ 0] << 30) | (mask_m[ 1] << 28)
-          | (mask_m[ 2] << 26) | (mask_m[ 3] << 24)
-          | (mask_m[ 4] << 22) | (mask_m[ 5] << 20)
-          | (mask_m[ 6] << 18) | (mask_m[ 7] << 16)
-          | (mask_m[ 8] << 14) | (mask_m[ 9] << 12)
-          | (mask_m[10] << 10) | (mask_m[11] <<  8)
-          | (mask_m[12] <<  6) | (mask_m[13] <<  4)
-          | (mask_m[14] <<  2) | (mask_m[15] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_MASK_CTL, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_MASK2_M_00_15, tmp_mask);
-
-    tmp_mask =             (mask_p[15] << 28)
-          | (mask_p[14] << 26) | (mask_p[13] << 24)
-          | (mask_p[12] << 22) | (mask_p[11] << 20)
-          | (mask_p[10] << 18) | (mask_p[ 9] << 16)
-          | (mask_p[ 8] << 14) | (mask_p[ 7] << 12)
-          | (mask_p[ 6] << 10) | (mask_p[ 5] <<  8)
-          | (mask_p[ 4] <<  6) | (mask_p[ 3] <<  4)
-          | (mask_p[ 2] <<  2) | (mask_p[ 1] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_1, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_MASK2_P_15_01, tmp_mask);
-
-    tmp_mask =             (mask_p[30] << 28)
-          | (mask_p[29] << 26) | (mask_p[28] << 24)
-          | (mask_p[27] << 22) | (mask_p[26] << 20)
-          | (mask_p[25] << 18) | (mask_p[24] << 16)
-          | (mask_p[23] << 14) | (mask_p[22] << 12)
-          | (mask_p[21] << 10) | (mask_p[20] <<  8)
-          | (mask_p[19] <<  6) | (mask_p[18] <<  4)
-          | (mask_p[17] <<  2) | (mask_p[16] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_2, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_MASK2_P_30_16, tmp_mask);
-
-    tmp_mask =             (mask_p[45] << 28)
-          | (mask_p[44] << 26) | (mask_p[43] << 24)
-          | (mask_p[42] << 22) | (mask_p[41] << 20)
-          | (mask_p[40] << 18) | (mask_p[39] << 16)
-          | (mask_p[38] << 14) | (mask_p[37] << 12)
-          | (mask_p[36] << 10) | (mask_p[35] <<  8)
-          | (mask_p[34] <<  6) | (mask_p[33] <<  4)
-          | (mask_p[32] <<  2) | (mask_p[31] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_3, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_MASK2_P_45_31, tmp_mask);
-
-    tmp_mask = (mask_p[61] << 30) | (mask_p[60] << 28)
-          | (mask_p[59] << 26) | (mask_p[58] << 24)
-          | (mask_p[57] << 22) | (mask_p[56] << 20)
-          | (mask_p[55] << 18) | (mask_p[54] << 16)
-          | (mask_p[53] << 14) | (mask_p[52] << 12)
-          | (mask_p[51] << 10) | (mask_p[50] <<  8)
-          | (mask_p[49] <<  6) | (mask_p[48] <<  4)
-          | (mask_p[47] <<  2) | (mask_p[46] <<  0);
-    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_4, tmp_mask);
-    OS_REG_WRITE(ah, AR_PHY_MASK2_P_61_45, tmp_mask);
-}
-#endif /* AH_SUPPORT_AR9280 */
-
-/*
  * Set a limit on the overall output power.  Used for dynamic
  * transmit power control and the like.
  *
@@ -1620,21 +995,14 @@ ar5416PhyDisable(struct ath_hal *ah)
 HAL_BOOL
 ar5416SetResetReg(struct ath_hal *ah, uint32_t type)
 {
-	/*
-	 * Set force wake
-	 */
-	OS_REG_WRITE(ah, AR_RTC_FORCE_WAKE,
-	     AR_RTC_FORCE_WAKE_EN | AR_RTC_FORCE_WAKE_ON_INT);
-
 	switch (type) {
 	case HAL_RESET_POWER_ON:
 		return ar5416SetResetPowerOn(ah);
-		break;
 	case HAL_RESET_WARM:
 	case HAL_RESET_COLD:
 		return ar5416SetReset(ah, type);
-		break;
 	default:
+		HALASSERT(AH_FALSE);
 		return AH_FALSE;
 	}
 }
@@ -1703,11 +1071,11 @@ ar5416SetReset(struct ath_hal *ah, int type)
     case HAL_RESET_WARM:
             OS_REG_WRITE(ah, AR_RTC_RC, AR_RTC_RC_MAC_WARM);
             break;
-        case HAL_RESET_COLD:
+    case HAL_RESET_COLD:
             OS_REG_WRITE(ah, AR_RTC_RC, AR_RTC_RC_MAC_WARM|AR_RTC_RC_MAC_COLD);
             break;
-        default:
-            HALASSERT(0);
+    default:
+            HALASSERT(AH_FALSE);
             break;
     }
 
@@ -1755,7 +1123,8 @@ ar5416InitPLL(struct ath_hal *ah, const struct ieee80211_channel *chan)
 {
 	uint32_t pll;
 
-	if (AR_SREV_MERLIN_20(ah) && chan != AH_NULL) {
+	if (AR_SREV_MERLIN_20(ah) &&
+	    chan != AH_NULL && IEEE80211_IS_CHAN_5GHZ(chan)) {
 		/*
 		 * PLL WAR for Merlin 2.0/2.1
 		 * When doing fast clock, set PLL to 0x142c
diff --git a/sys/dev/ath/ath_hal/ar5416/ar9280.c b/sys/dev/ath/ath_hal/ar5416/ar9280.c
new file mode 100644
index 0000000..72720f7
--- /dev/null
+++ b/sys/dev/ath/ath_hal/ar5416/ar9280.c
@@ -0,0 +1,361 @@
+/*
+ * Copyright (c) 2008-2009 Sam Leffler, Errno Consulting
+ * Copyright (c) 2008 Atheros Communications, Inc.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * $FreeBSD$
+ */
+#include "opt_ah.h"
+
+/*
+ * NB: Merlin and later have a simpler RF backend.
+ */
+#include "ah.h"
+#include "ah_internal.h"
+
+#include "ah_eeprom_v14.h"
+
+#include "ar5416/ar9280.h"
+#include "ar5416/ar5416reg.h"
+#include "ar5416/ar5416phy.h"
+
+#define N(a)    (sizeof(a)/sizeof(a[0]))
+
+struct ar9280State {
+	RF_HAL_FUNCS	base;		/* public state, must be first */
+	uint16_t	pcdacTable[1];	/* XXX */
+};
+#define	AR9280(ah)	((struct ar9280State *) AH5212(ah)->ah_rfHal)
+
+static HAL_BOOL ar9280GetChannelMaxMinPower(struct ath_hal *,
+	const struct ieee80211_channel *, int16_t *maxPow,int16_t *minPow);
+int16_t ar9280GetNfAdjust(struct ath_hal *ah, const HAL_CHANNEL_INTERNAL *c);
+
+static void
+ar9280WriteRegs(struct ath_hal *ah, u_int modesIndex, u_int freqIndex,
+	int writes)
+{
+	(void) ath_hal_ini_write(ah, &AH5416(ah)->ah_ini_bb_rfgain,
+		freqIndex, writes);
+}
+
+/*
+ * Take the MHz channel value and set the Channel value
+ *
+ * ASSUMES: Writes enabled to analog bus
+ *
+ * Actual Expression,
+ *
+ * For 2GHz channel, 
+ * Channel Frequency = (3/4) * freq_ref * (chansel[8:0] + chanfrac[16:0]/2^17) 
+ * (freq_ref = 40MHz)
+ *
+ * For 5GHz channel,
+ * Channel Frequency = (3/2) * freq_ref * (chansel[8:0] + chanfrac[16:0]/2^10)
+ * (freq_ref = 40MHz/(24>>amodeRefSel))
+ *
+ * For 5GHz channels which are 5MHz spaced,
+ * Channel Frequency = (3/2) * freq_ref * (chansel[8:0] + chanfrac[16:0]/2^17)
+ * (freq_ref = 40MHz)
+ */
+static HAL_BOOL
+ar9280SetChannel(struct ath_hal *ah, const struct ieee80211_channel *chan)
+{
+	uint16_t bMode, fracMode, aModeRefSel = 0;
+	uint32_t freq, ndiv, channelSel = 0, channelFrac = 0, reg32 = 0;
+	CHAN_CENTERS centers;
+	uint32_t refDivA = 24;
+
+	OS_MARK(ah, AH_MARK_SETCHANNEL, chan->ic_freq);
+
+	ar5416GetChannelCenters(ah, chan, &centers);
+	freq = centers.synth_center;
+
+	reg32 = OS_REG_READ(ah, AR_PHY_SYNTH_CONTROL);
+	reg32 &= 0xc0000000;
+
+	if (freq < 4800) {     /* 2 GHz, fractional mode */
+		uint32_t txctl;
+
+		bMode = 1;
+		fracMode = 1;
+		aModeRefSel = 0;       
+		channelSel = (freq * 0x10000)/15;
+
+		txctl = OS_REG_READ(ah, AR_PHY_CCK_TX_CTRL);
+		if (freq == 2484) {
+			/* Enable channel spreading for channel 14 */
+			OS_REG_WRITE(ah, AR_PHY_CCK_TX_CTRL,
+			    txctl | AR_PHY_CCK_TX_CTRL_JAPAN);
+		} else {
+			OS_REG_WRITE(ah, AR_PHY_CCK_TX_CTRL,
+			    txctl &~ AR_PHY_CCK_TX_CTRL_JAPAN);
+		}     
+	} else {
+		bMode = 0;
+		fracMode = 0;
+
+		if ((freq % 20) == 0) {
+			aModeRefSel = 3;
+		} else if ((freq % 10) == 0) {
+			aModeRefSel = 2;
+		} else {
+			aModeRefSel = 0;
+			/* Enable 2G (fractional) mode for channels which are 5MHz spaced */
+			fracMode = 1;
+			refDivA = 1;
+			channelSel = (freq * 0x8000)/15;
+
+			/* RefDivA setting */
+			OS_REG_RMW_FIELD(ah, AR_AN_SYNTH9,
+			    AR_AN_SYNTH9_REFDIVA, refDivA);
+		}
+		if (!fracMode) {
+			ndiv = (freq * (refDivA >> aModeRefSel))/60;
+			channelSel =  ndiv & 0x1ff;         
+			channelFrac = (ndiv & 0xfffffe00) * 2;
+			channelSel = (channelSel << 17) | channelFrac;
+		}
+	}
+
+	reg32 = reg32 | (bMode << 29) | (fracMode << 28) |
+	    (aModeRefSel << 26) | (channelSel);
+
+	OS_REG_WRITE(ah, AR_PHY_SYNTH_CONTROL, reg32);
+
+	AH_PRIVATE(ah)->ah_curchan = chan;
+
+	return AH_TRUE;
+}
+
+/*
+ * Return a reference to the requested RF Bank.
+ */
+static uint32_t *
+ar9280GetRfBank(struct ath_hal *ah, int bank)
+{
+	HALDEBUG(ah, HAL_DEBUG_ANY, "%s: unknown RF Bank %d requested\n",
+	    __func__, bank);
+	return AH_NULL;
+}
+
+/*
+ * Reads EEPROM header info from device structure and programs
+ * all rf registers
+ */
+static HAL_BOOL
+ar9280SetRfRegs(struct ath_hal *ah, const struct ieee80211_channel *chan,
+                uint16_t modesIndex, uint16_t *rfXpdGain)
+{
+	return AH_TRUE;		/* nothing to do */
+}
+
+/*
+ * Read the transmit power levels from the structures taken from EEPROM
+ * Interpolate read transmit power values for this channel
+ * Organize the transmit power values into a table for writing into the hardware
+ */
+
+static HAL_BOOL
+ar9280SetPowerTable(struct ath_hal *ah, int16_t *pPowerMin, int16_t *pPowerMax, 
+	const struct ieee80211_channel *chan, uint16_t *rfXpdGain)
+{
+	return AH_TRUE;
+}
+
+#if 0
+static int16_t
+ar9280GetMinPower(struct ath_hal *ah, EXPN_DATA_PER_CHANNEL_5112 *data)
+{
+    int i, minIndex;
+    int16_t minGain,minPwr,minPcdac,retVal;
+
+    /* Assume NUM_POINTS_XPD0 > 0 */
+    minGain = data->pDataPerXPD[0].xpd_gain;
+    for (minIndex=0,i=1; i<NUM_XPD_PER_CHANNEL; i++) {
+        if (data->pDataPerXPD[i].xpd_gain < minGain) {
+            minIndex = i;
+            minGain = data->pDataPerXPD[i].xpd_gain;
+        }
+    }
+    minPwr = data->pDataPerXPD[minIndex].pwr_t4[0];
+    minPcdac = data->pDataPerXPD[minIndex].pcdac[0];
+    for (i=1; i<NUM_POINTS_XPD0; i++) {
+        if (data->pDataPerXPD[minIndex].pwr_t4[i] < minPwr) {
+            minPwr = data->pDataPerXPD[minIndex].pwr_t4[i];
+            minPcdac = data->pDataPerXPD[minIndex].pcdac[i];
+        }
+    }
+    retVal = minPwr - (minPcdac*2);
+    return(retVal);
+}
+#endif
+
+static HAL_BOOL
+ar9280GetChannelMaxMinPower(struct ath_hal *ah,
+	const struct ieee80211_channel *chan,
+	int16_t *maxPow, int16_t *minPow)
+{
+#if 0
+    struct ath_hal_5212 *ahp = AH5212(ah);
+    int numChannels=0,i,last;
+    int totalD, totalF,totalMin;
+    EXPN_DATA_PER_CHANNEL_5112 *data=AH_NULL;
+    EEPROM_POWER_EXPN_5112 *powerArray=AH_NULL;
+
+    *maxPow = 0;
+    if (IS_CHAN_A(chan)) {
+        powerArray = ahp->ah_modePowerArray5112;
+        data = powerArray[headerInfo11A].pDataPerChannel;
+        numChannels = powerArray[headerInfo11A].numChannels;
+    } else if (IS_CHAN_G(chan) || IS_CHAN_108G(chan)) {
+        /* XXX - is this correct? Should we also use the same power for turbo G? */
+        powerArray = ahp->ah_modePowerArray5112;
+        data = powerArray[headerInfo11G].pDataPerChannel;
+        numChannels = powerArray[headerInfo11G].numChannels;
+    } else if (IS_CHAN_B(chan)) {
+        powerArray = ahp->ah_modePowerArray5112;
+        data = powerArray[headerInfo11B].pDataPerChannel;
+        numChannels = powerArray[headerInfo11B].numChannels;
+    } else {
+        return (AH_TRUE);
+    }
+    /* Make sure the channel is in the range of the TP values
+     *  (freq piers)
+     */
+    if ((numChannels < 1) ||
+        (chan->channel < data[0].channelValue) ||
+        (chan->channel > data[numChannels-1].channelValue))
+        return(AH_FALSE);
+
+    /* Linearly interpolate the power value now */
+    for (last=0,i=0;
+         (i<numChannels) && (chan->channel > data[i].channelValue);
+         last=i++);
+    totalD = data[i].channelValue - data[last].channelValue;
+    if (totalD > 0) {
+        totalF = data[i].maxPower_t4 - data[last].maxPower_t4;
+        *maxPow = (int8_t) ((totalF*(chan->channel-data[last].channelValue) + data[last].maxPower_t4*totalD)/totalD);
+
+        totalMin = ar9280GetMinPower(ah,&data[i]) - ar9280GetMinPower(ah, &data[last]);
+        *minPow = (int8_t) ((totalMin*(chan->channel-data[last].channelValue) + ar9280GetMinPower(ah, &data[last])*totalD)/totalD);
+        return (AH_TRUE);
+    } else {
+        if (chan->channel == data[i].channelValue) {
+            *maxPow = data[i].maxPower_t4;
+            *minPow = ar9280GetMinPower(ah, &data[i]);
+            return(AH_TRUE);
+        } else
+            return(AH_FALSE);
+    }
+#else
+	*maxPow = *minPow = 0;
+	return AH_FALSE;
+#endif
+}
+
+static void
+ar9280GetNoiseFloor(struct ath_hal *ah, int16_t nfarray[])
+{
+	int16_t nf;
+
+	nf = MS(OS_REG_READ(ah, AR_PHY_CCA), AR9280_PHY_MINCCA_PWR);
+	if (nf & 0x100)
+		nf = 0 - ((nf ^ 0x1ff) + 1);
+	HALDEBUG(ah, HAL_DEBUG_NFCAL,
+	    "NF calibrated [ctl] [chain 0] is %d\n", nf);
+	nfarray[0] = nf;
+
+	nf = MS(OS_REG_READ(ah, AR_PHY_CH1_CCA), AR9280_PHY_CH1_MINCCA_PWR);
+	if (nf & 0x100)
+		nf = 0 - ((nf ^ 0x1ff) + 1);
+	HALDEBUG(ah, HAL_DEBUG_NFCAL,
+	    "NF calibrated [ctl] [chain 1] is %d\n", nf);
+	nfarray[1] = nf;
+
+	nf = MS(OS_REG_READ(ah, AR_PHY_EXT_CCA), AR9280_PHY_EXT_MINCCA_PWR);
+	if (nf & 0x100)
+		nf = 0 - ((nf ^ 0x1ff) + 1);
+	HALDEBUG(ah, HAL_DEBUG_NFCAL,
+	    "NF calibrated [ext] [chain 0] is %d\n", nf);
+	nfarray[3] = nf;
+
+	nf = MS(OS_REG_READ(ah, AR_PHY_CH1_EXT_CCA), AR9280_PHY_CH1_EXT_MINCCA_PWR);
+	if (nf & 0x100)
+		nf = 0 - ((nf ^ 0x1ff) + 1);
+	HALDEBUG(ah, HAL_DEBUG_NFCAL,
+	    "NF calibrated [ext] [chain 1] is %d\n", nf);
+	nfarray[4] = nf;
+}
+
+/*
+ * Adjust NF based on statistical values for 5GHz frequencies.
+ * Stubbed:Not used by Fowl
+ */
+int16_t
+ar9280GetNfAdjust(struct ath_hal *ah, const HAL_CHANNEL_INTERNAL *c)
+{
+	return 0;
+}
+
+/*
+ * Free memory for analog bank scratch buffers
+ */
+static void
+ar9280RfDetach(struct ath_hal *ah)
+{
+	struct ath_hal_5212 *ahp = AH5212(ah);
+
+	HALASSERT(ahp->ah_rfHal != AH_NULL);
+	ath_hal_free(ahp->ah_rfHal);
+	ahp->ah_rfHal = AH_NULL;
+}
+
+HAL_BOOL
+ar9280RfAttach(struct ath_hal *ah, HAL_STATUS *status)
+{
+	struct ath_hal_5212 *ahp = AH5212(ah);
+	struct ar9280State *priv;
+
+	HALDEBUG(ah, HAL_DEBUG_ATTACH, "%s: attach AR9280 radio\n", __func__);
+
+	HALASSERT(ahp->ah_rfHal == AH_NULL);
+	priv = ath_hal_malloc(sizeof(struct ar9280State));
+	if (priv == AH_NULL) {
+		HALDEBUG(ah, HAL_DEBUG_ANY,
+		    "%s: cannot allocate private state\n", __func__);
+		*status = HAL_ENOMEM;		/* XXX */
+		return AH_FALSE;
+	}
+	priv->base.rfDetach		= ar9280RfDetach;
+	priv->base.writeRegs		= ar9280WriteRegs;
+	priv->base.getRfBank		= ar9280GetRfBank;
+	priv->base.setChannel		= ar9280SetChannel;
+	priv->base.setRfRegs		= ar9280SetRfRegs;
+	priv->base.setPowerTable	= ar9280SetPowerTable;
+	priv->base.getChannelMaxMinPower = ar9280GetChannelMaxMinPower;
+	priv->base.getNfAdjust		= ar9280GetNfAdjust;
+
+	ahp->ah_pcdacTable = priv->pcdacTable;
+	ahp->ah_pcdacTableSize = sizeof(priv->pcdacTable);
+	ahp->ah_rfHal = &priv->base;
+	/*
+	 * Set noise floor adjust method; we arrange a
+	 * direct call instead of thunking.
+	 */
+	AH_PRIVATE(ah)->ah_getNfAdjust = priv->base.getNfAdjust;
+	AH_PRIVATE(ah)->ah_getNoiseFloor = ar9280GetNoiseFloor;
+
+	return AH_TRUE;
+}
diff --git a/sys/dev/ath/ath_hal/ar5416/ar9280.h b/sys/dev/ath/ath_hal/ar5416/ar9280.h
new file mode 100644
index 0000000..f2e97d96
--- /dev/null
+++ b/sys/dev/ath/ath_hal/ar5416/ar9280.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2008-2009 Sam Leffler, Errno Consulting
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * $FreeBSD$
+ */
+#ifndef _ATH_AR9280_H_
+#define _ATH_AR9280_H_
+
+#include "ar5416/ar5416.h"
+
+struct ath_hal_9280 {
+	struct ath_hal_5416 ah_5416;
+
+	HAL_INI_ARRAY	ah_ini_xmodes;
+	HAL_INI_ARRAY	ah_ini_rxgain;
+	HAL_INI_ARRAY	ah_ini_txgain;
+};
+#define	AH9280(_ah)	((struct ath_hal_9280 *)(_ah))
+
+#define	AR9280_DEFAULT_RXCHAINMASK	3
+#define	AR9280_DEFAULT_TXCHAINMASK	1
+
+HAL_BOOL ar9280RfAttach(struct ath_hal *, HAL_STATUS *);
+
+struct ath_hal;
+
+HAL_BOOL	ar9280SetAntennaSwitch(struct ath_hal *, HAL_ANT_SETTING);
+#endif	/* _ATH_AR9280_H_ */
diff --git a/sys/dev/ath/ath_hal/ar5416/ar9280_attach.c b/sys/dev/ath/ath_hal/ar5416/ar9280_attach.c
new file mode 100644
index 0000000..d998af0
--- /dev/null
+++ b/sys/dev/ath/ath_hal/ar5416/ar9280_attach.c
@@ -0,0 +1,738 @@
+/*
+ * Copyright (c) 2008-2009 Sam Leffler, Errno Consulting
+ * Copyright (c) 2008 Atheros Communications, Inc.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * $FreeBSD$
+ */
+#include "opt_ah.h"
+
+#include "ah.h"
+#include "ah_internal.h"
+#include "ah_devid.h"
+
+#include "ah_eeprom_v14.h"		/* XXX for tx/rx gain */
+
+#include "ar5416/ar9280.h"
+#include "ar5416/ar5416reg.h"
+#include "ar5416/ar5416phy.h"
+
+#include "ar5416/ar9280v1.ini"
+#include "ar5416/ar9280v2.ini"
+
+static const HAL_PERCAL_DATA ar9280_iq_cal = {		/* single sample */
+	.calName = "IQ", .calType = IQ_MISMATCH_CAL,
+	.calNumSamples	= MIN_CAL_SAMPLES,
+	.calCountMax	= PER_MAX_LOG_COUNT,
+	.calCollect	= ar5416IQCalCollect,
+	.calPostProc	= ar5416IQCalibration
+};
+static const HAL_PERCAL_DATA ar9280_adc_gain_cal = {	/* single sample */
+	.calName = "ADC Gain", .calType = ADC_GAIN_CAL,
+	.calNumSamples	= MIN_CAL_SAMPLES,
+	.calCountMax	= PER_MIN_LOG_COUNT,
+	.calCollect	= ar5416AdcGainCalCollect,
+	.calPostProc	= ar5416AdcGainCalibration
+};
+static const HAL_PERCAL_DATA ar9280_adc_dc_cal = {	/* single sample */
+	.calName = "ADC DC", .calType = ADC_DC_CAL,
+	.calNumSamples	= MIN_CAL_SAMPLES,
+	.calCountMax	= PER_MIN_LOG_COUNT,
+	.calCollect	= ar5416AdcDcCalCollect,
+	.calPostProc	= ar5416AdcDcCalibration
+};
+static const HAL_PERCAL_DATA ar9280_adc_init_dc_cal = {
+	.calName = "ADC Init DC", .calType = ADC_DC_INIT_CAL,
+	.calNumSamples	= MIN_CAL_SAMPLES,
+	.calCountMax	= INIT_LOG_COUNT,
+	.calCollect	= ar5416AdcDcCalCollect,
+	.calPostProc	= ar5416AdcDcCalibration
+};
+
+static void ar9280ConfigPCIE(struct ath_hal *ah, HAL_BOOL restore);
+static HAL_BOOL ar9280FillCapabilityInfo(struct ath_hal *ah);
+static void ar9280WriteIni(struct ath_hal *ah,
+	const struct ieee80211_channel *chan);
+static void ar9280SpurMitigate(struct ath_hal *ah,
+	const struct ieee80211_channel *chan);
+
+static void
+ar9280AniSetup(struct ath_hal *ah)
+{
+	/* NB: disable ANI for reliable RIFS rx */
+	ar5212AniAttach(ah, AH_NULL, AH_NULL, AH_FALSE);
+}
+
+/*
+ * Attach for an AR9280 part.
+ */
+static struct ath_hal *
+ar9280Attach(uint16_t devid, HAL_SOFTC sc,
+	HAL_BUS_TAG st, HAL_BUS_HANDLE sh, HAL_STATUS *status)
+{
+	struct ath_hal_9280 *ahp9280;
+	struct ath_hal_5212 *ahp;
+	struct ath_hal *ah;
+	uint32_t val;
+	HAL_STATUS ecode;
+	HAL_BOOL rfStatus;
+
+	HALDEBUG(AH_NULL, HAL_DEBUG_ATTACH, "%s: sc %p st %p sh %p\n",
+	    __func__, sc, (void*) st, (void*) sh);
+
+	/* NB: memory is returned zero'd */
+	ahp9280 = ath_hal_malloc(sizeof (struct ath_hal_9280));
+	if (ahp9280 == AH_NULL) {
+		HALDEBUG(AH_NULL, HAL_DEBUG_ANY,
+		    "%s: cannot allocate memory for state block\n", __func__);
+		*status = HAL_ENOMEM;
+		return AH_NULL;
+	}
+	ahp = AH5212(ahp9280);
+	ah = &ahp->ah_priv.h;
+
+	ar5416InitState(AH5416(ah), devid, sc, st, sh, status);
+
+	/* XXX override with 9280 specific state */
+	/* override 5416 methods for our needs */
+	ah->ah_setAntennaSwitch		= ar9280SetAntennaSwitch;
+	ah->ah_configPCIE		= ar9280ConfigPCIE;
+
+	AH5416(ah)->ah_cal.iqCalData.calData = &ar9280_iq_cal;
+	AH5416(ah)->ah_cal.adcGainCalData.calData = &ar9280_adc_gain_cal;
+	AH5416(ah)->ah_cal.adcDcCalData.calData = &ar9280_adc_dc_cal;
+	AH5416(ah)->ah_cal.adcDcCalInitData.calData = &ar9280_adc_init_dc_cal;
+	AH5416(ah)->ah_cal.suppCals = ADC_GAIN_CAL | ADC_DC_CAL | IQ_MISMATCH_CAL;
+
+	AH5416(ah)->ah_spurMitigate	= ar9280SpurMitigate;
+	AH5416(ah)->ah_writeIni		= ar9280WriteIni;
+	AH5416(ah)->ah_rx_chainmask	= AR9280_DEFAULT_RXCHAINMASK;
+	AH5416(ah)->ah_tx_chainmask	= AR9280_DEFAULT_TXCHAINMASK;
+
+	if (!ar5416SetResetReg(ah, HAL_RESET_POWER_ON)) {
+		/* reset chip */
+		HALDEBUG(ah, HAL_DEBUG_ANY, "%s: couldn't reset chip\n",
+		    __func__);
+		ecode = HAL_EIO;
+		goto bad;
+	}
+
+	if (!ar5416SetPowerMode(ah, HAL_PM_AWAKE, AH_TRUE)) {
+		HALDEBUG(ah, HAL_DEBUG_ANY, "%s: couldn't wakeup chip\n",
+		    __func__);
+		ecode = HAL_EIO;
+		goto bad;
+	}
+	/* Read Revisions from Chips before taking out of reset */
+	val = OS_REG_READ(ah, AR_SREV);
+	HALDEBUG(ah, HAL_DEBUG_ATTACH,
+	    "%s: ID 0x%x VERSION 0x%x TYPE 0x%x REVISION 0x%x\n",
+	    __func__, MS(val, AR_XSREV_ID), MS(val, AR_XSREV_VERSION),
+	    MS(val, AR_XSREV_TYPE), MS(val, AR_XSREV_REVISION));
+	/* NB: include chip type to differentiate from pre-Sowl versions */
+	AH_PRIVATE(ah)->ah_macVersion =
+	    (val & AR_XSREV_VERSION) >> AR_XSREV_TYPE_S;
+	AH_PRIVATE(ah)->ah_macRev = MS(val, AR_XSREV_REVISION);
+	AH_PRIVATE(ah)->ah_ispcie = (val & AR_XSREV_TYPE_HOST_MODE) == 0;
+
+	/* setup common ini data; rf backends handle remainder */
+	if (AR_SREV_MERLIN_20_OR_LATER(ah)) {
+		HAL_INI_INIT(&ahp->ah_ini_modes, ar9280Modes_v2, 6);
+		HAL_INI_INIT(&ahp->ah_ini_common, ar9280Common_v2, 2);
+		HAL_INI_INIT(&AH5416(ah)->ah_ini_pcieserdes,
+		    ar9280PciePhy_clkreq_always_on_L1_v2, 2);
+		HAL_INI_INIT(&ahp9280->ah_ini_xmodes,
+		    ar9280Modes_fast_clock_v2, 3);
+	} else {
+		HAL_INI_INIT(&ahp->ah_ini_modes, ar9280Modes_v1, 6);
+		HAL_INI_INIT(&ahp->ah_ini_common, ar9280Common_v1, 2);
+		HAL_INI_INIT(&AH5416(ah)->ah_ini_pcieserdes,
+		    ar9280PciePhy_v1, 2);
+	}
+	ar5416AttachPCIE(ah);
+
+	ecode = ath_hal_v14EepromAttach(ah);
+	if (ecode != HAL_OK)
+		goto bad;
+
+	if (!ar5416ChipReset(ah, AH_NULL)) {	/* reset chip */
+		HALDEBUG(ah, HAL_DEBUG_ANY, "%s: chip reset failed\n", __func__);
+		ecode = HAL_EIO;
+		goto bad;
+	}
+
+	AH_PRIVATE(ah)->ah_phyRev = OS_REG_READ(ah, AR_PHY_CHIP_ID);
+
+	if (!ar5212ChipTest(ah)) {
+		HALDEBUG(ah, HAL_DEBUG_ANY, "%s: hardware self-test failed\n",
+		    __func__);
+		ecode = HAL_ESELFTEST;
+		goto bad;
+	}
+
+	/*
+	 * Set correct Baseband to analog shift
+	 * setting to access analog chips.
+	 */
+	OS_REG_WRITE(ah, AR_PHY(0), 0x00000007);
+
+	/* Read Radio Chip Rev Extract */
+	AH_PRIVATE(ah)->ah_analog5GhzRev = ar5416GetRadioRev(ah);
+	switch (AH_PRIVATE(ah)->ah_analog5GhzRev & AR_RADIO_SREV_MAJOR) {
+        case AR_RAD2133_SREV_MAJOR:	/* Sowl: 2G/3x3 */
+	case AR_RAD5133_SREV_MAJOR:	/* Sowl: 2+5G/3x3 */
+		break;
+	default:
+		if (AH_PRIVATE(ah)->ah_analog5GhzRev == 0) {
+			AH_PRIVATE(ah)->ah_analog5GhzRev =
+				AR_RAD5133_SREV_MAJOR;
+			break;
+		}
+#ifdef AH_DEBUG
+		HALDEBUG(ah, HAL_DEBUG_ANY,
+		    "%s: 5G Radio Chip Rev 0x%02X is not supported by "
+		    "this driver\n", __func__,
+		    AH_PRIVATE(ah)->ah_analog5GhzRev);
+		ecode = HAL_ENOTSUPP;
+		goto bad;
+#endif
+	}
+	rfStatus = ar9280RfAttach(ah, &ecode);
+	if (!rfStatus) {
+		HALDEBUG(ah, HAL_DEBUG_ANY, "%s: RF setup failed, status %u\n",
+		    __func__, ecode);
+		goto bad;
+	}
+
+	if (AR_SREV_MERLIN_20_OR_LATER(ah)) {
+		/* setup rxgain table */
+		switch (ath_hal_eepromGet(ah, AR_EEP_RXGAIN_TYPE, AH_NULL)) {
+		case AR5416_EEP_RXGAIN_13dB_BACKOFF:
+			HAL_INI_INIT(&ahp9280->ah_ini_rxgain,
+			    ar9280Modes_backoff_13db_rxgain_v2, 6);
+			break;
+		case AR5416_EEP_RXGAIN_23dB_BACKOFF:
+			HAL_INI_INIT(&ahp9280->ah_ini_rxgain,
+			    ar9280Modes_backoff_23db_rxgain_v2, 6);
+			break;
+		case AR5416_EEP_RXGAIN_ORIG:
+			HAL_INI_INIT(&ahp9280->ah_ini_rxgain,
+			    ar9280Modes_original_rxgain_v2, 6);
+			break;
+		default:
+			HALASSERT(AH_FALSE);
+			goto bad;		/* XXX ? try to continue */
+		}
+	}
+	if (AR_SREV_MERLIN_20_OR_LATER(ah)) {
+		/* setp txgain table */
+		switch (ath_hal_eepromGet(ah, AR_EEP_TXGAIN_TYPE, AH_NULL)) {
+		case AR5416_EEP_TXGAIN_HIGH_POWER:
+			HAL_INI_INIT(&ahp9280->ah_ini_txgain,
+			    ar9280Modes_high_power_tx_gain_v2, 6);
+			break;
+		case AR5416_EEP_TXGAIN_ORIG:
+			HAL_INI_INIT(&ahp9280->ah_ini_txgain,
+			    ar9280Modes_original_tx_gain_v2, 6);
+			break;
+		default:
+			HALASSERT(AH_FALSE);
+			goto bad;		/* XXX ? try to continue */
+		}
+	}
+
+	/*
+	 * Got everything we need now to setup the capabilities.
+	 */
+	if (!ar9280FillCapabilityInfo(ah)) {
+		ecode = HAL_EEREAD;
+		goto bad;
+	}
+
+	ecode = ath_hal_eepromGet(ah, AR_EEP_MACADDR, ahp->ah_macaddr);
+	if (ecode != HAL_OK) {
+		HALDEBUG(ah, HAL_DEBUG_ANY,
+		    "%s: error getting mac address from EEPROM\n", __func__);
+		goto bad;
+        }
+	/* XXX How about the serial number ? */
+	/* Read Reg Domain */
+	AH_PRIVATE(ah)->ah_currentRD =
+	    ath_hal_eepromGet(ah, AR_EEP_REGDMN_0, AH_NULL);
+
+	/*
+	 * ah_miscMode is populated by ar5416FillCapabilityInfo()
+	 * starting from griffin. Set here to make sure that
+	 * AR_MISC_MODE_MIC_NEW_LOC_ENABLE is set before a GTK is
+	 * placed into hardware.
+	 */
+	if (ahp->ah_miscMode != 0)
+		OS_REG_WRITE(ah, AR_MISC_MODE, ahp->ah_miscMode);
+
+	ar9280AniSetup(ah);			/* Anti Noise Immunity */
+	ar5416InitNfHistBuff(AH5416(ah)->ah_cal.nfCalHist);
+
+	HALDEBUG(ah, HAL_DEBUG_ATTACH, "%s: return\n", __func__);
+
+	return ah;
+bad:
+	if (ah != AH_NULL)
+		ah->ah_detach(ah);
+	if (status)
+		*status = ecode;
+	return AH_NULL;
+}
+
+static void
+ar9280ConfigPCIE(struct ath_hal *ah, HAL_BOOL restore)
+{
+	if (AH_PRIVATE(ah)->ah_ispcie && !restore) {
+		ath_hal_ini_write(ah, &AH5416(ah)->ah_ini_pcieserdes, 1, 0);
+		OS_DELAY(1000);
+		OS_REG_SET_BIT(ah, AR_PCIE_PM_CTRL, AR_PCIE_PM_CTRL_ENA);
+		OS_REG_WRITE(ah, AR_WA, AR9280_WA_DEFAULT);
+	}
+}
+
+static void
+ar9280WriteIni(struct ath_hal *ah, const struct ieee80211_channel *chan)
+{
+	u_int modesIndex, freqIndex;
+	int regWrites = 0;
+
+	/* Setup the indices for the next set of register array writes */
+	/* XXX Ignore 11n dynamic mode on the AR5416 for the moment */
+	if (IEEE80211_IS_CHAN_2GHZ(chan)) {
+		freqIndex = 2;
+		if (IEEE80211_IS_CHAN_HT40(chan))
+			modesIndex = 3;
+		else if (IEEE80211_IS_CHAN_108G(chan))
+			modesIndex = 5;
+		else
+			modesIndex = 4;
+	} else {
+		freqIndex = 1;
+		if (IEEE80211_IS_CHAN_HT40(chan) ||
+		    IEEE80211_IS_CHAN_TURBO(chan))
+			modesIndex = 2;
+		else
+			modesIndex = 1;
+	}
+
+	/* Set correct Baseband to analog shift setting to access analog chips. */
+	OS_REG_WRITE(ah, AR_PHY(0), 0x00000007);
+	OS_REG_WRITE(ah, AR_PHY_ADC_SERIAL_CTL, AR_PHY_SEL_INTERNAL_ADDAC);
+
+	/* XXX Merlin ini fixups */
+	/* XXX Merlin 100us delay for shift registers */
+	regWrites = ath_hal_ini_write(ah, &AH5212(ah)->ah_ini_modes,
+	    modesIndex, regWrites);
+	if (AR_SREV_MERLIN_20_OR_LATER(ah)) {
+		regWrites = ath_hal_ini_write(ah, &AH9280(ah)->ah_ini_rxgain,
+		    modesIndex, regWrites);
+		regWrites = ath_hal_ini_write(ah, &AH9280(ah)->ah_ini_txgain,
+		    modesIndex, regWrites);
+	}
+	/* XXX Merlin 100us delay for shift registers */
+	regWrites = ath_hal_ini_write(ah, &AH5212(ah)->ah_ini_common,
+	    1, regWrites);
+
+	if (AR_SREV_MERLIN_20(ah) && IS_5GHZ_FAST_CLOCK_EN(ah, chan)) {
+		/* 5GHz channels w/ Fast Clock use different modal values */
+		regWrites = ath_hal_ini_write(ah, &AH9280(ah)->ah_ini_xmodes,
+		    modesIndex, regWrites);
+	}
+}
+
+#define	AR_BASE_FREQ_2GHZ	2300
+#define	AR_BASE_FREQ_5GHZ	4900
+#define	AR_SPUR_FEEQ_BOUND_HT40	19
+#define	AR_SPUR_FEEQ_BOUND_HT20	10
+
+static void
+ar9280SpurMitigate(struct ath_hal *ah, const struct ieee80211_channel *chan)
+{
+    static const int pilot_mask_reg[4] = { AR_PHY_TIMING7, AR_PHY_TIMING8,
+                AR_PHY_PILOT_MASK_01_30, AR_PHY_PILOT_MASK_31_60 };
+    static const int chan_mask_reg[4] = { AR_PHY_TIMING9, AR_PHY_TIMING10,
+                AR_PHY_CHANNEL_MASK_01_30, AR_PHY_CHANNEL_MASK_31_60 };
+    static int inc[4] = { 0, 100, 0, 0 };
+
+    int bb_spur = AR_NO_SPUR;
+    int freq;
+    int bin, cur_bin;
+    int bb_spur_off, spur_subchannel_sd;
+    int spur_freq_sd;
+    int spur_delta_phase;
+    int denominator;
+    int upper, lower, cur_vit_mask;
+    int tmp, newVal;
+    int i;
+    CHAN_CENTERS centers;
+
+    int8_t mask_m[123];
+    int8_t mask_p[123];
+    int8_t mask_amt;
+    int tmp_mask;
+    int cur_bb_spur;
+    HAL_BOOL is2GHz = IEEE80211_IS_CHAN_2GHZ(chan);
+
+    OS_MEMZERO(&mask_m, sizeof(int8_t) * 123);
+    OS_MEMZERO(&mask_p, sizeof(int8_t) * 123);
+
+    ar5416GetChannelCenters(ah, chan, &centers);
+    freq = centers.synth_center;
+
+    /*
+     * Need to verify range +/- 9.38 for static ht20 and +/- 18.75 for ht40,
+     * otherwise spur is out-of-band and can be ignored.
+     */
+    for (i = 0; i < AR5416_EEPROM_MODAL_SPURS; i++) {
+        cur_bb_spur = ath_hal_getSpurChan(ah, i, is2GHz);
+        /* Get actual spur freq in MHz from EEPROM read value */ 
+        if (is2GHz) {
+            cur_bb_spur =  (cur_bb_spur / 10) + AR_BASE_FREQ_2GHZ;
+        } else {
+            cur_bb_spur =  (cur_bb_spur / 10) + AR_BASE_FREQ_5GHZ;
+        }
+
+        if (AR_NO_SPUR == cur_bb_spur)
+            break;
+        cur_bb_spur = cur_bb_spur - freq;
+
+        if (IEEE80211_IS_CHAN_HT40(chan)) {
+            if ((cur_bb_spur > -AR_SPUR_FEEQ_BOUND_HT40) && 
+                (cur_bb_spur < AR_SPUR_FEEQ_BOUND_HT40)) {
+                bb_spur = cur_bb_spur;
+                break;
+            }
+        } else if ((cur_bb_spur > -AR_SPUR_FEEQ_BOUND_HT20) &&
+                   (cur_bb_spur < AR_SPUR_FEEQ_BOUND_HT20)) {
+            bb_spur = cur_bb_spur;
+            break;
+        }
+    }
+
+    if (AR_NO_SPUR == bb_spur) {
+#if 1
+        /*
+         * MRC CCK can interfere with beacon detection and cause deaf/mute.
+         * Disable MRC CCK for now.
+         */
+        OS_REG_CLR_BIT(ah, AR_PHY_FORCE_CLKEN_CCK, AR_PHY_FORCE_CLKEN_CCK_MRC_MUX);
+#else
+        /* Enable MRC CCK if no spur is found in this channel. */
+        OS_REG_SET_BIT(ah, AR_PHY_FORCE_CLKEN_CCK, AR_PHY_FORCE_CLKEN_CCK_MRC_MUX);
+#endif
+        return;
+    } else {
+        /* 
+         * For Merlin, spur can break CCK MRC algorithm. Disable CCK MRC if spur
+         * is found in this channel.
+         */
+        OS_REG_CLR_BIT(ah, AR_PHY_FORCE_CLKEN_CCK, AR_PHY_FORCE_CLKEN_CCK_MRC_MUX);
+    }
+
+    bin = bb_spur * 320;
+
+    tmp = OS_REG_READ(ah, AR_PHY_TIMING_CTRL4_CHAIN(0));
+
+    newVal = tmp | (AR_PHY_TIMING_CTRL4_ENABLE_SPUR_RSSI |
+        AR_PHY_TIMING_CTRL4_ENABLE_SPUR_FILTER |
+        AR_PHY_TIMING_CTRL4_ENABLE_CHAN_MASK |
+        AR_PHY_TIMING_CTRL4_ENABLE_PILOT_MASK);
+    OS_REG_WRITE(ah, AR_PHY_TIMING_CTRL4_CHAIN(0), newVal);
+
+    newVal = (AR_PHY_SPUR_REG_MASK_RATE_CNTL |
+        AR_PHY_SPUR_REG_ENABLE_MASK_PPM |
+        AR_PHY_SPUR_REG_MASK_RATE_SELECT |
+        AR_PHY_SPUR_REG_ENABLE_VIT_SPUR_RSSI |
+        SM(AR5416_SPUR_RSSI_THRESH, AR_PHY_SPUR_REG_SPUR_RSSI_THRESH));
+    OS_REG_WRITE(ah, AR_PHY_SPUR_REG, newVal);
+
+    /* Pick control or extn channel to cancel the spur */
+    if (IEEE80211_IS_CHAN_HT40(chan)) {
+        if (bb_spur < 0) {
+            spur_subchannel_sd = 1;
+            bb_spur_off = bb_spur + 10;
+        } else {
+            spur_subchannel_sd = 0;
+            bb_spur_off = bb_spur - 10;
+        }
+    } else {
+        spur_subchannel_sd = 0;
+        bb_spur_off = bb_spur;
+    }
+
+    /*
+     * spur_delta_phase = bb_spur/40 * 2**21 for static ht20,
+     * /80 for dyn2040.
+     */
+    if (IEEE80211_IS_CHAN_HT40(chan))
+        spur_delta_phase = ((bb_spur * 262144) / 10) & AR_PHY_TIMING11_SPUR_DELTA_PHASE;    
+    else
+        spur_delta_phase = ((bb_spur * 524288) / 10) & AR_PHY_TIMING11_SPUR_DELTA_PHASE;
+
+    /*
+     * in 11A mode the denominator of spur_freq_sd should be 40 and
+     * it should be 44 in 11G
+     */
+    denominator = IEEE80211_IS_CHAN_2GHZ(chan) ? 44 : 40;
+    spur_freq_sd = ((bb_spur_off * 2048) / denominator) & 0x3ff;
+
+    newVal = (AR_PHY_TIMING11_USE_SPUR_IN_AGC |
+        SM(spur_freq_sd, AR_PHY_TIMING11_SPUR_FREQ_SD) |
+        SM(spur_delta_phase, AR_PHY_TIMING11_SPUR_DELTA_PHASE));
+    OS_REG_WRITE(ah, AR_PHY_TIMING11, newVal);
+
+    /* Choose to cancel between control and extension channels */
+    newVal = spur_subchannel_sd << AR_PHY_SFCORR_SPUR_SUBCHNL_SD_S;
+    OS_REG_WRITE(ah, AR_PHY_SFCORR_EXT, newVal);
+
+    /*
+     * ============================================
+     * Set Pilot and Channel Masks
+     *
+     * pilot mask 1 [31:0] = +6..-26, no 0 bin
+     * pilot mask 2 [19:0] = +26..+7
+     *
+     * channel mask 1 [31:0] = +6..-26, no 0 bin
+     * channel mask 2 [19:0] = +26..+7
+     */
+    cur_bin = -6000;
+    upper = bin + 100;
+    lower = bin - 100;
+
+    for (i = 0; i < 4; i++) {
+        int pilot_mask = 0;
+        int chan_mask  = 0;
+        int bp         = 0;
+        for (bp = 0; bp < 30; bp++) {
+            if ((cur_bin > lower) && (cur_bin < upper)) {
+                pilot_mask = pilot_mask | 0x1 << bp;
+                chan_mask  = chan_mask | 0x1 << bp;
+            }
+            cur_bin += 100;
+        }
+        cur_bin += inc[i];
+        OS_REG_WRITE(ah, pilot_mask_reg[i], pilot_mask);
+        OS_REG_WRITE(ah, chan_mask_reg[i], chan_mask);
+    }
+
+    /* =================================================
+     * viterbi mask 1 based on channel magnitude
+     * four levels 0-3
+     *  - mask (-27 to 27) (reg 64,0x9900 to 67,0x990c)
+     *      [1 2 2 1] for -9.6 or [1 2 1] for +16
+     *  - enable_mask_ppm, all bins move with freq
+     *
+     *  - mask_select,    8 bits for rates (reg 67,0x990c)
+     *  - mask_rate_cntl, 8 bits for rates (reg 67,0x990c)
+     *      choose which mask to use mask or mask2
+     */
+
+    /*
+     * viterbi mask 2  2nd set for per data rate puncturing
+     * four levels 0-3
+     *  - mask_select, 8 bits for rates (reg 67)
+     *  - mask (-27 to 27) (reg 98,0x9988 to 101,0x9994)
+     *      [1 2 2 1] for -9.6 or [1 2 1] for +16
+     */
+    cur_vit_mask = 6100;
+    upper        = bin + 120;
+    lower        = bin - 120;
+
+    for (i = 0; i < 123; i++) {
+        if ((cur_vit_mask > lower) && (cur_vit_mask < upper)) {
+            if ((abs(cur_vit_mask - bin)) < 75) {
+                mask_amt = 1;
+            } else {
+                mask_amt = 0;
+            }
+            if (cur_vit_mask < 0) {
+                mask_m[abs(cur_vit_mask / 100)] = mask_amt;
+            } else {
+                mask_p[cur_vit_mask / 100] = mask_amt;
+            }
+        }
+        cur_vit_mask -= 100;
+    }
+
+    tmp_mask = (mask_m[46] << 30) | (mask_m[47] << 28)
+          | (mask_m[48] << 26) | (mask_m[49] << 24)
+          | (mask_m[50] << 22) | (mask_m[51] << 20)
+          | (mask_m[52] << 18) | (mask_m[53] << 16)
+          | (mask_m[54] << 14) | (mask_m[55] << 12)
+          | (mask_m[56] << 10) | (mask_m[57] <<  8)
+          | (mask_m[58] <<  6) | (mask_m[59] <<  4)
+          | (mask_m[60] <<  2) | (mask_m[61] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_BIN_MASK_1, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_VIT_MASK2_M_46_61, tmp_mask);
+
+    tmp_mask =             (mask_m[31] << 28)
+          | (mask_m[32] << 26) | (mask_m[33] << 24)
+          | (mask_m[34] << 22) | (mask_m[35] << 20)
+          | (mask_m[36] << 18) | (mask_m[37] << 16)
+          | (mask_m[48] << 14) | (mask_m[39] << 12)
+          | (mask_m[40] << 10) | (mask_m[41] <<  8)
+          | (mask_m[42] <<  6) | (mask_m[43] <<  4)
+          | (mask_m[44] <<  2) | (mask_m[45] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_BIN_MASK_2, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_MASK2_M_31_45, tmp_mask);
+
+    tmp_mask = (mask_m[16] << 30) | (mask_m[16] << 28)
+          | (mask_m[18] << 26) | (mask_m[18] << 24)
+          | (mask_m[20] << 22) | (mask_m[20] << 20)
+          | (mask_m[22] << 18) | (mask_m[22] << 16)
+          | (mask_m[24] << 14) | (mask_m[24] << 12)
+          | (mask_m[25] << 10) | (mask_m[26] <<  8)
+          | (mask_m[27] <<  6) | (mask_m[28] <<  4)
+          | (mask_m[29] <<  2) | (mask_m[30] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_BIN_MASK_3, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_MASK2_M_16_30, tmp_mask);
+
+    tmp_mask = (mask_m[ 0] << 30) | (mask_m[ 1] << 28)
+          | (mask_m[ 2] << 26) | (mask_m[ 3] << 24)
+          | (mask_m[ 4] << 22) | (mask_m[ 5] << 20)
+          | (mask_m[ 6] << 18) | (mask_m[ 7] << 16)
+          | (mask_m[ 8] << 14) | (mask_m[ 9] << 12)
+          | (mask_m[10] << 10) | (mask_m[11] <<  8)
+          | (mask_m[12] <<  6) | (mask_m[13] <<  4)
+          | (mask_m[14] <<  2) | (mask_m[15] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_MASK_CTL, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_MASK2_M_00_15, tmp_mask);
+
+    tmp_mask =             (mask_p[15] << 28)
+          | (mask_p[14] << 26) | (mask_p[13] << 24)
+          | (mask_p[12] << 22) | (mask_p[11] << 20)
+          | (mask_p[10] << 18) | (mask_p[ 9] << 16)
+          | (mask_p[ 8] << 14) | (mask_p[ 7] << 12)
+          | (mask_p[ 6] << 10) | (mask_p[ 5] <<  8)
+          | (mask_p[ 4] <<  6) | (mask_p[ 3] <<  4)
+          | (mask_p[ 2] <<  2) | (mask_p[ 1] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_1, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_MASK2_P_15_01, tmp_mask);
+
+    tmp_mask =             (mask_p[30] << 28)
+          | (mask_p[29] << 26) | (mask_p[28] << 24)
+          | (mask_p[27] << 22) | (mask_p[26] << 20)
+          | (mask_p[25] << 18) | (mask_p[24] << 16)
+          | (mask_p[23] << 14) | (mask_p[22] << 12)
+          | (mask_p[21] << 10) | (mask_p[20] <<  8)
+          | (mask_p[19] <<  6) | (mask_p[18] <<  4)
+          | (mask_p[17] <<  2) | (mask_p[16] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_2, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_MASK2_P_30_16, tmp_mask);
+
+    tmp_mask =             (mask_p[45] << 28)
+          | (mask_p[44] << 26) | (mask_p[43] << 24)
+          | (mask_p[42] << 22) | (mask_p[41] << 20)
+          | (mask_p[40] << 18) | (mask_p[39] << 16)
+          | (mask_p[38] << 14) | (mask_p[37] << 12)
+          | (mask_p[36] << 10) | (mask_p[35] <<  8)
+          | (mask_p[34] <<  6) | (mask_p[33] <<  4)
+          | (mask_p[32] <<  2) | (mask_p[31] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_3, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_MASK2_P_45_31, tmp_mask);
+
+    tmp_mask = (mask_p[61] << 30) | (mask_p[60] << 28)
+          | (mask_p[59] << 26) | (mask_p[58] << 24)
+          | (mask_p[57] << 22) | (mask_p[56] << 20)
+          | (mask_p[55] << 18) | (mask_p[54] << 16)
+          | (mask_p[53] << 14) | (mask_p[52] << 12)
+          | (mask_p[51] << 10) | (mask_p[50] <<  8)
+          | (mask_p[49] <<  6) | (mask_p[48] <<  4)
+          | (mask_p[47] <<  2) | (mask_p[46] <<  0);
+    OS_REG_WRITE(ah, AR_PHY_BIN_MASK2_4, tmp_mask);
+    OS_REG_WRITE(ah, AR_PHY_MASK2_P_61_45, tmp_mask);
+}
+
+/*
+ * Fill all software cached or static hardware state information.
+ * Return failure if capabilities are to come from EEPROM and
+ * cannot be read.
+ */
+static HAL_BOOL
+ar9280FillCapabilityInfo(struct ath_hal *ah)
+{
+	HAL_CAPABILITIES *pCap = &AH_PRIVATE(ah)->ah_caps;
+
+	if (!ar5416FillCapabilityInfo(ah))
+		return AH_FALSE;
+	pCap->halNumGpioPins = 10;
+	pCap->halWowSupport = AH_TRUE;
+	pCap->halWowMatchPatternExact = AH_TRUE;
+#if 0
+	pCap->halWowMatchPatternDword = AH_TRUE;
+#endif
+	pCap->halCSTSupport = AH_TRUE;
+	pCap->halRifsRxSupport = AH_TRUE;
+	pCap->halRifsTxSupport = AH_TRUE;
+	pCap->halRtsAggrLimit = 64*1024;	/* 802.11n max */
+	pCap->halExtChanDfsSupport = AH_TRUE;
+#if 0
+	/* XXX bluetooth */
+	pCap->halBtCoexSupport = AH_TRUE;
+#endif
+	pCap->halAutoSleepSupport = AH_FALSE;	/* XXX? */
+#if 0
+	pCap->hal4kbSplitTransSupport = AH_FALSE;
+#endif
+	pCap->halRxStbcSupport = 1;
+	pCap->halTxStbcSupport = 1;
+
+	return AH_TRUE;
+}
+
+HAL_BOOL
+ar9280SetAntennaSwitch(struct ath_hal *ah, HAL_ANT_SETTING settings)
+{
+#define ANTENNA0_CHAINMASK    0x1
+#define ANTENNA1_CHAINMASK    0x2
+	struct ath_hal_5416 *ahp = AH5416(ah);
+
+	/* Antenna selection is done by setting the tx/rx chainmasks approp. */
+	switch (settings) {
+	case HAL_ANT_FIXED_A:
+		/* Enable first antenna only */
+		ahp->ah_tx_chainmask = ANTENNA0_CHAINMASK;
+		ahp->ah_rx_chainmask = ANTENNA0_CHAINMASK;
+		break;
+	case HAL_ANT_FIXED_B:
+		/* Enable second antenna only, after checking capability */
+		if (AH_PRIVATE(ah)->ah_caps.halTxChainMask > ANTENNA1_CHAINMASK)
+			ahp->ah_tx_chainmask = ANTENNA1_CHAINMASK;
+		ahp->ah_rx_chainmask = ANTENNA1_CHAINMASK;
+		break;
+	case HAL_ANT_VARIABLE:
+		/* Restore original chainmask settings */
+		/* XXX */
+		ahp->ah_tx_chainmask = AR5416_DEFAULT_TXCHAINMASK;
+		ahp->ah_rx_chainmask = AR5416_DEFAULT_RXCHAINMASK;
+		break;
+	}
+	return AH_TRUE;
+#undef ANTENNA0_CHAINMASK
+#undef ANTENNA1_CHAINMASK
+}
+
+static const char*
+ar9280Probe(uint16_t vendorid, uint16_t devid)
+{
+	if (vendorid == ATHEROS_VENDOR_ID &&
+	    (devid == AR9280_DEVID_PCI || devid == AR9280_DEVID_PCIE))
+		return "Atheros 9280";
+	return AH_NULL;
+}
+AH_CHIP(AR9280, ar9280Probe, ar9280Attach);
diff --git a/sys/dev/ath/ath_hal/ar5416/ar9280v1.ini b/sys/dev/ath/ath_hal/ar5416/ar9280v1.ini
new file mode 100644
index 0000000..96e0f44
--- /dev/null
+++ b/sys/dev/ath/ath_hal/ar5416/ar9280v1.ini
@@ -0,0 +1,582 @@
+/*
+ * Copyright (c) 2002-2009 Sam Leffler, Errno Consulting
+ * Copyright (c) 2002-2008 Atheros Communications, Inc.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * $FreeBSD$
+ */
+/* Auto Generated PCI Register Writes.  Created: 10/12/07 */
+
+static const uint32_t ar9280Modes_v1[][6] = {
+    { 0x00001030, 0x00000230, 0x00000460, 0x000002c0, 0x00000160, 0x000001e0 },
+    { 0x00001070, 0x00000168, 0x000002d0, 0x00000318, 0x0000018c, 0x000001e0 },
+    { 0x000010b0, 0x00000e60, 0x00001cc0, 0x00007c70, 0x00003e38, 0x00001180 },
+    { 0x000010f0, 0x0000a000, 0x00014000, 0x00016000, 0x0000b000, 0x00014008 },
+    { 0x00008014, 0x03e803e8, 0x07d007d0, 0x10801080, 0x08400840, 0x06e006e0 },
+    { 0x0000801c, 0x128d8027, 0x128d804f, 0x12e00057, 0x12e0002b, 0x0988004f },
+    { 0x00009804, 0x00000300, 0x000003c4, 0x000003c4, 0x00000300, 0x00000303 },
+    { 0x00009820, 0x02020200, 0x02020200, 0x02020200, 0x02020200, 0x02020200 },
+    { 0x00009824, 0x00000e0e, 0x00000e0e, 0x00000e0e, 0x00000e0e, 0x00000e0e },
+    { 0x00009828, 0x0a020001, 0x0a020001, 0x0a020001, 0x0a020001, 0x0a020001 },
+    { 0x00009834, 0x00000e0e, 0x00000e0e, 0x00000e0e, 0x00000e0e, 0x00000e0e },
+    { 0x00009838, 0x00000007, 0x00000007, 0x00000007, 0x00000007, 0x00000007 },
+    { 0x00009844, 0x1372161e, 0x1372161e, 0x137216a0, 0x137216a0, 0x137216a0 },
+    { 0x00009848, 0x00028566, 0x00028566, 0x00028563, 0x00028563, 0x00028563 },
+    { 0x0000a848, 0x00028566, 0x00028566, 0x00028563, 0x00028563, 0x00028563 },
+    { 0x00009850, 0x6d4000e2, 0x6d4000e2, 0x6d4000e2, 0x6d4000e2, 0x6d4000e2 },
+    { 0x00009858, 0x7ec82d2e, 0x7ec82d2e, 0x7ec82d2e, 0x7ec82d2e, 0x7ec82d2e },
+    { 0x0000985c, 0x3139605e, 0x3139605e, 0x3139605e, 0x3139605e, 0x3139605e },
+    { 0x00009860, 0x00049d18, 0x00049d18, 0x00049d20, 0x00049d20, 0x00049d18 },
+    { 0x0000c864, 0x0001ce00, 0x0001ce00, 0x0001ce00, 0x0001ce00, 0x0001ce00 },
+    { 0x00009868, 0x5ac64190, 0x5ac64190, 0x5ac64190, 0x5ac64190, 0x5ac64190 },
+    { 0x0000986c, 0x06903081, 0x06903081, 0x06903881, 0x06903881, 0x06903881 },
+    { 0x00009914, 0x000007d0, 0x000007d0, 0x00000898, 0x00000898, 0x000007d0 },
+    { 0x00009918, 0x0000000a, 0x00000014, 0x00000016, 0x0000000b, 0x00000016 },
+    { 0x00009924, 0xd00a8a07, 0xd00a8a07, 0xd00a8a0d, 0xd00a8a0d, 0xd00a8a0d },
+    { 0x00009944, 0xdfbc1010, 0xdfbc1010, 0xdfbc1010, 0xdfbc1010, 0xdfbc1010 },
+    { 0x00009960, 0x00000010, 0x00000010, 0x00000010, 0x00000010, 0x00000010 },
+    { 0x0000a960, 0x00000010, 0x00000010, 0x00000010, 0x00000010, 0x00000010 },
+    { 0x00009964, 0x00000210, 0x00000210, 0x00000210, 0x00000210, 0x00000210 },
+    { 0x0000c9b8, 0x0000001a, 0x0000001a, 0x0000001a, 0x0000001a, 0x0000001a },
+    { 0x0000c9bc, 0x00000600, 0x00000600, 0x00000c00, 0x00000c00, 0x00000c00 },
+    { 0x000099c0, 0x05eea6d4, 0x05eea6d4, 0x05eea6d4, 0x05eea6d4, 0x05eea6d4 },
+    { 0x000099c4, 0x06336f77, 0x06336f77, 0x06336f77, 0x06336f77, 0x06336f77 },
+    { 0x000099c8, 0x60f6532c, 0x60f6532c, 0x60f6532c, 0x60f6532c, 0x60f6532c },
+    { 0x000099cc, 0x08f186c8, 0x08f186c8, 0x08f186c8, 0x08f186c8, 0x08f186c8 },
+    { 0x000099d0, 0x00046384, 0x00046384, 0x00046384, 0x00046384, 0x00046384 },
+    { 0x000099d4, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    { 0x000099d8, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    { 0x00009a00, 0x00008184, 0x00008184, 0x00000214, 0x00000214, 0x00000214 },
+    { 0x00009a04, 0x00008188, 0x00008188, 0x00000218, 0x00000218, 0x00000218 },
+    { 0x00009a08, 0x0000818c, 0x0000818c, 0x00000224, 0x00000224, 0x00000224 },
+    { 0x00009a0c, 0x00008190, 0x00008190, 0x00000228, 0x00000228, 0x00000228 },
+    { 0x00009a10, 0x00008194, 0x00008194, 0x0000022c, 0x0000022c, 0x0000022c },
+    { 0x00009a14, 0x00008200, 0x00008200, 0x00000230, 0x00000230, 0x00000230 },
+    { 0x00009a18, 0x00008204, 0x00008204, 0x000002a4, 0x000002a4, 0x000002a4 },
+    { 0x00009a1c, 0x00008208, 0x00008208, 0x000002a8, 0x000002a8, 0x000002a8 },
+    { 0x00009a20, 0x0000820c, 0x0000820c, 0x000002ac, 0x000002ac, 0x000002ac },
+    { 0x00009a24, 0x00008210, 0x00008210, 0x000002b0, 0x000002b0, 0x000002b0 },
+    { 0x00009a28, 0x00008214, 0x00008214, 0x000002b4, 0x000002b4, 0x000002b4 },
+    { 0x00009a2c, 0x00008280, 0x00008280, 0x000002b8, 0x000002b8, 0x000002b8 },
+    { 0x00009a30, 0x00008284, 0x00008284, 0x00000390, 0x00000390, 0x00000390 },
+    { 0x00009a34, 0x00008288, 0x00008288, 0x00000394, 0x00000394, 0x00000394 },
+    { 0x00009a38, 0x0000828c, 0x0000828c, 0x00000398, 0x00000398, 0x00000398 },
+    { 0x00009a3c, 0x00008290, 0x00008290, 0x00000334, 0x00000334, 0x00000334 },
+    { 0x00009a40, 0x00008300, 0x00008300, 0x00000338, 0x00000338, 0x00000338 },
+    { 0x00009a44, 0x00008304, 0x00008304, 0x000003ac, 0x000003ac, 0x000003ac },
+    { 0x00009a48, 0x00008308, 0x00008308, 0x000003b0, 0x000003b0, 0x000003b0 },
+    { 0x00009a4c, 0x0000830c, 0x0000830c, 0x000003b4, 0x000003b4, 0x000003b4 },
+    { 0x00009a50, 0x00008310, 0x00008310, 0x000003b8, 0x000003b8, 0x000003b8 },
+    { 0x00009a54, 0x00008314, 0x00008314, 0x000003a5, 0x000003a5, 0x000003a5 },
+    { 0x00009a58, 0x00008380, 0x00008380, 0x000003a9, 0x000003a9, 0x000003a9 },
+    { 0x00009a5c, 0x00008384, 0x00008384, 0x000003ad, 0x000003ad, 0x000003ad },
+    { 0x00009a60, 0x00008388, 0x00008388, 0x00008194, 0x00008194, 0x00008194 },
+    { 0x00009a64, 0x0000838c, 0x0000838c, 0x000081a0, 0x000081a0, 0x000081a0 },
+    { 0x00009a68, 0x00008390, 0x00008390, 0x0000820c, 0x0000820c, 0x0000820c },
+    { 0x00009a6c, 0x00008394, 0x00008394, 0x000081a8, 0x000081a8, 0x000081a8 },
+    { 0x00009a70, 0x0000a380, 0x0000a380, 0x00008284, 0x00008284, 0x00008284 },
+    { 0x00009a74, 0x0000a384, 0x0000a384, 0x00008288, 0x00008288, 0x00008288 },
+    { 0x00009a78, 0x0000a388, 0x0000a388, 0x00008224, 0x00008224, 0x00008224 },
+    { 0x00009a7c, 0x0000a38c, 0x0000a38c, 0x00008290, 0x00008290, 0x00008290 },
+    { 0x00009a80, 0x0000a390, 0x0000a390, 0x00008300, 0x00008300, 0x00008300 },
+    { 0x00009a84, 0x0000a394, 0x0000a394, 0x00008304, 0x00008304, 0x00008304 },
+    { 0x00009a88, 0x0000a780, 0x0000a780, 0x00008308, 0x00008308, 0x00008308 },
+    { 0x00009a8c, 0x0000a784, 0x0000a784, 0x0000830c, 0x0000830c, 0x0000830c },
+    { 0x00009a90, 0x0000a788, 0x0000a788, 0x00008380, 0x00008380, 0x00008380 },
+    { 0x00009a94, 0x0000a78c, 0x0000a78c, 0x00008384, 0x00008384, 0x00008384 },
+    { 0x00009a98, 0x0000a790, 0x0000a790, 0x00008700, 0x00008700, 0x00008700 },
+    { 0x00009a9c, 0x0000a794, 0x0000a794, 0x00008704, 0x00008704, 0x00008704 },
+    { 0x00009aa0, 0x0000ab84, 0x0000ab84, 0x00008708, 0x00008708, 0x00008708 },
+    { 0x00009aa4, 0x0000ab88, 0x0000ab88, 0x0000870c, 0x0000870c, 0x0000870c },
+    { 0x00009aa8, 0x0000ab8c, 0x0000ab8c, 0x00008780, 0x00008780, 0x00008780 },
+    { 0x00009aac, 0x0000ab90, 0x0000ab90, 0x00008784, 0x00008784, 0x00008784 },
+    { 0x00009ab0, 0x0000ab94, 0x0000ab94, 0x00008b00, 0x00008b00, 0x00008b00 },
+    { 0x00009ab4, 0x0000af80, 0x0000af80, 0x00008b04, 0x00008b04, 0x00008b04 },
+    { 0x00009ab8, 0x0000af84, 0x0000af84, 0x00008b08, 0x00008b08, 0x00008b08 },
+    { 0x00009abc, 0x0000af88, 0x0000af88, 0x00008b0c, 0x00008b0c, 0x00008b0c },
+    { 0x00009ac0, 0x0000af8c, 0x0000af8c, 0x00008b80, 0x00008b80, 0x00008b80 },
+    { 0x00009ac4, 0x0000af90, 0x0000af90, 0x00008b84, 0x00008b84, 0x00008b84 },
+    { 0x00009ac8, 0x0000af94, 0x0000af94, 0x00008b88, 0x00008b88, 0x00008b88 },
+    { 0x00009acc, 0x0000b380, 0x0000b380, 0x00008b8c, 0x00008b8c, 0x00008b8c },
+    { 0x00009ad0, 0x0000b384, 0x0000b384, 0x00008b90, 0x00008b90, 0x00008b90 },
+    { 0x00009ad4, 0x0000b388, 0x0000b388, 0x00008f80, 0x00008f80, 0x00008f80 },
+    { 0x00009ad8, 0x0000b38c, 0x0000b38c, 0x00008f84, 0x00008f84, 0x00008f84 },
+    { 0x00009adc, 0x0000b390, 0x0000b390, 0x00008f88, 0x00008f88, 0x00008f88 },
+    { 0x00009ae0, 0x0000b394, 0x0000b394, 0x00008f8c, 0x00008f8c, 0x00008f8c },
+    { 0x00009ae4, 0x0000b398, 0x0000b398, 0x00008f90, 0x00008f90, 0x00008f90 },
+    { 0x00009ae8, 0x0000b780, 0x0000b780, 0x0000930c, 0x0000930c, 0x0000930c },
+    { 0x00009aec, 0x0000b784, 0x0000b784, 0x00009310, 0x00009310, 0x00009310 },
+    { 0x00009af0, 0x0000b788, 0x0000b788, 0x00009384, 0x00009384, 0x00009384 },
+    { 0x00009af4, 0x0000b78c, 0x0000b78c, 0x00009388, 0x00009388, 0x00009388 },
+    { 0x00009af8, 0x0000b790, 0x0000b790, 0x00009324, 0x00009324, 0x00009324 },
+    { 0x00009afc, 0x0000b794, 0x0000b794, 0x00009704, 0x00009704, 0x00009704 },
+    { 0x00009b00, 0x0000b798, 0x0000b798, 0x000096a4, 0x000096a4, 0x000096a4 },
+    { 0x00009b04, 0x0000d784, 0x0000d784, 0x000096a8, 0x000096a8, 0x000096a8 },
+    { 0x00009b08, 0x0000d788, 0x0000d788, 0x00009710, 0x00009710, 0x00009710 },
+    { 0x00009b0c, 0x0000d78c, 0x0000d78c, 0x00009714, 0x00009714, 0x00009714 },
+    { 0x00009b10, 0x0000d790, 0x0000d790, 0x00009720, 0x00009720, 0x00009720 },
+    { 0x00009b14, 0x0000f780, 0x0000f780, 0x00009724, 0x00009724, 0x00009724 },
+    { 0x00009b18, 0x0000f784, 0x0000f784, 0x00009728, 0x00009728, 0x00009728 },
+    { 0x00009b1c, 0x0000f788, 0x0000f788, 0x0000972c, 0x0000972c, 0x0000972c },
+    { 0x00009b20, 0x0000f78c, 0x0000f78c, 0x000097a0, 0x000097a0, 0x000097a0 },
+    { 0x00009b24, 0x0000f790, 0x0000f790, 0x000097a4, 0x000097a4, 0x000097a4 },
+    { 0x00009b28, 0x0000f794, 0x0000f794, 0x000097a8, 0x000097a8, 0x000097a8 },
+    { 0x00009b2c, 0x0000f7a4, 0x0000f7a4, 0x000097b0, 0x000097b0, 0x000097b0 },
+    { 0x00009b30, 0x0000f7a8, 0x0000f7a8, 0x000097b4, 0x000097b4, 0x000097b4 },
+    { 0x00009b34, 0x0000f7ac, 0x0000f7ac, 0x000097b8, 0x000097b8, 0x000097b8 },
+    { 0x00009b38, 0x0000f7b0, 0x0000f7b0, 0x000097a5, 0x000097a5, 0x000097a5 },
+    { 0x00009b3c, 0x0000f7b4, 0x0000f7b4, 0x000097a9, 0x000097a9, 0x000097a9 },
+    { 0x00009b40, 0x0000f7a1, 0x0000f7a1, 0x000097ad, 0x000097ad, 0x000097ad },
+    { 0x00009b44, 0x0000f7a5, 0x0000f7a5, 0x000097b1, 0x000097b1, 0x000097b1 },
+    { 0x00009b48, 0x0000f7a9, 0x0000f7a9, 0x000097b5, 0x000097b5, 0x000097b5 },
+    { 0x00009b4c, 0x0000f7ad, 0x0000f7ad, 0x000097b9, 0x000097b9, 0x000097b9 },
+    { 0x00009b50, 0x0000f7b1, 0x0000f7b1, 0x000097c5, 0x000097c5, 0x000097c5 },
+    { 0x00009b54, 0x0000f7b5, 0x0000f7b5, 0x000097c9, 0x000097c9, 0x000097c9 },
+    { 0x00009b58, 0x0000f7c5, 0x0000f7c5, 0x000097d1, 0x000097d1, 0x000097d1 },
+    { 0x00009b5c, 0x0000f7c9, 0x0000f7c9, 0x000097d5, 0x000097d5, 0x000097d5 },
+    { 0x00009b60, 0x0000f7cd, 0x0000f7cd, 0x000097d9, 0x000097d9, 0x000097d9 },
+    { 0x00009b64, 0x0000f7d1, 0x0000f7d1, 0x000097c6, 0x000097c6, 0x000097c6 },
+    { 0x00009b68, 0x0000f7d5, 0x0000f7d5, 0x000097ca, 0x000097ca, 0x000097ca },
+    { 0x00009b6c, 0x0000f7c2, 0x0000f7c2, 0x000097ce, 0x000097ce, 0x000097ce },
+    { 0x00009b70, 0x0000f7c6, 0x0000f7c6, 0x000097d2, 0x000097d2, 0x000097d2 },
+    { 0x00009b74, 0x0000f7ca, 0x0000f7ca, 0x000097d6, 0x000097d6, 0x000097d6 },
+    { 0x00009b78, 0x0000f7ce, 0x0000f7ce, 0x000097c3, 0x000097c3, 0x000097c3 },
+    { 0x00009b7c, 0x0000f7d2, 0x0000f7d2, 0x000097c7, 0x000097c7, 0x000097c7 },
+    { 0x00009b80, 0x0000f7d6, 0x0000f7d6, 0x000097cb, 0x000097cb, 0x000097cb },
+    { 0x00009b84, 0x0000f7c3, 0x0000f7c3, 0x000097cf, 0x000097cf, 0x000097cf },
+    { 0x00009b88, 0x0000f7c7, 0x0000f7c7, 0x000097d7, 0x000097d7, 0x000097d7 },
+    { 0x00009b8c, 0x0000f7cb, 0x0000f7cb, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009b90, 0x0000f7d3, 0x0000f7d3, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009b94, 0x0000f7d7, 0x0000f7d7, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009b98, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009b9c, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009ba0, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009ba4, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009ba8, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bac, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bb0, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bb4, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bb8, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bbc, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bc0, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bc4, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bc8, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bcc, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bd0, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bd4, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bd8, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bdc, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009be0, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009be4, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009be8, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bec, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bf0, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bf4, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bf8, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bfc, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x0000a204, 0x00000444, 0x00000444, 0x00000444, 0x00000444, 0x00000444 },
+    { 0x0000a208, 0x803e4788, 0x803e4788, 0x803e4788, 0x803e4788, 0x803e4788 },
+    { 0x0000a20c, 0x000c6019, 0x000c6019, 0x000c6019, 0x000c6019, 0x000c6019 },
+    { 0x0000b20c, 0x000c6019, 0x000c6019, 0x000c6019, 0x000c6019, 0x000c6019 },
+    { 0x0000a21c, 0x1883800a, 0x1883800a, 0x1883800a, 0x1883800a, 0x1883800a },
+    { 0x0000a230, 0x00000000, 0x00000000, 0x00000210, 0x00000108, 0x00000000 },
+    { 0x0000a274, 0x0a19c652, 0x0a19c652, 0x0a1aa652, 0x0a1aa652, 0x0a1aa652 },
+    { 0x0000a300, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    { 0x0000a304, 0x00003002, 0x00003002, 0x00003002, 0x00003002, 0x00003002 },
+    { 0x0000a308, 0x00006004, 0x00006004, 0x00008009, 0x00008009, 0x00008009 },
+    { 0x0000a30c, 0x0000a006, 0x0000a006, 0x0000b00b, 0x0000b00b, 0x0000b00b },
+    { 0x0000a310, 0x0000e012, 0x0000e012, 0x0000e012, 0x0000e012, 0x0000e012 },
+    { 0x0000a314, 0x00011014, 0x00011014, 0x00012048, 0x00012048, 0x00012048 },
+    { 0x0000a318, 0x0001504a, 0x0001504a, 0x0001604a, 0x0001604a, 0x0001604a },
+    { 0x0000a31c, 0x0001904c, 0x0001904c, 0x0001a211, 0x0001a211, 0x0001a211 },
+    { 0x0000a320, 0x0001c04e, 0x0001c04e, 0x0001e213, 0x0001e213, 0x0001e213 },
+    { 0x0000a324, 0x00020092, 0x00020092, 0x0002121b, 0x0002121b, 0x0002121b },
+    { 0x0000a328, 0x0002410a, 0x0002410a, 0x00024412, 0x00024412, 0x00024412 },
+    { 0x0000a32c, 0x0002710c, 0x0002710c, 0x00028414, 0x00028414, 0x00028414 },
+    { 0x0000a330, 0x0002b18b, 0x0002b18b, 0x0002b44a, 0x0002b44a, 0x0002b44a },
+    { 0x0000a334, 0x0002e1cc, 0x0002e1cc, 0x00030649, 0x00030649, 0x00030649 },
+    { 0x0000a338, 0x000321ec, 0x000321ec, 0x0003364b, 0x0003364b, 0x0003364b },
+    { 0x0000a33c, 0x000321ec, 0x000321ec, 0x00038a49, 0x00038a49, 0x00038a49 },
+    { 0x0000a340, 0x000321ec, 0x000321ec, 0x0003be48, 0x0003be48, 0x0003be48 },
+    { 0x0000a344, 0x000321ec, 0x000321ec, 0x0003ee4a, 0x0003ee4a, 0x0003ee4a },
+    { 0x0000a348, 0x000321ec, 0x000321ec, 0x00042e88, 0x00042e88, 0x00042e88 },
+    { 0x0000a34c, 0x000321ec, 0x000321ec, 0x00046e8a, 0x00046e8a, 0x00046e8a },
+    { 0x0000a350, 0x000321ec, 0x000321ec, 0x00049ec9, 0x00049ec9, 0x00049ec9 },
+    { 0x0000a354, 0x000321ec, 0x000321ec, 0x0004bf42, 0x0004bf42, 0x0004bf42 },
+    { 0x0000784c, 0x0e4f048c, 0x0e4f048c, 0x0e4d048c, 0x0e4d048c, 0x0e4d048c },
+    { 0x00007854, 0x12031828, 0x12031828, 0x12035828, 0x12035828, 0x12035828 },
+    { 0x00007870, 0x807ec400, 0x807ec400, 0x807ec000, 0x807ec000, 0x807ec000 },
+    { 0x0000788c, 0x00010000, 0x00010000, 0x00110000, 0x00110000, 0x00110000 },
+};
+
+static const uint32_t ar9280Common_v1[][2] = {
+    { 0x0000000c, 0x00000000 },
+    { 0x00000030, 0x00020015 },
+    { 0x00000034, 0x00000005 },
+    { 0x00000040, 0x00000000 },
+    { 0x00000044, 0x00000008 },
+    { 0x00000048, 0x00000008 },
+    { 0x0000004c, 0x00000010 },
+    { 0x00000050, 0x00000000 },
+    { 0x00000054, 0x0000001f },
+    { 0x00000800, 0x00000000 },
+    { 0x00000804, 0x00000000 },
+    { 0x00000808, 0x00000000 },
+    { 0x0000080c, 0x00000000 },
+    { 0x00000810, 0x00000000 },
+    { 0x00000814, 0x00000000 },
+    { 0x00000818, 0x00000000 },
+    { 0x0000081c, 0x00000000 },
+    { 0x00000820, 0x00000000 },
+    { 0x00000824, 0x00000000 },
+    { 0x00001040, 0x002ffc0f },
+    { 0x00001044, 0x002ffc0f },
+    { 0x00001048, 0x002ffc0f },
+    { 0x0000104c, 0x002ffc0f },
+    { 0x00001050, 0x002ffc0f },
+    { 0x00001054, 0x002ffc0f },
+    { 0x00001058, 0x002ffc0f },
+    { 0x0000105c, 0x002ffc0f },
+    { 0x00001060, 0x002ffc0f },
+    { 0x00001064, 0x002ffc0f },
+    { 0x00001230, 0x00000000 },
+    { 0x00001270, 0x00000000 },
+    { 0x00001038, 0x00000000 },
+    { 0x00001078, 0x00000000 },
+    { 0x000010b8, 0x00000000 },
+    { 0x000010f8, 0x00000000 },
+    { 0x00001138, 0x00000000 },
+    { 0x00001178, 0x00000000 },
+    { 0x000011b8, 0x00000000 },
+    { 0x000011f8, 0x00000000 },
+    { 0x00001238, 0x00000000 },
+    { 0x00001278, 0x00000000 },
+    { 0x000012b8, 0x00000000 },
+    { 0x000012f8, 0x00000000 },
+    { 0x00001338, 0x00000000 },
+    { 0x00001378, 0x00000000 },
+    { 0x000013b8, 0x00000000 },
+    { 0x000013f8, 0x00000000 },
+    { 0x00001438, 0x00000000 },
+    { 0x00001478, 0x00000000 },
+    { 0x000014b8, 0x00000000 },
+    { 0x000014f8, 0x00000000 },
+    { 0x00001538, 0x00000000 },
+    { 0x00001578, 0x00000000 },
+    { 0x000015b8, 0x00000000 },
+    { 0x000015f8, 0x00000000 },
+    { 0x00001638, 0x00000000 },
+    { 0x00001678, 0x00000000 },
+    { 0x000016b8, 0x00000000 },
+    { 0x000016f8, 0x00000000 },
+    { 0x00001738, 0x00000000 },
+    { 0x00001778, 0x00000000 },
+    { 0x000017b8, 0x00000000 },
+    { 0x000017f8, 0x00000000 },
+    { 0x0000103c, 0x00000000 },
+    { 0x0000107c, 0x00000000 },
+    { 0x000010bc, 0x00000000 },
+    { 0x000010fc, 0x00000000 },
+    { 0x0000113c, 0x00000000 },
+    { 0x0000117c, 0x00000000 },
+    { 0x000011bc, 0x00000000 },
+    { 0x000011fc, 0x00000000 },
+    { 0x0000123c, 0x00000000 },
+    { 0x0000127c, 0x00000000 },
+    { 0x000012bc, 0x00000000 },
+    { 0x000012fc, 0x00000000 },
+    { 0x0000133c, 0x00000000 },
+    { 0x0000137c, 0x00000000 },
+    { 0x000013bc, 0x00000000 },
+    { 0x000013fc, 0x00000000 },
+    { 0x0000143c, 0x00000000 },
+    { 0x0000147c, 0x00000000 },
+    { 0x00004030, 0x00000002 },
+    { 0x0000403c, 0x00000002 },
+    { 0x00004024, 0x0000001f },
+    { 0x00007010, 0x00000033 },
+    { 0x00007038, 0x000004c2 },
+    { 0x00008004, 0x00000000 },
+    { 0x00008008, 0x00000000 },
+    { 0x0000800c, 0x00000000 },
+    { 0x00008018, 0x00000700 },
+    { 0x00008020, 0x00000000 },
+    { 0x00008038, 0x00000000 },
+    { 0x0000803c, 0x00000000 },
+    { 0x00008048, 0x40000000 },
+    { 0x00008054, 0x00000000 },
+    { 0x00008058, 0x00000000 },
+    { 0x0000805c, 0x000fc78f },
+    { 0x00008060, 0x0000000f },
+    { 0x00008064, 0x00000000 },
+    { 0x00008070, 0x00000000 },
+    { 0x000080c0, 0x2a82301a },
+    { 0x000080c4, 0x05dc01e0 },
+    { 0x000080c8, 0x1f402710 },
+    { 0x000080cc, 0x01f40000 },
+    { 0x000080d0, 0x00001e00 },
+    { 0x000080d4, 0x00000000 },
+    { 0x000080d8, 0x00400000 },
+    { 0x000080e0, 0xffffffff },
+    { 0x000080e4, 0x0000ffff },
+    { 0x000080e8, 0x003f3f3f },
+    { 0x000080ec, 0x00000000 },
+    { 0x000080f0, 0x00000000 },
+    { 0x000080f4, 0x00000000 },
+    { 0x000080f8, 0x00000000 },
+    { 0x000080fc, 0x00020000 },
+    { 0x00008100, 0x00020000 },
+    { 0x00008104, 0x00000001 },
+    { 0x00008108, 0x00000052 },
+    { 0x0000810c, 0x00000000 },
+    { 0x00008110, 0x00000168 },
+    { 0x00008118, 0x000100aa },
+    { 0x0000811c, 0x00003210 },
+    { 0x00008120, 0x08f04800 },
+    { 0x00008124, 0x00000000 },
+    { 0x00008128, 0x00000000 },
+    { 0x0000812c, 0x00000000 },
+    { 0x00008130, 0x00000000 },
+    { 0x00008134, 0x00000000 },
+    { 0x00008138, 0x00000000 },
+    { 0x0000813c, 0x00000000 },
+    { 0x00008144, 0x00000000 },
+    { 0x00008168, 0x00000000 },
+    { 0x0000816c, 0x00000000 },
+    { 0x00008170, 0x32143320 },
+    { 0x00008174, 0xfaa4fa50 },
+    { 0x00008178, 0x00000100 },
+    { 0x0000817c, 0x00000000 },
+    { 0x000081c4, 0x00000000 },
+    { 0x000081d0, 0x00003210 },
+    { 0x000081ec, 0x00000000 },
+    { 0x000081f0, 0x00000000 },
+    { 0x000081f4, 0x00000000 },
+    { 0x000081f8, 0x00000000 },
+    { 0x000081fc, 0x00000000 },
+    { 0x00008200, 0x00000000 },
+    { 0x00008204, 0x00000000 },
+    { 0x00008208, 0x00000000 },
+    { 0x0000820c, 0x00000000 },
+    { 0x00008210, 0x00000000 },
+    { 0x00008214, 0x00000000 },
+    { 0x00008218, 0x00000000 },
+    { 0x0000821c, 0x00000000 },
+    { 0x00008220, 0x00000000 },
+    { 0x00008224, 0x00000000 },
+    { 0x00008228, 0x00000000 },
+    { 0x0000822c, 0x00000000 },
+    { 0x00008230, 0x00000000 },
+    { 0x00008234, 0x00000000 },
+    { 0x00008238, 0x00000000 },
+    { 0x0000823c, 0x00000000 },
+    { 0x00008240, 0x00100000 },
+    { 0x00008244, 0x0010f400 },
+    { 0x00008248, 0x00000100 },
+    { 0x0000824c, 0x0001e800 },
+    { 0x00008250, 0x00000000 },
+    { 0x00008254, 0x00000000 },
+    { 0x00008258, 0x00000000 },
+    { 0x0000825c, 0x400000ff },
+    { 0x00008260, 0x00080922 },
+    { 0x00008270, 0x00000000 },
+    { 0x00008274, 0x40000000 },
+    { 0x00008278, 0x003e4180 },
+    { 0x0000827c, 0x00000000 },
+    { 0x00008284, 0x0000002c },
+    { 0x00008288, 0x0000002c },
+    { 0x0000828c, 0x00000000 },
+    { 0x00008294, 0x00000000 },
+    { 0x00008298, 0x00000000 },
+    { 0x00008300, 0x00000000 },
+    { 0x00008304, 0x00000000 },
+    { 0x00008308, 0x00000000 },
+    { 0x0000830c, 0x00000000 },
+    { 0x00008310, 0x00000000 },
+    { 0x00008314, 0x00000000 },
+    { 0x00008318, 0x00000000 },
+    { 0x00008328, 0x00000000 },
+    { 0x0000832c, 0x00000007 },
+    { 0x00008330, 0x00000302 },
+    { 0x00008334, 0x00000e00 },
+    { 0x00008338, 0x00000000 },
+    { 0x0000833c, 0x00000000 },
+    { 0x00008340, 0x000107ff },
+    { 0x00008344, 0x00000000 },
+    { 0x00009808, 0x00000000 },
+    { 0x0000980c, 0xaf268e30 },
+    { 0x00009810, 0xfd14e000 },
+    { 0x00009814, 0x9c0a9f6b },
+    { 0x0000981c, 0x00000000 },
+    { 0x0000982c, 0x0000a000 },
+    { 0x00009830, 0x00000000 },
+    { 0x0000983c, 0x00200400 },
+    { 0x00009840, 0x206a01ae },
+    { 0x0000984c, 0x0040233c },
+    { 0x0000a84c, 0x0040233c },
+    { 0x00009854, 0x00000044 },
+    { 0x00009900, 0x00000000 },
+    { 0x00009904, 0x00000000 },
+    { 0x00009908, 0x00000000 },
+    { 0x0000990c, 0x00000000 },
+    { 0x0000991c, 0x10000fff },
+    { 0x00009920, 0x04900000 },
+    { 0x0000a920, 0x04900000 },
+    { 0x00009928, 0x00000001 },
+    { 0x0000992c, 0x00000004 },
+    { 0x00009934, 0x1e1f2022 },
+    { 0x00009938, 0x0a0b0c0d },
+    { 0x0000993c, 0x00000000 },
+    { 0x00009948, 0x9280c00a },
+    { 0x0000994c, 0x00020028 },
+    { 0x00009954, 0xe250a51e },
+    { 0x00009958, 0x3388ffff },
+    { 0x00009940, 0x00781204 },
+    { 0x0000c95c, 0x004b6a8e },
+    { 0x0000c968, 0x000003ce },
+    { 0x00009970, 0x190fb514 },
+    { 0x00009974, 0x00000000 },
+    { 0x00009978, 0x00000001 },
+    { 0x0000997c, 0x00000000 },
+    { 0x00009980, 0x00000000 },
+    { 0x00009984, 0x00000000 },
+    { 0x00009988, 0x00000000 },
+    { 0x0000998c, 0x00000000 },
+    { 0x00009990, 0x00000000 },
+    { 0x00009994, 0x00000000 },
+    { 0x00009998, 0x00000000 },
+    { 0x0000999c, 0x00000000 },
+    { 0x000099a0, 0x00000000 },
+    { 0x000099a4, 0x00000001 },
+    { 0x000099a8, 0x201fff00 },
+    { 0x000099ac, 0x006f00c4 },
+    { 0x000099b0, 0x03051000 },
+    { 0x000099b4, 0x00000820 },
+    { 0x000099dc, 0x00000000 },
+    { 0x000099e0, 0x00000000 },
+    { 0x000099e4, 0xaaaaaaaa },
+    { 0x000099e8, 0x3c466478 },
+    { 0x000099ec, 0x0cc80caa },
+    { 0x000099fc, 0x00001042 },
+    { 0x0000a210, 0x4080a333 },
+    { 0x0000a214, 0x40206c10 },
+    { 0x0000a218, 0x009c4060 },
+    { 0x0000a220, 0x01834061 },
+    { 0x0000a224, 0x00000400 },
+    { 0x0000a228, 0x000003b5 },
+    { 0x0000a22c, 0x23277200 },
+    { 0x0000a234, 0x20202020 },
+    { 0x0000a238, 0x20202020 },
+    { 0x0000a23c, 0x13c889af },
+    { 0x0000a240, 0x38490a20 },
+    { 0x0000a244, 0x00007bb6 },
+    { 0x0000a248, 0x0fff3ffc },
+    { 0x0000a24c, 0x00000001 },
+    { 0x0000a250, 0x001da000 },
+    { 0x0000a254, 0x00000000 },
+    { 0x0000a258, 0x0cdbd380 },
+    { 0x0000a25c, 0x0f0f0f01 },
+    { 0x0000a260, 0xdfa91f01 },
+    { 0x0000a268, 0x00000000 },
+    { 0x0000a26c, 0x0ebae9c6 },
+    { 0x0000b26c, 0x0ebae9c6 },
+    { 0x0000d270, 0x00820820 },
+    { 0x0000a278, 0x1ce739ce },
+    { 0x0000a27c, 0x050701ce },
+    { 0x0000a358, 0x7999aa0f },
+    { 0x0000d35c, 0x07ffffef },
+    { 0x0000d360, 0x0fffffe7 },
+    { 0x0000d364, 0x17ffffe5 },
+    { 0x0000d368, 0x1fffffe4 },
+    { 0x0000d36c, 0x37ffffe3 },
+    { 0x0000d370, 0x3fffffe3 },
+    { 0x0000d374, 0x57ffffe3 },
+    { 0x0000d378, 0x5fffffe2 },
+    { 0x0000d37c, 0x7fffffe2 },
+    { 0x0000d380, 0x7f3c7bba },
+    { 0x0000d384, 0xf3307ff0 },
+    { 0x0000a388, 0x0c000000 },
+    { 0x0000a38c, 0x20202020 },
+    { 0x0000a390, 0x20202020 },
+    { 0x0000a394, 0x1ce739ce },
+    { 0x0000a398, 0x000001ce },
+    { 0x0000a39c, 0x00000001 },
+    { 0x0000a3a0, 0x00000000 },
+    { 0x0000a3a4, 0x00000000 },
+    { 0x0000a3a8, 0x00000000 },
+    { 0x0000a3ac, 0x00000000 },
+    { 0x0000a3b0, 0x00000000 },
+    { 0x0000a3b4, 0x00000000 },
+    { 0x0000a3b8, 0x00000000 },
+    { 0x0000a3bc, 0x00000000 },
+    { 0x0000a3c0, 0x00000000 },
+    { 0x0000a3c4, 0x00000000 },
+    { 0x0000a3c8, 0x00000246 },
+    { 0x0000a3cc, 0x20202020 },
+    { 0x0000a3d0, 0x20202020 },
+    { 0x0000a3d4, 0x20202020 },
+    { 0x0000a3dc, 0x1ce739ce },
+    { 0x0000a3e0, 0x000001ce },
+    { 0x0000a3e4, 0x00000000 },
+    { 0x0000a3e8, 0x18c43433 },
+    { 0x0000a3ec, 0x00f38081 },
+    { 0x00007800, 0x00040000 },
+    { 0x00007804, 0xdb005012 },
+    { 0x00007808, 0x04924914 },
+    { 0x0000780c, 0x21084210 },
+    { 0x00007810, 0x6d801300 },
+    { 0x00007814, 0x0019beff },
+    { 0x00007818, 0x07e40000 },
+    { 0x0000781c, 0x00492000 },
+    { 0x00007820, 0x92492480 },
+    { 0x00007824, 0x00040000 },
+    { 0x00007828, 0xdb005012 },
+    { 0x0000782c, 0x04924914 },
+    { 0x00007830, 0x21084210 },
+    { 0x00007834, 0x6d801300 },
+    { 0x00007838, 0x0019beff },
+    { 0x0000783c, 0x07e40000 },
+    { 0x00007840, 0x00492000 },
+    { 0x00007844, 0x92492480 },
+    { 0x00007848, 0x00120000 },
+    { 0x00007850, 0x54214514 },
+    { 0x00007858, 0x92592692 },
+    { 0x00007860, 0x52802000 },
+    { 0x00007864, 0x0a8e370e },
+    { 0x00007868, 0xc0102850 },
+    { 0x0000786c, 0x812d4000 },
+    { 0x00007874, 0x001b6db0 },
+    { 0x00007878, 0x00376b63 },
+    { 0x0000787c, 0x06db6db6 },
+    { 0x00007880, 0x006d8000 },
+    { 0x00007884, 0xffeffffe },
+    { 0x00007888, 0xffeffffe },
+    { 0x00007890, 0x00060aeb },
+    { 0x00007894, 0x5a108000 },
+    { 0x00007898, 0x2a850160 },
+};
+
+/* hand-crafted from code that does explicit register writes */
+static const uint32_t ar9280PciePhy_v1[][2] = {
+    { AR_PCIE_SERDES,  0x9248fd00 },
+    { AR_PCIE_SERDES,  0x24924924 },
+    { AR_PCIE_SERDES,  0xa8000019 },
+    { AR_PCIE_SERDES,  0x13160820 },
+    { AR_PCIE_SERDES,  0xe5980560 },
+    { AR_PCIE_SERDES,  0x401deffd },
+    { AR_PCIE_SERDES,  0x1aaabe40 },
+    { AR_PCIE_SERDES,  0xbe105554 },
+    { AR_PCIE_SERDES,  0x00043007 },
+    { AR_PCIE_SERDES2, 0x00000000 },
+};
diff --git a/sys/dev/ath/ath_hal/ar5416/ar9280v2.ini b/sys/dev/ath/ath_hal/ar5416/ar9280v2.ini
new file mode 100644
index 0000000..aa8e524
--- /dev/null
+++ b/sys/dev/ath/ath_hal/ar5416/ar9280v2.ini
@@ -0,0 +1,941 @@
+/*
+ * Copyright (c) 2002-2009 Sam Leffler, Errno Consulting
+ * Copyright (c) 2002-2008 Atheros Communications, Inc.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * $FreeBSD$
+ */
+/* Auto Generated PCI Register Writes.  Created: 10/15/08 */
+
+static const uint32_t ar9280Modes_v2[][6] = {
+    /* Address      5G-HT20     5G-HT40     2G-HT40     2G-HT20     Turbo   */
+    { 0x00001030, 0x00000230, 0x00000460, 0x000002c0, 0x00000160, 0x000001e0 },
+    { 0x00001070, 0x00000168, 0x000002d0, 0x00000318, 0x0000018c, 0x000001e0 },
+    { 0x000010b0, 0x00000e60, 0x00001cc0, 0x00007c70, 0x00003e38, 0x00001180 },
+    { 0x000010f0, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000008 },
+    { 0x00008014, 0x03e803e8, 0x07d007d0, 0x10801600, 0x08400b00, 0x06e006e0 },
+    { 0x0000801c, 0x128d8027, 0x128d804f, 0x12e00057, 0x12e0002b, 0x0988004f },
+    { 0x00008318, 0x00003e80, 0x00007d00, 0x00006880, 0x00003440, 0x00006880 },
+    { 0x00009804, 0x00000300, 0x000003c4, 0x000003c4, 0x00000300, 0x00000303 },
+    { 0x00009820, 0x02020200, 0x02020200, 0x02020200, 0x02020200, 0x02020200 },
+    { 0x00009824, 0x00000e0e, 0x00000e0e, 0x00000e0e, 0x00000e0e, 0x00000e0e },
+    { 0x00009828, 0x0a020001, 0x0a020001, 0x0a020001, 0x0a020001, 0x0a020001 },
+    { 0x00009834, 0x00000e0e, 0x00000e0e, 0x00000e0e, 0x00000e0e, 0x00000e0e },
+    { 0x00009838, 0x00000007, 0x00000007, 0x00000007, 0x00000007, 0x00000007 },
+    { 0x00009840, 0x206a022e, 0x206a022e, 0x206a012e, 0x206a012e, 0x206a012e },
+    { 0x00009844, 0x0372161e, 0x0372161e, 0x037216a0, 0x037216a0, 0x037216a0 },
+    { 0x00009850, 0x6c4000e2, 0x6c4000e2, 0x6d4000e2, 0x6c4000e2, 0x6c4000e2 },
+    { 0x00009858, 0x7ec88d2e, 0x7ec88d2e, 0x7ec84d2e, 0x7ec84d2e, 0x7ec84d2e },
+    { 0x0000985c, 0x31395d5e, 0x31395d5e, 0x3139605e, 0x31395d5e, 0x31395d5e },
+    { 0x00009860, 0x00048d18, 0x00048d18, 0x00048d20, 0x00048d20, 0x00048d18 },
+    { 0x00009864, 0x0001ce00, 0x0001ce00, 0x0001ce00, 0x0001ce00, 0x0001ce00 },
+    { 0x00009868, 0x5ac640d0, 0x5ac640d0, 0x5ac640d0, 0x5ac640d0, 0x5ac640d0 },
+    { 0x0000986c, 0x06903081, 0x06903081, 0x06903881, 0x06903881, 0x06903881 },
+    { 0x00009914, 0x000007d0, 0x00000fa0, 0x00001130, 0x00000898, 0x000007d0 },
+    { 0x00009918, 0x0000000a, 0x00000014, 0x00000016, 0x0000000b, 0x00000016 },
+    { 0x00009924, 0xd00a8a0b, 0xd00a8a0b, 0xd00a8a0d, 0xd00a8a0d, 0xd00a8a0d },
+    { 0x00009944, 0xffbc1010, 0xffbc1010, 0xffbc1010, 0xffbc1010, 0xffbc1010 },
+    { 0x00009960, 0x00000010, 0x00000010, 0x00000010, 0x00000010, 0x00000010 },
+    { 0x0000a960, 0x00000010, 0x00000010, 0x00000010, 0x00000010, 0x00000010 },
+    { 0x00009964, 0x00000210, 0x00000210, 0x00000210, 0x00000210, 0x00000210 },
+    { 0x000099b8, 0x0000001c, 0x0000001c, 0x0000001c, 0x0000001c, 0x0000001c },
+    { 0x000099bc, 0x00000a00, 0x00000a00, 0x00000c00, 0x00000c00, 0x00000c00 },
+    { 0x000099c0, 0x05eea6d4, 0x05eea6d4, 0x05eea6d4, 0x05eea6d4, 0x05eea6d4 },
+    { 0x000099c4, 0x06336f77, 0x06336f77, 0x06336f77, 0x06336f77, 0x06336f77 },
+    { 0x000099c8, 0x60f65329, 0x60f65329, 0x60f65329, 0x60f65329, 0x60f65329 },
+    { 0x000099cc, 0x08f186c8, 0x08f186c8, 0x08f186c8, 0x08f186c8, 0x08f186c8 },
+    { 0x000099d0, 0x00046384, 0x00046384, 0x00046384, 0x00046384, 0x00046384 },
+    { 0x000099d4, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    { 0x000099d8, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    { 0x0000a204, 0x00000444, 0x00000444, 0x00000444, 0x00000444, 0x00000444 },
+    { 0x0000a20c, 0x00000014, 0x00000014, 0x0001f019, 0x0001f019, 0x0001f019 },
+    { 0x0000b20c, 0x00000014, 0x00000014, 0x0001f019, 0x0001f019, 0x0001f019 },
+    { 0x0000a21c, 0x1883800a, 0x1883800a, 0x1883800a, 0x1883800a, 0x1883800a },
+    { 0x0000a230, 0x00000000, 0x00000000, 0x00000210, 0x00000108, 0x00000000 },
+    { 0x0000a250, 0x001ff000, 0x001ff000, 0x0004a000, 0x0004a000, 0x0004a000 },
+    { 0x0000a274, 0x0a19c652, 0x0a19c652, 0x0a1aa652, 0x0a1aa652, 0x0a1aa652 },
+    { 0x0000a358, 0x7999aa02, 0x7999aa02, 0x7999aa0e, 0x7999aa0e, 0x7999aa0e },
+    { 0x0000a3d8, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    { 0x00007894, 0x5a508000, 0x5a508000, 0x5a508000, 0x5a508000, 0x5a508000 },
+};
+
+static const uint32_t ar9280Common_v2[][2] = {
+    { 0x0000000c, 0x00000000 },
+    { 0x00000030, 0x00020015 },
+    { 0x00000034, 0x00000005 },
+    { 0x00000040, 0x00000000 },
+    { 0x00000044, 0x00000008 },
+    { 0x00000048, 0x00000008 },
+    { 0x0000004c, 0x00000010 },
+    { 0x00000050, 0x00000000 },
+    { 0x00000054, 0x0000001f },
+    { 0x00000800, 0x00000000 },
+    { 0x00000804, 0x00000000 },
+    { 0x00000808, 0x00000000 },
+    { 0x0000080c, 0x00000000 },
+    { 0x00000810, 0x00000000 },
+    { 0x00000814, 0x00000000 },
+    { 0x00000818, 0x00000000 },
+    { 0x0000081c, 0x00000000 },
+    { 0x00000820, 0x00000000 },
+    { 0x00000824, 0x00000000 },
+    { 0x00001040, 0x002ffc0f },
+    { 0x00001044, 0x002ffc0f },
+    { 0x00001048, 0x002ffc0f },
+    { 0x0000104c, 0x002ffc0f },
+    { 0x00001050, 0x002ffc0f },
+    { 0x00001054, 0x002ffc0f },
+    { 0x00001058, 0x002ffc0f },
+    { 0x0000105c, 0x002ffc0f },
+    { 0x00001060, 0x002ffc0f },
+    { 0x00001064, 0x002ffc0f },
+    { 0x00001230, 0x00000000 },
+    { 0x00001270, 0x00000000 },
+    { 0x00001038, 0x00000000 },
+    { 0x00001078, 0x00000000 },
+    { 0x000010b8, 0x00000000 },
+    { 0x000010f8, 0x00000000 },
+    { 0x00001138, 0x00000000 },
+    { 0x00001178, 0x00000000 },
+    { 0x000011b8, 0x00000000 },
+    { 0x000011f8, 0x00000000 },
+    { 0x00001238, 0x00000000 },
+    { 0x00001278, 0x00000000 },
+    { 0x000012b8, 0x00000000 },
+    { 0x000012f8, 0x00000000 },
+    { 0x00001338, 0x00000000 },
+    { 0x00001378, 0x00000000 },
+    { 0x000013b8, 0x00000000 },
+    { 0x000013f8, 0x00000000 },
+    { 0x00001438, 0x00000000 },
+    { 0x00001478, 0x00000000 },
+    { 0x000014b8, 0x00000000 },
+    { 0x000014f8, 0x00000000 },
+    { 0x00001538, 0x00000000 },
+    { 0x00001578, 0x00000000 },
+    { 0x000015b8, 0x00000000 },
+    { 0x000015f8, 0x00000000 },
+    { 0x00001638, 0x00000000 },
+    { 0x00001678, 0x00000000 },
+    { 0x000016b8, 0x00000000 },
+    { 0x000016f8, 0x00000000 },
+    { 0x00001738, 0x00000000 },
+    { 0x00001778, 0x00000000 },
+    { 0x000017b8, 0x00000000 },
+    { 0x000017f8, 0x00000000 },
+    { 0x0000103c, 0x00000000 },
+    { 0x0000107c, 0x00000000 },
+    { 0x000010bc, 0x00000000 },
+    { 0x000010fc, 0x00000000 },
+    { 0x0000113c, 0x00000000 },
+    { 0x0000117c, 0x00000000 },
+    { 0x000011bc, 0x00000000 },
+    { 0x000011fc, 0x00000000 },
+    { 0x0000123c, 0x00000000 },
+    { 0x0000127c, 0x00000000 },
+    { 0x000012bc, 0x00000000 },
+    { 0x000012fc, 0x00000000 },
+    { 0x0000133c, 0x00000000 },
+    { 0x0000137c, 0x00000000 },
+    { 0x000013bc, 0x00000000 },
+    { 0x000013fc, 0x00000000 },
+    { 0x0000143c, 0x00000000 },
+    { 0x0000147c, 0x00000000 },
+    { 0x00004030, 0x00000002 },
+    { 0x0000403c, 0x00000002 },
+    { 0x00004024, 0x0000001f },
+    { 0x00004060, 0x00000000 },
+    { 0x00004064, 0x00000000 },
+    { 0x00007010, 0x00000033 },
+    { 0x00007034, 0x00000002 },
+    { 0x00007038, 0x000004c2 },
+    { 0x00008004, 0x00000000 },
+    { 0x00008008, 0x00000000 },
+    { 0x0000800c, 0x00000000 },
+    { 0x00008018, 0x00000700 },
+    { 0x00008020, 0x00000000 },
+    { 0x00008038, 0x00000000 },
+    { 0x0000803c, 0x00000000 },
+    { 0x00008048, 0x40000000 },
+    { 0x00008054, 0x00000000 },
+    { 0x00008058, 0x00000000 },
+    { 0x0000805c, 0x000fc78f },
+    { 0x00008060, 0x0000000f },
+    { 0x00008064, 0x00000000 },
+    { 0x00008070, 0x00000000 },
+    { 0x000080c0, 0x2a80001a },
+    { 0x000080c4, 0x05dc01e0 },
+    { 0x000080c8, 0x1f402710 },
+    { 0x000080cc, 0x01f40000 },
+    { 0x000080d0, 0x00001e00 },
+    { 0x000080d4, 0x00000000 },
+    { 0x000080d8, 0x00400000 },
+    { 0x000080e0, 0xffffffff },
+    { 0x000080e4, 0x0000ffff },
+    { 0x000080e8, 0x003f3f3f },
+    { 0x000080ec, 0x00000000 },
+    { 0x000080f0, 0x00000000 },
+    { 0x000080f4, 0x00000000 },
+    { 0x000080f8, 0x00000000 },
+    { 0x000080fc, 0x00020000 },
+    { 0x00008100, 0x00020000 },
+    { 0x00008104, 0x00000001 },
+    { 0x00008108, 0x00000052 },
+    { 0x0000810c, 0x00000000 },
+    { 0x00008110, 0x00000168 },
+    { 0x00008118, 0x000100aa },
+    { 0x0000811c, 0x00003210 },
+    { 0x00008120, 0x08f04800 },
+    { 0x00008124, 0x00000000 },
+    { 0x00008128, 0x00000000 },
+    { 0x0000812c, 0x00000000 },
+    { 0x00008130, 0x00000000 },
+    { 0x00008134, 0x00000000 },
+    { 0x00008138, 0x00000000 },
+    { 0x0000813c, 0x00000000 },
+    { 0x00008144, 0xffffffff },
+    { 0x00008168, 0x00000000 },
+    { 0x0000816c, 0x00000000 },
+    { 0x00008170, 0x32143320 },
+    { 0x00008174, 0xfaa4fa50 },
+    { 0x00008178, 0x00000100 },
+    { 0x0000817c, 0x00000000 },
+    { 0x000081c0, 0x00000000 },
+    { 0x000081d0, 0x00003210 },
+    { 0x000081ec, 0x00000000 },
+    { 0x000081f0, 0x00000000 },
+    { 0x000081f4, 0x00000000 },
+    { 0x000081f8, 0x00000000 },
+    { 0x000081fc, 0x00000000 },
+    { 0x00008200, 0x00000000 },
+    { 0x00008204, 0x00000000 },
+    { 0x00008208, 0x00000000 },
+    { 0x0000820c, 0x00000000 },
+    { 0x00008210, 0x00000000 },
+    { 0x00008214, 0x00000000 },
+    { 0x00008218, 0x00000000 },
+    { 0x0000821c, 0x00000000 },
+    { 0x00008220, 0x00000000 },
+    { 0x00008224, 0x00000000 },
+    { 0x00008228, 0x00000000 },
+    { 0x0000822c, 0x00000000 },
+    { 0x00008230, 0x00000000 },
+    { 0x00008234, 0x00000000 },
+    { 0x00008238, 0x00000000 },
+    { 0x0000823c, 0x00000000 },
+    { 0x00008240, 0x00100000 },
+    { 0x00008244, 0x0010f400 },
+    { 0x00008248, 0x00000100 },
+    { 0x0000824c, 0x0001e800 },
+    { 0x00008250, 0x00000000 },
+    { 0x00008254, 0x00000000 },
+    { 0x00008258, 0x00000000 },
+    { 0x0000825c, 0x400000ff },
+    { 0x00008260, 0x00080922 },
+    { 0x00008264, 0xa8a00010 },
+    { 0x00008270, 0x00000000 },
+    { 0x00008274, 0x40000000 },
+    { 0x00008278, 0x003e4180 },
+    { 0x0000827c, 0x00000000 },
+    { 0x00008284, 0x0000002c },
+    { 0x00008288, 0x0000002c },
+    { 0x0000828c, 0x00000000 },
+    { 0x00008294, 0x00000000 },
+    { 0x00008298, 0x00000000 },
+    { 0x0000829c, 0x00000000 },
+    { 0x00008300, 0x00000040 },
+    { 0x00008314, 0x00000000 },
+    { 0x00008328, 0x00000000 },
+    { 0x0000832c, 0x00000007 },
+    { 0x00008330, 0x00000302 },
+    { 0x00008334, 0x00000e00 },
+    { 0x00008338, 0x00ff0000 },
+    { 0x0000833c, 0x00000000 },
+    { 0x00008340, 0x000107ff },
+    { 0x00008344, 0x00581043 },
+    { 0x00009808, 0x00000000 },
+    { 0x0000980c, 0xafa68e30 },
+    { 0x00009810, 0xfd14e000 },
+    { 0x00009814, 0x9c0a9f6b },
+    { 0x0000981c, 0x00000000 },
+    { 0x0000982c, 0x0000a000 },
+    { 0x00009830, 0x00000000 },
+    { 0x0000983c, 0x00200400 },
+    { 0x0000984c, 0x0040233c },
+    { 0x0000a84c, 0x0040233c },
+    { 0x00009854, 0x00000044 },
+    { 0x00009900, 0x00000000 },
+    { 0x00009904, 0x00000000 },
+    { 0x00009908, 0x00000000 },
+    { 0x0000990c, 0x00000000 },
+    { 0x00009910, 0x01002310 },
+    { 0x0000991c, 0x10000fff },
+    { 0x00009920, 0x04900000 },
+    { 0x0000a920, 0x04900000 },
+    { 0x00009928, 0x00000001 },
+    { 0x0000992c, 0x00000004 },
+    { 0x00009934, 0x1e1f2022 },
+    { 0x00009938, 0x0a0b0c0d },
+    { 0x0000993c, 0x00000000 },
+    { 0x00009948, 0x9280c00a },
+    { 0x0000994c, 0x00020028 },
+    { 0x00009954, 0x5f3ca3de },
+    { 0x00009958, 0x2108ecff },
+    { 0x00009940, 0x14750604 },
+    { 0x0000c95c, 0x004b6a8e },
+    { 0x00009968, 0x000003ce },
+    { 0x00009970, 0x190fb515 },
+    { 0x00009974, 0x00000000 },
+    { 0x00009978, 0x00000001 },
+    { 0x0000997c, 0x00000000 },
+    { 0x00009980, 0x00000000 },
+    { 0x00009984, 0x00000000 },
+    { 0x00009988, 0x00000000 },
+    { 0x0000998c, 0x00000000 },
+    { 0x00009990, 0x00000000 },
+    { 0x00009994, 0x00000000 },
+    { 0x00009998, 0x00000000 },
+    { 0x0000999c, 0x00000000 },
+    { 0x000099a0, 0x00000000 },
+    { 0x000099a4, 0x00000001 },
+    { 0x000099a8, 0x201fff00 },
+    { 0x000099ac, 0x006f0000 },
+    { 0x000099b0, 0x03051000 },
+    { 0x000099b4, 0x00000820 },
+    { 0x000099dc, 0x00000000 },
+    { 0x000099e0, 0x00000000 },
+    { 0x000099e4, 0xaaaaaaaa },
+    { 0x000099e8, 0x3c466478 },
+    { 0x000099ec, 0x0cc80caa },
+    { 0x000099f0, 0x00000000 },
+    { 0x000099fc, 0x00001042 },
+    { 0x0000a208, 0x803e4788 },
+    { 0x0000a210, 0x4080a333 },
+    { 0x0000a214, 0x40206c10 },
+    { 0x0000a218, 0x009c4060 },
+    { 0x0000a220, 0x01834061 },
+    { 0x0000a224, 0x00000400 },
+    { 0x0000a228, 0x000003b5 },
+    { 0x0000a22c, 0x233f7180 },
+    { 0x0000a234, 0x20202020 },
+    { 0x0000a238, 0x20202020 },
+    { 0x0000a23c, 0x13c88000 },
+    { 0x0000a240, 0x38490a20 },
+    { 0x0000a244, 0x00007bb6 },
+    { 0x0000a248, 0x0fff3ffc },
+    { 0x0000a24c, 0x00000000 },
+    { 0x0000a254, 0x00000000 },
+    { 0x0000a258, 0x0cdbd380 },
+    { 0x0000a25c, 0x0f0f0f01 },
+    { 0x0000a260, 0xdfa91f01 },
+    { 0x0000a268, 0x00000000 },
+    { 0x0000a26c, 0x0ebae9c6 },
+    { 0x0000b26c, 0x0ebae9c6 },
+    { 0x0000d270, 0x00820820 },
+    { 0x0000a278, 0x1ce739ce },
+    { 0x0000d35c, 0x07ffffef },
+    { 0x0000d360, 0x0fffffe7 },
+    { 0x0000d364, 0x17ffffe5 },
+    { 0x0000d368, 0x1fffffe4 },
+    { 0x0000d36c, 0x37ffffe3 },
+    { 0x0000d370, 0x3fffffe3 },
+    { 0x0000d374, 0x57ffffe3 },
+    { 0x0000d378, 0x5fffffe2 },
+    { 0x0000d37c, 0x7fffffe2 },
+    { 0x0000d380, 0x7f3c7bba },
+    { 0x0000d384, 0xf3307ff0 },
+    { 0x0000a388, 0x0c000000 },
+    { 0x0000a38c, 0x20202020 },
+    { 0x0000a390, 0x20202020 },
+    { 0x0000a394, 0x1ce739ce },
+    { 0x0000a398, 0x000001ce },
+    { 0x0000a39c, 0x00000001 },
+    { 0x0000a3a0, 0x00000000 },
+    { 0x0000a3a4, 0x00000000 },
+    { 0x0000a3a8, 0x00000000 },
+    { 0x0000a3ac, 0x00000000 },
+    { 0x0000a3b0, 0x00000000 },
+    { 0x0000a3b4, 0x00000000 },
+    { 0x0000a3b8, 0x00000000 },
+    { 0x0000a3bc, 0x00000000 },
+    { 0x0000a3c0, 0x00000000 },
+    { 0x0000a3c4, 0x00000000 },
+    { 0x0000a3c8, 0x00000246 },
+    { 0x0000a3cc, 0x20202020 },
+    { 0x0000a3d0, 0x20202020 },
+    { 0x0000a3d4, 0x20202020 },
+    { 0x0000a3dc, 0x1ce739ce },
+    { 0x0000a3e0, 0x000001ce },
+    { 0x0000a3e4, 0x00000000 },
+    { 0x0000a3e8, 0x18c43433 },
+    { 0x0000a3ec, 0x00f70081 },
+    { 0x00007800, 0x00040000 },
+    { 0x00007804, 0xdb005012 },
+    { 0x00007808, 0x04924914 },
+    { 0x0000780c, 0x21084210 },
+    { 0x00007810, 0x6d801300 },
+    { 0x00007818, 0x07e41000 },
+    { 0x0000781c, 0x00392000 },
+    { 0x00007820, 0x92592480 },
+    { 0x00007824, 0x00040000 },
+    { 0x00007828, 0xdb005012 },
+    { 0x0000782c, 0x04924914 },
+    { 0x00007830, 0x21084210 },
+    { 0x00007834, 0x6d801300 },
+    { 0x0000783c, 0x07e40000 },
+    { 0x00007840, 0x00392000 },
+    { 0x00007844, 0x92592480 },
+    { 0x00007848, 0x00100000 },
+    { 0x0000784c, 0x773f0567 },
+    { 0x00007850, 0x54214514 },
+    { 0x00007854, 0x12035828 },
+    { 0x00007858, 0x9259269a },
+    { 0x00007860, 0x52802000 },
+    { 0x00007864, 0x0a8e370e },
+    { 0x00007868, 0xc0102850 },
+    { 0x0000786c, 0x812d4000 },
+    { 0x00007870, 0x807ec400 },
+    { 0x00007874, 0x001b6db0 },
+    { 0x00007878, 0x00376b63 },
+    { 0x0000787c, 0x06db6db6 },
+    { 0x00007880, 0x006d8000 },
+    { 0x00007884, 0xffeffffe },
+    { 0x00007888, 0xffeffffe },
+    { 0x0000788c, 0x00010000 },
+    { 0x00007890, 0x02060aeb },
+    { 0x00007898, 0x2a850160 },
+};
+
+static const uint32_t ar9280Modes_fast_clock_v2[][3] = {
+    /* Address      5G-HT20     5G-HT40  */
+    { 0x00001030, 0x00000268, 0x000004d0 },
+    { 0x00001070, 0x0000018c, 0x00000318 },
+    { 0x000010b0, 0x00000fd0, 0x00001fa0 },
+    { 0x00008014, 0x044c044c, 0x08980898 },
+    { 0x0000801c, 0x148ec02b, 0x148ec057 },
+    { 0x00008318, 0x000044c0, 0x00008980 },
+    { 0x00009820, 0x02020200, 0x02020200 },
+    { 0x00009824, 0x00000f0f, 0x00000f0f },
+    { 0x00009828, 0x0b020001, 0x0b020001 },
+    { 0x00009834, 0x00000f0f, 0x00000f0f },
+    { 0x00009844, 0x03721821, 0x03721821 },
+    { 0x00009914, 0x00000898, 0x00001130 },
+    { 0x00009918, 0x0000000b, 0x00000016 },
+    { 0x00009944, 0xdfbc1210, 0xdfbc1210 },
+};
+
+static const uint32_t ar9280Modes_backoff_23db_rxgain_v2[][6] = {
+    /* Address      5G-HT20     5G-HT40     2G-HT40     2G-HT20     Turbo   */
+    { 0x00009a00, 0x00008184, 0x00008184, 0x00000290, 0x00000290, 0x00000290 },
+    { 0x00009a04, 0x00008188, 0x00008188, 0x00000300, 0x00000300, 0x00000300 },
+    { 0x00009a08, 0x0000818c, 0x0000818c, 0x00000304, 0x00000304, 0x00000304 },
+    { 0x00009a0c, 0x00008190, 0x00008190, 0x00000308, 0x00000308, 0x00000308 },
+    { 0x00009a10, 0x00008194, 0x00008194, 0x0000030c, 0x0000030c, 0x0000030c },
+    { 0x00009a14, 0x00008200, 0x00008200, 0x00008000, 0x00008000, 0x00008000 },
+    { 0x00009a18, 0x00008204, 0x00008204, 0x00008004, 0x00008004, 0x00008004 },
+    { 0x00009a1c, 0x00008208, 0x00008208, 0x00008008, 0x00008008, 0x00008008 },
+    { 0x00009a20, 0x0000820c, 0x0000820c, 0x0000800c, 0x0000800c, 0x0000800c },
+    { 0x00009a24, 0x00008210, 0x00008210, 0x00008080, 0x00008080, 0x00008080 },
+    { 0x00009a28, 0x00008214, 0x00008214, 0x00008084, 0x00008084, 0x00008084 },
+    { 0x00009a2c, 0x00008280, 0x00008280, 0x00008088, 0x00008088, 0x00008088 },
+    { 0x00009a30, 0x00008284, 0x00008284, 0x0000808c, 0x0000808c, 0x0000808c },
+    { 0x00009a34, 0x00008288, 0x00008288, 0x00008100, 0x00008100, 0x00008100 },
+    { 0x00009a38, 0x0000828c, 0x0000828c, 0x00008104, 0x00008104, 0x00008104 },
+    { 0x00009a3c, 0x00008290, 0x00008290, 0x00008108, 0x00008108, 0x00008108 },
+    { 0x00009a40, 0x00008300, 0x00008300, 0x0000810c, 0x0000810c, 0x0000810c },
+    { 0x00009a44, 0x00008304, 0x00008304, 0x00008110, 0x00008110, 0x00008110 },
+    { 0x00009a48, 0x00008308, 0x00008308, 0x00008114, 0x00008114, 0x00008114 },
+    { 0x00009a4c, 0x0000830c, 0x0000830c, 0x00008180, 0x00008180, 0x00008180 },
+    { 0x00009a50, 0x00008310, 0x00008310, 0x00008184, 0x00008184, 0x00008184 },
+    { 0x00009a54, 0x00008314, 0x00008314, 0x00008188, 0x00008188, 0x00008188 },
+    { 0x00009a58, 0x00008380, 0x00008380, 0x0000818c, 0x0000818c, 0x0000818c },
+    { 0x00009a5c, 0x00008384, 0x00008384, 0x00008190, 0x00008190, 0x00008190 },
+    { 0x00009a60, 0x00008388, 0x00008388, 0x00008194, 0x00008194, 0x00008194 },
+    { 0x00009a64, 0x0000838c, 0x0000838c, 0x000081a0, 0x000081a0, 0x000081a0 },
+    { 0x00009a68, 0x00008390, 0x00008390, 0x0000820c, 0x0000820c, 0x0000820c },
+    { 0x00009a6c, 0x00008394, 0x00008394, 0x000081a8, 0x000081a8, 0x000081a8 },
+    { 0x00009a70, 0x0000a380, 0x0000a380, 0x00008284, 0x00008284, 0x00008284 },
+    { 0x00009a74, 0x0000a384, 0x0000a384, 0x00008288, 0x00008288, 0x00008288 },
+    { 0x00009a78, 0x0000a388, 0x0000a388, 0x00008224, 0x00008224, 0x00008224 },
+    { 0x00009a7c, 0x0000a38c, 0x0000a38c, 0x00008290, 0x00008290, 0x00008290 },
+    { 0x00009a80, 0x0000a390, 0x0000a390, 0x00008300, 0x00008300, 0x00008300 },
+    { 0x00009a84, 0x0000a394, 0x0000a394, 0x00008304, 0x00008304, 0x00008304 },
+    { 0x00009a88, 0x0000a780, 0x0000a780, 0x00008308, 0x00008308, 0x00008308 },
+    { 0x00009a8c, 0x0000a784, 0x0000a784, 0x0000830c, 0x0000830c, 0x0000830c },
+    { 0x00009a90, 0x0000a788, 0x0000a788, 0x00008380, 0x00008380, 0x00008380 },
+    { 0x00009a94, 0x0000a78c, 0x0000a78c, 0x00008384, 0x00008384, 0x00008384 },
+    { 0x00009a98, 0x0000a790, 0x0000a790, 0x00008700, 0x00008700, 0x00008700 },
+    { 0x00009a9c, 0x0000a794, 0x0000a794, 0x00008704, 0x00008704, 0x00008704 },
+    { 0x00009aa0, 0x0000ab84, 0x0000ab84, 0x00008708, 0x00008708, 0x00008708 },
+    { 0x00009aa4, 0x0000ab88, 0x0000ab88, 0x0000870c, 0x0000870c, 0x0000870c },
+    { 0x00009aa8, 0x0000ab8c, 0x0000ab8c, 0x00008780, 0x00008780, 0x00008780 },
+    { 0x00009aac, 0x0000ab90, 0x0000ab90, 0x00008784, 0x00008784, 0x00008784 },
+    { 0x00009ab0, 0x0000ab94, 0x0000ab94, 0x00008b00, 0x00008b00, 0x00008b00 },
+    { 0x00009ab4, 0x0000af80, 0x0000af80, 0x00008b04, 0x00008b04, 0x00008b04 },
+    { 0x00009ab8, 0x0000af84, 0x0000af84, 0x00008b08, 0x00008b08, 0x00008b08 },
+    { 0x00009abc, 0x0000af88, 0x0000af88, 0x00008b0c, 0x00008b0c, 0x00008b0c },
+    { 0x00009ac0, 0x0000af8c, 0x0000af8c, 0x00008b10, 0x00008b10, 0x00008b10 },
+    { 0x00009ac4, 0x0000af90, 0x0000af90, 0x00008b14, 0x00008b14, 0x00008b14 },
+    { 0x00009ac8, 0x0000af94, 0x0000af94, 0x00008b01, 0x00008b01, 0x00008b01 },
+    { 0x00009acc, 0x0000b380, 0x0000b380, 0x00008b05, 0x00008b05, 0x00008b05 },
+    { 0x00009ad0, 0x0000b384, 0x0000b384, 0x00008b09, 0x00008b09, 0x00008b09 },
+    { 0x00009ad4, 0x0000b388, 0x0000b388, 0x00008b0d, 0x00008b0d, 0x00008b0d },
+    { 0x00009ad8, 0x0000b38c, 0x0000b38c, 0x00008b11, 0x00008b11, 0x00008b11 },
+    { 0x00009adc, 0x0000b390, 0x0000b390, 0x00008b15, 0x00008b15, 0x00008b15 },
+    { 0x00009ae0, 0x0000b394, 0x0000b394, 0x00008b02, 0x00008b02, 0x00008b02 },
+    { 0x00009ae4, 0x0000b398, 0x0000b398, 0x00008b06, 0x00008b06, 0x00008b06 },
+    { 0x00009ae8, 0x0000b780, 0x0000b780, 0x00008b0a, 0x00008b0a, 0x00008b0a },
+    { 0x00009aec, 0x0000b784, 0x0000b784, 0x00008b0e, 0x00008b0e, 0x00008b0e },
+    { 0x00009af0, 0x0000b788, 0x0000b788, 0x00008b12, 0x00008b12, 0x00008b12 },
+    { 0x00009af4, 0x0000b78c, 0x0000b78c, 0x00008b16, 0x00008b16, 0x00008b16 },
+    { 0x00009af8, 0x0000b790, 0x0000b790, 0x00008b03, 0x00008b03, 0x00008b03 },
+    { 0x00009afc, 0x0000b794, 0x0000b794, 0x00008b07, 0x00008b07, 0x00008b07 },
+    { 0x00009b00, 0x0000b798, 0x0000b798, 0x00008b0b, 0x00008b0b, 0x00008b0b },
+    { 0x00009b04, 0x0000d784, 0x0000d784, 0x00008b0f, 0x00008b0f, 0x00008b0f },
+    { 0x00009b08, 0x0000d788, 0x0000d788, 0x00008b13, 0x00008b13, 0x00008b13 },
+    { 0x00009b0c, 0x0000d78c, 0x0000d78c, 0x00008b17, 0x00008b17, 0x00008b17 },
+    { 0x00009b10, 0x0000d790, 0x0000d790, 0x00008b23, 0x00008b23, 0x00008b23 },
+    { 0x00009b14, 0x0000f780, 0x0000f780, 0x00008b27, 0x00008b27, 0x00008b27 },
+    { 0x00009b18, 0x0000f784, 0x0000f784, 0x00008b2b, 0x00008b2b, 0x00008b2b },
+    { 0x00009b1c, 0x0000f788, 0x0000f788, 0x00008b2f, 0x00008b2f, 0x00008b2f },
+    { 0x00009b20, 0x0000f78c, 0x0000f78c, 0x00008b33, 0x00008b33, 0x00008b33 },
+    { 0x00009b24, 0x0000f790, 0x0000f790, 0x00008b37, 0x00008b37, 0x00008b37 },
+    { 0x00009b28, 0x0000f794, 0x0000f794, 0x00008b43, 0x00008b43, 0x00008b43 },
+    { 0x00009b2c, 0x0000f7a4, 0x0000f7a4, 0x00008b47, 0x00008b47, 0x00008b47 },
+    { 0x00009b30, 0x0000f7a8, 0x0000f7a8, 0x00008b4b, 0x00008b4b, 0x00008b4b },
+    { 0x00009b34, 0x0000f7ac, 0x0000f7ac, 0x00008b4f, 0x00008b4f, 0x00008b4f },
+    { 0x00009b38, 0x0000f7b0, 0x0000f7b0, 0x00008b53, 0x00008b53, 0x00008b53 },
+    { 0x00009b3c, 0x0000f7b4, 0x0000f7b4, 0x00008b57, 0x00008b57, 0x00008b57 },
+    { 0x00009b40, 0x0000f7a1, 0x0000f7a1, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b44, 0x0000f7a5, 0x0000f7a5, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b48, 0x0000f7a9, 0x0000f7a9, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b4c, 0x0000f7ad, 0x0000f7ad, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b50, 0x0000f7b1, 0x0000f7b1, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b54, 0x0000f7b5, 0x0000f7b5, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b58, 0x0000f7c5, 0x0000f7c5, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b5c, 0x0000f7c9, 0x0000f7c9, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b60, 0x0000f7cd, 0x0000f7cd, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b64, 0x0000f7d1, 0x0000f7d1, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b68, 0x0000f7d5, 0x0000f7d5, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b6c, 0x0000f7c2, 0x0000f7c2, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b70, 0x0000f7c6, 0x0000f7c6, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b74, 0x0000f7ca, 0x0000f7ca, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b78, 0x0000f7ce, 0x0000f7ce, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b7c, 0x0000f7d2, 0x0000f7d2, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b80, 0x0000f7d6, 0x0000f7d6, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b84, 0x0000f7c3, 0x0000f7c3, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b88, 0x0000f7c7, 0x0000f7c7, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b8c, 0x0000f7cb, 0x0000f7cb, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b90, 0x0000f7d3, 0x0000f7d3, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b94, 0x0000f7d7, 0x0000f7d7, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b98, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009b9c, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009ba0, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009ba4, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009ba8, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bac, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bb0, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bb4, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bb8, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bbc, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bc0, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bc4, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bc8, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bcc, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bd0, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bd4, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bd8, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bdc, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009be0, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009be4, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009be8, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bec, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bf0, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bf4, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bf8, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009bfc, 0x0000f7db, 0x0000f7db, 0x00008b5b, 0x00008b5b, 0x00008b5b },
+    { 0x00009848, 0x00001066, 0x00001066, 0x00001050, 0x00001050, 0x00001050 },
+    { 0x0000a848, 0x00001066, 0x00001066, 0x00001050, 0x00001050, 0x00001050 },
+};
+
+static const uint32_t ar9280Modes_original_rxgain_v2[][6] = {
+    /* Address      5G-HT20     5G-HT40     2G-HT40     2G-HT20     Turbo   */
+    { 0x00009a00, 0x00008184, 0x00008184, 0x00000290, 0x00000290, 0x00000290 },
+    { 0x00009a04, 0x00008188, 0x00008188, 0x00000300, 0x00000300, 0x00000300 },
+    { 0x00009a08, 0x0000818c, 0x0000818c, 0x00000304, 0x00000304, 0x00000304 },
+    { 0x00009a0c, 0x00008190, 0x00008190, 0x00000308, 0x00000308, 0x00000308 },
+    { 0x00009a10, 0x00008194, 0x00008194, 0x0000030c, 0x0000030c, 0x0000030c },
+    { 0x00009a14, 0x00008200, 0x00008200, 0x00008000, 0x00008000, 0x00008000 },
+    { 0x00009a18, 0x00008204, 0x00008204, 0x00008004, 0x00008004, 0x00008004 },
+    { 0x00009a1c, 0x00008208, 0x00008208, 0x00008008, 0x00008008, 0x00008008 },
+    { 0x00009a20, 0x0000820c, 0x0000820c, 0x0000800c, 0x0000800c, 0x0000800c },
+    { 0x00009a24, 0x00008210, 0x00008210, 0x00008080, 0x00008080, 0x00008080 },
+    { 0x00009a28, 0x00008214, 0x00008214, 0x00008084, 0x00008084, 0x00008084 },
+    { 0x00009a2c, 0x00008280, 0x00008280, 0x00008088, 0x00008088, 0x00008088 },
+    { 0x00009a30, 0x00008284, 0x00008284, 0x0000808c, 0x0000808c, 0x0000808c },
+    { 0x00009a34, 0x00008288, 0x00008288, 0x00008100, 0x00008100, 0x00008100 },
+    { 0x00009a38, 0x0000828c, 0x0000828c, 0x00008104, 0x00008104, 0x00008104 },
+    { 0x00009a3c, 0x00008290, 0x00008290, 0x00008108, 0x00008108, 0x00008108 },
+    { 0x00009a40, 0x00008300, 0x00008300, 0x0000810c, 0x0000810c, 0x0000810c },
+    { 0x00009a44, 0x00008304, 0x00008304, 0x00008110, 0x00008110, 0x00008110 },
+    { 0x00009a48, 0x00008308, 0x00008308, 0x00008114, 0x00008114, 0x00008114 },
+    { 0x00009a4c, 0x0000830c, 0x0000830c, 0x00008180, 0x00008180, 0x00008180 },
+    { 0x00009a50, 0x00008310, 0x00008310, 0x00008184, 0x00008184, 0x00008184 },
+    { 0x00009a54, 0x00008314, 0x00008314, 0x00008188, 0x00008188, 0x00008188 },
+    { 0x00009a58, 0x00008380, 0x00008380, 0x0000818c, 0x0000818c, 0x0000818c },
+    { 0x00009a5c, 0x00008384, 0x00008384, 0x00008190, 0x00008190, 0x00008190 },
+    { 0x00009a60, 0x00008388, 0x00008388, 0x00008194, 0x00008194, 0x00008194 },
+    { 0x00009a64, 0x0000838c, 0x0000838c, 0x000081a0, 0x000081a0, 0x000081a0 },
+    { 0x00009a68, 0x00008390, 0x00008390, 0x0000820c, 0x0000820c, 0x0000820c },
+    { 0x00009a6c, 0x00008394, 0x00008394, 0x000081a8, 0x000081a8, 0x000081a8 },
+    { 0x00009a70, 0x0000a380, 0x0000a380, 0x00008284, 0x00008284, 0x00008284 },
+    { 0x00009a74, 0x0000a384, 0x0000a384, 0x00008288, 0x00008288, 0x00008288 },
+    { 0x00009a78, 0x0000a388, 0x0000a388, 0x00008224, 0x00008224, 0x00008224 },
+    { 0x00009a7c, 0x0000a38c, 0x0000a38c, 0x00008290, 0x00008290, 0x00008290 },
+    { 0x00009a80, 0x0000a390, 0x0000a390, 0x00008300, 0x00008300, 0x00008300 },
+    { 0x00009a84, 0x0000a394, 0x0000a394, 0x00008304, 0x00008304, 0x00008304 },
+    { 0x00009a88, 0x0000a780, 0x0000a780, 0x00008308, 0x00008308, 0x00008308 },
+    { 0x00009a8c, 0x0000a784, 0x0000a784, 0x0000830c, 0x0000830c, 0x0000830c },
+    { 0x00009a90, 0x0000a788, 0x0000a788, 0x00008380, 0x00008380, 0x00008380 },
+    { 0x00009a94, 0x0000a78c, 0x0000a78c, 0x00008384, 0x00008384, 0x00008384 },
+    { 0x00009a98, 0x0000a790, 0x0000a790, 0x00008700, 0x00008700, 0x00008700 },
+    { 0x00009a9c, 0x0000a794, 0x0000a794, 0x00008704, 0x00008704, 0x00008704 },
+    { 0x00009aa0, 0x0000ab84, 0x0000ab84, 0x00008708, 0x00008708, 0x00008708 },
+    { 0x00009aa4, 0x0000ab88, 0x0000ab88, 0x0000870c, 0x0000870c, 0x0000870c },
+    { 0x00009aa8, 0x0000ab8c, 0x0000ab8c, 0x00008780, 0x00008780, 0x00008780 },
+    { 0x00009aac, 0x0000ab90, 0x0000ab90, 0x00008784, 0x00008784, 0x00008784 },
+    { 0x00009ab0, 0x0000ab94, 0x0000ab94, 0x00008b00, 0x00008b00, 0x00008b00 },
+    { 0x00009ab4, 0x0000af80, 0x0000af80, 0x00008b04, 0x00008b04, 0x00008b04 },
+    { 0x00009ab8, 0x0000af84, 0x0000af84, 0x00008b08, 0x00008b08, 0x00008b08 },
+    { 0x00009abc, 0x0000af88, 0x0000af88, 0x00008b0c, 0x00008b0c, 0x00008b0c },
+    { 0x00009ac0, 0x0000af8c, 0x0000af8c, 0x00008b80, 0x00008b80, 0x00008b80 },
+    { 0x00009ac4, 0x0000af90, 0x0000af90, 0x00008b84, 0x00008b84, 0x00008b84 },
+    { 0x00009ac8, 0x0000af94, 0x0000af94, 0x00008b88, 0x00008b88, 0x00008b88 },
+    { 0x00009acc, 0x0000b380, 0x0000b380, 0x00008b8c, 0x00008b8c, 0x00008b8c },
+    { 0x00009ad0, 0x0000b384, 0x0000b384, 0x00008b90, 0x00008b90, 0x00008b90 },
+    { 0x00009ad4, 0x0000b388, 0x0000b388, 0x00008f80, 0x00008f80, 0x00008f80 },
+    { 0x00009ad8, 0x0000b38c, 0x0000b38c, 0x00008f84, 0x00008f84, 0x00008f84 },
+    { 0x00009adc, 0x0000b390, 0x0000b390, 0x00008f88, 0x00008f88, 0x00008f88 },
+    { 0x00009ae0, 0x0000b394, 0x0000b394, 0x00008f8c, 0x00008f8c, 0x00008f8c },
+    { 0x00009ae4, 0x0000b398, 0x0000b398, 0x00008f90, 0x00008f90, 0x00008f90 },
+    { 0x00009ae8, 0x0000b780, 0x0000b780, 0x0000930c, 0x0000930c, 0x0000930c },
+    { 0x00009aec, 0x0000b784, 0x0000b784, 0x00009310, 0x00009310, 0x00009310 },
+    { 0x00009af0, 0x0000b788, 0x0000b788, 0x00009384, 0x00009384, 0x00009384 },
+    { 0x00009af4, 0x0000b78c, 0x0000b78c, 0x00009388, 0x00009388, 0x00009388 },
+    { 0x00009af8, 0x0000b790, 0x0000b790, 0x00009324, 0x00009324, 0x00009324 },
+    { 0x00009afc, 0x0000b794, 0x0000b794, 0x00009704, 0x00009704, 0x00009704 },
+    { 0x00009b00, 0x0000b798, 0x0000b798, 0x000096a4, 0x000096a4, 0x000096a4 },
+    { 0x00009b04, 0x0000d784, 0x0000d784, 0x000096a8, 0x000096a8, 0x000096a8 },
+    { 0x00009b08, 0x0000d788, 0x0000d788, 0x00009710, 0x00009710, 0x00009710 },
+    { 0x00009b0c, 0x0000d78c, 0x0000d78c, 0x00009714, 0x00009714, 0x00009714 },
+    { 0x00009b10, 0x0000d790, 0x0000d790, 0x00009720, 0x00009720, 0x00009720 },
+    { 0x00009b14, 0x0000f780, 0x0000f780, 0x00009724, 0x00009724, 0x00009724 },
+    { 0x00009b18, 0x0000f784, 0x0000f784, 0x00009728, 0x00009728, 0x00009728 },
+    { 0x00009b1c, 0x0000f788, 0x0000f788, 0x0000972c, 0x0000972c, 0x0000972c },
+    { 0x00009b20, 0x0000f78c, 0x0000f78c, 0x000097a0, 0x000097a0, 0x000097a0 },
+    { 0x00009b24, 0x0000f790, 0x0000f790, 0x000097a4, 0x000097a4, 0x000097a4 },
+    { 0x00009b28, 0x0000f794, 0x0000f794, 0x000097a8, 0x000097a8, 0x000097a8 },
+    { 0x00009b2c, 0x0000f7a4, 0x0000f7a4, 0x000097b0, 0x000097b0, 0x000097b0 },
+    { 0x00009b30, 0x0000f7a8, 0x0000f7a8, 0x000097b4, 0x000097b4, 0x000097b4 },
+    { 0x00009b34, 0x0000f7ac, 0x0000f7ac, 0x000097b8, 0x000097b8, 0x000097b8 },
+    { 0x00009b38, 0x0000f7b0, 0x0000f7b0, 0x000097a5, 0x000097a5, 0x000097a5 },
+    { 0x00009b3c, 0x0000f7b4, 0x0000f7b4, 0x000097a9, 0x000097a9, 0x000097a9 },
+    { 0x00009b40, 0x0000f7a1, 0x0000f7a1, 0x000097ad, 0x000097ad, 0x000097ad },
+    { 0x00009b44, 0x0000f7a5, 0x0000f7a5, 0x000097b1, 0x000097b1, 0x000097b1 },
+    { 0x00009b48, 0x0000f7a9, 0x0000f7a9, 0x000097b5, 0x000097b5, 0x000097b5 },
+    { 0x00009b4c, 0x0000f7ad, 0x0000f7ad, 0x000097b9, 0x000097b9, 0x000097b9 },
+    { 0x00009b50, 0x0000f7b1, 0x0000f7b1, 0x000097c5, 0x000097c5, 0x000097c5 },
+    { 0x00009b54, 0x0000f7b5, 0x0000f7b5, 0x000097c9, 0x000097c9, 0x000097c9 },
+    { 0x00009b58, 0x0000f7c5, 0x0000f7c5, 0x000097d1, 0x000097d1, 0x000097d1 },
+    { 0x00009b5c, 0x0000f7c9, 0x0000f7c9, 0x000097d5, 0x000097d5, 0x000097d5 },
+    { 0x00009b60, 0x0000f7cd, 0x0000f7cd, 0x000097d9, 0x000097d9, 0x000097d9 },
+    { 0x00009b64, 0x0000f7d1, 0x0000f7d1, 0x000097c6, 0x000097c6, 0x000097c6 },
+    { 0x00009b68, 0x0000f7d5, 0x0000f7d5, 0x000097ca, 0x000097ca, 0x000097ca },
+    { 0x00009b6c, 0x0000f7c2, 0x0000f7c2, 0x000097ce, 0x000097ce, 0x000097ce },
+    { 0x00009b70, 0x0000f7c6, 0x0000f7c6, 0x000097d2, 0x000097d2, 0x000097d2 },
+    { 0x00009b74, 0x0000f7ca, 0x0000f7ca, 0x000097d6, 0x000097d6, 0x000097d6 },
+    { 0x00009b78, 0x0000f7ce, 0x0000f7ce, 0x000097c3, 0x000097c3, 0x000097c3 },
+    { 0x00009b7c, 0x0000f7d2, 0x0000f7d2, 0x000097c7, 0x000097c7, 0x000097c7 },
+    { 0x00009b80, 0x0000f7d6, 0x0000f7d6, 0x000097cb, 0x000097cb, 0x000097cb },
+    { 0x00009b84, 0x0000f7c3, 0x0000f7c3, 0x000097cf, 0x000097cf, 0x000097cf },
+    { 0x00009b88, 0x0000f7c7, 0x0000f7c7, 0x000097d7, 0x000097d7, 0x000097d7 },
+    { 0x00009b8c, 0x0000f7cb, 0x0000f7cb, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009b90, 0x0000f7d3, 0x0000f7d3, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009b94, 0x0000f7d7, 0x0000f7d7, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009b98, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009b9c, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009ba0, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009ba4, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009ba8, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bac, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bb0, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bb4, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bb8, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bbc, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bc0, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bc4, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bc8, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bcc, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bd0, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bd4, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bd8, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bdc, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009be0, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009be4, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009be8, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bec, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bf0, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bf4, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bf8, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009bfc, 0x0000f7db, 0x0000f7db, 0x000097db, 0x000097db, 0x000097db },
+    { 0x00009848, 0x00001066, 0x00001066, 0x00001063, 0x00001063, 0x00001063 },
+    { 0x0000a848, 0x00001066, 0x00001066, 0x00001063, 0x00001063, 0x00001063 },
+};
+
+static const uint32_t ar9280Modes_backoff_13db_rxgain_v2[][6] = {
+    /* Address      5G-HT20     5G-HT40     2G-HT40     2G-HT20     Turbo   */
+    { 0x00009a00, 0x00008184, 0x00008184, 0x00000290, 0x00000290, 0x00000290 },
+    { 0x00009a04, 0x00008188, 0x00008188, 0x00000300, 0x00000300, 0x00000300 },
+    { 0x00009a08, 0x0000818c, 0x0000818c, 0x00000304, 0x00000304, 0x00000304 },
+    { 0x00009a0c, 0x00008190, 0x00008190, 0x00000308, 0x00000308, 0x00000308 },
+    { 0x00009a10, 0x00008194, 0x00008194, 0x0000030c, 0x0000030c, 0x0000030c },
+    { 0x00009a14, 0x00008200, 0x00008200, 0x00008000, 0x00008000, 0x00008000 },
+    { 0x00009a18, 0x00008204, 0x00008204, 0x00008004, 0x00008004, 0x00008004 },
+    { 0x00009a1c, 0x00008208, 0x00008208, 0x00008008, 0x00008008, 0x00008008 },
+    { 0x00009a20, 0x0000820c, 0x0000820c, 0x0000800c, 0x0000800c, 0x0000800c },
+    { 0x00009a24, 0x00008210, 0x00008210, 0x00008080, 0x00008080, 0x00008080 },
+    { 0x00009a28, 0x00008214, 0x00008214, 0x00008084, 0x00008084, 0x00008084 },
+    { 0x00009a2c, 0x00008280, 0x00008280, 0x00008088, 0x00008088, 0x00008088 },
+    { 0x00009a30, 0x00008284, 0x00008284, 0x0000808c, 0x0000808c, 0x0000808c },
+    { 0x00009a34, 0x00008288, 0x00008288, 0x00008100, 0x00008100, 0x00008100 },
+    { 0x00009a38, 0x0000828c, 0x0000828c, 0x00008104, 0x00008104, 0x00008104 },
+    { 0x00009a3c, 0x00008290, 0x00008290, 0x00008108, 0x00008108, 0x00008108 },
+    { 0x00009a40, 0x00008300, 0x00008300, 0x0000810c, 0x0000810c, 0x0000810c },
+    { 0x00009a44, 0x00008304, 0x00008304, 0x00008110, 0x00008110, 0x00008110 },
+    { 0x00009a48, 0x00008308, 0x00008308, 0x00008114, 0x00008114, 0x00008114 },
+    { 0x00009a4c, 0x0000830c, 0x0000830c, 0x00008180, 0x00008180, 0x00008180 },
+    { 0x00009a50, 0x00008310, 0x00008310, 0x00008184, 0x00008184, 0x00008184 },
+    { 0x00009a54, 0x00008314, 0x00008314, 0x00008188, 0x00008188, 0x00008188 },
+    { 0x00009a58, 0x00008380, 0x00008380, 0x0000818c, 0x0000818c, 0x0000818c },
+    { 0x00009a5c, 0x00008384, 0x00008384, 0x00008190, 0x00008190, 0x00008190 },
+    { 0x00009a60, 0x00008388, 0x00008388, 0x00008194, 0x00008194, 0x00008194 },
+    { 0x00009a64, 0x0000838c, 0x0000838c, 0x000081a0, 0x000081a0, 0x000081a0 },
+    { 0x00009a68, 0x00008390, 0x00008390, 0x0000820c, 0x0000820c, 0x0000820c },
+    { 0x00009a6c, 0x00008394, 0x00008394, 0x000081a8, 0x000081a8, 0x000081a8 },
+    { 0x00009a70, 0x0000a380, 0x0000a380, 0x00008284, 0x00008284, 0x00008284 },
+    { 0x00009a74, 0x0000a384, 0x0000a384, 0x00008288, 0x00008288, 0x00008288 },
+    { 0x00009a78, 0x0000a388, 0x0000a388, 0x00008224, 0x00008224, 0x00008224 },
+    { 0x00009a7c, 0x0000a38c, 0x0000a38c, 0x00008290, 0x00008290, 0x00008290 },
+    { 0x00009a80, 0x0000a390, 0x0000a390, 0x00008300, 0x00008300, 0x00008300 },
+    { 0x00009a84, 0x0000a394, 0x0000a394, 0x00008304, 0x00008304, 0x00008304 },
+    { 0x00009a88, 0x0000a780, 0x0000a780, 0x00008308, 0x00008308, 0x00008308 },
+    { 0x00009a8c, 0x0000a784, 0x0000a784, 0x0000830c, 0x0000830c, 0x0000830c },
+    { 0x00009a90, 0x0000a788, 0x0000a788, 0x00008380, 0x00008380, 0x00008380 },
+    { 0x00009a94, 0x0000a78c, 0x0000a78c, 0x00008384, 0x00008384, 0x00008384 },
+    { 0x00009a98, 0x0000a790, 0x0000a790, 0x00008700, 0x00008700, 0x00008700 },
+    { 0x00009a9c, 0x0000a794, 0x0000a794, 0x00008704, 0x00008704, 0x00008704 },
+    { 0x00009aa0, 0x0000ab84, 0x0000ab84, 0x00008708, 0x00008708, 0x00008708 },
+    { 0x00009aa4, 0x0000ab88, 0x0000ab88, 0x0000870c, 0x0000870c, 0x0000870c },
+    { 0x00009aa8, 0x0000ab8c, 0x0000ab8c, 0x00008780, 0x00008780, 0x00008780 },
+    { 0x00009aac, 0x0000ab90, 0x0000ab90, 0x00008784, 0x00008784, 0x00008784 },
+    { 0x00009ab0, 0x0000ab94, 0x0000ab94, 0x00008b00, 0x00008b00, 0x00008b00 },
+    { 0x00009ab4, 0x0000af80, 0x0000af80, 0x00008b04, 0x00008b04, 0x00008b04 },
+    { 0x00009ab8, 0x0000af84, 0x0000af84, 0x00008b08, 0x00008b08, 0x00008b08 },
+    { 0x00009abc, 0x0000af88, 0x0000af88, 0x00008b0c, 0x00008b0c, 0x00008b0c },
+    { 0x00009ac0, 0x0000af8c, 0x0000af8c, 0x00008b80, 0x00008b80, 0x00008b80 },
+    { 0x00009ac4, 0x0000af90, 0x0000af90, 0x00008b84, 0x00008b84, 0x00008b84 },
+    { 0x00009ac8, 0x0000af94, 0x0000af94, 0x00008b88, 0x00008b88, 0x00008b88 },
+    { 0x00009acc, 0x0000b380, 0x0000b380, 0x00008b8c, 0x00008b8c, 0x00008b8c },
+    { 0x00009ad0, 0x0000b384, 0x0000b384, 0x00008b90, 0x00008b90, 0x00008b90 },
+    { 0x00009ad4, 0x0000b388, 0x0000b388, 0x00008f80, 0x00008f80, 0x00008f80 },
+    { 0x00009ad8, 0x0000b38c, 0x0000b38c, 0x00008f84, 0x00008f84, 0x00008f84 },
+    { 0x00009adc, 0x0000b390, 0x0000b390, 0x00008f88, 0x00008f88, 0x00008f88 },
+    { 0x00009ae0, 0x0000b394, 0x0000b394, 0x00008f8c, 0x00008f8c, 0x00008f8c },
+    { 0x00009ae4, 0x0000b398, 0x0000b398, 0x00008f90, 0x00008f90, 0x00008f90 },
+    { 0x00009ae8, 0x0000b780, 0x0000b780, 0x00009310, 0x00009310, 0x00009310 },
+    { 0x00009aec, 0x0000b784, 0x0000b784, 0x00009314, 0x00009314, 0x00009314 },
+    { 0x00009af0, 0x0000b788, 0x0000b788, 0x00009320, 0x00009320, 0x00009320 },
+    { 0x00009af4, 0x0000b78c, 0x0000b78c, 0x00009324, 0x00009324, 0x00009324 },
+    { 0x00009af8, 0x0000b790, 0x0000b790, 0x00009328, 0x00009328, 0x00009328 },
+    { 0x00009afc, 0x0000b794, 0x0000b794, 0x0000932c, 0x0000932c, 0x0000932c },
+    { 0x00009b00, 0x0000b798, 0x0000b798, 0x00009330, 0x00009330, 0x00009330 },
+    { 0x00009b04, 0x0000d784, 0x0000d784, 0x00009334, 0x00009334, 0x00009334 },
+    { 0x00009b08, 0x0000d788, 0x0000d788, 0x00009321, 0x00009321, 0x00009321 },
+    { 0x00009b0c, 0x0000d78c, 0x0000d78c, 0x00009325, 0x00009325, 0x00009325 },
+    { 0x00009b10, 0x0000d790, 0x0000d790, 0x00009329, 0x00009329, 0x00009329 },
+    { 0x00009b14, 0x0000f780, 0x0000f780, 0x0000932d, 0x0000932d, 0x0000932d },
+    { 0x00009b18, 0x0000f784, 0x0000f784, 0x00009331, 0x00009331, 0x00009331 },
+    { 0x00009b1c, 0x0000f788, 0x0000f788, 0x00009335, 0x00009335, 0x00009335 },
+    { 0x00009b20, 0x0000f78c, 0x0000f78c, 0x00009322, 0x00009322, 0x00009322 },
+    { 0x00009b24, 0x0000f790, 0x0000f790, 0x00009326, 0x00009326, 0x00009326 },
+    { 0x00009b28, 0x0000f794, 0x0000f794, 0x0000932a, 0x0000932a, 0x0000932a },
+    { 0x00009b2c, 0x0000f7a4, 0x0000f7a4, 0x0000932e, 0x0000932e, 0x0000932e },
+    { 0x00009b30, 0x0000f7a8, 0x0000f7a8, 0x00009332, 0x00009332, 0x00009332 },
+    { 0x00009b34, 0x0000f7ac, 0x0000f7ac, 0x00009336, 0x00009336, 0x00009336 },
+    { 0x00009b38, 0x0000f7b0, 0x0000f7b0, 0x00009323, 0x00009323, 0x00009323 },
+    { 0x00009b3c, 0x0000f7b4, 0x0000f7b4, 0x00009327, 0x00009327, 0x00009327 },
+    { 0x00009b40, 0x0000f7a1, 0x0000f7a1, 0x0000932b, 0x0000932b, 0x0000932b },
+    { 0x00009b44, 0x0000f7a5, 0x0000f7a5, 0x0000932f, 0x0000932f, 0x0000932f },
+    { 0x00009b48, 0x0000f7a9, 0x0000f7a9, 0x00009333, 0x00009333, 0x00009333 },
+    { 0x00009b4c, 0x0000f7ad, 0x0000f7ad, 0x00009337, 0x00009337, 0x00009337 },
+    { 0x00009b50, 0x0000f7b1, 0x0000f7b1, 0x00009343, 0x00009343, 0x00009343 },
+    { 0x00009b54, 0x0000f7b5, 0x0000f7b5, 0x00009347, 0x00009347, 0x00009347 },
+    { 0x00009b58, 0x0000f7c5, 0x0000f7c5, 0x0000934b, 0x0000934b, 0x0000934b },
+    { 0x00009b5c, 0x0000f7c9, 0x0000f7c9, 0x0000934f, 0x0000934f, 0x0000934f },
+    { 0x00009b60, 0x0000f7cd, 0x0000f7cd, 0x00009353, 0x00009353, 0x00009353 },
+    { 0x00009b64, 0x0000f7d1, 0x0000f7d1, 0x00009357, 0x00009357, 0x00009357 },
+    { 0x00009b68, 0x0000f7d5, 0x0000f7d5, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009b6c, 0x0000f7c2, 0x0000f7c2, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009b70, 0x0000f7c6, 0x0000f7c6, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009b74, 0x0000f7ca, 0x0000f7ca, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009b78, 0x0000f7ce, 0x0000f7ce, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009b7c, 0x0000f7d2, 0x0000f7d2, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009b80, 0x0000f7d6, 0x0000f7d6, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009b84, 0x0000f7c3, 0x0000f7c3, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009b88, 0x0000f7c7, 0x0000f7c7, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009b8c, 0x0000f7cb, 0x0000f7cb, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009b90, 0x0000f7d3, 0x0000f7d3, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009b94, 0x0000f7d7, 0x0000f7d7, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009b98, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009b9c, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009ba0, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009ba4, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009ba8, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bac, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bb0, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bb4, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bb8, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bbc, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bc0, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bc4, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bc8, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bcc, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bd0, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bd4, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bd8, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bdc, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009be0, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009be4, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009be8, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bec, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bf0, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bf4, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bf8, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009bfc, 0x0000f7db, 0x0000f7db, 0x0000935b, 0x0000935b, 0x0000935b },
+    { 0x00009848, 0x00001066, 0x00001066, 0x0000105a, 0x0000105a, 0x0000105a },
+    { 0x0000a848, 0x00001066, 0x00001066, 0x0000105a, 0x0000105a, 0x0000105a },
+};
+
+static const uint32_t ar9280Modes_high_power_tx_gain_v2[][6] = {
+    /* Address      5G-HT20     5G-HT40     2G-HT40     2G-HT20     Turbo   */
+    { 0x0000a300, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    { 0x0000a304, 0x00003002, 0x00003002, 0x00004002, 0x00004002, 0x00004002 },
+    { 0x0000a308, 0x00006004, 0x00006004, 0x00007008, 0x00007008, 0x00007008 },
+    { 0x0000a30c, 0x0000a006, 0x0000a006, 0x0000c010, 0x0000c010, 0x0000c010 },
+    { 0x0000a310, 0x0000e012, 0x0000e012, 0x00010012, 0x00010012, 0x00010012 },
+    { 0x0000a314, 0x00011014, 0x00011014, 0x00013014, 0x00013014, 0x00013014 },
+    { 0x0000a318, 0x0001504a, 0x0001504a, 0x0001820a, 0x0001820a, 0x0001820a },
+    { 0x0000a31c, 0x0001904c, 0x0001904c, 0x0001b211, 0x0001b211, 0x0001b211 },
+    { 0x0000a320, 0x0001c04e, 0x0001c04e, 0x0001e213, 0x0001e213, 0x0001e213 },
+    { 0x0000a324, 0x00020092, 0x00020092, 0x00022411, 0x00022411, 0x00022411 },
+    { 0x0000a328, 0x0002410a, 0x0002410a, 0x00025413, 0x00025413, 0x00025413 },
+    { 0x0000a32c, 0x0002710c, 0x0002710c, 0x00029811, 0x00029811, 0x00029811 },
+    { 0x0000a330, 0x0002b18b, 0x0002b18b, 0x0002c813, 0x0002c813, 0x0002c813 },
+    { 0x0000a334, 0x0002e1cc, 0x0002e1cc, 0x00030a14, 0x00030a14, 0x00030a14 },
+    { 0x0000a338, 0x000321ec, 0x000321ec, 0x00035a50, 0x00035a50, 0x00035a50 },
+    { 0x0000a33c, 0x000321ec, 0x000321ec, 0x00039c4c, 0x00039c4c, 0x00039c4c },
+    { 0x0000a340, 0x000321ec, 0x000321ec, 0x0003de8a, 0x0003de8a, 0x0003de8a },
+    { 0x0000a344, 0x000321ec, 0x000321ec, 0x00042e92, 0x00042e92, 0x00042e92 },
+    { 0x0000a348, 0x000321ec, 0x000321ec, 0x00046ed2, 0x00046ed2, 0x00046ed2 },
+    { 0x0000a34c, 0x000321ec, 0x000321ec, 0x0004bed5, 0x0004bed5, 0x0004bed5 },
+    { 0x0000a350, 0x000321ec, 0x000321ec, 0x0004ff54, 0x0004ff54, 0x0004ff54 },
+    { 0x0000a354, 0x000321ec, 0x000321ec, 0x00053fd5, 0x00053fd5, 0x00053fd5 },
+    { 0x00007814, 0x00198eff, 0x00198eff, 0x00198eff, 0x00198eff, 0x00198eff },
+    { 0x00007838, 0x00198eff, 0x00198eff, 0x00198eff, 0x00198eff, 0x00198eff },
+    { 0x0000a27c, 0x050739ce, 0x050739ce, 0x050739ce, 0x050739ce, 0x050739ce },
+};
+
+static const uint32_t ar9280Modes_original_tx_gain_v2[][6] = {
+    /* Address      5G-HT20     5G-HT40     2G-HT40     2G-HT20     Turbo   */
+    { 0x0000a300, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 },
+    { 0x0000a304, 0x00003002, 0x00003002, 0x00003002, 0x00003002, 0x00003002 },
+    { 0x0000a308, 0x00006004, 0x00006004, 0x00008009, 0x00008009, 0x00008009 },
+    { 0x0000a30c, 0x0000a006, 0x0000a006, 0x0000b00b, 0x0000b00b, 0x0000b00b },
+    { 0x0000a310, 0x0000e012, 0x0000e012, 0x0000e012, 0x0000e012, 0x0000e012 },
+    { 0x0000a314, 0x00011014, 0x00011014, 0x00012048, 0x00012048, 0x00012048 },
+    { 0x0000a318, 0x0001504a, 0x0001504a, 0x0001604a, 0x0001604a, 0x0001604a },
+    { 0x0000a31c, 0x0001904c, 0x0001904c, 0x0001a211, 0x0001a211, 0x0001a211 },
+    { 0x0000a320, 0x0001c04e, 0x0001c04e, 0x0001e213, 0x0001e213, 0x0001e213 },
+    { 0x0000a324, 0x00020092, 0x00020092, 0x0002121b, 0x0002121b, 0x0002121b },
+    { 0x0000a328, 0x0002410a, 0x0002410a, 0x00024412, 0x00024412, 0x00024412 },
+    { 0x0000a32c, 0x0002710c, 0x0002710c, 0x00028414, 0x00028414, 0x00028414 },
+    { 0x0000a330, 0x0002b18b, 0x0002b18b, 0x0002b44a, 0x0002b44a, 0x0002b44a },
+    { 0x0000a334, 0x0002e1cc, 0x0002e1cc, 0x00030649, 0x00030649, 0x00030649 },
+    { 0x0000a338, 0x000321ec, 0x000321ec, 0x0003364b, 0x0003364b, 0x0003364b },
+    { 0x0000a33c, 0x000321ec, 0x000321ec, 0x00038a49, 0x00038a49, 0x00038a49 },
+    { 0x0000a340, 0x000321ec, 0x000321ec, 0x0003be48, 0x0003be48, 0x0003be48 },
+    { 0x0000a344, 0x000321ec, 0x000321ec, 0x0003ee4a, 0x0003ee4a, 0x0003ee4a },
+    { 0x0000a348, 0x000321ec, 0x000321ec, 0x00042e88, 0x00042e88, 0x00042e88 },
+    { 0x0000a34c, 0x000321ec, 0x000321ec, 0x00046e8a, 0x00046e8a, 0x00046e8a },
+    { 0x0000a350, 0x000321ec, 0x000321ec, 0x00049ec9, 0x00049ec9, 0x00049ec9 },
+    { 0x0000a354, 0x000321ec, 0x000321ec, 0x0004bf42, 0x0004bf42, 0x0004bf42 },
+    { 0x00007814, 0x0019beff, 0x0019beff, 0x0019beff, 0x0019beff, 0x0019beff },
+    { 0x00007838, 0x0019beff, 0x0019beff, 0x0019beff, 0x0019beff, 0x0019beff },
+    { 0x0000a27c, 0x050701ce, 0x050701ce, 0x050701ce, 0x050701ce, 0x050701ce },
+};
+
+/* Auto generated PCI-E PHY config for Merlin with CLKREQ de-asserted in L1 mode.             */
+/* In L1 mode, deassert CLKREQ, power consumption will be lower than leaving CLKREQ asserted. */
+static const uint32_t ar9280PciePhy_clkreq_off_L1_v2[][2] = {
+    {0x00004040,  0x9248fd00 },
+    {0x00004040,  0x24924924 },
+    {0x00004040,  0xa8000019 },
+    {0x00004040,  0x13160820 },
+    {0x00004040,  0xe5980560 },
+    {0x00004040,  0xc01dcffc },
+    {0x00004040,  0x1aaabe41 },
+    {0x00004040,  0xbe105554 },
+    {0x00004040,  0x00043007 },
+    {0x00004044,  0x00000000 },
+};
+
+/* Auto generated PCI-E PHY config for Merlin with CLKREQ always asserted in L1 mode */
+/* In L1 mode leave CLKREQ asserted, power consumption will be little high.          */
+static const uint32_t ar9280PciePhy_clkreq_always_on_L1_v2[][2] = {
+    {0x00004040,  0x9248fd00 },
+    {0x00004040,  0x24924924 },
+    {0x00004040,  0xa8000019 },
+    {0x00004040,  0x13160820 },
+    {0x00004040,  0xe5980560 },
+    {0x00004040,  0xc01dcffd },
+    {0x00004040,  0x1aaabe41 },
+    {0x00004040,  0xbe105554 },
+    {0x00004040,  0x00043007 },
+    {0x00004044,  0x00000000 },
+};
+
+/* Auto generated PCI-E PHY config for Merlin with WOW */
+static const uint32_t ar9280PciePhy_AWOW_merlin[][2] = {
+    {0x00004040,  0x9248fd00 },
+    {0x00004040,  0x24924924 },
+    {0x00004040,  0xa8000019 },
+    {0x00004040,  0x13160820 },
+    {0x00004040,  0xe5980560 },
+    {0x00004040,  0xc01ddffd },
+    {0x00004040,  0x1aaabe41 },
+    {0x00004040,  0xbe105554 },
+    {0x00004040,  0x00043007 },
+    {0x00004044,  0x00000000 },
+};
+
diff --git a/sys/dev/ath/if_ath.c b/sys/dev/ath/if_ath.c
index 3a401fe..32fbbe8 100644
--- a/sys/dev/ath/if_ath.c
+++ b/sys/dev/ath/if_ath.c
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
 #include "opt_ath.h"
+#include "opt_wlan.h"
 
 #include <sys/param.h>
 #include <sys/systm.h> 
@@ -70,7 +71,10 @@ __FBSDID("$FreeBSD$");
 
 #include <net80211/ieee80211_var.h>
 #include <net80211/ieee80211_regdomain.h>
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_SUPERG
+#include <net80211/ieee80211_superg.h>
+#endif
+#ifdef IEEE80211_SUPPORT_TDMA
 #include <net80211/ieee80211_tdma.h>
 #endif
 
@@ -89,11 +93,6 @@ __FBSDID("$FreeBSD$");
 #endif
 
 /*
- * We require a HAL w/ the changes for split tx/rx MIC.
- */
-CTASSERT(HAL_ABI_VERSION > 0x06052200);
-
-/*
  * ATH_BCBUF determines the number of vap's that can transmit
  * beacons and also (currently) the number of vap's that can
  * have unique mac addresses/bssid.  When staggering beacons
@@ -218,14 +217,14 @@ static int	ath_raw_xmit(struct ieee80211_node *,
 static void	ath_bpfattach(struct ath_softc *);
 static void	ath_announce(struct ath_softc *);
 
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 static void	ath_tdma_settimers(struct ath_softc *sc, u_int32_t nexttbtt,
 		    u_int32_t bintval);
 static void	ath_tdma_bintvalsetup(struct ath_softc *sc,
 		    const struct ieee80211_tdma_state *tdma);
 static void	ath_tdma_config(struct ath_softc *sc, struct ieee80211vap *vap);
 static void	ath_tdma_update(struct ieee80211_node *ni,
-		    const struct ieee80211_tdma_param *tdma);
+		    const struct ieee80211_tdma_param *tdma, int);
 static void	ath_tdma_beacon_send(struct ath_softc *sc,
 		    struct ieee80211vap *vap);
 
@@ -260,7 +259,7 @@ ath_hal_getcca(struct ath_hal *ah)
 #define	TDMA_EP_RND(x,mul) \
 	((((x)%(mul)) >= ((mul)/2)) ? ((x) + ((mul) - 1)) / (mul) : (x)/(mul))
 #define	TDMA_AVG(x)		TDMA_EP_RND(x, TDMA_EP_MULTIPLIER)
-#endif /* ATH_SUPPORT_TDMA */
+#endif /* IEEE80211_SUPPORT_TDMA */
 
 SYSCTL_DECL(_hw_ath);
 
@@ -358,6 +357,7 @@ ath_attach(u_int16_t devid, struct ath_softc *sc)
 	HAL_STATUS status;
 	int error = 0, i;
 	u_int wmodes;
+	uint8_t macaddr[IEEE80211_ADDR_LEN];
 
 	DPRINTF(sc, ATH_DEBUG_ANY, "%s: devid 0x%x\n", __func__, devid);
 
@@ -380,13 +380,6 @@ ath_attach(u_int16_t devid, struct ath_softc *sc)
 		error = ENXIO;
 		goto bad;
 	}
-	if (ah->ah_abi != HAL_ABI_VERSION) {
-		if_printf(ifp, "HAL ABI mismatch detected "
-			"(HAL:0x%x != driver:0x%x)\n",
-			ah->ah_abi, HAL_ABI_VERSION);
-		error = ENXIO;
-		goto bad;
-	}
 	sc->sc_ah = ah;
 	sc->sc_invalid = 0;	/* ready to go, enable interrupt handling */
 #ifdef	ATH_DEBUG
@@ -669,7 +662,7 @@ ath_attach(u_int16_t devid, struct ath_softc *sc)
 	wmodes = ath_hal_getwirelessmodes(ah);
 	if (wmodes & (HAL_MODE_108G|HAL_MODE_TURBO))
 		ic->ic_caps |= IEEE80211_C_TURBOP;
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 	if (ath_hal_macversion(ah) > 0x78) {
 		ic->ic_caps |= IEEE80211_C_TDMA; /* capable of TDMA */
 		ic->ic_tdma_update = ath_tdma_update;
@@ -693,14 +686,14 @@ ath_attach(u_int16_t devid, struct ath_softc *sc)
 	sc->sc_hasveol = ath_hal_hasveol(ah);
 
 	/* get mac address from hardware */
-	ath_hal_getmac(ah, ic->ic_myaddr);
+	ath_hal_getmac(ah, macaddr);
 	if (sc->sc_hasbmask)
 		ath_hal_getbssidmask(ah, sc->sc_hwbssidmask);
 
 	/* NB: used to size node table key mapping array */
 	ic->ic_max_keyix = sc->sc_keymax;
 	/* call MI attach routine. */
-	ieee80211_ifattach(ic);
+	ieee80211_ifattach(ic, macaddr);
 	ic->ic_setregdomain = ath_setregdomain;
 	ic->ic_getradiocaps = ath_getradiocaps;
 	sc->sc_opmode = HAL_M_STA;
@@ -898,7 +891,7 @@ ath_vap_create(struct ieee80211com *ic,
 		needbeacon = 1;
 		break;
 	case IEEE80211_M_AHDEMO:
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 		if (flags & IEEE80211_CLONE_TDMA) {
 			needbeacon = 1;
 			flags |= IEEE80211_CLONE_NOBEACONS;
@@ -1023,7 +1016,7 @@ ath_vap_create(struct ieee80211com *ic,
 		sc->sc_opmode = HAL_M_STA;
 		break;
 	case IEEE80211_M_AHDEMO:
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 		if (vap->iv_caps & IEEE80211_C_TDMA) {
 			sc->sc_tdma = 1;
 			/* NB: disable tsf adjust */
@@ -1129,7 +1122,7 @@ ath_vap_delete(struct ieee80211vap *vap)
 	}
 	if (vap->iv_opmode != IEEE80211_M_WDS)
 		sc->sc_nvaps--;
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 	/* TDMA operation ceases when the last vap is destroyed */
 	if (sc->sc_tdma && sc->sc_nvaps == 0) {
 		sc->sc_tdma = 0;
@@ -1290,7 +1283,7 @@ ath_intr(void *arg)
 			 * this is too slow to meet timing constraints
 			 * under load.
 			 */
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 			if (sc->sc_tdma) {
 				if (sc->sc_tdmaswba == 0) {
 					struct ieee80211com *ic = ifp->if_l2com;
@@ -1303,7 +1296,17 @@ ath_intr(void *arg)
 					sc->sc_tdmaswba--;
 			} else
 #endif
+			{
 				ath_beacon_proc(sc, 0);
+#ifdef IEEE80211_SUPPORT_SUPERG
+				/*
+				 * Schedule the rx taskq in case there's no
+				 * traffic so any frames held on the staging
+				 * queue are aged and potentially flushed.
+				 */
+				taskqueue_enqueue(sc->sc_tq, &sc->sc_rxtask);
+#endif
+			}
 		}
 		if (status & HAL_INT_RXEOL) {
 			/*
@@ -1637,7 +1640,7 @@ ath_reset(struct ifnet *ifp)
 	 */
 	ath_chan_change(sc, ic->ic_curchan);
 	if (sc->sc_beacons) {
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 		if (sc->sc_tdma)
 			ath_tdma_config(sc, NULL);
 		else
@@ -1672,320 +1675,6 @@ ath_reset_vap(struct ieee80211vap *vap, u_long cmd)
 	return ath_reset(ifp);
 }
 
-static int 
-ath_ff_always(struct ath_txq *txq, struct ath_buf *bf)
-{
-	return 0;
-}
-
-#if 0
-static int 
-ath_ff_ageflushtestdone(struct ath_txq *txq, struct ath_buf *bf)
-{
-	return (txq->axq_curage - bf->bf_age) < ATH_FF_STAGEMAX;
-}
-#endif
-
-/*
- * Flush FF staging queue.
- */
-static void
-ath_ff_stageq_flush(struct ath_softc *sc, struct ath_txq *txq,
-	int (*ath_ff_flushdonetest)(struct ath_txq *txq, struct ath_buf *bf))
-{
-	struct ath_buf *bf;
-	struct ieee80211_node *ni;
-	int pktlen, pri;
-	
-	for (;;) {
-		ATH_TXQ_LOCK(txq);
-		/*
-		 * Go from the back (oldest) to front so we can
-		 * stop early based on the age of the entry.
-		 */
-		bf = TAILQ_LAST(&txq->axq_stageq, axq_headtype);
-		if (bf == NULL || ath_ff_flushdonetest(txq, bf)) {
-			ATH_TXQ_UNLOCK(txq);
-			break;
-		}
-
-		ni = bf->bf_node;
-		pri = M_WME_GETAC(bf->bf_m);
-		KASSERT(ATH_NODE(ni)->an_ff_buf[pri],
-			("no bf on staging queue %p", bf));
-		ATH_NODE(ni)->an_ff_buf[pri] = NULL;
-		TAILQ_REMOVE(&txq->axq_stageq, bf, bf_stagelist);
-		
-		ATH_TXQ_UNLOCK(txq);
-
-		DPRINTF(sc, ATH_DEBUG_FF, "%s: flush frame, age %u\n",
-			__func__, bf->bf_age);
-
-		sc->sc_stats.ast_ff_flush++;
-		
-		/* encap and xmit */
-		bf->bf_m = ieee80211_encap(ni, bf->bf_m);
-		if (bf->bf_m == NULL) {
-			DPRINTF(sc, ATH_DEBUG_XMIT | ATH_DEBUG_FF,
-				"%s: discard, encapsulation failure\n",
-				__func__);
-			sc->sc_stats.ast_tx_encap++;
-			goto bad;
-		}
-		pktlen = bf->bf_m->m_pkthdr.len; /* NB: don't reference below */
-		if (ath_tx_start(sc, ni, bf, bf->bf_m) == 0) {
-#if 0 /*XXX*/
-			ifp->if_opackets++;
-#endif
-			continue;
-		}
-	bad:
-		if (ni != NULL)
-			ieee80211_free_node(ni);
-		bf->bf_node = NULL;
-		if (bf->bf_m != NULL) {
-			m_freem(bf->bf_m);
-			bf->bf_m = NULL;
-		}
-
-		ATH_TXBUF_LOCK(sc);
-		STAILQ_INSERT_HEAD(&sc->sc_txbuf, bf, bf_list);
-		ATH_TXBUF_UNLOCK(sc);
-	}
-}
-
-static __inline u_int32_t
-ath_ff_approx_txtime(struct ath_softc *sc, struct ath_node *an, struct mbuf *m)
-{
-	struct ieee80211com *ic = sc->sc_ifp->if_l2com;
-	u_int32_t framelen;
-	struct ath_buf *bf;
-
-	/*
-	 * Approximate the frame length to be transmitted. A swag to add
-	 * the following maximal values to the skb payload:
-	 *   - 32: 802.11 encap + CRC
-	 *   - 24: encryption overhead (if wep bit)
-	 *   - 4 + 6: fast-frame header and padding
-	 *   - 16: 2 LLC FF tunnel headers
-	 *   - 14: 1 802.3 FF tunnel header (skb already accounts for 2nd)
-	 */
-	framelen = m->m_pkthdr.len + 32 + 4 + 6 + 16 + 14;
-	if (ic->ic_flags & IEEE80211_F_PRIVACY)
-		framelen += 24;
-	bf = an->an_ff_buf[M_WME_GETAC(m)];
-	if (bf != NULL)
-		framelen += bf->bf_m->m_pkthdr.len;
-	return ath_hal_computetxtime(sc->sc_ah, sc->sc_currates, framelen,
-			sc->sc_lastdatarix, AH_FALSE);
-}
-
-/*
- * Determine if a data frame may be aggregated via ff tunnelling.
- * Note the caller is responsible for checking if the destination
- * supports fast frames.
- *
- *  NB: allowing EAPOL frames to be aggregated with other unicast traffic.
- *      Do 802.1x EAPOL frames proceed in the clear? Then they couldn't
- *      be aggregated with other types of frames when encryption is on?
- *
- *  NB: assumes lock on an_ff_buf effectively held by txq lock mechanism.
- */
-static __inline int 
-ath_ff_can_aggregate(struct ath_softc *sc,
-	struct ath_node *an, struct mbuf *m, int *flushq)
-{
-	struct ieee80211com *ic = sc->sc_ifp->if_l2com;
-	struct ath_txq *txq;
-	u_int32_t txoplimit;
-	u_int pri;
-
-	*flushq = 0;
-
-	/*
-	 * If there is no frame to combine with and the txq has
-	 * fewer frames than the minimum required; then do not
-	 * attempt to aggregate this frame.
-	 */
-	pri = M_WME_GETAC(m);
-	txq = sc->sc_ac2q[pri];
-	if (an->an_ff_buf[pri] == NULL && txq->axq_depth < sc->sc_fftxqmin)
-		return 0;
-	/*
-	 * When not in station mode never aggregate a multicast
-	 * frame; this insures, for example, that a combined frame
-	 * does not require multiple encryption keys when using
-	 * 802.1x/WPA.
-	 */
-	if (ic->ic_opmode != IEEE80211_M_STA &&
-	    ETHER_IS_MULTICAST(mtod(m, struct ether_header *)->ether_dhost))
-		return 0;		
-	/*
-	 * Consult the max bursting interval to insure a combined
-	 * frame fits within the TxOp window.
-	 */
-	txoplimit = IEEE80211_TXOP_TO_US(
-		ic->ic_wme.wme_chanParams.cap_wmeParams[pri].wmep_txopLimit);
-	if (txoplimit != 0 && ath_ff_approx_txtime(sc, an, m) > txoplimit) {
-		DPRINTF(sc, ATH_DEBUG_XMIT | ATH_DEBUG_FF,
-			"%s: FF TxOp violation\n", __func__);
-		if (an->an_ff_buf[pri] != NULL)
-			*flushq = 1;
-		return 0;
-	}
-	return 1;		/* try to aggregate */
-}
-
-/*
- * Check if the supplied frame can be partnered with an existing
- * or pending frame.  Return a reference to any frame that should be
- * sent on return; otherwise return NULL.
- */
-static struct mbuf *
-ath_ff_check(struct ath_softc *sc, struct ath_txq *txq,
-	struct ath_buf *bf, struct mbuf *m, struct ieee80211_node *ni)
-{
-	struct ath_node *an = ATH_NODE(ni);
-	struct ath_buf *bfstaged;
-	int ff_flush, pri;
-
-	/*
-	 * Check if the supplied frame can be aggregated.
-	 *
-	 * NB: we use the txq lock to protect references to
-	 *     an->an_ff_txbuf in ath_ff_can_aggregate().
-	 */
-	ATH_TXQ_LOCK(txq);
-	pri = M_WME_GETAC(m);
-	if (ath_ff_can_aggregate(sc, an, m, &ff_flush)) {
-		struct ath_buf *bfstaged = an->an_ff_buf[pri];
-		if (bfstaged != NULL) {
-			/*
-			 * A frame is available for partnering; remove
-			 * it, chain it to this one, and encapsulate.
-			 */
-			an->an_ff_buf[pri] = NULL;
-			TAILQ_REMOVE(&txq->axq_stageq, bfstaged, bf_stagelist);
-			ATH_TXQ_UNLOCK(txq);
-
-			/* 
-			 * Chain mbufs and add FF magic.
-			 */
-			DPRINTF(sc, ATH_DEBUG_FF,
-				"[%s] aggregate fast-frame, age %u\n",
-				ether_sprintf(ni->ni_macaddr), txq->axq_curage);
-			m->m_nextpkt = NULL;
-			bfstaged->bf_m->m_nextpkt = m;
-			m = bfstaged->bf_m;
-			bfstaged->bf_m = NULL;
-			m->m_flags |= M_FF;
-			/*
-			 * Release the node reference held while
-			 * the packet sat on an_ff_buf[]
-			 */
-			bfstaged->bf_node = NULL;
-			ieee80211_free_node(ni);
-
-			/*
-			 * Return bfstaged to the free list.
-			 */
-			ATH_TXBUF_LOCK(sc);
-			STAILQ_INSERT_HEAD(&sc->sc_txbuf, bfstaged, bf_list);
-			ATH_TXBUF_UNLOCK(sc);
-
-			return m;		/* ready to go */
-		} else {
-			/*
-			 * No frame available, queue this frame to wait
-			 * for a partner.  Note that we hold the buffer
-			 * and a reference to the node; we need the
-			 * buffer in particular so we're certain we
-			 * can flush the frame at a later time.
-			 */
-			DPRINTF(sc, ATH_DEBUG_FF,
-				"[%s] stage fast-frame, age %u\n",
-				ether_sprintf(ni->ni_macaddr), txq->axq_curage);
-
-			bf->bf_m = m;
-			bf->bf_node = ni;	/* NB: held reference */
-			bf->bf_age = txq->axq_curage;
-			an->an_ff_buf[pri] = bf;
-			TAILQ_INSERT_HEAD(&txq->axq_stageq, bf, bf_stagelist);
-			ATH_TXQ_UNLOCK(txq);
-
-			return NULL;		/* consumed */
-		}
-	}
-	/*
-	 * Frame could not be aggregated, it needs to be returned
-	 * to the caller for immediate transmission.  In addition
-	 * we check if we should first flush a frame from the
-	 * staging queue before sending this one.
-	 *
-	 * NB: ath_ff_can_aggregate only marks ff_flush if a frame
-	 *     is present to flush.
-	 */
-	if (ff_flush) {
-		int pktlen;
-
-		bfstaged = an->an_ff_buf[pri];
-		an->an_ff_buf[pri] = NULL;
-		TAILQ_REMOVE(&txq->axq_stageq, bfstaged, bf_stagelist);
-		ATH_TXQ_UNLOCK(txq);
-
-		DPRINTF(sc, ATH_DEBUG_FF, "[%s] flush staged frame\n",
-			ether_sprintf(an->an_node.ni_macaddr));
-
-		/* encap and xmit */
-		bfstaged->bf_m = ieee80211_encap(ni, bfstaged->bf_m);
-		if (bfstaged->bf_m == NULL) {
-			DPRINTF(sc, ATH_DEBUG_XMIT | ATH_DEBUG_FF,
-				"%s: discard, encap failure\n", __func__);
-			sc->sc_stats.ast_tx_encap++;
-			goto ff_flushbad;
-		}
-		pktlen = bfstaged->bf_m->m_pkthdr.len;
-		if (ath_tx_start(sc, ni, bfstaged, bfstaged->bf_m)) {
-			DPRINTF(sc, ATH_DEBUG_XMIT,
-				"%s: discard, xmit failure\n", __func__);
-	ff_flushbad:
-			/*
-			 * Unable to transmit frame that was on the staging
-			 * queue.  Reclaim the node reference and other
-			 * resources.
-			 */
-			if (ni != NULL)
-				ieee80211_free_node(ni);
-			bfstaged->bf_node = NULL;
-			if (bfstaged->bf_m != NULL) {
-				m_freem(bfstaged->bf_m);
-				bfstaged->bf_m = NULL;
-			}
-
-			ATH_TXBUF_LOCK(sc);
-			STAILQ_INSERT_HEAD(&sc->sc_txbuf, bfstaged, bf_list);
-			ATH_TXBUF_UNLOCK(sc);
-		} else {
-#if 0
-			ifp->if_opackets++;
-#endif
-		}
-	} else {
-		if (an->an_ff_buf[pri] != NULL) {
-			/* 
-			 * XXX: out-of-order condition only occurs for AP
-			 * mode and multicast.  There may be no valid way
-			 * to get this condition.
-			 */
-			DPRINTF(sc, ATH_DEBUG_FF, "[%s] out-of-order frame\n",
-				ether_sprintf(an->an_node.ni_macaddr));
-			/* XXX stat */
-		}
-		ATH_TXQ_UNLOCK(txq);
-	}
-	return m;
-}
-
 static struct ath_buf *
 _ath_getbuf_locked(struct ath_softc *sc)
 {
@@ -2080,9 +1769,7 @@ ath_start(struct ifnet *ifp)
 	struct ieee80211_node *ni;
 	struct ath_buf *bf;
 	struct mbuf *m, *next;
-	struct ath_txq *txq;
 	ath_bufhead frags;
-	int pri;
 
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || sc->sc_invalid)
 		return;
@@ -2101,47 +1788,14 @@ ath_start(struct ifnet *ifp)
 			ATH_TXBUF_UNLOCK(sc);
 			break;
 		}
-		STAILQ_INIT(&frags);
 		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
-		pri = M_WME_GETAC(m);
-		txq = sc->sc_ac2q[pri];
-		if (IEEE80211_ATH_CAP(ni->ni_vap, ni, IEEE80211_NODE_FF)) {
-			/*
-			 * Check queue length; if too deep drop this
-			 * frame (tail drop considered good).
-			 */
-			if (txq->axq_depth >= sc->sc_fftxqmax) {
-				DPRINTF(sc, ATH_DEBUG_FF,
-				    "[%s] tail drop on q %u depth %u\n",
-				    ether_sprintf(ni->ni_macaddr),
-				    txq->axq_qnum, txq->axq_depth);
-				sc->sc_stats.ast_tx_qfull++;
-				m_freem(m);
-				goto reclaim;
-			}
-			m = ath_ff_check(sc, txq, bf, m, ni);
-			if (m == NULL) {
-				/* NB: ni ref & bf held on stageq */
-				continue;
-			}
-		}
-		ifp->if_opackets++;
-		/*
-		 * Encapsulate the packet in prep for transmission.
-		 */
-		m = ieee80211_encap(ni, m);
-		if (m == NULL) {
-			DPRINTF(sc, ATH_DEBUG_XMIT,
-			    "%s: encapsulation failure\n", __func__);
-			sc->sc_stats.ast_tx_encap++;
-			goto bad;
-		}
 		/*
 		 * Check for fragmentation.  If this frame
 		 * has been broken up verify we have enough
 		 * buffers to send all the fragments so all
 		 * go out or none...
 		 */
+		STAILQ_INIT(&frags);
 		if ((m->m_flags & M_FRAG) && 
 		    !ath_txfrag_setup(sc, &frags, m, ni)) {
 			DPRINTF(sc, ATH_DEBUG_XMIT,
@@ -2150,6 +1804,7 @@ ath_start(struct ifnet *ifp)
 			ath_freetx(m);
 			goto bad;
 		}
+		ifp->if_opackets++;
 	nextfrag:
 		/*
 		 * Pass the frame to the h/w for transmission.
@@ -2199,13 +1854,6 @@ ath_start(struct ifnet *ifp)
 		}
 
 		sc->sc_wd_timer = 5;
-#if 0
-		/*
-		 * Flush stale frames from the fast-frame staging queue.
-		 */
-		if (ic->ic_opmode != IEEE80211_M_STA)
-			ath_ff_stageq_flush(sc, txq, ath_ff_ageflushtestdone);
-#endif
 	}
 }
 
@@ -2689,17 +2337,8 @@ ath_calcrxfilter(struct ath_softc *sc)
 	u_int32_t rfilt;
 
 	rfilt = HAL_RX_FILTER_UCAST | HAL_RX_FILTER_BCAST | HAL_RX_FILTER_MCAST;
-#if HAL_ABI_VERSION < 0x08011600
-	rfilt |= (ath_hal_getrxfilter(sc->sc_ah) &
-		(HAL_RX_FILTER_PHYRADAR | HAL_RX_FILTER_PHYERR));
-#elif HAL_ABI_VERSION < 0x08060100
-	if (ic->ic_opmode == IEEE80211_M_STA &&
-	    !sc->sc_needmib && !sc->sc_scanning)
-		rfilt |= HAL_RX_FILTER_PHYERR;
-#else
 	if (!sc->sc_needmib && !sc->sc_scanning)
 		rfilt |= HAL_RX_FILTER_PHYERR;
-#endif
 	if (ic->ic_opmode != IEEE80211_M_STA)
 		rfilt |= HAL_RX_FILTER_PROBEREQ;
 	if (ic->ic_opmode == IEEE80211_M_MONITOR || (ifp->if_flags & IFF_PROMISC))
@@ -2776,7 +2415,6 @@ static void
 ath_mode_init(struct ath_softc *sc)
 {
 	struct ifnet *ifp = sc->sc_ifp;
-	struct ieee80211com *ic = ifp->if_l2com;
 	struct ath_hal *ah = sc->sc_ah;
 	u_int32_t rfilt;
 
@@ -2787,16 +2425,8 @@ ath_mode_init(struct ath_softc *sc)
 	/* configure operational mode */
 	ath_hal_setopmode(ah);
 
-	/*
-	 * Handle any link-level address change.  Note that we only
-	 * need to force ic_myaddr; any other addresses are handled
-	 * as a byproduct of the ifnet code marking the interface
-	 * down then up.
-	 *
-	 * XXX should get from lladdr instead of arpcom but that's more work
-	 */
-	IEEE80211_ADDR_COPY(ic->ic_myaddr, IF_LLADDR(ifp));
-	ath_hal_setmac(ah, ic->ic_myaddr);
+	/* handle any link-level address change */
+	ath_hal_setmac(ah, IF_LLADDR(ifp));
 
 	/* calculate and install multicast filter */
 	ath_update_mcast(ifp);
@@ -4305,7 +3935,7 @@ rx_accept:
 			/*
 			 * Sending station is known, dispatch directly.
 			 */
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 			sc->sc_tdmars = rs;
 #endif
 			type = ieee80211_input(ni, m,
@@ -4363,10 +3993,18 @@ rx_next:
 	if (ngood)
 		sc->sc_lastrx = tsf;
 
-	if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0 &&
-	    !IFQ_IS_EMPTY(&ifp->if_snd))
-		ath_start(ifp);
-
+	if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
+#ifdef IEEE80211_SUPPORT_SUPERG
+		if (ic->ic_stageqdepth) {
+			ieee80211_age_stageq(ic, WME_AC_VO, 100);
+			ieee80211_age_stageq(ic, WME_AC_VI, 100);
+			ieee80211_age_stageq(ic, WME_AC_BE, 100);
+			ieee80211_age_stageq(ic, WME_AC_BK, 100);
+		}
+#endif
+		if (!IFQ_IS_EMPTY(&ifp->if_snd))
+			ath_start(ifp);
+	}
 #undef PA2DESC
 }
 
@@ -4374,13 +4012,12 @@ static void
 ath_txq_init(struct ath_softc *sc, struct ath_txq *txq, int qnum)
 {
 	txq->axq_qnum = qnum;
+	txq->axq_ac = 0;
 	txq->axq_depth = 0;
 	txq->axq_intrcnt = 0;
 	txq->axq_link = NULL;
 	STAILQ_INIT(&txq->axq_q);
 	ATH_TXQ_LOCK_INIT(sc, txq);
-	TAILQ_INIT(&txq->axq_stageq);
-	txq->axq_curage = 0;
 }
 
 /*
@@ -4457,6 +4094,7 @@ ath_tx_setup(struct ath_softc *sc, int ac, int haltype)
 	}
 	txq = ath_txq_setup(sc, HAL_TX_QUEUE_DATA, haltype);
 	if (txq != NULL) {
+		txq->axq_ac = ac;
 		sc->sc_ac2q[ac] = txq;
 		return 1;
 	} else
@@ -4480,7 +4118,7 @@ ath_txq_update(struct ath_softc *sc, int ac)
 	HAL_TXQ_INFO qi;
 
 	ath_hal_gettxqueueprops(ah, txq->axq_qnum, &qi);
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 	if (sc->sc_tdma) {
 		/*
 		 * AIFS is zero so there's no pre-transmit wait.  The
@@ -4514,7 +4152,7 @@ ath_txq_update(struct ath_softc *sc, int ac)
 		qi.tqi_cwmax = ATH_EXPONENT_TO_VALUE(wmep->wmep_logcwmax);
 		qi.tqi_readyTime = 0;
 		qi.tqi_burstTime = ATH_TXOP_TO_US(wmep->wmep_txopLimit);
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 	}
 #endif
 
@@ -4704,7 +4342,7 @@ ath_tx_handoff(struct ath_softc *sc, struct ath_txq *txq, struct ath_buf *bf)
 	KASSERT((bf->bf_flags & ATH_BUF_BUSY) == 0,
 	     ("busy status 0x%x", bf->bf_flags));
 	if (txq->axq_qnum != ATH_TXQ_SWQ) {
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 		int qbusy;
 
 		ATH_TXQ_INSERT_TAIL(txq, bf, bf_list);
@@ -4774,7 +4412,7 @@ ath_tx_handoff(struct ath_softc *sc, struct ath_txq *txq, struct ath_buf *bf)
 			    txq->axq_qnum, txq->axq_link,
 			    (caddr_t)bf->bf_daddr, bf->bf_desc, txq->axq_depth);
 		}
-#endif /* ATH_SUPPORT_TDMA */
+#endif /* IEEE80211_SUPPORT_TDMA */
 		txq->axq_link = &bf->bf_desc[bf->bf_nseg - 1].ds_link;
 		ath_hal_txstart(ah, txq->axq_qnum);
 	} else {
@@ -5009,7 +4647,7 @@ ath_tx_start(struct ath_softc *sc, struct ieee80211_node *ni, struct ath_buf *bf
 	}
 	if (flags & HAL_TXDESC_NOACK)		/* NB: avoid double counting */
 		sc->sc_stats.ast_tx_noack++;
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 	if (sc->sc_tdma && (flags & HAL_TXDESC_NOACK) == 0) {
 		DPRINTF(sc, ATH_DEBUG_TDMA,
 		    "%s: discard frame, ACK required w/ TDMA\n", __func__);
@@ -5251,7 +4889,7 @@ ath_tx_processq(struct ath_softc *sc, struct ath_txq *txq)
 			break;
 		}
 		ATH_TXQ_REMOVE_HEAD(txq, bf_list);
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 		if (txq->axq_depth > 0) {
 			/*
 			 * More frames follow.  Mark the buffer busy
@@ -5320,13 +4958,6 @@ ath_tx_processq(struct ath_softc *sc, struct ath_txq *txq)
 				ieee80211_process_callback(ni, bf->bf_m,
 				    (bf->bf_txflags & HAL_TXDESC_NOACK) == 0 ?
 				        ts->ts_status : HAL_TXERR_XRETRY);
-			/*
-			 * Reclaim reference to node.
-			 *
-			 * NB: the node may be reclaimed here if, for example
-			 *     this is a DEAUTH message that was sent and the
-			 *     node was timed out due to inactivity.
-			 */
 			ieee80211_free_node(ni);
 		}
 		bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap,
@@ -5344,11 +4975,13 @@ ath_tx_processq(struct ath_softc *sc, struct ath_txq *txq)
 		STAILQ_INSERT_TAIL(&sc->sc_txbuf, bf, bf_list);
 		ATH_TXBUF_UNLOCK(sc);
 	}
+#ifdef IEEE80211_SUPPORT_SUPERG
 	/*
 	 * Flush fast-frame staging queue when traffic slows.
 	 */
 	if (txq->axq_depth <= 1)
-		ath_ff_stageq_flush(sc, txq, ath_ff_always);
+		ieee80211_flush_stageq(ic, txq->axq_ac);
+#endif
 	return nacked;
 }
 
@@ -5962,7 +5595,7 @@ ath_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 		    ni->ni_capinfo, ieee80211_chan2ieee(ic, ic->ic_curchan));
 
 		switch (vap->iv_opmode) {
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 		case IEEE80211_M_AHDEMO:
 			if ((vap->iv_caps & IEEE80211_C_TDMA) == 0)
 				break;
@@ -5996,7 +5629,7 @@ ath_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 			    ni->ni_tstamp.tsf != 0) {
 				sc->sc_syncbeacon = 1;
 			} else if (!sc->sc_beacons) {
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 				if (vap->iv_caps & IEEE80211_C_TDMA)
 					ath_tdma_config(sc, vap);
 				else
@@ -6064,7 +5697,7 @@ ath_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 			taskqueue_block(sc->sc_tq);
 			sc->sc_beacons = 0;
 		}
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 		ath_hal_setcca(ah, AH_TRUE);
 #endif
 	}
@@ -6273,10 +5906,6 @@ ath_rate_setup(struct ath_softc *sc, u_int mode)
 		break;
 	case IEEE80211_MODE_TURBO_A:
 		rt = ath_hal_getratetable(ah, HAL_MODE_108A);
-#if HAL_ABI_VERSION < 0x07013100
-		if (rt == NULL)		/* XXX bandaid for old hal's */
-			rt = ath_hal_getratetable(ah, HAL_MODE_TURBO);
-#endif
 		break;
 	case IEEE80211_MODE_TURBO_G:
 		rt = ath_hal_getratetable(ah, HAL_MODE_108G);
@@ -6576,7 +6205,7 @@ ath_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		sc->sc_stats.ast_rx_packets = ifp->if_ipackets;
 		sc->sc_stats.ast_tx_rssi = ATH_RSSI(sc->sc_halstats.ns_avgtxrssi);
 		sc->sc_stats.ast_rx_rssi = ATH_RSSI(sc->sc_halstats.ns_avgrssi);
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 		sc->sc_stats.ast_tdma_tsfadjp = TDMA_AVG(sc->sc_avgtsfdeltap);
 		sc->sc_stats.ast_tdma_tsfadjm = TDMA_AVG(sc->sc_avgtsfdeltam);
 #endif
@@ -6866,7 +6495,7 @@ ath_sysctl_intmit(SYSCTL_HANDLER_ARGS)
 	return !ath_hal_setintmit(sc->sc_ah, intmit) ? EINVAL : 0;
 }
 
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 static int
 ath_sysctl_setcca(SYSCTL_HANDLER_ARGS)
 {
@@ -6880,7 +6509,7 @@ ath_sysctl_setcca(SYSCTL_HANDLER_ARGS)
 	sc->sc_setcca = (setcca != 0);
 	return 0;
 }
-#endif /* ATH_SUPPORT_TDMA */
+#endif /* IEEE80211_SUPPORT_TDMA */
 
 static void
 ath_sysctlattach(struct ath_softc *sc)
@@ -6952,16 +6581,6 @@ ath_sysctlattach(struct ath_softc *sc)
 			"tpcts", CTLTYPE_INT | CTLFLAG_RW, sc, 0,
 			ath_sysctl_tpcts, "I", "tx power for cts frames");
 	}
-	if (ath_hal_hasfastframes(sc->sc_ah)) {
-		sc->sc_fftxqmin = ATH_FF_TXQMIN;
-		SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
-			"fftxqmin", CTLFLAG_RW, &sc->sc_fftxqmin, 0,
-			"min frames before fast-frame staging");
-		sc->sc_fftxqmax = ATH_FF_TXQMAX;
-		SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
-			"fftxqmax", CTLFLAG_RW, &sc->sc_fftxqmax, 0,
-			"max queued frames before tail drop");
-	}
 	if (ath_hal_hasrfsilent(ah)) {
 		SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 			"rfsilent", CTLTYPE_INT | CTLFLAG_RW, sc, 0,
@@ -6979,7 +6598,7 @@ ath_sysctlattach(struct ath_softc *sc)
 	SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 		"monpass", CTLFLAG_RW, &sc->sc_monpass, 0,
 		"mask of error frames to pass when monitoring");
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 	if (ath_hal_macversion(ah) > 0x78) {
 		sc->sc_tdmadbaprep = 2;
 		SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
@@ -7318,7 +6937,7 @@ ath_announce(struct ath_softc *sc)
 		if_printf(ifp, "using %u tx buffers\n", ath_txbuf);
 }
 
-#ifdef ATH_SUPPORT_TDMA
+#ifdef IEEE80211_SUPPORT_TDMA
 static __inline uint32_t
 ath_hal_getnexttbtt(struct ath_hal *ah)
 {
@@ -7477,7 +7096,7 @@ ath_tdma_config(struct ath_softc *sc, struct ieee80211vap *vap)
  */
 static void
 ath_tdma_update(struct ieee80211_node *ni,
-	const struct ieee80211_tdma_param *tdma)
+	const struct ieee80211_tdma_param *tdma, int changed)
 {
 #define	TSF_TO_TU(_h,_l) \
 	((((u_int32_t)(_h)) << 22) | (((u_int32_t)(_l)) >> 10))
@@ -7498,7 +7117,7 @@ ath_tdma_update(struct ieee80211_node *ni,
 	/*
 	 * Check for and adopt configuration changes.
 	 */
-	if (isset(ATH_VAP(vap)->av_boff.bo_flags, IEEE80211_BEACON_TDMA)) {
+	if (changed != 0) {
 		const struct ieee80211_tdma_state *ts = vap->iv_tdma;
 
 		ath_tdma_bintvalsetup(sc, ts);
@@ -7678,4 +7297,4 @@ ath_tdma_beacon_send(struct ath_softc *sc, struct ieee80211vap *vap)
 		vap->iv_bss->ni_tstamp.tsf = ath_hal_gettsf64(ah);
 	}
 }
-#endif /* ATH_SUPPORT_TDMA */
+#endif /* IEEE80211_SUPPORT_TDMA */
diff --git a/sys/dev/ath/if_athvar.h b/sys/dev/ath/if_athvar.h
index 540b76a..8f88b59 100644
--- a/sys/dev/ath/if_athvar.h
+++ b/sys/dev/ath/if_athvar.h
@@ -71,10 +71,6 @@
 #define	ATH_KEYMAX	128		/* max key cache size we handle */
 #define	ATH_KEYBYTES	(ATH_KEYMAX/NBBY)	/* storage space in bytes */
 
-#define	ATH_FF_TXQMIN	2		/* min txq depth for staging */
-#define	ATH_FF_TXQMAX	50		/* maximum # of queued frames allowed */
-#define	ATH_FF_STAGEMAX	5		/* max waiting period for staged frame*/
-
 struct taskqueue;
 struct kthread;
 struct ath_buf;
@@ -106,8 +102,6 @@ struct ath_node {
 
 struct ath_buf {
 	STAILQ_ENTRY(ath_buf)	bf_list;
-	TAILQ_ENTRY(ath_buf)	bf_stagelist;	/* stage queue list */
-	u_int32_t		bf_age;		/* age when placed on stageq */
 	int			bf_nseg;
 	uint16_t		bf_txflags;	/* tx descriptor flags */
 	uint16_t		bf_flags;	/* status flags (below) */
@@ -151,6 +145,7 @@ struct ath_descdma {
 struct ath_txq {
 	u_int			axq_qnum;	/* hardware q number */
 #define	ATH_TXQ_SWQ	(HAL_NUM_TX_QUEUES+1)	/* qnum for s/w only queue */
+	u_int			axq_ac;		/* WME AC */
 	u_int			axq_flags;
 #define	ATH_TXQ_PUTPENDING	0x0001		/* ath_hal_puttxbuf pending */
 	u_int			axq_depth;	/* queue depth (stat only) */
@@ -159,13 +154,6 @@ struct ath_txq {
 	STAILQ_HEAD(, ath_buf)	axq_q;		/* transmit queue */
 	struct mtx		axq_lock;	/* lock on q and link */
 	char			axq_name[12];	/* e.g. "ath0_txq4" */
-	/*
-	 * Fast-frame state.  The staging queue holds awaiting
-	 * a fast-frame pairing.  Buffers on this queue are
-	 * assigned an ``age'' and flushed when they wait too long.
-	 */
-	TAILQ_HEAD(axq_headtype, ath_buf) axq_stageq;
-	u_int32_t		axq_curage;	/* queue age */
 };
 
 #define	ATH_TXQ_LOCK_INIT(_sc, _tq) do { \
@@ -181,7 +169,6 @@ struct ath_txq {
 #define ATH_TXQ_INSERT_TAIL(_tq, _elm, _field) do { \
 	STAILQ_INSERT_TAIL(&(_tq)->axq_q, (_elm), _field); \
 	(_tq)->axq_depth++; \
-	(_tq)->axq_curage++; \
 } while (0)
 #define ATH_TXQ_REMOVE_HEAD(_tq, _field) do { \
 	STAILQ_REMOVE_HEAD(&(_tq)->axq_q, _field); \
@@ -345,18 +332,15 @@ struct ath_softc {
 	int			sc_lastlongcal;	/* last long cal completed */
 	int			sc_lastcalreset;/* last cal reset done */
 	HAL_NODE_STATS		sc_halstats;	/* station-mode rssi stats */
-#ifdef ATH_SUPPORT_TDMA
 	u_int			sc_tdmadbaprep;	/* TDMA DBA prep time */
 	u_int			sc_tdmaswbaprep;/* TDMA SWBA prep time */
 	u_int			sc_tdmaswba;	/* TDMA SWBA counter */
 	u_int32_t		sc_tdmabintval;	/* TDMA beacon interval (TU) */
 	u_int32_t		sc_tdmaguard;	/* TDMA guard time (usec) */
 	u_int			sc_tdmaslotlen;	/* TDMA slot length (usec) */
-	u_int			sc_tdmabintcnt;	/* TDMA beacon intvl (slots) */
 	struct ath_rx_status	*sc_tdmars;	/* TDMA status of last rx */
 	u_int32_t		sc_avgtsfdeltap;/* TDMA slot adjust (+) */
 	u_int32_t		sc_avgtsfdeltam;/* TDMA slot adjust (-) */
-#endif
 };
 
 #define	ATH_LOCK_INIT(_sc) \
@@ -462,16 +446,10 @@ void	ath_intr(void *);
 	((*(_ah)->ah_setChannel)((_ah), (_chan)))
 #define	ath_hal_calibrate(_ah, _chan, _iqcal) \
 	((*(_ah)->ah_perCalibration)((_ah), (_chan), (_iqcal)))
-#if HAL_ABI_VERSION >= 0x08111000
 #define	ath_hal_calibrateN(_ah, _chan, _lcal, _isdone) \
 	((*(_ah)->ah_perCalibrationN)((_ah), (_chan), 0x1, (_lcal), (_isdone)))
 #define	ath_hal_calreset(_ah, _chan) \
 	((*(_ah)->ah_resetCalValid)((_ah), (_chan)))
-#else
-#define	ath_hal_calibrateN(_ah, _chan, _lcal, _isdone) \
-	ath_hal_calibrate(_ah, _chan, _isdone)
-#define	ath_hal_calreset(_ah, _chan)	(0)
-#endif
 #define	ath_hal_setledstate(_ah, _state) \
 	((*(_ah)->ah_setLedState)((_ah), (_state)))
 #define	ath_hal_beaconinit(_ah, _nextb, _bperiod) \
@@ -545,19 +523,8 @@ void	ath_intr(void *);
 	(ath_hal_getcapability(_ah, HAL_CAP_CIPHER, _cipher, NULL) == HAL_OK)
 #define	ath_hal_getregdomain(_ah, _prd) \
 	(ath_hal_getcapability(_ah, HAL_CAP_REG_DMN, 0, (_prd)) == HAL_OK)
-#if HAL_ABI_VERSION < 0x08090100
-/* XXX wrong for anything but amd64 and i386 */
-#if defined(__LP64__)
-#define	ath_hal_setregdomain(_ah, _rd) \
-	(*(uint16_t *)(((uint8_t *)&(_ah)[1]) + 176) = (_rd))
-#else
-#define	ath_hal_setregdomain(_ah, _rd) \
-	(*(uint16_t *)(((uint8_t *)&(_ah)[1]) + 128) = (_rd))
-#endif
-#else
 #define	ath_hal_setregdomain(_ah, _rd) \
 	ath_hal_setcapability(_ah, HAL_CAP_REG_DMN, 0, _rd, NULL)
-#endif
 #define	ath_hal_getcountrycode(_ah, _pcc) \
 	(*(_pcc) = (_ah)->ah_countryCode)
 #define	ath_hal_gettkipmic(_ah) \
@@ -656,31 +623,6 @@ void	ath_intr(void *);
 	ath_hal_setcapability(_ah, HAL_CAP_INTMIT, 1, _v, NULL)
 #define	ath_hal_getchannoise(_ah, _c) \
 	((*(_ah)->ah_getChanNoise)((_ah), (_c)))
-#if HAL_ABI_VERSION < 0x05122200
-#define	HAL_TXQ_TXOKINT_ENABLE	TXQ_FLAG_TXOKINT_ENABLE
-#define	HAL_TXQ_TXERRINT_ENABLE	TXQ_FLAG_TXERRINT_ENABLE
-#define	HAL_TXQ_TXDESCINT_ENABLE TXQ_FLAG_TXDESCINT_ENABLE
-#define	HAL_TXQ_TXEOLINT_ENABLE	TXQ_FLAG_TXEOLINT_ENABLE
-#define	HAL_TXQ_TXURNINT_ENABLE	TXQ_FLAG_TXURNINT_ENABLE
-#endif
-#if HAL_ABI_VERSION < 0x06102501
-#define	ath_hal_ispublicsafetysku(ah) \
-	(((ah)->ah_regdomain == 0 && (ah)->ah_countryCode == 842) || \
-	 (ah)->ah_regdomain == 0x12)
-#endif
-#if HAL_ABI_VERSION < 0x06122400
-/* XXX yech, can't get to regdomain so just hack a compat shim */
-#define	ath_hal_isgsmsku(ah) \
-	((ah)->ah_countryCode == 843)
-#endif
-#if HAL_ABI_VERSION < 0x07050400
-/* compat shims so code compilers--it won't work though */
-#define	CHANNEL_HT20		0x10000
-#define	CHANNEL_HT40PLUS 	0x20000
-#define	CHANNEL_HT40MINUS 	0x40000
-#define	HAL_MODE_11NG_HT20	0x008000
-#define HAL_MODE_11NA_HT20  	0x010000
-#endif
 
 #define	ath_hal_setuprxdesc(_ah, _ds, _size, _intreq) \
 	((*(_ah)->ah_setupRxDesc)((_ah), (_ds), (_size), (_intreq)))
diff --git a/sys/dev/atkbdc/psm.c b/sys/dev/atkbdc/psm.c
index 338d8f2..93c9acb 100644
--- a/sys/dev/atkbdc/psm.c
+++ b/sys/dev/atkbdc/psm.c
@@ -70,7 +70,10 @@ __FBSDID("$FreeBSD$");
 #include <sys/module.h>
 #include <sys/bus.h>
 #include <sys/conf.h>
+#include <sys/filio.h>
 #include <sys/poll.h>
+#include <sys/sigio.h>
+#include <sys/signalvar.h>
 #include <sys/syslog.h>
 #include <machine/bus.h>
 #include <sys/rman.h>
@@ -299,6 +302,7 @@ struct psm_softc {		/* Driver status information */
 	struct cdev	*bdev;
 	int		lasterr;
 	int		cmdcount;
+	struct sigio	*async;		/* Processes waiting for SIGIO */
 };
 static devclass_t psm_devclass;
 #define	PSM_SOFTC(unit)	\
@@ -1490,6 +1494,7 @@ psmopen(struct cdev *dev, int flag, int fmt, struct thread *td)
 	sc->mode.level = sc->dflt_mode.level;
 	sc->mode.protocol = sc->dflt_mode.protocol;
 	sc->watchdog = FALSE;
+	sc->async = NULL;
 
 	/* flush the event queue */
 	sc->queue.count = 0;
@@ -1629,6 +1634,12 @@ psmclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 	/* remove anything left in the output buffer */
 	empty_aux_buffer(sc->kbdc, 10);
 
+	/* clean up and sigio requests */
+	if (sc->async != NULL) {
+		funsetown(&sc->async);
+		sc->async = NULL;
+	}
+
 	/* close is almost always successful */
 	sc->state &= ~PSM_OPEN;
 	kbdc_lock(sc->kbdc, FALSE);
@@ -2190,6 +2201,15 @@ psmioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		break;
 #endif /* MOUSE_GETHWID */
 
+	case FIONBIO:
+	case FIOASYNC:
+		break;
+	case FIOSETOWN:
+		error = fsetown(*(int *)addr, &sc->async);
+		break;
+	case FIOGETOWN:
+		*(int *) addr = fgetown(&sc->async);
+		break;
 	default:
 		return (ENOTTY);
 	}
@@ -3454,6 +3474,9 @@ next:
 		wakeup(sc);
 	}
 	selwakeuppri(&sc->rsel, PZERO);
+	if (sc->async != NULL) {
+		pgsigio(&sc->async, SIGIO, 0);
+	}
 	sc->state &= ~PSM_SOFTARMED;
 	splx(s);
 }
diff --git a/sys/dev/bge/if_bge.c b/sys/dev/bge/if_bge.c
index e9e5b93..1bfd509 100644
--- a/sys/dev/bge/if_bge.c
+++ b/sys/dev/bge/if_bge.c
@@ -384,6 +384,7 @@ static uint32_t bge_readreg_ind(struct bge_softc *, int);
 #endif
 static void bge_writemem_direct(struct bge_softc *, int, int);
 static void bge_writereg_ind(struct bge_softc *, int, int);
+static void bge_set_max_readrq(struct bge_softc *, int);
 
 static int bge_miibus_readreg(device_t, int, int);
 static int bge_miibus_writereg(device_t, int, int, int);
@@ -523,6 +524,34 @@ bge_writemem_ind(struct bge_softc *sc, int off, int val)
 	pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
 }
 
+/*
+ * PCI Express only
+ */
+static void
+bge_set_max_readrq(struct bge_softc *sc, int expr_ptr)
+{
+	device_t dev;
+	uint16_t val;
+
+	KASSERT((sc->bge_flags & BGE_FLAG_PCIE) && expr_ptr != 0,
+	    ("%s: not applicable", __func__));
+
+	dev = sc->bge_dev;
+
+	val = pci_read_config(dev, expr_ptr + BGE_PCIE_DEVCTL, 2);
+	if ((val & BGE_PCIE_DEVCTL_MAX_READRQ_MASK) !=
+	    BGE_PCIE_DEVCTL_MAX_READRQ_4096) {
+		if (bootverbose)
+			device_printf(dev, "adjust device control 0x%04x ",
+			    val);
+		val &= ~BGE_PCIE_DEVCTL_MAX_READRQ_MASK;
+		val |= BGE_PCIE_DEVCTL_MAX_READRQ_4096;
+		pci_write_config(dev, expr_ptr + BGE_PCIE_DEVCTL, val, 2);
+		if (bootverbose)
+			printf("-> 0x%04x\n", val);
+	}
+}
+
 #ifdef notdef
 static uint32_t
 bge_readreg_ind(struct bge_softc *sc, int off)
@@ -1266,8 +1295,7 @@ bge_stop_fw(sc)
 }
 
 /*
- * Do endian, PCI and DMA initialization. Also check the on-board ROM
- * self-test results.
+ * Do endian, PCI and DMA initialization.
  */
 static int
 bge_chipinit(struct bge_softc *sc)
@@ -1278,18 +1306,6 @@ bge_chipinit(struct bge_softc *sc)
 	/* Set endianness before we access any non-PCI registers. */
 	pci_write_config(sc->bge_dev, BGE_PCI_MISC_CTL, BGE_INIT, 4);
 
-	/*
-	 * Check the 'ROM failed' bit on the RX CPU to see if
-	 * self-tests passed. Skip this check when there's no
-	 * chip containing the Ethernet address fitted, since
-	 * in that case it will always fail.
-	 */
-	if ((sc->bge_flags & BGE_FLAG_EADDR) &&
-	    CSR_READ_4(sc, BGE_RXCPU_MODE) & BGE_RXCPUMODE_ROMFAIL) {
-		device_printf(sc->bge_dev, "RX CPU self-diagnostics failed!\n");
-		return (ENODEV);
-	}
-
 	/* Clear the MAC control register */
 	CSR_WRITE_4(sc, BGE_MAC_MODE, 0);
 
@@ -1387,9 +1403,11 @@ bge_chipinit(struct bge_softc *sc)
 
 	/*
 	 * Disable memory write invalidate.  Apparently it is not supported
-	 * properly by these devices.
+	 * properly by these devices.  Also ensure that INTx isn't disabled,
+	 * as these chips need it even when using MSI.
 	 */
-	PCI_CLRBIT(sc->bge_dev, BGE_PCI_CMD, PCIM_CMD_MWIEN, 4);
+	PCI_CLRBIT(sc->bge_dev, BGE_PCI_CMD,
+	    PCIM_CMD_INTxDIS | PCIM_CMD_MWIEN, 4);
 
 	/* Set the timer prescaler (always 66Mhz) */
 	CSR_WRITE_4(sc, BGE_MISC_CFG, BGE_32BITTIME_66MHZ);
@@ -1742,14 +1760,18 @@ bge_blockinit(struct bge_softc *sc)
 	/* Enable host coalescing bug fix. */
 	if (sc->bge_asicrev == BGE_ASICREV_BCM5755 ||
 	    sc->bge_asicrev == BGE_ASICREV_BCM5787)
-			val |= 1 << 29;
+		val |= 1 << 29;
 
 	/* Turn on write DMA state machine */
 	CSR_WRITE_4(sc, BGE_WDMA_MODE, val);
+	DELAY(40);
 
 	/* Turn on read DMA state machine */
-	CSR_WRITE_4(sc, BGE_RDMA_MODE,
-	    BGE_RDMAMODE_ENABLE | BGE_RDMAMODE_ALL_ATTNS);
+	val = BGE_RDMAMODE_ENABLE | BGE_RDMAMODE_ALL_ATTNS;
+	if (sc->bge_flags & BGE_FLAG_PCIE)
+		val |= BGE_RDMAMODE_FIFO_LONG_BURST;
+	CSR_WRITE_4(sc, BGE_RDMA_MODE, val);
+	DELAY(40);
 
 	/* Turn on RX data completion state machine */
 	CSR_WRITE_4(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
@@ -2387,7 +2409,7 @@ bge_attach(device_t dev)
 		goto fail;
 	}
 
-	/* Save ASIC rev. */
+	/* Save various chip information. */
 	sc->bge_chipid =
 	    pci_read_config(dev, BGE_PCI_MISC_CTL, 4) &
 	    BGE_PCIMISCCTL_ASICREV;
@@ -2470,14 +2492,17 @@ bge_attach(device_t dev)
 		 * Found a PCI Express capabilities register, this
 		 * must be a PCI Express device.
 		 */
-		if (reg != 0)
+		if (reg != 0) {
 			sc->bge_flags |= BGE_FLAG_PCIE;
 #else
 	if (BGE_IS_5705_PLUS(sc)) {
 		reg = pci_read_config(dev, BGE_PCIE_CAPID_REG, 4);
-		if ((reg & 0xFF) == BGE_PCIE_CAPID)
+		if ((reg & 0xFF) == BGE_PCIE_CAPID) {
 			sc->bge_flags |= BGE_FLAG_PCIE;
+			reg = BGE_PCIE_CAPID;
 #endif
+			bge_set_max_readrq(sc, reg);
+		}
 	} else {
 		/*
 		 * Check if the device is in PCI-X Mode.
@@ -2522,6 +2547,13 @@ bge_attach(device_t dev)
 		goto fail;
 	}
 
+	if (bootverbose)
+		device_printf(dev,
+		    "CHIP ID 0x%08x; ASIC REV 0x%02x; CHIP REV 0x%02x; %s\n",
+		    sc->bge_chipid, sc->bge_asicrev, sc->bge_chiprev,
+		    (sc->bge_flags & BGE_FLAG_PCIX) ? "PCI-X" :
+		    ((sc->bge_flags & BGE_FLAG_PCIE) ? "PCI-E" : "PCI"));
+
 	BGE_LOCK_INIT(sc, device_get_nameunit(dev));
 
 	/* Try to reset the chip. */
@@ -3882,6 +3914,7 @@ bge_ifmedia_upd_locked(struct ifnet *ifp)
 {
 	struct bge_softc *sc = ifp->if_softc;
 	struct mii_data *mii;
+	struct mii_softc *miisc;
 	struct ifmedia *ifm;
 
 	BGE_LOCK_ASSERT(sc);
@@ -3932,12 +3965,9 @@ bge_ifmedia_upd_locked(struct ifnet *ifp)
 
 	sc->bge_link_evt++;
 	mii = device_get_softc(sc->bge_miibus);
-	if (mii->mii_instance) {
-		struct mii_softc *miisc;
-		for (miisc = LIST_FIRST(&mii->mii_phys); miisc != NULL;
-		    miisc = LIST_NEXT(miisc, mii_list))
+	if (mii->mii_instance)
+		LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
 			mii_phy_reset(miisc);
-	}
 	mii_mediachg(mii);
 
 	/*
diff --git a/sys/dev/bge/if_bgereg.h b/sys/dev/bge/if_bgereg.h
index b10da2e..66e36b5 100644
--- a/sys/dev/bge/if_bgereg.h
+++ b/sys/dev/bge/if_bgereg.h
@@ -176,6 +176,22 @@
 #define	BGE_PCI_MSI_ADDR_LO		0x60
 #define	BGE_PCI_MSI_DATA		0x64
 
+/*
+ * PCI Express definitions
+ * According to
+ * PCI Express base specification, REV. 1.0a
+ */
+
+/* PCI Express device control, 16bits */
+#define	BGE_PCIE_DEVCTL			0x08
+#define	BGE_PCIE_DEVCTL_MAX_READRQ_MASK	0x7000
+#define	BGE_PCIE_DEVCTL_MAX_READRQ_128	0x0000
+#define	BGE_PCIE_DEVCTL_MAX_READRQ_256	0x1000
+#define	BGE_PCIE_DEVCTL_MAX_READRQ_512	0x2000
+#define	BGE_PCIE_DEVCTL_MAX_READRQ_1024	0x3000
+#define	BGE_PCIE_DEVCTL_MAX_READRQ_2048	0x4000
+#define	BGE_PCIE_DEVCTL_MAX_READRQ_4096	0x5000
+
 /* PCI MSI. ??? */
 #define	BGE_PCIE_CAPID_REG		0xD0
 #define	BGE_PCIE_CAPID			0x10
@@ -388,6 +404,9 @@
 #ifndef PCIM_CMD_MWIEN
 #define	PCIM_CMD_MWIEN			0x0010
 #endif
+#ifndef PCIM_CMD_INTxDIS
+#define	PCIM_CMD_INTxDIS		0x0400
+#endif
 
 /*
  * High priority mailbox registers
@@ -1359,6 +1378,8 @@
 #define	BGE_RDMAMODE_PCI_FIFOOREAD_ATTN	0x00000100
 #define	BGE_RDMAMODE_LOCWRITE_TOOBIG	0x00000200
 #define	BGE_RDMAMODE_ALL_ATTNS		0x000003FC
+#define	BGE_RDMAMODE_FIFO_SIZE_128	0x00020000
+#define	BGE_RDMAMODE_FIFO_LONG_BURST	0x00030000
 
 /* Read DMA status register */
 #define	BGE_RDMASTAT_PCI_TGT_ABRT_ATTN	0x00000004
diff --git a/sys/dev/cardbus/cardbus_cis.c b/sys/dev/cardbus/cardbus_cis.c
index 2f68003..7e58949 100644
--- a/sys/dev/cardbus/cardbus_cis.c
+++ b/sys/dev/cardbus/cardbus_cis.c
@@ -451,8 +451,7 @@ cardbus_read_tuple_init(device_t cbdev, device_t child, uint32_t *start,
 	space = *start & PCIM_CIS_ASI_MASK;
 	switch (space) {
 	case PCIM_CIS_ASI_CONFIG:
-		if (cardbus_cis_debug)
-			device_printf(cbdev, "CIS in PCI config space\n");
+		DEVPRINTF((cbdev, "CIS in PCI config space\n"));
 		/* CIS in PCI config space need no initialization */
 		return (CIS_CONFIG_SPACE);
 	case PCIM_CIS_ASI_BAR0:
@@ -462,13 +461,11 @@ cardbus_read_tuple_init(device_t cbdev, device_t child, uint32_t *start,
 	case PCIM_CIS_ASI_BAR4:
 	case PCIM_CIS_ASI_BAR5:
 		*rid = PCIR_BAR(space - PCIM_CIS_ASI_BAR0);
-		if (cardbus_cis_debug)
-			device_printf(cbdev, "CIS in BAR %#x\n", *rid);
+		DEVPRINTF((cbdev, "CIS in BAR %#x\n", *rid));
 		break;
 	case PCIM_CIS_ASI_ROM:
 		*rid = PCIR_BIOS;
-		if (cardbus_cis_debug)
-			device_printf(cbdev, "CIS in option rom\n");
+		DEVPRINTF((cbdev, "CIS in option rom\n"));
 		break;
 	default:
 		device_printf(cbdev, "Unable to read CIS: Unknown space: %d\n",
@@ -484,6 +481,7 @@ cardbus_read_tuple_init(device_t cbdev, device_t child, uint32_t *start,
 		    "to read CIS.\n");
 		return (NULL);
 	}
+	DEVPRINTF((cbdev, "CIS Mapped to %#lx\n", rman_get_start(res)));
 	if (*rid == PCIR_BIOS)
 		pci_write_config(child, *rid,
 		    rman_get_start(res) | PCIM_BIOS_ENABLE, 4);
@@ -558,8 +556,7 @@ cardbus_read_tuple_init(device_t cbdev, device_t child, uint32_t *start,
 	} else {
 		*start = *start & PCIM_CIS_ADDR_MASK;
 	}
-	if (cardbus_cis_debug)
-		device_printf(cbdev, "CIS offset is %#x\n", *start);
+	DEVPRINTF((cbdev, "CIS offset is %#x\n", *start));
 
 	return (res);
 }
@@ -598,13 +595,10 @@ cardbus_parse_cis(device_t cbdev, device_t child,
 	bzero(tupledata, MAXTUPLESIZE);
 	expect_linktarget = TRUE;
 	if ((start = pci_read_config(child, PCIR_CIS, 4)) == 0) {
-		if (cardbus_cis_debug)
-			device_printf(cbdev,
-			    "Warning: CIS pointer 0 (no CIS present)\n");
+		DEVPRINTF((cbdev, "Warning: CIS pointer is 0: (no CIS)\n"));
 		return (ENXIO);
 	}
-	if (cardbus_cis_debug)
-		device_printf(cbdev, "CIS pointer is %#x\n", start);
+	DEVPRINTF((cbdev, "CIS pointer is %#x\n", start));
 	off = 0;
 	res = cardbus_read_tuple_init(cbdev, child, &start, &rid);
 	if (res == NULL) {
diff --git a/sys/dev/cxgb/cxgb_main.c b/sys/dev/cxgb/cxgb_main.c
index 593a8d6..cc09aa0 100644
--- a/sys/dev/cxgb/cxgb_main.c
+++ b/sys/dev/cxgb/cxgb_main.c
@@ -365,8 +365,8 @@ cxgb_controller_probe(device_t dev)
 }
 
 #define FW_FNAME "cxgb_t3fw"
-#define TPEEPROM_NAME "t3%c_tp_eeprom"
-#define TPSRAM_NAME "t3%c_protocol_sram"
+#define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
+#define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
 
 static int
 upgrade_fw(adapter_t *sc)
@@ -1292,10 +1292,8 @@ void t3_os_link_fault_handler(struct adapter *sc, int port_id)
 {
 	struct port_info *pi = &sc->port[port_id];
 
-	ADAPTER_LOCK(sc);
 	pi->link_fault = 1;
 	taskqueue_enqueue(sc->tq, &pi->link_fault_task);
-	ADAPTER_UNLOCK(sc);
 }
 
 void
@@ -1595,8 +1593,9 @@ update_tpeeprom(struct adapter *adap)
 
 	tpeeprom = firmware_get(name);
 	if (tpeeprom == NULL) {
-		device_printf(adap->dev, "could not load TP EEPROM: unable to load %s\n",
-		    TPEEPROM_NAME);
+		device_printf(adap->dev,
+			      "could not load TP EEPROM: unable to load %s\n",
+			      name);
 		return;
 	}
 
@@ -1607,7 +1606,9 @@ update_tpeeprom(struct adapter *adap)
 		goto release_tpeeprom;
 
 	if (len != TP_SRAM_LEN) {
-		device_printf(adap->dev, "%s length is wrong len=%d expected=%d\n", TPEEPROM_NAME, len, TP_SRAM_LEN);
+		device_printf(adap->dev,
+			      "%s length is wrong len=%d expected=%d\n", name,
+			      len, TP_SRAM_LEN);
 		return;
 	}
 	
@@ -1619,7 +1620,8 @@ update_tpeeprom(struct adapter *adap)
 			"Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
 			 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 	} else 
-		device_printf(adap->dev, "Protocol SRAM image update in EEPROM failed\n");
+		device_printf(adap->dev,
+			      "Protocol SRAM image update in EEPROM failed\n");
 
 release_tpeeprom:
 	firmware_put(tpeeprom, FIRMWARE_UNLOAD);
@@ -1912,6 +1914,7 @@ cxgb_init_locked(struct port_info *p)
 	device_printf(sc->dev, "enabling interrupts on port=%d\n", p->port_id);
 	t3_port_intr_enable(sc, p->port_id);
 
+ 	callout_reset(&sc->cxgb_tick_ch, CXGB_TICKS(sc), cxgb_tick, sc);
 	t3_sge_reset_adapter(sc);
 
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
@@ -2570,7 +2573,7 @@ cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
 			mmd = mid->phy_id >> 8;
 			if (!mmd)
 				mmd = MDIO_DEV_PCS;
-			else if (mmd > MDIO_DEV_XGXS)
+			else if (mmd > MDIO_DEV_VEND2)
 				return (EINVAL);
 
 			error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
@@ -2592,7 +2595,7 @@ cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
 			mmd = mid->phy_id >> 8;
 			if (!mmd)
 				mmd = MDIO_DEV_PCS;
-			else if (mmd > MDIO_DEV_XGXS)
+			else if (mmd > MDIO_DEV_VEND2)
 				return (EINVAL);
 			
 			error = phy->mdio_write(sc, mid->phy_id & 0x1f,
diff --git a/sys/dev/cxgb/cxgb_offload.c b/sys/dev/cxgb/cxgb_offload.c
index b1b9d55..2ae83bd 100644
--- a/sys/dev/cxgb/cxgb_offload.c
+++ b/sys/dev/cxgb/cxgb_offload.c
@@ -94,6 +94,9 @@ register_tdev(struct t3cdev *tdev)
 static inline void
 unregister_tdev(struct t3cdev *tdev)
 {
+	if (!inited)
+		return;
+
 	mtx_lock(&cxgb_db_lock);
 	TAILQ_REMOVE(&ofld_dev_list, tdev, entry);
 	mtx_unlock(&cxgb_db_lock);	
diff --git a/sys/dev/cxgb/cxgb_sge.c b/sys/dev/cxgb/cxgb_sge.c
index 6c5a17e..47f36fd 100644
--- a/sys/dev/cxgb/cxgb_sge.c
+++ b/sys/dev/cxgb/cxgb_sge.c
@@ -2879,6 +2879,7 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
 			eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mbuf, r);
 #endif
 #ifdef IFNET_MULTIQUEUE
+			rspq->rspq_mh.mh_head->m_flags |= M_FLOWID;
 			rspq->rspq_mh.mh_head->m_pkthdr.flowid = rss_hash;
 #endif			
 			ethpad = 2;
diff --git a/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c b/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c
index f11aaf0..6582b97 100644
--- a/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c
+++ b/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c
@@ -3829,7 +3829,7 @@ socket_act_establish(struct socket *so, struct mbuf *m)
 #endif
 
 	toep->tp_state = tp->t_state;
-	V_tcpstat.tcps_connects++;
+	TCPSTAT_INC(tcps_connects);
 				
 }
 
diff --git a/sys/dev/dc/dcphy.c b/sys/dev/dc/dcphy.c
index 2033577..a4df704 100644
--- a/sys/dev/dc/dcphy.c
+++ b/sys/dev/dc/dcphy.c
@@ -154,13 +154,12 @@ dcphy_attach(device_t dev)
 	sc->mii_service = dcphy_service;
 	sc->mii_pdata = mii;
 
-	sc->mii_flags |= MIIF_NOISOLATE;
-	mii->mii_instance++;
-
-#define	ADD(m, c)	ifmedia_add(&mii->mii_media, (m), (c), NULL)
+	/*
+	 * Apparently, we can neither isolate nor do loopback.
+	 */
+	sc->mii_flags |= MIIF_NOISOLATE | MIIF_NOLOOP;
 
-	ADD(IFM_MAKEWORD(IFM_ETHER, IFM_NONE, 0, sc->mii_inst),
-	    BMCR_ISO);
+	mii->mii_instance++;
 
 	/*dcphy_reset(sc);*/
 	dc_sc = mii->mii_ifp->if_softc;
@@ -186,9 +185,8 @@ dcphy_attach(device_t dev)
 
 	sc->mii_capabilities &= ma->mii_capmask;
 	device_printf(dev, " ");
-	mii_add_media(sc);
+	mii_phy_add_media(sc);
 	printf("\n");
-#undef ADD
 
 	MIIBUS_MEDIAINIT(sc->mii_dev);
 	return (0);
diff --git a/sys/dev/dc/pnphy.c b/sys/dev/dc/pnphy.c
index e78ccf3..d1282a7 100644
--- a/sys/dev/dc/pnphy.c
+++ b/sys/dev/dc/pnphy.c
@@ -137,24 +137,19 @@ pnphy_attach(device_t dev)
 	sc->mii_service = pnphy_service;
 	sc->mii_pdata = mii;
 
-	sc->mii_flags |= MIIF_NOISOLATE;
-	mii->mii_instance++;
+	/*
+	 * Apparently, we can neither isolate nor do loopback.
+	 */
+	sc->mii_flags |= MIIF_NOISOLATE | MIIF_NOLOOP;
 
-#define	ADD(m, c)	ifmedia_add(&mii->mii_media, (m), (c), NULL)
+	mii->mii_instance++;
 
 	sc->mii_capabilities =
 	    BMSR_100TXFDX | BMSR_100TXHDX | BMSR_10TFDX | BMSR_10THDX;
 	sc->mii_capabilities &= ma->mii_capmask;
 	device_printf(dev, " ");
-	mii_add_media(sc);
+	mii_phy_add_media(sc);
 	printf("\n");
-	ADD(IFM_MAKEWORD(IFM_ETHER, IFM_NONE, 0, sc->mii_inst),
-	    BMCR_ISO);
-
-	ADD(IFM_MAKEWORD(IFM_ETHER, IFM_100_TX, IFM_LOOP, sc->mii_inst),
-	    BMCR_LOOP|BMCR_S100);
-
-#undef ADD
 
 	MIIBUS_MEDIAINIT(sc->mii_dev);
 	return (0);
diff --git a/sys/dev/dcons/dcons_os.c b/sys/dev/dcons/dcons_os.c
index f4d4181..49967eb 100644
--- a/sys/dev/dcons/dcons_os.c
+++ b/sys/dev/dcons/dcons_os.c
@@ -72,7 +72,7 @@
 
 
 #ifndef DCONS_POLL_HZ
-#define DCONS_POLL_HZ	100
+#define DCONS_POLL_HZ	25
 #endif
 
 #ifndef DCONS_BUF_SIZE
@@ -360,6 +360,7 @@ dcons_attach_port(int port, char *name, int flags)
 	tp = tty_alloc(&dcons_ttydevsw, dc, NULL);
 	dc->flags = flags;
 	dc->tty   = tp;
+	tty_init_console(tp, 0);
 	tty_makedev(tp, NULL, "%s", name);
 	return(0);
 }
@@ -402,8 +403,9 @@ dcons_modevent(module_t mode, int type, void *data)
 	switch (type) {
 	case MOD_LOAD:
 		ret = dcons_drv_init(1);
-		if (ret == 0) {
+		if (ret != -1)
 			dcons_attach();
+		if (ret == 0) {
 			dcons_cnprobe(&dcons_consdev);
 			dcons_cninit(&dcons_consdev);
 			cnadd(&dcons_consdev);
diff --git a/sys/dev/drm/ati_pcigart.c b/sys/dev/drm/ati_pcigart.c
index 3073cb1..e3fecd1 100644
--- a/sys/dev/drm/ati_pcigart.c
+++ b/sys/dev/drm/ati_pcigart.c
@@ -75,14 +75,14 @@ drm_ati_alloc_pcigart_table(struct drm_device *dev,
 	    NULL, NULL, /* filtfunc, filtfuncargs */
 	    gart_info->table_size, 1, /* maxsize, nsegs */
 	    gart_info->table_size, /* maxsegsize */
-	    BUS_DMA_ALLOCNOW, NULL, NULL, /* flags, lockfunc, lockfuncargs */
+	    0, NULL, NULL, /* flags, lockfunc, lockfuncargs */
 	    &dmah->tag);
 	if (ret != 0) {
 		free(dmah, DRM_MEM_DMA);
 		return ENOMEM;
 	}
 
-	flags = BUS_DMA_NOWAIT | BUS_DMA_ZERO;
+	flags = BUS_DMA_WAITOK | BUS_DMA_ZERO;
 	if (gart_info->gart_reg_if == DRM_ATI_GART_IGP)
 	    flags |= BUS_DMA_NOCACHE;
 	
@@ -95,7 +95,8 @@ drm_ati_alloc_pcigart_table(struct drm_device *dev,
 	DRM_LOCK();
 
 	ret = bus_dmamap_load(dmah->tag, dmah->map, dmah->vaddr,
-	    gart_info->table_size, drm_ati_alloc_pcigart_table_cb, dmah, 0);
+	    gart_info->table_size, drm_ati_alloc_pcigart_table_cb, dmah,
+	    BUS_DMA_NOWAIT);
 	if (ret != 0) {
 		bus_dmamem_free(dmah->tag, dmah->vaddr, dmah->map);
 		bus_dma_tag_destroy(dmah->tag);
@@ -103,7 +104,7 @@ drm_ati_alloc_pcigart_table(struct drm_device *dev,
 		return ENOMEM;
 	}
 
-	dev->sg->dmah = dmah;
+	gart_info->dmah = dmah;
 
 	return 0;
 }
@@ -112,12 +113,12 @@ static void
 drm_ati_free_pcigart_table(struct drm_device *dev,
 			   struct drm_ati_pcigart_info *gart_info)
 {
-	struct drm_dma_handle *dmah = dev->sg->dmah;
+	struct drm_dma_handle *dmah = gart_info->dmah;
 
 	bus_dmamem_free(dmah->tag, dmah->vaddr, dmah->map);
 	bus_dma_tag_destroy(dmah->tag);
 	free(dmah, DRM_MEM_DMA);
-	dev->sg->dmah = NULL;
+	gart_info->dmah = NULL;
 }
 
 int
@@ -133,7 +134,7 @@ drm_ati_pcigart_cleanup(struct drm_device *dev,
 	if (gart_info->bus_addr) {
 		if (gart_info->gart_table_location == DRM_ATI_GART_MAIN) {
 			gart_info->bus_addr = 0;
-			if (dev->sg->dmah)
+			if (gart_info->dmah)
 				drm_ati_free_pcigart_table(dev, gart_info);
 		}
 	}
@@ -168,8 +169,8 @@ drm_ati_pcigart_init(struct drm_device *dev,
 			goto done;
 		}
 
-		address = (void *)dev->sg->dmah->vaddr;
-		bus_address = dev->sg->dmah->busaddr;
+		address = (void *)gart_info->dmah->vaddr;
+		bus_address = gart_info->dmah->busaddr;
 	} else {
 		address = gart_info->addr;
 		bus_address = gart_info->bus_addr;
diff --git a/sys/dev/drm/drmP.h b/sys/dev/drm/drmP.h
index 0b6ba60..602d906 100644
--- a/sys/dev/drm/drmP.h
+++ b/sys/dev/drm/drmP.h
@@ -240,17 +240,23 @@ typedef u_int8_t u8;
 #endif
 
 #define DRM_READ8(map, offset)						\
-	*(volatile u_int8_t *) (((unsigned long)(map)->handle) + (offset))
+	*(volatile u_int8_t *)(((vm_offset_t)(map)->handle) +		\
+	    (vm_offset_t)(offset))
 #define DRM_READ16(map, offset)						\
-	*(volatile u_int16_t *) (((unsigned long)(map)->handle) + (offset))
+	*(volatile u_int16_t *)(((vm_offset_t)(map)->handle) +		\
+	    (vm_offset_t)(offset))
 #define DRM_READ32(map, offset)						\
-	*(volatile u_int32_t *)(((unsigned long)(map)->handle) + (offset))
+	*(volatile u_int32_t *)(((vm_offset_t)(map)->handle) +		\
+	    (vm_offset_t)(offset))
 #define DRM_WRITE8(map, offset, val)					\
-	*(volatile u_int8_t *) (((unsigned long)(map)->handle) + (offset)) = val
+	*(volatile u_int8_t *)(((vm_offset_t)(map)->handle) +		\
+	    (vm_offset_t)(offset)) = val
 #define DRM_WRITE16(map, offset, val)					\
-	*(volatile u_int16_t *) (((unsigned long)(map)->handle) + (offset)) = val
+	*(volatile u_int16_t *)(((vm_offset_t)(map)->handle) +		\
+	    (vm_offset_t)(offset)) = val
 #define DRM_WRITE32(map, offset, val)					\
-	*(volatile u_int32_t *)(((unsigned long)(map)->handle) + (offset)) = val
+	*(volatile u_int32_t *)(((vm_offset_t)(map)->handle) +		\
+	    (vm_offset_t)(offset)) = val
 
 #define DRM_VERIFYAREA_READ( uaddr, size )		\
 	(!useracc(__DECONST(caddr_t, uaddr), size, VM_PROT_READ))
@@ -474,9 +480,7 @@ typedef struct drm_sg_mem {
 	void			 *virtual;
 	int			  pages;
 	dma_addr_t		 *busaddr;
-	struct drm_dma_handle	 *sg_dmah;	/* Handle for sg_pages   */
 	struct drm_dma_handle	 *dmah;		/* Handle to PCI memory  */
-						/* for ATI PCIGART table */
 } drm_sg_mem_t;
 
 typedef TAILQ_HEAD(drm_map_list, drm_local_map) drm_map_list_t;
@@ -537,6 +541,7 @@ struct drm_ati_pcigart_info {
 	struct drm_dma_handle *table_handle;
 	drm_local_map_t mapping;
 	int table_size;
+	struct drm_dma_handle *dmah; /* handle for ATI PCIGART table */
 };
 
 #ifndef DMA_BIT_MASK
@@ -607,7 +612,7 @@ struct drm_driver_info {
 };
 
 /* Length for the array of resource pointers for drm_get_resource_*. */
-#define DRM_MAX_PCI_RESOURCE	3
+#define DRM_MAX_PCI_RESOURCE	6
 
 /** 
  * DRM device functions structure
diff --git a/sys/dev/drm/drm_bufs.c b/sys/dev/drm/drm_bufs.c
index c7d0288..6016c98 100644
--- a/sys/dev/drm/drm_bufs.c
+++ b/sys/dev/drm/drm_bufs.c
@@ -216,7 +216,7 @@ int drm_addmap(struct drm_device * dev, unsigned long offset,
 			DRM_LOCK();
 			return EINVAL;
 		}
-		map->offset = map->offset + dev->sg->handle;
+		map->offset += dev->sg->handle;
 		break;
 	case _DRM_CONSISTENT:
 		/* Unfortunately, we don't get any alignment specification from
@@ -1106,7 +1106,7 @@ int drm_order(unsigned long size)
 	if (size == 0)
 		return 0;
 
-	order = ffsl(size) - 1;
+	order = flsl(size) - 1;
 	if (size & ~(1ul << order))
 		++order;
 
diff --git a/sys/dev/drm/drm_drv.c b/sys/dev/drm/drm_drv.c
index 79e81d2..1612120 100644
--- a/sys/dev/drm/drm_drv.c
+++ b/sys/dev/drm/drm_drv.c
@@ -182,7 +182,10 @@ int drm_probe(device_t kdev, drm_pci_id_list_t *idlist)
 
 	id_entry = drm_find_description(vendor, device, idlist);
 	if (id_entry != NULL) {
-		device_set_desc(kdev, id_entry->name);
+		if (!device_get_desc(kdev)) {
+			DRM_DEBUG("desc : %s\n", device_get_desc(kdev));
+			device_set_desc(kdev, id_entry->name);
+		}
 		return 0;
 	}
 
@@ -290,7 +293,8 @@ drm_pci_id_list_t *drm_find_description(int vendor, int device,
 	
 	for (i = 0; idlist[i].vendor != 0; i++) {
 		if ((idlist[i].vendor == vendor) &&
-		    (idlist[i].device == device)) {
+		    ((idlist[i].device == device) ||
+		    (idlist[i].device == 0))) {
 			return &idlist[i];
 		}
 	}
@@ -666,7 +670,7 @@ void drm_close(void *data)
 			}
 			/* Contention */
 			retcode = mtx_sleep((void *)&dev->lock.lock_queue,
-			    &dev->dev_lock, PZERO | PCATCH, "drmlk2", 0);
+			    &dev->dev_lock, PCATCH, "drmlk2", 0);
 			if (retcode)
 				break;
 		}
diff --git a/sys/dev/drm/drm_irq.c b/sys/dev/drm/drm_irq.c
index 00e1c9d..a13192c 100644
--- a/sys/dev/drm/drm_irq.c
+++ b/sys/dev/drm/drm_irq.c
@@ -80,13 +80,14 @@ static void vblank_disable_fn(void *arg)
 	}
 	callout_deactivate(&dev->vblank_disable_timer);
 
-	DRM_DEBUG("vblank_disable_allowed=%d\n", dev->vblank_disable_allowed);
+	DRM_DEBUG("vblank_disable: %s\n", dev->vblank_disable_allowed ?
+		"allowed" : "denied");
 	if (!dev->vblank_disable_allowed)
 		return;
 
 	for (i = 0; i < dev->num_crtcs; i++) {
 		if (atomic_read(&dev->vblank[i].refcount) == 0 &&
-		    dev->vblank[i].enabled) {
+		    dev->vblank[i].enabled && !dev->vblank[i].inmodeset) {
 			DRM_DEBUG("disabling vblank on crtc %d\n", i);
 			dev->vblank[i].last =
 			    dev->driver->get_vblank_counter(dev, i);
@@ -98,15 +99,13 @@ static void vblank_disable_fn(void *arg)
 
 void drm_vblank_cleanup(struct drm_device *dev)
 {
-	unsigned long irqflags;
-
 	/* Bail if the driver didn't call drm_vblank_init() */
 	if (dev->num_crtcs == 0)
 		return;
 
-	DRM_SPINLOCK_IRQSAVE(&dev->vbl_lock, irqflags);
+	DRM_SPINLOCK(&dev->vbl_lock);
 	callout_stop(&dev->vblank_disable_timer);
-	DRM_SPINUNLOCK_IRQRESTORE(&dev->vbl_lock, irqflags);
+	DRM_SPINUNLOCK(&dev->vbl_lock);
 
 	callout_drain(&dev->vblank_disable_timer);
 
@@ -151,7 +150,7 @@ err:
 
 int drm_irq_install(struct drm_device *dev)
 {
-	int retcode;
+	int crtc, retcode;
 
 	if (dev->irq == 0 || dev->dev_private == NULL)
 		return EINVAL;
@@ -188,6 +187,17 @@ int drm_irq_install(struct drm_device *dev)
 	DRM_LOCK();
 	dev->driver->irq_postinstall(dev);
 	DRM_UNLOCK();
+	if (dev->driver->enable_vblank) {
+		DRM_SPINLOCK(&dev->vbl_lock);
+		for( crtc = 0 ; crtc < dev->num_crtcs ; crtc++) {
+			if (dev->driver->enable_vblank(dev, crtc) == 0) {
+				dev->vblank[crtc].enabled = 1;
+			}
+		}
+		callout_reset(&dev->vblank_disable_timer, 5 * DRM_HZ,
+		    (timeout_t *)vblank_disable_fn, (void *)dev);
+		DRM_SPINUNLOCK(&dev->vbl_lock);
+	}
 
 	return 0;
 err:
@@ -200,11 +210,27 @@ err:
 
 int drm_irq_uninstall(struct drm_device *dev)
 {
+	int crtc;
+
 	if (!dev->irq_enabled)
 		return EINVAL;
 
 	dev->irq_enabled = 0;
 
+	/*
+	* Wake up any waiters so they don't hang.
+	*/
+	DRM_SPINLOCK(&dev->vbl_lock);
+	for (crtc = 0; crtc < dev->num_crtcs; crtc++) {
+		if (dev->vblank[crtc].enabled) {
+			DRM_WAKEUP(&dev->vblank[crtc].queue);
+			dev->vblank[crtc].last =
+			    dev->driver->get_vblank_counter(dev, crtc);
+			dev->vblank[crtc].enabled = 0;
+		}
+	}
+	DRM_SPINUNLOCK(&dev->vbl_lock);
+
 	DRM_DEBUG("irq=%d\n", dev->irq);
 
 	dev->driver->irq_uninstall(dev);
@@ -277,16 +303,15 @@ static void drm_update_vblank_count(struct drm_device *dev, int crtc)
 
 int drm_vblank_get(struct drm_device *dev, int crtc)
 {
-	unsigned long irqflags;
 	int ret = 0;
 
-	DRM_SPINLOCK_IRQSAVE(&dev->vbl_lock, irqflags);
+	DRM_SPINLOCK(&dev->vbl_lock);
 	/* Going from 0->1 means we have to enable interrupts again */
 	atomic_add_acq_int(&dev->vblank[crtc].refcount, 1);
-	DRM_DEBUG("vblank refcount = %d\n", dev->vblank[crtc].refcount);
 	if (dev->vblank[crtc].refcount == 1 &&
 	    !dev->vblank[crtc].enabled) {
 		ret = dev->driver->enable_vblank(dev, crtc);
+		DRM_DEBUG("enabling vblank on crtc %d, ret: %d\n", crtc, ret);
 		if (ret)
 			atomic_dec(&dev->vblank[crtc].refcount);
 		else {
@@ -294,30 +319,30 @@ int drm_vblank_get(struct drm_device *dev, int crtc)
 			drm_update_vblank_count(dev, crtc);
 		}
 	}
-	DRM_SPINUNLOCK_IRQRESTORE(&dev->vbl_lock, irqflags);
+	DRM_SPINUNLOCK(&dev->vbl_lock);
 
 	return ret;
 }
 
 void drm_vblank_put(struct drm_device *dev, int crtc)
 {
-	unsigned long irqflags;
+	KASSERT(atomic_read(&dev->vblank[crtc].refcount) > 0,
+	    ("invalid refcount"));
 
-	DRM_SPINLOCK_IRQSAVE(&dev->vbl_lock, irqflags);
 	/* Last user schedules interrupt disable */
 	atomic_subtract_acq_int(&dev->vblank[crtc].refcount, 1);
-	DRM_DEBUG("vblank refcount = %d\n", dev->vblank[crtc].refcount);
+
+	DRM_SPINLOCK(&dev->vbl_lock);
 	if (dev->vblank[crtc].refcount == 0)
 	    callout_reset(&dev->vblank_disable_timer, 5 * DRM_HZ,
 		(timeout_t *)vblank_disable_fn, (void *)dev);
-	DRM_SPINUNLOCK_IRQRESTORE(&dev->vbl_lock, irqflags);
+	DRM_SPINUNLOCK(&dev->vbl_lock);
 }
 
 int drm_modeset_ctl(struct drm_device *dev, void *data,
 		    struct drm_file *file_priv)
 {
 	struct drm_modeset_ctl *modeset = data;
-	unsigned long irqflags;
 	int crtc, ret = 0;
 
 	DRM_DEBUG("num_crtcs=%d\n", dev->num_crtcs);
@@ -343,18 +368,22 @@ int drm_modeset_ctl(struct drm_device *dev, void *data,
 	case _DRM_PRE_MODESET:
 		DRM_DEBUG("pre-modeset\n");
 		if (!dev->vblank[crtc].inmodeset) {
-			dev->vblank[crtc].inmodeset = 1;
-			drm_vblank_get(dev, crtc);
+			dev->vblank[crtc].inmodeset = 0x1;
+			if (drm_vblank_get(dev, crtc) == 0)
+				dev->vblank[crtc].inmodeset |= 0x2;
 		}
 		break;
 	case _DRM_POST_MODESET:
 		DRM_DEBUG("post-modeset\n");
 		if (dev->vblank[crtc].inmodeset) {
-			DRM_SPINLOCK_IRQSAVE(&dev->vbl_lock, irqflags);
+			DRM_SPINLOCK(&dev->vbl_lock);
 			dev->vblank_disable_allowed = 1;
+			DRM_SPINUNLOCK(&dev->vbl_lock);
+
+			if (dev->vblank[crtc].inmodeset & 0x2)
+				drm_vblank_put(dev, crtc);
+
 			dev->vblank[crtc].inmodeset = 0;
-			DRM_SPINUNLOCK_IRQRESTORE(&dev->vbl_lock, irqflags);
-			drm_vblank_put(dev, crtc);
 		}
 		break;
 	default:
@@ -434,27 +463,29 @@ int drm_wait_vblank(struct drm_device *dev, void *data, struct drm_file *file_pr
 	} else {
 		DRM_DEBUG("waiting on vblank count %d, crtc %d\n",
 		    vblwait->request.sequence, crtc);
-		for ( ret = 0 ; !ret && !((drm_vblank_count(dev, crtc) -
-		    vblwait->request.sequence) <= (1 << 23)) ; ) {
+		dev->vblank[crtc].last = vblwait->request.sequence;
+		for ( ret = 0 ; !ret && !(((drm_vblank_count(dev, crtc) -
+		    vblwait->request.sequence) <= (1 << 23)) ||
+		    !dev->irq_enabled) ; ) {
 			mtx_lock(&dev->irq_lock);
-			if (!((drm_vblank_count(dev, crtc) -
-			    vblwait->request.sequence) <= (1 << 23)))
+			if (!(((drm_vblank_count(dev, crtc) -
+			    vblwait->request.sequence) <= (1 << 23)) ||
+			    !dev->irq_enabled))
 				ret = mtx_sleep(&dev->vblank[crtc].queue,
 				    &dev->irq_lock, PCATCH, "vblwtq",
 				    3 * DRM_HZ);
 			mtx_unlock(&dev->irq_lock);
 		}
 
-		DRM_DEBUG("return = %d\n", ret);
-		if (ret != EINTR) {
+		if (ret != EINTR && ret != ERESTART) {
 			struct timeval now;
 
 			microtime(&now);
 			vblwait->reply.tval_sec = now.tv_sec;
 			vblwait->reply.tval_usec = now.tv_usec;
 			vblwait->reply.sequence = drm_vblank_count(dev, crtc);
-			DRM_DEBUG("returning %d to client\n",
-			    vblwait->reply.sequence);
+			DRM_DEBUG("returning %d to client, irq_enabled %d\n",
+			    vblwait->reply.sequence, dev->irq_enabled);
 		} else {
 			DRM_DEBUG("vblank wait interrupted by signal\n");
 		}
diff --git a/sys/dev/drm/drm_linux_list.h b/sys/dev/drm/drm_linux_list.h
index d2142a4..809d8ed 100644
--- a/sys/dev/drm/drm_linux_list.h
+++ b/sys/dev/drm/drm_linux_list.h
@@ -67,6 +67,10 @@ list_del(struct list_head *entry) {
 #define list_for_each(entry, head)				\
     for (entry = (head)->next; entry != head; entry = (entry)->next)
 
+#define list_for_each_prev(entry, head) \
+        for (entry = (head)->prev; entry != (head); \
+                entry = entry->prev)
+
 #define list_for_each_safe(entry, temp, head)			\
     for (entry = (head)->next, temp = (entry)->next;		\
 	entry != head; 						\
diff --git a/sys/dev/drm/drm_lock.c b/sys/dev/drm/drm_lock.c
index bb8a834..28573c8 100644
--- a/sys/dev/drm/drm_lock.c
+++ b/sys/dev/drm/drm_lock.c
@@ -82,12 +82,17 @@ int drm_lock(struct drm_device *dev, void *data, struct drm_file *file_priv)
 
 		/* Contention */
 		ret = mtx_sleep((void *)&dev->lock.lock_queue, &dev->dev_lock,
-		    PZERO | PCATCH, "drmlk2", 0);
+		    PCATCH, "drmlk2", 0);
 		if (ret != 0)
 			break;
 	}
 	DRM_UNLOCK();
-	DRM_DEBUG("%d %s\n", lock->context, ret ? "interrupted" : "has lock");
+
+	if (ret == ERESTART)
+		DRM_DEBUG("restarting syscall\n");
+	else
+		DRM_DEBUG("%d %s\n", lock->context,
+		    ret ? "interrupted" : "has lock");
 
 	if (ret != 0)
 		return ret;
diff --git a/sys/dev/drm/drm_pci.c b/sys/dev/drm/drm_pci.c
index 6d46c5d..afd6604 100644
--- a/sys/dev/drm/drm_pci.c
+++ b/sys/dev/drm/drm_pci.c
@@ -91,7 +91,7 @@ drm_pci_alloc(struct drm_device *dev, size_t size,
 	}
 
 	ret = bus_dmamem_alloc(dmah->tag, &dmah->vaddr,
-	    BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmah->map);
+	    BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_NOCACHE, &dmah->map);
 	if (ret != 0) {
 		bus_dma_tag_destroy(dmah->tag);
 		free(dmah, DRM_MEM_DMA);
diff --git a/sys/dev/drm/drm_pciids.h b/sys/dev/drm/drm_pciids.h
index b7e7ecc..fddf70a 100644
--- a/sys/dev/drm/drm_pciids.h
+++ b/sys/dev/drm/drm_pciids.h
@@ -320,6 +320,8 @@
 	{0x1002, 0x9612, CHIP_RS780|RADEON_NEW_MEMMAP|RADEON_IS_IGP, "ATI Radeon HD 3200 Graphics"}, \
 	{0x1002, 0x9613, CHIP_RS780|RADEON_NEW_MEMMAP|RADEON_IS_IGP, "ATI Radeon 3100 Graphics"}, \
 	{0x1002, 0x9614, CHIP_RS780|RADEON_NEW_MEMMAP|RADEON_IS_IGP, "ATI Radeon 3300 Graphics"}, \
+	{0x1002, 0x9615, CHIP_RS780|RADEON_NEW_MEMMAP|RADEON_IS_IGP, "ATI Radeon 3200 Graphics"}, \
+	{0x1002, 0x9616, CHIP_RS780|RADEON_NEW_MEMMAP|RADEON_IS_IGP, "ATI Radeon 3000 Graphics"}, \
 	{0x1002, 0x9440, CHIP_RV770|RADEON_NEW_MEMMAP, "ATI Radeon 4800 Series"}, \
 	{0x1002, 0x9441, CHIP_RV770|RADEON_NEW_MEMMAP, "ATI Radeon 4870 X2"}, \
 	{0x1002, 0x9442, CHIP_RV770|RADEON_NEW_MEMMAP, "ATI Radeon 4800 Series"}, \
@@ -334,6 +336,8 @@
 	{0x1002, 0x944B, CHIP_RV770|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon HD 4850 X2"}, \
 	{0x1002, 0x945A, CHIP_RV770|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon HD 4870"}, \
 	{0x1002, 0x945B, CHIP_RV770|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI Mobility Radeon M98"}, \
+	{0x1002, 0x9460, CHIP_RV770|RADEON_NEW_MEMMAP, "ATI Radeon 4800 Series"}, \
+	{0x1002, 0x9462, CHIP_RV770|RADEON_NEW_MEMMAP, "ATI Radeon 4800 Series"}, \
 	{0x1002, 0x946A, CHIP_RV770|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI FirePro M7750"}, \
 	{0x1002, 0x946B, CHIP_RV770|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI M98"}, \
 	{0x1002, 0x947A, CHIP_RV770|RADEON_IS_MOBILITY|RADEON_NEW_MEMMAP, "ATI M98"}, \
diff --git a/sys/dev/drm/drm_scatter.c b/sys/dev/drm/drm_scatter.c
index c884df1..5f8b29b 100644
--- a/sys/dev/drm/drm_scatter.c
+++ b/sys/dev/drm/drm_scatter.c
@@ -92,7 +92,7 @@ drm_sg_alloc(struct drm_device *dev, struct drm_scatter_gather *request)
 	}
 
 	ret = bus_dmamem_alloc(dmah->tag, &dmah->vaddr,
-	    BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmah->map);
+	    BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_NOCACHE, &dmah->map);
 	if (ret != 0) {
 		bus_dma_tag_destroy(dmah->tag);
 		free(dmah, DRM_MEM_DMA);
@@ -102,8 +102,7 @@ drm_sg_alloc(struct drm_device *dev, struct drm_scatter_gather *request)
 	}
 
 	ret = bus_dmamap_load(dmah->tag, dmah->map, dmah->vaddr,
-	    request->size, drm_sg_alloc_cb, entry,
-	    BUS_DMA_NOWAIT | BUS_DMA_NOCACHE);
+	    request->size, drm_sg_alloc_cb, entry, BUS_DMA_NOWAIT);
 	if (ret != 0) {
 		bus_dmamem_free(dmah->tag, dmah->vaddr, dmah->map);
 		bus_dma_tag_destroy(dmah->tag);
@@ -113,7 +112,7 @@ drm_sg_alloc(struct drm_device *dev, struct drm_scatter_gather *request)
 		return ENOMEM;
 	}
 
-	entry->sg_dmah = dmah;
+	entry->dmah = dmah;
 	entry->handle = (unsigned long)dmah->vaddr;
 	
 	DRM_DEBUG("sg alloc handle  = %08lx\n", entry->handle);
@@ -161,7 +160,7 @@ drm_sg_alloc_ioctl(struct drm_device *dev, void *data,
 void
 drm_sg_cleanup(struct drm_sg_mem *entry)
 {
-	struct drm_dma_handle *dmah = entry->sg_dmah;
+	struct drm_dma_handle *dmah = entry->dmah;
 
 	bus_dmamap_unload(dmah->tag, dmah->map);
 	bus_dmamem_free(dmah->tag, dmah->vaddr, dmah->map);
diff --git a/sys/dev/drm/drm_vm.c b/sys/dev/drm/drm_vm.c
index e00684b..7d5cd8d 100644
--- a/sys/dev/drm/drm_vm.c
+++ b/sys/dev/drm/drm_vm.c
@@ -54,7 +54,7 @@ int drm_mmap(struct cdev *kdev, vm_offset_t offset, vm_paddr_t *paddr,
 	if (file_priv && !file_priv->authenticated)
 		return EACCES;
 
-	if (dev->dma && offset >= 0 && offset < ptoa(dev->dma->page_count)) {
+	if (dev->dma && offset < ptoa(dev->dma->page_count)) {
 		drm_device_dma_t *dma = dev->dma;
 
 		DRM_SPINLOCK(&dev->dma_lock);
@@ -86,8 +86,14 @@ int drm_mmap(struct cdev *kdev, vm_offset_t offset, vm_paddr_t *paddr,
 	}
 
 	if (map == NULL) {
+		DRM_DEBUG("Can't find map, requested offset = %016lx\n",
+		    (unsigned long)offset);
+		TAILQ_FOREACH(map, &dev->maplist, link) {
+			DRM_DEBUG("map offset = %016lx, handle = %016lx\n",
+			    (unsigned long)map->offset,
+			    (unsigned long)map->handle);
+		}
 		DRM_UNLOCK();
-		DRM_DEBUG("can't find map\n");
 		return -1;
 	}
 	if (((map->flags&_DRM_RESTRICTED) && !DRM_SUSER(DRM_CURPROC))) {
diff --git a/sys/dev/drm/i915_dma.c b/sys/dev/drm/i915_dma.c
index 1996831..a701a5e 100644
--- a/sys/dev/drm/i915_dma.c
+++ b/sys/dev/drm/i915_dma.c
@@ -193,7 +193,7 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
 		dev_priv->ring.map.flags = 0;
 		dev_priv->ring.map.mtrr = 0;
 
-		drm_core_ioremap(&dev_priv->ring.map, dev);
+		drm_core_ioremap_wc(&dev_priv->ring.map, dev);
 
 		if (dev_priv->ring.map.handle == NULL) {
 			i915_dma_cleanup(dev);
@@ -209,7 +209,7 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
 	dev_priv->back_offset = init->back_offset;
 	dev_priv->front_offset = init->front_offset;
 	dev_priv->current_page = 0;
-	dev_priv->sarea_priv->pf_current_page = dev_priv->current_page;
+	dev_priv->sarea_priv->pf_current_page = 0;
 
 	/* Allow hardware batchbuffers unless told otherwise.
 	 */
@@ -439,8 +439,7 @@ static void i915_emit_breadcrumb(struct drm_device *dev)
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	RING_LOCALS;
 
-	dev_priv->counter++;
-	if (dev_priv->counter > 0x7FFFFFFFUL)
+	if (++dev_priv->counter > 0x7FFFFFFFUL)
 		dev_priv->counter = 0;
 	if (dev_priv->sarea_priv)
 		dev_priv->sarea_priv->last_enqueue = dev_priv->counter;
@@ -574,7 +573,10 @@ static int i915_dispatch_flip(struct drm_device * dev)
 	OUT_RING(0);
 	ADVANCE_LP_RING();
 
-	dev_priv->sarea_priv->last_enqueue = dev_priv->counter++;
+	if (++dev_priv->counter > 0x7FFFFFFFUL)
+		dev_priv->counter = 0;
+	if (dev_priv->sarea_priv)
+		dev_priv->sarea_priv->last_enqueue = dev_priv->counter;
 
 	BEGIN_LP_RING(4);
 	OUT_RING(MI_STORE_DWORD_INDEX);
@@ -721,7 +723,7 @@ static int i915_flip_bufs(struct drm_device *dev, void *data,
 
 	DRM_DEBUG("%s\n", __func__);
 
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
+	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	ret = i915_dispatch_flip(dev);
 
@@ -758,7 +760,7 @@ static int i915_getparam(struct drm_device *dev, void *data,
 		value = 0;
 		break;
 	default:
-		DRM_ERROR("Unknown parameter %d\n", param->param);
+		DRM_DEBUG("Unknown parameter %d\n", param->param);
 		return -EINVAL;
 	}
 
@@ -791,7 +793,7 @@ static int i915_setparam(struct drm_device *dev, void *data,
 		dev_priv->allow_batchbuffer = param->value;
 		break;
 	default:
-		DRM_ERROR("unknown parameter %d\n", param->param);
+		DRM_DEBUG("unknown parameter %d\n", param->param);
 		return -EINVAL;
 	}
 
@@ -822,7 +824,7 @@ static int i915_set_status_page(struct drm_device *dev, void *data,
 	dev_priv->hws_map.flags = 0;
 	dev_priv->hws_map.mtrr = 0;
 
-	drm_core_ioremap(&dev_priv->hws_map, dev);
+	drm_core_ioremap_wc(&dev_priv->hws_map, dev);
 	if (dev_priv->hws_map.handle == NULL) {
 		i915_dma_cleanup(dev);
 		dev_priv->status_gfx_addr = 0;
@@ -880,8 +882,12 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 	/* Init HWS */
 	if (!I915_NEED_GFX_HWS(dev)) {
 		ret = i915_init_phys_hws(dev);
-		if (ret != 0)
+		if (ret != 0) {
+			drm_rmmap(dev, dev_priv->mmio_map);
+			drm_free(dev_priv, sizeof(struct drm_i915_private),
+			    DRM_MEM_DRIVER);
 			return ret;
+		}
 	}
 #ifdef __linux__
 	/* On the 945G/GM, the chipset reports the MSI capability on the
@@ -901,6 +907,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 	intel_opregion_init(dev);
 #endif
 	DRM_SPININIT(&dev_priv->user_irq_lock, "userirq");
+	dev_priv->user_irq_refcount = 0;
 
 	ret = drm_vblank_init(dev, I915_NUM_PIPE);
 
diff --git a/sys/dev/drm/i915_drv.c b/sys/dev/drm/i915_drv.c
index 9715814..39db236 100644
--- a/sys/dev/drm/i915_drv.c
+++ b/sys/dev/drm/i915_drv.c
@@ -46,11 +46,8 @@ static drm_pci_id_list_t i915_pciidlist[] = {
 static int i915_suspend(device_t kdev)
 {
 	struct drm_device *dev = device_get_softc(kdev);
-	struct drm_i915_private *dev_priv = dev->dev_private;
 
-	if (!dev || !dev_priv) {
-		DRM_ERROR("dev: 0x%lx, dev_priv: 0x%lx\n",
-			(unsigned long) dev, (unsigned long) dev_priv);
+	if (!dev || !dev->dev_private) {
 		DRM_ERROR("DRM not initialized, aborting suspend.\n");
 		return -ENODEV;
 	}
diff --git a/sys/dev/drm/i915_drv.h b/sys/dev/drm/i915_drv.h
index 0935a70..92c8f74 100644
--- a/sys/dev/drm/i915_drv.h
+++ b/sys/dev/drm/i915_drv.h
@@ -151,6 +151,8 @@ typedef struct drm_i915_private {
 	u32 saveDSPACNTR;
 	u32 saveDSPBCNTR;
 	u32 saveDSPARB;
+	u32 saveRENDERSTANDBY;
+	u32 saveHWS;
 	u32 savePIPEACONF;
 	u32 savePIPEBCONF;
 	u32 savePIPEASRC;
@@ -232,8 +234,8 @@ typedef struct drm_i915_private {
 	u8 saveAR_INDEX;
 	u8 saveAR[21];
 	u8 saveDACMASK;
-	u8 saveDACDATA[256*3]; /* 256 3-byte colors */
 	u8 saveCR[37];
+
 	struct {
 #ifdef __linux__
 		struct drm_mm gtt_space;
@@ -651,7 +653,8 @@ extern int i915_wait_ring(struct drm_device * dev, int n, const char *caller);
 
 #define IS_G4X(dev) ((dev)->pci_device == 0x2E02 || \
 		     (dev)->pci_device == 0x2E12 || \
-		     (dev)->pci_device == 0x2E22)
+		     (dev)->pci_device == 0x2E22 || \
+		     IS_GM45(dev))
 
 #define IS_G33(dev)    ((dev)->pci_device == 0x29C2 ||	\
 			(dev)->pci_device == 0x29B2 ||	\
diff --git a/sys/dev/drm/i915_irq.c b/sys/dev/drm/i915_irq.c
index 1117f62..7583031 100644
--- a/sys/dev/drm/i915_irq.c
+++ b/sys/dev/drm/i915_irq.c
@@ -43,21 +43,28 @@ __FBSDID("$FreeBSD$");
  * we leave them always unmasked in IMR and then control enabling them through
  * PIPESTAT alone.
  */
-#define I915_INTERRUPT_ENABLE_FIX (I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | \
-				   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT)
+#define I915_INTERRUPT_ENABLE_FIX	(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | \
+				   	 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT)
 
 /** Interrupts that we mask and unmask at runtime. */
-#define I915_INTERRUPT_ENABLE_VAR (I915_USER_INTERRUPT)
+#define I915_INTERRUPT_ENABLE_VAR	(I915_USER_INTERRUPT)
 
 /** These are all of the interrupts used by the driver */
-#define I915_INTERRUPT_ENABLE_MASK (I915_INTERRUPT_ENABLE_FIX | \
-				    I915_INTERRUPT_ENABLE_VAR)
+#define I915_INTERRUPT_ENABLE_MASK	(I915_INTERRUPT_ENABLE_FIX | \
+				    	 I915_INTERRUPT_ENABLE_VAR)
+
+#define I915_PIPE_VBLANK_STATUS		(PIPE_START_VBLANK_INTERRUPT_STATUS |\
+					 PIPE_VBLANK_INTERRUPT_STATUS)
+ 
+#define I915_PIPE_VBLANK_ENABLE		(PIPE_START_VBLANK_INTERRUPT_ENABLE |\
+					 PIPE_VBLANK_INTERRUPT_ENABLE)
+  
+#define DRM_I915_VBLANK_PIPE_ALL	(DRM_I915_VBLANK_PIPE_A | \
+					 DRM_I915_VBLANK_PIPE_B)
 
 static inline void
 i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
 {
-	DRM_DEBUG("irq_enable_reg = 0x%08x, mask = 0x%08x\n",
-	    dev_priv->irq_mask_reg, mask);
 	mask &= I915_INTERRUPT_ENABLE_VAR;
 	if ((dev_priv->irq_mask_reg & mask) != 0) {
 		dev_priv->irq_mask_reg &= ~mask;
@@ -189,59 +196,84 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 	u32 iir, new_iir;
 	u32 pipea_stats, pipeb_stats;
+	u32 vblank_status;
+	u32 vblank_enable;
+	int irq_received;
 
 	atomic_inc(&dev_priv->irq_received);
 
-	for (iir = I915_READ(IIR) ; iir != 0 ; iir = new_iir) {
+	iir = I915_READ(IIR);
 
-		pipea_stats = pipeb_stats = 0;
+	if (IS_I965G(dev)) {
+		vblank_status = I915_START_VBLANK_INTERRUPT_STATUS;
+		vblank_enable = PIPE_START_VBLANK_INTERRUPT_ENABLE;
+	} else {
+		vblank_status = I915_VBLANK_INTERRUPT_STATUS;
+		vblank_enable = I915_VBLANK_INTERRUPT_ENABLE;
+	}
+
+	for (;;) {
+		irq_received = iir != 0;
+
+		/* Can't rely on pipestat interrupt bit in iir as it might
+		 * have been cleared after the pipestat interrupt was received.
+		 * It doesn't set the bit in iir again, but it still produces
+		 * interrupts (for non-MSI).
+		 */
+		DRM_SPINLOCK(&dev_priv->user_irq_lock);
+		pipea_stats = I915_READ(PIPEASTAT);
+		pipeb_stats = I915_READ(PIPEBSTAT);
 
 		/*
 		 * Clear the PIPE(A|B)STAT regs before the IIR
 		 */
-		if (iir & I915_DISPLAY_PIPE_A_EVENT_INTERRUPT) {
-			DRM_SPINLOCK(&dev_priv->user_irq_lock);
-			pipea_stats = I915_READ(PIPEASTAT);
+		if (pipea_stats & 0x8000ffff) {
 			I915_WRITE(PIPEASTAT, pipea_stats);
-			DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
+			irq_received = 1;
 		}
 
-		if (iir & I915_DISPLAY_PIPE_B_EVENT_INTERRUPT) {
-			DRM_SPINLOCK(&dev_priv->user_irq_lock);
-			pipeb_stats = I915_READ(PIPEBSTAT);
+		if (pipeb_stats & 0x8000ffff) {
 			I915_WRITE(PIPEBSTAT, pipeb_stats);
-			DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
+			irq_received = 1;
 		}
+		DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
 
-		I915_WRITE(IIR, iir);
-		new_iir = I915_READ(IIR);
+		if (!irq_received)
+			break;
 
-		DRM_DEBUG("iir = 0x%08x, pipestats a = 0x%08x, b = 0x%08x\n",
-		    iir, pipea_stats, pipeb_stats);
+		I915_WRITE(IIR, iir);
+		new_iir = I915_READ(IIR); /* Flush posted writes */
 
 		if (dev_priv->sarea_priv)
 			dev_priv->sarea_priv->last_dispatch =
 			    READ_BREADCRUMB(dev_priv);
 
 		if (iir & I915_USER_INTERRUPT) {
-#ifdef I915_HAVE_GEM
-			dev_priv->mm.irq_gem_seqno = i915_get_gem_seqno(dev);
-#endif
 			DRM_WAKEUP(&dev_priv->irq_queue);
 		}
 
-		if (pipea_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS |
-		    PIPE_VBLANK_INTERRUPT_STATUS))
+		if (pipea_stats & vblank_status)
 			drm_handle_vblank(dev, 0);
 
-		if (pipeb_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS |
-		    PIPE_VBLANK_INTERRUPT_STATUS))
+		if (pipeb_stats & vblank_status)
 			drm_handle_vblank(dev, 1);
-#ifdef __linux__
-		if ((pipeb_stats & I915_LEGACY_BLC_EVENT_STATUS) ||
-		    (iir & I915_ASLE_INTERRUPT))
-			opregion_asle_intr(dev);
-#endif
+
+		/* With MSI, interrupts are only generated when iir
+		 * transitions from zero to nonzero.  If another bit got
+		 * set while we were handling the existing iir bits, then
+		 * we would never get another interrupt.
+		 *
+		 * This is fine on non-MSI as well, as if we hit this path
+		 * we avoid exiting the interrupt handler only to generate
+		 * another one.
+		 *
+		 * Note that for MSI this could cause a stray interrupt report
+		 * if an interrupt landed in the time between writing IIR and
+		 * the posting read.  This should be rare enough to never
+		 * trigger the 99% of 100,000 interrupts test for disabling
+		 * stray interrupts.
+		 */
+		iir = new_iir;
 	}
 }
 
@@ -252,14 +284,13 @@ static int i915_emit_irq(struct drm_device * dev)
 
 	i915_kernel_lost_context(dev);
 
-	DRM_DEBUG("\n");
-
-	dev_priv->counter++;
-	if (dev_priv->counter > 0x7FFFFFFFUL)
-		dev_priv->counter = 1;
+	if (++dev_priv->counter > 0x7FFFFFFFUL)
+		dev_priv->counter = 0;
 	if (dev_priv->sarea_priv)
 		dev_priv->sarea_priv->last_enqueue = dev_priv->counter;
 
+	DRM_DEBUG("emitting: %d\n", dev_priv->counter);
+
 	BEGIN_LP_RING(4);
 	OUT_RING(MI_STORE_DWORD_INDEX);
 	OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
@@ -273,27 +304,25 @@ static int i915_emit_irq(struct drm_device * dev)
 void i915_user_irq_get(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	unsigned long irqflags;
 
 	DRM_DEBUG("\n");
-	DRM_SPINLOCK_IRQSAVE(&dev_priv->user_irq_lock, irqflags);
+	DRM_SPINLOCK(&dev_priv->user_irq_lock);
 	if (dev->irq_enabled && (++dev_priv->user_irq_refcount == 1))
 		i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
-	DRM_SPINUNLOCK_IRQRESTORE(&dev_priv->user_irq_lock, irqflags);
+	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
 }
 
 void i915_user_irq_put(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	unsigned long irqflags;
-
-	DRM_SPINLOCK_IRQSAVE(&dev_priv->user_irq_lock, irqflags);
-#ifdef __linux__
-	BUG_ON(dev->irq_enabled && dev_priv->user_irq_refcount <= 0);
-#endif
-	if (dev->irq_enabled && (--dev_priv->user_irq_refcount == 0))
-		i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
-	DRM_SPINUNLOCK_IRQRESTORE(&dev_priv->user_irq_lock, irqflags);
+
+	DRM_SPINLOCK(&dev_priv->user_irq_lock);
+	if (dev->irq_enabled) {
+		KASSERT(dev_priv->user_irq_refcount > 0, ("invalid refcount"));
+		if (--dev_priv->user_irq_refcount == 0)
+			i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
+	}
+	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
 }
 
 static int i915_wait_irq(struct drm_device * dev, int irq_nr)
@@ -301,9 +330,6 @@ static int i915_wait_irq(struct drm_device * dev, int irq_nr)
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 	int ret = 0;
 
-	DRM_DEBUG("irq_nr=%d breadcrumb=%d\n", irq_nr,
-		  READ_BREADCRUMB(dev_priv));
-
 	if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
 		if (dev_priv->sarea_priv) {
 			dev_priv->sarea_priv->last_dispatch =
@@ -315,20 +341,22 @@ static int i915_wait_irq(struct drm_device * dev, int irq_nr)
 	if (dev_priv->sarea_priv)
 		dev_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
 
+	DRM_DEBUG("irq_nr=%d breadcrumb=%d\n", irq_nr,
+		  READ_BREADCRUMB(dev_priv));
+
 	i915_user_irq_get(dev);
 	DRM_WAIT_ON(ret, dev_priv->irq_queue, 3 * DRM_HZ,
 		    READ_BREADCRUMB(dev_priv) >= irq_nr);
 	i915_user_irq_put(dev);
 
+	if (ret == -ERESTART)
+		DRM_DEBUG("restarting syscall\n");
+
 	if (ret == -EBUSY) {
 		DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
 			  READ_BREADCRUMB(dev_priv), (int)dev_priv->counter);
 	}
 
-	if (dev_priv->sarea_priv)
-		dev_priv->sarea_priv->last_dispatch =
-			READ_BREADCRUMB(dev_priv);
-
 	return ret;
 }
 
@@ -341,13 +369,13 @@ int i915_irq_emit(struct drm_device *dev, void *data,
 	drm_i915_irq_emit_t *emit = data;
 	int result;
 
-	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
-
 	if (!dev_priv) {
 		DRM_ERROR("called with no initialization\n");
 		return -EINVAL;
 	}
 
+	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
+
 	result = i915_emit_irq(dev);
 
 	if (DRM_COPY_TO_USER(emit->irq_seq, &result, sizeof(int))) {
@@ -380,21 +408,21 @@ int i915_irq_wait(struct drm_device *dev, void *data,
 int i915_enable_vblank(struct drm_device *dev, int pipe)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	unsigned long irqflags;
-	u32 pipestat;
+	int pipeconf_reg = (pipe == 0) ? PIPEACONF : PIPEBCONF;
+	u32 pipeconf;
 
-	/*
-	 * Older chips didn't have the start vblank interrupt,
-	 * but
-	 */
-	if (IS_I965G (dev))
-		pipestat = PIPE_START_VBLANK_INTERRUPT_ENABLE;
-	else
-		pipestat = PIPE_VBLANK_INTERRUPT_ENABLE;
+	pipeconf = I915_READ(pipeconf_reg);
+	if (!(pipeconf & PIPEACONF_ENABLE))
+		return -EINVAL;
 
-	DRM_SPINLOCK_IRQSAVE(&dev_priv->user_irq_lock, irqflags);
-	i915_enable_pipestat(dev_priv, pipe, pipestat);
-	DRM_SPINUNLOCK_IRQRESTORE(&dev_priv->user_irq_lock, irqflags);
+	DRM_SPINLOCK(&dev_priv->user_irq_lock);
+	if (IS_I965G(dev))
+		i915_enable_pipestat(dev_priv, pipe,
+				     PIPE_START_VBLANK_INTERRUPT_ENABLE);
+	else
+		i915_enable_pipestat(dev_priv, pipe,
+				     PIPE_VBLANK_INTERRUPT_ENABLE);
+	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
 	return 0;
 }
 
@@ -404,12 +432,12 @@ int i915_enable_vblank(struct drm_device *dev, int pipe)
 void i915_disable_vblank(struct drm_device *dev, int pipe)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	unsigned long irqflags;
 
-	DRM_SPINLOCK_IRQSAVE(&dev_priv->user_irq_lock, irqflags);
+	DRM_SPINLOCK(&dev_priv->user_irq_lock);
 	i915_disable_pipestat(dev_priv, pipe,
-	    PIPE_START_VBLANK_INTERRUPT_ENABLE | PIPE_VBLANK_INTERRUPT_ENABLE);
-	DRM_SPINUNLOCK_IRQRESTORE(&dev_priv->user_irq_lock, irqflags);
+			      PIPE_VBLANK_INTERRUPT_ENABLE |
+			      PIPE_START_VBLANK_INTERRUPT_ENABLE);
+	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
 }
 
 /* Set the vblank monitor pipe
@@ -463,7 +491,6 @@ int i915_vblank_swap(struct drm_device *dev, void *data,
 	 * Context switching to userland and back is plenty fast enough for
 	 * meeting the requirements of vblank swapping.
 	 */
-
 	return -EINVAL;
 }
 
@@ -473,6 +500,8 @@ void i915_driver_irq_preinstall(struct drm_device * dev)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 
+	atomic_set_int(&dev_priv->irq_received, 0);
+
 	I915_WRITE(HWSTAM, 0xeffe);
 	I915_WRITE(PIPEASTAT, 0);
 	I915_WRITE(PIPEBSTAT, 0);
@@ -505,13 +534,6 @@ int i915_driver_irq_postinstall(struct drm_device *dev)
 	I915_WRITE(IER, I915_INTERRUPT_ENABLE_MASK);
 	I915_WRITE(IMR, dev_priv->irq_mask_reg);
 	(void) I915_READ(IER);
-#ifdef __linux__
-	opregion_enable_asle(dev);
-#endif
-	DRM_INIT_WAITQUEUE(&dev_priv->irq_queue);
-
-	i915_enable_vblank(dev, 0);
-	i915_enable_vblank(dev, 1);
 
 	return 0;
 }
diff --git a/sys/dev/drm/i915_reg.h b/sys/dev/drm/i915_reg.h
index a9142d0..1a32c5f 100644
--- a/sys/dev/drm/i915_reg.h
+++ b/sys/dev/drm/i915_reg.h
@@ -38,7 +38,7 @@ __FBSDID("$FreeBSD$");
 #define INTEL_GMCH_MEM_64M	0x1
 #define INTEL_GMCH_MEM_128M	0
 
-#define INTEL_855_GMCH_GMS_MASK		(0x7 << 4)
+#define INTEL_GMCH_GMS_MASK		(0xf << 4)
 #define INTEL_855_GMCH_GMS_DISABLED	(0x0 << 4)
 #define INTEL_855_GMCH_GMS_STOLEN_1M	(0x1 << 4)
 #define INTEL_855_GMCH_GMS_STOLEN_4M	(0x2 << 4)
@@ -48,6 +48,12 @@ __FBSDID("$FreeBSD$");
 
 #define INTEL_915G_GMCH_GMS_STOLEN_48M	(0x6 << 4)
 #define INTEL_915G_GMCH_GMS_STOLEN_64M	(0x7 << 4)
+#define INTEL_GMCH_GMS_STOLEN_128M	(0x8 << 4)
+#define INTEL_GMCH_GMS_STOLEN_256M	(0x9 << 4)
+#define INTEL_GMCH_GMS_STOLEN_96M	(0xa << 4)
+#define INTEL_GMCH_GMS_STOLEN_160M	(0xb << 4)
+#define INTEL_GMCH_GMS_STOLEN_224M	(0xc << 4)
+#define INTEL_GMCH_GMS_STOLEN_352M	(0xd << 4)
 
 /* PCI config space */
 
@@ -178,9 +184,27 @@ __FBSDID("$FreeBSD$");
 #define   DISPLAY_PLANE_B           (1<<20)
 
 /*
- * Instruction and interrupt control regs
+ * Fence registers
  */
+#define FENCE_REG_830_0			0x2000
+#define FENCE_REG_945_8			0x3000
+#define   I830_FENCE_START_MASK		0x07f80000
+#define   I830_FENCE_TILING_Y_SHIFT	12
+#define   I830_FENCE_SIZE_BITS(size)	((ffs((size) >> 19) - 1) << 8)
+#define   I830_FENCE_PITCH_SHIFT	4
+#define   I830_FENCE_REG_VALID		(1<<0)
+
+#define   I915_FENCE_START_MASK		0x0ff00000
+#define   I915_FENCE_SIZE_BITS(size)	((ffs((size) >> 20) - 1) << 8)
+
+#define FENCE_REG_965_0			0x03000
+#define   I965_FENCE_PITCH_SHIFT	2
+#define   I965_FENCE_TILING_Y_SHIFT	1
+#define   I965_FENCE_REG_VALID		(1<<0)
 
+/*
+ * Instruction and interrupt control regs
+ */
 #define PRB0_TAIL	0x02030
 #define PRB0_HEAD	0x02034
 #define PRB0_START	0x02038
@@ -248,6 +272,7 @@ __FBSDID("$FreeBSD$");
 #define   CM0_RC_OP_FLUSH_DISABLE (1<<0)
 #define GFX_FLSH_CNTL	0x02170 /* 915+ only */
 
+
 /*
  * Framebuffer compression (915+ only)
  */
@@ -525,11 +550,17 @@ __FBSDID("$FreeBSD$");
 #define DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED	(2 << 0)
 #define DCC_ADDRESSING_MODE_MASK			(3 << 0)
 #define DCC_CHANNEL_XOR_DISABLE				(1 << 10)
+#define DCC_CHANNEL_XOR_BIT_17				(1 << 9)
 
 /** 965 MCH register controlling DRAM channel configuration */
 #define C0DRB3			0x10206
 #define C1DRB3			0x10606
 
+/** GM965 GM45 render standby register */
+#define MCHBAR_RENDER_STANDBY	0x111B8
+
+#define PEG_BAND_GAP_DATA	0x14d68
+
 /*
  * Overlay regs
  */
@@ -593,6 +624,9 @@ __FBSDID("$FreeBSD$");
 
 /* Hotplug control (945+ only) */
 #define PORT_HOTPLUG_EN		0x61110
+#define   HDMIB_HOTPLUG_INT_EN			(1 << 29)
+#define   HDMIC_HOTPLUG_INT_EN			(1 << 28)
+#define   HDMID_HOTPLUG_INT_EN			(1 << 27)
 #define   SDVOB_HOTPLUG_INT_EN			(1 << 26)
 #define   SDVOC_HOTPLUG_INT_EN			(1 << 25)
 #define   TV_HOTPLUG_INT_EN			(1 << 18)
@@ -600,6 +634,9 @@ __FBSDID("$FreeBSD$");
 #define   CRT_HOTPLUG_FORCE_DETECT		(1 << 3)
 
 #define PORT_HOTPLUG_STAT	0x61114
+#define   HDMIB_HOTPLUG_INT_STATUS		(1 << 29)
+#define   HDMIC_HOTPLUG_INT_STATUS		(1 << 28)
+#define   HDMID_HOTPLUG_INT_STATUS		(1 << 27)
 #define   CRT_HOTPLUG_INT_STATUS		(1 << 11)
 #define   TV_HOTPLUG_INT_STATUS			(1 << 10)
 #define   CRT_HOTPLUG_MONITOR_MASK		(3 << 8)
@@ -629,7 +666,16 @@ __FBSDID("$FreeBSD$");
 #define   SDVO_PHASE_SELECT_DEFAULT	(6 << 19)
 #define   SDVO_CLOCK_OUTPUT_INVERT	(1 << 18)
 #define   SDVOC_GANG_MODE		(1 << 16)
+#define   SDVO_ENCODING_SDVO		(0x0 << 10)
+#define   SDVO_ENCODING_HDMI		(0x2 << 10)
+/** Requird for HDMI operation */
+#define   SDVO_NULL_PACKETS_DURING_VSYNC (1 << 9)
 #define   SDVO_BORDER_ENABLE		(1 << 7)
+#define   SDVO_AUDIO_ENABLE		(1 << 6)
+/** New with 965, default is to be set */
+#define   SDVO_VSYNC_ACTIVE_HIGH	(1 << 4)
+/** New with 965, default is to be set */
+#define   SDVO_HSYNC_ACTIVE_HIGH	(1 << 3)
 #define   SDVOB_PCIE_CONCURRENCY	(1 << 3)
 #define   SDVO_DETECTED			(1 << 2)
 /* Bits to be preserved when writing */
@@ -1403,6 +1449,7 @@ __FBSDID("$FreeBSD$");
 #define PIPEB_FRMCOUNT_GM45	0x71040
 #define PIPEB_FLIPCOUNT_GM45	0x71044
 
+
 /* Display B control */
 #define DSPBCNTR		0x71180
 #define   DISPPLANE_ALPHA_TRANS_ENABLE		(1<<15)
diff --git a/sys/dev/drm/i915_suspend.c b/sys/dev/drm/i915_suspend.c
index 9f8fad6..0692f52 100644
--- a/sys/dev/drm/i915_suspend.c
+++ b/sys/dev/drm/i915_suspend.c
@@ -125,11 +125,6 @@ static void i915_save_vga(struct drm_device *dev)
 
 	/* VGA color palette registers */
 	dev_priv->saveDACMASK = I915_READ8(VGA_DACMASK);
-	/* DACCRX automatically increments during read */
-	I915_WRITE8(VGA_DACRX, 0);
-	/* Read 3 bytes of color data from each index */
-	for (i = 0; i < 256 * 3; i++)
-		dev_priv->saveDACDATA[i] = I915_READ8(VGA_DACDATA);
 
 	/* MSR bits */
 	dev_priv->saveMSR = I915_READ8(VGA_MSR_READ);
@@ -231,12 +226,6 @@ static void i915_restore_vga(struct drm_device *dev)
 
 	/* VGA color palette registers */
 	I915_WRITE8(VGA_DACMASK, dev_priv->saveDACMASK);
-	/* DACCRX automatically increments during read */
-	I915_WRITE8(VGA_DACWX, 0);
-	/* Read 3 bytes of color data from each index */
-	for (i = 0; i < 256 * 3; i++)
-		I915_WRITE8(VGA_DACDATA, dev_priv->saveDACDATA[i]);
-
 }
 
 int i915_save_state(struct drm_device *dev)
@@ -250,6 +239,13 @@ int i915_save_state(struct drm_device *dev)
 	pci_read_config_byte(dev->pdev, LBB, &dev_priv->saveLBB);
 #endif
 
+	/* Render Standby */
+	if (IS_I965G(dev) && IS_MOBILE(dev))
+		dev_priv->saveRENDERSTANDBY = I915_READ(MCHBAR_RENDER_STANDBY);
+
+	/* Hardware status page */
+	dev_priv->saveHWS = I915_READ(HWS_PGA);
+
 	/* Display arbitration control */
 	dev_priv->saveDSPARB = I915_READ(DSPARB);
 
@@ -379,6 +375,14 @@ int i915_restore_state(struct drm_device *dev)
 	pci_write_config_byte(dev->pdev, LBB, dev_priv->saveLBB);
 #endif
 
+	/* Render Standby */
+	if (IS_I965G(dev) && IS_MOBILE(dev))
+		I915_WRITE(MCHBAR_RENDER_STANDBY, dev_priv->saveRENDERSTANDBY);
+
+	/* Hardware status page */
+	I915_WRITE(HWS_PGA, dev_priv->saveHWS);
+
+	/* Display arbitration */
 	I915_WRITE(DSPARB, dev_priv->saveDSPARB);
 
 	/* Pipe & plane A info */
@@ -511,7 +515,7 @@ int i915_restore_state(struct drm_device *dev)
 
 	for (i = 0; i < 16; i++) {
 		I915_WRITE(SWF00 + (i << 2), dev_priv->saveSWF0[i]);
-		I915_WRITE(SWF10 + (i << 2), dev_priv->saveSWF1[i+7]);
+		I915_WRITE(SWF10 + (i << 2), dev_priv->saveSWF1[i]);
 	}
 	for (i = 0; i < 3; i++)
 		I915_WRITE(SWF30 + (i << 2), dev_priv->saveSWF2[i]);
diff --git a/sys/dev/drm/mga_irq.c b/sys/dev/drm/mga_irq.c
index 005d43d..8e28987 100644
--- a/sys/dev/drm/mga_irq.c
+++ b/sys/dev/drm/mga_irq.c
@@ -139,6 +139,9 @@ int mga_driver_fence_wait(struct drm_device * dev, unsigned int *sequence)
 		    (((cur_fence = atomic_read(&dev_priv->last_fence_retired))
 		      - *sequence) <= (1 << 23)));
 
+	if (ret == -ERESTART)
+		DRM_DEBUG("restarting syscall\n");
+
 	*sequence = cur_fence;
 
 	return ret;
diff --git a/sys/dev/drm/r300_cmdbuf.c b/sys/dev/drm/r300_cmdbuf.c
index b9e9f0f..8251562 100644
--- a/sys/dev/drm/r300_cmdbuf.c
+++ b/sys/dev/drm/r300_cmdbuf.c
@@ -208,6 +208,10 @@ void r300_init_reg_flags(struct drm_device *dev)
 	ADD_RANGE(0x42C0, 2);
 	ADD_RANGE(R300_RS_CNTL_0, 2);
 
+	ADD_RANGE(R300_SU_REG_DEST, 1);
+	if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV530)
+		ADD_RANGE(RV530_FG_ZBREG_DEST, 1);
+
 	ADD_RANGE(R300_SC_HYPERZ, 2);
 	ADD_RANGE(0x43E8, 1);
 
@@ -232,7 +236,11 @@ void r300_init_reg_flags(struct drm_device *dev)
 	ADD_RANGE_MARK(R300_ZB_DEPTHOFFSET, 1, MARK_CHECK_OFFSET);	/* check offset */
 	ADD_RANGE(R300_ZB_DEPTHPITCH, 1);
 	ADD_RANGE(R300_ZB_DEPTHCLEARVALUE, 1);
-	ADD_RANGE(R300_ZB_ZMASK_OFFSET, 13);
+	ADD_RANGE(R300_ZB_ZMASK_OFFSET, 5);
+	ADD_RANGE(R300_ZB_HIZ_OFFSET, 5);
+	ADD_RANGE(R300_ZB_ZPASS_DATA, 1);
+	ADD_RANGE_MARK(R300_ZB_ZPASS_ADDR, 1, MARK_CHECK_OFFSET);       /* check offset */
+	ADD_RANGE(R300_ZB_DEPTHXY_OFFSET, 1)
 
 	ADD_RANGE(R300_TX_FILTER_0, 16);
 	ADD_RANGE(R300_TX_FILTER1_0, 16);
diff --git a/sys/dev/drm/r300_reg.h b/sys/dev/drm/r300_reg.h
index c92779e..928440f 100644
--- a/sys/dev/drm/r300_reg.h
+++ b/sys/dev/drm/r300_reg.h
@@ -1776,6 +1776,11 @@ __FBSDID("$FreeBSD$");
 #define R500_RB3D_COLOR_CLEAR_VALUE_AR  0x46c0
 #define R500_RB3D_CONSTANT_COLOR_AR     0x4ef8
 
+#define R300_SU_REG_DEST                0x42c8
+#define RV530_FG_ZBREG_DEST             0x4be8
+#define R300_ZB_ZPASS_DATA              0x4f58
+#define R300_ZB_ZPASS_ADDR              0x4f5c
+
 #endif /* _R300_REG_H */
 
 /* *INDENT-ON* */
diff --git a/sys/dev/drm/r600_cp.c b/sys/dev/drm/r600_cp.c
index 9cf4f31..f9b16af 100644
--- a/sys/dev/drm/r600_cp.c
+++ b/sys/dev/drm/r600_cp.c
@@ -174,7 +174,6 @@ int r600_page_table_init(struct drm_device *dev)
 		if (entry->busaddr[i] == 0) {
 			DRM_ERROR("unable to map PCIGART pages!\n");
 			r600_page_table_cleanup(dev, gart_info);
-			ret = -EINVAL;
 			goto done;
 		}
 #endif
@@ -193,6 +192,7 @@ int r600_page_table_init(struct drm_device *dev)
 			entry_addr += ATI_PCIGART_PAGE_SIZE;
 		}
 	}
+	ret = 1;
 #ifdef __linux__
 done:
 #endif
@@ -282,8 +282,50 @@ static void r600_vm_init(struct drm_device *dev)
 /* load r600 microcode */
 static void r600_cp_load_microcode(drm_radeon_private_t *dev_priv)
 {
+	const u32 (*cp)[3];
+	const u32 *pfp;
 	int i;
 
+	switch (dev_priv->flags & RADEON_FAMILY_MASK) {
+	case CHIP_R600:
+		DRM_INFO("Loading R600 Microcode\n");
+		cp  = R600_cp_microcode;
+		pfp = R600_pfp_microcode;
+		break;
+	case CHIP_RV610:
+		DRM_INFO("Loading RV610 Microcode\n");
+		cp  = RV610_cp_microcode;
+		pfp = RV610_pfp_microcode;
+		break;
+	case CHIP_RV630:
+		DRM_INFO("Loading RV630 Microcode\n");
+		cp  = RV630_cp_microcode;
+		pfp = RV630_pfp_microcode;
+		break;
+	case CHIP_RV620:
+		DRM_INFO("Loading RV620 Microcode\n");
+		cp  = RV620_cp_microcode;
+		pfp = RV620_pfp_microcode;
+		break;
+	case CHIP_RV635:
+		DRM_INFO("Loading RV635 Microcode\n");
+		cp  = RV635_cp_microcode;
+		pfp = RV635_pfp_microcode;
+		break;
+	case CHIP_RV670:
+		DRM_INFO("Loading RV670 Microcode\n");
+		cp  = RV670_cp_microcode;
+		pfp = RV670_pfp_microcode;
+		break;
+	case CHIP_RS780:
+		DRM_INFO("Loading RS780 Microcode\n");
+		cp  = RS780_cp_microcode;
+		pfp = RS780_pfp_microcode;
+		break;
+	default:
+		return;
+	}
+
 	r600_do_cp_stop(dev_priv);
 
 	RADEON_WRITE(R600_CP_RB_CNTL,
@@ -298,116 +340,19 @@ static void r600_cp_load_microcode(drm_radeon_private_t *dev_priv)
 
 	RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600)) {
-		DRM_INFO("Loading R600 CP Microcode\n");
-		for (i = 0; i < PM4_UCODE_SIZE; i++) {
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     R600_cp_microcode[i][0]);
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     R600_cp_microcode[i][1]);
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     R600_cp_microcode[i][2]);
-		}
-
-		RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-		DRM_INFO("Loading R600 PFP Microcode\n");
-		for (i = 0; i < PFP_UCODE_SIZE; i++)
-			RADEON_WRITE(R600_CP_PFP_UCODE_DATA, R600_pfp_microcode[i]);
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610)) {
-		DRM_INFO("Loading RV610 CP Microcode\n");
-		for (i = 0; i < PM4_UCODE_SIZE; i++) {
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV610_cp_microcode[i][0]);
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV610_cp_microcode[i][1]);
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV610_cp_microcode[i][2]);
-		}
-
-		RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-		DRM_INFO("Loading RV610 PFP Microcode\n");
-		for (i = 0; i < PFP_UCODE_SIZE; i++)
-			RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV610_pfp_microcode[i]);
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV630)) {
-		DRM_INFO("Loading RV630 CP Microcode\n");
-		for (i = 0; i < PM4_UCODE_SIZE; i++) {
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV630_cp_microcode[i][0]);
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV630_cp_microcode[i][1]);
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV630_cp_microcode[i][2]);
-		}
-
-		RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-		DRM_INFO("Loading RV630 PFP Microcode\n");
-		for (i = 0; i < PFP_UCODE_SIZE; i++)
-			RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV630_pfp_microcode[i]);
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620)) {
-		DRM_INFO("Loading RV620 CP Microcode\n");
-		for (i = 0; i < PM4_UCODE_SIZE; i++) {
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV620_cp_microcode[i][0]);
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV620_cp_microcode[i][1]);
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV620_cp_microcode[i][2]);
-		}
-
-		RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-		DRM_INFO("Loading RV620 PFP Microcode\n");
-		for (i = 0; i < PFP_UCODE_SIZE; i++)
-			RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV620_pfp_microcode[i]);
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV635)) {
-		DRM_INFO("Loading RV635 CP Microcode\n");
-		for (i = 0; i < PM4_UCODE_SIZE; i++) {
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV635_cp_microcode[i][0]);
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV635_cp_microcode[i][1]);
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV635_cp_microcode[i][2]);
-		}
-
-		RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-		DRM_INFO("Loading RV635 PFP Microcode\n");
-		for (i = 0; i < PFP_UCODE_SIZE; i++)
-			RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV635_pfp_microcode[i]);
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV670)) {
-		DRM_INFO("Loading RV670 CP Microcode\n");
-		for (i = 0; i < PM4_UCODE_SIZE; i++) {
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV670_cp_microcode[i][0]);
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV670_cp_microcode[i][1]);
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV670_cp_microcode[i][2]);
-		}
+	for (i = 0; i < PM4_UCODE_SIZE; i++) {
+		RADEON_WRITE(R600_CP_ME_RAM_DATA, cp[i][0]);
+		RADEON_WRITE(R600_CP_ME_RAM_DATA, cp[i][1]);
+		RADEON_WRITE(R600_CP_ME_RAM_DATA, cp[i][2]);
+	}
 
-		RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-		DRM_INFO("Loading RV670 PFP Microcode\n");
-		for (i = 0; i < PFP_UCODE_SIZE; i++)
-			RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV670_pfp_microcode[i]);
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780)) {
-		DRM_INFO("Loading RS780 CP Microcode\n");
-		for (i = 0; i < PM4_UCODE_SIZE; i++) {
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV670_cp_microcode[i][0]);
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV670_cp_microcode[i][1]);
-			RADEON_WRITE(R600_CP_ME_RAM_DATA,
-				     RV670_cp_microcode[i][2]);
-		}
+	RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
+	for (i = 0; i < PFP_UCODE_SIZE; i++)
+		RADEON_WRITE(R600_CP_PFP_UCODE_DATA, pfp[i]);
 
-		RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-		DRM_INFO("Loading RS780 PFP Microcode\n");
-		for (i = 0; i < PFP_UCODE_SIZE; i++)
-			RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV670_pfp_microcode[i]);
-	}
 	RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 	RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 	RADEON_WRITE(R600_CP_ME_RAM_RADDR, 0);
-
 }
 
 static void r700_vm_init(struct drm_device *dev)
@@ -465,8 +410,30 @@ static void r700_vm_init(struct drm_device *dev)
 /* load r600 microcode */
 static void r700_cp_load_microcode(drm_radeon_private_t *dev_priv)
 {
+	const u32 *pfp;
+	const u32 *cp;
 	int i;
 
+	switch (dev_priv->flags & RADEON_FAMILY_MASK) {
+	case CHIP_RV770:
+		DRM_INFO("Loading RV770/RV790 Microcode\n");
+		pfp = RV770_pfp_microcode;
+		cp  = RV770_cp_microcode;
+		break;
+	case CHIP_RV730:
+		DRM_INFO("Loading RV730 Microcode\n");
+		pfp = RV730_pfp_microcode;
+		cp  = RV730_cp_microcode;
+		break;
+	case CHIP_RV710:
+		DRM_INFO("Loading RV710 Microcode\n");
+		pfp = RV710_pfp_microcode;
+		cp  = RV710_cp_microcode;
+		break;
+	default:
+		return;
+	}
+
 	r600_do_cp_stop(dev_priv);
 
 	RADEON_WRITE(R600_CP_RB_CNTL,
@@ -479,51 +446,19 @@ static void r700_cp_load_microcode(drm_radeon_private_t *dev_priv)
 	DRM_UDELAY(15000);
 	RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
 
+	RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
+	for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
+		RADEON_WRITE(R600_CP_PFP_UCODE_DATA, pfp[i]);
+	RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV770)) {
-		RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-		DRM_INFO("Loading RV770 PFP Microcode\n");
-		for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
-			RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV770_pfp_microcode[i]);
-		RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-
-		RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
-		DRM_INFO("Loading RV770 CP Microcode\n");
-		for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
-			RADEON_WRITE(R600_CP_ME_RAM_DATA, RV770_cp_microcode[i]);
-		RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
-
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV730)) {
-		RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-		DRM_INFO("Loading RV730 PFP Microcode\n");
-		for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
-			RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV730_pfp_microcode[i]);
-		RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-
-		RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
-		DRM_INFO("Loading RV730 CP Microcode\n");
-		for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
-			RADEON_WRITE(R600_CP_ME_RAM_DATA, RV730_cp_microcode[i]);
-		RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
-
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV710)) {
-		RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-		DRM_INFO("Loading RV710 PFP Microcode\n");
-		for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
-			RADEON_WRITE(R600_CP_PFP_UCODE_DATA, RV710_pfp_microcode[i]);
-		RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
-
-		RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
-		DRM_INFO("Loading RV710 CP Microcode\n");
-		for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
-			RADEON_WRITE(R600_CP_ME_RAM_DATA, RV710_cp_microcode[i]);
-		RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
+	RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
+	for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
+		RADEON_WRITE(R600_CP_ME_RAM_DATA, cp[i]);
+	RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 
-	}
 	RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 	RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 	RADEON_WRITE(R600_CP_ME_RAM_RADDR, 0);
-
 }
 
 static void r600_test_writeback(drm_radeon_private_t *dev_priv)
@@ -1740,9 +1675,6 @@ static void r600_cp_init_ring_buffer(struct drm_device *dev,
 
 	RADEON_WRITE(R600_CP_DEBUG, (1 << 27) | (1 << 28));
 
-	/* Start with assuming that writeback doesn't work */
-	dev_priv->writeback_works = 0;
-
 	/* Initialize the scratch register pointer.  This will cause
 	 * the scratch register values to be written out to memory
 	 * whenever they are updated.
@@ -2103,7 +2035,7 @@ int r600_do_init_cp(struct drm_device *dev, drm_radeon_init_t *init,
 			  dev_priv->gart_info.addr,
 			  dev_priv->pcigart_offset);
 
-		if (r600_page_table_init(dev)) {
+		if (!r600_page_table_init(dev)) {
 			DRM_ERROR("Failed to init GART table\n");
 			r600_do_cleanup_cp(dev);
 			return -EINVAL;
diff --git a/sys/dev/drm/radeon_cp.c b/sys/dev/drm/radeon_cp.c
index 972751c..8f723ce 100644
--- a/sys/dev/drm/radeon_cp.c
+++ b/sys/dev/drm/radeon_cp.c
@@ -436,7 +436,7 @@ static void radeon_init_pipes(drm_radeon_private_t *dev_priv)
 
 	if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
 		RADEON_WRITE_PLL(R500_DYN_SCLK_PWMEM_PIPE, (1 | ((gb_pipe_sel >> 8) & 0xf) << 4));
-		RADEON_WRITE(R500_SU_REG_DEST, ((1 << dev_priv->num_gb_pipes) - 1));
+		RADEON_WRITE(R300_SU_REG_DEST, ((1 << dev_priv->num_gb_pipes) - 1));
 	}
 	RADEON_WRITE(R300_GB_TILE_CONFIG, gb_tile_config);
 	radeon_do_wait_for_idle(dev_priv);
@@ -455,88 +455,71 @@ static void radeon_init_pipes(drm_radeon_private_t *dev_priv)
 /* Load the microcode for the CP */
 static void radeon_cp_load_microcode(drm_radeon_private_t * dev_priv)
 {
+	const u32 (*cp)[2];
 	int i;
-	DRM_DEBUG("\n");
 
-	radeon_do_wait_for_idle(dev_priv);
+	DRM_DEBUG("\n");
 
-	RADEON_WRITE(RADEON_CP_ME_RAM_ADDR, 0);
-	if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R100) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV100) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV200) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS100) ||
-	    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS200)) {
+	switch (dev_priv->flags & RADEON_FAMILY_MASK) {
+	case CHIP_R100:
+	case CHIP_RV100:
+	case CHIP_RV200:
+	case CHIP_RS100:
+	case CHIP_RS200:
 		DRM_INFO("Loading R100 Microcode\n");
-		for (i = 0; i < 256; i++) {
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
-				     R100_cp_microcode[i][1]);
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
-				     R100_cp_microcode[i][0]);
-		}
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R200) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV250) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV280) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS300)) {
+		cp = R100_cp_microcode;
+		break;
+	case CHIP_R200:
+	case CHIP_RV250:
+	case CHIP_RV280:
+	case CHIP_RS300:
 		DRM_INFO("Loading R200 Microcode\n");
-		for (i = 0; i < 256; i++) {
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
-				     R200_cp_microcode[i][1]);
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
-				     R200_cp_microcode[i][0]);
-		}
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV350) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV380) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
+		cp = R200_cp_microcode;
+		break;
+	case CHIP_R300:
+	case CHIP_R350:
+	case CHIP_RV350:
+	case CHIP_RV380:
+	case CHIP_RS400:
+	case CHIP_RS480:
 		DRM_INFO("Loading R300 Microcode\n");
-		for (i = 0; i < 256; i++) {
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
-				     R300_cp_microcode[i][1]);
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
-				     R300_cp_microcode[i][0]);
-		}
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R420) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R423) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV410)) {
+		cp = R300_cp_microcode;
+		break;
+	case CHIP_R420:
+	case CHIP_R423:
+	case CHIP_RV410:
 		DRM_INFO("Loading R400 Microcode\n");
-		for (i = 0; i < 256; i++) {
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
-				     R420_cp_microcode[i][1]);
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
-				     R420_cp_microcode[i][0]);
-		}
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740)) {
+		cp = R420_cp_microcode;
+		break;
+	case CHIP_RS690:
+	case CHIP_RS740:
 		DRM_INFO("Loading RS690/RS740 Microcode\n");
-		for (i = 0; i < 256; i++) {
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
-				     RS690_cp_microcode[i][1]);
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
-				     RS690_cp_microcode[i][0]);
-		}
-	} else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600) {
+		cp = RS690_cp_microcode;
+		break;
+	case CHIP_RS600:
 		DRM_INFO("Loading RS600 Microcode\n");
-		for (i = 0; i < 256; i++) {
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
-				     RS600_cp_microcode[i][1]);
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
-				     RS600_cp_microcode[i][0]);
-		}
-	} else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R520) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV530) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R580) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV560) ||
-		   ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV570)) {
+		cp = RS600_cp_microcode;
+		break;
+	case CHIP_RV515:
+	case CHIP_R520:
+	case CHIP_RV530:
+	case CHIP_R580:
+	case CHIP_RV560:
+	case CHIP_RV570:
 		DRM_INFO("Loading R500 Microcode\n");
-		for (i = 0; i < 256; i++) {
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAH,
-				     R520_cp_microcode[i][1]);
-			RADEON_WRITE(RADEON_CP_ME_RAM_DATAL,
-				     R520_cp_microcode[i][0]);
-		}
+		cp = R520_cp_microcode;
+		break;
+	default:
+		return;
+	}
+
+	radeon_do_wait_for_idle(dev_priv);
+
+	RADEON_WRITE(RADEON_CP_ME_RAM_ADDR, 0);
+
+	for (i = 0; i != 256; i++) {
+		RADEON_WRITE(RADEON_CP_ME_RAM_DATAH, cp[i][1]);
+		RADEON_WRITE(RADEON_CP_ME_RAM_DATAL, cp[i][0]);
 	}
 }
 
@@ -1708,26 +1691,14 @@ void radeon_do_release(struct drm_device * dev)
 			if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) {
 				while ((ret = r600_do_cp_idle(dev_priv)) != 0) {
 					DRM_DEBUG("radeon_do_cp_idle %d\n", ret);
-#ifdef __linux__
-					schedule();
-#elsif defined(__FreeBSD__)
 					mtx_sleep(&ret, &dev->dev_lock, 0,
 					    "rdnrel", 1);
-#else
-					tsleep(&ret, PZERO, "rdnrel", 1);
-#endif
 				}
 			} else {
 				while ((ret = radeon_do_cp_idle(dev_priv)) != 0) {
 					DRM_DEBUG("radeon_do_cp_idle %d\n", ret);
-#ifdef __linux__
-					schedule();
-#elsif defined(__FreeBSD__)
 					mtx_sleep(&ret, &dev->dev_lock, 0,
 					    "rdnrel", 1);
-#else
-					tsleep(&ret, PZERO, "rdnrel", 1);
-#endif
 				}
 			}
 			if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) {
diff --git a/sys/dev/drm/radeon_drv.h b/sys/dev/drm/radeon_drv.h
index 138a1e2..57f57a9 100644
--- a/sys/dev/drm/radeon_drv.h
+++ b/sys/dev/drm/radeon_drv.h
@@ -684,7 +684,6 @@ extern void r600_page_table_cleanup(struct drm_device *dev, struct drm_ati_pciga
 /* pipe config regs */
 #define R400_GB_PIPE_SELECT             0x402c
 #define R500_DYN_SCLK_PWMEM_PIPE        0x000d /* PLL */
-#define R500_SU_REG_DEST                0x42c8
 #define R300_GB_TILE_CONFIG             0x4018
 #       define R300_ENABLE_TILING       (1 << 0)
 #       define R300_PIPE_COUNT_RV350    (0 << 1)
diff --git a/sys/dev/drm/radeon_irq.c b/sys/dev/drm/radeon_irq.c
index 58b957d..ce301eb 100644
--- a/sys/dev/drm/radeon_irq.c
+++ b/sys/dev/drm/radeon_irq.c
@@ -229,6 +229,7 @@ irqreturn_t radeon_driver_irq_handler(DRM_IRQ_ARGS)
 				RADEON_WRITE(RADEON_AIC_CNTL,
 				    tmp | RS400_MSI_REARM);
 				break;
+			case CHIP_RS600:
 			case CHIP_RS690:
 			case CHIP_RS740:
 				tmp = RADEON_READ(RADEON_BUS_CNTL) &
@@ -281,6 +282,9 @@ static int radeon_wait_irq(struct drm_device * dev, int swi_nr)
 	DRM_WAIT_ON(ret, dev_priv->swi_queue, 3 * DRM_HZ,
 		    RADEON_READ(RADEON_LAST_SWI_REG) >= swi_nr);
 
+	if (ret == -ERESTART)
+		DRM_DEBUG("restarting syscall");
+
 	return ret;
 }
 
diff --git a/sys/dev/e1000/e1000_80003es2lan.c b/sys/dev/e1000/e1000_80003es2lan.c
index fa7272e..5c06086 100644
--- a/sys/dev/e1000/e1000_80003es2lan.c
+++ b/sys/dev/e1000/e1000_80003es2lan.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -43,9 +43,7 @@ static s32  e1000_init_phy_params_80003es2lan(struct e1000_hw *hw);
 static s32  e1000_init_nvm_params_80003es2lan(struct e1000_hw *hw);
 static s32  e1000_init_mac_params_80003es2lan(struct e1000_hw *hw);
 static s32  e1000_acquire_phy_80003es2lan(struct e1000_hw *hw);
-static s32  e1000_acquire_mac_csr_80003es2lan(struct e1000_hw *hw);
 static void e1000_release_phy_80003es2lan(struct e1000_hw *hw);
-static void e1000_release_mac_csr_80003es2lan(struct e1000_hw *hw);
 static s32  e1000_acquire_nvm_80003es2lan(struct e1000_hw *hw);
 static void e1000_release_nvm_80003es2lan(struct e1000_hw *hw);
 static s32  e1000_read_phy_reg_gg82563_80003es2lan(struct e1000_hw *hw,
@@ -276,6 +274,8 @@ static s32 e1000_init_mac_params_80003es2lan(struct e1000_hw *hw)
 	mac->ops.mta_set = e1000_mta_set_generic;
 	/* read mac address */
 	mac->ops.read_mac_addr = e1000_read_mac_addr_80003es2lan;
+	/* ID LED init */
+	mac->ops.id_led_init = e1000_id_led_init_generic;
 	/* blink LED */
 	mac->ops.blink_led = e1000_blink_led_generic;
 	/* setup LED */
@@ -802,17 +802,16 @@ static s32 e1000_get_cable_length_80003es2lan(struct e1000_hw *hw)
 
 	index = phy_data & GG82563_DSPD_CABLE_LENGTH;
 
-	if (index < GG82563_CABLE_LENGTH_TABLE_SIZE + 5) {
-		phy->min_cable_length = e1000_gg82563_cable_length_table[index];
-		phy->max_cable_length =
-		                 e1000_gg82563_cable_length_table[index+5];
-
-		phy->cable_length = (phy->min_cable_length +
-		                     phy->max_cable_length) / 2;
-	} else {
+	if (index >= GG82563_CABLE_LENGTH_TABLE_SIZE + 5) {
 		ret_val = E1000_ERR_PHY;
+		goto out;
 	}
 
+	phy->min_cable_length = e1000_gg82563_cable_length_table[index];
+	phy->max_cable_length = e1000_gg82563_cable_length_table[index+5];
+
+	phy->cable_length = (phy->min_cable_length + phy->max_cable_length) / 2;
+
 out:
 	return ret_val;
 }
@@ -892,7 +891,7 @@ static s32 e1000_reset_hw_80003es2lan(struct e1000_hw *hw)
 	E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff);
 	icr = E1000_READ_REG(hw, E1000_ICR);
 
-	e1000_check_alt_mac_addr_generic(hw);
+	ret_val = e1000_check_alt_mac_addr_generic(hw);
 
 out:
 	return ret_val;
@@ -916,7 +915,7 @@ static s32 e1000_init_hw_80003es2lan(struct e1000_hw *hw)
 	e1000_initialize_hw_bits_80003es2lan(hw);
 
 	/* Initialize identification LED */
-	ret_val = e1000_id_led_init_generic(hw);
+	ret_val = mac->ops.id_led_init(hw);
 	if (ret_val) {
 		DEBUGOUT("Error initializing identification LED\n");
 		/* This is not fatal and we should not stop init due to this */
@@ -1104,9 +1103,9 @@ static s32 e1000_copper_link_setup_gg82563_80003es2lan(struct e1000_hw *hw)
 
 	/* Bypass Rx and Tx FIFO's */
 	ret_val = e1000_write_kmrn_reg_80003es2lan(hw,
-	                        E1000_KMRNCTRLSTA_OFFSET_FIFO_CTRL,
-	                        E1000_KMRNCTRLSTA_FIFO_CTRL_RX_BYPASS |
-	                                E1000_KMRNCTRLSTA_FIFO_CTRL_TX_BYPASS);
+					E1000_KMRNCTRLSTA_OFFSET_FIFO_CTRL,
+					E1000_KMRNCTRLSTA_FIFO_CTRL_RX_BYPASS |
+					E1000_KMRNCTRLSTA_FIFO_CTRL_TX_BYPASS);
 	if (ret_val)
 		goto out;
 
@@ -1147,22 +1146,19 @@ static s32 e1000_copper_link_setup_gg82563_80003es2lan(struct e1000_hw *hw)
 	if (!(hw->mac.ops.check_mng_mode(hw))) {
 		/* Enable Electrical Idle on the PHY */
 		data |= GG82563_PMCR_ENABLE_ELECTRICAL_IDLE;
-		ret_val = hw->phy.ops.write_reg(hw,
-		                                GG82563_PHY_PWR_MGMT_CTRL,
+		ret_val = hw->phy.ops.write_reg(hw, GG82563_PHY_PWR_MGMT_CTRL,
 		                                data);
 		if (ret_val)
 			goto out;
-		ret_val = hw->phy.ops.read_reg(hw,
-			                       GG82563_PHY_KMRN_MODE_CTRL,
-			                       &data);
-			if (ret_val)
-				goto out;
+
+		ret_val = hw->phy.ops.read_reg(hw, GG82563_PHY_KMRN_MODE_CTRL,
+		                               &data);
+		if (ret_val)
+			goto out;
 
 		data &= ~GG82563_KMCR_PASS_FALSE_CARRIER;
-		ret_val = hw->phy.ops.write_reg(hw,
-		                                GG82563_PHY_KMRN_MODE_CTRL,
+		ret_val = hw->phy.ops.write_reg(hw, GG82563_PHY_KMRN_MODE_CTRL,
 		                                data);
-
 		if (ret_val)
 			goto out;
 	}
@@ -1261,7 +1257,6 @@ static s32 e1000_cfg_on_link_up_80003es2lan(struct e1000_hw *hw)
 	DEBUGFUNC("e1000_configure_on_link_up");
 
 	if (hw->phy.media_type == e1000_media_type_copper) {
-
 		ret_val = e1000_get_speed_and_duplex_copper_generic(hw,
 		                                                    &speed,
 		                                                    &duplex);
@@ -1393,7 +1388,8 @@ out:
  *  using the kumeran interface.  The information retrieved is stored in data.
  *  Release the semaphore before exiting.
  **/
-s32 e1000_read_kmrn_reg_80003es2lan(struct e1000_hw *hw, u32 offset, u16 *data)
+static s32 e1000_read_kmrn_reg_80003es2lan(struct e1000_hw *hw, u32 offset,
+                                           u16 *data)
 {
 	u32 kmrnctrlsta;
 	s32 ret_val = E1000_SUCCESS;
@@ -1429,7 +1425,8 @@ out:
  *  at the offset using the kumeran interface.  Release semaphore
  *  before exiting.
  **/
-s32 e1000_write_kmrn_reg_80003es2lan(struct e1000_hw *hw, u32 offset, u16 data)
+static s32 e1000_write_kmrn_reg_80003es2lan(struct e1000_hw *hw, u32 offset,
+                                            u16 data)
 {
 	u32 kmrnctrlsta;
 	s32 ret_val = E1000_SUCCESS;
@@ -1461,9 +1458,19 @@ static s32 e1000_read_mac_addr_80003es2lan(struct e1000_hw *hw)
 	s32 ret_val = E1000_SUCCESS;
 
 	DEBUGFUNC("e1000_read_mac_addr_80003es2lan");
-	if (e1000_check_alt_mac_addr_generic(hw))
-		ret_val = e1000_read_mac_addr_generic(hw);
 
+	/*
+	 * If there's an alternate MAC address place it in RAR0
+	 * so that it will override the Si installed default perm
+	 * address.
+	 */
+	ret_val = e1000_check_alt_mac_addr_generic(hw);
+	if (ret_val)
+		goto out;
+
+	ret_val = e1000_read_mac_addr_generic(hw);
+
+out:
 	return ret_val;
 }
 
diff --git a/sys/dev/e1000/e1000_82540.c b/sys/dev/e1000/e1000_82540.c
index 6967261..2cd1057 100644
--- a/sys/dev/e1000/e1000_82540.c
+++ b/sys/dev/e1000/e1000_82540.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -229,6 +229,8 @@ static s32 e1000_init_mac_params_82540(struct e1000_hw *hw)
 	mac->ops.clear_vfta = e1000_clear_vfta_generic;
 	/* setting MTA */
 	mac->ops.mta_set = e1000_mta_set_generic;
+	/* ID LED init */
+	mac->ops.id_led_init = e1000_id_led_init_generic;
 	/* setup LED */
 	mac->ops.setup_led = e1000_setup_led_generic;
 	/* cleanup LED */
@@ -332,7 +334,7 @@ static s32 e1000_init_hw_82540(struct e1000_hw *hw)
 	DEBUGFUNC("e1000_init_hw_82540");
 
 	/* Initialize identification LED */
-	ret_val = e1000_id_led_init_generic(hw);
+	ret_val = mac->ops.id_led_init(hw);
 	if (ret_val) {
 		DEBUGOUT("Error initializing identification LED\n");
 		/* This is not fatal and we should not stop init due to this */
diff --git a/sys/dev/e1000/e1000_82541.c b/sys/dev/e1000/e1000_82541.c
index 03d1103..a7f896e 100644
--- a/sys/dev/e1000/e1000_82541.c
+++ b/sys/dev/e1000/e1000_82541.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -261,6 +261,8 @@ static s32 e1000_init_mac_params_82541(struct e1000_hw *hw)
 	mac->ops.clear_vfta = e1000_clear_vfta_generic;
 	/* setting MTA */
 	mac->ops.mta_set = e1000_mta_set_generic;
+	/* ID LED init */
+	mac->ops.id_led_init = e1000_id_led_init_generic;
 	/* setup LED */
 	mac->ops.setup_led = e1000_setup_led_82541;
 	/* cleanup LED */
@@ -381,7 +383,7 @@ static s32 e1000_init_hw_82541(struct e1000_hw *hw)
 	DEBUGFUNC("e1000_init_hw_82541");
 
 	/* Initialize identification LED */
-	ret_val = e1000_id_led_init_generic(hw);
+	ret_val = mac->ops.id_led_init(hw);
 	if (ret_val) {
 		DEBUGOUT("Error initializing identification LED\n");
 		/* This is not fatal and we should not stop init due to this */
diff --git a/sys/dev/e1000/e1000_82571.c b/sys/dev/e1000/e1000_82571.c
index db7852a..5d16136 100644
--- a/sys/dev/e1000/e1000_82571.c
+++ b/sys/dev/e1000/e1000_82571.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -46,6 +46,7 @@
  * 82573E Gigabit Ethernet Controller (Copper)
  * 82573L Gigabit Ethernet Controller
  * 82574L Gigabit Network Connection
+ * 82574L Gigabit Network Connection
  */
 
 #include "e1000_api.h"
@@ -67,11 +68,9 @@ static s32  e1000_init_hw_82571(struct e1000_hw *hw);
 static void e1000_clear_vfta_82571(struct e1000_hw *hw);
 static bool e1000_check_mng_mode_82574(struct e1000_hw *hw);
 static s32 e1000_led_on_82574(struct e1000_hw *hw);
-static void e1000_update_mc_addr_list_82571(struct e1000_hw *hw,
-                                           u8 *mc_addr_list, u32 mc_addr_count,
-                                           u32 rar_used_count, u32 rar_count);
 static s32  e1000_setup_link_82571(struct e1000_hw *hw);
 static s32  e1000_setup_copper_link_82571(struct e1000_hw *hw);
+static s32  e1000_check_for_serdes_link_82571(struct e1000_hw *hw);
 static s32  e1000_setup_fiber_serdes_link_82571(struct e1000_hw *hw);
 static s32  e1000_valid_led_default_82571(struct e1000_hw *hw, u16 *data);
 static void e1000_clear_hw_cntrs_82571(struct e1000_hw *hw);
@@ -330,7 +329,7 @@ static s32 e1000_init_mac_params_82571(struct e1000_hw *hw)
 		mac->ops.check_for_link = e1000_check_for_fiber_link_generic;
 		break;
 	case e1000_media_type_internal_serdes:
-		mac->ops.check_for_link = e1000_check_for_serdes_link_generic;
+		mac->ops.check_for_link = e1000_check_for_serdes_link_82571;
 		break;
 	default:
 		ret_val = -E1000_ERR_CONFIG;
@@ -347,7 +346,7 @@ static s32 e1000_init_mac_params_82571(struct e1000_hw *hw)
 		break;
 	}
 	/* multicast address update */
-	mac->ops.update_mc_addr_list = e1000_update_mc_addr_list_82571;
+	mac->ops.update_mc_addr_list = e1000_update_mc_addr_list_generic;
 	/* writing VFTA */
 	mac->ops.write_vfta = e1000_write_vfta_generic;
 	/* clearing VFTA */
@@ -356,6 +355,8 @@ static s32 e1000_init_mac_params_82571(struct e1000_hw *hw)
 	mac->ops.mta_set = e1000_mta_set_generic;
 	/* read mac address */
 	mac->ops.read_mac_addr = e1000_read_mac_addr_82571;
+	/* ID LED init */
+	mac->ops.id_led_init = e1000_id_led_init_generic;
 	/* blink LED */
 	mac->ops.blink_led = e1000_blink_led_generic;
 	/* setup LED */
@@ -528,8 +529,14 @@ static s32 e1000_acquire_nvm_82571(struct e1000_hw *hw)
 	if (ret_val)
 		goto out;
 
-	if (hw->mac.type != e1000_82573 && hw->mac.type != e1000_82574)
+	switch (hw->mac.type) {
+	case e1000_82574:
+	case e1000_82573:
+		break;
+	default:
 		ret_val = e1000_acquire_nvm_generic(hw);
+		break;
+	}
 
 	if (ret_val)
 		e1000_put_hw_semaphore_82571(hw);
@@ -876,7 +883,9 @@ static s32 e1000_reset_hw_82571(struct e1000_hw *hw)
 	 * Must acquire the MDIO ownership before MAC reset.
 	 * Ownership defaults to firmware after a reset.
 	 */
-	if (hw->mac.type == e1000_82573 || hw->mac.type == e1000_82574) {
+	switch (hw->mac.type) {
+	case e1000_82574:
+	case e1000_82573:
 		extcnf_ctrl = E1000_READ_REG(hw, E1000_EXTCNF_CTRL);
 		extcnf_ctrl |= E1000_EXTCNF_CTRL_MDIO_SW_OWNERSHIP;
 
@@ -892,6 +901,9 @@ static s32 e1000_reset_hw_82571(struct e1000_hw *hw)
 			msec_delay(2);
 			i++;
 		} while (i < MDIO_OWNERSHIP_TIMEOUT);
+		break;
+	default:
+		break;
 	}
 
 	ctrl = E1000_READ_REG(hw, E1000_CTRL);
@@ -917,15 +929,30 @@ static s32 e1000_reset_hw_82571(struct e1000_hw *hw)
 	 * Need to wait for Phy configuration completion before accessing
 	 * NVM and Phy.
 	 */
-	if (hw->mac.type == e1000_82573 || hw->mac.type == e1000_82574)
+
+	switch (hw->mac.type) {
+	case e1000_82574:
+	case e1000_82573:
 		msec_delay(25);
+		break;
+	default:
+		break;
+	}
 
 	/* Clear any pending interrupt events. */
 	E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff);
 	icr = E1000_READ_REG(hw, E1000_ICR);
 
-	if (!(e1000_check_alt_mac_addr_generic(hw)))
-		e1000_set_laa_state_82571(hw, TRUE);
+	/* Install any alternate MAC address into RAR0 */
+	ret_val = e1000_check_alt_mac_addr_generic(hw);
+	if (ret_val)
+		goto out;
+
+	e1000_set_laa_state_82571(hw, TRUE);
+
+	/* Reinitialize the 82571 serdes link state machine */
+	if (hw->phy.media_type == e1000_media_type_internal_serdes)
+		hw->mac.serdes_link_state = e1000_serdes_link_down;
 
 out:
 	return ret_val;
@@ -949,7 +976,7 @@ static s32 e1000_init_hw_82571(struct e1000_hw *hw)
 	e1000_initialize_hw_bits_82571(hw);
 
 	/* Initialize identification LED */
-	ret_val = e1000_id_led_init_generic(hw);
+	ret_val = mac->ops.id_led_init(hw);
 	if (ret_val) {
 		DEBUGOUT("Error initializing identification LED\n");
 		/* This is not fatal and we should not stop init due to this */
@@ -985,17 +1012,21 @@ static s32 e1000_init_hw_82571(struct e1000_hw *hw)
 	E1000_WRITE_REG(hw, E1000_TXDCTL(0), reg_data);
 
 	/* ...for both queues. */
-	if (mac->type != e1000_82573 && mac->type != e1000_82574) {
-		reg_data = E1000_READ_REG(hw, E1000_TXDCTL(1));
-		reg_data = (reg_data & ~E1000_TXDCTL_WTHRESH) |
-		           E1000_TXDCTL_FULL_TX_DESC_WB |
-		           E1000_TXDCTL_COUNT_DESC;
-		E1000_WRITE_REG(hw, E1000_TXDCTL(1), reg_data);
-	} else {
+	switch (mac->type) {
+	case e1000_82574:
+	case e1000_82573:
 		e1000_enable_tx_pkt_filtering_generic(hw);
 		reg_data = E1000_READ_REG(hw, E1000_GCR);
 		reg_data |= E1000_GCR_L1_ACT_WITHOUT_L0S_RX;
 		E1000_WRITE_REG(hw, E1000_GCR, reg_data);
+		break;
+	default:
+		reg_data = E1000_READ_REG(hw, E1000_TXDCTL(1));
+		reg_data = (reg_data & ~E1000_TXDCTL_WTHRESH) |
+					E1000_TXDCTL_FULL_TX_DESC_WB |
+					E1000_TXDCTL_COUNT_DESC;
+		E1000_WRITE_REG(hw, E1000_TXDCTL(1), reg_data);
+		break;
 	}
 
 	/*
@@ -1062,25 +1093,70 @@ static void e1000_initialize_hw_bits_82571(struct e1000_hw *hw)
 	}
 
 	/* Device Control */
-	if (hw->mac.type == e1000_82573 || hw->mac.type == e1000_82574) {
+
+	switch (hw->mac.type) {
+	case e1000_82574:
+	case e1000_82573:
 		reg = E1000_READ_REG(hw, E1000_CTRL);
 		reg &= ~(1 << 29);
 		E1000_WRITE_REG(hw, E1000_CTRL, reg);
+		break;
+	default:
+		break;
 	}
 
 	/* Extended Device Control */
-	if (hw->mac.type == e1000_82573 || hw->mac.type == e1000_82574) {
+	switch (hw->mac.type) {
+	case e1000_82574:
+	case e1000_82573:
 		reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
 		reg &= ~(1 << 23);
 		reg |= (1 << 22);
 		E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
+		break;
+	default:
+		break;
 	}
 
-	/* PCI-Ex Control Register */
-	if (hw->mac.type == e1000_82574) {
+
+	if (hw->mac.type == e1000_82571) {
+		reg = E1000_READ_REG(hw, E1000_PBA_ECC);
+		reg |= E1000_PBA_ECC_CORR_EN;
+		E1000_WRITE_REG(hw, E1000_PBA_ECC, reg);
+	}
+
+	/*
+	 * Workaround for hardware errata.
+	 * Ensure that DMA Dynamic Clock gating is disabled on 82571 and 82572
+	 */
+
+	if ((hw->mac.type == e1000_82571) ||
+	   (hw->mac.type == e1000_82572)) {
+		reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
+		reg &= ~E1000_CTRL_EXT_DMA_DYN_CLK_EN;
+		E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
+	}
+
+	/* PCI-Ex Control Registers */
+
+	switch (hw->mac.type) {
+	case e1000_82574:
 		reg = E1000_READ_REG(hw, E1000_GCR);
 		reg |= (1 << 22);
 		E1000_WRITE_REG(hw, E1000_GCR, reg);
+		/*
+		 * Workaround for hardware errata.
+		 * apply workaround for hardware errata documented in errata
+		 * docs Fixes issue where some error prone or unreliable PCIe
+		 * completions are occurring, particularly with ASPM enabled.
+		 * Without fix, issue can cause tx timeouts.
+		 */
+		reg = E1000_READ_REG(hw, E1000_GCR2);
+		reg |= 1;
+		E1000_WRITE_REG(hw, E1000_GCR2, reg);
+		break;
+	default:
+		break;
 	}
 
 	return;
@@ -1102,31 +1178,38 @@ static void e1000_clear_vfta_82571(struct e1000_hw *hw)
 
 	DEBUGFUNC("e1000_clear_vfta_82571");
 
-	if (hw->mac.type == e1000_82573 || hw->mac.type == e1000_82574) {
+	switch (hw->mac.type) {
+	case e1000_82574:
+	case e1000_82573:
 		if (hw->mng_cookie.vlan_id != 0) {
 			/*
-			 * The VFTA is a 4096b bit-field, each identifying
-			 * a single VLAN ID.  The following operations
-			 * determine which 32b entry (i.e. offset) into the
-			 * array we want to set the VLAN ID (i.e. bit) of
-			 * the manageability unit.
-			 */
+			*The VFTA is a 4096b bit-field, each identifying
+			*a single VLAN ID.  The following operations
+			*determine which 32b entry (i.e. offset) into the
+			*array we want to set the VLAN ID (i.e. bit) of
+			*the manageability unit.
+			*/
 			vfta_offset = (hw->mng_cookie.vlan_id >>
-			               E1000_VFTA_ENTRY_SHIFT) &
-			              E1000_VFTA_ENTRY_MASK;
+				E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK;
 			vfta_bit_in_reg = 1 << (hw->mng_cookie.vlan_id &
-			                       E1000_VFTA_ENTRY_BIT_SHIFT_MASK);
+				E1000_VFTA_ENTRY_BIT_SHIFT_MASK);
 		}
-	}
-	for (offset = 0; offset < E1000_VLAN_FILTER_TBL_SIZE; offset++) {
-		/*
-		 * If the offset we want to clear is the same offset of the
-		 * manageability VLAN ID, then clear all bits except that of
-		 * the manageability unit.
-		 */
-		vfta_value = (offset == vfta_offset) ? vfta_bit_in_reg : 0;
-		E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, offset, vfta_value);
-		E1000_WRITE_FLUSH(hw);
+
+		for (offset = 0; offset < E1000_VLAN_FILTER_TBL_SIZE; offset++) {
+			/*
+			*If the offset we want to clear is the same offset of
+			*the manageability VLAN ID, then clear all bits except
+			*that of the manageability unit
+			*/
+			vfta_value = (offset == vfta_offset) ?
+							vfta_bit_in_reg : 0;
+			E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, offset,
+				vfta_value);
+			E1000_WRITE_FLUSH(hw);
+		}
+		break;
+	default:
+		break;
 	}
 }
 
@@ -1176,31 +1259,6 @@ static s32 e1000_led_on_82574(struct e1000_hw *hw)
 	return E1000_SUCCESS;
 }
 
-/**
- *  e1000_update_mc_addr_list_82571 - Update Multicast addresses
- *  @hw: pointer to the HW structure
- *  @mc_addr_list: array of multicast addresses to program
- *  @mc_addr_count: number of multicast addresses to program
- *  @rar_used_count: the first RAR register free to program
- *  @rar_count: total number of supported Receive Address Registers
- *
- *  Updates the Receive Address Registers and Multicast Table Array.
- *  The caller must have a packed mc_addr_list of multicast addresses.
- *  The parameter rar_count will usually be hw->mac.rar_entry_count
- *  unless there are workarounds that change this.
- **/
-static void e1000_update_mc_addr_list_82571(struct e1000_hw *hw,
-                                           u8 *mc_addr_list, u32 mc_addr_count,
-                                           u32 rar_used_count, u32 rar_count)
-{
-	DEBUGFUNC("e1000_update_mc_addr_list_82571");
-
-	if (e1000_get_laa_state_82571(hw))
-		rar_count--;
-
-	e1000_update_mc_addr_list_generic(hw, mc_addr_list, mc_addr_count,
-	                                  rar_used_count, rar_count);
-}
 
 /**
  *  e1000_setup_link_82571 - Setup flow control and link settings
@@ -1221,10 +1279,15 @@ static s32 e1000_setup_link_82571(struct e1000_hw *hw)
 	 * the default flow control setting, so we explicitly
 	 * set it to full.
 	 */
-	if ((hw->mac.type == e1000_82573 || hw->mac.type == e1000_82574) &&
-	    hw->fc.requested_mode == e1000_fc_default)
-		hw->fc.requested_mode = e1000_fc_full;
-
+	switch (hw->mac.type) {
+	case e1000_82574:
+	case e1000_82573:
+		if (hw->fc.requested_mode == e1000_fc_default)
+			hw->fc.requested_mode = e1000_fc_full;
+		break;
+	default:
+		break;
+	}
 	return e1000_setup_link_generic(hw);
 }
 
@@ -1306,6 +1369,133 @@ static s32 e1000_setup_fiber_serdes_link_82571(struct e1000_hw *hw)
 }
 
 /**
+ *  e1000_check_for_serdes_link_82571 - Check for link (Serdes)
+ *  @hw: pointer to the HW structure
+ *
+ *  Checks for link up on the hardware.  If link is not up and we have
+ *  a signal, then we need to force link up.
+ **/
+s32 e1000_check_for_serdes_link_82571(struct e1000_hw *hw)
+{
+	struct e1000_mac_info *mac = &hw->mac;
+	u32 rxcw;
+	u32 ctrl;
+	u32 status;
+	s32 ret_val = E1000_SUCCESS;
+
+	DEBUGFUNC("e1000_check_for_serdes_link_82571");
+
+	ctrl = E1000_READ_REG(hw, E1000_CTRL);
+	status = E1000_READ_REG(hw, E1000_STATUS);
+	rxcw = E1000_READ_REG(hw, E1000_RXCW);
+
+	if ((rxcw & E1000_RXCW_SYNCH) && !(rxcw & E1000_RXCW_IV)) {
+
+		/* Receiver is synchronized with no invalid bits.  */
+		switch (mac->serdes_link_state) {
+		case e1000_serdes_link_autoneg_complete:
+			if (!(status & E1000_STATUS_LU)) {
+				/*
+				 * We have lost link, retry autoneg before
+				 * reporting link failure
+				 */
+				mac->serdes_link_state =
+				    e1000_serdes_link_autoneg_progress;
+				DEBUGOUT("AN_UP     -> AN_PROG\n");
+			}
+		break;
+
+		case e1000_serdes_link_forced_up:
+			/*
+			 * If we are receiving /C/ ordered sets, re-enable
+			 * auto-negotiation in the TXCW register and disable
+			 * forced link in the Device Control register in an
+			 * attempt to auto-negotiate with our link partner.
+			 */
+			if (rxcw & E1000_RXCW_C) {
+				/* Enable autoneg, and unforce link up */
+				E1000_WRITE_REG(hw, E1000_TXCW, mac->txcw);
+				E1000_WRITE_REG(hw, E1000_CTRL,
+				    (ctrl & ~E1000_CTRL_SLU));
+				mac->serdes_link_state =
+				    e1000_serdes_link_autoneg_progress;
+				DEBUGOUT("FORCED_UP -> AN_PROG\n");
+			}
+			break;
+
+		case e1000_serdes_link_autoneg_progress:
+			/*
+			 * If the LU bit is set in the STATUS register,
+			 * autoneg has completed sucessfully. If not,
+			 * try foring the link because the far end may be
+			 * available but not capable of autonegotiation.
+			 */
+			if (status & E1000_STATUS_LU)  {
+				mac->serdes_link_state =
+				    e1000_serdes_link_autoneg_complete;
+				DEBUGOUT("AN_PROG   -> AN_UP\n");
+			} else {
+				/*
+				 * Disable autoneg, force link up and
+				 * full duplex, and change state to forced
+				 */
+				E1000_WRITE_REG(hw, E1000_TXCW,
+				    (mac->txcw & ~E1000_TXCW_ANE));
+				ctrl |= (E1000_CTRL_SLU | E1000_CTRL_FD);
+				E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
+
+				/* Configure Flow Control after link up. */
+				ret_val =
+				    e1000_config_fc_after_link_up_generic(hw);
+				if (ret_val) {
+					DEBUGOUT("Error config flow control\n");
+					break;
+				}
+				mac->serdes_link_state =
+				    e1000_serdes_link_forced_up;
+				DEBUGOUT("AN_PROG   -> FORCED_UP\n");
+			}
+			mac->serdes_has_link = TRUE;
+			break;
+
+		case e1000_serdes_link_down:
+		default:
+			/* The link was down but the receiver has now gained
+			 * valid sync, so lets see if we can bring the link
+			 * up. */
+			E1000_WRITE_REG(hw, E1000_TXCW, mac->txcw);
+			E1000_WRITE_REG(hw, E1000_CTRL,
+			    (ctrl & ~E1000_CTRL_SLU));
+			mac->serdes_link_state =
+			    e1000_serdes_link_autoneg_progress;
+			DEBUGOUT("DOWN      -> AN_PROG\n");
+			break;
+		}
+	} else {
+		if (!(rxcw & E1000_RXCW_SYNCH)) {
+			mac->serdes_has_link = FALSE;
+			mac->serdes_link_state = e1000_serdes_link_down;
+			DEBUGOUT("ANYSTATE  -> DOWN\n");
+		} else {
+			/*
+			 * We have sync, and can tolerate one
+			 * invalid (IV) codeword before declaring
+			 * link down, so reread to look again
+			 */
+			usec_delay(10);
+			rxcw = E1000_READ_REG(hw, E1000_RXCW);
+			if (rxcw & E1000_RXCW_IV) {
+				mac->serdes_link_state = e1000_serdes_link_down;
+				mac->serdes_has_link = FALSE;
+				DEBUGOUT("ANYSTATE  -> DOWN\n");
+			}
+		}
+	}
+
+	return ret_val;
+}
+
+/**
  *  e1000_valid_led_default_82571 - Verify a valid default LED config
  *  @hw: pointer to the HW structure
  *  @data: pointer to the NVM (EEPROM)
@@ -1325,11 +1515,19 @@ static s32 e1000_valid_led_default_82571(struct e1000_hw *hw, u16 *data)
 		goto out;
 	}
 
-	if ((hw->mac.type == e1000_82573 || hw->mac.type == e1000_82574) &&
-	    *data == ID_LED_RESERVED_F746)
-		*data = ID_LED_DEFAULT_82573;
-	else if (*data == ID_LED_RESERVED_0000 || *data == ID_LED_RESERVED_FFFF)
-		*data = ID_LED_DEFAULT;
+	switch (hw->mac.type) {
+	case e1000_82574:
+	case e1000_82573:
+		if(*data == ID_LED_RESERVED_F746)
+			*data = ID_LED_DEFAULT_82573;
+		break;
+	default:
+		if (*data == ID_LED_RESERVED_0000 ||
+			*data == ID_LED_RESERVED_FFFF)
+			*data = ID_LED_DEFAULT;
+		break;
+	}
+
 out:
 	return ret_val;
 }
@@ -1435,6 +1633,7 @@ out:
 	return ret_val;
 }
 
+
 /**
  *  e1000_read_mac_addr_82571 - Read device MAC address
  *  @hw: pointer to the HW structure
@@ -1444,9 +1643,19 @@ static s32 e1000_read_mac_addr_82571(struct e1000_hw *hw)
 	s32 ret_val = E1000_SUCCESS;
 
 	DEBUGFUNC("e1000_read_mac_addr_82571");
-	if (e1000_check_alt_mac_addr_generic(hw))
-		ret_val = e1000_read_mac_addr_generic(hw);
 
+	/*
+	 * If there's an alternate MAC address place it in RAR0
+	 * so that it will override the Si installed default perm
+	 * address.
+	 */
+	ret_val = e1000_check_alt_mac_addr_generic(hw);
+	if (ret_val)
+		goto out;
+
+	ret_val = e1000_read_mac_addr_generic(hw);
+
+out:
 	return ret_val;
 }
 
diff --git a/sys/dev/e1000/e1000_82575.c b/sys/dev/e1000/e1000_82575.c
index 3d16447..d7ed6c8 100644
--- a/sys/dev/e1000/e1000_82575.c
+++ b/sys/dev/e1000/e1000_82575.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -36,6 +36,7 @@
  * 82575EB Gigabit Network Connection
  * 82575EB Gigabit Backplane Connection
  * 82575GB Gigabit Network Connection
+ * 82575GB Gigabit Network Connection
  * 82576 Gigabit Network Connection
  */
 
@@ -75,11 +76,6 @@ static bool e1000_sgmii_active_82575(struct e1000_hw *hw);
 static s32  e1000_reset_init_script_82575(struct e1000_hw *hw);
 static s32  e1000_read_mac_addr_82575(struct e1000_hw *hw);
 static void e1000_power_down_phy_copper_82575(struct e1000_hw *hw);
-
-static void e1000_init_rx_addrs_82575(struct e1000_hw *hw, u16 rar_count);
-static void e1000_update_mc_addr_list_82575(struct e1000_hw *hw,
-                                           u8 *mc_addr_list, u32 mc_addr_count,
-                                           u32 rar_used_count, u32 rar_count);
 void e1000_shutdown_fiber_serdes_link_82575(struct e1000_hw *hw);
 
 /**
@@ -281,13 +277,15 @@ static s32 e1000_init_mac_params_82575(struct e1000_hw *hw)
 	/* read mac address */
 	mac->ops.read_mac_addr = e1000_read_mac_addr_82575;
 	/* multicast address update */
-	mac->ops.update_mc_addr_list = e1000_update_mc_addr_list_82575;
+	mac->ops.update_mc_addr_list = e1000_update_mc_addr_list_generic;
 	/* writing VFTA */
 	mac->ops.write_vfta = e1000_write_vfta_generic;
 	/* clearing VFTA */
 	mac->ops.clear_vfta = e1000_clear_vfta_generic;
 	/* setting MTA */
 	mac->ops.mta_set = e1000_mta_set_generic;
+	/* ID LED init */
+	mac->ops.id_led_init = e1000_id_led_init_generic;
 	/* blink LED */
 	mac->ops.blink_led = e1000_blink_led_generic;
 	/* setup LED */
@@ -854,11 +852,18 @@ static s32 e1000_check_for_link_82575(struct e1000_hw *hw)
 
 	/* SGMII link check is done through the PCS register. */
 	if ((hw->phy.media_type != e1000_media_type_copper) ||
-	    (e1000_sgmii_active_82575(hw)))
+	    (e1000_sgmii_active_82575(hw))) {
 		ret_val = e1000_get_pcs_speed_and_duplex_82575(hw, &speed,
 		                                               &duplex);
-	else
+		/*
+		 * Use this flag to determine if link needs to be checked or
+		 * not.  If we have link clear the flag so that we do not
+		 * continue to check for link.
+		 */
+		hw->mac.get_link_status = !hw->mac.serdes_has_link;
+	} else {
 		ret_val = e1000_check_for_copper_link_generic(hw);
+	}
 
 	return ret_val;
 }
@@ -921,101 +926,6 @@ static s32 e1000_get_pcs_speed_and_duplex_82575(struct e1000_hw *hw,
 }
 
 /**
- *  e1000_init_rx_addrs_82575 - Initialize receive address's
- *  @hw: pointer to the HW structure
- *  @rar_count: receive address registers
- *
- *  Setups the receive address registers by setting the base receive address
- *  register to the devices MAC address and clearing all the other receive
- *  address registers to 0.
- **/
-static void e1000_init_rx_addrs_82575(struct e1000_hw *hw, u16 rar_count)
-{
-	u32 i;
-	u8 addr[6] = {0,0,0,0,0,0};
-	/*
-	 * This function is essentially the same as that of
-	 * e1000_init_rx_addrs_generic. However it also takes care
-	 * of the special case where the register offset of the
-	 * second set of RARs begins elsewhere. This is implicitly taken care by
-	 * function e1000_rar_set_generic.
-	 */
-
-	DEBUGFUNC("e1000_init_rx_addrs_82575");
-
-	/* Setup the receive address */
-	DEBUGOUT("Programming MAC Address into RAR[0]\n");
-	hw->mac.ops.rar_set(hw, hw->mac.addr, 0);
-
-	/* Zero out the other (rar_entry_count - 1) receive addresses */
-	DEBUGOUT1("Clearing RAR[1-%u]\n", rar_count-1);
-	for (i = 1; i < rar_count; i++) {
-	    hw->mac.ops.rar_set(hw, addr, i);
-	}
-}
-
-/**
- *  e1000_update_mc_addr_list_82575 - Update Multicast addresses
- *  @hw: pointer to the HW structure
- *  @mc_addr_list: array of multicast addresses to program
- *  @mc_addr_count: number of multicast addresses to program
- *  @rar_used_count: the first RAR register free to program
- *  @rar_count: total number of supported Receive Address Registers
- *
- *  Updates the Receive Address Registers and Multicast Table Array.
- *  The caller must have a packed mc_addr_list of multicast addresses.
- *  The parameter rar_count will usually be hw->mac.rar_entry_count
- *  unless there are workarounds that change this.
- **/
-static void e1000_update_mc_addr_list_82575(struct e1000_hw *hw,
-                                     u8 *mc_addr_list, u32 mc_addr_count,
-                                     u32 rar_used_count, u32 rar_count)
-{
-	u32 hash_value;
-	u32 i;
-	u8 addr[6] = {0,0,0,0,0,0};
-	/*
-	 * This function is essentially the same as that of 
-	 * e1000_update_mc_addr_list_generic. However it also takes care 
-	 * of the special case where the register offset of the 
-	 * second set of RARs begins elsewhere. This is implicitly taken care by 
-	 * function e1000_rar_set_generic.
-	 */
-
-	DEBUGFUNC("e1000_update_mc_addr_list_82575");
-
-	/*
-	 * Load the first set of multicast addresses into the exact
-	 * filters (RAR).  If there are not enough to fill the RAR
-	 * array, clear the filters.
-	 */
-	for (i = rar_used_count; i < rar_count; i++) {
-		if (mc_addr_count) {
-			e1000_rar_set_generic(hw, mc_addr_list, i);
-			mc_addr_count--;
-			mc_addr_list += ETH_ADDR_LEN;
-		} else {
-			e1000_rar_set_generic(hw, addr, i);
-		}
-	}
-
-	/* Clear the old settings from the MTA */
-	DEBUGOUT("Clearing MTA\n");
-	for (i = 0; i < hw->mac.mta_reg_count; i++) {
-		E1000_WRITE_REG_ARRAY(hw, E1000_MTA, i, 0);
-		E1000_WRITE_FLUSH(hw);
-	}
-
-	/* Load any remaining multicast addresses into the hash table. */
-	for (; mc_addr_count > 0; mc_addr_count--) {
-		hash_value = e1000_hash_mc_addr(hw, mc_addr_list);
-		DEBUGOUT1("Hash value = 0x%03X\n", hash_value);
-		hw->mac.ops.mta_set(hw, hash_value);
-		mc_addr_list += ETH_ADDR_LEN;
-	}
-}
-
-/**
  *  e1000_shutdown_fiber_serdes_link_82575 - Remove link during power down
  *  @hw: pointer to the HW structure
  *
@@ -1060,6 +970,253 @@ void e1000_shutdown_fiber_serdes_link_82575(struct e1000_hw *hw)
 }
 
 /**
+ *  e1000_vmdq_loopback_enable_pf- Enables VM to VM queue loopback replication
+ *  @hw: pointer to the HW structure
+ **/
+void e1000_vmdq_loopback_enable_pf(struct e1000_hw *hw)
+{
+	u32 reg;
+
+	reg = E1000_READ_REG(hw, E1000_DTXSWC);
+	reg |= E1000_DTXSWC_VMDQ_LOOPBACK_EN;
+	E1000_WRITE_REG(hw, E1000_DTXSWC, reg);
+}
+
+/**
+ *  e1000_vmdq_loopback_disable_pf - Disable VM to VM queue loopbk replication
+ *  @hw: pointer to the HW structure
+ **/
+void e1000_vmdq_loopback_disable_pf(struct e1000_hw *hw)
+{
+	u32 reg;
+
+	reg = E1000_READ_REG(hw, E1000_DTXSWC);
+	reg &= ~(E1000_DTXSWC_VMDQ_LOOPBACK_EN);
+	E1000_WRITE_REG(hw, E1000_DTXSWC, reg);
+}
+
+/**
+ *  e1000_vmdq_replication_enable_pf - Enable replication of brdcst & multicst
+ *  @hw: pointer to the HW structure
+ *
+ *  Enables replication of broadcast and multicast packets from the network
+ *  to VM's which have their respective broadcast and multicast accept
+ *  bits set in the VM Offload Register.  This gives the PF driver per
+ *  VM granularity control over which VM's get replicated broadcast traffic.
+ **/
+void e1000_vmdq_replication_enable_pf(struct e1000_hw *hw, u32 enables)
+{
+	u32 reg;
+	u32 i;
+
+	for (i = 0; i < MAX_NUM_VFS; i++) {
+		if (enables & (1 << i)) {
+			reg = E1000_READ_REG(hw, E1000_VMOLR(i));
+			reg |= (E1000_VMOLR_AUPE |
+				E1000_VMOLR_BAM |
+				E1000_VMOLR_MPME);
+			E1000_WRITE_REG(hw, E1000_VMOLR(i), reg);
+		}
+	}
+
+	reg = E1000_READ_REG(hw, E1000_VT_CTL);
+	reg |= E1000_VT_CTL_VM_REPL_EN;
+	E1000_WRITE_REG(hw, E1000_VT_CTL, reg);
+}
+
+/**
+ *  e1000_vmdq_replication_disable_pf - Disable replication of brdcst & multicst
+ *  @hw: pointer to the HW structure
+ *
+ *  Disables replication of broadcast and multicast packets to the VM's.
+ **/
+void e1000_vmdq_replication_disable_pf(struct e1000_hw *hw)
+{
+	u32 reg;
+
+	reg = E1000_READ_REG(hw, E1000_VT_CTL);
+	reg &= ~(E1000_VT_CTL_VM_REPL_EN);
+	E1000_WRITE_REG(hw, E1000_VT_CTL, reg);
+}
+
+/**
+ *  e1000_vmdq_enable_replication_mode_pf - Enables replication mode in the device
+ *  @hw: pointer to the HW structure
+ **/
+void e1000_vmdq_enable_replication_mode_pf(struct e1000_hw *hw)
+{
+	u32 reg;
+
+	reg = E1000_READ_REG(hw, E1000_VT_CTL);
+	reg |= E1000_VT_CTL_VM_REPL_EN;
+	E1000_WRITE_REG(hw, E1000_VT_CTL, reg);
+}
+
+/**
+ *  e1000_vmdq_broadcast_replication_enable_pf - Enable replication of brdcst
+ *  @hw: pointer to the HW structure
+ *  @enables: PoolSet Bit - if set to ALL_QUEUES, apply to all pools.
+ *
+ *  Enables replication of broadcast packets from the network
+ *  to VM's which have their respective broadcast accept
+ *  bits set in the VM Offload Register.  This gives the PF driver per
+ *  VM granularity control over which VM's get replicated broadcast traffic.
+ **/
+void e1000_vmdq_broadcast_replication_enable_pf(struct e1000_hw *hw,
+						u32 enables)
+{
+	u32 reg;
+	u32 i;
+
+	for (i = 0; i < MAX_NUM_VFS; i++) {
+		if ((enables == ALL_QUEUES) || (enables & (1 << i))) {
+			reg = E1000_READ_REG(hw, E1000_VMOLR(i));
+			reg |= E1000_VMOLR_BAM;
+			E1000_WRITE_REG(hw, E1000_VMOLR(i), reg);
+		}
+	}
+}
+
+/**
+ *  e1000_vmdq_broadcast_replication_disable_pf - Disable replication
+ *  of broadcast packets
+ *  @hw: pointer to the HW structure
+ *  @disables: PoolSet Bit - if set to ALL_QUEUES, apply to all pools.
+ *
+ *  Disables replication of broadcast packets for specific pools.
+ *  If bam/mpe is disabled on all pools then replication mode is
+ *  turned off.
+ **/
+void e1000_vmdq_broadcast_replication_disable_pf(struct e1000_hw *hw,
+						 u32 disables)
+{
+	u32 reg;
+	u32 i;
+	u32 oneenabled = 0;
+
+	for (i = 0; i < MAX_NUM_VFS; i++) {
+		reg = E1000_READ_REG(hw, E1000_VMOLR(i));
+		if ((disables == ALL_QUEUES) || (disables & (1 << i))) {
+			reg &= ~(E1000_VMOLR_BAM);
+			E1000_WRITE_REG(hw, E1000_VMOLR(i), reg);
+		}
+		if (!oneenabled && (reg & (E1000_VMOLR_AUPE |
+				E1000_VMOLR_BAM |
+				E1000_VMOLR_MPME)))
+				oneenabled = 1;
+	}
+	if (!oneenabled) {
+		reg = E1000_READ_REG(hw, E1000_VT_CTL);
+		reg &= ~(E1000_VT_CTL_VM_REPL_EN);
+		E1000_WRITE_REG(hw, E1000_VT_CTL, reg);
+	}
+}
+
+/**
+ *  e1000_vmdq_multicast_promiscuous_enable_pf - Enable promiscuous reception
+ *  @hw: pointer to the HW structure
+ *  @enables: PoolSet Bit - if set to ALL_QUEUES, apply to all pools.
+ *
+ *  Enables promiscuous reception of multicast packets from the network
+ *  to VM's which have their respective multicast promiscuous mode enable
+ *  bits set in the VM Offload Register.  This gives the PF driver per
+ *  VM granularity control over which VM's get all multicast traffic.
+ **/
+void e1000_vmdq_multicast_promiscuous_enable_pf(struct e1000_hw *hw,
+						u32 enables)
+{
+	u32 reg;
+	u32 i;
+
+	for (i = 0; i < MAX_NUM_VFS; i++) {
+		if ((enables == ALL_QUEUES) || (enables & (1 << i))) {
+			reg = E1000_READ_REG(hw, E1000_VMOLR(i));
+			reg |= E1000_VMOLR_MPME;
+			E1000_WRITE_REG(hw, E1000_VMOLR(i), reg);
+		}
+	}
+}
+
+/**
+ *  e1000_vmdq_multicast_promiscuous_disable_pf - Disable promiscuous
+ *  reception of multicast packets
+ *  @hw: pointer to the HW structure
+ *  @disables: PoolSet Bit - if set to ALL_QUEUES, apply to all pools.
+ *
+ *  Disables promiscuous reception of multicast packets for specific pools.
+ *  If bam/mpe is disabled on all pools then replication mode is
+ *  turned off.
+ **/
+void e1000_vmdq_multicast_promiscuous_disable_pf(struct e1000_hw *hw,
+						 u32 disables)
+{
+	u32 reg;
+	u32 i;
+	u32 oneenabled = 0;
+
+	for (i = 0; i < MAX_NUM_VFS; i++) {
+		reg = E1000_READ_REG(hw, E1000_VMOLR(i));
+		if ((disables == ALL_QUEUES) || (disables & (1 << i))) {
+			reg &= ~(E1000_VMOLR_MPME);
+			E1000_WRITE_REG(hw, E1000_VMOLR(i), reg);
+		}
+		if (!oneenabled && (reg & (E1000_VMOLR_AUPE |
+				E1000_VMOLR_BAM |
+				E1000_VMOLR_MPME)))
+				oneenabled = 1;
+	}
+	if (!oneenabled) {
+		reg = E1000_READ_REG(hw, E1000_VT_CTL);
+		reg &= ~(E1000_VT_CTL_VM_REPL_EN);
+		E1000_WRITE_REG(hw, E1000_VT_CTL, reg);
+	}
+}
+
+/**
+ *  e1000_vmdq_aupe_enable_pf - Enable acceptance of untagged packets
+ *  @hw: pointer to the HW structure
+ *  @enables: PoolSet Bit - if set to ALL_QUEUES, apply to all pools.
+ *
+ *  Enables acceptance of packets from the network which do not have
+ *  a VLAN tag but match the exact MAC filter of a given VM.
+ **/
+void e1000_vmdq_aupe_enable_pf(struct e1000_hw *hw, u32 enables)
+{
+	u32 reg;
+	u32 i;
+
+	for (i = 0; i < MAX_NUM_VFS; i++) {
+	if ((enables == ALL_QUEUES) || (enables & (1 << i))) {
+			reg = E1000_READ_REG(hw, E1000_VMOLR(i));
+			reg |= E1000_VMOLR_AUPE;
+			E1000_WRITE_REG(hw, E1000_VMOLR(i), reg);
+		}
+	}
+}
+
+/**
+ *  e1000_vmdq_aupe_disable_pf - Disable acceptance of untagged packets
+ *  @hw: pointer to the HW structure
+ *  @disables: PoolSet Bit - if set to ALL_QUEUES, apply to all pools.
+ *
+ *  Disables acceptance of packets from the network which do not have
+ *  a VLAN tag but match the exact MAC filter of a given VM.
+ **/
+void e1000_vmdq_aupe_disable_pf(struct e1000_hw *hw, u32 disables)
+{
+	u32 reg;
+	u32 i;
+
+	for (i = 0; i < MAX_NUM_VFS; i++) {
+		if ((disables == ALL_QUEUES) || (disables & (1 << i))) {
+			reg = E1000_READ_REG(hw, E1000_VMOLR(i));
+			reg &= ~E1000_VMOLR_AUPE;
+			E1000_WRITE_REG(hw, E1000_VMOLR(i), reg);
+		}
+	}
+}
+
+/**
  *  e1000_reset_hw_82575 - Reset hardware
  *  @hw: pointer to the HW structure
  *
@@ -1113,7 +1270,8 @@ static s32 e1000_reset_hw_82575(struct e1000_hw *hw)
 	E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff);
 	icr = E1000_READ_REG(hw, E1000_ICR);
 
-	e1000_check_alt_mac_addr_generic(hw);
+	/* Install any alternate MAC address into RAR0 */
+	ret_val = e1000_check_alt_mac_addr_generic(hw);
 
 	return ret_val;
 }
@@ -1133,7 +1291,7 @@ static s32 e1000_init_hw_82575(struct e1000_hw *hw)
 	DEBUGFUNC("e1000_init_hw_82575");
 
 	/* Initialize identification LED */
-	ret_val = e1000_id_led_init_generic(hw);
+	ret_val = mac->ops.id_led_init(hw);
 	if (ret_val) {
 		DEBUGOUT("Error initializing identification LED\n");
 		/* This is not fatal and we should not stop init due to this */
@@ -1144,7 +1302,8 @@ static s32 e1000_init_hw_82575(struct e1000_hw *hw)
 	mac->ops.clear_vfta(hw);
 
 	/* Setup the receive address */
-	e1000_init_rx_addrs_82575(hw, rar_count);
+	e1000_init_rx_addrs_generic(hw, rar_count);
+
 	/* Zero out the Multicast HASH table */
 	DEBUGOUT("Zeroing the MTA\n");
 	for (i = 0; i < mac->mta_reg_count; i++)
@@ -1502,9 +1661,19 @@ static s32 e1000_read_mac_addr_82575(struct e1000_hw *hw)
 	s32 ret_val = E1000_SUCCESS;
 
 	DEBUGFUNC("e1000_read_mac_addr_82575");
-	if (e1000_check_alt_mac_addr_generic(hw))
-		ret_val = e1000_read_mac_addr_generic(hw);
 
+	/*
+	 * If there's an alternate MAC address place it in RAR0
+	 * so that it will override the Si installed default perm
+	 * address.
+	 */
+	ret_val = e1000_check_alt_mac_addr_generic(hw);
+	if (ret_val)
+		goto out;
+
+	ret_val = e1000_read_mac_addr_generic(hw);
+
+out:
 	return ret_val;
 }
 
diff --git a/sys/dev/e1000/e1000_82575.h b/sys/dev/e1000/e1000_82575.h
index 56321e4..1b7ce89 100644
--- a/sys/dev/e1000/e1000_82575.h
+++ b/sys/dev/e1000/e1000_82575.h
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -128,6 +128,7 @@ struct e1000_adv_context_desc {
 #define E1000_SRRCTL_DESCTYPE_HDR_REPLICATION           0x06000000
 #define E1000_SRRCTL_DESCTYPE_HDR_REPLICATION_LARGE_PKT 0x08000000
 #define E1000_SRRCTL_DESCTYPE_MASK                      0x0E000000
+#define E1000_SRRCTL_DROP_EN                            0x80000000
 
 #define E1000_SRRCTL_BSIZEPKT_MASK      0x0000007F
 #define E1000_SRRCTL_BSIZEHDR_MASK      0x00003F00
@@ -137,6 +138,7 @@ struct e1000_adv_context_desc {
 
 #define E1000_MRQC_ENABLE_RSS_4Q            0x00000002
 #define E1000_MRQC_ENABLE_VMDQ              0x00000003
+#define E1000_MRQC_ENABLE_VMDQ_RSS_2Q       0x00000005
 #define E1000_MRQC_RSS_FIELD_IPV4_UDP       0x00400000
 #define E1000_MRQC_RSS_FIELD_IPV6_UDP       0x00800000
 #define E1000_MRQC_RSS_FIELD_IPV6_UDP_EX    0x01000000
@@ -313,6 +315,7 @@ struct e1000_adv_tx_context_desc {
 #define E1000_ADVTXD_TUCMD_IPV6    0x00000000  /* IP Packet Type: 0=IPv6 */
 #define E1000_ADVTXD_TUCMD_L4T_UDP 0x00000000  /* L4 Packet TYPE of UDP */
 #define E1000_ADVTXD_TUCMD_L4T_TCP 0x00000800  /* L4 Packet TYPE of TCP */
+#define E1000_ADVTXD_TUCMD_L4T_SCTP 0x00001000  /* L4 Packet TYPE of SCTP */
 #define E1000_ADVTXD_TUCMD_IPSEC_TYPE_ESP    0x00002000 /* IPSec Type ESP */
 /* IPSec Encrypt Enable for ESP */
 #define E1000_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN  0x00004000
@@ -381,6 +384,8 @@ struct e1000_adv_tx_context_desc {
 #define E1000_DTXSWC_MAC_SPOOF_MASK   0x000000FF /* Per VF MAC spoof control */
 #define E1000_DTXSWC_VLAN_SPOOF_MASK  0x0000FF00 /* Per VF VLAN spoof control */
 #define E1000_DTXSWC_LLE_MASK         0x00FF0000 /* Per VF Local LB enables */
+#define E1000_DTXSWC_VLAN_SPOOF_SHIFT 8
+#define E1000_DTXSWC_LLE_SHIFT        16
 #define E1000_DTXSWC_VMDQ_LOOPBACK_EN (1 << 31)  /* global VF LB enable */
 
 /* Easy defines for setting default pool, would normally be left a zero */
@@ -393,82 +398,44 @@ struct e1000_adv_tx_context_desc {
 #define E1000_VT_CTL_VM_REPL_EN         (1 << 30)
 
 /* Per VM Offload register setup */
+#define E1000_VMOLR_RLPML_MASK 0x00003FFF /* Long Packet Maximum Length mask */
 #define E1000_VMOLR_LPE        0x00010000 /* Accept Long packet */
+#define E1000_VMOLR_RSSE       0x00020000 /* Enable RSS */
 #define E1000_VMOLR_AUPE       0x01000000 /* Accept untagged packets */
+#define E1000_VMOLR_ROMPE      0x02000000 /* Accept overflow multicast */
+#define E1000_VMOLR_ROPE       0x04000000 /* Accept overflow unicast */
 #define E1000_VMOLR_BAM        0x08000000 /* Accept Broadcast packets */
 #define E1000_VMOLR_MPME       0x10000000 /* Multicast promiscuous mode */
 #define E1000_VMOLR_STRVLAN    0x40000000 /* Vlan stripping enable */
+#define E1000_VMOLR_STRCRC     0x80000000 /* CRC stripping enable */
 
-#define E1000_V2PMAILBOX_REQ   0x00000001 /* Request for PF Ready bit */
-#define E1000_V2PMAILBOX_ACK   0x00000002 /* Ack PF message received */
-#define E1000_V2PMAILBOX_VFU   0x00000004 /* VF owns the mailbox buffer */
-#define E1000_V2PMAILBOX_PFU   0x00000008 /* PF owns the mailbox buffer */
-#define E1000_V2PMAILBOX_PFSTS 0x00000010 /* PF wrote a message in the MB */
-#define E1000_V2PMAILBOX_PFACK 0x00000020 /* PF ack the previous VF msg */
-#define E1000_V2PMAILBOX_RSTI  0x00000040 /* PF has reset indication */
-
-#define E1000_P2VMAILBOX_STS   0x00000001 /* Initiate message send to VF */
-#define E1000_P2VMAILBOX_ACK   0x00000002 /* Ack message recv'd from VF */
-#define E1000_P2VMAILBOX_VFU   0x00000004 /* VF owns the mailbox buffer */
-#define E1000_P2VMAILBOX_PFU   0x00000008 /* PF owns the mailbox buffer */
-#define E1000_P2VMAILBOX_RVFU  0x00000010 /* Reset VFU - used when VF stuck */
-
-#define E1000_VFMAILBOX_SIZE   16 /* 16 32 bit words - 64 bytes */
-
-/* If it's a E1000_VF_* msg then it originates in the VF and is sent to the
- * PF.  The reverse is TRUE if it is E1000_PF_*.
- * Message ACK's are the value or'd with 0xF0000000
- */
-#define E1000_VT_MSGTYPE_ACK      0xF0000000  /* Messages below or'd with
-                                               * this are the ACK */
-#define E1000_VT_MSGTYPE_NACK     0xFF000000  /* Messages below or'd with
-                                               * this are the NACK */
-#define E1000_VT_MSGINFO_SHIFT    16
-/* bits 23:16 are used for exra info for certain messages */
-#define E1000_VT_MSGINFO_MASK     (0xFF << E1000_VT_MSGINFO_SHIFT)
-
-#define E1000_VF_MSGTYPE_REQ_MAC  1 /* VF needs to know its MAC */
-#define E1000_VF_MSGTYPE_VFLR     2 /* VF notifies VFLR to PF */
-#define E1000_VF_SET_MULTICAST    3 /* VF requests PF to set MC addr */
-#define E1000_VF_SET_VLAN         4 /* VF requests PF to set VLAN */
-
-/* Add 100h to all PF msgs, leaves room for up to 255 discrete message types
- * from VF to PF - way more than we'll ever need */
-#define E1000_PF_MSGTYPE_RESET    (1 + 0x100) /* PF notifies global reset
-                                               * imminent to VF */
-#define E1000_PF_MSGTYPE_LSC      (2 + 0x100) /* PF notifies VF of LSC... VF
-                                               * will see extra msg info for
-                                               * status */
-
-#define E1000_PF_MSG_LSCDOWN      (1 << E1000_VT_MSGINFO_SHIFT)
-#define E1000_PF_MSG_LSCUP        (2 << E1000_VT_MSGINFO_SHIFT)
+#define E1000_VLVF_ARRAY_SIZE     32
+#define E1000_VLVF_VLANID_MASK    0x00000FFF
+#define E1000_VLVF_POOLSEL_SHIFT  12
+#define E1000_VLVF_POOLSEL_MASK   (0xFF << E1000_VLVF_POOLSEL_SHIFT)
+#define E1000_VLVF_LVLAN          0x00100000
+#define E1000_VLVF_VLANID_ENABLE  0x80000000
+
+#define E1000_VF_INIT_TIMEOUT 200 /* Number of retries to clear RSTI */
+
+#define E1000_IOVCTL 0x05BBC
+#define E1000_IOVCTL_REUSE_VFQ 0x00000001
 
 #define ALL_QUEUES   0xFFFF
 
-s32  e1000_send_mail_to_pf_vf(struct e1000_hw *hw, u32 *msg,
-                              s16 size);
-s32  e1000_receive_mail_from_pf_vf(struct e1000_hw *hw,
-                                   u32 *msg, s16 size);
-s32  e1000_send_mail_to_vf(struct e1000_hw *hw, u32 *msg,
-                           u32 vf_number, s16 size);
-s32  e1000_receive_mail_from_vf(struct e1000_hw *hw, u32 *msg,
-                                u32 vf_number, s16 size);
-void e1000_vmdq_loopback_enable_vf(struct e1000_hw *hw);
-void e1000_vmdq_loopback_disable_vf(struct e1000_hw *hw);
-void e1000_vmdq_replication_enable_vf(struct e1000_hw *hw, u32 enables);
-void e1000_vmdq_replication_disable_vf(struct e1000_hw *hw);
-void e1000_vmdq_enable_replication_mode_vf(struct e1000_hw *hw);
-void e1000_vmdq_broadcast_replication_enable_vf(struct e1000_hw *hw,
+void e1000_vmdq_loopback_enable_pf(struct e1000_hw *hw);
+void e1000_vmdq_loopback_disable_pf(struct e1000_hw *hw);
+void e1000_vmdq_replication_enable_pf(struct e1000_hw *hw, u32 enables);
+void e1000_vmdq_replication_disable_pf(struct e1000_hw *hw);
+void e1000_vmdq_enable_replication_mode_pf(struct e1000_hw *hw);
+void e1000_vmdq_broadcast_replication_enable_pf(struct e1000_hw *hw,
 						u32 enables);
-void e1000_vmdq_multicast_replication_enable_vf(struct e1000_hw *hw,
+void e1000_vmdq_multicast_promiscuous_enable_pf(struct e1000_hw *hw,
 						u32 enables);
-void e1000_vmdq_broadcast_replication_disable_vf(struct e1000_hw *hw,
+void e1000_vmdq_broadcast_replication_disable_pf(struct e1000_hw *hw,
 						u32 disables);
-void e1000_vmdq_multicast_replication_disable_vf(struct e1000_hw *hw,
+void e1000_vmdq_multicast_promiscuous_disable_pf(struct e1000_hw *hw,
 						u32 disables);
-bool e1000_check_for_pf_ack_vf(struct e1000_hw *hw);
-
-bool e1000_check_for_pf_mail_vf(struct e1000_hw *hw, u32*);
-
-
-#endif
+void e1000_vmdq_aupe_enable_pf(struct e1000_hw *hw, u32 enables);
+void e1000_vmdq_aupe_disable_pf(struct e1000_hw *hw, u32 disables);
+#endif /* _E1000_82575_H_ */
diff --git a/sys/dev/e1000/e1000_api.c b/sys/dev/e1000/e1000_api.c
index 47bffb5..1f47d34 100644
--- a/sys/dev/e1000/e1000_api.c
+++ b/sys/dev/e1000/e1000_api.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -112,6 +112,7 @@ out:
 	return ret_val;
 }
 
+
 /**
  *  e1000_set_mac_type - Sets MAC type
  *  @hw: pointer to the HW structure
@@ -250,12 +251,14 @@ s32 e1000_set_mac_type(struct e1000_hw *hw)
 	case E1000_DEV_ID_82575EB_COPPER:
 	case E1000_DEV_ID_82575EB_FIBER_SERDES:
 	case E1000_DEV_ID_82575GB_QUAD_COPPER:
+	case E1000_DEV_ID_82575GB_QUAD_COPPER_PM:
 		mac->type = e1000_82575;
 		break;
 	case E1000_DEV_ID_82576:
 	case E1000_DEV_ID_82576_FIBER:
 	case E1000_DEV_ID_82576_SERDES:
 	case E1000_DEV_ID_82576_QUAD_COPPER:
+	case E1000_DEV_ID_82576_NS:
 		mac->type = e1000_82576;
 		break;
 	default:
@@ -370,7 +373,6 @@ s32 e1000_setup_init_funcs(struct e1000_hw *hw, bool init_device)
 		ret_val = e1000_init_phy_params(hw);
 		if (ret_val)
 			goto out;
-
 	}
 
 out:
@@ -426,26 +428,16 @@ void e1000_write_vfta(struct e1000_hw *hw, u32 offset, u32 value)
  *  @hw: pointer to the HW structure
  *  @mc_addr_list: array of multicast addresses to program
  *  @mc_addr_count: number of multicast addresses to program
- *  @rar_used_count: the first RAR register free to program
- *  @rar_count: total number of supported Receive Address Registers
  *
- *  Updates the Receive Address Registers and Multicast Table Array.
+ *  Updates the Multicast Table Array.
  *  The caller must have a packed mc_addr_list of multicast addresses.
- *  The parameter rar_count will usually be hw->mac.rar_entry_count
- *  unless there are workarounds that change this.  Currently no func pointer
- *  exists and all implementations are handled in the generic version of this
- *  function.
  **/
 void e1000_update_mc_addr_list(struct e1000_hw *hw, u8 *mc_addr_list,
-                               u32 mc_addr_count, u32 rar_used_count,
-                               u32 rar_count)
+                               u32 mc_addr_count)
 {
 	if (hw->mac.ops.update_mc_addr_list)
-		hw->mac.ops.update_mc_addr_list(hw,
-		                                mc_addr_list,
-		                                mc_addr_count,
-		                                rar_used_count,
-		                                rar_count);
+		hw->mac.ops.update_mc_addr_list(hw, mc_addr_list,
+		                                mc_addr_count);
 }
 
 /**
@@ -617,6 +609,21 @@ s32 e1000_blink_led(struct e1000_hw *hw)
 }
 
 /**
+ *  e1000_id_led_init - store LED configurations in SW
+ *  @hw: pointer to the HW structure
+ *
+ *  Initializes the LED config in SW. This is a function pointer entry point
+ *  called by drivers.
+ **/
+s32 e1000_id_led_init(struct e1000_hw *hw)
+{
+	if (hw->mac.ops.id_led_init)
+		return hw->mac.ops.id_led_init(hw);
+
+	return E1000_SUCCESS;
+}
+
+/**
  *  e1000_led_on - Turn on SW controllable LED
  *  @hw: pointer to the HW structure
  *
diff --git a/sys/dev/e1000/e1000_api.h b/sys/dev/e1000/e1000_api.h
index 4629ff0..b492e57 100644
--- a/sys/dev/e1000/e1000_api.h
+++ b/sys/dev/e1000/e1000_api.h
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -70,14 +70,14 @@ void e1000_rar_set(struct e1000_hw *hw, u8 *addr, u32 index);
 void e1000_mta_set(struct e1000_hw *hw, u32 hash_value);
 u32  e1000_hash_mc_addr(struct e1000_hw *hw, u8 *mc_addr);
 void e1000_update_mc_addr_list(struct e1000_hw *hw,
-                               u8 *mc_addr_list, u32 mc_addr_count,
-                               u32 rar_used_count, u32 rar_count);
+                               u8 *mc_addr_list, u32 mc_addr_count);
 s32  e1000_setup_led(struct e1000_hw *hw);
 s32  e1000_cleanup_led(struct e1000_hw *hw);
 s32  e1000_check_reset_block(struct e1000_hw *hw);
 s32  e1000_blink_led(struct e1000_hw *hw);
 s32  e1000_led_on(struct e1000_hw *hw);
 s32  e1000_led_off(struct e1000_hw *hw);
+s32 e1000_id_led_init(struct e1000_hw *hw);
 void e1000_reset_adaptive(struct e1000_hw *hw);
 void e1000_update_adaptive(struct e1000_hw *hw);
 s32  e1000_get_cable_length(struct e1000_hw *hw);
diff --git a/sys/dev/e1000/e1000_defines.h b/sys/dev/e1000/e1000_defines.h
index 81259a1..1f6c198 100644
--- a/sys/dev/e1000/e1000_defines.h
+++ b/sys/dev/e1000/e1000_defines.h
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -144,6 +144,7 @@
 #define E1000_CTRL_EXT_PFRSTD    0x00004000
 #define E1000_CTRL_EXT_SPD_BYPS  0x00008000 /* Speed Select Bypass */
 #define E1000_CTRL_EXT_RO_DIS    0x00020000 /* Relaxed Ordering disable */
+#define E1000_CTRL_EXT_DMA_DYN_CLK_EN 0x00080000 /* DMA Dynamic Clock Gating */
 #define E1000_CTRL_EXT_LINK_MODE_MASK 0x00C00000
 #define E1000_CTRL_EXT_LINK_MODE_GMII 0x00000000
 #define E1000_CTRL_EXT_LINK_MODE_TBI  0x00C00000
@@ -162,8 +163,6 @@
 #define E1000_CTRL_EXT_DRV_LOAD       0x10000000 /* Driver loaded bit for FW */
 /* IAME enable bit (27) was removed in >= 82575 */
 #define E1000_CTRL_EXT_IAME           0x08000000 /* Int acknowledge Auto-mask */
-#define E1000_CTRL_EXT_INT_TIMER_CLR  0x20000000 /* Clear Interrupt timers
-                                                  * after IMS clear */
 #define E1000_CRTL_EXT_PB_PAREN       0x01000000 /* packet buffer parity error
                                                   * detection enabled */
 #define E1000_CTRL_EXT_DF_PAREN       0x02000000 /* descriptor FIFO parity
@@ -402,6 +401,7 @@
 #define E1000_CTRL_SWDPIN0  0x00040000  /* SWDPIN 0 value */
 #define E1000_CTRL_SWDPIN1  0x00080000  /* SWDPIN 1 value */
 #define E1000_CTRL_SWDPIN2  0x00100000  /* SWDPIN 2 value */
+#define E1000_CTRL_ADVD3WUC 0x00100000  /* D3 WUC */
 #define E1000_CTRL_SWDPIN3  0x00200000  /* SWDPIN 3 value */
 #define E1000_CTRL_SWDPIO0  0x00400000  /* SWDPIN 0 Input or output */
 #define E1000_CTRL_SWDPIO1  0x00800000  /* SWDPIN 1 input or output */
@@ -692,14 +692,19 @@
 /* PBA constants */
 #define E1000_PBA_6K  0x0006	/* 6KB */
 #define E1000_PBA_8K  0x0008    /* 8KB */
+#define E1000_PBA_10K 0x000A    /* 10KB */
 #define E1000_PBA_12K 0x000C    /* 12KB */
+#define E1000_PBA_14K 0x000E    /* 14KB */
 #define E1000_PBA_16K 0x0010    /* 16KB */
+#define E1000_PBA_18K 0x0012
 #define E1000_PBA_20K 0x0014
 #define E1000_PBA_22K 0x0016
 #define E1000_PBA_24K 0x0018
+#define E1000_PBA_26K 0x001A
 #define E1000_PBA_30K 0x001E
 #define E1000_PBA_32K 0x0020
 #define E1000_PBA_34K 0x0022
+#define E1000_PBA_35K 0x0023
 #define E1000_PBA_38K 0x0026
 #define E1000_PBA_40K 0x0028
 #define E1000_PBA_48K 0x0030    /* 48KB */
@@ -761,6 +766,13 @@
 #define E1000_ICR_TXQ1          0x00800000 /* Tx Queue 1 Interrupt */
 #define E1000_ICR_OTHER         0x01000000 /* Other Interrupts */
 
+/* PBA ECC Register */
+#define E1000_PBA_ECC_COUNTER_MASK  0xFFF00000 /* ECC counter mask */
+#define E1000_PBA_ECC_COUNTER_SHIFT 20         /* ECC counter shift value */
+#define E1000_PBA_ECC_CORR_EN       0x00000001 /* Enable ECC error correction */
+#define E1000_PBA_ECC_STAT_CLR      0x00000002 /* Clear ECC error counter */
+#define E1000_PBA_ECC_INT_EN        0x00000004 /* Enable ICR bit 5 on ECC error */
+
 /* Extended Interrupt Cause Read */
 #define E1000_EICR_RX_QUEUE0    0x00000001 /* Rx Queue 0 Interrupt */
 #define E1000_EICR_RX_QUEUE1    0x00000002 /* Rx Queue 1 Interrupt */
@@ -906,6 +918,8 @@
 #define E1000_EICS_TCP_TIMER    E1000_EICR_TCP_TIMER /* TCP Timer */
 #define E1000_EICS_OTHER        E1000_EICR_OTHER   /* Interrupt Cause Active */
 
+#define E1000_EITR_ITR_INT_MASK 0x0000FFFF
+
 /* Transmit Descriptor Control */
 #define E1000_TXDCTL_PTHRESH 0x0000003F /* TXDCTL Prefetch Threshold */
 #define E1000_TXDCTL_HTHRESH 0x00003F00 /* TXDCTL Host Threshold */
@@ -936,6 +950,10 @@
  */
 #define E1000_RAR_ENTRIES     15
 #define E1000_RAH_AV  0x80000000        /* Receive descriptor valid */
+#define E1000_RAL_MAC_ADDR_LEN 4
+#define E1000_RAH_MAC_ADDR_LEN 2
+#define E1000_RAH_POOL_MASK 0x03FC0000
+#define E1000_RAH_POOL_1 0x00040000
 
 /* Error Codes */
 #define E1000_SUCCESS      0
@@ -951,6 +969,7 @@
 #define E1000_BLK_PHY_RESET   12
 #define E1000_ERR_SWFW_SYNC 13
 #define E1000_NOT_IMPLEMENTED 14
+#define E1000_ERR_MBX      15
 
 /* Loop limit on how long we wait for auto-negotiation to complete */
 #define FIBER_LINK_UP_LIMIT               50
@@ -1145,6 +1164,7 @@
 #define E1000_EECD_SHADV     0x00200000 /* Shadow RAM Data Valid */
 #define E1000_EECD_SEC1VAL   0x00400000 /* Sector One Valid */
 #define E1000_EECD_SECVAL_SHIFT      22
+#define E1000_EECD_SEC1VAL_VALID_MASK (E1000_EECD_AUTO_RD | E1000_EECD_PRES)
 
 #define E1000_NVM_SWDPIN0   0x0001   /* SWDPIN 0 NVM Value */
 #define E1000_NVM_LED_LOGIC 0x0020   /* Led Logic Word */
diff --git a/sys/dev/e1000/e1000_hw.h b/sys/dev/e1000/e1000_hw.h
index 277d8d1..d92ea40 100644
--- a/sys/dev/e1000/e1000_hw.h
+++ b/sys/dev/e1000/e1000_hw.h
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -94,6 +94,7 @@ struct e1000_hw;
 #define E1000_DEV_ID_82573E_IAMT              0x108C
 #define E1000_DEV_ID_82573L                   0x109A
 #define E1000_DEV_ID_82574L                   0x10D3
+#define E1000_DEV_ID_82574LA                  0x10F6
 #define E1000_DEV_ID_80003ES2LAN_COPPER_DPT   0x1096
 #define E1000_DEV_ID_80003ES2LAN_SERDES_DPT   0x1098
 #define E1000_DEV_ID_80003ES2LAN_COPPER_SPT   0x10BA
@@ -123,10 +124,11 @@ struct e1000_hw;
 #define E1000_DEV_ID_82576_FIBER              0x10E6
 #define E1000_DEV_ID_82576_SERDES             0x10E7
 #define E1000_DEV_ID_82576_QUAD_COPPER        0x10E8
-#define E1000_DEV_ID_82576_VF                 0x10CA
+#define E1000_DEV_ID_82576_NS                 0x150A
 #define E1000_DEV_ID_82575EB_COPPER           0x10A7
 #define E1000_DEV_ID_82575EB_FIBER_SERDES     0x10A9
 #define E1000_DEV_ID_82575GB_QUAD_COPPER      0x10D6
+#define E1000_DEV_ID_82575GB_QUAD_COPPER_PM   0x10E2
 #define E1000_REVISION_0 0
 #define E1000_REVISION_1 1
 #define E1000_REVISION_2 2
@@ -136,6 +138,9 @@ struct e1000_hw;
 #define E1000_FUNC_0     0
 #define E1000_FUNC_1     1
 
+#define E1000_ALT_MAC_ADDRESS_OFFSET_LAN0   0
+#define E1000_ALT_MAC_ADDRESS_OFFSET_LAN1   3
+
 enum e1000_mac_type {
 	e1000_undefined = 0,
 	e1000_82542,
@@ -160,7 +165,6 @@ enum e1000_mac_type {
 	e1000_ich10lan,
 	e1000_82575,
 	e1000_82576,
-	e1000_vfadapt,
 	e1000_num_macs  /* List is 1-based, so subtract 1 for TRUE count. */
 };
 
@@ -279,6 +283,13 @@ enum e1000_smart_speed {
 	e1000_smart_speed_off
 };
 
+enum e1000_serdes_link_state {
+	e1000_serdes_link_down = 0,
+	e1000_serdes_link_autoneg_progress,
+	e1000_serdes_link_autoneg_complete,
+	e1000_serdes_link_forced_up
+};
+
 /* Receive Descriptor */
 struct e1000_rx_desc {
 	__le64 buffer_addr; /* Address of the descriptor's data buffer */
@@ -496,37 +507,6 @@ struct e1000_hw_stats {
 	u64 doosync;
 };
 
-struct e1000_vf_stats {
-	u64 base_gprc;
-	u64 base_gptc;
-	u64 base_gorc;
-	u64 base_gotc;
-	u64 base_mprc;
-	u64 base_gotlbc;
-	u64 base_gptlbc;
-	u64 base_gorlbc;
-	u64 base_gprlbc;
-
-	u32 last_gprc;
-	u32 last_gptc;
-	u32 last_gorc;
-	u32 last_gotc;
-	u32 last_mprc;
-	u32 last_gotlbc;
-	u32 last_gptlbc;
-	u32 last_gorlbc;
-	u32 last_gprlbc;
-
-	u64 gprc;
-	u64 gptc;
-	u64 gorc;
-	u64 gotc;
-	u64 mprc;
-	u64 gotlbc;
-	u64 gptlbc;
-	u64 gorlbc;
-	u64 gprlbc;
-};
 
 struct e1000_phy_stats {
 	u32 idle_errors;
@@ -581,6 +561,7 @@ struct e1000_host_mng_command_info {
 struct e1000_mac_operations {
 	/* Function pointers for the MAC. */
 	s32  (*init_params)(struct e1000_hw *);
+	s32  (*id_led_init)(struct e1000_hw *);
 	s32  (*blink_led)(struct e1000_hw *);
 	s32  (*check_for_link)(struct e1000_hw *);
 	bool (*check_mng_mode)(struct e1000_hw *hw);
@@ -592,7 +573,7 @@ struct e1000_mac_operations {
 	s32  (*get_link_up_info)(struct e1000_hw *, u16 *, u16 *);
 	s32  (*led_on)(struct e1000_hw *);
 	s32  (*led_off)(struct e1000_hw *);
-	void (*update_mc_addr_list)(struct e1000_hw *, u8 *, u32, u32, u32);
+	void (*update_mc_addr_list)(struct e1000_hw *, u8 *, u32);
 	s32  (*reset_hw)(struct e1000_hw *);
 	s32  (*init_hw)(struct e1000_hw *);
 	void (*shutdown_serdes)(struct e1000_hw *);
@@ -666,6 +647,10 @@ struct e1000_mac_info {
 	u16 ifs_ratio;
 	u16 ifs_step_size;
 	u16 mta_reg_count;
+#define MAX_MTA_REG 128	/* this must be the maximum size of the MTA register
+			 * table in all supported adapters
+			 */
+	u32 mta_shadow[MAX_MTA_REG];
 	u16 rar_entry_count;
 
 	u8  forced_speed_duplex;
@@ -678,6 +663,7 @@ struct e1000_mac_info {
 	bool get_link_status;
 	bool in_ifs_mode;
 	bool report_tx_early;
+	enum e1000_serdes_link_state serdes_link_state;
 	bool serdes_has_link;
 	bool tx_pkt_filtering;
 };
@@ -785,12 +771,15 @@ struct e1000_dev_spec_ich8lan {
 
 struct e1000_dev_spec_82575 {
 	bool sgmii_active;
+	bool global_device_reset;
 };
 
 struct e1000_dev_spec_vf {
 	u32	vf_number;
+	u32	v2p_mailbox;
 };
 
+
 struct e1000_hw {
 	void *back;
 
diff --git a/sys/dev/e1000/e1000_ich8lan.c b/sys/dev/e1000/e1000_ich8lan.c
index f368998..1225e55 100644
--- a/sys/dev/e1000/e1000_ich8lan.c
+++ b/sys/dev/e1000/e1000_ich8lan.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -197,7 +197,7 @@ static s32 e1000_init_phy_params_ich8lan(struct e1000_hw *hw)
 		phy->ops.read_reg  = e1000_read_phy_reg_bm;
 		ret_val = e1000_determine_phy_address(hw);
 		if (ret_val) {
-			DEBUGOUT("Cannot determine PHY address. Erroring out\n");
+			DEBUGOUT("Cannot determine PHY addr. Erroring out\n");
 			goto out;
 		}
 	}
@@ -319,6 +319,9 @@ out:
 static s32 e1000_init_mac_params_ich8lan(struct e1000_hw *hw)
 {
 	struct e1000_mac_info *mac = &hw->mac;
+#if defined(NAHUM4) && !defined(NO_PCH_A_SUPPORT)
+	u16 pci_cfg;
+#endif
 
 	DEBUGFUNC("e1000_init_mac_params_ich8lan");
 
@@ -360,18 +363,30 @@ static s32 e1000_init_mac_params_ich8lan(struct e1000_hw *hw)
 	mac->ops.update_mc_addr_list = e1000_update_mc_addr_list_generic;
 	/* setting MTA */
 	mac->ops.mta_set = e1000_mta_set_generic;
-	/* blink LED */
-	mac->ops.blink_led = e1000_blink_led_generic;
-	/* setup LED */
-	mac->ops.setup_led = e1000_setup_led_generic;
-	/* cleanup LED */
-	mac->ops.cleanup_led = e1000_cleanup_led_ich8lan;
-	/* turn on/off LED */
-	mac->ops.led_on = e1000_led_on_ich8lan;
-	mac->ops.led_off = e1000_led_off_ich8lan;
 	/* clear hardware counters */
 	mac->ops.clear_hw_cntrs = e1000_clear_hw_cntrs_ich8lan;
 
+	/* LED operations */
+	switch (mac->type) {
+	case e1000_ich8lan:
+	case e1000_ich9lan:
+	case e1000_ich10lan:
+		/* ID LED init */
+		mac->ops.id_led_init = e1000_id_led_init_generic;
+		/* blink LED */
+		mac->ops.blink_led = e1000_blink_led_generic;
+		/* setup LED */
+		mac->ops.setup_led = e1000_setup_led_generic;
+		/* cleanup LED */
+		mac->ops.cleanup_led = e1000_cleanup_led_ich8lan;
+		/* turn on/off LED */
+		mac->ops.led_on = e1000_led_on_ich8lan;
+		mac->ops.led_off = e1000_led_off_ich8lan;
+		break;
+	default:
+		break;
+	}
+
 	/* Enable PCS Lock-loss workaround for ICH8 */
 	if (mac->type == e1000_ich8lan)
 		e1000_set_kmrn_lock_loss_workaround_ich8lan(hw, TRUE);
@@ -993,48 +1008,65 @@ out:
  *  @bank:  pointer to the variable that returns the active bank
  *
  *  Reads signature byte from the NVM using the flash access registers.
+ *  Word 0x13 bits 15:14 = 10b indicate a valid signature for that bank.
  **/
 static s32 e1000_valid_nvm_bank_detect_ich8lan(struct e1000_hw *hw, u32 *bank)
 {
-	s32 ret_val = E1000_SUCCESS;
+	u32 eecd;
 	struct e1000_nvm_info *nvm = &hw->nvm;
-	/* flash bank size is in words */
 	u32 bank1_offset = nvm->flash_bank_size * sizeof(u16);
 	u32 act_offset = E1000_ICH_NVM_SIG_WORD * 2 + 1;
-	u8 bank_high_byte = 0;
+	u8 sig_byte = 0;
+	s32 ret_val = E1000_SUCCESS;
 
-	if (hw->mac.type != e1000_ich10lan) {
-		if (E1000_READ_REG(hw, E1000_EECD) & E1000_EECD_SEC1VAL)
-			*bank = 1;
-		else
-			*bank = 0;
-	} else {
-		/*
-		 * Make sure the signature for bank 0 is valid,
-		 * if not check for bank1
-		 */
-		e1000_read_flash_byte_ich8lan(hw, act_offset, &bank_high_byte);
-		if ((bank_high_byte & 0xC0) == 0x80) {
+	switch (hw->mac.type) {
+	case e1000_ich8lan:
+	case e1000_ich9lan:
+		eecd = E1000_READ_REG(hw, E1000_EECD);
+		if ((eecd & E1000_EECD_SEC1VAL_VALID_MASK) ==
+		    E1000_EECD_SEC1VAL_VALID_MASK) {
+			if (eecd & E1000_EECD_SEC1VAL)
+				*bank = 1;
+			else
+				*bank = 0;
+
+			goto out;
+		}
+		DEBUGOUT("Unable to determine valid NVM bank via EEC - "
+		         "reading flash signature\n");
+		/* fall-thru */
+	default:
+		/* set bank to 0 in case flash read fails */
+		*bank = 0;
+
+		/* Check bank 0 */
+		ret_val = e1000_read_flash_byte_ich8lan(hw, act_offset,
+		                                        &sig_byte);
+		if (ret_val)
+			goto out;
+		if ((sig_byte & E1000_ICH_NVM_VALID_SIG_MASK) ==
+			E1000_ICH_NVM_SIG_VALUE) {
 			*bank = 0;
-		} else {
-			/*
-			 * find if segment 1 is valid by verifying
-			 * bit 15:14 = 10b in word 0x13
-			 */
-			e1000_read_flash_byte_ich8lan(hw,
-			                              act_offset + bank1_offset,
-			                              &bank_high_byte);
+			goto out;
+		}
 
-			/* bank1 has a valid signature equivalent to SEC1V */
-			if ((bank_high_byte & 0xC0) == 0x80) {
-				*bank = 1;
-			} else {
-				DEBUGOUT("ERROR: EEPROM not present\n");
-				ret_val = -E1000_ERR_NVM;
-			}
+		/* Check bank 1 */
+		ret_val = e1000_read_flash_byte_ich8lan(hw, act_offset +
+		                                        bank1_offset,
+					                &sig_byte);
+		if (ret_val)
+			goto out;
+		if ((sig_byte & E1000_ICH_NVM_VALID_SIG_MASK) ==
+			E1000_ICH_NVM_SIG_VALUE) {
+			*bank = 1;
+			goto out;
 		}
-	}
 
+		DEBUGOUT("ERROR: No valid NVM bank present\n");
+		ret_val = -E1000_ERR_NVM;
+		break;
+	}
+out:
 	return ret_val;
 }
 
@@ -1072,7 +1104,7 @@ static s32 e1000_read_nvm_ich8lan(struct e1000_hw *hw, u16 offset, u16 words,
 
 	ret_val = e1000_valid_nvm_bank_detect_ich8lan(hw, &bank);
 	if (ret_val != E1000_SUCCESS)
-		goto out;
+		goto release;
 
 	act_offset = (bank) ? nvm->flash_bank_size : 0;
 	act_offset += offset;
@@ -1091,9 +1123,13 @@ static s32 e1000_read_nvm_ich8lan(struct e1000_hw *hw, u16 offset, u16 words,
 		}
 	}
 
+release:
 	nvm->ops.release(hw);
 
 out:
+	if (ret_val)
+		DEBUGOUT1("NVM read error: %d\n", ret_val);
+
 	return ret_val;
 }
 
@@ -1426,17 +1462,27 @@ static s32 e1000_update_nvm_checksum_ich8lan(struct e1000_hw *hw)
 	 * is going to be written
 	 */
 	ret_val =  e1000_valid_nvm_bank_detect_ich8lan(hw, &bank);
-	if (ret_val != E1000_SUCCESS)
+	if (ret_val != E1000_SUCCESS) {
+		nvm->ops.release(hw);
 		goto out;
+	}
 
 	if (bank == 0) {
 		new_bank_offset = nvm->flash_bank_size;
 		old_bank_offset = 0;
-		e1000_erase_flash_bank_ich8lan(hw, 1);
+		ret_val = e1000_erase_flash_bank_ich8lan(hw, 1);
+		if (ret_val) {
+			nvm->ops.release(hw);
+			goto out;
+		}
 	} else {
 		old_bank_offset = nvm->flash_bank_size;
 		new_bank_offset = 0;
-		e1000_erase_flash_bank_ich8lan(hw, 0);
+		ret_val = e1000_erase_flash_bank_ich8lan(hw, 0);
+		if (ret_val) {
+			nvm->ops.release(hw);
+			goto out;
+		}
 	}
 
 	for (i = 0; i < E1000_SHADOW_RAM_WORDS; i++) {
@@ -1448,9 +1494,11 @@ static s32 e1000_update_nvm_checksum_ich8lan(struct e1000_hw *hw)
 		if (dev_spec->shadow_ram[i].modified) {
 			data = dev_spec->shadow_ram[i].value;
 		} else {
-			e1000_read_flash_word_ich8lan(hw,
-			                              i + old_bank_offset,
-			                              &data);
+			ret_val = e1000_read_flash_word_ich8lan(hw, i +
+			                                        old_bank_offset,
+			                                        &data);
+			if (ret_val)
+				break;
 		}
 
 		/*
@@ -1500,7 +1548,11 @@ static s32 e1000_update_nvm_checksum_ich8lan(struct e1000_hw *hw)
 	 * and we need to change bit 14 to 0b
 	 */
 	act_offset = new_bank_offset + E1000_ICH_NVM_SIG_WORD;
-	e1000_read_flash_word_ich8lan(hw, act_offset, &data);
+	ret_val = e1000_read_flash_word_ich8lan(hw, act_offset, &data);
+	if (ret_val) {
+		nvm->ops.release(hw);
+		goto out;
+	}
 	data &= 0xBFFF;
 	ret_val = e1000_retry_write_flash_byte_ich8lan(hw,
 	                                               act_offset * 2 + 1,
@@ -1539,6 +1591,9 @@ static s32 e1000_update_nvm_checksum_ich8lan(struct e1000_hw *hw)
 	msec_delay(10);
 
 out:
+	if (ret_val)
+		DEBUGOUT1("NVM update error: %d\n", ret_val);
+
 	return ret_val;
 }
 
@@ -1997,11 +2052,10 @@ static s32 e1000_init_hw_ich8lan(struct e1000_hw *hw)
 	e1000_initialize_hw_bits_ich8lan(hw);
 
 	/* Initialize identification LED */
-	ret_val = e1000_id_led_init_generic(hw);
-	if (ret_val) {
-		DEBUGOUT("Error initializing identification LED\n");
+	ret_val = mac->ops.id_led_init(hw);
+	if (ret_val)
 		/* This is not fatal and we should not stop init due to this */
-	}
+		DEBUGOUT("Error initializing identification LED\n");
 
 	/* Setup the receive address. */
 	e1000_init_rx_addrs_generic(hw, mac->rar_entry_count);
@@ -2140,7 +2194,7 @@ static s32 e1000_setup_link_ich8lan(struct e1000_hw *hw)
 	hw->fc.current_mode = hw->fc.requested_mode;
 
 	DEBUGOUT1("After fix-ups FlowControl is now = %x\n",
-                                                    hw->fc.current_mode);
+		hw->fc.current_mode);
 
 	/* Continue to configure the copper link. */
 	ret_val = hw->mac.ops.setup_physical_interface(hw);
@@ -2195,17 +2249,18 @@ static s32 e1000_setup_copper_link_ich8lan(struct e1000_hw *hw)
 	if (ret_val)
 		goto out;
 
-	if (hw->phy.type == e1000_phy_igp_3) {
+	switch (hw->phy.type) {
+	case e1000_phy_igp_3:
 		ret_val = e1000_copper_link_setup_igp(hw);
 		if (ret_val)
 			goto out;
-	} else if (hw->phy.type == e1000_phy_bm) {
+		break;
+	case e1000_phy_bm:
 		ret_val = e1000_copper_link_setup_m88(hw);
 		if (ret_val)
 			goto out;
-	}
-
-	if (hw->phy.type == e1000_phy_ife) {
+		break;
+	case e1000_phy_ife:
 		ret_val = hw->phy.ops.read_reg(hw, IFE_PHY_MDIX_CONTROL,
 		                               &reg_data);
 		if (ret_val)
@@ -2229,6 +2284,9 @@ static s32 e1000_setup_copper_link_ich8lan(struct e1000_hw *hw)
 		                                reg_data);
 		if (ret_val)
 			goto out;
+		break;
+	default:
+		break;
 	}
 	ret_val = e1000_setup_copper_link_generic(hw);
 
@@ -2476,18 +2534,21 @@ out:
  *  'LPLU Enabled' and 'Gig Disable' to force link speed negotiation
  *  to a lower speed.
  *
- *  Should only be called for ICH9 and ICH10 devices.
+ *  Should only be called for applicable parts.
  **/
 void e1000_disable_gig_wol_ich8lan(struct e1000_hw *hw)
 {
 	u32 phy_ctrl;
 
-	if ((hw->mac.type == e1000_ich10lan) ||
-	    (hw->mac.type == e1000_ich9lan)) {
+	switch (hw->mac.type) {
+	case e1000_ich9lan:
+	case e1000_ich10lan:
 		phy_ctrl = E1000_READ_REG(hw, E1000_PHY_CTRL);
 		phy_ctrl |= E1000_PHY_CTRL_D0A_LPLU |
 		            E1000_PHY_CTRL_GBE_DISABLE;
 		E1000_WRITE_REG(hw, E1000_PHY_CTRL, phy_ctrl);
+	default:
+		break;
 	}
 
 	return;
diff --git a/sys/dev/e1000/e1000_ich8lan.h b/sys/dev/e1000/e1000_ich8lan.h
index 79dffcd..6316021 100644
--- a/sys/dev/e1000/e1000_ich8lan.h
+++ b/sys/dev/e1000/e1000_ich8lan.h
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -41,9 +41,10 @@
 #define ICH_FLASH_FADDR                  0x0008
 #define ICH_FLASH_FDATA0                 0x0010
 
-#define ICH_FLASH_READ_COMMAND_TIMEOUT   500
-#define ICH_FLASH_WRITE_COMMAND_TIMEOUT  500
-#define ICH_FLASH_ERASE_COMMAND_TIMEOUT  3000000
+/* Requires up to 10 seconds when MNG might be accessing part. */
+#define ICH_FLASH_READ_COMMAND_TIMEOUT   10000000
+#define ICH_FLASH_WRITE_COMMAND_TIMEOUT  10000000
+#define ICH_FLASH_ERASE_COMMAND_TIMEOUT  10000000
 #define ICH_FLASH_LINEAR_ADDR_MASK       0x00FFFFFF
 #define ICH_FLASH_CYCLE_REPEAT_COUNT     10
 
@@ -70,12 +71,14 @@
 #define E1000_ICH_MNG_IAMT_MODE          0x2
 
 #define ID_LED_DEFAULT_ICH8LAN  ((ID_LED_DEF1_DEF2 << 12) | \
-                                 (ID_LED_DEF1_OFF2 <<  8) | \
-                                 (ID_LED_DEF1_ON2  <<  4) | \
+                                 (ID_LED_OFF1_OFF2 <<  8) | \
+                                 (ID_LED_OFF1_ON2  <<  4) | \
                                  (ID_LED_DEF1_DEF2))
 
 #define E1000_ICH_NVM_SIG_WORD           0x13
 #define E1000_ICH_NVM_SIG_MASK           0xC000
+#define E1000_ICH_NVM_VALID_SIG_MASK     0xC0
+#define E1000_ICH_NVM_SIG_VALUE          0x80
 
 #define E1000_ICH8_LAN_INIT_TIMEOUT      1500
 
@@ -99,6 +102,25 @@
 #define IGP3_VR_CTRL_MODE_SHUTDOWN           0x0200
 #define IGP3_PM_CTRL_FORCE_PWR_DOWN          0x0020
 
+/* PHY Wakeup Registers and defines */
+#define BM_RCTL         PHY_REG(BM_WUC_PAGE, 0)
+#define BM_WUC          PHY_REG(BM_WUC_PAGE, 1)
+#define BM_WUFC         PHY_REG(BM_WUC_PAGE, 2)
+#define BM_WUS          PHY_REG(BM_WUC_PAGE, 3)
+#define BM_RAR_L(_i)    (BM_PHY_REG(BM_WUC_PAGE, 16 + ((_i) << 2)))
+#define BM_RAR_M(_i)    (BM_PHY_REG(BM_WUC_PAGE, 17 + ((_i) << 2)))
+#define BM_RAR_H(_i)    (BM_PHY_REG(BM_WUC_PAGE, 18 + ((_i) << 2)))
+#define BM_RAR_CTRL(_i) (BM_PHY_REG(BM_WUC_PAGE, 19 + ((_i) << 2)))
+#define BM_MTA(_i)      (BM_PHY_REG(BM_WUC_PAGE, 128 + ((_i) << 1)))
+
+#define BM_RCTL_UPE           0x0001          /* Unicast Promiscuous Mode */
+#define BM_RCTL_MPE           0x0002          /* Multicast Promiscuous Mode */
+#define BM_RCTL_MO_SHIFT      3               /* Multicast Offset Shift */
+#define BM_RCTL_MO_MASK       (3 << 3)        /* Multicast Offset Mask */
+#define BM_RCTL_BAM           0x0020          /* Broadcast Accept Mode */
+#define BM_RCTL_PMCF          0x0040          /* Pass MAC Control Frames */
+#define BM_RCTL_RFCE          0x0080          /* Rx Flow Control Enable */
+
 /*
  * Additional interrupts need to be handled for ICH family:
  *  DSW = The FW changed the status of the DISSW bit in FWSM
@@ -128,5 +150,8 @@ void e1000_set_kmrn_lock_loss_workaround_ich8lan(struct e1000_hw *hw,
 void e1000_igp3_phy_powerdown_workaround_ich8lan(struct e1000_hw *hw);
 void e1000_gig_downshift_workaround_ich8lan(struct e1000_hw *hw);
 void e1000_disable_gig_wol_ich8lan(struct e1000_hw *hw);
+#if defined(HANKSVILLE_HW) && !defined(NO_PCH_A_SUPPORT)
+s32 e1000_hv_phy_powerdown_workaround_ich8lan(struct e1000_hw *hw);
+#endif
 
 #endif
diff --git a/sys/dev/e1000/e1000_mac.c b/sys/dev/e1000/e1000_mac.c
index fcec342..435d14e 100644
--- a/sys/dev/e1000/e1000_mac.c
+++ b/sys/dev/e1000/e1000_mac.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -35,6 +35,7 @@
 #include "e1000_api.h"
 
 static s32 e1000_validate_mdi_setting_generic(struct e1000_hw *hw);
+static void e1000_set_lan_id_multi_port_pcie(struct e1000_hw *hw);
 
 /**
  *  e1000_init_mac_ops_generic - Initialize MAC function pointers
@@ -126,7 +127,7 @@ bool e1000_null_mng_mode(struct e1000_hw *hw)
  *  e1000_null_update_mc - No-op function, return void
  *  @hw: pointer to the HW structure
  **/
-void e1000_null_update_mc(struct e1000_hw *hw, u8 *h, u32 a, u32 b, u32 c)
+void e1000_null_update_mc(struct e1000_hw *hw, u8 *h, u32 a)
 {
 	DEBUGFUNC("e1000_null_update_mc");
 	return;
@@ -261,18 +262,17 @@ s32 e1000_get_bus_info_pcie_generic(struct e1000_hw *hw)
  *  Determines the LAN function id by reading memory-mapped registers
  *  and swaps the port value if requested.
  **/
-void e1000_set_lan_id_multi_port_pcie(struct e1000_hw *hw)
+static void e1000_set_lan_id_multi_port_pcie(struct e1000_hw *hw)
 {
 	struct e1000_bus_info *bus = &hw->bus;
 	u32 reg;
 
+	/*
+	 * The status register reports the correct function number
+	 * for the device regardless of function swap state.
+	 */
 	reg = E1000_READ_REG(hw, E1000_STATUS);
 	bus->func = (reg & E1000_STATUS_FUNC_MASK) >> E1000_STATUS_FUNC_SHIFT;
-
-	/* check for a port swap */
-	reg = E1000_READ_REG(hw, E1000_FACTPS);
-	if (reg & E1000_FACTPS_LFS)
-		bus->func ^= 0x1;
 }
 
 /**
@@ -358,6 +358,7 @@ void e1000_write_vfta_generic(struct e1000_hw *hw, u32 offset, u32 value)
 void e1000_init_rx_addrs_generic(struct e1000_hw *hw, u16 rar_count)
 {
 	u32 i;
+	u8 mac_addr[ETH_ADDR_LEN] = {0};
 
 	DEBUGFUNC("e1000_init_rx_addrs_generic");
 
@@ -368,12 +369,8 @@ void e1000_init_rx_addrs_generic(struct e1000_hw *hw, u16 rar_count)
 
 	/* Zero out the other (rar_entry_count - 1) receive addresses */
 	DEBUGOUT1("Clearing RAR[1-%u]\n", rar_count-1);
-	for (i = 1; i < rar_count; i++) {
-		E1000_WRITE_REG_ARRAY(hw, E1000_RA, (i << 1), 0);
-		E1000_WRITE_FLUSH(hw);
-		E1000_WRITE_REG_ARRAY(hw, E1000_RA, ((i << 1) + 1), 0);
-		E1000_WRITE_FLUSH(hw);
-	}
+	for (i = 1; i < rar_count; i++)
+		hw->mac.ops.rar_set(hw, mac_addr, i);
 }
 
 /**
@@ -382,10 +379,11 @@ void e1000_init_rx_addrs_generic(struct e1000_hw *hw, u16 rar_count)
  *
  *  Checks the nvm for an alternate MAC address.  An alternate MAC address
  *  can be setup by pre-boot software and must be treated like a permanent
- *  address and must override the actual permanent MAC address.  If an
- *  alternate MAC address is found it is saved in the hw struct and
- *  programmed into RAR0 and the function returns success, otherwise the
- *  function returns an error.
+ *  address and must override the actual permanent MAC address. If an
+ *  alternate MAC address is found it is programmed into RAR0, replacing
+ *  the permanent address that was installed into RAR0 by the Si on reset.
+ *  This function will return SUCCESS unless it encounters an error while
+ *  reading the EEPROM.
  **/
 s32 e1000_check_alt_mac_addr_generic(struct e1000_hw *hw)
 {
@@ -404,13 +402,12 @@ s32 e1000_check_alt_mac_addr_generic(struct e1000_hw *hw)
 	}
 
 	if (nvm_alt_mac_addr_offset == 0xFFFF) {
-		ret_val = -(E1000_NOT_IMPLEMENTED);
+		/* There is no Alternate MAC Address */
 		goto out;
 	}
 
 	if (hw->bus.func == E1000_FUNC_1)
-		nvm_alt_mac_addr_offset += ETH_ADDR_LEN/sizeof(u16);
-
+		nvm_alt_mac_addr_offset += E1000_ALT_MAC_ADDRESS_OFFSET_LAN1;
 	for (i = 0; i < ETH_ADDR_LEN; i += 2) {
 		offset = nvm_alt_mac_addr_offset + (i >> 1);
 		ret_val = hw->nvm.ops.read(hw, offset, 1, &nvm_data);
@@ -425,14 +422,16 @@ s32 e1000_check_alt_mac_addr_generic(struct e1000_hw *hw)
 
 	/* if multicast bit is set, the alternate address will not be used */
 	if (alt_mac_addr[0] & 0x01) {
-		ret_val = -(E1000_NOT_IMPLEMENTED);
+		DEBUGOUT("Ignoring Alternate Mac Address with MC bit set\n");
 		goto out;
 	}
 
-	for (i = 0; i < ETH_ADDR_LEN; i++)
-		hw->mac.addr[i] = hw->mac.perm_addr[i] = alt_mac_addr[i];
-
-	hw->mac.ops.rar_set(hw, hw->mac.perm_addr, 0);
+	/*
+	 * We have a valid alternate MAC address, and we want to treat it the
+	 * same as the normal permanent MAC address stored by the HW into the
+	 * RAR. Do this by mapping this address into RAR0.
+	 */
+	hw->mac.ops.rar_set(hw, alt_mac_addr, 0);
 
 out:
 	return ret_val;
@@ -467,8 +466,15 @@ void e1000_rar_set_generic(struct e1000_hw *hw, u8 *addr, u32 index)
 	if (rar_low || rar_high)
 		rar_high |= E1000_RAH_AV;
 
+	/*
+	 * Some bridges will combine consecutive 32-bit writes into
+	 * a single burst write, which will malfunction on some parts.
+	 * The flushes avoid this.
+	 */
 	E1000_WRITE_REG(hw, E1000_RAL(index), rar_low);
+	E1000_WRITE_FLUSH(hw);
 	E1000_WRITE_REG(hw, E1000_RAH(index), rar_high);
+	E1000_WRITE_FLUSH(hw);
 }
 
 /**
@@ -512,55 +518,36 @@ void e1000_mta_set_generic(struct e1000_hw *hw, u32 hash_value)
  *  @hw: pointer to the HW structure
  *  @mc_addr_list: array of multicast addresses to program
  *  @mc_addr_count: number of multicast addresses to program
- *  @rar_used_count: the first RAR register free to program
- *  @rar_count: total number of supported Receive Address Registers
  *
- *  Updates the Receive Address Registers and Multicast Table Array.
+ *  Updates entire Multicast Table Array.
  *  The caller must have a packed mc_addr_list of multicast addresses.
- *  The parameter rar_count will usually be hw->mac.rar_entry_count
- *  unless there are workarounds that change this.
  **/
 void e1000_update_mc_addr_list_generic(struct e1000_hw *hw,
-                                       u8 *mc_addr_list, u32 mc_addr_count,
-                                       u32 rar_used_count, u32 rar_count)
+                                       u8 *mc_addr_list, u32 mc_addr_count)
 {
-	u32 hash_value;
-	u32 i;
+	u32 hash_value, hash_bit, hash_reg;
+	int i;
 
 	DEBUGFUNC("e1000_update_mc_addr_list_generic");
 
-	/*
-	 * Load the first set of multicast addresses into the exact
-	 * filters (RAR).  If there are not enough to fill the RAR
-	 * array, clear the filters.
-	 */
-	for (i = rar_used_count; i < rar_count; i++) {
-		if (mc_addr_count) {
-			hw->mac.ops.rar_set(hw, mc_addr_list, i);
-			mc_addr_count--;
-			mc_addr_list += ETH_ADDR_LEN;
-		} else {
-			E1000_WRITE_REG_ARRAY(hw, E1000_RA, i << 1, 0);
-			E1000_WRITE_FLUSH(hw);
-			E1000_WRITE_REG_ARRAY(hw, E1000_RA, (i << 1) + 1, 0);
-			E1000_WRITE_FLUSH(hw);
-		}
-	}
+	/* clear mta_shadow */
+	memset(&hw->mac.mta_shadow, 0, sizeof(hw->mac.mta_shadow));
 
-	/* Clear the old settings from the MTA */
-	DEBUGOUT("Clearing MTA\n");
-	for (i = 0; i < hw->mac.mta_reg_count; i++) {
-		E1000_WRITE_REG_ARRAY(hw, E1000_MTA, i, 0);
-		E1000_WRITE_FLUSH(hw);
-	}
-
-	/* Load any remaining multicast addresses into the hash table. */
-	for (; mc_addr_count > 0; mc_addr_count--) {
+	/* update mta_shadow from mc_addr_list */
+	for (i = 0; (u32) i < mc_addr_count; i++) {
 		hash_value = e1000_hash_mc_addr_generic(hw, mc_addr_list);
-		DEBUGOUT1("Hash value = 0x%03X\n", hash_value);
-		hw->mac.ops.mta_set(hw, hash_value);
-		mc_addr_list += ETH_ADDR_LEN;
+
+		hash_reg = (hash_value >> 5) & (hw->mac.mta_reg_count - 1);
+		hash_bit = hash_value & 0x1F;
+
+		hw->mac.mta_shadow[hash_reg] |= (1 << hash_bit);
+		mc_addr_list += (ETH_ADDR_LEN);
 	}
+
+	/* replace the entire MTA table */
+	for (i = hw->mac.mta_reg_count - 1; i >= 0; i--)
+		E1000_WRITE_REG_ARRAY(hw, E1000_MTA, i, hw->mac.mta_shadow[i]);
+	E1000_WRITE_FLUSH(hw);
 }
 
 /**
@@ -1022,7 +1009,7 @@ s32 e1000_setup_link_generic(struct e1000_hw *hw)
 	hw->fc.current_mode = hw->fc.requested_mode;
 
 	DEBUGOUT1("After fix-ups FlowControl is now = %x\n",
-	                                             hw->fc.current_mode);
+		hw->fc.current_mode);
 
 	/* Call the necessary media_type subroutine to configure the link. */
 	ret_val = hw->mac.ops.setup_physical_interface(hw);
diff --git a/sys/dev/e1000/e1000_mac.h b/sys/dev/e1000/e1000_mac.h
index 7f9baa9..b7a5b2c 100644
--- a/sys/dev/e1000/e1000_mac.h
+++ b/sys/dev/e1000/e1000_mac.h
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -44,7 +44,7 @@ void e1000_null_mac_generic(struct e1000_hw *hw);
 s32  e1000_null_ops_generic(struct e1000_hw *hw);
 s32  e1000_null_link_info(struct e1000_hw *hw, u16 *s, u16 *d);
 bool e1000_null_mng_mode(struct e1000_hw *hw);
-void e1000_null_update_mc(struct e1000_hw *hw, u8 *h, u32 a, u32 b, u32 c);
+void e1000_null_update_mc(struct e1000_hw *hw, u8 *h, u32 a);
 void e1000_null_write_vfta(struct e1000_hw *hw, u32 a, u32 b);
 void e1000_null_mta_set(struct e1000_hw *hw, u32 a);
 void e1000_null_rar_set(struct e1000_hw *hw, u8 *h, u32 a);
@@ -63,7 +63,6 @@ s32  e1000_get_bus_info_pci_generic(struct e1000_hw *hw);
 s32  e1000_get_bus_info_pcie_generic(struct e1000_hw *hw);
 void e1000_set_lan_id_single_port(struct e1000_hw *hw);
 void e1000_set_lan_id_multi_port_pci(struct e1000_hw *hw);
-void e1000_set_lan_id_multi_port_pcie(struct e1000_hw *hw);
 s32  e1000_get_hw_semaphore_generic(struct e1000_hw *hw);
 s32  e1000_get_speed_and_duplex_copper_generic(struct e1000_hw *hw, u16 *speed,
                                                u16 *duplex);
@@ -73,8 +72,7 @@ s32  e1000_id_led_init_generic(struct e1000_hw *hw);
 s32  e1000_led_on_generic(struct e1000_hw *hw);
 s32  e1000_led_off_generic(struct e1000_hw *hw);
 void e1000_update_mc_addr_list_generic(struct e1000_hw *hw,
-	                               u8 *mc_addr_list, u32 mc_addr_count,
-	                               u32 rar_used_count, u32 rar_count);
+	                               u8 *mc_addr_list, u32 mc_addr_count);
 s32  e1000_set_default_fc_generic(struct e1000_hw *hw);
 s32  e1000_set_fc_watermarks_generic(struct e1000_hw *hw);
 s32  e1000_setup_fiber_serdes_link_generic(struct e1000_hw *hw);
diff --git a/sys/dev/e1000/e1000_nvm.c b/sys/dev/e1000/e1000_nvm.c
index 8492d80..83557ef 100644
--- a/sys/dev/e1000/e1000_nvm.c
+++ b/sys/dev/e1000/e1000_nvm.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -34,6 +34,8 @@
 
 #include "e1000_api.h"
 
+static void e1000_reload_nvm_generic(struct e1000_hw *hw);
+
 /**
  *  e1000_init_nvm_ops_generic - Initialize NVM function pointers
  *  @hw: pointer to the HW structure
@@ -815,31 +817,23 @@ out:
  **/
 s32 e1000_read_mac_addr_generic(struct e1000_hw *hw)
 {
-	s32  ret_val = E1000_SUCCESS;
-	u16 offset, nvm_data, i;
+	u32 rar_high;
+	u32 rar_low;
+	u16 i;
 
-	DEBUGFUNC("e1000_read_mac_addr");
+	rar_high = E1000_READ_REG(hw, E1000_RAH(0));
+	rar_low = E1000_READ_REG(hw, E1000_RAL(0));
 
-	for (i = 0; i < ETH_ADDR_LEN; i += 2) {
-		offset = i >> 1;
-		ret_val = hw->nvm.ops.read(hw, offset, 1, &nvm_data);
-		if (ret_val) {
-			DEBUGOUT("NVM Read Error\n");
-			goto out;
-		}
-		hw->mac.perm_addr[i] = (u8)(nvm_data & 0xFF);
-		hw->mac.perm_addr[i+1] = (u8)(nvm_data >> 8);
-	}
+	for (i = 0; i < E1000_RAL_MAC_ADDR_LEN; i++)
+		hw->mac.perm_addr[i] = (u8)(rar_low >> (i*8));
 
-	/* Flip last bit of mac address if we're on second port */
-	if (hw->bus.func == E1000_FUNC_1)
-		hw->mac.perm_addr[5] ^= 1;
+	for (i = 0; i < E1000_RAH_MAC_ADDR_LEN; i++)
+		hw->mac.perm_addr[i+4] = (u8)(rar_high >> (i*8));
 
 	for (i = 0; i < ETH_ADDR_LEN; i++)
 		hw->mac.addr[i] = hw->mac.perm_addr[i];
 
-out:
-	return ret_val;
+	return E1000_SUCCESS;
 }
 
 /**
@@ -916,7 +910,7 @@ out:
  *  Reloads the EEPROM by setting the "Reinitialize from EEPROM" bit in the
  *  extended control register.
  **/
-void e1000_reload_nvm_generic(struct e1000_hw *hw)
+static void e1000_reload_nvm_generic(struct e1000_hw *hw)
 {
 	u32 ctrl_ext;
 
diff --git a/sys/dev/e1000/e1000_nvm.h b/sys/dev/e1000/e1000_nvm.h
index d0ab33c..974c407 100644
--- a/sys/dev/e1000/e1000_nvm.h
+++ b/sys/dev/e1000/e1000_nvm.h
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -61,7 +61,6 @@ s32  e1000_write_nvm_spi(struct e1000_hw *hw, u16 offset, u16 words,
 s32  e1000_update_nvm_checksum_generic(struct e1000_hw *hw);
 void e1000_stop_nvm(struct e1000_hw *hw);
 void e1000_release_nvm_generic(struct e1000_hw *hw);
-void e1000_reload_nvm_generic(struct e1000_hw *hw);
 
 #define E1000_STM_OPCODE  0xDB00
 
diff --git a/sys/dev/e1000/e1000_phy.c b/sys/dev/e1000/e1000_phy.c
index 2e4d046..751edbb 100644
--- a/sys/dev/e1000/e1000_phy.c
+++ b/sys/dev/e1000/e1000_phy.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -237,6 +237,12 @@ s32 e1000_read_phy_reg_mdic(struct e1000_hw *hw, u32 offset, u16 *data)
 
 	E1000_WRITE_REG(hw, E1000_MDIC, mdic);
 
+#if defined(HANKSVILLE_HW) && !defined(NO_PCH_A_SUPPORT)
+	/* Workaround for Si errata */
+	if ((hw->phy.type == e1000_phy_lsi) && (hw->revision_id <= 2 ))
+		msec_delay(10);
+
+#endif /* HANKSVILLE_HW && !NO_PCH_A_SUPPORT */
 	/*
 	 * Poll the ready bit to see if the MDI read completed
 	 * Increasing the time out as testing showed failures with
@@ -292,6 +298,12 @@ s32 e1000_write_phy_reg_mdic(struct e1000_hw *hw, u32 offset, u16 data)
 
 	E1000_WRITE_REG(hw, E1000_MDIC, mdic);
 
+#if defined(HANKSVILLE_HW) && !defined(NO_PCH_A_SUPPORT)
+	/* Workaround for Si errata */
+	if ((hw->phy.type == e1000_phy_lsi) && (hw->revision_id <= 2))
+		msec_delay(10);
+
+#endif /* HANKSVILLE_HW && !NO_PCH_A_SUPPORT */
 	/*
 	 * Poll the ready bit to see if the MDI read completed
 	 * Increasing the time out as testing showed failures with
@@ -563,8 +575,8 @@ s32 e1000_copper_link_setup_m88(struct e1000_hw *hw)
 	if (ret_val)
 		goto out;
 
-	/* For newer PHYs this bit is downshift enable */
-	if (phy->type == e1000_phy_m88)
+	/* For BM PHY this bit is downshift enable */
+	if (phy->type != e1000_phy_bm)
 		phy_data |= M88E1000_PSCR_ASSERT_CRS_ON_TX;
 
 	/*
@@ -1670,16 +1682,16 @@ s32 e1000_get_cable_length_m88(struct e1000_hw *hw)
 
 	index = (phy_data & M88E1000_PSSR_CABLE_LENGTH) >>
 	        M88E1000_PSSR_CABLE_LENGTH_SHIFT;
-	if (index < M88E1000_CABLE_LENGTH_TABLE_SIZE + 1) {
-		phy->min_cable_length = e1000_m88_cable_length_table[index];
-		phy->max_cable_length = e1000_m88_cable_length_table[index+1];
-
-		phy->cable_length = (phy->min_cable_length +
-		                     phy->max_cable_length) / 2;
-	} else {
+	if (index >= M88E1000_CABLE_LENGTH_TABLE_SIZE + 1) {
 		ret_val = E1000_ERR_PHY;
+		goto out;
 	}
 
+	phy->min_cable_length = e1000_m88_cable_length_table[index];
+	phy->max_cable_length = e1000_m88_cable_length_table[index+1];
+
+	phy->cable_length = (phy->min_cable_length + phy->max_cable_length) / 2;
+
 out:
 	return ret_val;
 }
@@ -2140,6 +2152,8 @@ s32 e1000_determine_phy_address(struct e1000_hw *hw)
 	u32 i;
 	enum e1000_phy_type phy_type = e1000_phy_unknown;
 
+	hw->phy.id = phy_type;
+
 	for (phy_addr = 0; phy_addr < E1000_MAX_PHY_ADDR; phy_addr++) {
 		hw->phy.addr = phy_addr;
 		i = 0;
@@ -2427,11 +2441,11 @@ static s32 e1000_access_phy_wakeup_reg_bm(struct e1000_hw *hw, u32 offset,
                                           u16 *data, bool read)
 {
 	s32 ret_val;
-	u16 reg = ((u16)offset);
+	u16 reg = BM_PHY_REG_NUM(offset);
 	u16 phy_reg = 0;
 	u8  phy_acquired = 1;
 
-	DEBUGFUNC("e1000_read_phy_wakeup_reg_bm");
+	DEBUGFUNC("e1000_access_phy_wakeup_reg_bm");
 
 	ret_val = hw->phy.ops.acquire(hw);
 	if (ret_val) {
@@ -2484,15 +2498,15 @@ static s32 e1000_access_phy_wakeup_reg_bm(struct e1000_hw *hw, u32 offset,
 	if (read) {
 	        /* Read the page 800 value using opcode 0x12 */
 		ret_val = e1000_read_phy_reg_mdic(hw, BM_WUC_DATA_OPCODE,
-							data);
+		                                  data);
 	} else {
-	        /* Read the page 800 value using opcode 0x12 */
+	        /* Write the page 800 value using opcode 0x12 */
 		ret_val = e1000_write_phy_reg_mdic(hw, BM_WUC_DATA_OPCODE,
-							*data);
+		                                   *data);
 	}
 
 	if (ret_val) {
-		DEBUGOUT("Could not read data value from page 800\n");
+		DEBUGOUT("Could not access data value from page 800\n");
 		goto out;
 	}
 
diff --git a/sys/dev/e1000/e1000_phy.h b/sys/dev/e1000/e1000_phy.h
index 06bcaf0..3e52368 100644
--- a/sys/dev/e1000/e1000_phy.h
+++ b/sys/dev/e1000/e1000_phy.h
@@ -1,6 +1,6 @@
-/*****************************************************************************
+/******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -99,14 +99,29 @@ s32 e1000_write_phy_reg_mdic(struct e1000_hw *hw, u32 offset, u16 data);
 #define IGP_PAGE_SHIFT                    5
 #define PHY_REG_MASK                      0x1F
 
+/* BM/HV Specific Registers */
+#define BM_PORT_CTRL_PAGE                 769
+#define BM_PCIE_PAGE                      770
 #define BM_WUC_PAGE                       800
 #define BM_WUC_ADDRESS_OPCODE             0x11
 #define BM_WUC_DATA_OPCODE                0x12
-#define BM_WUC_ENABLE_PAGE                769
+#define BM_WUC_ENABLE_PAGE                BM_PORT_CTRL_PAGE
 #define BM_WUC_ENABLE_REG                 17
 #define BM_WUC_ENABLE_BIT                 (1 << 2)
 #define BM_WUC_HOST_WU_BIT                (1 << 4)
 
+#define PHY_UPPER_SHIFT                   21
+#define BM_PHY_REG(page, reg) \
+	(((reg) & MAX_PHY_REG_ADDRESS) |\
+	 (((page) & 0xFFFF) << PHY_PAGE_SHIFT) |\
+	 (((reg) & ~MAX_PHY_REG_ADDRESS) << (PHY_UPPER_SHIFT - PHY_PAGE_SHIFT)))
+#define BM_PHY_REG_PAGE(offset) \
+	((u16)(((offset) >> PHY_PAGE_SHIFT) & 0xFFFF))
+#define BM_PHY_REG_NUM(offset) \
+	((u16)(((offset) & MAX_PHY_REG_ADDRESS) |\
+	 (((offset) >> (PHY_UPPER_SHIFT - PHY_PAGE_SHIFT)) &\
+		~MAX_PHY_REG_ADDRESS)))
+
 /* BM PHY Copper Specific Control 1 */
 #define BM_CS_CTRL1                       16
 #define BM_CS_CTRL1_ENERGY_DETECT         0x0300 /* Enable Energy Detect */
diff --git a/sys/dev/e1000/e1000_regs.h b/sys/dev/e1000/e1000_regs.h
index 1dbd56b..ac673a7 100644
--- a/sys/dev/e1000/e1000_regs.h
+++ b/sys/dev/e1000/e1000_regs.h
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -62,6 +62,7 @@
 #define E1000_FCTTV    0x00170  /* Flow Control Transmit Timer Value - RW */
 #define E1000_TXCW     0x00178  /* Tx Configuration Word - RW */
 #define E1000_RXCW     0x00180  /* Rx Configuration Word - RO */
+#define E1000_PBA_ECC  0x01100  /* PBA ECC Register */
 #define E1000_EICR     0x01580  /* Ext. Interrupt Cause Read - R/clr */
 #define E1000_EITR(_n) (0x01680 + (0x4 * (_n)))
 #define E1000_EICS     0x01520  /* Ext. Interrupt Cause Set - W0 */
@@ -269,15 +270,6 @@
 #define E1000_ICRXDMTC 0x04120  /* Interrupt Cause Rx Desc Min Thresh Count */
 #define E1000_ICRXOC   0x04124  /* Interrupt Cause Receiver Overrun Count */
 
-#define E1000_VFGPRC   0x00F10
-#define E1000_VFGORC   0x00F18
-#define E1000_VFMPRC   0x00F3C
-#define E1000_VFGPTC   0x00F14
-#define E1000_VFGOTC   0x00F34
-#define E1000_VFGOTLBC 0x00F50
-#define E1000_VFGPTLBC 0x00F44
-#define E1000_VFGORLBC 0x00F48
-#define E1000_VFGPRLBC 0x00F40
 #define E1000_LSECTXUT        0x04300  /* LinkSec Tx Untagged Packet Count - OutPktsUntagged */
 #define E1000_LSECTXPKTE      0x04304  /* LinkSec Encrypted Tx Packets Count - OutPktsEncrypted */
 #define E1000_LSECTXPKTP      0x04308  /* LinkSec Protected Tx Packet Count - OutPktsProtected */
@@ -387,6 +379,7 @@
 #define E1000_GIOCTL      0x05B44 /* GIO Analog Control Register */
 #define E1000_SCCTL       0x05B4C /* PCIc PLL Configuration Register */
 #define E1000_GCR         0x05B00 /* PCI-Ex Control */
+#define E1000_GCR2        0x05B64 /* PCI-Ex Control #2 */
 #define E1000_GSCL_1    0x05B10 /* PCI-Ex Statistic Control #1 */
 #define E1000_GSCL_2    0x05B14 /* PCI-Ex Statistic Control #2 */
 #define E1000_GSCL_3    0x05B18 /* PCI-Ex Statistic Control #3 */
@@ -429,7 +422,6 @@
 #define E1000_VFTE      0x00C90 /* VF Transmit Enables */
 #define E1000_QDE       0x02408 /* Queue Drop Enable - RW */
 #define E1000_DTXSWC    0x03500 /* DMA Tx Switch Control - RW */
-#define E1000_VLVF      0x05D00 /* VLAN Virtual Machine Filter - RW */
 #define E1000_RPLOLR    0x05AF0 /* Replication Offload - RW */
 #define E1000_UTA       0x0A000 /* Unicast Table Array - RW */
 #define E1000_IOVTCL    0x05BBC /* IOV Control Register */
@@ -440,6 +432,8 @@
 #define E1000_VMBMEM(_n)       (0x00800 + (64 * (_n)))
 #define E1000_VFVMBMEM(_n)     (0x00800 + (_n))
 #define E1000_VMOLR(_n)        (0x05AD0 + (4 * (_n)))
+#define E1000_VLVF(_n)         (0x05D00 + (4 * (_n))) /* VLAN Virtual Machine
+                                                       * Filter - RW */
 /* Time Sync */
 #define E1000_TSYNCRXCTL 0x0B620 /* Rx Time Sync Control register - RW */
 #define E1000_TSYNCTXCTL 0x0B614 /* Tx Time Sync Control register - RW */
diff --git a/sys/dev/e1000/if_em.c b/sys/dev/e1000/if_em.c
index f0cd60f..df54a53 100644
--- a/sys/dev/e1000/if_em.c
+++ b/sys/dev/e1000/if_em.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -93,7 +93,7 @@ int	em_display_debug_stats = 0;
 /*********************************************************************
  *  Driver version:
  *********************************************************************/
-char em_driver_version[] = "6.9.6";
+char em_driver_version[] = "6.9.9";
 
 
 /*********************************************************************
@@ -278,10 +278,8 @@ static void	em_set_multi(struct adapter *);
 static void	em_print_hw_stats(struct adapter *);
 static void	em_update_link_status(struct adapter *);
 static int	em_get_buf(struct adapter *, int);
-
 static void	em_register_vlan(void *, struct ifnet *, u16);
 static void	em_unregister_vlan(void *, struct ifnet *, u16);
-
 static int	em_xmit(struct adapter *, struct mbuf **);
 static void	em_smartspeed(struct adapter *);
 static int	em_82547_fifo_workaround(struct adapter *, int);
@@ -322,17 +320,19 @@ static void	em_irq_fast(void *);
 #else
 static int	em_irq_fast(void *);
 #endif
+
 /* MSIX handlers */
 static void	em_msix_tx(void *);
 static void	em_msix_rx(void *);
 static void	em_msix_link(void *);
-static void	em_add_rx_process_limit(struct adapter *, const char *,
-		    const char *, int *, int);
-static void	em_handle_rxtx(void *context, int pending);
 static void	em_handle_rx(void *context, int pending);
 static void	em_handle_tx(void *context, int pending);
+
+static void	em_handle_rxtx(void *context, int pending);
 static void	em_handle_link(void *context, int pending);
-#endif /* EM_LEGACY_IRQ */
+static void	em_add_rx_process_limit(struct adapter *, const char *,
+		    const char *, int *, int);
+#endif /* ~EM_LEGACY_IRQ */
 
 #ifdef DEVICE_POLLING
 static poll_handler_t em_poll;
@@ -514,8 +514,8 @@ em_attach(device_t dev)
 	** identified
 	*/
 	if ((adapter->hw.mac.type == e1000_ich8lan) ||
-	    (adapter->hw.mac.type == e1000_ich10lan) ||
-	    (adapter->hw.mac.type == e1000_ich9lan)) {
+	    (adapter->hw.mac.type == e1000_ich9lan) ||
+	    (adapter->hw.mac.type == e1000_ich10lan)) {
 		int rid = EM_BAR_TYPE_FLASH;
 		adapter->flash = bus_alloc_resource_any(dev,
 		    SYS_RES_MEMORY, &rid, RF_ACTIVE);
@@ -644,6 +644,13 @@ em_attach(device_t dev)
 	adapter->rx_desc_base =
 	    (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr;
 
+	/*
+	** Start from a known state, this is
+	** important in reading the nvm and
+	** mac from that.
+	*/
+	e1000_reset_hw(&adapter->hw);
+
 	/* Make sure we have a good EEPROM before we read from it */
 	if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 		/*
@@ -659,13 +666,6 @@ em_attach(device_t dev)
 		}
 	}
 
-	/* Initialize the hardware */
-	if (em_hardware_init(adapter)) {
-		device_printf(dev, "Unable to initialize the hardware\n");
-		error = EIO;
-		goto err_hw_init;
-	}
-
 	/* Copy the permanent MAC address out of the EEPROM */
 	if (e1000_read_mac_addr(&adapter->hw) < 0) {
 		device_printf(dev, "EEPROM read error while reading MAC"
@@ -680,6 +680,13 @@ em_attach(device_t dev)
 		goto err_hw_init;
 	}
 
+	/* Initialize the hardware */
+	if (em_hardware_init(adapter)) {
+		device_printf(dev, "Unable to initialize the hardware\n");
+		error = EIO;
+		goto err_hw_init;
+	}
+
 	/* Allocate transmit descriptors and buffers */
 	if (em_allocate_transmit_structures(adapter)) {
 		device_printf(dev, "Could not setup transmit structures\n");
@@ -1463,8 +1470,6 @@ em_init_locked(struct adapter *adapter)
 	/* Setup VLAN support, basic and offload if available */
 	E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
 
-	/* New register interface replaces this but
-	   waiting on kernel support to be added */
 	if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) &&
 	    ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)) {
 		u32 ctrl;
@@ -1473,6 +1478,7 @@ em_init_locked(struct adapter *adapter)
 		E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
 	}
 
+
 	/* Set hardware offload abilities */
 	ifp->if_hwassist = 0;
 	if (adapter->hw.mac.type >= e1000_82543) {
@@ -1622,49 +1628,35 @@ em_intr(void *arg)
 		return;
 
 	EM_CORE_LOCK(adapter);
-	for (;;) {
-		reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
-
-		if (adapter->hw.mac.type >= e1000_82571 &&
-	    	    (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
-			break;
-		else if (reg_icr == 0)
-			break;
-
-		/*
-		 * XXX: some laptops trigger several spurious interrupts
-		 * on em(4) when in the resume cycle. The ICR register
-		 * reports all-ones value in this case. Processing such
-		 * interrupts would lead to a freeze. I don't know why.
-		 */
-		if (reg_icr == 0xffffffff)
-			break;
+	reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
+	if ((reg_icr == 0xffffffff) || (reg_icr == 0)||
+	    (adapter->hw.mac.type >= e1000_82571 &&
+	    (reg_icr & E1000_ICR_INT_ASSERTED) == 0))
+			goto out;
 
-		EM_CORE_UNLOCK(adapter);
-		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
-			em_rxeof(adapter, -1);
-			EM_TX_LOCK(adapter);
-			em_txeof(adapter);
-			EM_TX_UNLOCK(adapter);
-		}
-		EM_CORE_LOCK(adapter);
+	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
+			goto out;
 
-		/* Link status change */
-		if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
-			callout_stop(&adapter->timer);
-			adapter->hw.mac.get_link_status = 1;
-			em_update_link_status(adapter);
-			/* Deal with TX cruft when link lost */
-			em_tx_purge(adapter);
-			callout_reset(&adapter->timer, hz,
-			    em_local_timer, adapter);
-		}
+	EM_TX_LOCK(adapter);
+	em_txeof(adapter);
+	em_rxeof(adapter, -1);
+	em_txeof(adapter);
+	EM_TX_UNLOCK(adapter);
 
-		if (reg_icr & E1000_ICR_RXO)
-			adapter->rx_overruns++;
+	if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
+		callout_stop(&adapter->timer);
+		adapter->hw.mac.get_link_status = 1;
+		em_update_link_status(adapter);
+		/* Deal with TX cruft when link lost */
+		em_tx_purge(adapter);
+		callout_reset(&adapter->timer, hz,
+		    em_local_timer, adapter);
 	}
-	EM_CORE_UNLOCK(adapter);
 
+	if (reg_icr & E1000_ICR_RXO)
+		adapter->rx_overruns++;
+out:
+	EM_CORE_UNLOCK(adapter);
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
 	    !IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 		em_start(ifp);
@@ -1713,33 +1705,6 @@ em_handle_rxtx(void *context, int pending)
 	em_enable_intr(adapter);
 }
 
-static void
-em_handle_rx(void *context, int pending)
-{
-	struct adapter	*adapter = context;
-	struct ifnet	*ifp = adapter->ifp;
-
-	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) &&
-	    (em_rxeof(adapter, adapter->rx_process_limit) != 0))
-		taskqueue_enqueue(adapter->tq, &adapter->rx_task);
-
-}
-
-static void
-em_handle_tx(void *context, int pending)
-{
-	struct adapter	*adapter = context;
-	struct ifnet	*ifp = adapter->ifp;
-
-	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
-		EM_TX_LOCK(adapter);
-		em_txeof(adapter);
-		if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
-			em_start_locked(ifp);
-		EM_TX_UNLOCK(adapter);
-	}
-}
-
 /*********************************************************************
  *
  *  Fast Legacy/MSI Combined Interrupt Service routine  
@@ -1868,6 +1833,33 @@ em_msix_link(void *arg)
 	    EM_MSIX_LINK | E1000_IMS_LSC);
 	return;
 }
+
+static void
+em_handle_rx(void *context, int pending)
+{
+	struct adapter	*adapter = context;
+	struct ifnet	*ifp = adapter->ifp;
+
+	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) &&
+	    (em_rxeof(adapter, adapter->rx_process_limit) != 0))
+		taskqueue_enqueue(adapter->tq, &adapter->rx_task);
+
+}
+
+static void
+em_handle_tx(void *context, int pending)
+{
+	struct adapter	*adapter = context;
+	struct ifnet	*ifp = adapter->ifp;
+
+	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+		EM_TX_LOCK(adapter);
+		em_txeof(adapter);
+		if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
+			em_start_locked(ifp);
+		EM_TX_UNLOCK(adapter);
+	}
+}
 #endif /* EM_FAST_IRQ */
 
 /*********************************************************************
@@ -2468,7 +2460,7 @@ em_set_multi(struct adapter *adapter)
 	struct ifnet	*ifp = adapter->ifp;
 	struct ifmultiaddr *ifma;
 	u32 reg_rctl = 0;
-	u8  mta[512]; /* Largest MTS is 4096 bits */
+	u8  *mta; /* Multicast array memory */
 	int mcnt = 0;
 
 	IOCTL_DEBUGOUT("em_set_multi: begin");
@@ -2483,6 +2475,13 @@ em_set_multi(struct adapter *adapter)
 		msec_delay(5);
 	}
 
+	/* Allocate temporary memory to setup array */
+	mta = malloc(sizeof(u8) *
+	    (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES),
+	    M_DEVBUF, M_NOWAIT | M_ZERO);
+	if (mta == NULL)
+		panic("em_set_multi memory failure\n");
+
 	IF_ADDR_LOCK(ifp);
 	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 		if (ifma->ifma_addr->sa_family != AF_LINK)
@@ -2502,8 +2501,7 @@ em_set_multi(struct adapter *adapter)
 		reg_rctl |= E1000_RCTL_MPE;
 		E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
 	} else
-		e1000_update_mc_addr_list(&adapter->hw, mta,
-		    mcnt, 1, adapter->hw.mac.rar_entry_count);
+		e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
 
 	if (adapter->hw.mac.type == e1000_82542 && 
 	    adapter->hw.revision_id == E1000_REVISION_2) {
@@ -2514,6 +2512,7 @@ em_set_multi(struct adapter *adapter)
 		if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
 			e1000_pci_set_mwi(&adapter->hw);
 	}
+	free(mta, M_DEVBUF);
 }
 
 
@@ -2925,6 +2924,7 @@ em_allocate_msix(struct adapter *adapter)
 	return (0);
 }
 
+
 static void
 em_free_pci_resources(struct adapter *adapter)
 {
@@ -2973,7 +2973,7 @@ em_free_pci_resources(struct adapter *adapter)
 }
 
 /*
- * Setup MSI/X
+ * Setup MSI or MSI/X
  */
 static int
 em_setup_msix(struct adapter *adapter)
diff --git a/sys/dev/e1000/if_igb.c b/sys/dev/e1000/if_igb.c
index 0f7dd7c..919cab1 100644
--- a/sys/dev/e1000/if_igb.c
+++ b/sys/dev/e1000/if_igb.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -34,7 +34,6 @@
 
 #ifdef HAVE_KERNEL_OPTION_HEADERS
 #include "opt_device_polling.h"
-#include "opt_inet.h"
 #endif
 
 #include <sys/param.h>
@@ -95,7 +94,7 @@ int	igb_display_debug_stats = 0;
 /*********************************************************************
  *  Driver version:
  *********************************************************************/
-char igb_driver_version[] = "version - 1.4.1";
+char igb_driver_version[] = "version - 1.5.2";
 
 
 /*********************************************************************
@@ -116,8 +115,11 @@ static igb_vendor_info_t igb_vendor_info_array[] =
 	{ 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER,
 						PCI_ANY_ID, PCI_ANY_ID, 0},
 	{ 0x8086, E1000_DEV_ID_82576,		PCI_ANY_ID, PCI_ANY_ID, 0},
+	{ 0x8086, E1000_DEV_ID_82576_NS,	PCI_ANY_ID, PCI_ANY_ID, 0},
 	{ 0x8086, E1000_DEV_ID_82576_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
 	{ 0x8086, E1000_DEV_ID_82576_SERDES,	PCI_ANY_ID, PCI_ANY_ID, 0},
+	{ 0x8086, E1000_DEV_ID_82576_QUAD_COPPER,
+						PCI_ANY_ID, PCI_ANY_ID, 0},
 	/* required last entry */
 	{ 0, 0, 0, 0, 0}
 };
@@ -152,7 +154,9 @@ static void	igb_identify_hardware(struct adapter *);
 static int	igb_allocate_pci_resources(struct adapter *);
 static int	igb_allocate_msix(struct adapter *);
 static int	igb_allocate_legacy(struct adapter *);
+#if __FreeBSD_version >= 602105
 static int	igb_setup_msix(struct adapter *);
+#endif
 static void	igb_free_pci_resources(struct adapter *);
 static void	igb_local_timer(void *);
 static int	igb_hardware_init(struct adapter *);
@@ -179,10 +183,7 @@ static void	igb_disable_intr(struct adapter *);
 static void	igb_update_stats_counters(struct adapter *);
 static bool	igb_txeof(struct tx_ring *);
 static bool	igb_rxeof(struct rx_ring *, int);
-#ifndef __NO_STRICT_ALIGNMENT
-static int	igb_fixup_rx(struct rx_ring *);
-#endif
-static void	igb_rx_checksum(u32, struct mbuf *);
+static void	igb_rx_checksum(u32, struct mbuf *, bool);
 static int	igb_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 static bool	igb_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
 static void	igb_set_promisc(struct adapter *);
@@ -190,11 +191,9 @@ static void	igb_disable_promisc(struct adapter *);
 static void	igb_set_multi(struct adapter *);
 static void	igb_print_hw_stats(struct adapter *);
 static void	igb_update_link_status(struct adapter *);
-static int	igb_get_buf(struct rx_ring *, int);
-
+static int	igb_get_buf(struct rx_ring *, int, u8);
 static void	igb_register_vlan(void *, struct ifnet *, u16);
 static void	igb_unregister_vlan(void *, struct ifnet *, u16);
-
 static int	igb_xmit(struct tx_ring *, struct mbuf **);
 static int	igb_dma_malloc(struct adapter *, bus_size_t,
 		    struct igb_dma_alloc *, int);
@@ -217,18 +216,24 @@ static int igb_tsync_init(struct adapter *);
 static void igb_tsync_disable(struct adapter *);
 #endif
 
+#if __FreeBSD_version > 700000
 static int	igb_irq_fast(void *);
+#else
+static void	igb_irq_fast(void *);
+#endif
+
 static void	igb_add_rx_process_limit(struct adapter *, const char *,
 		    const char *, int *, int);
 static void	igb_handle_rxtx(void *context, int pending);
 static void	igb_handle_tx(void *context, int pending);
 static void	igb_handle_rx(void *context, int pending);
-static void	igb_handle_link(void *context, int pending);
 
+#if __FreeBSD_version >= 602105
 /* These are MSIX only irq handlers */
 static void	igb_msix_rx(void *);
 static void	igb_msix_tx(void *);
 static void	igb_msix_link(void *);
+#endif
 
 /* Adaptive Interrupt Moderation */
 static void	igb_update_aim(struct rx_ring *);
@@ -271,8 +276,12 @@ TUNABLE_INT("hw.igb.txd", &igb_txd);
 ** These parameters are used in Adaptive
 ** Interrupt Moderation. The value is set
 ** into EITR and controls the interrupt
-** frequency. They can be modified but
-** be careful in tuning them.
+** frequency. A variable static scheme can
+** be created by changing the assigned value
+** of igb_ave_latency to the desired value,
+** and then set igb_enable_aim to FALSE.
+** This will result in all EITR registers
+** getting set to that value statically.
 */
 static int igb_enable_aim = TRUE;
 TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim);
@@ -289,7 +298,7 @@ TUNABLE_INT("hw.igb.bulk_latency", &igb_bulk_latency);
 ** 0, it will then be based on number of cpus.
 */
 static int igb_tx_queues = 1;
-static int igb_rx_queues = 4;
+static int igb_rx_queues = 1;
 TUNABLE_INT("hw.igb.tx_queues", &igb_tx_queues);
 TUNABLE_INT("hw.igb.rx_queues", &igb_rx_queues);
 
@@ -297,8 +306,8 @@ TUNABLE_INT("hw.igb.rx_queues", &igb_rx_queues);
 static int igb_rx_process_limit = 100;
 TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit);
 
-/* Flow control setting - default to none */
-static int igb_fc_setting = 0;
+/* Flow control setting - default to FULL */
+static int igb_fc_setting = e1000_fc_full;
 TUNABLE_INT("hw.igb.fc_setting", &igb_fc_setting);
 
 /*
@@ -310,6 +319,12 @@ TUNABLE_INT("hw.igb.fc_setting", &igb_fc_setting);
 static int igb_enable_lro = 1;
 TUNABLE_INT("hw.igb.enable_lro", &igb_enable_lro);
 
+/*
+ * Enable RX Header Split
+ */
+static int igb_rx_hdr_split = 1;
+TUNABLE_INT("hw.igb.rx_hdr_split", &igb_rx_hdr_split);
+
 extern int mp_ncpus;
 /*********************************************************************
  *  Device identification routine
@@ -426,6 +441,11 @@ igb_attach(device_t dev)
 	    OID_AUTO, "bulk_latency", CTLTYPE_INT|CTLFLAG_RW,
 	    &igb_bulk_latency, 1, "Bulk Latency");
  
+	SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
+	    SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
+	    OID_AUTO, "hdr_split", CTLTYPE_INT|CTLFLAG_RW,
+	    &igb_rx_hdr_split, 0, "RX Header Split");
+
 	callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 
 	/* Determine hardware and mac info */
@@ -475,7 +495,6 @@ igb_attach(device_t dev)
 	adapter->hw.mac.autoneg = DO_AUTO_NEG;
 	adapter->hw.phy.autoneg_wait_to_complete = FALSE;
 	adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
-	adapter->rx_buffer_len = 2048;
 
 	/* Copper options */
 	if (adapter->hw.phy.media_type == e1000_media_type_copper) {
@@ -499,6 +518,13 @@ igb_attach(device_t dev)
 		goto err_pci;
 	}
 
+	/*
+	** Start from a known state, this is
+	** important in reading the nvm and
+	** mac from that.
+	*/
+	e1000_reset_hw(&adapter->hw);
+
 	/* Make sure we have a good EEPROM before we read from it */
 	if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 		/*
@@ -514,27 +540,29 @@ igb_attach(device_t dev)
 		}
 	}
 
-	/* Initialize the hardware */
-	if (igb_hardware_init(adapter)) {
-		device_printf(dev, "Unable to initialize the hardware\n");
-		error = EIO;
-		goto err_late;
-	}
-
-	/* Copy the permanent MAC address out of the EEPROM */
+	/*
+	** Copy the permanent MAC address out of the EEPROM
+	*/
 	if (e1000_read_mac_addr(&adapter->hw) < 0) {
 		device_printf(dev, "EEPROM read error while reading MAC"
 		    " address\n");
 		error = EIO;
 		goto err_late;
 	}
-
+	/* Check its sanity */
 	if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
 		device_printf(dev, "Invalid MAC address\n");
 		error = EIO;
 		goto err_late;
 	}
 
+	/* Now Initialize the hardware */
+	if (igb_hardware_init(adapter)) {
+		device_printf(dev, "Unable to initialize the hardware\n");
+		error = EIO;
+		goto err_late;
+	}
+
 	/* 
 	** Configure Interrupts
 	*/
@@ -613,7 +641,11 @@ igb_detach(device_t dev)
 	INIT_DEBUGOUT("igb_detach: begin");
 
 	/* Make sure VLANS are not using driver */
+#if __FreeBSD_version >= 700000
 	if (adapter->ifp->if_vlantrunk != NULL) {
+#else
+	if (adapter->ifp->if_nvlans != 0) {
+#endif
 		device_printf(dev,"Vlan in use, detach first\n");
 		return (EBUSY);
 	}
@@ -919,13 +951,11 @@ igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 			reinit = 1;
 		}
-		if (mask & IFCAP_VLAN_HWFILTER) {
-			ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
-			reinit = 1;
-		}
 		if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
 			igb_init(adapter);
+#if __FreeBSD_version >= 700000
 		VLAN_CAPABILITIES(ifp);
+#endif
 		break;
 	    }
 
@@ -1124,7 +1154,6 @@ igb_init_locked(struct adapter *adapter)
 
 	E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
 
-	/* Vlan's enabled but HW Filtering off */
 	if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) &&
 	    ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)) {
 		u32 ctrl;
@@ -1135,8 +1164,14 @@ igb_init_locked(struct adapter *adapter)
 
 	/* Set hardware offload abilities */
 	ifp->if_hwassist = 0;
-	if (ifp->if_capenable & IFCAP_TXCSUM)
+	if (ifp->if_capenable & IFCAP_TXCSUM) {
 		ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
+#if __FreeBSD_version >= 800000
+		if (adapter->hw.mac.type == e1000_82576)
+			ifp->if_hwassist |= CSUM_SCTP;
+#endif
+	}
+
 	if (ifp->if_capenable & IFCAP_TSO4)
 		ifp->if_hwassist |= CSUM_TSO;
 
@@ -1150,6 +1185,15 @@ igb_init_locked(struct adapter *adapter)
 	/* Setup Multicast table */
 	igb_set_multi(adapter);
 
+	/*
+	** Figure out the desired mbuf pool
+	** for doing jumbo/packetsplit
+	*/
+	if (ifp->if_mtu > ETHERMTU)
+		adapter->rx_mbuf_sz = MJUMPAGESIZE;
+	else
+		adapter->rx_mbuf_sz = MCLBYTES;
+
 	/* Prepare receive descriptors and buffers */
 	if (igb_setup_receive_structures(adapter)) {
 		device_printf(dev, "Could not setup receive structures\n");
@@ -1211,24 +1255,6 @@ igb_init(void *arg)
 
 
 static void
-igb_handle_link(void *context, int pending)
-{
-	struct adapter	*adapter = context;
-	struct ifnet *ifp;
-
-	ifp = adapter->ifp;
-
-	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
-		return;
-
-	IGB_CORE_LOCK(adapter);
-	callout_stop(&adapter->timer);
-	igb_update_link_status(adapter);
-	callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
-	IGB_CORE_UNLOCK(adapter);
-}
-
-static void
 igb_handle_rxtx(void *context, int pending)
 {
 	struct adapter	*adapter = context;
@@ -1239,7 +1265,7 @@ igb_handle_rxtx(void *context, int pending)
 	ifp = adapter->ifp;
 
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
-		if (igb_rxeof(rxr, adapter->rx_process_limit) != 0)
+		if (igb_rxeof(rxr, adapter->rx_process_limit))
 			taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
 		IGB_TX_LOCK(txr);
 		igb_txeof(txr);
@@ -1255,23 +1281,23 @@ igb_handle_rxtx(void *context, int pending)
 static void
 igb_handle_rx(void *context, int pending)
 {
-	struct rx_ring	*rxr = context;
-	struct adapter	*adapter = rxr->adapter;
-	struct ifnet	*ifp = adapter->ifp;
-
+	struct rx_ring  *rxr = context;
+	struct adapter  *adapter = rxr->adapter;
+	struct ifnet    *ifp = adapter->ifp;
+            
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 		if (igb_rxeof(rxr, adapter->rx_process_limit) != 0)
 			/* More to clean, schedule another task */
 			taskqueue_enqueue(adapter->tq, &rxr->rx_task);
-
+                
 }
 
 static void
 igb_handle_tx(void *context, int pending)
 {
-	struct tx_ring	*txr = context;
-	struct adapter	*adapter = txr->adapter;
-	struct ifnet	*ifp = adapter->ifp;
+	struct tx_ring  *txr = context;
+	struct adapter  *adapter = txr->adapter;
+	struct ifnet    *ifp = adapter->ifp;
 
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 		IGB_TX_LOCK(txr);
@@ -1289,7 +1315,11 @@ igb_handle_tx(void *context, int pending)
  *  Interrupt Service routine  
  *
  *********************************************************************/
+#if __FreeBSD_version < 700000
+static void
+#else
 static int
+#endif
 igb_irq_fast(void *arg)
 {
 	struct adapter	*adapter = arg;
@@ -1320,7 +1350,7 @@ igb_irq_fast(void *arg)
 	/* Link status change */
 	if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
 		adapter->hw.mac.get_link_status = 1;
-		taskqueue_enqueue(adapter->tq, &adapter->link_task);
+		igb_update_link_status(adapter);
 	}
 
 	if (reg_icr & E1000_ICR_RXO)
@@ -1329,26 +1359,32 @@ igb_irq_fast(void *arg)
 }
 
 
+#if __FreeBSD_version >= 602105
 /*********************************************************************
  *
  *  MSIX TX Interrupt Service routine
  *
  **********************************************************************/
-
 static void
 igb_msix_tx(void *arg)
 {
 	struct tx_ring *txr = arg;
 	struct adapter *adapter = txr->adapter;
-	struct ifnet	*ifp = adapter->ifp;
+	u32		loop = IGB_MAX_LOOP;
+	bool		more;
 
 	++txr->tx_irq;
-	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
-		IGB_TX_LOCK(txr);
-		igb_txeof(txr);
-		IGB_TX_UNLOCK(txr);
-		taskqueue_enqueue(adapter->tq, &txr->tx_task);
-	}
+	IGB_TX_LOCK(txr);
+
+	do {
+		more = igb_txeof(txr);
+	} while (loop-- && more);
+
+	IGB_TX_UNLOCK(txr);
+
+	/* Schedule a clean task */
+	taskqueue_enqueue(adapter->tq, &txr->tx_task);
+
 	/* Reenable this interrupt */
 	E1000_WRITE_REG(&adapter->hw, E1000_EIMS, txr->eims);
 	return;
@@ -1365,25 +1401,55 @@ igb_msix_rx(void *arg)
 {
 	struct rx_ring *rxr = arg;
 	struct adapter *adapter = rxr->adapter;
-	u32		more, loop = 5;
+	u32		loop = IGB_MAX_LOOP;
+	bool		more;
 
 	++rxr->rx_irq;
 	do {
 		more = igb_rxeof(rxr, adapter->rx_process_limit);
-	} while (loop-- || more != 0);
-
-	taskqueue_enqueue(adapter->tq, &rxr->rx_task);
+	} while (loop-- && more);
 
 	/* Update interrupt rate */
 	if (igb_enable_aim == TRUE)
 		igb_update_aim(rxr);
 
+	/* Schedule another clean */
+	taskqueue_enqueue(adapter->tq, &rxr->rx_task);
+
 	/* Reenable this interrupt */
 	E1000_WRITE_REG(&adapter->hw, E1000_EIMS, rxr->eims);
 	return;
 }
 
 
+/*********************************************************************
+ *
+ *  MSIX Link Interrupt Service routine
+ *
+ **********************************************************************/
+
+static void
+igb_msix_link(void *arg)
+{
+	struct adapter	*adapter = arg;
+	u32       	icr;
+
+	++adapter->link_irq;
+	icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
+	if (!(icr & E1000_ICR_LSC))
+		goto spurious;
+	adapter->hw.mac.get_link_status = 1;
+	igb_update_link_status(adapter);
+
+spurious:
+	/* Rearm */
+	E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
+	E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
+	return;
+}
+#endif 
+
+
 /*
 ** Routine to adjust the RX EITR value based on traffic,
 ** its a simple three state model, but seems to help.
@@ -1436,33 +1502,6 @@ igb_update_aim(struct rx_ring *rxr)
 
 /*********************************************************************
  *
- *  MSIX Link Interrupt Service routine
- *
- **********************************************************************/
-
-static void
-igb_msix_link(void *arg)
-{
-	struct adapter	*adapter = arg;
-	u32       	icr;
-
-	++adapter->link_irq;
-	icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
-	if (!(icr & E1000_ICR_LSC))
-		goto spurious;
-	adapter->hw.mac.get_link_status = 1;
-	taskqueue_enqueue(adapter->tq, &adapter->link_task);
-
-spurious:
-	/* Rearm */
-	E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
-	E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
-	return;
-}
-
-
-/*********************************************************************
- *
  *  Media Ioctl callback
  *
  *  This routine is called whenever the user queries the status of
@@ -1589,7 +1628,7 @@ igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
 	struct adapter		*adapter = txr->adapter;
 	bus_dma_segment_t	segs[IGB_MAX_SCATTER];
 	bus_dmamap_t		map;
-	struct igb_buffer	*tx_buffer, *tx_buffer_mapped;
+	struct igb_tx_buffer	*tx_buffer, *tx_buffer_mapped;
 	union e1000_adv_tx_desc	*txd = NULL;
 	struct mbuf		*m_head;
 	u32			olinfo_status = 0, cmd_type_len = 0;
@@ -1639,7 +1678,7 @@ igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
 
 		m = m_defrag(*m_headp, M_DONTWAIT);
 		if (m == NULL) {
-			adapter->mbuf_alloc_failed++;
+			adapter->mbuf_defrag_failed++;
 			m_freem(*m_headp);
 			*m_headp = NULL;
 			return (ENOBUFS);
@@ -1692,7 +1731,7 @@ igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
 			return (ENXIO); 
 	} else
 		/* Do all other context descriptor setup */
-	offload = igb_tx_ctx_setup(txr, m_head);
+		offload = igb_tx_ctx_setup(txr, m_head);
 	if (offload == TRUE)
 		olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
 #ifdef IGB_TIMESYNC
@@ -1829,8 +1868,7 @@ igb_set_multi(struct adapter *adapter)
 		reg_rctl |= E1000_RCTL_MPE;
 		E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
 	} else
-		e1000_update_mc_addr_list(&adapter->hw, mta,
-		    mcnt, 1, adapter->hw.mac.rar_entry_count);
+		e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
 }
 
 
@@ -1861,6 +1899,9 @@ igb_local_timer(void *arg)
 	 */
 	igb_watchdog(adapter);
 
+	/* Trigger an RX interrupt on all queues */
+	E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->rx_mask);
+ 
 	callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
 
 }
@@ -2027,9 +2068,10 @@ igb_allocate_pci_resources(struct adapter *adapter)
 	adapter->num_tx_queues = 1; /* Defaults for Legacy or MSI */
 	adapter->num_rx_queues = 1;
 
+#if __FreeBSD_version >= 602105
 	/* This will setup either MSI/X or MSI */
 	adapter->msix = igb_setup_msix(adapter);
-
+#endif
 	adapter->hw.back = &adapter->osdep;
 
 	return (error);
@@ -2067,14 +2109,16 @@ igb_allocate_legacy(struct adapter *adapter)
 	 * processing contexts.
 	 */
 	TASK_INIT(&adapter->rxtx_task, 0, igb_handle_rxtx, adapter);
-	TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
 	adapter->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
 	    taskqueue_thread_enqueue, &adapter->tq);
 	taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
 	    device_get_nameunit(adapter->dev));
 	if ((error = bus_setup_intr(dev, adapter->res[0],
-	    INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL, adapter,
-	    &adapter->tag[0])) != 0) {
+	    INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast,
+#if __FreeBSD_version >= 700000
+		    NULL,
+#endif
+	    adapter, &adapter->tag[0])) != 0) {
 		device_printf(dev, "Failed to register fast interrupt "
 			    "handler: %d\n", error);
 		taskqueue_free(adapter->tq);
@@ -2086,6 +2130,7 @@ igb_allocate_legacy(struct adapter *adapter)
 }
 
 
+#if __FreeBSD_version >= 602105
 /*********************************************************************
  *
  *  Setup the MSIX Interrupt handlers: 
@@ -2115,8 +2160,11 @@ igb_allocate_msix(struct adapter *adapter)
 			return (ENXIO);
 		}
 		error = bus_setup_intr(dev, adapter->res[vector],
-	    	    INTR_TYPE_NET | INTR_MPSAFE, NULL, igb_msix_tx,
-		    txr, &adapter->tag[vector]);
+	    	    INTR_TYPE_NET | INTR_MPSAFE,
+#if __FreeBSD_version >= 700000
+		    NULL,
+#endif
+		    igb_msix_tx, txr, &adapter->tag[vector]);
 		if (error) {
 			adapter->res[vector] = NULL;
 			device_printf(dev, "Failed to register TX handler");
@@ -2146,13 +2194,17 @@ igb_allocate_msix(struct adapter *adapter)
 			return (ENXIO);
 		}
 		error = bus_setup_intr(dev, adapter->res[vector],
-	    	    INTR_TYPE_NET | INTR_MPSAFE, NULL, igb_msix_rx,
-		    rxr, &adapter->tag[vector]);
+	    	    INTR_TYPE_NET | INTR_MPSAFE,
+#if __FreeBSD_version >= 700000
+		    NULL,
+#endif
+		    igb_msix_rx, rxr, &adapter->tag[vector]);
 		if (error) {
 			adapter->res[vector] = NULL;
 			device_printf(dev, "Failed to register RX handler");
 			return (error);
 		}
+		/* Make tasklet for deferred handling - one per queue */
 		TASK_INIT(&rxr->rx_task, 0, igb_handle_rx, rxr);
 		if (adapter->hw.mac.type == e1000_82575) {
 			rxr->eims = E1000_EICR_RX_QUEUE0 << i;
@@ -2161,6 +2213,8 @@ igb_allocate_msix(struct adapter *adapter)
 			rxr->eims = 1 << vector;
 			rxr->msix = vector;
 		}
+		/* Get a mask for local timer */
+		adapter->rx_mask |= rxr->eims;
 	}
 
 	/* And Link */
@@ -2174,8 +2228,11 @@ igb_allocate_msix(struct adapter *adapter)
 		return (ENXIO);
 	}
 	if ((error = bus_setup_intr(dev, adapter->res[vector],
-	    INTR_TYPE_NET | INTR_MPSAFE, NULL, igb_msix_link,
-	    adapter, &adapter->tag[vector])) != 0) {
+	    INTR_TYPE_NET | INTR_MPSAFE,
+#if __FreeBSD_version >= 700000
+		    NULL,
+#endif
+	    igb_msix_link, adapter, &adapter->tag[vector])) != 0) {
 		device_printf(dev, "Failed to register Link handler");
 		return (error);
 	}
@@ -2184,9 +2241,6 @@ igb_allocate_msix(struct adapter *adapter)
 	else
 		adapter->linkvec = vector;
 
-	/* Make tasklet for deferred link interrupt handling */
-	TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
-
 	adapter->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
 	    taskqueue_thread_enqueue, &adapter->tq);
 	taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
@@ -2194,6 +2248,14 @@ igb_allocate_msix(struct adapter *adapter)
 
 	return (0);
 }
+#else /* FreeBSD 6.1/2 */
+static int
+igb_allocate_msix(struct adapter *adapter)
+{
+	return (1);
+}
+#endif
+
 
 static void
 igb_configure_queues(struct adapter *adapter)
@@ -2316,12 +2378,14 @@ igb_free_pci_resources(struct adapter *adapter)
 		}
 	}
 
+#if __FreeBSD_version >= 602105
 	if (adapter->msix)
 		pci_release_msi(dev);
 
 	if (adapter->msix_mem != NULL)
 		bus_release_resource(dev, SYS_RES_MEMORY,
 		    PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
+#endif
 
 	if (adapter->pci_mem != NULL)
 		bus_release_resource(dev, SYS_RES_MEMORY,
@@ -2329,6 +2393,7 @@ igb_free_pci_resources(struct adapter *adapter)
 
 }
 
+#if __FreeBSD_version >= 602105
 /*
  * Setup Either MSI/X or MSI
  */
@@ -2391,6 +2456,7 @@ msi:
                	device_printf(adapter->dev,"Using MSI interrupt\n");
 	return (msgs);
 }
+#endif /*  __FreeBSD_version >= 602105 */
 
 /*********************************************************************
  *
@@ -2487,11 +2553,12 @@ igb_setup_interface(device_t dev, struct adapter *adapter)
 	ifp->if_capabilities = ifp->if_capenable = 0;
 
 	ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
-	ifp->if_capabilities |= IFCAP_TSO4;
+	ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_VLAN_MTU;
+	ifp->if_capabilities |= IFCAP_JUMBO_MTU;
 	ifp->if_capenable = ifp->if_capabilities;
 
 	/*
-	 * Tell the upper layer(s) what we support.
+	 * Tell the upper layer(s) we support long frames.
 	 */
 	ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
 	ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER;
@@ -2750,7 +2817,7 @@ igb_allocate_transmit_buffers(struct tx_ring *txr)
 {
 	struct adapter *adapter = txr->adapter;
 	device_t dev = adapter->dev;
-	struct igb_buffer *txbuf;
+	struct igb_tx_buffer *txbuf;
 	int error, i;
 
 	/*
@@ -2773,7 +2840,7 @@ igb_allocate_transmit_buffers(struct tx_ring *txr)
 	}
 
 	if (!(txr->tx_buffers =
-	    (struct igb_buffer *) malloc(sizeof(struct igb_buffer) *
+	    (struct igb_tx_buffer *) malloc(sizeof(struct igb_tx_buffer) *
 	    adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
 		device_printf(dev, "Unable to allocate tx_buffer memory\n");
 		error = ENOMEM;
@@ -2806,7 +2873,7 @@ static void
 igb_setup_transmit_ring(struct tx_ring *txr)
 {
 	struct adapter *adapter = txr->adapter;
-	struct igb_buffer *txbuf;
+	struct igb_tx_buffer *txbuf;
 	int i;
 
 	/* Clear the old ring contents */
@@ -2936,7 +3003,7 @@ static void
 igb_free_transmit_buffers(struct tx_ring *txr)
 {
 	struct adapter *adapter = txr->adapter;
-	struct igb_buffer *tx_buffer;
+	struct igb_tx_buffer *tx_buffer;
 	int             i;
 
 	INIT_DEBUGOUT("free_transmit_ring: begin");
@@ -2978,6 +3045,7 @@ igb_free_transmit_buffers(struct tx_ring *txr)
 	return;
 }
 
+#if __FreeBSD_version >= 700000
 /**********************************************************************
  *
  *  Setup work for hardware segmentation offload (TSO) on
@@ -2989,7 +3057,7 @@ igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen)
 {
 	struct adapter *adapter = txr->adapter;
 	struct e1000_adv_tx_context_desc *TXD;
-	struct igb_buffer        *tx_buffer;
+	struct igb_tx_buffer        *tx_buffer;
 	u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
 	u32 mss_l4len_idx = 0;
 	u16 vtag = 0;
@@ -3065,7 +3133,13 @@ igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen)
 	txr->next_avail_desc = ctxd;
 	return TRUE;
 }
-
+#else	/* fake out for 6.2 */
+static boolean_t
+igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen)
+{
+	return (FALSE);
+}
+#endif
 
 /*********************************************************************
  *
@@ -3078,7 +3152,7 @@ igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 {
 	struct adapter *adapter = txr->adapter;
 	struct e1000_adv_tx_context_desc *TXD;
-	struct igb_buffer        *tx_buffer;
+	struct igb_tx_buffer        *tx_buffer;
 	uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0;
 	struct ether_vlan_header *eh;
 	struct ip *ip = NULL;
@@ -3086,24 +3160,38 @@ igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 	int  ehdrlen, ip_hlen = 0;
 	u16	etype;
 	u8	ipproto = 0;
-	bool	offload = TRUE;
+	bool	offload = FALSE;
+#if __FreeBSD_version >= 700000
 	u16 vtag = 0;
+#else
+	struct m_tag	*mtag;
+#endif
 
 	int ctxd = txr->next_avail_desc;
 	tx_buffer = &txr->tx_buffers[ctxd];
 	TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
 
-	if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
-		offload = FALSE; /* Only here to handle VLANs */
+	if (mp->m_pkthdr.csum_flags & CSUM_OFFLOAD)
+		offload = TRUE;
+
 	/*
 	** In advanced descriptors the vlan tag must 
 	** be placed into the descriptor itself.
 	*/
+#if __FreeBSD_version < 700000
+	mtag = VLAN_OUTPUT_TAG(ifp, mp);
+	if (mtag != NULL) {
+		vlan_macip_lens |=
+		    htole16(VLAN_TAG_VALUE(mtag)) << E1000_ADVTXD_VLAN_SHIFT;
+		offload = TRUE;
+	}
+#else
 	if (mp->m_flags & M_VLANTAG) {
 		vtag = htole16(mp->m_pkthdr.ether_vtag);
 		vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
-	} else if (offload == FALSE)
-		return FALSE;
+		offload = TRUE;
+	}
+#endif
 	/*
 	 * Determine where frame payload starts.
 	 * Jump over vlan headers if already present,
@@ -3125,10 +3213,8 @@ igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 		case ETHERTYPE_IP:
 			ip = (struct ip *)(mp->m_data + ehdrlen);
 			ip_hlen = ip->ip_hl << 2;
-			if (mp->m_len < ehdrlen + ip_hlen) {
-				offload = FALSE;
-				break;
-			}
+			if (mp->m_len < ehdrlen + ip_hlen)
+				return FALSE;
 			ipproto = ip->ip_p;
 			type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
 			break;
@@ -3142,12 +3228,10 @@ igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 			break;
 #ifdef IGB_TIMESYNC
 		case ETHERTYPE_IEEE1588:
-			offload = IGB_TIMESTAMP;
-			break;
+			return (IGB_TIMESTAMP);
 #endif
 		default:
-			offload = FALSE;
-			break;
+			return (FALSE);
 	}
 
 	vlan_macip_lens |= ip_hlen;
@@ -3155,8 +3239,10 @@ igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 
 	switch (ipproto) {
 		case IPPROTO_TCP:
-			if (mp->m_pkthdr.csum_flags & CSUM_TCP)
+			if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
 				type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
+				offload = TRUE;
+			}
 			break;
 		case IPPROTO_UDP:
 		{
@@ -3165,17 +3251,31 @@ igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 			struct udphdr *uh = (struct udphdr *)hdr;
 
 			if (uh->uh_dport == htons(TSYNC_PORT))
-				offload = IGB_TIMESTAMP;
+				return (IGB_TIMESTAMP);
 #endif
-			if (mp->m_pkthdr.csum_flags & CSUM_UDP)
+			if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
 				type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
+				offload = TRUE;
+			}
 			break;
 		}
-		default:
-			offload = FALSE;
+#if __FreeBSD_version >= 800000
+		case IPPROTO_SCTP:
+		{
+			if (mp->m_pkthdr.csum_flags & CSUM_SCTP) {
+				type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
+				offload = TRUE;
+			}
 			break;
+		}
+#endif
+		default:
+			return (FALSE);
 	}
 
+	if (offload != TRUE)
+		return (FALSE);
+
 	/* Now copy bits into descriptor */
 	TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
 	TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
@@ -3191,7 +3291,7 @@ igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 	txr->next_avail_desc = ctxd;
 	--txr->tx_avail;
 
-        return (offload);
+        return (TRUE);
 }
 
 
@@ -3208,7 +3308,7 @@ igb_txeof(struct tx_ring *txr)
 {
 	struct adapter	*adapter = txr->adapter;
         int first, last, done, num_avail;
-        struct igb_buffer *tx_buffer;
+        struct igb_tx_buffer *tx_buffer;
         struct e1000_tx_desc   *tx_desc, *eop_desc;
 	struct ifnet   *ifp = adapter->ifp;
 
@@ -3304,55 +3404,113 @@ igb_txeof(struct tx_ring *txr)
 
 /*********************************************************************
  *
- *  Get a buffer from system mbuf buffer pool.
+ *  Setup descriptor buffer(s) from system mbuf buffer pools.
+ *  		i - designates the ring index
+ *		clean - tells the function whether to update
+ *		        the header, the packet buffer, or both.
  *
  **********************************************************************/
 static int
-igb_get_buf(struct rx_ring *rxr, int i)
+igb_get_buf(struct rx_ring *rxr, int i, u8 clean)
 {
 	struct adapter		*adapter = rxr->adapter;
-	struct mbuf		*m;
-	bus_dma_segment_t	segs[1];
+	struct mbuf		*mh, *mp;
+	bus_dma_segment_t	seg[2];
 	bus_dmamap_t		map;
-	struct igb_buffer	*rx_buffer;
+	struct igb_rx_buffer	*rx_buffer;
 	int			error, nsegs;
+	int			merr = 0;
 
-	m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
-	if (m == NULL) {
-		adapter->mbuf_cluster_failed++;
-		return (ENOBUFS);
-	}
-	m->m_len = m->m_pkthdr.len = MCLBYTES;
 
-	if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
-		m_adj(m, ETHER_ALIGN);
+	rx_buffer = &rxr->rx_buffers[i];
 
+	/* First get our header and payload mbuf */
+	if (clean & IGB_CLEAN_HEADER) {
+		mh = m_gethdr(M_DONTWAIT, MT_DATA);
+		if (mh == NULL)
+			goto remap;
+	} else  /* reuse */
+		mh = rxr->rx_buffers[i].m_head;
+
+	mh->m_len = MHLEN;
+	mh->m_flags |= M_PKTHDR;
+
+	if (clean & IGB_CLEAN_PAYLOAD) {
+		mp = m_getjcl(M_DONTWAIT, MT_DATA,
+		    M_PKTHDR, adapter->rx_mbuf_sz);
+		if (mp == NULL)
+			goto remap;
+		mp->m_len = adapter->rx_mbuf_sz;
+		mp->m_flags &= ~M_PKTHDR;
+	} else {	/* reusing */
+		mp = rxr->rx_buffers[i].m_pack;
+		mp->m_len = adapter->rx_mbuf_sz;
+		mp->m_flags &= ~M_PKTHDR;
+	}
 	/*
-	 * Using memory from the mbuf cluster pool, invoke the
-	 * bus_dma machinery to arrange the memory mapping.
-	 */
+	** Need to create a chain for the following
+	** dmamap call at this point.
+	*/
+	mh->m_next = mp;
+	mh->m_pkthdr.len = mh->m_len + mp->m_len;
+
+	/* Get the memory mapping */
 	error = bus_dmamap_load_mbuf_sg(rxr->rxtag,
-	    rxr->rx_spare_map, m, segs, &nsegs, BUS_DMA_NOWAIT);
+	    rxr->rx_spare_map, mh, seg, &nsegs, BUS_DMA_NOWAIT);
 	if (error != 0) {
-		m_free(m);
+		printf("GET BUF: dmamap load failure - %d\n", error);
+		m_free(mh);
 		return (error);
 	}
 
-	/* If nsegs is wrong then the stack is corrupt. */
-	KASSERT(nsegs == 1, ("Too many segments returned!"));
-
-	rx_buffer = &rxr->rx_buffers[i];
+	/* Unload old mapping and update buffer struct */
 	if (rx_buffer->m_head != NULL)
-		bus_dmamap_unload(rxr->rxtag, rx_buffer->map);
-
+			bus_dmamap_unload(rxr->rxtag, rx_buffer->map);
 	map = rx_buffer->map;
 	rx_buffer->map = rxr->rx_spare_map;
 	rxr->rx_spare_map = map;
-	bus_dmamap_sync(rxr->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
-	rx_buffer->m_head = m;
+	rx_buffer->m_head = mh;
+	rx_buffer->m_pack = mp;
+	bus_dmamap_sync(rxr->rxtag,
+	    rx_buffer->map, BUS_DMASYNC_PREREAD);
+
+	/* Update descriptor */
+	rxr->rx_base[i].read.hdr_addr = htole64(seg[0].ds_addr);
+	rxr->rx_base[i].read.pkt_addr = htole64(seg[1].ds_addr);
 
-	rxr->rx_base[i].read.pkt_addr = htole64(segs[0].ds_addr);
 	return (0);
+
+	/*
+	** If we get here, we have an mbuf resource
+	** issue, so we discard the incoming packet
+	** and attempt to reuse existing mbufs next
+	** pass thru the ring, but to do so we must
+	** fix up the descriptor which had the address
+	** clobbered with writeback info.
+	*/
+remap:
+	adapter->mbuf_header_failed++;
+	merr = ENOBUFS;
+	/* Is there a reusable buffer? */
+	mh = rxr->rx_buffers[i].m_head;
+	if (mh == NULL) /* Nope, init error */
+		return (merr);
+	mp = rxr->rx_buffers[i].m_pack;
+	if (mp == NULL) /* Nope, init error */
+		return (merr);
+	/* Get our old mapping */
+	rx_buffer = &rxr->rx_buffers[i];
+	error = bus_dmamap_load_mbuf_sg(rxr->rxtag,
+	    rx_buffer->map, mh, seg, &nsegs, BUS_DMA_NOWAIT);
+	if (error != 0) {
+		/* We really have a problem */
+		m_free(mh);
+		return (error);
+	}
+	/* Now fix the descriptor as needed */
+	rxr->rx_base[i].read.hdr_addr = htole64(seg[0].ds_addr);
+	rxr->rx_base[i].read.pkt_addr = htole64(seg[1].ds_addr);
+	return (merr);
 }
 
 
@@ -3369,31 +3527,36 @@ igb_allocate_receive_buffers(struct rx_ring *rxr)
 {
 	struct	adapter 	*adapter = rxr->adapter;
 	device_t 		dev = adapter->dev;
-	struct igb_buffer 	*rxbuf;
+	struct igb_rx_buffer 	*rxbuf;
 	int             	i, bsize, error;
 
-	bsize = sizeof(struct igb_buffer) * adapter->num_rx_desc;
+	bsize = sizeof(struct igb_rx_buffer) * adapter->num_rx_desc;
 	if (!(rxr->rx_buffers =
-	    (struct igb_buffer *) malloc(bsize,
+	    (struct igb_rx_buffer *) malloc(bsize,
 	    M_DEVBUF, M_NOWAIT | M_ZERO))) {
 		device_printf(dev, "Unable to allocate rx_buffer memory\n");
 		error = ENOMEM;
 		goto fail;
 	}
 
+	/*
+	** The tag is made to accomodate the largest buffer size
+	** with packet split (hence the two segments, even though
+	** it may not always use this.
+	*/
 	if ((error = bus_dma_tag_create(NULL,		/* parent */
 				   PAGE_SIZE, 0,	/* alignment, bounds */
 				   BUS_SPACE_MAXADDR,	/* lowaddr */
 				   BUS_SPACE_MAXADDR,	/* highaddr */
 				   NULL, NULL,		/* filter, filterarg */
-				   MCLBYTES,		/* maxsize */
-				   1,			/* nsegments */
-				   MCLBYTES,		/* maxsegsize */
+				   MJUM16BYTES,		/* maxsize */
+				   2,			/* nsegments */
+				   MJUMPAGESIZE,	/* maxsegsize */
 				   0,			/* flags */
 				   NULL,		/* lockfunc */
 				   NULL,		/* lockfuncarg */
 				   &rxr->rxtag))) {
-		device_printf(dev, "Unable to create RX Small DMA tag\n");
+		device_printf(dev, "Unable to create RX DMA tag\n");
 		goto fail;
 	}
 
@@ -3401,7 +3564,8 @@ igb_allocate_receive_buffers(struct rx_ring *rxr)
         error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT,
 	     &rxr->rx_spare_map);
 	if (error) {
-		device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
+		device_printf(dev,
+		    "%s: bus_dmamap_create header spare failed: %d\n",
 		    __func__, error);
 		goto fail;
 	}
@@ -3411,7 +3575,7 @@ igb_allocate_receive_buffers(struct rx_ring *rxr)
 		error = bus_dmamap_create(rxr->rxtag,
 		    BUS_DMA_NOWAIT, &rxbuf->map);
 		if (error) {
-			device_printf(dev, "Unable to create Small RX DMA map\n");
+			device_printf(dev, "Unable to create RX DMA maps\n");
 			goto fail;
 		}
 	}
@@ -3434,36 +3598,40 @@ igb_setup_receive_ring(struct rx_ring *rxr)
 {
 	struct	adapter		*adapter;
 	device_t		dev;
-	struct igb_buffer	*rxbuf;
+	struct igb_rx_buffer	*rxbuf;
 	struct lro_ctrl		*lro = &rxr->lro;
-	int j, rsize;
+	int			j, rsize;
 
 	adapter = rxr->adapter;
 	dev = adapter->dev;
-	rsize = roundup2(adapter->num_rx_desc *
-	    sizeof(union e1000_adv_rx_desc), 4096);
+
 	/* Clear the ring contents */
+	rsize = roundup2(adapter->num_rx_desc *
+	    sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
 	bzero((void *)rxr->rx_base, rsize);
 
 	/*
-	** Free current RX buffers: the size buffer
-	** that is loaded is indicated by the buffer
-	** bigbuf value.
+	** Free current RX buffer structures and their mbufs
 	*/
 	for (int i = 0; i < adapter->num_rx_desc; i++) {
 		rxbuf = &rxr->rx_buffers[i];
-		if (rxbuf->m_head != NULL) {
-			bus_dmamap_sync(rxr->rxtag, rxbuf->map,
-			    BUS_DMASYNC_POSTREAD);
-			bus_dmamap_unload(rxr->rxtag, rxbuf->map);
+		bus_dmamap_sync(rxr->rxtag, rxbuf->map,
+		    BUS_DMASYNC_POSTREAD);
+		bus_dmamap_unload(rxr->rxtag, rxbuf->map);
+		if (rxbuf->m_head) {
+			rxbuf->m_head->m_next = rxbuf->m_pack;
 			m_freem(rxbuf->m_head);
-			rxbuf->m_head = NULL;
 		}
+		rxbuf->m_head = NULL;
+		rxbuf->m_pack = NULL;
 	}
 
+	/* Next replenish the ring */
 	for (j = 0; j < adapter->num_rx_desc; j++) {
-		if (igb_get_buf(rxr, j) == ENOBUFS) {
+		if (igb_get_buf(rxr, j, IGB_CLEAN_BOTH) == ENOBUFS) {
 			rxr->rx_buffers[j].m_head = NULL;
+			rxr->rx_buffers[j].m_pack = NULL;
+			rxr->rx_base[j].read.hdr_addr = 0;
 			rxr->rx_base[j].read.pkt_addr = 0;
 			goto fail;
 		}
@@ -3515,9 +3683,9 @@ static int
 igb_setup_receive_structures(struct adapter *adapter)
 {
 	struct rx_ring *rxr = adapter->rx_rings;
-	int j;
+	int i, j;
 
-	for (j = 0; j < adapter->num_rx_queues; j++, rxr++)
+	for (i = 0; i < adapter->num_rx_queues; i++, rxr++)
 		if (igb_setup_receive_ring(rxr))
 			goto fail;
 
@@ -3526,13 +3694,14 @@ fail:
 	/*
 	 * Free RX buffers allocated so far, we will only handle
 	 * the rings that completed, the failing case will have
-	 * cleaned up for itself. Clean up til 'j', the failure.
+	 * cleaned up for itself. The value of 'i' will be the
+	 * failed ring so we must pre-decrement it.
 	 */
-	for (int i = 0; i < j; i++) {
-		rxr = &adapter->rx_rings[i];
-		for (int n = 0; n < adapter->num_rx_desc; n++) {
-			struct igb_buffer *rxbuf;
-			rxbuf = &rxr->rx_buffers[n];
+	rxr = adapter->rx_rings;
+	for (--i; i > 0; i--, rxr++) {
+		for (j = 0; j < adapter->num_rx_desc; j++) {
+			struct igb_rx_buffer *rxbuf;
+			rxbuf = &rxr->rx_buffers[j];
 			if (rxbuf->m_head != NULL) {
 				bus_dmamap_sync(rxr->rxtag, rxbuf->map,
 			  	  BUS_DMASYNC_POSTREAD);
@@ -3556,7 +3725,7 @@ igb_initialize_receive_units(struct adapter *adapter)
 {
 	struct rx_ring	*rxr = adapter->rx_rings;
 	struct ifnet	*ifp = adapter->ifp;
-	u32		rctl, rxcsum, psize;
+	u32		rctl, rxcsum, psize, srrctl = 0;
 
 	INIT_DEBUGOUT("igb_initialize_receive_unit: begin");
 
@@ -3567,10 +3736,44 @@ igb_initialize_receive_units(struct adapter *adapter)
 	rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
 	E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
 
+	/*
+	** Set up for header split
+	*/
+	if (igb_rx_hdr_split) {
+		/* Use a standard mbuf for the header */
+		srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
+		srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
+	} else
+		srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
+
+	/*
+	** Set up for jumbo frames
+	*/
+	if (ifp->if_mtu > ETHERMTU) {
+		rctl |= E1000_RCTL_LPE;
+		srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
+		rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
+
+		/* Set maximum packet len */
+		psize = adapter->max_frame_size;
+		/* are we on a vlan? */
+#if __FreeBSD_version >= 700000
+		if (adapter->ifp->if_vlantrunk != NULL)
+#else
+		if (adapter->ifp->if_nvlans != 0)
+#endif
+			psize += VLAN_TAG_SIZE;
+		E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
+	} else {
+		rctl &= ~E1000_RCTL_LPE;
+		srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
+		rctl |= E1000_RCTL_SZ_2048;
+	}
+
 	/* Setup the Base and Length of the Rx Descriptor Rings */
 	for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) {
 		u64 bus_addr = rxr->rxdma.dma_paddr;
-		u32 rxdctl, srrctl;
+		u32 rxdctl;
 
 		E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(i),
 		    adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
@@ -3578,9 +3781,6 @@ igb_initialize_receive_units(struct adapter *adapter)
 		    (uint32_t)(bus_addr >> 32));
 		E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(i),
 		    (uint32_t)bus_addr);
-		/* Use Advanced Descriptor type */
-		srrctl = E1000_READ_REG(&adapter->hw, E1000_SRRCTL(i));
-		srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
 		E1000_WRITE_REG(&adapter->hw, E1000_SRRCTL(i), srrctl);
 		/* Enable this Queue */
 		rxdctl = E1000_READ_REG(&adapter->hw, E1000_RXDCTL(i));
@@ -3595,6 +3795,7 @@ igb_initialize_receive_units(struct adapter *adapter)
 	/*
 	** Setup for RX MultiQueue
 	*/
+	rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
 	if (adapter->num_rx_queues >1) {
 		u32 random[10], mrqc, shift = 0;
 		union igb_reta {
@@ -3611,7 +3812,7 @@ igb_initialize_receive_units(struct adapter *adapter)
 			    (i % adapter->num_rx_queues) << shift;
 			if ((i & 3) == 3)
 				E1000_WRITE_REG(&adapter->hw,
-				    E1000_RETA(i & ~3), reta.dword);
+				    E1000_RETA(i >> 2), reta.dword);
 		}
 		/* Now fill in hash table */
 		mrqc = E1000_MRQC_ENABLE_RSS_4Q;
@@ -3636,14 +3837,25 @@ igb_initialize_receive_units(struct adapter *adapter)
 		** this is not the same as TCP/IP checksums which
 		** still work.
 		*/
-		rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
 		rxcsum |= E1000_RXCSUM_PCSD;
-		E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum);
-	} else if (ifp->if_capenable & IFCAP_RXCSUM) {
-		rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
-		rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
-		E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum);
+#if __FreeBSD_version >= 800000
+		/* For SCTP Offload */
+		if ((adapter->hw.mac.type == e1000_82576)
+		    && (ifp->if_capenable & IFCAP_RXCSUM))
+			rxcsum |= E1000_RXCSUM_CRCOFL;
+#endif
+	} else {
+		/* Non RSS setup */
+		if (ifp->if_capenable & IFCAP_RXCSUM) {
+			rxcsum |= E1000_RXCSUM_IPPCSE;
+#if __FreeBSD_version >= 800000
+			if (adapter->hw.mac.type == e1000_82576)
+				rxcsum |= E1000_RXCSUM_CRCOFL;
+#endif
+		} else
+			rxcsum &= ~E1000_RXCSUM_TUOFL;
 	}
+	E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum);
 
 	/* Setup the Receive Control Register */
 	rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
@@ -3653,39 +3865,9 @@ igb_initialize_receive_units(struct adapter *adapter)
 
 	/* Make sure VLAN Filters are off */
 	rctl &= ~E1000_RCTL_VFE;
-
+	/* Don't store bad packets */
 	rctl &= ~E1000_RCTL_SBP;
 
-	switch (adapter->rx_buffer_len) {
-	default:
-	case 2048:
-		rctl |= E1000_RCTL_SZ_2048;
-		break;
-	case 4096:
-		rctl |= E1000_RCTL_SZ_4096 |
-		    E1000_RCTL_BSEX | E1000_RCTL_LPE;
-		break;
-	case 8192:
-		rctl |= E1000_RCTL_SZ_8192 |
-		    E1000_RCTL_BSEX | E1000_RCTL_LPE;
-		break;
-	case 16384:
-		rctl |= E1000_RCTL_SZ_16384 |
-		    E1000_RCTL_BSEX | E1000_RCTL_LPE;
-		break;
-	}
-
-	if (ifp->if_mtu > ETHERMTU) {
-		/* Set maximum packet len */
-		psize = adapter->max_frame_size;
-		/* are we on a vlan? */
-		if (adapter->ifp->if_vlantrunk != NULL)
-			psize += VLAN_TAG_SIZE;
-		E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
-		rctl |= E1000_RCTL_LPE;
-	} else
-		rctl &= ~E1000_RCTL_LPE;
-
 	/* Enable Receives */
 	E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
 
@@ -3730,7 +3912,7 @@ static void
 igb_free_receive_buffers(struct rx_ring *rxr)
 {
 	struct adapter	*adapter = rxr->adapter;
-	struct igb_buffer *rx_buffer;
+	struct igb_rx_buffer *rx_buffer;
 
 	INIT_DEBUGOUT("free_receive_structures: begin");
 
@@ -3780,7 +3962,7 @@ igb_free_receive_buffers(struct rx_ring *rxr)
  *  We loop at most count times if count is > 0, or until done if
  *  count < 0.
  *
- *  Return TRUE if all clean, FALSE otherwise
+ *  Return TRUE if more to clean, FALSE otherwise
  *********************************************************************/
 static bool
 igb_rxeof(struct rx_ring *rxr, int count)
@@ -3789,120 +3971,172 @@ igb_rxeof(struct rx_ring *rxr, int count)
 	struct ifnet		*ifp;
 	struct lro_ctrl		*lro = &rxr->lro;
 	struct lro_entry	*queued;
-	struct mbuf		*mp;
-	uint8_t			accept_frame = 0;
-	uint8_t			eop = 0;
-	uint16_t 		len, desc_len, prev_len_adj;
 	int			i;
 	u32			staterr;
 	union e1000_adv_rx_desc	*cur;
 
+
 	IGB_RX_LOCK(rxr);
 	ifp = adapter->ifp;
 	i = rxr->next_to_check;
 	cur = &rxr->rx_base[i];
 	staterr = cur->wb.upper.status_error;
 
-	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
-	    BUS_DMASYNC_POSTREAD);
-
 	if (!(staterr & E1000_RXD_STAT_DD)) {
 		IGB_RX_UNLOCK(rxr);
 		return FALSE;
 	}
 
+	/* Sync the ring */
+	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
+	    BUS_DMASYNC_POSTREAD);
+
+	/* Main clean loop */
 	while ((staterr & E1000_RXD_STAT_DD) &&
 	    (count != 0) &&
 	    (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
-		struct mbuf *m = NULL;
+		struct mbuf *sendmp, *mh, *mp;
+		u16 hlen, plen, hdr, ptype, len_adj;
+		u8 dopayload, accept_frame, eop;
+ 
+		accept_frame = 1;
+		hlen = plen = len_adj = 0;
+		sendmp = mh = mp = NULL;
+		ptype = (u16)cur->wb.lower.lo_dword.data;
 
-		mp = rxr->rx_buffers[i].m_head;
-		/*
-		 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT
-		 * needs to access the last received byte in the mbuf.
-		 */
+		/* Sync the buffers */
 		bus_dmamap_sync(rxr->rxtag, rxr->rx_buffers[i].map,
-		    BUS_DMASYNC_POSTREAD);
+			    BUS_DMASYNC_POSTREAD);
+
+		/*
+		** The way the hardware is configured to
+		** split, it will ONLY use the header buffer
+		** when header split is enabled, otherwise we
+		** get normal behavior, ie, both header and
+		** payload are DMA'd into the payload buffer.
+		**
+		** The fmp test is to catch the case where a
+		** packet spans multiple descriptors, in that
+		** case only the first header is valid.
+		*/
+		if ((igb_rx_hdr_split) && (rxr->fmp == NULL)){
+			hdr = le16toh(cur->
+			    wb.lower.lo_dword.hs_rss.hdr_info);
+			hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
+			    E1000_RXDADV_HDRBUFLEN_SHIFT;
+			if (hlen > IGB_HDR_BUF)
+				hlen = IGB_HDR_BUF;
+			plen = le16toh(cur->wb.upper.length);
+			/* Handle the header mbuf */
+			mh = rxr->rx_buffers[i].m_head;
+			mh->m_len = hlen;
+			dopayload = IGB_CLEAN_HEADER;
+			/*
+			** Get the payload length, this
+			** could be zero if its a small
+			** packet.
+			*/
+			if (plen) {
+				mp = rxr->rx_buffers[i].m_pack;
+				mp->m_len = plen;
+				mp->m_next = NULL;
+				mp->m_flags &= ~M_PKTHDR;
+				mh->m_next = mp;
+				mh->m_flags |= M_PKTHDR;
+				dopayload = IGB_CLEAN_BOTH;
+				rxr->rx_split_packets++;
+			} else {  /* small packets */
+				mh->m_flags &= ~M_PKTHDR;
+				mh->m_next = NULL;
+			}
+		} else {
+			/*
+			** Either no header split, or a
+			** secondary piece of a fragmented
+			** split packet.
+			*/
+			mh = rxr->rx_buffers[i].m_pack;
+			mh->m_flags |= M_PKTHDR;
+			mh->m_len = le16toh(cur->wb.upper.length);
+			dopayload = IGB_CLEAN_PAYLOAD;
+		}
 
-		accept_frame = 1;
-		prev_len_adj = 0;
-		desc_len = le16toh(cur->wb.upper.length);
 		if (staterr & E1000_RXD_STAT_EOP) {
 			count--;
 			eop = 1;
-			if (desc_len < ETHER_CRC_LEN) {
-				len = 0;
-				prev_len_adj = ETHER_CRC_LEN - desc_len;
-			} else
-				len = desc_len - ETHER_CRC_LEN;
-		} else {
+			/*
+			** Strip CRC and account for frag
+			*/
+			if (mp) { 
+				if (mp->m_len < ETHER_CRC_LEN) {
+					/* a frag, how much is left? */
+					len_adj = ETHER_CRC_LEN - mp->m_len;
+					mp->m_len = 0;
+				} else
+					mp->m_len -= ETHER_CRC_LEN;
+			} else { /* not split */
+				if (mh->m_len < ETHER_CRC_LEN) {
+					len_adj = ETHER_CRC_LEN - mh->m_len;
+					mh->m_len = 0;
+				} else
+					mh->m_len -= ETHER_CRC_LEN;
+			}
+		} else 
 			eop = 0;
-			len = desc_len;
-		}
-
-		if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
-			u32	pkt_len = desc_len;
-
-			if (rxr->fmp != NULL)
-				pkt_len += rxr->fmp->m_pkthdr.len;
 
+		if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK)
 			accept_frame = 0;
-		}
 
 		if (accept_frame) {
-			if (igb_get_buf(rxr, i) != 0) {
+			if (igb_get_buf(rxr, i, dopayload) != 0) {
 				ifp->if_iqdrops++;
 				goto discard;
 			}
-
-			/* Assign correct length to the current fragment */
-			mp->m_len = len;
-
+			/* Initial frame - setup */
 			if (rxr->fmp == NULL) {
-				mp->m_pkthdr.len = len;
-				rxr->fmp = mp; /* Store the first mbuf */
-				rxr->lmp = mp;
+				mh->m_flags |= M_PKTHDR;
+				mh->m_pkthdr.len = mh->m_len;
+				rxr->fmp = mh; /* Store the first mbuf */
+				rxr->lmp = mh;
+				if (mp) { /* Add payload if split */
+					mh->m_pkthdr.len += mp->m_len;
+					rxr->lmp = mh->m_next;
+				}
 			} else {
 				/* Chain mbuf's together */
-				mp->m_flags &= ~M_PKTHDR;
-				/*
-				 * Adjust length of previous mbuf in chain if
-				 * we received less than 4 bytes in the last
-				 * descriptor.
-				 */
-				if (prev_len_adj > 0) {
-					rxr->lmp->m_len -= prev_len_adj;
-					rxr->fmp->m_pkthdr.len -=
-					    prev_len_adj;
-				}
-				rxr->lmp->m_next = mp;
+				mh->m_flags &= ~M_PKTHDR;
+				rxr->lmp->m_next = mh;
 				rxr->lmp = rxr->lmp->m_next;
-				rxr->fmp->m_pkthdr.len += len;
+				rxr->fmp->m_pkthdr.len += mh->m_len;
+				/* Adjust for CRC frag */
+				if (len_adj) {
+					rxr->lmp->m_len -= len_adj;
+					rxr->fmp->m_pkthdr.len -= len_adj;
+				}
 			}
 
 			if (eop) {
+				bool sctp = ((ptype & 0x40) != 0);
 				rxr->fmp->m_pkthdr.rcvif = ifp;
 				ifp->if_ipackets++;
 				rxr->rx_packets++;
+				/* capture data for AIM */
 				rxr->bytes += rxr->fmp->m_pkthdr.len;
-				rxr->rx_bytes += rxr->bytes;
-
-				igb_rx_checksum(staterr, rxr->fmp);
-#ifndef __NO_STRICT_ALIGNMENT
-				if (adapter->max_frame_size >
-				    (MCLBYTES - ETHER_ALIGN) &&
-				    igb_fixup_rx(rxr) != 0)
-					goto skip;
-#endif
+				rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
+
+				igb_rx_checksum(staterr, rxr->fmp, sctp);
 				if (staterr & E1000_RXD_STAT_VP) {
+#if __FreeBSD_version >= 700000
 					rxr->fmp->m_pkthdr.ether_vtag =
 					    le16toh(cur->wb.upper.vlan);
 					rxr->fmp->m_flags |= M_VLANTAG;
-				}
-#ifndef __NO_STRICT_ALIGNMENT
-skip:
+#else
+					VLAN_INPUT_TAG_NEW(ifp, rxr->fmp,
+					    (le16toh(cur->wb.upper.vlan) &
+					    E1000_RXD_SPC_VLAN_MASK));
 #endif
-				m = rxr->fmp;
+				}
+				sendmp = rxr->fmp;
 				rxr->fmp = NULL;
 				rxr->lmp = NULL;
 			}
@@ -3910,23 +4144,27 @@ skip:
 			ifp->if_ierrors++;
 discard:
 			/* Reuse loaded DMA map and just update mbuf chain */
-			mp = rxr->rx_buffers[i].m_head;
-			mp->m_len = mp->m_pkthdr.len = MCLBYTES;
+			if (hlen) {
+				mh = rxr->rx_buffers[i].m_head;
+				mh->m_len = MHLEN;
+				mh->m_next = NULL;
+			}
+			mp = rxr->rx_buffers[i].m_pack;
+			mp->m_len = mp->m_pkthdr.len = adapter->rx_mbuf_sz;
 			mp->m_data = mp->m_ext.ext_buf;
 			mp->m_next = NULL;
 			if (adapter->max_frame_size <=
 			    (MCLBYTES - ETHER_ALIGN))
 				m_adj(mp, ETHER_ALIGN);
 			if (rxr->fmp != NULL) {
+				/* handles the whole chain */
 				m_freem(rxr->fmp);
 				rxr->fmp = NULL;
 				rxr->lmp = NULL;
 			}
-			m = NULL;
+			sendmp = NULL;
 		}
 
-		/* Zero out the receive descriptors status. */
-		cur->wb.upper.status_error = 0;
 		bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
@@ -3935,16 +4173,19 @@ discard:
 		/* Advance our pointers to the next descriptor. */
 		if (++i == adapter->num_rx_desc)
 			i = 0;
-
-		if (m != NULL) {
-			rxr->next_to_check = i;
+ 
+		/*
+		** Note that we hold the RX lock thru
+		** the following call so this ring's
+		** next_to_check is not gonna change.
+		*/
+		if (sendmp != NULL) {
 			/* Use LRO if possible */   
-			if ((!lro->lro_cnt) || (tcp_lro_rx(lro, m, 0))) {
+			if ((!lro->lro_cnt) || (tcp_lro_rx(lro, sendmp, 0)))
 				/* Pass up to the stack */
-				(*ifp->if_input)(ifp, m);
-				i = rxr->next_to_check;
-			}
+				(*ifp->if_input)(ifp, sendmp);
 		}
+
 		/* Get the next descriptor */
 		cur = &rxr->rx_base[i];
 		staterr = cur->wb.upper.status_error;
@@ -3965,60 +4206,18 @@ discard:
 
 	IGB_RX_UNLOCK(rxr);
 
-	if (!((staterr) & E1000_RXD_STAT_DD))
-		return FALSE;
-
-	return TRUE;
-}
-
-#ifndef __NO_STRICT_ALIGNMENT
-/*
- * When jumbo frames are enabled we should realign entire payload on
- * architecures with strict alignment. This is serious design mistake of 8254x
- * as it nullifies DMA operations. 8254x just allows RX buffer size to be
- * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
- * payload. On architecures without strict alignment restrictions 8254x still
- * performs unaligned memory access which would reduce the performance too.
- * To avoid copying over an entire frame to align, we allocate a new mbuf and
- * copy ethernet header to the new mbuf. The new mbuf is prepended into the
- * existing mbuf chain.
- *
- * Be aware, best performance of the 8254x is achived only when jumbo frame is
- * not used at all on architectures with strict alignment.
- */
-static int
-igb_fixup_rx(struct rx_ring *rxr)
-{
-	struct adapter *adapter = rxr->adapter;
-	struct mbuf *m, *n;
-	int error;
-
-	error = 0;
-	m = rxr->fmp;
-	if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
-		bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
-		m->m_data += ETHER_HDR_LEN;
-	} else {
-		MGETHDR(n, M_DONTWAIT, MT_DATA);
-		if (n != NULL) {
-			bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
-			m->m_data += ETHER_HDR_LEN;
-			m->m_len -= ETHER_HDR_LEN;
-			n->m_len = ETHER_HDR_LEN;
-			M_MOVE_PKTHDR(n, m);
-			n->m_next = m;
-			rxr->fmp = n;
-		} else {
-			adapter->dropped_pkts++;
-			m_freem(rxr->fmp);
-			rxr->fmp = NULL;
-			error = ENOMEM;
-		}
+	/*
+	** We still have cleaning to do?
+	** Schedule another interrupt if so.
+	*/
+	if (staterr & E1000_RXD_STAT_DD) {
+		E1000_WRITE_REG(&adapter->hw, E1000_EICS, rxr->eims);
+		return TRUE;
 	}
 
-	return (error);
+	return FALSE;
 }
-#endif
+
 
 /*********************************************************************
  *
@@ -4028,7 +4227,7 @@ igb_fixup_rx(struct rx_ring *rxr)
  *
  *********************************************************************/
 static void
-igb_rx_checksum(u32 staterr, struct mbuf *mp)
+igb_rx_checksum(u32 staterr, struct mbuf *mp, bool sctp)
 {
 	u16 status = (u16)staterr;
 	u8  errors = (u8) (staterr >> 24);
@@ -4045,17 +4244,21 @@ igb_rx_checksum(u32 staterr, struct mbuf *mp)
 			/* IP Checksum Good */
 			mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
 			mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
-
 		} else
 			mp->m_pkthdr.csum_flags = 0;
 	}
 
-	if (status & E1000_RXD_STAT_TCPCS) {
+	if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
+		u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
+#if __FreeBSD_version >= 800000
+		if (sctp) /* reassign */
+			type = CSUM_SCTP_VALID;
+#endif
 		/* Did it pass? */
 		if (!(errors & E1000_RXD_ERR_TCPE)) {
-			mp->m_pkthdr.csum_flags |=
-			(CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
-			mp->m_pkthdr.csum_data = htons(0xffff);
+			mp->m_pkthdr.csum_flags = type;
+			if (!sctp)
+				mp->m_pkthdr.csum_data = htons(0xffff);
 		}
 	}
 	return;
@@ -4071,10 +4274,6 @@ igb_register_vlan(void *unused, struct ifnet *ifp, u16 vtag)
 	struct adapter	*adapter = ifp->if_softc;
 	u32		ctrl, rctl, index, vfta;
 
-	/* Shouldn't happen */
-	if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
-		return;
-
 	ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
 	ctrl |= E1000_CTRL_VME;
 	E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
@@ -4107,10 +4306,6 @@ igb_unregister_vlan(void *unused, struct ifnet *ifp, u16 vtag)
 	struct adapter	*adapter = ifp->if_softc;
 	u32		index, vfta;
 
-	/* Shouldn't happen */
-	if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
-		return;
-
 	/* Remove entry in the hardware filter table */
 	index = ((vtag >> 5) & 0x7F);
 	vfta = E1000_READ_REG_ARRAY(&adapter->hw, E1000_VFTA, index);
@@ -4172,8 +4367,6 @@ igb_disable_intr(struct adapter *adapter)
 static void
 igb_init_manageability(struct adapter *adapter)
 {
-	/* A shared code workaround */
-#define E1000_82542_MANC2H E1000_MANC2H
 	if (adapter->has_manage) {
 		int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
 		int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
@@ -4183,12 +4376,9 @@ igb_init_manageability(struct adapter *adapter)
 
                 /* enable receiving management packets to the host */
 		manc |= E1000_MANC_EN_MNG2HOST;
-#define E1000_MNG2HOST_PORT_623 (1 << 5)
-#define E1000_MNG2HOST_PORT_664 (1 << 6)
-		manc2h |= E1000_MNG2HOST_PORT_623;
-		manc2h |= E1000_MNG2HOST_PORT_664;
+		manc2h |= 1 << 5;  /* Mng Port 623 */
+		manc2h |= 1 << 6;  /* Mng Port 664 */
 		E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
-
 		E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
 	}
 }
@@ -4430,6 +4620,8 @@ igb_print_debug_info(struct adapter *adapter)
 		    E1000_READ_REG(&adapter->hw, E1000_RDT(i)));
 		device_printf(dev, "RX(%d) Packets received = %lld\n", rxr->me,
 		    (long long)rxr->rx_packets);
+		device_printf(dev, "RX(%d) Split Packets = %lld\n", rxr->me,
+		    (long long)rxr->rx_split_packets);
 		device_printf(dev, "RX(%d) Byte count = %lld\n", rxr->me,
 		    (long long)rxr->rx_bytes);
 		device_printf(dev, "RX(%d) MSIX IRQ Handled = %lld\n", rxr->me,
@@ -4442,10 +4634,12 @@ igb_print_debug_info(struct adapter *adapter)
 
 	device_printf(dev, "LINK MSIX IRQ Handled = %u\n", adapter->link_irq);
 
-	device_printf(dev, "Std mbuf failed = %ld\n",
-	    adapter->mbuf_alloc_failed);
-	device_printf(dev, "Std mbuf cluster failed = %ld\n",
-	    adapter->mbuf_cluster_failed);
+	device_printf(dev, "Mbuf defrag failed = %ld\n",
+	    adapter->mbuf_defrag_failed);
+	device_printf(dev, "Std mbuf header failed = %ld\n",
+	    adapter->mbuf_header_failed);
+	device_printf(dev, "Std mbuf packet failed = %ld\n",
+	    adapter->mbuf_packet_failed);
 	device_printf(dev, "Driver dropped packets = %ld\n",
 	    adapter->dropped_pkts);
 	device_printf(dev, "Driver tx dma failure in xmit = %ld\n",
diff --git a/sys/dev/e1000/if_igb.h b/sys/dev/e1000/if_igb.h
index c0e5aaa..025a03c 100644
--- a/sys/dev/e1000/if_igb.h
+++ b/sys/dev/e1000/if_igb.h
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -172,7 +172,7 @@
 #define IGB_DEFAULT_PBA			0x00000030
 #define IGB_SMARTSPEED_DOWNSHIFT	3
 #define IGB_SMARTSPEED_MAX		15
-#define IGB_MAX_INTR			10
+#define IGB_MAX_LOOP			10
 #define IGB_RX_PTHRESH			16
 #define IGB_RX_HTHRESH			8
 #define IGB_RX_WTHRESH			1
@@ -184,14 +184,18 @@
 #define IGB_FC_PAUSE_TIME		0x0680
 #define IGB_EEPROM_APME			0x400;
 
-#define MAX_INTS_PER_SEC	8000
-#define DEFAULT_ITR		1000000000/(MAX_INTS_PER_SEC * 256)
-
 /* Code compatilbility between 6 and 7 */
 #ifndef ETHER_BPF_MTAP
 #define ETHER_BPF_MTAP			BPF_MTAP
 #endif
 
+#if __FreeBSD_version < 700000
+#define CSUM_TSO                0
+#define IFCAP_TSO4              0
+#define FILTER_STRAY
+#define FILTER_HANDLED
+#endif
+
 /*
  * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be
  * multiple of 128 bytes. So we align TDBA/RDBA on 128 byte boundary. This will
@@ -230,9 +234,21 @@
 #define IGB_MAX_SCATTER		64
 #define IGB_TSO_SIZE		(65535 + sizeof(struct ether_vlan_header))
 #define IGB_TSO_SEG_SIZE	4096	/* Max dma segment size */
+#define IGB_HDR_BUF		128
 #define ETH_ZLEN		60
 #define ETH_ADDR_LEN		6
-#define CSUM_OFFLOAD		7	/* Offload bits in mbuf flag */
+
+/* Offload bits in mbuf flag */
+#if __FreeBSD_version >= 800000
+#define CSUM_OFFLOAD		(CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP)
+#else
+#define CSUM_OFFLOAD		(CSUM_IP|CSUM_TCP|CSUM_UDP)
+#endif
+
+/* Header split codes for get_buf */
+#define IGB_CLEAN_HEADER		1
+#define IGB_CLEAN_PAYLOAD		2
+#define IGB_CLEAN_BOTH			3
 
 /*
  * Interrupt Moderation parameters
@@ -305,7 +321,7 @@ struct tx_ring {
 	u32			next_avail_desc;
 	u32			next_to_clean;
 	volatile u16		tx_avail;
-	struct igb_buffer	*tx_buffers;
+	struct igb_tx_buffer	*tx_buffers;
 	bus_dma_tag_t		txtag;		/* dma tag for tx */
 	u32			watchdog_timer;
 	u64			no_desc_avail;
@@ -329,7 +345,7 @@ struct rx_ring {
 	char			mtx_name[16];
 	u32			last_cleaned;
 	u32			next_to_check;
-	struct igb_buffer	*rx_buffers;
+	struct igb_rx_buffer	*rx_buffers;
 	bus_dma_tag_t		rxtag;		/* dma tag for tx */
 	bus_dmamap_t		rx_spare_map;
 	/*
@@ -344,6 +360,7 @@ struct rx_ring {
 
 	/* Soft stats */
 	u64			rx_irq;
+	u64			rx_split_packets;
 	u64			rx_packets;
 	u64			rx_bytes;
 };
@@ -380,6 +397,7 @@ struct adapter {
 	struct taskqueue *tq;           /* private task queue */
 	eventhandler_tag vlan_attach;
 	eventhandler_tag vlan_detach;
+
 	/* Management and WOL features */
 	int		wol;
 	int		has_manage;
@@ -402,15 +420,18 @@ struct adapter {
 	 * Receive rings
 	 */
 	struct rx_ring		*rx_rings;
+	bool			rx_hdr_split;
         u16			num_rx_desc;
         u16			num_rx_queues;
 	int			rx_process_limit;
-	u32			rx_buffer_len;
+	u32			rx_mbuf_sz;
+	u32			rx_mask;
 
 	/* Misc stats maintained by the driver */
 	unsigned long	dropped_pkts;
-	unsigned long	mbuf_alloc_failed;
-	unsigned long	mbuf_cluster_failed;
+	unsigned long	mbuf_defrag_failed;
+	unsigned long	mbuf_header_failed;
+	unsigned long	mbuf_packet_failed;
 	unsigned long	no_tx_map_avail;
         unsigned long	no_tx_dma_setup;
 	unsigned long	watchdog_events;
@@ -443,12 +464,18 @@ typedef struct _igb_vendor_info_t {
 } igb_vendor_info_t;
 
 
-struct igb_buffer {
+struct igb_tx_buffer {
 	int		next_eop;  /* Index of the desc to watch */
         struct mbuf    *m_head;
         bus_dmamap_t    map;         /* bus_dma map for packet */
 };
 
+struct igb_rx_buffer {
+        struct mbuf    *m_head;
+        struct mbuf    *m_pack;
+        bus_dmamap_t    map;         /* bus_dma map for packet */
+};
+
 #define	IGB_CORE_LOCK_INIT(_sc, _name) \
 	mtx_init(&(_sc)->core_mtx, _name, "IGB Core Lock", MTX_DEF)
 #define	IGB_CORE_LOCK_DESTROY(_sc)	mtx_destroy(&(_sc)->core_mtx)
diff --git a/sys/dev/ed/ax88x90reg.h b/sys/dev/ed/ax88x90reg.h
index 9d2eeed..d28d5aa 100644
--- a/sys/dev/ed/ax88x90reg.h
+++ b/sys/dev/ed/ax88x90reg.h
@@ -30,7 +30,7 @@
 /* AX88x90 based miibus defines */
 #define ED_AX88X90_MIIBUS	0x04	/* MII bus register on ASIC */
 #define ED_AX88X90_MII_CLK	    0x01
-#define ED_AX88X90_MII_DIROUT	    0x02
+#define ED_AX88X90_MII_DIRIN	    0x02
 #define ED_AX88X90_MII_DATAIN	    0x04
 #define ED_AX88X90_MII_DATAOUT	    0x08
 #define	ED_AX88X90_TEST		0x05	/* "test" register on asic */
diff --git a/sys/dev/ed/dl100xxreg.h b/sys/dev/ed/dl100xxreg.h
index a4659b0..3b9f1c9 100644
--- a/sys/dev/ed/dl100xxreg.h
+++ b/sys/dev/ed/dl100xxreg.h
@@ -29,14 +29,15 @@
 
 /* Dlink chipset used on some Netgear and Dlink PCMCIA cards */
 #define ED_DL100XX_MIIBUS	0x0c	/* MII bus register on ASIC */
-#define ED_DL100XX_DIAG		0x0d
-#define ED_DL100XX_COLLISON_DIS	    4	/* Disable collision detection */
+#define ED_DL10022_DIAG		0x0d
+#define ED_DL10022_COLLISON_DIS	    4	/* Disable collision detection */
 
-#define ED_DL100XX_MII_RESET1	0x04
-#define ED_DL100XX_MII_RESET2	0x08
+#define ED_DL10022_MII_RESET1	0x04
+#define ED_DL10022_MII_RESET2	0x08
 
 #define ED_DL100XX_MII_DATAIN	0x10
-#define ED_DL100XX_MII_DIROUT_22	0x20
-#define ED_DL100XX_MII_DIROUT_19	0x10
+#define ED_DL10022_MII_DIROUT	0x20
+#define ED_DL10019_MII_DIROUT	0x10
+#define ED_DL100XX_MII_DIROUT	(ED_DL10022_MII_DIROUT | ED_DL10019_MII_DIROUT)
 #define ED_DL100XX_MII_DATAOUT	0x40
 #define ED_DL100XX_MII_CLK	0x80
diff --git a/sys/dev/ed/if_ed.c b/sys/dev/ed/if_ed.c
index a11beda..f9a32bf 100644
--- a/sys/dev/ed/if_ed.c
+++ b/sys/dev/ed/if_ed.c
@@ -389,8 +389,11 @@ ed_detach(device_t dev)
 		callout_drain(&sc->tick_ch);
 		ether_ifdetach(ifp);
 	}
-	bus_teardown_intr(dev, sc->irq_res, sc->irq_handle);
+	if (sc->irq_res != NULL && sc->irq_handle)
+		bus_teardown_intr(dev, sc->irq_res, sc->irq_handle);
 	ed_release_resources(dev);
+	if (sc->miibus)
+		device_delete_child(dev, sc->miibus);
 	ED_LOCK_DESTROY(sc);
 	bus_generic_detach(dev);
 	return (0);
@@ -426,10 +429,19 @@ ed_stop_hw(struct ed_softc *sc)
 	 * Wait for interface to enter stopped state, but limit # of checks to
 	 * 'n' (about 5ms). It shouldn't even take 5us on modern DS8390's, but
 	 * just in case it's an old one.
+	 *
+	 * The AX88x90 chips don't seem to implement this behavor.  The
+	 * datasheets say it is only turned on when the chip enters a RESET
+	 * state and is silent about behavior for the stopped state we just
+	 * entered.
 	 */
-	if (sc->chip_type != ED_CHIP_TYPE_AX88190)
-		while (((ed_nic_inb(sc, ED_P0_ISR) & ED_ISR_RST) == 0) && --n)
-			continue;
+	if (sc->chip_type == ED_CHIP_TYPE_AX88190 ||
+	    sc->chip_type == ED_CHIP_TYPE_AX88790)
+		return;
+	while (((ed_nic_inb(sc, ED_P0_ISR) & ED_ISR_RST) == 0) && --n)
+		continue;
+	if (n <= 0)
+		device_printf(sc->dev, "ed_stop_hw RST never set\n");
 }
 
 /*
@@ -806,14 +818,15 @@ ed_rint(struct ed_softc *sc)
 			/*
 			 * Length is a wild value. There's a good chance that
 			 * this was caused by the NIC being old and buggy.
-			 * The bug is that the length low byte is duplicated in
-			 * the high byte. Try to recalculate the length based on
-			 * the pointer to the next packet.
-			 */
-			/*
-			 * NOTE: sc->next_packet is pointing at the current packet.
+			 * The bug is that the length low byte is duplicated
+			 * in the high byte. Try to recalculate the length
+			 * based on the pointer to the next packet.  Also,
+			 * need ot preserve offset into page.
+			 *
+			 * NOTE: sc->next_packet is pointing at the current
+			 * packet.
 			 */
-			len &= ED_PAGE_SIZE - 1;	/* preserve offset into page */
+			len &= ED_PAGE_SIZE - 1;
 			if (packet_hdr.next_packet >= sc->next_packet)
 				len += (packet_hdr.next_packet -
 				    sc->next_packet) * ED_PAGE_SIZE;
@@ -834,14 +847,14 @@ ed_rint(struct ed_softc *sc)
 		}
 
 		/*
-		 * Be fairly liberal about what we allow as a "reasonable" length
-		 * so that a [crufty] packet will make it to BPF (and can thus
-		 * be analyzed). Note that all that is really important is that
-		 * we have a length that will fit into one mbuf cluster or less;
-		 * the upper layer protocols can then figure out the length from
-		 * their own length field(s).
-		 * But make sure that we have at least a full ethernet header
-		 * or we would be unable to call ether_input() later.
+		 * Be fairly liberal about what we allow as a "reasonable"
+		 * length so that a [crufty] packet will make it to BPF (and
+		 * can thus be analyzed). Note that all that is really
+		 * important is that we have a length that will fit into one
+		 * mbuf cluster or less; the upper layer protocols can then
+		 * figure out the length from their own length field(s).  But
+		 * make sure that we have at least a full ethernet header or
+		 * we would be unable to call ether_input() later.
 		 */
 		if ((len >= sizeof(struct ed_ring) + ETHER_HDR_LEN) &&
 		    (len <= MCLBYTES) &&
@@ -914,10 +927,10 @@ edintr(void *arg)
 	ed_nic_outb(sc, ED_P0_CR, sc->cr_proto | ED_CR_STA);
 
 	/*
-	 * loop until there are no more new interrupts.  When the card
-	 * goes away, the hardware will read back 0xff.  Looking at
-	 * the interrupts, it would appear that 0xff is impossible,
-	 * or at least extremely unlikely.
+	 * loop until there are no more new interrupts.  When the card goes
+	 * away, the hardware will read back 0xff.  Looking at the interrupts,
+	 * it would appear that 0xff is impossible, or at least extremely
+	 * unlikely.
 	 */
 	while ((isr = ed_nic_inb(sc, ED_P0_ISR)) != 0 && isr != 0xff) {
 
@@ -928,12 +941,14 @@ edintr(void *arg)
 		 */
 		ed_nic_outb(sc, ED_P0_ISR, isr);
 
-		/* 
-		 * XXX workaround for AX88190
+		/*
+		 * The AX88190 and AX88190A has problems acking an interrupt
+		 * and having them clear.  This interferes with top-level loop
+		 * here.  Wait for all the bits to clear.
+		 *
 		 * We limit this to 5000 iterations.  At 1us per inb/outb,
-		 * this translates to about 15ms, which should be plenty
-		 * of time, and also gives protection in the card eject
-		 * case.
+		 * this translates to about 15ms, which should be plenty of
+		 * time, and also gives protection in the card eject case.
 		 */
 		if (sc->chip_type == ED_CHIP_TYPE_AX88190) {
 			count = 5000;		/* 15ms */
@@ -1309,7 +1324,7 @@ ed_shmem_readmem16(struct ed_softc *sc, bus_size_t src, uint8_t *dst,
     uint16_t amount)
 {
 	bus_space_read_region_2(sc->mem_bst, sc->mem_bsh, src, (uint16_t *)dst,
-	    amount + 1 / 2);
+	    (amount + 1) / 2);
 }
 
 /*
@@ -1528,7 +1543,8 @@ ed_setrcr(struct ed_softc *sc)
 	ED_ASSERT_LOCKED(sc);
 
 	/* Bit 6 in AX88190 RCR register must be set. */
-	if (sc->chip_type == ED_CHIP_TYPE_AX88190)
+	if (sc->chip_type == ED_CHIP_TYPE_AX88190 ||
+	    sc->chip_type == ED_CHIP_TYPE_AX88790)
 		reg1 = ED_RCR_INTT;
 	else
 		reg1 = 0x00;
@@ -1723,3 +1739,38 @@ ed_shmem_write_mbufs(struct ed_softc *sc, struct mbuf *m, bus_size_t dst)
 	}
 	return (len);
 }
+
+/*
+ * Generic ifmedia support.  By default, the DP8390-based cards don't know
+ * what their network attachment really is, or even if it is valid (except
+ * upon successful transmission of a packet).  To play nicer with dhclient, as
+ * well as to fit in with a framework where some cards can provde more
+ * detailed information, make sure that we use this as a fallback.
+ */
+static int
+ed_gen_ifmedia_ioctl(struct ed_softc *sc, struct ifreq *ifr, u_long command)
+{
+	return (ifmedia_ioctl(sc->ifp, ifr, &sc->ifmedia, command));
+}
+
+static int
+ed_gen_ifmedia_upd(struct ifnet *ifp)
+{
+	return 0;
+}
+
+static void
+ed_gen_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
+{
+	ifmr->ifm_active = IFM_ETHER | IFM_AUTO;
+	ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE;
+}
+
+void
+ed_gen_ifmedia_init(struct ed_softc *sc)
+{
+	sc->sc_media_ioctl = &ed_gen_ifmedia_ioctl;
+	ifmedia_init(&sc->ifmedia, 0, ed_gen_ifmedia_upd, ed_gen_ifmedia_sts);
+	ifmedia_add(&sc->ifmedia, IFM_ETHER | IFM_AUTO, 0, 0);
+	ifmedia_set(&sc->ifmedia, IFM_ETHER | IFM_AUTO);
+}
diff --git a/sys/dev/ed/if_ed_cbus.c b/sys/dev/ed/if_ed_cbus.c
index dc4fabb..bcc0d85 100644
--- a/sys/dev/ed/if_ed_cbus.c
+++ b/sys/dev/ed/if_ed_cbus.c
@@ -248,7 +248,8 @@ ed_cbus_attach(dev)
 		ed_release_resources(dev);
 		return (error);
 	}
-
+	if (sc->sc_media_ioctl == NULL)
+		ed_gen_ifmedia_init(sc);
 	return ed_attach(dev);
 }
 
diff --git a/sys/dev/ed/if_ed_isa.c b/sys/dev/ed/if_ed_isa.c
index 7d30614..1beac27 100644
--- a/sys/dev/ed/if_ed_isa.c
+++ b/sys/dev/ed/if_ed_isa.c
@@ -175,7 +175,8 @@ ed_isa_attach(device_t dev)
 		ed_release_resources(dev);
 		return (error);
 	}
-
+	if (sc->sc_media_ioctl == NULL)
+		ed_gen_ifmedia_init(sc);
 	return ed_attach(dev);
 }
 
diff --git a/sys/dev/ed/if_ed_pccard.c b/sys/dev/ed/if_ed_pccard.c
index 28ccfc2..fa8a86d 100644
--- a/sys/dev/ed/if_ed_pccard.c
+++ b/sys/dev/ed/if_ed_pccard.c
@@ -45,6 +45,7 @@
  *	2 loopback
  *      1 gdlink (tpi mode only) 1 tp good, 0 tp bad
  *	0 0-no mam, 1 mam connected
+ *
  * NS83926 appears to be a NS pcmcia glue chip used on the IBM Ethernet II
  * and the NEC PC9801N-J12 ccr base 0x2000!
  *
@@ -64,8 +65,6 @@
  *	0-1 PHY01	00 auto, 01 res, 10 10B5, 11 TPI
  *	2 GDLINK	1 disable checking of link
  *	6 LINK		0 bad link, 1 good link
- * TMI tc5299 10/100 chip, has a different MII interaction than
- * dl100xx and ax88x90.
  *
  * EN5017A, EN5020	no data, but very popular
  * Other chips?
@@ -114,9 +113,12 @@
  * memory somewhere that isn't in the CIS.  Some new chipsets have it
  * in special registers in the ASIC part of the chip.
  *
- * For those cards that have the MAC adress stored in attribute memory,
- * nearly all of them have it at a fixed offset (0xff0).  We use that
- * offset as a source of last resource if other offsets have failed.
+ * For those cards that have the MAC adress stored in attribute memory
+ * outside of a FUNCE entry in the CIS, nearly all of them have it at
+ * a fixed offset (0xff0).  We use that offset as a source of last
+ * resource if other offsets have failed.  This is the address of the
+ * National Semiconductor DP83903A, which is the only chip's datasheet
+ * I've found.
  */
 #define ED_DEFAULT_MAC_OFFSET	0xff0
 
@@ -126,6 +128,7 @@ static const struct ed_product {
 #define	NE2000DVF_DL100XX	0x0001		/* chip is D-Link DL10019/22 */
 #define	NE2000DVF_AX88X90	0x0002		/* chip is ASIX AX88[17]90 */
 #define NE2000DVF_TC5299J	0x0004		/* chip is Tamarack TC5299J */
+#define NE2000DVF_TOSHIBA	0x0008		/* Toshiba DP83902A */
 #define NE2000DVF_ENADDR	0x0100		/* Get MAC from attr mem */
 #define NE2000DVF_ANYFUNC	0x0200		/* Allow any function type */
 #define NE2000DVF_MODEM		0x0400		/* Has a modem/serial */
@@ -138,6 +141,7 @@ static const struct ed_product {
 	{ PCMCIA_CARD(BILLIONTON, CFLT10N), 0},
 	{ PCMCIA_CARD(BILLIONTON, LNA100B), NE2000DVF_AX88X90},
 	{ PCMCIA_CARD(BILLIONTON, LNT10TN), 0},
+	{ PCMCIA_CARD(BROMAX, AXNET), NE2000DVF_AX88X90},
 	{ PCMCIA_CARD(BROMAX, IPORT), 0},
 	{ PCMCIA_CARD(BROMAX, IPORT2), 0},
 	{ PCMCIA_CARD(BUFFALO, LPC2_CLT), 0},
@@ -161,23 +165,24 @@ static const struct ed_product {
 	{ PCMCIA_CARD(COREGA, LAPCCTXD), 0},
 	{ PCMCIA_CARD(DAYNA, COMMUNICARD_E_1), 0},
 	{ PCMCIA_CARD(DAYNA, COMMUNICARD_E_2), 0},
-	{ PCMCIA_CARD(DLINK, DE650), 0 },
+	{ PCMCIA_CARD(DLINK, DE650), NE2000DVF_ANYFUNC },
 	{ PCMCIA_CARD(DLINK, DE660), 0 },
 	{ PCMCIA_CARD(DLINK, DE660PLUS), 0},
 	{ PCMCIA_CARD(DYNALINK, L10C), 0},
 	{ PCMCIA_CARD(EDIMAX, EP4000A), 0},
-	{ PCMCIA_CARD(EPSON, EEN10B), NE2000DVF_ENADDR, 0xff0},
+	{ PCMCIA_CARD(EPSON, EEN10B), 0},
 	{ PCMCIA_CARD(EXP, THINLANCOMBO), 0},
 	{ PCMCIA_CARD(GLOBALVILLAGE, LANMODEM), 0},
 	{ PCMCIA_CARD(GREY_CELL, TDK3000), 0},
 	{ PCMCIA_CARD(GREY_CELL, DMF650TX),
 	    NE2000DVF_ANYFUNC | NE2000DVF_DL100XX | NE2000DVF_MODEM},
 	{ PCMCIA_CARD(IBM, HOME_AND_AWAY), 0},
-	{ PCMCIA_CARD(IBM, INFOMOVER), NE2000DVF_ENADDR, 0xff0},
+	{ PCMCIA_CARD(IBM, INFOMOVER), 0},
 	{ PCMCIA_CARD(IODATA3, PCLAT), 0},
 	{ PCMCIA_CARD(KINGSTON, CIO10T), 0},
 	{ PCMCIA_CARD(KINGSTON, KNE2), 0},
 	{ PCMCIA_CARD(LANTECH, FASTNETTX), NE2000DVF_AX88X90},
+	/* Same ID for many different cards, including generic NE2000 */
 	{ PCMCIA_CARD(LINKSYS, COMBO_ECARD),
 	    NE2000DVF_DL100XX | NE2000DVF_AX88X90},
 	{ PCMCIA_CARD(LINKSYS, ECARD_1), 0},
@@ -188,6 +193,7 @@ static const struct ed_product {
 	{ PCMCIA_CARD(MAGICRAM, ETHER), 0},
 	{ PCMCIA_CARD(MELCO, LPC3_CLX), NE2000DVF_AX88X90},
 	{ PCMCIA_CARD(MELCO, LPC3_TX), NE2000DVF_AX88X90},
+	{ PCMCIA_CARD(MITSUBISHI, B8895), NE2000DVF_ANYFUNC}, /* NG */
 	{ PCMCIA_CARD(MICRORESEARCH, MR10TPC), 0},
 	{ PCMCIA_CARD(NDC, ND5100_E), 0},
 	{ PCMCIA_CARD(NETGEAR, FA410TXC), NE2000DVF_DL100XX},
@@ -206,6 +212,7 @@ static const struct ed_product {
 	{ PCMCIA_CARD(RACORE, FASTENET), NE2000DVF_AX88X90},
 	{ PCMCIA_CARD(RACORE, 8041TX), NE2000DVF_AX88X90 | NE2000DVF_TC5299J},
 	{ PCMCIA_CARD(RELIA, COMBO), 0},
+	{ PCMCIA_CARD(RIOS, PCCARD3), 0},
 	{ PCMCIA_CARD(RPTI, EP400), 0},
 	{ PCMCIA_CARD(RPTI, EP401), 0},
 	{ PCMCIA_CARD(SMC, EZCARD), 0},
@@ -223,6 +230,7 @@ static const struct ed_product {
 	{ PCMCIA_CARD(TELECOMDEVICE, LM5LT), 0 },
 	{ PCMCIA_CARD(TELECOMDEVICE, TCD_HPC100), NE2000DVF_AX88X90},
 	{ PCMCIA_CARD(TJ, PTJ_LAN_T), 0 },
+	{ PCMCIA_CARD(TOSHIBA2, LANCT00A), NE2000DVF_ANYFUNC | NE2000DVF_TOSHIBA},
 	{ PCMCIA_CARD(ZONET, ZEN), 0},
 	{ { NULL } }
 };
@@ -232,6 +240,7 @@ static const struct ed_product {
  */
 static int	ed_pccard_probe(device_t);
 static int	ed_pccard_attach(device_t);
+static void	ed_pccard_tick(void *);
 
 static int	ed_pccard_dl100xx(device_t dev, const struct ed_product *);
 static void	ed_pccard_dl100xx_mii_reset(struct ed_softc *sc);
@@ -240,7 +249,6 @@ static void	ed_pccard_dl100xx_mii_writebits(struct ed_softc *sc, u_int val,
     int nbits);
 
 static int	ed_pccard_ax88x90(device_t dev, const struct ed_product *);
-static void	ed_pccard_ax88x90_mii_reset(struct ed_softc *sc);
 static u_int	ed_pccard_ax88x90_mii_readbits(struct ed_softc *sc, int nbits);
 static void	ed_pccard_ax88x90_mii_writebits(struct ed_softc *sc, u_int val,
     int nbits);
@@ -250,7 +258,6 @@ static int	ed_ifmedia_upd(struct ifnet *);
 static void	ed_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 
 static int	ed_pccard_tc5299j(device_t dev, const struct ed_product *);
-static void	ed_pccard_tc5299j_mii_reset(struct ed_softc *sc);
 static u_int	ed_pccard_tc5299j_mii_readbits(struct ed_softc *sc, int nbits);
 static void	ed_pccard_tc5299j_mii_writebits(struct ed_softc *sc, u_int val,
     int nbits);
@@ -359,7 +366,7 @@ ed_pccard_rom_mac(device_t dev, uint8_t *enaddr)
 	 */
 	ed_pio_readmem(sc, 0, romdata, 32);
 	if (bootverbose)
-		printf("ROM DATA: %32D\n", romdata, " ");
+		device_printf(dev, "ROM DATA: %32D\n", romdata, " ");
 	if (romdata[28] != 0x57 || romdata[30] != 0x57)
 		return (0);
 	for (i = 0; i < ETHER_ADDR_LEN; i++)
@@ -378,6 +385,26 @@ ed_pccard_add_modem(device_t dev)
 }
 
 static int
+ed_pccard_kick_phy(struct ed_softc *sc)
+{
+	struct mii_softc *miisc;
+	struct mii_data *mii;
+
+	/*
+	 * Many of the PHYs that wind up on PC Cards are weird in
+	 * this way.  Generally, we don't need to worry so much about
+	 * the Isolation protocol since there's only one PHY in
+	 * these designs, so this workaround is reasonable.
+	 */
+	mii = device_get_softc(sc->miibus);
+	LIST_FOREACH(miisc, &mii->mii_phys, mii_list) {
+		miisc->mii_flags |= MIIF_FORCEANEG;
+		mii_phy_reset(miisc);
+	}
+	return (mii_mediachg(mii));
+}
+
+static int
 ed_pccard_media_ioctl(struct ed_softc *sc, struct ifreq *ifr, u_long command)
 {
 	struct mii_data *mii;
@@ -400,40 +427,18 @@ ed_pccard_mediachg(struct ed_softc *sc)
 	mii_mediachg(mii);
 }
 
-static void
-ed_pccard_tick(void *arg)
-{
-	struct ed_softc *sc = arg;
-	struct mii_data *mii;
-	int media = 0;
-
-	ED_ASSERT_LOCKED(sc);
-	if (sc->miibus != NULL) {
-		mii = device_get_softc(sc->miibus);
-		media = mii->mii_media_status;
-		mii_tick(mii);
-		if (mii->mii_media_status & IFM_ACTIVE &&
-		    media != mii->mii_media_status && 0 &&
-		    sc->chip_type == ED_CHIP_TYPE_DL10022) {
-			ed_asic_outb(sc, ED_DL100XX_DIAG,
-			    (mii->mii_media_active & IFM_FDX) ?
-			    ED_DL100XX_COLLISON_DIS : 0);
-		}
-		
-	}
-	callout_reset(&sc->tick_ch, hz, ed_pccard_tick, sc);
-}
-
 static int
 ed_pccard_attach(device_t dev)
 {
 	u_char sum;
 	u_char enaddr[ETHER_ADDR_LEN];
 	const struct ed_product *pp;
-	int	error, i;
+	int	error, i, flags;
 	struct ed_softc *sc = device_get_softc(dev);
 	u_long size;
+	static uint16_t *intr_vals[] = {NULL, NULL};
 
+	sc->dev = dev;
 	if ((pp = (const struct ed_product *) pccard_product_lookup(dev, 
 	    (const struct pccard_product *) ed_pccard_products,
 	    sizeof(ed_pccard_products[0]), NULL)) == NULL)
@@ -464,12 +469,14 @@ ed_pccard_attach(device_t dev)
 		goto bad;
 
 	/*
-	 * Determine which chipset we are.  All the PC Card chipsets have the
-	 * ASIC and NIC offsets in the same place.
+	 * Determine which chipset we are.  Almost all the PC Card chipsets
+	 * have the Novel ASIC and NIC offsets.  There's 2 known cards that
+	 * follow the WD80x3 conventions, which are handled as a special case.
 	 */
 	sc->asic_offset = ED_NOVELL_ASIC_OFFSET;
 	sc->nic_offset  = ED_NOVELL_NIC_OFFSET;
 	error = ENXIO;
+	flags = device_get_flags(dev);
 	if (error != 0)
 		error = ed_pccard_dl100xx(dev, pp);
 	if (error != 0)
@@ -477,7 +484,14 @@ ed_pccard_attach(device_t dev)
 	if (error != 0)
 		error = ed_pccard_tc5299j(dev, pp);
 	if (error != 0)
-		error = ed_probe_Novell_generic(dev, device_get_flags(dev));
+		error = ed_probe_Novell_generic(dev, flags);
+	if (error != 0 && (pp->flags & NE2000DVF_TOSHIBA)) {
+		flags |= ED_FLAGS_TOSH_ETHER;
+		flags |= ED_FLAGS_PCCARD;
+		sc->asic_offset = ED_WD_ASIC_OFFSET;
+		sc->nic_offset  = ED_WD_NIC_OFFSET;
+		error = ed_probe_WD80x3_generic(dev, flags, intr_vals);
+	}
 	if (error)
 		goto bad;
 
@@ -489,16 +503,18 @@ ed_pccard_attach(device_t dev)
 	}	      
 
 	/*
-	 * For the older cards, we have to get the MAC address from
-	 * the card in some way.  Let's try the standard PCMCIA way
-	 * first.  If that fails, then check to see if we have valid
-	 * data from the standard NE-2000 data roms.  If that fails,
-	 * check to see if the card has a hint about where to look in
-	 * its CIS.  If that fails, maybe we should look at some
-	 * default value.  In all fails, we should fail the attach,
-	 * but don't right now.
+	 * There are several ways to get the MAC address for the card.
+	 * Some of the above probe routines can fill in the enaddr.  If
+	 * not, we run through a number of 'well known' locations:
+	 *	(1) From the PC Card FUNCE
+	 *	(2) From offset 0 in the shared memory
+	 *	(3) From a hinted offset in attribute memory
+	 *	(4) From 0xff0 in attribute memory
+	 * If we can't get a non-zero MAC address from this list, we fail.
 	 */
-	if (sc->chip_type == ED_CHIP_TYPE_DP8390) {
+	for (i = 0, sum = 0; i < ETHER_ADDR_LEN; i++)
+		sum |= sc->enaddr[i];
+	if (sum == 0) {
 		pccard_get_ether(dev, enaddr);
 		if (bootverbose)
 			device_printf(dev, "CIS MAC %6D\n", enaddr, ":");
@@ -547,16 +563,15 @@ ed_pccard_attach(device_t dev)
 		ed_pccard_dl100xx_mii_reset(sc);
 		(void)mii_phy_probe(dev, &sc->miibus, ed_ifmedia_upd,
 		    ed_ifmedia_sts);
-	} else if (sc->chip_type == ED_CHIP_TYPE_AX88190) {
-		ed_pccard_ax88x90_mii_reset(sc);
+	} else if (sc->chip_type == ED_CHIP_TYPE_AX88190 ||
+	    sc->chip_type == ED_CHIP_TYPE_AX88790) {
 		if ((error = mii_phy_probe(dev, &sc->miibus, ed_ifmedia_upd,
 		     ed_ifmedia_sts)) != 0) {
-			device_printf(dev, "Missing mii!\n");
+			device_printf(dev, "Missing mii %d!\n", error);
 			goto bad;
 		}
 		    
 	} else if (sc->chip_type == ED_CHIP_TYPE_TC5299J) {
-		ed_pccard_tc5299j_mii_reset(sc);
 		if ((error = mii_phy_probe(dev, &sc->miibus, ed_ifmedia_upd,
 		     ed_ifmedia_sts)) != 0) {
 			device_printf(dev, "Missing mii!\n");
@@ -568,6 +583,10 @@ ed_pccard_attach(device_t dev)
 		sc->sc_tick = ed_pccard_tick;
 		sc->sc_mediachg = ed_pccard_mediachg;
 		sc->sc_media_ioctl = ed_pccard_media_ioctl;
+		ed_pccard_kick_phy(sc);
+	} else {
+		printf("Generic ifmedia\n");
+		ed_gen_ifmedia_init(sc);
 	}
 	if (sc->modem_rid != -1)
 		ed_pccard_add_modem(dev);
@@ -580,10 +599,6 @@ bad:
 /*
  * Probe the Ethernet MAC addrees for PCMCIA Linksys EtherFast 10/100 
  * and compatible cards (DL10019C Ethernet controller).
- *
- * Note: The PAO patches try to use more memory for the card, but that
- * seems to fail for my card.  A future optimization would add this back
- * conditionally.
  */
 static int
 ed_pccard_dl100xx(device_t dev, const struct ed_product *pp)
@@ -591,6 +606,7 @@ ed_pccard_dl100xx(device_t dev, const struct ed_product *pp)
 	struct ed_softc *sc = device_get_softc(dev);
 	u_char sum;
 	uint8_t id;
+	u_int   memsize;
 	int i, error;
 
 	if (!(pp->flags & NE2000DVF_DL100XX))
@@ -624,8 +640,26 @@ ed_pccard_dl100xx(device_t dev, const struct ed_product *pp)
 	ed_nic_outb(sc, ED_P0_DCR, ED_DCR_WTS | ED_DCR_FT1 | ED_DCR_LS);
 	id = ed_asic_inb(sc, 0xf);
 	sc->isa16bit = 1;
-	sc->vendor = ED_VENDOR_NOVELL;
+	/*
+	 * Hard code values based on the datasheet.  We're NE-2000 compatible
+	 * NIC with 24kb of packet memory starting at 24k offset.  These
+	 * cards also work with 16k at 16k, but don't work with 24k at 16k
+	 * or 32k at 16k.
+	 */
 	sc->type = ED_TYPE_NE2000;
+	sc->mem_start = 24 * 1024;
+	memsize = sc->mem_size = 24 * 1024;
+	sc->mem_end = sc->mem_start + memsize;
+	sc->tx_page_start = memsize / ED_PAGE_SIZE;
+	sc->txb_cnt = 3;
+	sc->rec_page_start = sc->tx_page_start + sc->txb_cnt * ED_TXBUF_SIZE;
+	sc->rec_page_stop = sc->tx_page_start + memsize / ED_PAGE_SIZE;
+
+	sc->mem_ring = sc->mem_start + sc->txb_cnt * ED_PAGE_SIZE * ED_TXBUF_SIZE;
+
+	ed_nic_outb(sc, ED_P0_PSTART, sc->mem_start / ED_PAGE_SIZE);
+	ed_nic_outb(sc, ED_P0_PSTOP, sc->mem_end / ED_PAGE_SIZE);
+	sc->vendor = ED_VENDOR_NOVELL;
 	sc->chip_type = (id & 0x90) == 0x90 ?
 	    ED_CHIP_TYPE_DL10022 : ED_CHIP_TYPE_DL10019;
 	sc->type_str = ((id & 0x90) == 0x90) ? "DL10022" : "DL10019";
@@ -646,15 +680,15 @@ ed_pccard_dl100xx_mii_reset(struct ed_softc *sc)
 	if (sc->chip_type != ED_CHIP_TYPE_DL10022)
 		return;
 
-	ed_asic_outb(sc, ED_DL100XX_MIIBUS, ED_DL100XX_MII_RESET2);
+	ed_asic_outb(sc, ED_DL100XX_MIIBUS, ED_DL10022_MII_RESET2);
 	DELAY(10);
 	ed_asic_outb(sc, ED_DL100XX_MIIBUS,
-	    ED_DL100XX_MII_RESET2 | ED_DL100XX_MII_RESET1);
+	    ED_DL10022_MII_RESET2 | ED_DL10022_MII_RESET1);
 	DELAY(10);
-	ed_asic_outb(sc, ED_DL100XX_MIIBUS, ED_DL100XX_MII_RESET2);
+	ed_asic_outb(sc, ED_DL100XX_MIIBUS, ED_DL10022_MII_RESET2);
 	DELAY(10);
 	ed_asic_outb(sc, ED_DL100XX_MIIBUS,
-	    ED_DL100XX_MII_RESET2 | ED_DL100XX_MII_RESET1);
+	    ED_DL10022_MII_RESET2 | ED_DL10022_MII_RESET1);
 	DELAY(10);
 	ed_asic_outb(sc, ED_DL100XX_MIIBUS, 0);
 }
@@ -664,21 +698,14 @@ ed_pccard_dl100xx_mii_writebits(struct ed_softc *sc, u_int val, int nbits)
 {
 	int i;
 
-	if (sc->chip_type == ED_CHIP_TYPE_DL10022)
-		DL100XX_MIISET(sc, ED_DL100XX_MII_DIROUT_22);
-	else
-		DL100XX_MIISET(sc, ED_DL100XX_MII_DIROUT_19);
-
+	DL100XX_MIISET(sc, ED_DL100XX_MII_DIROUT);
 	for (i = nbits - 1; i >= 0; i--) {
 		if ((val >> i) & 1)
 			DL100XX_MIISET(sc, ED_DL100XX_MII_DATAOUT);
 		else
 			DL100XX_MIICLR(sc, ED_DL100XX_MII_DATAOUT);
-		DELAY(10);
 		DL100XX_MIISET(sc, ED_DL100XX_MII_CLK);
-		DELAY(10);
 		DL100XX_MIICLR(sc, ED_DL100XX_MII_CLK);
-		DELAY(10);
 	}
 }
 
@@ -688,66 +715,157 @@ ed_pccard_dl100xx_mii_readbits(struct ed_softc *sc, int nbits)
 	int i;
 	u_int val = 0;
 
-	if (sc->chip_type == ED_CHIP_TYPE_DL10022)
-		DL100XX_MIICLR(sc, ED_DL100XX_MII_DIROUT_22);
-	else
-		DL100XX_MIICLR(sc, ED_DL100XX_MII_DIROUT_19);
-
+	DL100XX_MIICLR(sc, ED_DL100XX_MII_DIROUT);
 	for (i = nbits - 1; i >= 0; i--) {
 		DL100XX_MIISET(sc, ED_DL100XX_MII_CLK);
-		DELAY(10);
 		val <<= 1;
 		if (ed_asic_inb(sc, ED_DL100XX_MIIBUS) & ED_DL100XX_MII_DATAIN)
 			val++;
 		DL100XX_MIICLR(sc, ED_DL100XX_MII_CLK);
-		DELAY(10);
 	}
 	return val;
 }
 
-static int
-ed_pccard_ax88x90_geteprom(struct ed_softc *sc)
+static void
+ed_pccard_ax88x90_reset(struct ed_softc *sc)
 {
-	int prom[16],i;
-	u_char tmp;
-	struct {
-		unsigned char offset, value;
-	} pg_seq[] = {
-						/* Select Page0 */
-		{ED_P0_CR, ED_CR_RD2 | ED_CR_STP | ED_CR_PAGE_0},
-		{ED_P0_DCR, 0x01},
-		{ED_P0_RBCR0, 0x00},		/* Clear the count regs. */
-		{ED_P0_RBCR1, 0x00},
-		{ED_P0_IMR, 0x00},		/* Mask completion irq. */
-		{ED_P0_ISR, 0xff},
-		{ED_P0_RCR, ED_RCR_MON | ED_RCR_INTT}, /* Set To Monitor */
-		{ED_P0_TCR, ED_TCR_LB0},	/* loopback mode. */
-		{ED_P0_RBCR0, 32},
-		{ED_P0_RBCR1, 0x00},
-		{ED_P0_RSAR0, 0x00},
-		{ED_P0_RSAR1, 0x04},
-		{ED_P0_CR, ED_CR_RD0 | ED_CR_STA | ED_CR_PAGE_0},
-	};
+	int i;
 
 	/* Reset Card */
-	tmp = ed_asic_inb(sc, ED_NOVELL_RESET);
-	ed_asic_outb(sc, ED_NOVELL_RESET, tmp);
-	DELAY(5000);
 	ed_nic_outb(sc, ED_P0_CR, ED_CR_RD2 | ED_CR_STP | ED_CR_PAGE_0);
-	DELAY(5000);
-
-	/* Card Settings */
-	for (i = 0; i < sizeof(pg_seq) / sizeof(pg_seq[0]); i++)
-		ed_nic_outb(sc, pg_seq[i].offset, pg_seq[i].value);
-
-	/* Get Data */
-	for (i = 0; i < ETHER_ADDR_LEN / 2; i++)
-		prom[i] = ed_asic_inw(sc, 0);
-	for (i = 0; i < ETHER_ADDR_LEN; i += 2) {
-		sc->enaddr[i] = prom[i / 2] & 0xff;
-		sc->enaddr[i + 1] = (prom[i / 2] >> 8) & 0xff;
+	ed_asic_outb(sc, ED_NOVELL_RESET, ed_asic_inb(sc, ED_NOVELL_RESET));
+
+	/* Wait for the RST bit to assert, but cap it at 10ms */
+	for (i = 10000; !(ed_nic_inb(sc, ED_P0_ISR) & ED_ISR_RST) && i > 0;
+	     i--)
+		continue;
+	ed_nic_outb(sc, ED_P0_ISR, ED_ISR_RST);	/* ACK INTR */
+	if (i == 0)
+		device_printf(sc->dev, "Reset didn't finish\n");
+}
+
+/*
+ * Probe and vendor-specific initialization routine for ax88x90 boards
+ */
+static int
+ed_probe_ax88x90_generic(device_t dev, int flags)
+{
+	struct ed_softc *sc = device_get_softc(dev);
+	u_int   memsize;
+	static char test_pattern[32] = "THIS is A memory TEST pattern";
+	char    test_buffer[32];
+
+	ed_pccard_ax88x90_reset(sc);
+	DELAY(10*1000);
+
+	/* Make sure that we really have an 8390 based board */
+	if (!ed_probe_generic8390(sc))
+		return (ENXIO);
+
+	sc->vendor = ED_VENDOR_NOVELL;
+	sc->mem_shared = 0;
+	sc->cr_proto = ED_CR_RD2;
+
+	/*
+	 * This prevents packets from being stored in the NIC memory when the
+	 * readmem routine turns on the start bit in the CR.  We write some
+	 * bytes in word mode and verify we can read them back.  If we can't
+	 * then we don't have an AX88x90 chip here.
+	 */
+	sc->isa16bit = 1;
+	ed_nic_outb(sc, ED_P0_RCR, ED_RCR_MON);
+	ed_nic_outb(sc, ED_P0_DCR, ED_DCR_WTS | ED_DCR_FT1 | ED_DCR_LS);
+	ed_pio_writemem(sc, test_pattern, 16384, sizeof(test_pattern));
+	ed_pio_readmem(sc, 16384, test_buffer, sizeof(test_pattern));
+	if (bcmp(test_pattern, test_buffer, sizeof(test_pattern)) != 0)
+		return (ENXIO);
+
+	/*
+	 * Hard code values based on the datasheet.  We're NE-2000 compatible
+	 * NIC with 16kb of packet memory starting at 16k offset.
+	 */
+	sc->type = ED_TYPE_NE2000;
+	memsize = sc->mem_size = 16*1024;
+	sc->mem_start = 16 * 1024;
+	if (ed_asic_inb(sc, ED_AX88X90_TEST) != 0)
+		sc->chip_type = ED_CHIP_TYPE_AX88790;
+	else {
+		sc->chip_type = ED_CHIP_TYPE_AX88190;
+		/*
+		 * The AX88190 (not A) has external 64k SRAM.  Probe for this
+		 * here.  Most of the cards I have either use the AX88190A
+		 * part, or have only 32k SRAM for some reason, so I don't
+		 * know if this works or not.
+		 */
+		ed_pio_writemem(sc, test_pattern, 32768, sizeof(test_pattern));
+		ed_pio_readmem(sc, 32768, test_buffer, sizeof(test_pattern));
+		if (bcmp(test_pattern, test_buffer, sizeof(test_pattern)) == 0) {
+			sc->mem_start = 2*1024;
+			memsize = sc->mem_size = 62 * 1024;
+		}
+	}
+	sc->mem_end = sc->mem_start + memsize;
+	sc->tx_page_start = memsize / ED_PAGE_SIZE;
+	if (sc->mem_size > 16 * 1024)
+		sc->txb_cnt = 3;
+	else
+		sc->txb_cnt = 2;
+	sc->rec_page_start = sc->tx_page_start + sc->txb_cnt * ED_TXBUF_SIZE;
+	sc->rec_page_stop = sc->tx_page_start + memsize / ED_PAGE_SIZE;
+
+	sc->mem_ring = sc->mem_start + sc->txb_cnt * ED_PAGE_SIZE * ED_TXBUF_SIZE;
+
+	ed_nic_outb(sc, ED_P0_PSTART, sc->mem_start / ED_PAGE_SIZE);
+	ed_nic_outb(sc, ED_P0_PSTOP, sc->mem_end / ED_PAGE_SIZE);
+
+	/* Get the mac before we go -- It's just at 0x400 in "SRAM" */
+	ed_pio_readmem(sc, 0x400, sc->enaddr, ETHER_ADDR_LEN);
+
+	/* clear any pending interrupts that might have occurred above */
+	ed_nic_outb(sc, ED_P0_ISR, 0xff);
+	sc->sc_write_mbufs = ed_pio_write_mbufs;
+	return (0);
+}
+
+static int
+ed_pccard_ax88x90_check_mii(device_t dev, struct ed_softc *sc)
+{
+	int	i, id;
+
+	/*
+	 * All AX88x90 devices have MII and a PHY, so we use this to weed out
+	 * chips that would otherwise make it through the tests we have after
+	 * this point.
+	 */
+	for (i = 0; i < 32; i++) {
+		id = ed_miibus_readreg(dev, i, MII_BMSR);
+		if (id != 0 && id != 0xffff)
+			break;
+	}
+	/*
+	 * Found one, we're good.
+	 */
+	if (i != 32)
+		return (0);
+	/*
+	 * Didn't find anything, so try to power up and try again.  The PHY
+	 * may be not responding because we're in power down mode.
+	 */
+	if (sc->chip_type == ED_CHIP_TYPE_AX88190)
+		return (ENXIO);
+	pccard_ccr_write_1(dev, PCCARD_CCR_STATUS, PCCARD_CCR_STATUS_PWRDWN);
+	for (i = 0; i < 32; i++) {
+		id = ed_miibus_readreg(dev, i, MII_BMSR);
+		if (id != 0 && id != 0xffff)
+			break;
 	}
+	/*
+	 * Still no joy?  We're AFU, punt.
+	 */
+	if (i == 32)
+		return (ENXIO);
 	return (0);
+	
 }
 
 /*
@@ -756,8 +874,8 @@ ed_pccard_ax88x90_geteprom(struct ed_softc *sc)
 static int
 ed_pccard_ax88x90(device_t dev, const struct ed_product *pp)
 {
-	int	error, iobase, i, id;
-	char *ts;
+	int	error;
+	int iobase;
 	struct	ed_softc *sc = device_get_softc(dev);
 
 	if (!(pp->flags & NE2000DVF_AX88X90))
@@ -769,93 +887,48 @@ ed_pccard_ax88x90(device_t dev, const struct ed_product *pp)
 	/*
 	 * Set the IOBASE Register.  The AX88x90 cards are potentially
 	 * multifunction cards, and thus requires a slight workaround.
-	 * We write the address the card is at.
+	 * We write the address the card is at, on the off chance that this
+	 * card is not MFC.
+	 * XXX I'm not sure that this is still needed...
 	 */
 	iobase = rman_get_start(sc->port_res);
 	pccard_ccr_write_1(dev, PCCARD_CCR_IOBASE0, iobase & 0xff);
 	pccard_ccr_write_1(dev, PCCARD_CCR_IOBASE1, (iobase >> 8) & 0xff);
 
-	ts = "AX88190";
-	if (ed_asic_inb(sc, ED_AX88X90_TEST) != 0) {
-		/*
-		 * AX88790 (and I think AX88190A) chips need to be
-		 * powered down.  There's an erratum that says we should
-		 * power down the PHY for 2.5s, but this seems to power
-		 * down the whole card.  I'm unsure why this was done, but
-		 * appears to be required for proper operation.
-		 */
-		pccard_ccr_write_1(dev, PCCARD_CCR_STATUS,
-		    PCCARD_CCR_STATUS_PWRDWN);
-		/*
-		 * Linux axnet driver selects the internal phy for the ax88790
-		 */
-		ed_asic_outb(sc, ED_AX88X90_GPIO, ED_AX88X90_GPIO_INT_PHY);
-		ts = "AX88790";
-	}
-
-	/*
-	 * Check to see if we have a MII PHY ID at any of the first 17
-	 * locations.  All AX88x90 devices have MII and a PHY, so we use
-	 * this to weed out chips that would otherwise make it through
-	 * the tests we have after this point.
-	 */
 	sc->mii_readbits = ed_pccard_ax88x90_mii_readbits;
 	sc->mii_writebits = ed_pccard_ax88x90_mii_writebits;
-	for (i = 0; i < 17; i++) {
-		id = ed_miibus_readreg(dev, i, MII_PHYIDR1);
-		if (id != 0 && id != 0xffff)
-			break;
-	}
-	if (i == 17) {
-		sc->mii_readbits = 0;
-		sc->mii_writebits = 0;
-		return (ENXIO);
+	error = ed_probe_ax88x90_generic(dev, device_get_flags(dev));
+	if (error) {
+		if (bootverbose)
+			device_printf(dev, "probe ax88x90 failed %d\n",
+			    error);
+		goto fail;
 	}
-	
-	sc->chip_type = ED_CHIP_TYPE_AX88190;
-	error = ed_pccard_ax88x90_geteprom(sc);
+	error = ed_pccard_ax88x90_check_mii(dev, sc);
 	if (error)
-		return (error);
-	error = ed_probe_Novell_generic(dev, device_get_flags(dev));
-	if (bootverbose)
-		device_printf(dev, "probe novel returns %d\n", error);
-	if (error == 0) {
-		sc->vendor = ED_VENDOR_NOVELL;
-		sc->type = ED_TYPE_NE2000;
-		sc->chip_type = ED_CHIP_TYPE_AX88190;
-		sc->type_str = ts;
-	}
+		goto fail;
+	sc->vendor = ED_VENDOR_NOVELL;
+	sc->type = ED_TYPE_NE2000;
+	if (sc->chip_type == ED_CHIP_TYPE_AX88190)
+		sc->type_str = "AX88190";
+	else
+		sc->type_str = "AX88790";
+	return (0);
+fail:;
+	sc->mii_readbits = 0;
+	sc->mii_writebits = 0;
 	return (error);
 }
 
-/* MII bit-twiddling routines for cards using AX88x90 chipset */
-#define AX88X90_MIISET(sc, x) ed_asic_outb(sc, ED_AX88X90_MIIBUS, \
-    ed_asic_inb(sc, ED_AX88X90_MIIBUS) | (x))
-#define AX88X90_MIICLR(sc, x) ed_asic_outb(sc, ED_AX88X90_MIIBUS, \
-    ed_asic_inb(sc, ED_AX88X90_MIIBUS) & ~(x))
-
-static void
-ed_pccard_ax88x90_mii_reset(struct ed_softc *sc)
-{
-	/* Do nothing! */
-}
-
 static void
 ed_pccard_ax88x90_mii_writebits(struct ed_softc *sc, u_int val, int nbits)
 {
-	int i;
+	int i, data;
 
-	AX88X90_MIICLR(sc, ED_AX88X90_MII_DIROUT);
 	for (i = nbits - 1; i >= 0; i--) {
-		if ((val >> i) & 1)
-			AX88X90_MIISET(sc, ED_AX88X90_MII_DATAOUT);
-		else
-			AX88X90_MIICLR(sc, ED_AX88X90_MII_DATAOUT);
-		DELAY(10);
-		AX88X90_MIISET(sc, ED_AX88X90_MII_CLK);
-		DELAY(10);
-		AX88X90_MIICLR(sc, ED_AX88X90_MII_CLK);
-		DELAY(10);
+		data = (val >> i) & 1 ? ED_AX88X90_MII_DATAOUT : 0;
+		ed_asic_outb(sc, ED_AX88X90_MIIBUS, data);
+		ed_asic_outb(sc, ED_AX88X90_MIIBUS, data | ED_AX88X90_MII_CLK);
 	}
 }
 
@@ -864,16 +937,15 @@ ed_pccard_ax88x90_mii_readbits(struct ed_softc *sc, int nbits)
 {
 	int i;
 	u_int val = 0;
+	uint8_t mdio;
 
-	AX88X90_MIISET(sc, ED_AX88X90_MII_DIROUT);
+	mdio = ED_AX88X90_MII_DIRIN;
 	for (i = nbits - 1; i >= 0; i--) {
-		AX88X90_MIISET(sc, ED_AX88X90_MII_CLK);
-		DELAY(10);
+		ed_asic_outb(sc, ED_AX88X90_MIIBUS, mdio);
 		val <<= 1;
 		if (ed_asic_inb(sc, ED_AX88X90_MIIBUS) & ED_AX88X90_MII_DATAIN)
 			val++;
-		AX88X90_MIICLR(sc, ED_AX88X90_MII_CLK);
-		DELAY(10);
+		ed_asic_outb(sc, ED_AX88X90_MIIBUS, mdio | ED_AX88X90_MII_CLK);
 	}
 	return val;
 }
@@ -934,36 +1006,24 @@ ed_pccard_tc5299j(device_t dev, const struct ed_product *pp)
 	return (0);
 }
 
-/* MII bit-twiddling routines for cards using TC5299J chipset */
-#define TC5299J_MIISET(sc, x) ed_nic_outb(sc, ED_TC5299J_MIIBUS, \
-    ed_nic_inb(sc, ED_TC5299J_MIIBUS) | (x))
-#define TC5299J_MIICLR(sc, x) ed_nic_outb(sc, ED_TC5299J_MIIBUS, \
-    ed_nic_inb(sc, ED_TC5299J_MIIBUS) & ~(x))
-
-static void
-ed_pccard_tc5299j_mii_reset(struct ed_softc *sc)
-{
-	/* Do nothing! */
-}
-
 static void
 ed_pccard_tc5299j_mii_writebits(struct ed_softc *sc, u_int val, int nbits)
 {
 	int i;
-	uint8_t cr;
+	uint8_t cr, data;
 
+	/* Select page 3 */
 	cr = ed_nic_inb(sc, ED_P0_CR);
 	ed_nic_outb(sc, ED_P0_CR, cr | ED_CR_PAGE_3);
 
-	TC5299J_MIICLR(sc, ED_TC5299J_MII_DIROUT);
 	for (i = nbits - 1; i >= 0; i--) {
-		if ((val >> i) & 1)
-			TC5299J_MIISET(sc, ED_TC5299J_MII_DATAOUT);
-		else
-			TC5299J_MIICLR(sc, ED_TC5299J_MII_DATAOUT);
-		TC5299J_MIISET(sc, ED_TC5299J_MII_CLK);
-		TC5299J_MIICLR(sc, ED_TC5299J_MII_CLK);
+		data = (val >> i) & 1 ? ED_TC5299J_MII_DATAOUT : 0;
+		ed_nic_outb(sc, ED_TC5299J_MIIBUS, data);
+		ed_nic_outb(sc, ED_TC5299J_MIIBUS, data | ED_TC5299J_MII_CLK);
 	}
+	ed_nic_outb(sc, ED_TC5299J_MIIBUS, 0);
+		
+	/* Restore prior page */
 	ed_nic_outb(sc, ED_P0_CR, cr);
 }
 
@@ -974,17 +1034,21 @@ ed_pccard_tc5299j_mii_readbits(struct ed_softc *sc, int nbits)
 	u_int val = 0;
 	uint8_t cr;
 
+	/* Select page 3 */
 	cr = ed_nic_inb(sc, ED_P0_CR);
 	ed_nic_outb(sc, ED_P0_CR, cr | ED_CR_PAGE_3);
 
-	TC5299J_MIISET(sc, ED_TC5299J_MII_DIROUT);
+	ed_asic_outb(sc, ED_TC5299J_MIIBUS, ED_TC5299J_MII_DIROUT);
 	for (i = nbits - 1; i >= 0; i--) {
-		TC5299J_MIISET(sc, ED_TC5299J_MII_CLK);
+		ed_nic_outb(sc, ED_TC5299J_MIIBUS,
+		    ED_TC5299J_MII_CLK | ED_TC5299J_MII_DIROUT);
 		val <<= 1;
 		if (ed_nic_inb(sc, ED_TC5299J_MIIBUS) & ED_TC5299J_MII_DATAIN)
 			val++;
-		TC5299J_MIICLR(sc, ED_TC5299J_MII_CLK);
+		ed_nic_outb(sc, ED_TC5299J_MIIBUS, ED_TC5299J_MII_DIROUT);
 	}
+
+	/* Restore prior page */
 	ed_nic_outb(sc, ED_P0_CR, cr);
 	return val;
 }
@@ -998,23 +1062,45 @@ ed_miibus_readreg(device_t dev, int phy, int reg)
 	struct ed_softc *sc;
 	int failed, val;
 
+	sc = device_get_softc(dev);
 	/*
-	 * The AX88790 seem to have phy 0..f external, and 0x10 internal.
-	 * but they also seem to have a bogus one that shows up at phy
-	 * 0x11 through 0x1f.
+	 * The AX88790 has an interesting quirk.  It has an internal phy that
+	 * needs a special bit set to access, but can also have additional
+	 * external PHYs set for things like HomeNET media.  When accessing
+	 * the internal PHY, a bit has to be set, when accessing the external
+	 * PHYs, it must be clear.  See Errata 1, page 51, in the AX88790
+	 * datasheet for more details.
+	 *
+	 * Also, PHYs above 16 appear to be phantoms on some cards, but not
+	 * others.  Registers read for this are often the same as prior values
+	 * read.  Filter all register requests to 17-31.
+	 *
+	 * I can't explain it, since I don't have the DL100xx data sheets, but
+	 * the DL100xx chips do 13-bits before the 'ACK' but, but the AX88x90
+	 * chips have 14.  The linux pcnet and axnet drivers confirm this.
 	 */
-	if (phy >= 0x11)
-		return (0);
+	if (sc->chip_type == ED_CHIP_TYPE_AX88790) {
+		if (phy > 0x10)
+			return (0);
+		if (phy == 0x10)
+			ed_asic_outb(sc, ED_AX88X90_GPIO,
+			    ED_AX88X90_GPIO_INT_PHY);
+		else
+			ed_asic_outb(sc, ED_AX88X90_GPIO, 0);
+	}
 
-	sc = device_get_softc(dev);
 	(*sc->mii_writebits)(sc, 0xffffffff, 32);
 	(*sc->mii_writebits)(sc, ED_MII_STARTDELIM, ED_MII_STARTDELIM_BITS);
 	(*sc->mii_writebits)(sc, ED_MII_READOP, ED_MII_OP_BITS);
 	(*sc->mii_writebits)(sc, phy, ED_MII_PHY_BITS);
 	(*sc->mii_writebits)(sc, reg, ED_MII_REG_BITS);
+	if (sc->chip_type == ED_CHIP_TYPE_AX88790 ||
+	    sc->chip_type == ED_CHIP_TYPE_AX88190)
+		(*sc->mii_readbits)(sc, ED_MII_ACK_BITS);
 	failed = (*sc->mii_readbits)(sc, ED_MII_ACK_BITS);
 	val = (*sc->mii_readbits)(sc, ED_MII_DATA_BITS);
 	(*sc->mii_writebits)(sc, ED_MII_IDLE, ED_MII_IDLE_BITS);
+/*	printf("Reading phy %d reg %#x returning %#x (valid %d)\n", phy, reg, val, !failed); */
 	return (failed ? 0 : val);
 }
 
@@ -1023,15 +1109,18 @@ ed_miibus_writereg(device_t dev, int phy, int reg, int data)
 {
 	struct ed_softc *sc;
 
-	/*
-	 * The AX88790 seem to have phy 0..f external, and 0x10 internal.
-	 * but they also seem to have a bogus one that shows up at phy
-	 * 0x11 through 0x1f.
-	 */
-	if (phy >= 0x11)
-		return (0);
-
+/*	printf("Writing phy %d reg %#x data %#x\n", phy, reg, data); */
 	sc = device_get_softc(dev);
+	/* See ed_miibus_readreg for details */
+	if (sc->chip_type == ED_CHIP_TYPE_AX88790) {
+		if (phy > 0x10)
+			return (0);
+		if (phy == 0x10)
+			ed_asic_outb(sc, ED_AX88X90_GPIO,
+			    ED_AX88X90_GPIO_INT_PHY);
+		else
+			ed_asic_outb(sc, ED_AX88X90_GPIO, 0);
+	}
 	(*sc->mii_writebits)(sc, 0xffffffff, 32);
 	(*sc->mii_writebits)(sc, ED_MII_STARTDELIM, ED_MII_STARTDELIM_BITS);
 	(*sc->mii_writebits)(sc, ED_MII_WRITEOP, ED_MII_OP_BITS);
@@ -1047,14 +1136,15 @@ static int
 ed_ifmedia_upd(struct ifnet *ifp)
 {
 	struct ed_softc *sc;
-	struct mii_data *mii;
+	int error;
 
 	sc = ifp->if_softc;
 	if (sc->miibus == NULL)
 		return (ENXIO);
-	
-	mii = device_get_softc(sc->miibus);
-	return mii_mediachg(mii);
+	ED_LOCK(sc);
+	error = ed_pccard_kick_phy(sc);
+	ED_UNLOCK(sc);
+	return (error);
 }
 
 static void
@@ -1083,6 +1173,37 @@ ed_child_detached(device_t dev, device_t child)
 		sc->miibus = NULL;
 }
 
+static void
+ed_pccard_tick(void *arg)
+{
+	struct ed_softc *sc = arg;
+	struct mii_data *mii;
+	int media = 0;
+
+	ED_ASSERT_LOCKED(sc);
+	if (sc->miibus != NULL) {
+		mii = device_get_softc(sc->miibus);
+		media = mii->mii_media_status;
+		mii_tick(mii);
+		if (mii->mii_media_status & IFM_ACTIVE &&
+		    media != mii->mii_media_status) {
+			if (sc->chip_type == ED_CHIP_TYPE_DL10022) {
+				ed_asic_outb(sc, ED_DL10022_DIAG,
+				    (mii->mii_media_active & IFM_FDX) ?
+				    ED_DL10022_COLLISON_DIS : 0);
+#ifdef notyet
+			} else if (sc->chip_type == ED_CHIP_TYPE_DL10019) {
+				write_asic(sc, ED_DL10019_MAGIC,
+				    (mii->mii_media_active & IFM_FDX) ?
+				    DL19FDUPLX : 0);
+#endif
+			}
+		}
+		
+	}
+	callout_reset(&sc->tick_ch, hz, ed_pccard_tick, sc);
+}
+
 static device_method_t ed_pccard_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		ed_pccard_probe),
diff --git a/sys/dev/ed/if_ed_pci.c b/sys/dev/ed/if_ed_pci.c
index a2831b8..1b2e17d 100644
--- a/sys/dev/ed/if_ed_pci.c
+++ b/sys/dev/ed/if_ed_pci.c
@@ -91,11 +91,9 @@ ed_pci_attach(device_t dev)
 	int	error = ENXIO;
 
 	/*
-	 * If this card claims to be a RTL8029, probe it as such.
-	 * However, allow that probe to fail.  Some versions of qemu
-	 * claim to be a 8029 in the PCI register, but it doesn't
-	 * implement the 8029 specific registers.  In that case, fall
-	 * back to a normal NE2000.
+	 * Probe RTL8029 cards, but allow failure and try as a generic
+	 * ne-2000.  QEMU 0.9 and earlier use the RTL8029 PCI ID, but
+	 * are areally just generic ne-2000 cards.
 	 */
 	if (pci_get_devid(dev) == ED_RTL8029_PCI_ID)
 		error = ed_probe_RTL80x9(dev, PCIR_BAR(0), flags);
@@ -118,7 +116,8 @@ ed_pci_attach(device_t dev)
 		ed_release_resources(dev);
 		return (error);
 	}
-
+	if (sc->sc_media_ioctl == NULL)
+		ed_gen_ifmedia_init(sc);
 	error = ed_attach(dev);
 	if (error)
 		ed_release_resources(dev);
diff --git a/sys/dev/ed/if_ed_wd80x3.c b/sys/dev/ed/if_ed_wd80x3.c
index b3cb439..4ff2286 100644
--- a/sys/dev/ed/if_ed_wd80x3.c
+++ b/sys/dev/ed/if_ed_wd80x3.c
@@ -119,7 +119,6 @@ ed_probe_WD80x3_generic(device_t dev, int flags, uint16_t *intr_vals[])
 		sum += ed_asic_inb(sc, ED_WD_PROM + i);
 
 	if (sum != totalsum) {
-
 		/*
 		 * Checksum is invalid. This often happens with cheap WD8003E
 		 * clones.  In this case, the checksum byte (the eighth byte)
@@ -268,9 +267,11 @@ ed_probe_WD80x3_generic(device_t dev, int flags, uint16_t *intr_vals[])
 		printf("%x -> %x\n", i, ed_asic_inb(sc, i));
 #endif
 	pmem = rman_get_start(sc->mem_res);
-	error = ed_isa_mem_ok(dev, pmem, memsize);
-	if (error)
-		return (error);
+	if (!(flags & ED_FLAGS_PCCARD)) {
+		error = ed_isa_mem_ok(dev, pmem, memsize);
+		if (error)
+			return (error);
+	}
 
 	/*
 	 * (note that if the user specifies both of the following flags that
diff --git a/sys/dev/ed/if_edreg.h b/sys/dev/ed/if_edreg.h
index 3f2b719..9820dfd 100644
--- a/sys/dev/ed/if_edreg.h
+++ b/sys/dev/ed/if_edreg.h
@@ -1066,19 +1066,23 @@ struct ed_ring	{
 /*
  * Chip types.
  */
-
-#define ED_CHIP_TYPE_DP8390	0
-#define ED_CHIP_TYPE_WD790	1
-#define ED_CHIP_TYPE_AX88190	2
-#define ED_CHIP_TYPE_DL10019	3
-#define ED_CHIP_TYPE_DL10022	4
-#define ED_CHIP_TYPE_TC5299J	5
-#define ED_CHIP_TYPE_RTL8019	6
-#define ED_CHIP_TYPE_RTL8029	7
+#define ED_CHIP_TYPE_AX88190	0
+#define ED_CHIP_TYPE_AX88790	1
+#define ED_CHIP_TYPE_DL10019	2
+#define ED_CHIP_TYPE_DL10022	3
+#define ED_CHIP_TYPE_DP8390	4
+#define ED_CHIP_TYPE_NS83903	5
+#define ED_CHIP_TYPE_NS83926	6
+#define ED_CHIP_TYPE_RTL8019	7
+#define ED_CHIP_TYPE_RTL8029	8
+#define ED_CHIP_TYPE_TC3299	9
+#define ED_CHIP_TYPE_TC5299J	10
+#define ED_CHIP_TYPE_W89C926	11
+#define ED_CHIP_TYPE_WD790	12
 
 /*
  * MII bus definitions.  These are common to both DL100xx and AX88x90
- * MII definitions, most likely because they are standards based.
+ * MII definitions, because they are standards based.
  */
 #define ED_MII_STARTDELIM	0x01
 #define ED_MII_WRITEOP		0x01
@@ -1091,6 +1095,6 @@ struct ed_ring	{
 #define ED_MII_PHY_BITS		5
 #define ED_MII_REG_BITS		5
 #define ED_MII_TURNAROUND_BITS	2
-#define ED_MII_DATA_BITS	16
 #define ED_MII_ACK_BITS		1
+#define ED_MII_DATA_BITS	16
 #define ED_MII_IDLE_BITS	1
diff --git a/sys/dev/ed/if_edvar.h b/sys/dev/ed/if_edvar.h
index 170558e..27dd669 100644
--- a/sys/dev/ed/if_edvar.h
+++ b/sys/dev/ed/if_edvar.h
@@ -226,6 +226,8 @@ u_short	ed_pio_write_mbufs(struct ed_softc *, struct mbuf *, bus_size_t);
 void	ed_disable_16bit_access(struct ed_softc *);
 void	ed_enable_16bit_access(struct ed_softc *);
 
+void	ed_gen_ifmedia_init(struct ed_softc *);
+
 driver_intr_t	edintr;
 
 extern devclass_t ed_devclass;
@@ -267,12 +269,15 @@ extern devclass_t ed_devclass;
 #define ED_FLAGS_FORCE_PIO		0x0010
 
 /*
+ * This forces a PC Card, and disables ISA memory range checks
+ */
+#define ED_FLAGS_PCCARD			0x0020
+
+/*
  * These are flags describing the chip type.
  */
 #define ED_FLAGS_TOSH_ETHER		0x10000
 #define ED_FLAGS_GWETHER		0x20000
-#define ED_FLAGS_AX88190		0x30000
-#define ED_FLAGS_LINKSYS		0x80000
 
 #define ED_FLAGS_GETTYPE(flg)		((flg) & 0xff0000)
 
diff --git a/sys/dev/ep/if_ep.c b/sys/dev/ep/if_ep.c
index 3b91369..c56639d 100644
--- a/sys/dev/ep/if_ep.c
+++ b/sys/dev/ep/if_ep.c
@@ -435,6 +435,8 @@ epinit_locked(struct ep_softc *sc)
 	if (!sc->epb.mii_trans)
 		ep_ifmedia_upd(ifp);
 
+	if (sc->stat & F_HAS_TX_PLL)
+		CSR_WRITE_2(sc, EP_COMMAND, TX_PLL_ENABLE);
 	CSR_WRITE_2(sc, EP_COMMAND, RX_ENABLE);
 	CSR_WRITE_2(sc, EP_COMMAND, TX_ENABLE);
 
@@ -473,7 +475,7 @@ epstart_locked(struct ifnet *ifp)
 	struct ep_softc *sc;
 	u_int len;
 	struct mbuf *m, *m0;
-	int pad;
+	int pad, started;
 
 	sc = ifp->if_softc;
 	if (sc->gone)
@@ -482,11 +484,15 @@ epstart_locked(struct ifnet *ifp)
 	EP_BUSY_WAIT(sc);
 	if (ifp->if_drv_flags & IFF_DRV_OACTIVE)
 		return;
+	started = 0;
 startagain:
 	/* Sneak a peek at the next packet */
 	IFQ_DRV_DEQUEUE(&ifp->if_snd, m0);
 	if (m0 == NULL)
 		return;
+	if (!started && (sc->stat & F_HAS_TX_PLL))
+		CSR_WRITE_2(sc, EP_COMMAND, TX_PLL_ENABLE);
+	started++;
 	for (len = 0, m = m0; m != NULL; m = m->m_next)
 		len += m->m_len;
 
@@ -895,8 +901,24 @@ static void
 ep_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 {
 	struct ep_softc *sc = ifp->if_softc;
+	uint16_t ms;
 
-	ifmr->ifm_active = sc->ifmedia.ifm_media;
+	switch (IFM_SUBTYPE(sc->ifmedia.ifm_media)) {
+	case IFM_10_T:
+		GO_WINDOW(sc, 4);
+		ms = CSR_READ_2(sc, EP_W4_MEDIA_TYPE);
+		GO_WINDOW(sc, 0);
+		ifmr->ifm_status = IFM_AVALID;
+		if (ms & MT_LB) {
+			ifmr->ifm_status |= IFM_ACTIVE;
+			ifmr->ifm_active = IFM_ETHER | IFM_10_T;
+		} else {
+			ifmr->ifm_active = IFM_ETHER | IFM_NONE;
+		}
+	default:
+		ifmr->ifm_active = sc->ifmedia.ifm_media;
+		break;
+	}
 }
 
 static int
diff --git a/sys/dev/ep/if_ep_pccard.c b/sys/dev/ep/if_ep_pccard.c
index 4309358..98ebaf4 100644
--- a/sys/dev/ep/if_ep_pccard.c
+++ b/sys/dev/ep/if_ep_pccard.c
@@ -68,9 +68,10 @@ struct ep_pccard_product
 
 #define EP_CHIP_589	1	/* Classic 3c5x9 chipset */
 #define EP_CHIP_574	2	/* Roadrunner */
+#define EP_CHIP_C1	3	/* 3c1 */
 
 static const struct ep_pccard_product ep_pccard_products[] = {
-	{ PCMCIA_CARD(3COM, 3C1),		EP_CHIP_589 },
+	{ PCMCIA_CARD(3COM, 3C1),		EP_CHIP_C1 },
 	{ PCMCIA_CARD(3COM, 3C562),		EP_CHIP_589 },
 	{ PCMCIA_CARD(3COM, 3C589),		EP_CHIP_589 },
 	{ PCMCIA_CARD(3COM, 3CXEM556),		EP_CHIP_589 },
@@ -144,19 +145,21 @@ ep_pccard_attach(device_t dev)
 	if ((pp = ep_pccard_lookup(dev)) == NULL)
 		panic("ep_pccard_attach: can't find product in attach.");
 
-	if (pp->chipset == EP_CHIP_589) {
-		sc->epb.mii_trans = 0;
-		sc->epb.cmd_off = 0;
-	} else {
+	if (pp->chipset == EP_CHIP_574) {
 		sc->epb.mii_trans = 1;
 		sc->epb.cmd_off = 2;
+	} else {
+		sc->epb.mii_trans = 0;
+		sc->epb.cmd_off = 0;
 	}
-
 	if ((error = ep_alloc(dev))) {
 		device_printf(dev, "ep_alloc() failed! (%d)\n", error);
 		goto bad;
 	}
 
+	if (pp->chipset == EP_CHIP_C1)
+		sc->stat |= F_HAS_TX_PLL;
+	
 	/* ROM size = 0, ROM base = 0 */
 	/* For now, ignore AUTO SELECT feature of 3C589B and later. */
 	error = ep_get_e(sc, EEPROM_ADDR_CFG, &result);
diff --git a/sys/dev/ep/if_epreg.h b/sys/dev/ep/if_epreg.h
index 25ab182..f3c269f 100644
--- a/sys/dev/ep/if_epreg.h
+++ b/sys/dev/ep/if_epreg.h
@@ -156,6 +156,7 @@
  * Window 2 registers. Station Address Setup/Read
  */
 /* Read/Write */
+#define EP_W2_PHY_MGMT		0x0c
 #define EP_W2_ALT_EEPROM	0x0a
 #define EP_W2_ADDR_5		0x05
 #define EP_W2_ADDR_4		0x04
@@ -406,6 +407,17 @@
 #define LINKBEAT_ENABLE		0x80
 #define	ENABLE_UTP		(JABBER_GUARD_ENABLE | LINKBEAT_ENABLE)
 #define DISABLE_UTP		0x0
+#define	MT_CSD			0x0004	/* CRC Strip disable */
+#define	MT_SQE			0x0008	/* SQE Enable */
+#define	MT_C0			0x0010	/* Collison */
+#define	MT_CS			0x0020	/* Carrier Sense */
+#define MT_J			0x0200	/* Jabber detected */
+#define MT_PL			0x0400	/* Plarity Inverted */
+#define MT_LB			0x0800	/* Link Beat detected */
+#define MT_SQ			0x1000	/* SQE Present */
+#define MT_IN			0x2000	/* Reserved */
+#define MT_CE			0x4000	/* Coax */
+#define MT_TPE			0x8000
 
 /*
  * Misc defines for various things.
diff --git a/sys/dev/ep/if_epvar.h b/sys/dev/ep/if_epvar.h
index 704fe02..922a264 100644
--- a/sys/dev/ep/if_epvar.h
+++ b/sys/dev/ep/if_epvar.h
@@ -57,6 +57,7 @@ struct ep_softc {
 #define F_ENADDR_SKIP		0x002
 #define	F_PROMISC		0x008
 #define	F_ACCESS_32_BITS	0x100
+#define	F_HAS_TX_PLL		0x200
 
 	int gone;		/* adapter is not present (for PCCARD) */
 	struct ep_board epb;
diff --git a/sys/dev/fe/if_fe_pccard.c b/sys/dev/fe/if_fe_pccard.c
index 40c10ee..e1cb10d 100644
--- a/sys/dev/fe/if_fe_pccard.c
+++ b/sys/dev/fe/if_fe_pccard.c
@@ -62,9 +62,10 @@ static int fe_pccard_detach(device_t);
 static const struct fe_pccard_product {
 	struct pccard_product mpp_product;
 	int mpp_flags;
+	int mpp_cfe;
 #define MPP_MBH10302 1
 #define MPP_ANYFUNC 2
-#define MPP_SKIP_TO_CFE_10 4
+#define MPP_SKIP_TO_CFE 4
 } fe_pccard_products[] = {
 	/* These need to be first */
 	{ PCMCIA_CARD(FUJITSU2, FMV_J181), MPP_MBH10302 },
@@ -72,16 +73,17 @@ static const struct fe_pccard_product {
 	{ PCMCIA_CARD(FUJITSU2, FMV_J182A), 0 },
 	{ PCMCIA_CARD(FUJITSU2, ITCFJ182A), 0 },
 	/* These need to be second */
+	{ PCMCIA_CARD(TDK, LAK_CD011), 0 }, 
 	{ PCMCIA_CARD(TDK, LAK_CD021BX), 0 }, 
 	{ PCMCIA_CARD(TDK, LAK_CF010), 0 }, 
 #if 0 /* XXX 86960-based? */
 	{ PCMCIA_CARD(TDK, LAK_DFL9610), 0 }, 
 #endif
-	{ PCMCIA_CARD(CONTEC, CNETPC), 0 },
+	{ PCMCIA_CARD(CONTEC, CNETPC), MPP_SKIP_TO_CFE, 2 },
 	{ PCMCIA_CARD(FUJITSU, LA501), 0 },
 	{ PCMCIA_CARD(FUJITSU, LA10S), 0 },
 	{ PCMCIA_CARD(FUJITSU, NE200T), MPP_MBH10302 },/* Sold by Eagle */
-	{ PCMCIA_CARD(HITACHI, HT_4840), MPP_MBH10302 | MPP_SKIP_TO_CFE_10},
+	{ PCMCIA_CARD(HITACHI, HT_4840), MPP_MBH10302 | MPP_SKIP_TO_CFE, 10 },
 	{ PCMCIA_CARD(RATOC, REX_R280), 0 },
 	{ PCMCIA_CARD(XIRCOM, CE), MPP_ANYFUNC },
 	{ { NULL } }
@@ -108,13 +110,13 @@ fe_pccard_probe(device_t dev)
 			return (error);
 		if (fcn != PCCARD_FUNCTION_NETWORK)
 			return (ENXIO);
-		if (pp->mpp_flags & MPP_SKIP_TO_CFE_10) {
-			for (i = 10; i < 27; i++) {
+		if (pp->mpp_flags & MPP_SKIP_TO_CFE) {
+			for (i = pp->mpp_cfe; i < 32; i++) {
 				if (pccard_select_cfe(dev, i) == 0)
 					goto good;
 			}
 			device_printf(dev,
-			    "Hitachi HT-4840-11 workaround failed\n");
+			    "Failed to map CFE %d or higher\n", pp->mpp_cfe);
 			return ENXIO;
 		}
 	good:;
diff --git a/sys/dev/firewire/firewire.c b/sys/dev/firewire/firewire.c
index 05cf834..c2484ae 100644
--- a/sys/dev/firewire/firewire.c
+++ b/sys/dev/firewire/firewire.c
@@ -685,7 +685,8 @@ fw_init_crom(struct firewire_comm *fc)
 	src->businfo.cyc_clk_acc = 100;
 	src->businfo.max_rec = fc->maxrec;
 	src->businfo.max_rom = MAXROM_4;
-	src->businfo.generation = 0;
+#define FW_GENERATION_CHANGEABLE 2
+	src->businfo.generation = FW_GENERATION_CHANGEABLE;
 	src->businfo.link_spd = fc->speed;
 
 	src->businfo.eui64.hi = fc->eui.hi;
@@ -734,6 +735,7 @@ fw_busreset(struct firewire_comm *fc, uint32_t new_status)
 	struct firewire_dev_comm *fdc;
 	struct crom_src *src;
 	device_t *devlistp;
+	uint32_t *newrom;
 	int i, devcnt;
 
 	FW_GLOCK_ASSERT(fc);
@@ -759,18 +761,31 @@ fw_busreset(struct firewire_comm *fc, uint32_t new_status)
 	}
 
 	src = &fc->crom_src_buf->src;
-	/*
-	 * If the old config rom needs to be overwritten,
-	 * bump the businfo.generation indicator to 
-	 * indicate that we need to be reprobed
-	 */
-	if (bcmp(src, fc->config_rom, CROMSIZE) != 0) {
-		/* generation is a 2 bit field */
-		/* valid values are only from 0 - 3 */
-		src->businfo.generation = 1;
-		bcopy(src, (void *)fc->config_rom, CROMSIZE);
-	} else
-		src->businfo.generation = 0;
+        /*
+         * If the old config rom needs to be overwritten,
+         * bump the businfo.generation indicator to 
+         * indicate that we need to be reprobed
+         * See 1394a-2000 8.3.2.5.4 for more details.
+         * generation starts at 2 and rolls over at 0xF
+         * back to 2.
+         * 
+         * A generation of 0 indicates a device
+         * that is not 1394a-2000 compliant.
+         * A generation of 1 indicates a device that
+         * does not change it's Bus Info Block or 
+         * Configuration ROM.
+         */
+#define FW_MAX_GENERATION 0xF
+	newrom = malloc(CROMSIZE, M_FW, M_NOWAIT | M_ZERO);
+	src = &fc->crom_src_buf->src;
+	crom_load(src, newrom, CROMSIZE);
+	if (bcmp(newrom, fc->config_rom, CROMSIZE) != 0) {
+		if ( src->businfo.generation++ > FW_MAX_GENERATION )
+			src->businfo.generation = FW_GENERATION_CHANGEABLE;
+		bcopy(newrom, (void *)fc->config_rom, CROMSIZE);
+	}
+	free(newrom, M_FW);
+
 }
 
 /* Call once after reboot */
@@ -1590,6 +1605,10 @@ fw_explore_node(struct fw_device *dfwdev)
 		}
 		fwdev->fc = fc;
 		fwdev->eui = binfo->eui64;
+		fwdev->dst = dfwdev->dst;
+		fwdev->maxrec = dfwdev->maxrec;
+		fwdev->status = dfwdev->status;
+
 		/*
 		 * Pre-1394a-2000 didn't have link_spd in
 		 * the Bus Info block, so try and use the 
@@ -1599,7 +1618,7 @@ fw_explore_node(struct fw_device *dfwdev)
 		 * ignore the speed map alltogether. SWB
 		 */
 		if ( binfo->link_spd == FWSPD_S100 /* 0 */) {
-			device_printf(fc->bdev, "%s"
+			device_printf(fc->bdev, "%s: "
 				"Pre 1394a-2000 detected\n",
 				__func__);
 			fwdev->speed = fc->speed_map->speed[fc->nodeid][node];
@@ -1609,22 +1628,24 @@ fw_explore_node(struct fw_device *dfwdev)
 		 * Test this speed with a read to the CSRROM.
 		 * If it fails, slow down the speed and retry.
 		 */
-		while (fwdev->speed > 0) {
+		while (fwdev->speed > FWSPD_S100 /* 0 */) {
 			err = fw_explore_read_quads(fwdev, CSRROMOFF,
             				&speed_test, 1);
-			if (err)
+			if (err) {
+				device_printf(fc->bdev, "%s: fwdev->speed(%s)"
+						" decremented due to negotiation\n",
+						__func__,
+						linkspeed[fwdev->speed]);
 				fwdev->speed--;
-			else
+			} else
 				break;
 				
 		}
-		if (fwdev->speed != binfo->link_spd)
-			device_printf(fc->bdev, "%s: fwdev->speed(%s)"
-						" set lower than binfo->link_spd(%s)\n",
-						__func__,
-						linkspeed[fwdev->speed],
-						linkspeed[binfo->link_spd]);
-		/* inesrt into sorted fwdev list */
+
+		/*
+		 * If the fwdev is not found in the 
+		 * fc->devices TAILQ, then we will add it.
+		 */
 		pfwdev = NULL;
 		STAILQ_FOREACH(tfwdev, &fc->devices, link) {
 			if (tfwdev->eui.hi > fwdev->eui.hi ||
@@ -1641,15 +1662,15 @@ fw_explore_node(struct fw_device *dfwdev)
 		device_printf(fc->bdev, "New %s device ID:%08x%08x\n",
 		    linkspeed[fwdev->speed],
 		    fwdev->eui.hi, fwdev->eui.lo);
-	}
-	fwdev->dst = node;
-	fwdev->status = FWDEVINIT;
-
-	/* unchanged ? */
-	if (bcmp(&csr[0], &fwdev->csrrom[0], sizeof(uint32_t) * 5) == 0) {
-		if (firewire_debug)
-			device_printf(fc->dev, "node%d: crom unchanged\n", node);
-		return (0);
+	} else {
+		fwdev->dst = node;
+		fwdev->status = FWDEVINIT;
+		/* unchanged ? */
+		if (bcmp(&csr[0], &fwdev->csrrom[0], sizeof(uint32_t) * 5) == 0) {
+			if (firewire_debug)
+				device_printf(fc->dev, "node%d: crom unchanged\n", node);
+			return (0);
+		}
 	}
 
 	bzero(&fwdev->csrrom[0], CROMSIZE);
@@ -1661,6 +1682,9 @@ fw_explore_node(struct fw_device *dfwdev)
 	err = fw_explore_csrblock(fwdev, 0x14, 1); /* root directory */
 
 	if (err) {
+		if (firewire_debug)
+			device_printf(fc->dev, "%s: explore csrblock failed err(%d)\n",
+					__func__, err);
 		fwdev->status = FWDEVINVAL;
 		fwdev->csrrom[0] = 0;
 	}
diff --git a/sys/dev/firewire/sbp.h b/sys/dev/firewire/sbp.h
index 9f85fab..fcfa514 100644
--- a/sys/dev/firewire/sbp.h
+++ b/sys/dev/firewire/sbp.h
@@ -121,7 +121,7 @@ struct sbp_status{
 /*   3: Page size not supported */
 /*   4: Access denied */
 #define STATUS_ACCESS_DENY	4
-/*   5: Logical unit not supported */
+#define STATUS_LUR		5
 /*   6: Maximum payload too small */
 /*   7: Reserved for future standardization */
 /*   8: Resource unavailabe */
diff --git a/sys/dev/hptiop/hptiop.h b/sys/dev/hptiop/hptiop.h
index 0f74592..eb48676 100644
--- a/sys/dev/hptiop/hptiop.h
+++ b/sys/dev/hptiop/hptiop.h
@@ -260,7 +260,7 @@ struct hpt_iop_ioctl_param {
 	unsigned long    lpOutBuffer;           /* output data buffer */
 	u_int32_t        nOutBufferSize;        /* size of output data buffer */
 	unsigned long    lpBytesReturned;       /* count of HPT_U8s returned */
-};
+} __packed;
 
 #define HPT_IOCTL_FLAG_OPEN 1
 #define HPT_CTL_CODE_BSD_TO_IOP(x) ((x)-0xff00)
diff --git a/sys/dev/hptmv/access601.h b/sys/dev/hptmv/access601.h
index 26eaab2..51f1ca3 100644
--- a/sys/dev/hptmv/access601.h
+++ b/sys/dev/hptmv/access601.h
@@ -28,6 +28,8 @@
 #ifndef _ACCESS601_H_
 #define _ACCESS601_H_
 
+#ifndef FOR_DEMO
+
 void HPTLIBAPI BeepOn(MV_BUS_ADDR_T BaseAddr);
 void HPTLIBAPI BeepOff(MV_BUS_ADDR_T BaseAddr);
 UCHAR HPTLIBAPI check_protect_circuit(MV_BUS_ADDR_T BaseAddr);
@@ -40,4 +42,21 @@ void HPTLIBAPI set_fail_leds(MV_SATA_ADAPTER *pAdapter, UCHAR mask);
 #define set_fail_leds(pAdapter, mask)
 #endif
 
+int HPTLIBAPI sx508x_ioctl(MV_SATA_ADAPTER *pSataAdapter, UCHAR *indata, ULONG inlen,
+			UCHAR *outdata, ULONG maxoutlen, ULONG *poutlen);
+
+MV_BOOLEAN HPTLIBAPI sx508x_flash_access(MV_SATA_ADAPTER *pSataAdapter,
+			MV_U32 offset, void *value, int size, int reading);
+#else 
+
+#define BeepOn(addr)
+#define BeepOff(addr)
+#define check_protect_circuit(addr) 1
+#define set_fail_led(pAdapter, channel, state)
+#define set_fail_leds(pAdapter, mask)
+#define sx508x_ioctl(pSataAdapter, indata, inlen, outdata, maxoutlen, poutlen) 0
+#define sx508x_flash_access(pSataAdapter, offset, value, size, reading) 0
+
+#endif
+
 #endif
diff --git a/sys/dev/hptmv/amd64-elf.raid.o.uu b/sys/dev/hptmv/amd64-elf.raid.o.uu
index 5b439c9..333a8f6 100644
--- a/sys/dev/hptmv/amd64-elf.raid.o.uu
+++ b/sys/dev/hptmv/amd64-elf.raid.o.uu
@@ -26,1881 +26,1909 @@
  * $FreeBSD$
  */
 begin 644 hptmvraid.o
-M?T5,1@(!`0D```````````$`/@`!`````````````````````````"#=````
-M`````````$```````$``#@`+`$B)7"302(EL)-A,B60DX$R);"3H3(ET)/!,
-MB7PD^$B![$@"``!(B?M,BV\02(UOD$R-9"00N`(```"`?P$`=0N`?P(!&<"#
-MX`+_P(A%`$C'10@`````@$T!!$B+0Q!(B440BU,8B5482`^V10!(BP3%````
-M`$B)16"`?0`#=2:-0O:)11B+<QB#[@I,B>&Z(````$B)W^@`````08$\)/,6
-M>%IT&D@/MD4`2(L$Q0````!(B45H@$T!!.FE!```OI````!,B>?H`````(3`
-M==5!]D0D"0)T'<9#"`%!#[9$)`J(0PLZ0PES"P^V\$B)W^@`````O@`"``!,
-MB>?H`````(3`=!=)C;PDD````+IP`0``O@````#H`````$$/MI0DDP```+X!
-M````(=8/MD,$@^#\B=&#X0()\`G(O@0````AUH/@\XG1@^$("?`)R+X0````
-M(=:#X,^)T8/A(`GP"<B)T8/A0(/@/X/B@`G("="(0P1!#[94)`F#X@$/MD4!
-M@^#^"="(10%!@'PD$`,/A`____]!@'PD$`!U%,9%``!(BP4`````2(E%:.FR
-M`P``08-\)`0`#X3G_O__N@````!!#[9$)`N)P8/X`'XQ2,?&`````$ACPDC!
-MX`1,`>"`>!`(#X>Z_O__2`^V0!!(@SS&``^$JO[____".=%_UD&+1"08B40D
-M#+H`````2&/"2&G`,`$``$J-G"BH1```2(-["`!U#HM#=$$[1"0$#X04`0``
-M_\*#^@]^TDF+G:!$``!(A=MT(4B+0Q!)B86@1```NC`!``"^`````$B)W^@`
-M````3(EK$$&+1"0$B4-T#[93?H/*`8A3?F9!#[9$)!2#X`$!P(/B_0G"B%-^
-M00^W1"069HF#@````(M4)`R)DX0```!!#[9$)!"(`T$/MD0D$XA#<@^VR+@!
-M````T^!FB4-\2,=#"`````!!BT0D#(E#&$$/MD0D$8A#<$@/M@-(BP3%````
-M`$B)0V!)C70D+$B-N\@```"Z$````.@`````28UT)#Q(C;O8````N@0```#H
-M`````$F-M"2`````2(V['`$``+H0````Z`````!)C70D0$B-N]P```"Z0```
-M`.@`````BT0D##F#A````'8&B8.$````0?9$)`D!=`2`2P$!10^V?"0208!\
-M)`L!#X:V`0``1(GZ#[;"3(NTPX@```!!BT0D*(E$)`Q-A?8/A3D!``!-B[6@
-M1```387V="%)BT8028F%H$0``+HP`0``O@````!,B??H`````$V);A!!BT0D
-M!$&)1G1!#[96?H/*`4&(5GYF00^V1"0D@^`!`<"#XOT)PD&(5GY!#[=$)"9F
-M08F&@````(M4)`Q!B9:$````00^V1"0A08A&<$$/MD0D($&(!DF)7@A!BT0D
-M'$&)1AA!#[9$)"-!B$9R#[;(N`$```#3X&9!B49\20^V!DB+!,4`````28E&
-M8$@/M@-(BP3%`````$F)1FA%B'X#1(GZ#[;"3(FTPX@```#^0W%)C70D+$F-
-MOL@```"Z$````.@`````28UT)#Q)C;[8````N@0```#H`````$F-M"2`````
-M28V^'`$``+H0````Z`````!)C70D0$F-OMP```"Z0````.@`````0?9$)`D!
-M=`5!@$X!`4D/MD0D(DF#O,:(`````'5S28FLQH@```!!#[9$)"*(10-!_D9Q
-M3(EU"$D/M@9(BP3%`````$B)16B+1"0,03F&A````'8]08F&A````.LT1(GZ
-M#[;"2(.\PX@`````=2-(B:S#B````$B)70C^0W%$B'T#2`^V`TB+!,4`````
-M2(E%:$B+G"08`@``2(NL)"`"``!,BZ0D*`(``$R+K"0P`@``3(NT)#@"``!,
-MB[PD0`(``$B!Q$@"``##9F9FD%532(/L"$B)\[T`````#[9&<(G"@_@`#X[(
-M````9F:02&/%2(.\PX@`````=`O_Q3GJ?^SIK````+@!````B>G3X&8)@X``
-M``!(B=[H`````$B)PDB%P`^$B````,8``T"(:`-(B5@(@#L(#Y3`P>`##[9*
-M`8/A]PG!B$H!2(L%`````$B)0F!(#[8#2(L$Q0````!(B4)H]D,!$'01B<B#
-MR!"(0@&+@H@```")0AA(8\5(B93#B`````^V0WZ#X/Z#R`J(0W['@X0`````
-M````_D-Q2(G6OP<```#H`````(!+?@1(@\0(6UW#9F9FD$B)7"3@2(EL).A,
-MB60D\$R);"3X2(/L*$F)_4B)\P^V!H/X!@^$EP```(/X!G\1@_@$#XR8`0``
-MZQMF9I!F9I"#^`@/A.<```!F9F:09F:0Z7L!```/MD9Q.D9P9F9FD`^%:@$`
-M`(!.`02`9G[^0;P`````@'YP``^$4@$``&:026/$2(NLPX@```!(A>UT((!]
-M``-V&DB)[DR)[^@`````]D5^!'0)@$M^!&9FD&:00?_$#[9#<$0YX'_$Z0\!
-M``"`3@$$@'YQ`G4*@&9^_I#I^P```(!F?OU(@[Z(`````'4=2(N&D````,9`
-M`P!(B8:(````2,>&D`````````!(BX.(````@#@#=0M(B=Y,B>_H`````(!+
-M?@3'@X0`````````Z:8```!F9I!F9I`/MD9Q.D9P=2"`3@$$#[9&?H/@_HA&
-M?F:#OH``````=&^#R`*(1G[K9P^V1G'_P`^V5G`YT'599H.^@`````!T++D`
-M````#[>&@````(G"J`%U"O_!B=#3^*@!=/9(8\%(@[S#B`````!U&^L&]D9^
-M`G43@$L!!(!+?@1(B=Y,B>_H`````,>#A``````````/MT-\]]@A@X0```!F
-M9F:02(-["`!U&?9#?@1T$^@`````B4-T2(G?Z`````!F9I!(BUPD"$B+;"00
-M3(MD)!A,BVPD($B#Q"C#9F9FD&9FD$%455-)B?R]`````&9F9I!(8\5(:<`P
-M`0``2HV<(*A$``"#>W0`="!(@WL(`'492(G>3(GGZ`````#V0P$$=`A(QT-H
-M`````/_%@_T/?L%;74%<PV9F9I!F9F:09F9FD$%7059!54%455-(@>P(`@``
-M28G]2(GU08G628GG2(GWZ`````"`?0`(=0M(B>Y,B>_H`````$&\`````(!]
-M<``/AL<```!)8\1(@[S%B``````/A*@```!(B[3%B````(`^`W8100^VUDR)
-M[^@`````Z8H```!)8\1(BYS%B````,9#`P!(QT,(`````,>#C`````````"`
-M8P'^2,=#:`````!%A/9U#8!]``9U!TB#?0@`="JZ``(``+X`````3(G_Z```
-M``!(C7MP3(GYNC````"^`````.@`````ZQ,/ME4!@^(!#[9#`8/@_@G0B$,!
-M2(G?Z`````!F9I!F9I!!_\1$.&5P#X<Y____NC`!``"^`````$B)[^@`````
-M28N%H$0``$B)11!)B:V@1```QT5T`````$B!Q`@"``!;74%<05U!7D%?PV9F
-MD&9FD$%6055!5%532('L``(``$B)TTB+!DR+:`A,BW8X28MN($0/MF`#2(GB
-M2(7M=`UF00^V1AB#X`$YR'4A2(G528-^*`!T#4R)]D'_5BB%P&:0=0JX````
-M`.F(````00^W3DC!X0E!#[9V1&9F9I!F9I!,BT4`OP```0!F@WT(`'0$#[=]
-M"#GYB?H/1M%$.>9U$6:)4PAFQT,*``!,B0-(@\,0*=>)T$D!P"G1=1O_QD$/
-MMD5P.<:X``````]$\$$/MT5\B<'!X0F%_W6Y9H%]"@"`=`9(@\40ZY=FQT/Z
-M`("X`0```$B!Q``"``!;74%<05U!7L-F9F:09F9FD&9F9I!,BT\(1(M&0`^V
-M3P,/MWY,.$Y$=0L/MT9*00'`ZP]FD#A.1'8(00^W07Q!`<`X3D1U!&8#?D@X
-M3D5U!68#?D:0@'Y0`'0..$Y$<QTX3D5FD'<1ZQ0X3D1S!69!`WE\.$Y%=@5F
-M00-Y?&:)>@Q$B4((PV9F9I!F9F:09F9FD&9FD%-)B?E)B?"+?@A$#[=6#$&[
-M`````$$/MDERB?[3[F9!QT!,``!!QD!0`4$/ME%PB?")T;H`````]_%!#[9)
-M<M/@08E`0$$/ME%PB?")T;H`````]_%!B%!$B=%!#[=!?/_((?AF08E`2D$/
-MMU%\9BG"9D0YTG(-9D6)4$A!N@````#K7&9!B5!(9D$ITD&[`0```+H!````
-MB=#3X&9!"4!._\%!.$EP=2>Y`````$&`>%``=`M!QD!0`.L49F9FD$$/MT!,
-M9D$#07QF08E`3)!F13E1?',)9D4K47SKNF:0N`$```#3X&9!"4!.08A(169%
-MB5!&00^WPUO#9F:09I!54TB#[`A(B?U(BUXX2(L&#[9(`[C^____T\!F(4-.
-M@'X9`70'#[9&&8A#&4B)[^@`````9H-[3@!U&8![&0!U!,9#&0%(BW,P2(G:
-M2(GOZ`````!(@\0(6UW#9F:09I!(B5PDT$B);"383(ED).!,B6PDZ$R)="3P
-M3(E\)/A(@^PX28G_2(GU3(LV0?9&`01U%<9&&0)(BW8P2(GJZ`````#IR@``
-M`$R)]^A1_O__9H-]3@!U&L9%&0)(BW4P2(GJ3(G_Z`````#IHP```&:00;T`
-M````08!^<``/A)`````/MT5.1(GIT_BH`71R3(G_Z`````!(B<-)8\5-BZ3&
-MB````$R)(TB):S@/ME48@^("#[9#&(/@_0G0B$,8#[95&(/B!(/@^PG0B$,8
-M#[9%#HA##DB)VDB)[DR)Y^@^_?__2,=#,`````!(QT,H`````$B)WDR)_T'_
-M5"1@9F:00?_%00^V1G!$.>@/CW#___](BUPD"$B+;"003(MD)!A,BVPD($R+
-M="0H3(M\)#!(@\0XPV9F9I!F9F:09F9FD$B#[`A(BTX(#[91`8!F`?N`80'[
-M_DEQ@$E^`4B#>6@`#Y7`]L($#Y7"#[;`A<)T!DB)SO]1:$B#Q`C#9F:09I!(
-M@^P(3(M&.$F+<"!(A?9T#69!#[9`&(/@`3G(=2))@W@H`'003(G&0?]0*+H!
-M````A<!U,+H`````ZREF9F:02(L&2(D"2(M&"$B)0@A(@\(02(U&"DB#QA!F
-M@S@`>>"Z`0```(G02(/$",-F9F:055-(B?L/M@>#Z`2#^`%W4+T`````@']P
-M`'0W9F:09I!(8\5(@[S#B`````!T$4B+O,.(````Z,7___^%P'4'N`````#K
-M)/_%#[9#<#GH9I!_SK@!````ZQ%F9F:09F:0#[9'`<#H`H/@`5M=PV9FD$B)
-M7"382(EL).!,B60DZ$R);"3P3(ET)/A(@^PH28G^2(GU2(M>.$R+(TR++H!^
-M&0$/A-D```#V0Q@"#X1_````@'L:``^%Q0```$'V1"1^`@^%N0```,9#&@%!
-M@'T#`4@9P(/@"$Z+K""(````387M=$),B>_H%?___X7`=#9(C7U`ND@```"^
-M`````.@`````00^V30.X`0```-/@9HE#3DR);0!(B>Y,B?=!_U5@Z<````!,
-MBVT`ZU5F9I!FD/9#&`1T2DR)[^C"_O__A<!T(DB-?4"Z2````+X`````Z```
-M``!(B>Y,B?=!_U5@Z8$```"`>QD!=!T/MU-.00^V30.X`0```-/@.<)U"6:0
-M#[9%&8A#&4B)[DR)]^@`````00^V30.X_O___]/`9B%#3F:#>TX`=3R`>QD`
-M=03&0QD!2(MS,$B)VDR)]^@`````9D'_3"0B9D&#?"0B`'4428M$)#!(A<!T
-M"DF+="0X3(GW_]!(BQPD2(ML)`A,BV0D$$R+;"083(MT)"!(@\0HPV9F9I!F
-M9F:09F:09F:02(/L"$B+1CB%R70:2(M`$$B)`@^W1@S!X`EFB4((9L=""@"`
-MZU%(@W@@`'0S]D`8`70M2(M(($B+`4B)`DB+00A(B4((2(/"$$B-00I(@\$0
-M9H,X`'G@ZQYF9F:09F:0O@````!(@W@H`'002(G&D/]0*(G&ZP6^`0```(GP
-M2(/$",-F9F:09F:09F:09F:02(E<).A(B6PD\$R)9"3X2(/L&$F)_$B)\TB+
-M;CA(BP8/MD@#N/[____3P&8A14Z`?AD!=`</MD89B$492(G>3(GGZ`````!F
-M@WU.`'55@'T9`'4?QD49`4B+30"+D80````[4PAU#`^W0PP!T(F!A````$B+
-M=3!(B>I,B>?H`````$B+10!FQT`@``!(BU4`2(/"*$C'Q@````!,B>?H````
-M`$B+'"1(BVPD"$R+9"002(/$&,-F9F:09F:09F:02(E<).A(B6PD\$R)9"3X
-M2(/L&$F)_$B)\TB+;CB`?AD!=`T/MD89B$49ZW1F9F:0@'X.('5*QD89`$B+
-M10!(BX"0````2(D&2(U^0+I(````O@````#H`````,9#&@#&0PXP#[9#&(/@
-M_8/(!(A#&$B+`TB)WDR)Y_]08.M=9I#&11D!2(M-`(N1A````#M6"'4-#[=&
-M#`'0B8&$````D$B)WDR)Y^@`````2(MU,$B)ZDR)Y^@`````2(M%`&;'0"``
-M`$B+50!(@\(H2,?&`````$R)Y^@`````2(L<)$B+;"0(3(MD)!!(@\08PV:0
-M05154TF)]$B)U4B+1CB%R71&2(M0$$B)50`/MT8,P>`)9HE%"&;'10H`@$B+
-M!H!X`P`/A`T!``!(C8(```$`9H-]"`!T"4@/MT4(2(T$`DB)10#I[0````^W
-M7@S!XPE(@W@@`'0L]D`8`70F2(M(($B+`4B)`DB+00A(B4((2(/"$$B-00I(
-M@\$09H,X`'G@ZS2Z`````$B#>"@`#X2H````2(GJ2(G&_U`HN@````"%P`^$
-MD@```.L,9HE9"&;'00H`@.M&2(GI9F9FD&9FD+H```$`9H-Y"`!T!`^W40A)
-MBP0D@'@#`'06.=IV%HG82`$!B=!F*=AFB4$(ZPYFD#G:<[9(@\$0*=-UQ$F+
-M!"2`>`,`#Y7`2#G-#Y7"#[;`A<)T(4B+`4B)10!(BT$(2(E%"$B#Q1!(C4$*
-M2(/!$&:#.`!YW[H!````B=!;74%<PV9F9I!F9F:09F9FD&9FD$B)7"3H2(EL
-M)/!,B60D^$B#[!A)B?Q(B?-(BVXX@'X9`70-#[9&&8A%&>F!````D$B+!H!X
-M`P!U-\9&&0!(BT4`2(N`D````$B)!DB-?D"Z2````+X`````Z`````#&0QH`
-M2(L#2(G>3(GG_U!@ZWL/MU8,P>()B=9(BWT02`'^Z`````"%P'0'QD49#.LA
-MD,9%&0%(BTT`BY&$````.U,(=0T/MT,,`=")@80```"02(G>3(GGZ`````!(
-MBW4P2(GJ3(GGZ`````!(BT4`9L=`(```2(M5`$B#PBA(Q\8`````3(GGZ```
-M``!(BQPD2(ML)`A,BV0D$$B#Q!C#9I!(B5PDX$B);"3H3(ED)/!,B6PD^$B#
-M["A)B?Q(B?-(BRY(QT4P`````/9%?@$/A54!```/MD8.@_@"#X3V````@_@"
-M?Q"#^`$/A)8```!FD.DS`0``@_@##X4J`0``BT8(B44<#[=&#&:)12`/MU8,
-MP>()2(M^$+X`````Z`````!FQT-.`P!!O0````!,B>?H`````$B)QL9`#C"+
-M0PB)1@@/MT,,9HE&#(!.&`1(B5XX2,=&,`````!(QT8H`````$ECQ4B+A,6(
-M````2(D&3(GG_U!@0?_%08/]`7ZNZ;<```"+1@B)11P/MT8,9HE%(.@`````
-M2(G&QD`.((M#"(E&"`^W0PQFB48,@$X8`DB)7CA(QT8P`````$C'1B@`````
-M2(N%B````$B)!DR)Y_]08.MEBT8(B44<#[=&#&:)12#H`````$B)QL9`#B"+
-M0PB)1@@/MT,,9HE&#(!.&`)(B5XX2,=&,`````!(QT8H`````$B+A8@```!(
-MB09,B>?_4&#K$\9#&09(BW,P2(G:3(GGZ`````!(C54H2,?&`````$R)Y^@`
-M````2(M<)`A(BVPD$$R+9"083(ML)"!(@\0HPV9FD&9FD&9FD$B)7"302(EL
-M)-A,B60DX$R);"3H3(ET)/!,B7PD^$B#[#A)B?])B?1(BR[V10$$=17&1AD"
-M2(MV,$R)XN@`````Z=H!``#V1A@@="1(QT4P`````$B)=3AF@WTB``^%O0$`
-M`.C(_?__Z;,!``!F9I!(@WTP`'4I9H-](`!F9F:0=#X/MT4@`T4<.T8(=C(/
-MMT8,`T8(.44<<R9F9I!F9I!)QT0D2`````!(C7TH3(GFZ`````#I9@$``&9F
-MD&9FD&;_12+V17X##X6B````0?9$)!@"#X26````2(N5B````$B+M9````"`
-M.@-U9(`^`W5?08M$)`B)P2N*D````$&)P$0KAI````"+NI0```"+MI0```")
-MR)DQT2G11(G`F4$QT$$IT$0YP7T3A?8/E$0D!X/_!0^7P"!$)`?K-X7_#Y5$
-M)`>#_@4/EL`(1"0'ZR2`O8(`````#Y1$)`</MDPD!XB-@@```.L+9F:09F:0
-MQD0D!P!F0<=$)$X``$&^`````$2)\$0/MNA*@[SMB`````!T>TJ+A.V(````
-M]D`!!'1M00^V5"08P.H"1#AT)`</E,`)T*@!=%9,B?_H`````$B)P[I`````
-M3(GF2(G'Z`````"X`0```$2)\=/@9D$)1"1.3(EC.$C'0S``````2,=#*```
-M``!*BX3MB````$B)`TB)WDR)__]08&9FD$'_QD&`_@$/AF;___]F9I!(BUPD
-M"$B+;"003(MD)!A,BVPD($R+="0H3(M\)#!(@\0XPV9F9I!F9I!F9I!F9I!5
-M4TB#[`A(B?5(BUX(#[96`X!F`?N`2WX!QX.$`````````/Y+<8![<0!U!H!C
-M`?OK%@^V0WZH`G0.@^#5B$-^A=)U!(!C`?OV0P$$=1=(@WMH``^$[P```$B)
-MWO]3:)#IXP```(72=21(BY.(````QD(#`4B+@Y````#&0`,`2(F#B````$B)
-MDY````!(@WL(`'0.2(M#"/9``00/A'X```!(BX.(````@#@#=7)(B=[H````
-M`$B)PDB%P'1BQ@`#QD`#`4B)6`A(BP4`````2(E"8$B+!0````!(B4)H]D,!
-M$'0-@$H!$(N"B````(E"&$C'10@`````2(F3D`````^V0WZ#X/Z#R`J(0W[^
-M0W%(B=:_!P```.@`````9I!(@WL(`'0+2(M;"$B#>P@`=?6+0W2)0WCH````
-M`(E#=$B)W^@`````2(/$"%M=PY!(B5PDV$B);"3@3(ED).A,B6PD\$R)="3X
-M2('L*`(``$F)U$R+-DB+;CA(BUT@0;T1````2(GB2(7;=`QF#[9%&(/@`3G(
-M=2A(B=-(@WTH`'0.2(GN_U4HA<!U%&9F9I"X`````.GE````9F:09F:0@'U%
-M_P^$O@````^W34C!X0FZ```!`&:#>P@`=`0/MU,(.=%V.T$/MD8#.D5$=11(
-MBP-)B00D2(M#"$F)1"0(28/$$"G12(/#$+H```$`9H-["`!T!`^W4PA!_\TY
-MT7?%00^V1@,Z141U'$B+`TF)!"1F08E,)`AF0<=$)`H`@+@!````ZUDYT7,W
-MB<A(`P-)B00DB=!F*<AF08E$)`AF@7L*`(!U#V9!QT0D"@"`N`$```#K*F9!
-MQT0D"@``28/$$$B#PQ!!_\U$B>I(P>($2(G>3(GGZ`````"X`0```$B+G"0`
-M`@``2(NL)`@"``!,BZ0D$`(``$R+K"08`@``3(NT)"`"``!(@<0H`@``PY`/
-MMD<#.D9$=0\/MT9(9HE"#(M&0(E"",,/MT9&9HE"#,=""`````##9F9FD&9F
-M9I!$BT8(1`^W3@Q!NP````"Y`````(!_<``/A((```!%#[?1B<A(BY3'B```
-M`$0Y0AAV7T2)1D"(3D2X`0```-/@#[=^3@G'9HE^3D.-!`([0AAV,T&[`0``
-M`(U!`8A&1?_!1(G8T^")P8GX"<AFB49.BT(89D0IP&:)1DAF02G!9D2)3D;K
-M&L9&1?]F1(E.2.L/1"M"&/_!#[9'<#G(=X.01(G8PV9F9I!F9F:09F9FD%53
-M2(/L"$B)_4B+7CA(BP8/MD@#N/[____3P&8A0TZ`?AD!=`</MD89B$,92(GO
-MZ`````!F@WM.`'49@'L9`'4$QD,9`4B+<S!(B=I(B>_H`````$B#Q`A;7<-F
-M9I!FD$B)7"302(EL)-A,B60DX$R);"3H3(ET)/!,B7PD^$B#[#A)B?](B?5,
-MBS9!]D8!!'45QD89`DB+=C!(B>KH`````.G*````3(GWZ*'^__]F@WU.`'4:
-MQD49`DB+=3!(B>I,B?_H`````.FC````9I!!O0````!!@'YP``^$D`````^W
-M14Y$B>G3^*@!='),B?_H`````$B)PTECQ4V+I,:(````3(DC2(EK.`^V51B#
-MX@(/MD,8@^#]"="(0Q@/ME48@^($@^#["="(0Q@/MD4.B$,.2(G:2(GN3(GG
-MZ-[]__](QT,P`````$C'0R@`````2(G>3(G_0?]4)&!F9I!!_\5!#[9&<$0Y
-MZ`^/</___TB+7"0(2(ML)!!,BV0D&$R+;"0@3(MT)"A,BWPD,$B#Q#C#9F9F
-MD&9F9I!F9F:02(/L"$B+1@B`9@'[@&`!^_Y(<8!(?@%(@WAH`'0&2(G&_U!H
-M2(/$",-F9F:09F:02&/.2(N4S^!7``!(A=)T"TB+`DB)A,_@5P``2(G0PY!(
-M8])(BX37X%<``$B)!DB)M-?@5P``PV9FD&9FD&9FD$%7059!54%455-(@^P(
-M28G_O0"```"!_@!```!W:&:]`$"!_@`@``!W7&:]`""!_@`0``!W4&:]`!"!
-M_@`(``!W1&:]``B!_@`$``!W.&:]``2!_@`"``!W+&:]``*!_@`!``!W(&:]
-M``&!_H````!W%&:]@`"#_D!W"X/^(1GM@^7@@\5`B=`/K\5$C;#_#P``0<'N
-M#,=$)`0`````N`````"!_0`0```/A]L```!!_\Y!@_[_#X3*````0;T`$```
-M3(G_Z`````!(B</_1"0$0;P`````1(GHN@````#W]8/X``^&CP```&9FD&9F
-MD+H*````@?T`0```=U:R"8']`"```'=,L@B!_0`0``!W0K('@?T`"```=SBR
-M!H']``0``'<NL@6!_0`"``!W)+($@?T``0``=QJR`X']@````'<0L@*#_4!W
-M"8/](`^7P`^VT$B)WDR)_^B"_O__B>A(`<-!_\1$B>BZ`````/?U1#G@#X=W
-M____0?_.08/^_P^%//___XM$)`1(@\0(6UU!7$%=05Y!7\-F9F:09F:055-(
-MB?V)\[@`````.;<X6```<E.^`P```.@"_O__2(G&2(7`=#Z)&/_+@_O_=#60
-MB=FZ`````$B#O3!8````=!=(BY4P6```2(L"2(F%,%@``/^-.%@``$B)5,X(
-M_\N#^_]US$B)\%M=PV9FD&9FD(L._\F#^?]T+69F9I!F9I!(8\%(BU3&"$B+
-MAS!8``!(B0)(B9<P6```_X<X6```_\F#^?]UVKH#````Z)#]___SPV9F9I!F
-M9F:09F:09F:055-(@^P(2(G]BQ__RX/[_W0>2&/#2(M\Q0BZ`!```+X`````
-MZ`````#_RX/[_W7B2(/$"%M=PV9FD&9FD&9FD(L/_\F#^?]T+DACP4B+1,<(
-MN@````!(@S@`=`:X`````,/_PDB#P`B!^O\!``!VYO_)@_G_==*X`0```,-F
-M9I!(@^P(NA````"^`````.@`````2(/$",-F9F:09F9FD$B#?A``=`Q(BU80
-M2(M&"$B)0@A(BU8(2(M&$$B)`DC'1@@`````N@````#HL/S___/#9F9FD&9F
-M9I!F9I!F9I!(C4]82(M'6$B)1A!(A<!T#$B+5UA(C4802(E""$B)3@A(B3'#
-M9F9FD&9F9I!F9I!(B5PDX$B);"3H3(ED)/!,B6PD^$B#["!)B?5!B=2)S4B+
-M7EA(A=MT(F9F9I!F9I`Y*W,.1#EC!'8(N`````"0ZRI(BUL02(7;=>6^````
-M`.C[^___2(G#1(D@B6@$2(G&3(GOZ&?___](B=A(BQPD2(ML)`A,BV0D$$R+
-M;"082(/$(,-F9F:09F9FD&9F9I`/MD=)#[;02(T4DDB-5-=0_\"(1TE(B3K&
-M0@D`QD(*`,9""P!(B=##9F9FD&9F9I!!5%5328G\2(GSO0`````/ME9&C4+]
-MT?@!PH/Z`'XED$ACQ4B+M,.@"@``3(GGZ*W]____Q0^V4T:-0OW1^`'".>I_
-MW+H'````2(G>3(GGZ%S[__];74%<PV9F9I!F9I!!5T%6055!5%532(/L"$F)
-M_8E4)`2)\T&)SHNW.%@``,'F`P^WR0^VTXU"_='X`<(/K\HYS@^#UP```$0/
-MMOM$B3PD08G,28V%L%<``$DYA;!7```/A-@```!)BZVP5P``2(M5"$B+10!(
-MB5`(2(D"2(EM`$B);0A(@WT@`'0F2(-]*`!T#$B+52A(BT4@2(E"($B+52!(
-MBT4H2(D"2,=%(`````!F1#EU1'5(.%U&=4-(B6T`2(EM"(M$)`2)14!!O```
-M``!!@_\`?B"+'"2026/\2,'G!$B-O"]P"P``Z&S]__]!_\1$.>-_Y$B)Z.DF
-M`0``2(GN3(GOZ*'^__]!BX4X6```P>`#1#G@#X(T____O@<```!,B>_H$?K_
-M_TB)Q;@`````2(7M#X3J````ZPZX`````.G>````9F9FD$B);0!(B6T(BT0D
-M!(E%0&9$B75$B%U&2,=%,`````!!O``````/MM.-0OW1^`'"@_H`#XZ?````
-M18GW9D'![P,/MM.-0OW1^$2--`)F9F:026/<00^W]TR)[^AQ^___2(F$W:`*
-M``!(A<!U/$'_S$&#_/]T'$ECQ$B+M,6@"@``3(GOZ+C[__]!_\Q!@_S_=>2Z
-M!P```$B)[DR)[^AO^?__N`````#K,DECQ$C!X`1(C1PH2(V[\`H``.A1_/__
-M2(V[<`L``.A%_/__0?_$13GF#X]Y____2(GH2(/$"%M=05Q!74%>05_#9F9F
-MD&9FD$%455-(B?6)TXG.#[9-<HG0T^@E_P$``$B+55!,C23"38L$)$V%P'0P
-M9F9FD&9FD$$Y6$!U&DF+4`A)BP!(B5`(2(D"38D`38E`"$R)P.MM38M`*$V%
-MP'77#[?.#[9U<(G:Z&G]__])B<!(A<!T2TB):#A)BP0D28E`*$B%P'0,28L4
-M)$F-0"A(B4(@38E@($V)!"0/MG5P08AP1@^V37*)V-/H#[95<(G1N@````#W
-M\4`HUO_.08AP1TR)P%M=05S#9F9FD&9F9I!F9I!F9I!!5T%6055!5%532('L
-M:`$``$B)?"1(2(ET)$!(B50D.`^V5AB)T-#H@_`!B<&#X0&)3"0T2(-^(`!T
-M!?;"`70<3(U\)%"Y`0```$R)^DB+="1`2(M\)$C_5BCK"4B+7"1`3(M[(,=$
-M)`P```$`9D&#?P@`=`E!#[='"(E$)`Q!O0````#'1"0P`````$B+5"1`@'IX
-M``^$=0$``$AC1"0P2(M,)$!(BT3!4$B)1"0@#[:88`P``(A<)"\XF&$,```/
-M@C,!```/MD0D+TB+5"0@.$)'#X0&`0``#[;`2(G1#[>4@D`,```/MX2!0@P`
-M`&:)1"00B=!FP>@##[?`B40D"$&)UD&#Y@=!#[?&B<;!Y@FX"````&9$*?!!
-MB<9F1#MT)!"+7"001`]'\T$/M\:)Q<'E"4@/MD0D+TB+5"0@2(N,PJ`*``"+
-M5"0(B?!)B<1,`V31"$0[;"0,=21)@\<0QT0D#````0!F08-_"`!T"4$/MT\(
-MB4PD#$&]`````)"+7"0,1"GK.>L/1]V#?"0T`'021(GN20,WB=I,B>?H````
-M`.L01(GO20,_B=I,B>;H`````$$!W2G==`>)V$D!Q.N29D0I="00=!3_1"0(
-MO@````!!O@@```#I/O____Y$)"\/MEPD+TB+1"0@.)AA#```#X/-_O___T0D
-M,$B+5"1`#[9">#M$)#`/CXO^__^Z`````$B+="1`2(M\)$C_5"0X2('$:`$`
-M`%M=05Q!74%>05_#9F9FD&9F9I!F9I!54TB#[`A(B?V%TG0$QD89"T&[````
-M`(!^>``/A,$```!(C9W`5P``2(V]L%<``)!)8\-,BT3&4$G'@#`,````````
-M38G"N0````!!@'A&`'9(#[;!2,'@!$P!T$B-D/`*``!!N0````"#N/`*````
-M=1*#>@0`=0R#>@@`=0:#>@P`=`9!N0$```!%A<EF9I!U)?_!03A*1G>XN```
-M``"%P'0=2(M#"$R)0PA)B1A)B4`(3(D`ZQNX`0```.OA9I!(BT<(3(E'"$F)
-M.$F)0`A,B0!!_\,/MD9X1#G8#X].____2(GOZ`8X``!(@\0(6UW#9F9FD&9F
-M9I!F9F:09F:02(E<)-!(B6PDV$R)9"3@3(EL).A,B70D\$R)?"3X2(/L2$F)
-M_4B)\TB+KC`,``#_CSQ8``!(C4802(M("$B+5A!(B4H(2(D12(E&$$B)0`A(
-MA>T/A(<"``#V11@@#X0=`0``3(ME``^W1D1!B<9$`W9`1`^W?D1(QX8P#```
-M`````$B-A[!7``!(BU`(2(EP"$B)!DB)5@A(B3*`OF(,```!=56`?0X"=49(
-M#[9&1TB+O,:@"@``Z!3W__^%P'4PQH-B#```#$&`3"1^"D$/MT0D?/?8(T-`
-M03F$)(0```!V#D&)A"2$````9F:09F:0@+MB#````70:#[:#8@P``(A%&4B)
-M[DR)[^CB-@``Z6T#``!!#[=$)'R+50C_P@^OPD0Y\'8M00^WST2)\DR)YDR)
-M[^BX^O__2(FH,`P``$B)15!(B<9,B>_H`A```.DM`P``00^W1"1\00.$)(0`
-M``!$.?!U"$6)M"2$````2(GN3(GOZ'<V``#I`@,``&:0#[9%>?_`B$5Y.D5X
-M#X6.`@``]D48!`^$T@```$&^`````(!]>``/AK(```!(C8>P5P``2(E$)!!(
-MC9?`5P``2(E4)`A$B?(/ML)(BUS%4$C'@S`,````````@+MB#````75-28G?
-M0;P`````@'M&`'8@1(G@#[;X2,'G!$J-O#_P"@``Z`?V__]!_\1%.&=&=^!(
-MBU0D$$B+0@A(B5H(2(D32(E#"$B)&.LF9F:09I`/MH-B#```B$492(M4)`A(
-MBT((2(E:"$B)$TB)0PA(B1A!_\9$.'5X#X=F____2(GN3(GOZ(<U``#IL@$`
-M`$&^`````(!]>`!V+69FD&9FD$2)\@^VPDB+7,50@+MB#````70*#[:#8@P`
-M`(A%&4'_QD0X=7AWV8!]&0!U%TC'P@````!(B>Y,B>_H`?K__^E<`0``N@``
-M``!(B>Y,B>_H+/S__^E'`0``9F9FD&9FD("^8@P```$/A8,```!)B?2]````
-M`(!^1@!V)&9FD&:0B>@/MOA(P><$2HV\)_`*``#H^O3____%03AL)$9WX4B#
-MNS@,````="E(BX,X#```2(F#,`P``$C'@S@,````````2(G>3(GOZ`,.``#I
-MS@```$F-A;!7``!(BU`(2(E8"$B)`TB)4PA(B1KK6F9FD$B-A\!7``!(BU`(
-M2(EP"$B)!DB)5@A(B3)(@[XX#````'0T2(N&.`P```^VEF(,``"(4!E(BY8X
-M#```2,?&`````.@`````2,>#.`P```````#IN0```$B+<S!,B>_H7?3__TC'
-M0S``````2(M#.$B#>"@`=!5(C5`H2,?&`````$R)[^@`````ZR5)@[U(6```
-M`'0;28V52%@``$C'Q@````!,B>_H`````&9FD&:008.]0%@```!T5H"[8@P`
-M``%U34B+>SA(@W]``'0PZ-TQ``"%P'4G2(M#.$B+4$A(BW!`3(GOZ`````!(
-MBT,X2,=`0`````!!_XU`6```08.]0%@```!T"$R)[^@`````2(M<)!A(BVPD
-M($R+9"0H3(ML)#!,BW0D.$R+?"1`2(/$2,-F9F:09F:09F:09F:02(/L"$B+
-M%H!Z2@!T$\9"2@!(Q\8`````Z`````!F9I!(@\0(PV9F9I!F9F:09F:005=!
-M5D%505154TB#[`A)B=1!B<](BT8X2(L02`^V0!%,B[3"H`H``(M&""M"0,'@
-M"0^W;@S!Y0F)P8'A_P\``$&)Q4'![0R[`!```"G+.>L/1]U%A?]T#$ECU8G(
-M20-$U@CK#TECQ8G/20-\Q@CH`````$F)!"1F08E<)`@YZW089D''1"0*``!)
-M@\000?_%*=VY`````.NN9D''1"0*`("X`0```$B#Q`A;74%<05U!7D%?PV9F
-M9I!F9F:09F:0#[9'$@)'$XA'$DB-5Q2^`0```.L/@'\2`'D&N`````##_D<2
-M#[9/$H/Y'W<3B?#3X(5'%`^5P`^VP.M49F9FD(/Y/W<2@^D@B?#3X(5"!`^5
-MP`^VP.LY@_E?=Q2#Z4")\-/@A4((#Y7`#[;`ZR)FD+@`````@_E_=Q:#Z6")
-M\-/@A4(,#Y7`#[;`9F:09F:0A<!T@<9'$P%,C4<4#[97$KX!````ZP/^1Q,/
-MMD<3C0P"@_D?=P^)\-/@A4<4#Y7`#[;`ZTZ#^3]W%H/I((GPT^!!A4`$#Y7`
-M#[;`ZS9F9I"#^5]W$X/I0(GPT^!!A4`(#Y7`#[;`ZQNX`````(/Y?W<1@^E@
-MB?#3X$&%0`P/E<`/ML"%P'60L`'#9F:09F:02(E<).A(B6PD\$R)9"3X2(/L
-M&$B)_4B+7CA,BR,/MD89B$,*Z`````"`>PH!=7%(B=_HJ?[__X7`=!7&0PH`
-M2(G>2(GOZ,8```#K8V9F9I"`>P@!=4H/MD,003A$)$>0=#XX0Q%U.0^V\$B)
-M\$C!X`1*C;0@<`L``$B-2Q2+!@M#%(D&BT8$"T$$B48$BT8("T$(B48(BT8,
-M"T$,B48,D,9#"0)(B=Y(B>_H0?W__TB+'"1(BVPD"$R+9"002(/$&,-F9F:0
-M9F9FD&9FD&9FD$B)7"3P3(ED)/A(@^P828G\2(GSQD83`,9&$@!(B??H[/W_
-M_TB)WDR)Y^@1````2(M<)`A,BV0D$$B#Q!C#9I!(B5PDX$B);"3H3(ED)/!,
-MB6PD^$B#["A)B?U(B?-(BRY(BU4X2`^V1A!,BZ3"B````.@`````2(G&3(D@
-M@'L(`74-@$@8`L9`#B#K"V9FD(!(&`3&0`XP#[9#$@-%0(E&"&8/MD,39HE&
-M#$C'1B@`````2(E>.$C'1C``````3(GO0?]4)&!(BUPD"$B+;"003(MD)!A,
-MBVPD($B#Q"C#05154T&Y`````$&[`````$B)_4&\`0```.L808/[?W8/1(G(
-MZ5X!``!F9F:09F:00?_#1(G:08/['W<21(G@1(G9T^"%!P^5P`^VP.M208/[
-M/W<3C4K@1(GCT^.%700/E<`/ML#K.4&#^U]W$XU*P$2)X-/@A44(#Y7`#[;`
-MZR"X`````$&#^W]W%8U*H$2)X]/CA5T,#Y7`#[;`9F9FD(7`#X1S____0;@!
-M````2(GZ0;H!````ZPIF9F:09F:00?_`0XT,`X/Y'W<41(G0T^"%`@^5P`^V
-MP.M59F:09I"#^3]W$X/I($2)T-/@A4($#Y7`#[;`ZSB#^5]W$X/I0$2)T]/C
-MA5H(#Y7`#[;`ZR"X`````(/Y?W<6@^E@1(G0T^"%0@P/E<`/ML!F9I!FD(7`
-M=8Q$B=E%`<-!N@@```!$B<J)R,'H`X@$EHG(@^`'B$26`42)TRC#B%R6`@^V
-MPT0YP',*02G``<%!_\'KT42)R$2(1(8"0?_!08/[?P^&K_[__T2)R%M=05S#
-M9F9FD$B)7"302(EL)-A,B60DX$R);"3H3(ET)/!,B7PD^$B![%@!``!(B7PD
-M"$F)]4B++DB+7A"`>QD`#X3C````2(-[$`!U"TB)<Q#I9`$``&:0QD,9`$B-
-M="0@2(G?Z`_^__]!B<9(#[9#&$R+I,6@"@``2(M#$$@/MD`83(N\Q:`*``!)
-M#[9%&$B+K,6@"@``NP````!$.?,/C0,!``!F9I!(8],/MD24(8G!P>$)#[9$
-ME")!B<!!P>`)1`^V3)0@1(G(#[;0B<A(B<=)`WS7"$B)?"002(G&2`-TU0A(
-MB70D&$D#1-0(2#G'=0I$B<+H`````.L@2(M4)!A(BW0D$$2)R$0/MLB)STL#
-M?,P(1(G!Z`````#_PT0Y\WR$Z7\```!(C70D($B)W^A"_?__08G&2`^V0QA,
-MBZ3%H`H``$D/MD483(N\Q:`*``"[`````$0Y\WU+2&/3#[9$E"&)P<'A"0^V
-M1)0B08G`0<'@"4@/MD24((G.2(GW20-\Q`A(B7PD$$D#=,<(2(ET)!A$B<+H
-M`````/_#1#GS?+EF9F:0N@````!,B>Y(BWPD".A^`@``2(N<)"@!``!(BZPD
-M,`$``$R+I"0X`0``3(NL)$`!``!,B[0D2`$``$R+O"10`0``2('$6`$``,-F
-M9I!F9I!(B5PDT$B);"383(ED).!,B6PDZ$R)="3P3(E\)/A(@>Q(`0``28G_
-M2(GU2(L>2(UT)!!(C7T0Z$3\__]!B<1(#[9%($R+M,.@"@``2`^V12%,BZS#
-MH`H``+L`````1#GC?3U(8\,/MG2$$<'F"0^V5(02P>()2`^V1(00B?9(B?=)
-M`WS&"$B)/"1)`W3%"$B)="0(Z`````#_PT0YXWS#N@````!(B>Y,B?_HD`$`
-M`$B+G"08`0``2(NL)"`!``!,BZ0D*`$``$R+K"0P`0``3(NT)#@!``!,B[PD
-M0`$``$B!Q$@!``##9F9FD&9F9I!!5T%6055!5%532('LB`$``$F)_TB)]4R+
-M)DB+7A!(C;0D@````$B)W^AB^___08G&2`^V0QA-BZS$H`H``+L`````@'T@
-M`'8C#[9-(&9FD`^VPT@/ME0H&$F+E-2@"@``2(E4Q$#_PSC9=^1!O`````!%
-M.?0/C:T```!F9I!FD$ECQ`^VE(2!````08G20<'B"0^VE(2"````08G30<'C
-M"0^VC(2`````NP````"`?2``=BA$#[;)18G0#[9U()`/ML-(BU3$0$J+?,H(
-M28T4.$B)%,3_PT`XWG?D#[;11(G02(G'20-\U0A(.3PD=1!(BW0D"$2)VN@`
-M````ZQV02(M4)`A(BS0D#[;!1(G720-\Q0A$B=GH`````$'_Q$4Y]`^,6/__
-M_[H`````2(GN3(G_Z!@```!(@<2(`0``6UU!7$%=05Y!7\-F9I!F9I!(B5PD
-MZ$B);"3P3(ED)/A(@^P828G\2(GSB=7&1@D"@_H!&<"#X/6#P`R(1@KH3O;_
-M_X!["`-U+TB+0Q!(BW`02(7V=")(QT`0`````,9&"0*#_0$9P(/@]8/`#(A&
-M"DR)Y^@9]O__2(L<)$B+;"0(3(MD)!!(@\08PV9FD&9FD`^V5@N%TG1&B=!(
-M#[:$1[`)``!(C02`2(U$QU"`>`D"=`:X`````,.`>`H!=!+&1@D"#[9`"HA&
-M"K@`````PY")T`^VE$>Q"0``A=)UNK@!````PV9F9I!F9F:09F9FD$%7059!
-M54%455-(@^P(28G^2(GUQD9*`("^8PP````/A(L```!!O`````!(C5U0@'U)
-M`'1.@'L)`74X@'L(`W4I2(M#$(!X&0!T'TB#>!``=!A(QT`0`````,9#"0(/
-MMH5B#```B$,*ZPG&14H!Z;<!``!!_\1(@\,H#[9%240YX'^R2(GN3(GWZ&+P
-M___IEP$```^V0PJ(A6(,``#&A6,,```!Z7S___]F9F:09F:00;\`````0;T!
-M````0;P`````2(U=4(!]20`/A#0!``"`>PD`=7I(B=Y(B>_HO_[__X7`=%O&
-M0PD!3(GW2(G>#[9#"(/X!G=(B<#_),4`````Z'KW___K.&9F9I!F9F:0Z-OY
-M___K*>C$^___9F9FD.L>Z*G\___K%\9%2P'&1@D"QD8*`>AF]/__9F:09F:0
-M@'L)`G400?_'9F9FD&9FD(!["0)T"T&]`````.F/````@'L*`0^$A0```(!]
-M2P"0#X0=____2(M%.$@/ME,02(N$T(@```#V0`$$=%]!O0````#&0PD!QD,*
-M`$R)]TB)W@^V0PB#^`9W1HG`_R3%`````.C(]O__ZS9F9I!F9I#H*_G__^LI
-MZ!3[__]F9F:0ZQ[H^?O__^L7QD5+`<9&"0+&1@H!Z+;S___K!,9#"@%!_\1(
-M@\,H#[9%240YX`^/S/[__T6%_P^%H_[__T6%[704QH5B#````4B)[DR)]^C,
-M[O__ZP3&14H!2(/$"%M=05Q!74%>05_#9F9FD&9FD$%7059!54%455-(@>QX
-M`0``2(E\)$A(B?5(BUXX3(NF,`P``$&]`````$V%Y'4E#[=.1(M60`'12(G>
-MZ!#G__](B44PN/____](@WTP``^$L2(``$B+1"1(_X`\6```2(U%$$B+5"1(
-M2('"T%<``$B+2@A(B4((2(E5$$B)2`A(B0%F@[N``````'1$]D-^`74+BX.$
-M````.T5`=R=!O@`````/MX.`````B<*H`74.0?_&B=!$B?'3^*@!=/)$B'5(
-MZS#&14@(ZRIF9I!F9I#V0WX"=!J+@X0````[14!W#P^V14>(14CK"F9FD&9F
-MD,9%2`C&14D`QH6P"0```,9%2P#&A6(,````QH5C#````$C'A3@,````````
-M387D#X1@#P``0?9$)!@@#X2/!P``00^V1"0.@_@"#X1N!```@_@"?Q"#^`$/
-MA`4!``!FD.F>(0``@_@##X65(0``2(U<)&!(B=_H5>7__TF)VK\`````#[=U
-M1+L`````0;D@````0;L!````@_\?=S\/ML-)C12"1(G)*?DY\0]'SH/Y('4(
-MQP+_____ZPU$B=C3X/_(B?G3X`D"1(G(*?@Y\',5C70^X+\`````ZP.#[R#_
-MPX#[`W:R2(N]H`H``.A>Y/__0;\`````@'U&``^&`2$``$2)^P^VPTC!X`1(
-MC1PH2(V[\`H``.BTY/__2(V[<`L``.BHY/__2(GOZ,#E___&0`@"1(AX$,9`
-M$0!(BU0D8$B)4!1(BU0D:$B)4!Q!_\=$.'U&=ZOIIR```+@!````ZU5(B>^Y
-M`````(!]1@"0=D$/ML%(P>`$2`'X2(V0\`H``+X`````@[CP"@```'42@WH$
-M`'4,@WH(`'4&@WH,`'0%O@$```"%]G6P_\$X3T9WO[@`````A<!T%TR)I3@,
-M``!(QX4P#````````.G"#0``2(U<)&!(B=_H\./__TF)VK\`````#[=U1+L`
-M````0;D@````0;L!````@_\?=S\/ML-)C12"1(G)*?DY\0]'SH/Y('4(QP+_
-M____ZPU$B=C3X/_(B?G3X`D"1(G(*?@Y\',5C70^X+\`````ZP.#[R#_PX#[
-M`W:R@'U("'4'#[9%1XA%2`^V74B(G0@,``#&A0D,```!2(M$)&!(B87P"P``
-M2(M$)&A(B87X"P``2,>%``P```````!(B>_H7.3__\9`"`*(6!"(6!%(BU0D
-M8$B)4!1(BU0D:$B)4!Q)B<2`?48"#X<``0``2(V=\`L``$B)[^@DY/__QD`(
-M!4B)6!#&0"``28G%28L,)/Z!L`D```^VL;`)``")]T@IR$B#Z%!(P?@#2+K-
-MS,S,S,S,S$@/K\*(A'FP"0``00^V1"0+B(1YL0D``$&(="0+0;\`````@'U&
-M``^&TQX``$B)TT0X?4AT<DB)[^BOX___QD`(`42(>!!$B'@12(M4)&!(B5`4
-M2(M4)&A(B5`<00^V52`/MLI&B'PI&/_"08A5($F+50#^@K`)```/MK*P"0``
-MB?%(*=!(@^A02,'X`T@/K\.(A$JP"0``00^V10N(A$JQ"0``08AU"T'_QT0X
-M?48/AWO____I1AX``$&_`````(!]1@`/AC8>``!(C9WP"P``2;[-S,S,S,S,
-MS$0X?4@/A+T```!(B>_H`./__\9`"`%$B'@01(AX$4B+5"1@2(E0%$B+5"1H
-M2(E0'$B)1"1`2(GOZ-7B___&0`@#2(E8$$2(>!A)B<5(BQ#^@K`)```/MK*P
-M"0``B?%(BT0D0$@IT$B#Z%!(P?@#20^OQHB$2K`)``!!#[9%"XB$2K$)``!!
-MB'4+28L4)/Z"L`D```^VLK`)``")\4R)Z$@IT$B#Z%!(P?@#20^OQHB$2K`)
-M``!!#[9$)`N(A$JQ"0``08AT)`M!_\=$.'U&#X<L____Z4P=``"X`0```.M9
-M2(GON0````"`?48`=D9F9F:0#[;!2,'@!$@!^$B-D/`*``"^`````(.X\`H`
-M``!U$X-Z!`!U#8-Z"`!U!X-Z#`"0=`6^`0```(7V=:S_P3A/1G>^N`````"%
-MP'073(FE.`P``$C'A3`,````````Z6,*``!(C5PD8$B)W^B1X/__28G:OP``
-M```/MW5$NP````!!N2````!!NP$```!F9F:0@_\?=S\/ML-)C12"1(G)*?DY
-M\0]'SH/Y('4(QP+_____ZPU$B=C3X/_(B?G3X`D"1(G(*?@Y\',5C70^X+\`
-M````ZP.#[R#_PX#[`W:R#[9%1XB%"`P``,:%"0P```!(BT0D8$B)A?`+``!(
-MBT0D:$B)A?@+``!(QX4`#````````(!]1@(/A[@```!(C9WP"P``2(GOZ/7@
-M___&0`@%2(E8$,9`(`!)B<5!OP````"`?48`#X;I&P``2+O-S,S,S,S,S$B)
-M[^C$X/__QD`(`42(>!!$B'@12(M4)&!(B5`42(M4)&A(B5`<00^V52`/MLI&
-MB'PI&/_"08A5($F+50#^@K`)```/MK*P"0``B?%(*=!(@^A02,'X`T@/K\.(
-MA$JP"0``00^V10N(A$JQ"0``08AU"T'_QT0X?49WA>E?&P``#[9=1TB)[^A`
-MX/__QD`(`8A8$(A8$4B+5"1@2(E0%$B+5"1H2(E0'$B)1"0X0;\`````@'U&
-M``^&(AL``$B-G?`+``!)O,W,S,S,S,S,1#A]1P^$O0```$B)[^CLW___QD`(
-M`42(>!!$B'@12(M4)&!(B5`42(M4)&A(B5`<2(E$)$!(B>_HP=___\9`"`-(
-MB5@01(AX&$F)Q4B+$/Z"L`D```^VLK`)``")\4B+1"1`2"G02(/H4$C!^`-)
-M#Z_$B(1*L`D``$$/MD4+B(1*L0D``$&(=0M)BU4`_H*P"0``#[:RL`D``(GQ
-M2(M$)#A(*=!(@^A02,'X`TD/K\2(A$JP"0``00^V10N(A$JQ"0``08AU"T'_
-MQT0X?48/ARS____I.!H``&9FD&:0387D#X2\!P``0?9$)!@"D`^$KP<``$0/
-MMKU@#```1#B]80P```^"50$``&9FD&:01#A]1P^$-@$``$2)^`^VV$B)V$C!
-MX`1,C:0$\````$R)Y^BIW?__#[>4G4`,```/M[R=0@P``+L`````0;D@````
-M0;H!````@_H?=T0/ML-)C32$1(G)*=$Y^0]'SX/Y('4-QP;_____ZQ)F9I!F
-MD$2)T-/@_\B)T=/@"09$B<@IT#GX<Q6-?!?@N@````#K`X/J(/_#@/L#=JU$
-MB?L/ML-(B<=(P><$2(T$+TB-D'`+``"+B'`+``#WT8E,)&"+0@3WT(E$)&2+
-M0@CWT(E$)&B+0@SWT(E$)&Q(C70\<$B-E#SP````(PJ)#HM"!"-$)&2)1@2+
-M0@@C1"1HB48(BT(,(T0D;(E&#$0X?4AU,;H`````@SX`=1*#?@0`=0R#?@@`
-M=0:#?@P`=`6Z`0```(72N`$```!$#T7H9F:09I!!_\=$.+UA#```#X.P_O__
-M183M#X2:!0``2(U\)&#H;=S__T&_`````(!]1@!V69!$.'U(=$E$.'U'=$-$
-MB?@/MM!(P>($2`'J2(V*\`H``(M$)&`+@O`*``")1"1@BT0D9`M!!(E$)&2+
-M1"1H"T$(B40D:(M$)&P+00R)1"1L0?_'1#A]1G>H3`^V14A)P>`$2HUT!'"+
-M?"1@(SZ)?"10BTPD9"-.!(E,)%2+5"1H(U8(B50D6(M$)&PC1@R)1"1<]]>)
-M?"1@]]&)3"1D]]*)5"1H]]")1"1L(SZ)?"1@(TX$B4PD9"-6"(E4)&@C1@R)
-M1"1L2(GOZ+3<___&0`@`2(E$)"BX`````(-\)&``=16#?"1D`'4.@WPD:`!U
-M!X-\)&P`=`6X`0```(7`#X2F`0``#[9%2(B%"`P``$B+1"1@2(F%\`L``$B+
-M1"1H2(F%^`L``,:%"0P```%(QX4`#````````$&_`````(!]1@`/A@,"``!,
-MC:7P"P``1#A]2`^$/P$``$0XO6`,``!W#T0XO6$,``!R!D0X?4=U+4B)[^@(
-MW/__QD`(`42(>!!$B'@12(M4)&!(B5`42(M4)&A(B5`<2(E$)$#K4D2)^@^V
-MPDC!X`1(C5P$<(M$)&`)`XM$)&0)0P2+1"1H"4,(BT0D;`E##$B)[^BQV___
-MQD`(`42(>!!$B'@12(L32(E0%$B+4PA(B5`<2(E$)$!(B>_HB=O__\9`"`-,
-MB6`01(AX&$F)Q4B+$/Z"L`D```^VLK`)``")\4B+1"1`2"G02(/H4$C!^`-(
-MO\W,S,S,S,S,2`^OQXB$2K`)``!!#[9%"XB$2K$)``!!B'4+1#A]1W5$2(M,
-M)"A(BQ'^@K`)```/MK*P"0``B?%,B>A(*=!(@^A02,'X`T@/K\>(A$JP"0``
-M2(M<)"@/MD,+B(1*L0D``$"(<PM!_\=$.'U&#X>J_O__Z:$```!!OP````"`
-M?48`#X:1````1#B]8`P``'=[1#B]80P``')R1#A]1W1L1#A]2'1F1(GZ#[;"
-M2,'@!$B-1`1PN@````"#.`!U$H-X!`!U#(-X"`!U!H-X#`!T!;H!````A=)T
-M,D2)^0^VP4C!X`1(C5P$<$B)[^A>VO__QD`(`42(>!!$B'@12(L32(E0%$B+
-M4PA(B5`<0?_'1#A]1@^';____[@`````@WPD4`!U%8-\)%0`=0Z#?"18`'4'
-M@WPD7`!T!;@!````A<`/A!L5``!$#[9E2$2(I2@,``!(BT0D4$B)A1`,``!(
-MBT0D6$B)A1@,``#&A2D,````2,>%(`P````````/MEU'2(GOZ,;9___&0`@!
-MB%@01(A@$4B+5"102(E0%$B+5"182(E0'$B)1"0X2,=$)#``````0;\`````
-M@'U&``^&GA0``$R-I1`,``!)OLW,S,S,S,S,1#A]2`^$80$``$0X?4</A%<!
-M```/MEU'2(GOZ%K9___&0`@!1(AX$(A8$4B+5"102(E0%$B+5"182(E0'$B)
-M1"1`2(L0_H*P"0``#[:RL`D``(GQ2(M$)"A(*=!(@^A02,'X`TB_S<S,S,S,
-MS,Q(#Z_'B(1*L`D``$B+7"1`#[9#"XB$2K$)``!`B',+2(-\)#``=#Q(BQ/^
-M@K`)```/MK*P"0``B?%(BT0D,$@IT$B#Z%!(P?@#2`^OQXB$2K`)```/MD,+
-MB(1*L0D``$"(<PL/MEU'2(GOZ)W8___&0`@#3(E@$(A8&$F)Q4B)1"0P2(L0
-M_H*P"0``#[:RL`D``(GQ2(M$)$!(*=!(@^A02,'X`TD/K\:(A$JP"0``00^V
-M10N(A$JQ"0``08AU"TF+50#^@K`)```/MK*P"0``B?%(BT0D.$@IT$B#Z%!(
-MP?@#20^OQHB$2K`)``!!#[9%"XB$2K$)``!!B'4+0?_'1#A]1@^'B/[__^D0
-M$P``1`^VO6`,``!$.+UA#```#X+[$@``1#A]1W1R1(GZ#[;"2,'@!$B-1`1P
-MN@````"#.`!U&8-X!`!U$X-X"`!U#8-X#`!T#&9F9I!F9I"Z`0```(72=#=$
-MB?D/ML%(P>`$2(U<!'!(B>_HD-?__\9`"`%$B'@01(AX$4B+$TB)4!1(BU,(
-M2(E0'&9FD&:00?_'1#B]80P```^#>/___^EN$@``@'U(!P^&TPD``$B-?"1@
-MZ"?6___&1"0G`$&_`````(!]1@`/AJ$```!FD$0X?4</A(@```!$B?L/ML-(
-MP>`$2`'H2(V0\`H``+D`````@[CP"@```'46@WH$`'40@WH(`'4*@WH,`&9F
-M9I!T!;D!````A<ET1T2)^`^VT$C!X@1(`>I(C8KP"@``BT0D8`N"\`H``(E$
-M)&"+1"1D"T$$B40D9(M$)&@+00B)1"1HBT0D;`M!#(E$)&S^1"0G0?_'1#A]
-M1@^'8?___P^V5"0G#[9%1H/H`]'X.<(/CE0%```/MEU'B)T(#```2(M$)&!(
-MB87P"P``2(M$)&A(B87X"P``QH4)#````4C'A0`,````````2(GOZ#_6___&
-M0`@"B%@0B%@12(M4)&!(B5`42(M4)&A(B5`<28G$2(GOZ!C6___&0`@&2(E$
-M)"A)BQ0D_H*P"0``#[:RL`D``(GQ2(M$)"A(*=!(@^A02,'X`TF^S<S,S,S,
-MS,Q)#Z_&B(1*L`D``$$/MD0D"XB$2K$)``!!B'0D"X!]1@(/ATD"``!(C9WP
-M"P``2(GOZ*S5___&0`@%2(E8$,9`(`!)B<5(BTPD*$B+$?Z"L`D```^VLK`)
-M``")\4@IT$B#Z%!(P?@#20^OQHB$2K`)``!(BUPD*`^V0PN(A$JQ"0``0(AS
-M"T&_`````(!]1@`/AE\0``!$.'U'#X2^`0``00^V12`/MM!&B'PJ&/_`08A%
-M($2)^`^V\$C!Y@1(C00N2(V(<`L``(N0<`L``/?2B50D4(M!!/?0B40D5(M!
-M"/?0B40D6(M!#/?0B40D7$B-3#1P(U0D8(D1BT0D9"-$)%2)002+1"1H(T0D
-M6(E!"(M$)&PC1"1<B4$,N`````"#.0!U$H-Y!`!U#(-Y"`!U!H-Y#`!T!;@!
-M````A<!T:T2)^@^VPDC!X`1(C5P$<$B)[^B+U/__QD`(`42(>!!$B'@12(L3
-M2(E0%$B+4PA(B5`<28M5`/Z"L`D```^VLK`)``")\4@IT$B#Z%!(P?@#20^O
-MQHB$2K`)``!!#[9%"XB$2K$)``!!B'4+1(GY#[;!2,'@!$B-A"CP"@``N@``
-M``"#.`!U$H-X!`!U#(-X"`!U!H-X#`!T!;H!````A=)FD'1Z1(GX#[;82(G8
-M2,'@!$B-G"CP"@``2(GOZ.'3___&0`@"1(AX$$2(>!%(BQ-(B5`42(M3"$B)
-M4!Q)B<1(BQ#^@K`)```/MK*P"0``B?%(BT0D*$@IT$B#Z%!(P?@#20^OQHB$
-M2K`)``!!#[9$)`N(A$JQ"0``08AT)`M!_\=$.'U&#X<K_O__Z84.``!!OP``
-M``"`?48`#X9U#@``2(V5\`L``$B)5"082;[-S,S,S,S,S$0X?4</A`D"``!$
-MB?D/MO%(P>8$2(T$+DB-B'`+``"+D'`+``#WTHE4)%"+003WT(E$)%2+00CW
-MT(E$)%B+00SWT(E$)%Q(C5PT<"-4)&")$XM$)&0C1"14B4,$BT0D:"-$)%B)
-M0PB+1"1L(T0D7(E##$B)[^C2TO__QD`(`TB+?"082(EX$$2(>!A)B<5(BT0D
-M*$B+$/Z"L`D```^VLK`)``")\4R)Z$@IT$B#Z%!(P?@#20^OQHB$2K`)``!(
-MBWPD*`^V1PN(A$JQ"0``0(AW"[@`````@SL`=1*#>P0`=0R#>P@`=0:#>PP`
-M=`6X`0```(7`=&M$B?H/ML)(P>`$2(U<!'!(B>_H/M+__\9`"`%$B'@01(AX
-M$4B+$TB)4!1(BU,(2(E0'$F+50#^@K`)```/MK*P"0``B?%(*=!(@^A02,'X
-M`TD/K\:(A$JP"0``00^V10N(A$JQ"0``08AU"T2)^0^VP4C!X`1(C80H\`H`
-M`+H`````@S@`=1*#>`0`=0R#>`@`=0:#>`P`=`6Z`0```(72='I$B?@/MMA(
-MB=A(P>`$2(V<*/`*``!(B>_HEM'__\9`"`)$B'@01(AX$4B+$TB)4!1(BU,(
-M2(E0'$F)Q$B+$/Z"L`D```^VLK`)``")\4B+1"0H2"G02(/H4$C!^`-)#Z_&
-MB(1*L`D``$$/MD0D"XB$2K$)``!!B'0D"T'_QT0X?48/A^#]___I.@P```^V
-M74=(B>_H&]'__\9`"`&(6!"(6!%(BU0D8$B)4!1(BU0D:$B)4!Q(B40D.`^V
-M74=(B>_H[M#__\9`"`*(6!"(6!%(BU0D8$B)4!1(BU0D:$B)4!Q)B<1(B>_H
-MQ]#__\9`"`9(B40D*$F+#"3^@;`)```/MKFP"0``B?Y(BT0D*$@IR$B#Z%!(
-MP?@#2+K-S,S,S,S,S$@/K\*(A'&P"0``00^V1"0+B(1QL0D``$&(?"0+0;\`
-M````@'U&``^&=PL``&9F9I!$.'U'#X3&`@``1(GZ#[;"2,'@!$@!Z$B-D/`*
-M``"Y`````(.X\`H```!U%H-Z!`!U$(-Z"`!U"H-Z#`!F9F:0=`6Y`0```(7)
-M#X2!`@``_DPD)P^V3"0G`DU&08G.1(G[1`^VXTG!Y`1)`>Q)C;PD\`H``$B)
-M?"002(GOZ-_/___&0`@!1(AX$$2(<!%)BY0D\`H``$B)4!1(BTPD$$B+40A(
-MB5`<2(E$)$`/MEU'2(GOZ*G/___&0`@$B%@@1(AP(4F+E"3P"@``2(E0$$B+
-M7"002(M3"$B)4!A)B<5(BQ#^@K`)```/MK*P"0``B?%(BT0D.$@IT$B#Z%!(
-MP?@#2+_-S,S,S,S,S$@/K\>(A$JP"0``00^V10N(A$JQ"0``08AU"TF+50#^
-M@K`)```/MK*P"0``B?%(BT0D0$@IT$B#Z%!(P?@#2`^OQXB$2K`)``!!#[9%
-M"XB$2K$)``!!B'4+2(M$)"A(BQ#^@K`)```/MK*P"0``B?%,B>A(*=!(@^A0
-M2,'X`T@/K\>(A$JP"0``2(M<)"@/MD,+B(1*L0D``$"(<PL/MEU'2(GOZ*W.
-M___&0`@$B%@@1(AX(4F+E"3P"@``2(E0$$B+?"002(M7"$B)4!A)B<5(BQ#^
-M@K`)```/MK*P"0``B?%(BT0D.$@IT$B#Z%!(P?@#2+O-S,S,S,S,S$@/K\.(
-MA$JP"0``00^V10N(A$JQ"0``08AU"TB+?"0H2(L7_H*P"0``#[:RL`D``(GQ
-M3(GH2"G02(/H4$C!^`-(#Z_#B(1*L`D```^V1PN(A$JQ"0``0(AW"TB)[^CX
-MS?__QD`(`D2(>!!$B'@128N4)/`*``!(B5`42(M,)!!(BU$(2(E0'$F)Q$B+
-M$/Z"L`D```^VLK`)``")\4B+1"0H2"G02(/H4$C!^`-(#Z_#B(1*L`D``$$/
-MMD0D"XB$2K$)``!!B'0D"Y!!_\=$.'U&#X<C_?__Z9$(```/MD5(.D5'#X6H
-M````0;\`````@'U&``^&=`@``)!$.'U'=']$B?L/ML-(P>`$2`'H2(V0\`H`
-M`+D`````@[CP"@```'48@WH$`'42@WH(`'4,@WH,`'0+9F:09F:0N0$```"%
-MR70\1(GX#[;82,'C!$@!ZTR-H_`*``!(B>_H^\S__\9`"`)$B'@01(AX$4B+
-MD_`*``!(B5`428M4)`A(B5`<0?_'1#A]1@^';O___^G<!P``2(U\)&#HG\O_
-M_TB)[^BWS/__QD`(!DB)1"0H0;\`````@'U&``^&&P$``$0X?4</A`0!``!$
-MB?H/ML)(P>`$2`'H2(V0\`H``+D`````@[CP"@```'44@WH$`'4.@WH(`'4(
-M@WH,`&:0=`6Y`0```(7)#X3!````1(GY#[;!2,'@!$R-)"A)C9PD\`H``(M$
-M)&!!"X0D\`H``(E$)&"+1"1D"T,$B40D9(M$)&@+0PB)1"1HBT0D;`M##(E$
-M)&Q$.'U(='1(B>_H_,O__\9`"`)$B'@01(AX$4F+E"3P"@``2(E0%$B+4PA(
-MB5`<28G$2(L(_H&P"0``#[:YL`D``(G^2(M$)"A(*<A(@^A02,'X`TBZS<S,
-MS,S,S,Q(#Z_"B(1QL`D``$$/MD0D"XB$<;$)``!!B'PD"T'_QT0X?48/A^7^
-M__\/MEU'2(GOZ'?+___&0`@"B%@0B%@12(M4)&!(B5`42(M4)&A(B5`<28G$
-M2(L(_H&P"0``#[:YL`D``(G^2(M$)"A(*<A(@^A02,'X`TBZS<S,S,S,S,Q(
-M#Z_"B(1QL`D``$$/MD0D"XB$<;$)``!!B'PD"T@/MG5(2(GW2(GP2,'@!$B-
-MM"AP"P``BP;WT(E$)%"+5@3WTHE4)%2+3@CWT8E,)%B+=@SWUD2+3"1@1"'(
-MB40D4$2+1"1D1"'"B50D5(M4)&@AT8E,)%B+1"1L(<:)="1<2,'G!$B-O"]P
-M"P``1",/1(E,)&!$(T<$1(E$)&0C5PB)5"1H(T<,B40D;+@`````187)=1!%
-MA<!U"X72=0>#?"1L`'0%N`$```"%P)`/A`\"```/MD5'B(4(#```2(M$)&!(
-MB87P"P``2(M$)&A(B87X"P``QH4)#````4C'A0`,````````0;\`````@'U&
-M``^&RP$``$R-M?`+``!)O,W,S,S,S,S,9F:09I!$.'U(=5I(B>_H\LG__\9`
-M"`-,B7`01(AX&$B+7"0H2(L3_H*P"0``#[:RL`D``(GQ2"G02(/H4$C!^`-)
-M#Z_$B(1*L`D```^V0PN(A$JQ"0``0(AS"^E-`0``9F:09I!$.'U'#X0^`0``
-M1(GZ#[;"2,'@!$B-7`1P2`'H2(VX<`L``(NP<`L``/?6B3.+3P3WT8E+!(M7
-M"/?2B5,(BT<,]]")0PPC="1@B3,C3"1DB4L$(U0D:(E3""-$)&R)0PQ(B>_H
-M.,G__\9`"`-,B7`01(AX&$F)Q4B+3"0H2(L1_H*P"0``#[:RL`D``(GQ2"G0
-M2(/H4$C!^`-)#Z_$B(1*L`D``$B+?"0H#[9'"XB$2K$)``!`B'<+N`````"#
+M?T5,1@(!`0D```````````$`/@`!`````````````````````````"CA````
+M`````````$```````$``#@`+`$B![$@"``!(B9PD&`(``$B)K"0@`@``3(FD
+M)"@"``!,B:PD,`(``$R)M"0X`@``3(F\)$`"``!(B?M,BV\02(UOB$R-9"00
+MN`(```"`?P$`=0N`?P(!&<"#X`+_P(A%`$C'10@`````@$T!!$B+0Q!(B440
+MBU,82(E5&$@/MD4`2(L$Q0````!(B45H@'T``W4N2(U"]DB)11B+<QB#[@I,
+MB>&Z(````$B)W^@`````08$\)/,6>%IT(&9FD&9FD$@/MD4`2(L$Q0````!(
+MB45P@$T!!.E:!0``OI````!,B>?H`````(3`==5!]D0D"0)T'<9#"`%!#[9$
+M)`J(0PLZ0PES"P^V\$B)W^@`````O@`"``!,B>?H`````(3`=!=)C;PDD```
+M`+IP`0``O@````#H`````$$/MI0DDP```+X!````(=8/MD,$@^#\B=&#X0()
+M\`G(O@0````AUH/@\XG1@^$("?`)R+X0````(=:#X,^)T8/A(`GP"<B)T8/A
+M0(/@/X/B@`G("="(0P2H`704T.B)QH/F`;H`````2(G?Z``````/MD,$J`1T
+M%<#H`XG&@^8!N@````!(B=_H``````^V0P2H$'00P.@%B<:#Y@%(B=_H````
+M``^V0P2H0'0.P.@'#[;P2(G?Z`````!!#[94)`F#X@$/MD4!@^#^"="(10%!
+M@'PD$`,/A*S^__]!@'PD$`!U%,9%``!(BP4`````2(E%<.D$!```08-\)`0`
+M#X2$_O__N@````!!#[9$)`N)P8/X`'XQ2,?&`````$ACPDC!X`1,`>"`>!`(
+M#X=7_O__2`^V0!!(@SS&``^$1_[____".=%_UD6+?"0808N$))@```!(P>`@
+M20G'N@````!(8\)(:<!(`0``2HV<*,A$``!(@WL(`'4.BT-\03M$)`0/A#$!
+M``#_PH/Z#W[228N=P$0``$B%VW0A2(M#$$F)A<!$``"Z2`$``+X`````2(G?
+MZ`````!,B6L008M$)`2)0WP/MI.2````@\H!B).2````9D$/MD0D%(/@`0'`
+M@^+]"<*(DY(```!!#[=$)!9FB8.4````3(F[F````$$/MD0D$(@#00^V1"03
+MB$-Z#[;(N`$```#3X&:)@Y````!(QT,(`````$&+5"0,2(E3&$&+A"24````
+M2,'@($@)T$B)0QA!#[9$)!&(0WA(#[8#2(L$Q0````!(B4-H28UT)"Q(C;O@
+M````NA````#H`````$F-="0\2(V[\````+H$````Z`````!)C;0D@````$B-
+MNS0!``"Z$````.@`````28UT)$!(C;OT````ND````#H`````$PYNY@```!V
+M!TR)NY@```!!]D0D"0%T!(!+`0%!#[9$)!*(1"0/08!\)`L!#X;:`0``#[;`
+M3(NTPZ````!%BWPD*$&+A"2@````2,'@($D)QTV%]@^%60$``$V+M<!$``!-
+MA?9T(4F+1A!)B87`1```ND@!``"^`````$R)]^@`````38EN$$&+1"0$08E&
+M?$$/MI:2````@\H!08B6D@```&9!#[9$)"2#X`$!P(/B_0G"08B6D@```$$/
+MMT0D)F9!B8:4````38F^F````$$/MD0D(4&(1GA!#[9$)"!!B`9)B5X(08M4
+M)!Q)B58808N$))P```!(P>`@2`G028E&&$$/MD0D(T&(1GH/MLBX`0```-/@
+M9D&)AI````!)#[8&2(L$Q0````!)B49H2`^V`TB+!,4`````28E&<`^V1"0/
+M08A&`T@/MD0D#TR)M,.@````_D-Y28UT)"Q)C;[@````NA````#H`````$F-
+M="0\28V^\````+H$````Z`````!)C;0D@````$F-OC0!``"Z$````.@`````
+M28UT)$!)C;[T````ND````#H`````$'V1"0)`70%08!.`0%)#[9$)"))@[S&
+MH`````!U<TF)K,:@````00^V1"0BB$4#0?Y&>4R)=0A)#[8&2(L$Q0````!(
+MB45P33F^F````'9!38F^F````.LX2`^V1"0/2(.\PZ``````=2=(B:S#H```
+M`$B)70C^0WD/MD0D#XA%`T@/M@-(BP3%`````$B)17!(BYPD&`(``$B+K"0@
+M`@``3(ND)"@"``!,BZPD,`(``$R+M"0X`@``3(N\)$`"``!(@<1(`@``PV9F
+M9I!54TB#[`A(B?.]``````^V1GB)PH/X``^.TP```&9FD$ACQ4B#O,.@````
+M`'0+_\4YZG_LZ;<```"X`0```(GIT^!F"8.4````2(G>Z`````!(B<)(A<`/
+MA),```#&``-`B&@#2(E8"(`["`^4P,'@`P^V2@&#X?<)P8A*`4B+!0````!(
+MB4)H2`^V`TB+!,4`````2(E"</9#`1!T$HG(@\@0B$(!BX*0````2(E"&$AC
+MQ4B)E,.@````#[:#D@```(/@_H/("HB#D@```$C'@Y@`````````_D-Y2(G6
+MOP<```#H`````("+D@````1(@\0(6UW#9F:09F:09F:02(/L*$B)7"0(2(EL
+M)!!,B60D&$R);"0@28G]2(GS#[8&@_@&#X29````@_@&?Q&#^`0/C+H!``#K
+M&V9FD&9FD(/X"`^$]P```&9F9I!F9I#IG0$```^V1GDZ1GAF9F:0#X6,`0``
+M@$X!!("FD@```/Y!O`````"`?G@`#X1Q`0``26/$2(NLPZ````!(A>UT(8!]
+M``-V&TB)[DR)[^@`````]H62````!'0'@(N2````!$'_Q`^V0WA$.>!_P^DO
+M`0``@$X!!(!^>0)U$X"FD@```/[I&0$``&9F9I!F9I"`II(```#]2(.^H```
+M``!U'4B+AJ@```#&0`,`2(F&H````$C'AJ@`````````2(N#H````(`X`W4+
+M2(G>3(GOZ`````"`BY(````$2,>#F`````````#IM@```&9F9I`/MD9Y.D9X
+M=2F`3@$$#[:&D@```(/@_HB&D@```&:#OI0`````='F#R`*(AI(```#K;@^V
+M1GG_P`^V5G@YT'5@9H.^E`````!T++D`````#[>&E````(G"J`%U"O_!B=#3
+M^*@!=/9(8\%(@[S#H`````!U(>L)]H:2`````G46@$L!!("+D@````1(B=Y,
+MB>_H`````$C'@Y@`````````2`^W@Y````!(]]A((8.8````2(-["`!U&?:#
+MD@````1T$.@`````B4-\2(G?Z`````!(BUPD"$B+;"003(MD)!A,BVPD($B#
+MQ"C#9F:09I!!5%5328G\O0````!F9F:02&/%2&G`2`$``$J-G"#(1```@WM\
+M`'0@2(-["`!U&4B)WDR)Y^@`````]D,!!'0(2,=#<`````#_Q8/]#W[!6UU!
+M7,-F9F:09F9FD&9F9I!!5T%6055!5%532('L"`(``$F)_4B)]4&)UDF)YTB)
+M]^@`````@'T`"'4+2(GN3(GOZ`````!!O`````"`?7@`#X;'````26/$2(.\
+MQ:``````#X2H````2(NTQ:````"`/@-V$4$/MM9,B>_H`````.F*````26/$
+M2(N<Q:````#&0P,`2,=#"`````#'@Y0`````````@&,!_DC'0W``````183V
+M=0V`?0`&=0=(@WT(`'0JN@`"``"^`````$R)_^@`````2(U[>$R)^;HP````
+MO@````#H`````.L3#[95`8/B`0^V0P&#X/X)T(A#`4B)W^@`````9F:09F:0
+M0?_$1#AE>`^'.?___[I(`0``O@````!(B>_H`````$F+A<!$``!(B44028FM
+MP$0``,=%?`````!(@<0(`@``6UU!7$%=05Y!7\-F9I!F9I!!5D%505154TB!
+M[``"``!(B=-(BP9,BV@(3(MV0$F+;BA$#[9@`TB)XDB%[70-9D$/MD8@@^`!
+M.<AU(4B)U4F#?C``=`U,B?9!_U8PA<!FD'4*N`````#IBP```$$/MTY4P>$)
+M00^V=E!F9F:09F:03(M%`+\```$`9H-]"`!T!`^W?0@Y^8GZ#T;11#GF=1%F
+MB5,(9L=#"@``3(D#2(/#$"G7B=!)`<`IT74>_\9!#[9%>#G&N``````/1/!!
+M#[>%D````(G!P>$)A?]UMF:!?0H`@'0&2(/%$.N49L=#^@"`N`$```!(@<0`
+M`@``6UU!7$%=05[#9F:09F:09F:03(M/"$R+1D@/MD\##[=^6#A.4'4+2`^W
+M1E9)`<#K$9`X3E!V"TD/MX&0````20'`.$Y0=0MF`WY49F9FD&9FD#A.474+
+M9@-^4F9F9I!F9I"`?EP`=`XX3E!S)3A.46:0=Q3K'#A.4',(9D$#N9`````X
+M3E%V"F9!`[F0````9I!FB7H03(E"",-F9F:09F:04TF)^4F)\$B+?@A$#[=6
+M$$&[`````$$/MDEZ2(G^2-/N9D''0%@``$'&0%P!20^V47A(B?!(B=&Z````
+M`$CW\4$/MDEZ2-/@28E`2$D/ME%X2(GP2(G1N@````!(]_%!B%!0B=%!#[>!
+MD````/_((?AF08E`5D$/MY&0````9BG"9D0YTG(-9D6)4%1!N@````#K8F9!
+MB5!49D$ITD&[`0```+H!````B=#3X&9!"4!:_\%!.$EX=2FY`````$&`>%P`
+M=`M!QD!<`.L69F9FD$$/MT!89D$#@9````!F08E`6&9%.9&0````<PIF12N1
+MD````.NRN`$```#3X&9!"4!:08A(469%B5!200^WPUO#9F9FD&9F9I!F9F:0
+M9F:055-(@^P(2(G]2(M>0$B+!@^V2`.X_O___]/`9B%#6H!^(0%T!P^V1B&(
+M0R%(B>_H`````&:#>UH`=1F`>R$`=03&0R$!2(MS.$B)VDB)[^@`````2(/$
+M"%M=PV9FD&:02(/L.$B)7"0(2(EL)!!,B60D&$R);"0@3(ET)"A,B7PD,$F)
+M_TB)]4R+-D'V1@$$=17&1B$"2(MV.$B)ZN@`````Z<H```!,B??H,?[__V:#
+M?5H`=1K&12$"2(MU.$B)ZDR)_^@`````Z:,```!FD$&]`````$&`?G@`#X20
+M````#[=%6D2)Z=/XJ`%T<DR)_^@`````2(G#26/%38NDQJ````!,B2-(B6M`
+M#[95((/B`@^V0R"#X/T)T(A#(`^V52"#X@2#X/L)T(A#(`^V11*(0Q)(B=I(
+MB>Y,B>?H#OW__TC'0S@`````2,=#,`````!(B=Y,B?]!_U0D:&9FD$'_Q4$/
+MMD9X1#GH#X]P____2(M<)`A(BVPD$$R+9"083(ML)"!,BW0D*$R+?"0P2(/$
+M.,-F9F:09F9FD&9F9I!(@^P(2(M."`^V40&`9@'[@&$!^_Y)>8")D@````%(
+M@WEP``^5P/;"!`^5P@^VP(7"=`U(B<[_47!F9F:09F:02(/$",-F9F:09F9F
+MD&9FD$B#[`A,BT9`28MP*$B%]G0-9D$/MD`@@^`!.<AU(DF#>#``=!!,B<9!
+M_U`PN@$```"%P'4PN@````#K*69F9I!(BP9(B0)(BT8(2(E""$B#PA!(C48*
+M2(/&$&:#.`!YX+H!````B=!(@\0(PV9F9I!54TB)^P^V!X/H!(/X`7=0O0``
+M``"`?W@`=#=F9I!FD$ACQ4B#O,.@`````'012(N\PZ````#HQ?___X7`=0>X
+M`````.LD_\4/MD-X.>AFD'_.N`$```#K$69F9I!F9I`/MD<!P.@"@^`!6UW#
+M9F:02(/L*$B)'"1(B6PD"$R)9"003(EL)!A,B70D($F)_DB)]4B+7D!,BR-,
+MBRZ`?B$!#X3:````]D,@`@^$@````(![(@`/A<8```!!]H0DD@````(/A;<`
+M``#&0R(!08!]`P%(&<"#X`A.BZP@H````$V%[71"3(GOZ!/___^%P'0V2(U]
+M2+I8````O@````#H`````$$/MDT#N`$```#3X&:)0UI,B6T`2(GN3(GW0?]5
+M:.F^````3(MM`.M39F:0]D,@!'1*3(GOZ,+^__^%P'0B2(U]2+I8````O@``
+M``#H`````$B)[DR)]T'_56CI@0```(![(0%T'0^W4UI!#[9-`[@!````T^`Y
+MPG4)9I`/MD4AB$,A2(GN3(GWZ`````!!#[9-`[C^____T\!F(4-:9H-[6@!U
+M/(![(0!U!,9#(0%(BW,X2(G:3(GWZ`````!F0?],)"IF08-\)"H`=11)BT0D
+M.$B%P'0*28MT)$!,B??_T$B+'"1(BVPD"$R+9"003(ML)!A,BW0D($B#Q"C#
+M9F9FD&9F9I!F9I!F9I!(@^P(2(M&0(7)=!I(BT`82(D"#[=&$,'@"6:)0@AF
+MQT(*`(#K44B#>"@`=#/V0"`!="U(BT@H2(L!2(D"2(M!"$B)0@A(@\(02(U!
+M"DB#P1!F@S@`>>#K'F9F9I!F9I"^`````$B#>#``=!!(B<:0_U`PB<;K!;X!
+M````B?!(@\0(PV9F9I!F9I!F9I!F9I!(@^P82(D<)$B);"0(3(ED)!!)B?Q(
+MB?-(BVY`2(L&#[9(`[C^____T\!F(45:@'XA`70'#[9&(8A%(4B)WDR)Y^@`
+M````9H-]6@!U6H!](0!U),9%(0%(BTT`2(N1F````$@[4PAU#T@/MT,02`'0
+M2(F!F````$B+=3A(B>I,B>?H`````$B+10!FQT`H``!(BU4`2(/",$C'Q@``
+M``!,B>?H`````$B+'"1(BVPD"$R+9"002(/$&,-F9I!F9I!(@^P82(D<)$B)
+M;"0(3(ED)!!)B?Q(B?-(BVY`@'XA`70.#[9&(8A%(>MY9F:09I"`?A(@=4K&
+M1B$`2(M%`$B+@*@```!(B09(C7Y(NE@```"^`````.@`````QD,B`,9#$C`/
+MMD,@@^#]@\@$B$,@2(L#2(G>3(GG_U!HZV%FD,9%(0%(BTT`2(N1F````$@[
+M5@AU#T@/MT802`'02(F!F````$B)WDR)Y^@`````2(MU.$B)ZDR)Y^@`````
+M2(M%`&;'0"@``$B+50!(@\(P2,?&`````$R)Y^@`````2(L<)$B+;"0(3(MD
+M)!!(@\08PV9F9I!F9F:09F:09F:005154TF)]$B)U4B+1D"%R71&2(M0&$B)
+M50`/MT80P>`)9HE%"&;'10H`@$B+!H!X`P`/A`T!``!(C8(```$`9H-]"`!T
+M"4@/MT4(2(T$`DB)10#I[0````^W7A#!XPE(@W@H`'0L]D`@`70F2(M(*$B+
+M`4B)`DB+00A(B4((2(/"$$B-00I(@\$09H,X`'G@ZS2Z`````$B#>#``#X2H
+M````2(GJ2(G&_U`PN@````"%P`^$D@```.L,9HE9"&;'00H`@.M&2(GI9F9F
+MD&9FD+H```$`9H-Y"`!T!`^W40A)BP0D@'@#`'06.=IV%HG82`$!B=!F*=AF
+MB4$(ZPYFD#G:<[9(@\$0*=-UQ$F+!"2`>`,`#Y7`2#G-#Y7"#[;`A<)T(4B+
+M`4B)10!(BT$(2(E%"$B#Q1!(C4$*2(/!$&:#.`!YW[H!````B=!;74%<PV9F
+M9I!F9F:09F9FD&9FD$B#[!A(B1PD2(EL)`A,B60D$$F)_$B)\TB+;D"`?B$!
+M=`X/MD8AB$4AZ88```!FD$B+!H!X`P!U-\9&(0!(BT4`2(N`J````$B)!DB-
+M?DBZ6````+X`````Z`````#&0R(`2(L#2(G>3(GG_U!HZW\/MU80P>()B=9(
+MBWT82`'^Z`````"%P'0'QD4A#.LED,9%(0%(BTT`2(N1F````$@[4PAU#T@/
+MMT,02`'02(F!F````$B)WDR)Y^@`````2(MU.$B)ZDR)Y^@`````2(M%`&;'
+M0"@``$B+50!(@\(P2,?&`````$R)Y^@`````2(L<)$B+;"0(3(MD)!!(@\08
+MPV9F9I!F9F:09F:09F:02(/L*$B)7"0(2(EL)!!,B60D&$R);"0@28G\2(GS
+M2(LN2,=%.`````#VA9(````!#X5J`0``#[9&$H/X`@^$`P$``(/X`G\.@_@!
+M#X2?````Z4H!``"#^`-F9F:09F:0#X4Z`0``2(M&"$B)12`/MT809HE%*`^W
+M5A#!X@E(BWX8O@````#H`````&;'0UH#`$&]`````$R)Y^@`````2(G&QD`2
+M,$B+0PA(B48(#[=#$&:)1A"`3B`$2(E>0$C'1C@`````2,=&,`````!)8\5(
+MBX3%H````$B)!DR)Y_]0:$'_Q4&#_0%^K.G#````2(M&"$B)12`/MT809HE%
+M*.@`````2(G&QD`2($B+0PA(B48(#[=#$&:)1A"`3B`"2(E>0$C'1C@`````
+M2,=&,`````!(BX6@````2(D&3(GG_U!HZVU(BT8(2(E%(`^W1A!FB44HZ```
+M``!(B<;&0!(@2(M#"$B)1@@/MT,09HE&$(!.(`)(B5Y`2,=&.`````!(QT8P
+M`````$B+A:````!(B09,B>?_4&CK%V9F9I#&0R$&2(MS.$B)VDR)Y^@`````
+M2(U5,$C'Q@````!,B>?H`````$B+7"0(2(ML)!!,BV0D&$R+;"0@2(/$*,.0
+M2(/L.$B)7"0(2(EL)!!,B60D&$R);"0@3(ET)"A,B7PD,$F)_TF)]$B++O9%
+M`01U%<9&(0)(BW8X3(GBZ`````#IV@$``/9&("!T)$C'13@`````2(EU0&:#
+M?2H`#X6]`0``Z+C]___ILP$``&9FD$B#?3@`=2EF@WTH`&9F9I!T/D@/MT4H
+M2`-%($@[1@AV+T@/MT802`-&"$@Y12!S($G'1"10`````$B-?3!,B>;H````
+M`.EF`0``9F:09F:09O]%*O:%D@````,/A9\```!!]D0D(`(/A),```!(BY6@
+M````2(NUJ````(`Z`W5D@#X#=5])BT0D"(G!*XJ8````08G`1"N&F````(NZ
+MG````(NVG````(G(F3'1*=%$B<"903'002G01#G!?1.%]@^41"0'@_\%#Y?`
+M($0D!^LTA?\/E40D!X/^!0^6P`A$)`?K(8"]E@`````/E$0D!P^V3"0'B(V6
+M````ZPAF9I#&1"0'`&9!QT0D6@``0;X`````1(GP1`^VZ$J#O.V@`````'1[
+M2HN$[:````#V0`$$=&U!#[94)"#`Z@)$.'0D!P^4P`G0J`%T5DR)_^@`````
+M2(G#ND@```!,B>9(B<?H`````+@!````1(GQT^!F00E$)%I,B6-`2,=#.```
+M``!(QT,P`````$J+A.V@````2(D#2(G>3(G__U!H9F:00?_&08#^`0^&9O__
+M_V9FD$B+7"0(2(ML)!!,BV0D&$R+;"0@3(MT)"A,BWPD,$B#Q#C#9F9FD&9F
+MD&9FD&9FD%532(/L"$B)]4B+7@@/ME8#@&8!^X"+D@````%(QX.8````````
+M`/Y+>8![>0!U!H!C`?OK'`^V@Y(```"H`G01@^#7B(.2````A=)U!(!C`?OV
+M0P$$=1U(@WMP``^$+`$``$B)WO]3<&9F9I!F9I#I&@$``(72=21(BY.@````
+MQD(#`4B+@Z@```#&0`,`2(F#H````$B)DZ@```!(@WL(`'0.2(M#"/9``00/
+MA(,```!(BX.@````@#@#=7=(B=[H`````$B)PDB%P'1GQ@`#QD`#`4B)6`A(
+MBP4`````2(E":$B+!0````!(B4)P]D,!$'0.@$H!$(N"D````$B)0AA(QT4(
+M`````$B)DZ@````/MH.2````@^#^@\@*B(.2````_D-Y2(G6OP<```#H````
+M`$B#>P@`=`]F9F:02(M;"$B#>P@`=?6+@X@```")@XP```"+@X0```")@X@`
+M``"+@X````")@X0```"+0WR)@X````#H`````(E#?,:#EP````%(B=_H````
+M`$B#Q`A;7<-F9F:09F:09F:02('L*`(``$B)G"0``@``2(FL)`@"``!,B:0D
+M$`(``$R)K"08`@``3(FT)"`"``!)B=1,BS9(BVY`2(M=*$&]$0```$B)XDB%
+MVW0,9@^V12"#X`$YR'4I2(G32(-],`!T#TB)[O]5,(7`=15F9I!FD+@`````
+MZ>4```!F9I!F9I"`?5'_#X2^````#[=-5,'A";H```$`9H-["`!T!`^W4P@Y
+MT78[00^V1@,Z15!U%$B+`TF)!"1(BT,(28E$)`A)@\00*=%(@\,0N@```0!F
+M@WL(`'0$#[=3"$'_S3G1=\5!#[9&`SI%4'4<2(L#28D$)&9!B4PD"&9!QT0D
+M"@"`N`$```#K63G1<S>)R$@#`TF)!"2)T&8IR&9!B40D"&:!>PH`@'4/9D''
+M1"0*`("X`0```.LJ9D''1"0*``!)@\002(/#$$'_S42)ZDC!X@1(B=Y,B>?H
+M`````+@!````2(N<)``"``!(BZPD"`(``$R+I"00`@``3(NL)!@"``!,B[0D
+M(`(``$B!Q"@"``##D`^V1P,Z1E!U%P^W1E1FB4(02(M&2$B)0@C#9F:09F:0
+M#[=&4F:)0A!(QT((`````,-F9F:09F9FD&9F9I!F9I!,BT8(1`^W3A!!NP``
+M``"Y`````(!_>``/A(,```!%#[?1B<A(BY3'H````$PY0AAV84R)1DB(3E"X
+M`0```-/@#[=^6@G'9HE^6DN-!`)(.T(8=C1!NP$```"-00&(1E'_P42)V-/@
+MB<&)^`G(9HE&6DB+0AAF1"G`9HE&5&9!*<%F1(E.4NL9QD91_V9$B4Y4ZPY,
+M*T(8_\$/MD=X.<AW@42)V,-F9F:09F9FD&9FD%532(/L"$B)_4B+7D!(BP8/
+MMD@#N/[____3P&8A0UJ`?B$!=`</MD8AB$,A2(GOZ`````!F@WM:`'49@'LA
+M`'4$QD,A`4B+<SA(B=I(B>_H`````$B#Q`A;7<-F9I!FD$B#[#A(B5PD"$B)
+M;"003(ED)!A,B6PD($R)="0H3(E\)#!)B?](B?5,BS9!]D8!!'45QD8A`DB+
+M=CA(B>KH`````.G*````3(GWZ*'^__]F@WU:`'4:QD4A`DB+=3A(B>I,B?_H
+M`````.FC````9I!!O0````!!@'YX``^$D`````^W15I$B>G3^*@!='),B?_H
+M`````$B)PTECQ4V+I,:@````3(DC2(EK0`^V52"#X@(/MD,@@^#]"="(0R`/
+MME4@@^($@^#["="(0R`/MD42B$,22(G:2(GN3(GGZ,[]__](QT,X`````$C'
+M0S``````2(G>3(G_0?]4)&AF9I!!_\5!#[9&>$0YZ`^/</___TB+7"0(2(ML
+M)!!,BV0D&$R+;"0@3(MT)"A,BWPD,$B#Q#C#9F9FD&9F9I!F9F:02(/L"$B+
+M1@B`9@'[@&`!^_Y(>8"(D@````%(@WAP`'0&2(G&_U!P2(/$",-F9F:02&/.
+M2(N4SX!9``!(A=)T"TB+`DB)A,^`60``2(G0PY!(8])(BX37@%D``$B)!DB)
+MM->`60``PV9FD&9FD&9FD$%7059!54%455-(@^P(28G_O0"```"!_@!```!W
+M:&:]`$"!_@`@``!W7&:]`""!_@`0``!W4&:]`!"!_@`(``!W1&:]``B!_@`$
+M``!W.&:]``2!_@`"``!W+&:]``*!_@`!``!W(&:]``&!_H````!W%&:]@`"#
+M_D!W"X/^(1GM@^7@@\5`B=`/K\5$C;#_#P``0<'N#,=$)`0`````N`````"!
+M_0`0```/A]L```!!_\Y!@_[_#X3*````0;T`$```3(G_Z`````!(B</_1"0$
+M0;P`````1(GHN@````#W]8/X``^&CP```&9FD&9FD+H*````@?T`0```=U:R
+M"8']`"```'=,L@B!_0`0``!W0K('@?T`"```=SBR!H']``0``'<NL@6!_0`"
+M``!W)+($@?T``0``=QJR`X']@````'<0L@*#_4!W"8/](`^7P`^VT$B)WDR)
+M_^B"_O__B>A(`<-!_\1$B>BZ`````/?U1#G@#X=W____0?_.08/^_P^%//__
+M_XM$)`1(@\0(6UU!7$%=05Y!7\-F9F:09F:055-(B?V)\[@`````.;?860``
+M<E.^`P```.@"_O__2(G&2(7`=#Z)&/_+@_O_=#60B=FZ`````$B#O=!9````
+M=!=(BY7060``2(L"2(F%T%D``/^-V%D``$B)5,X(_\N#^_]US$B)\%M=PV9F
+MD&9FD(L._\F#^?]T+69F9I!F9I!(8\%(BU3&"$B+A]!9``!(B0)(B9?060``
+M_X?860``_\F#^?]UVKH#````Z)#]___SPV9F9I!F9F:09F:09F:055-(@^P(
+M2(G]BQ__RX/[_W0>2&/#2(M\Q0BZ`!```+X`````Z`````#_RX/[_W7B2(/$
+M"%M=PV9FD&9FD&9FD(L/_\F#^?]T+DACP4B+1,<(N@````!(@S@`=`:X````
+M`,/_PDB#P`B!^O\!``!VYO_)@_G_==*X`0```,-F9I!(@^P(NA````"^````
+M`.@`````2(/$",-F9F:09F9FD$B#?A@`=`Q(BU882(M&$$B)0A!(BU802(M&
+M&$B)`DC'1A``````N@````#HL/S___/#9F9FD&9F9I!F9I!F9I!(C4]@2(M'
+M8$B)1AA(A<!T#$B+5V!(C4882(E"$$B)3A!(B3'#9F9FD&9F9I!F9I!(@^P@
+M2(D<)$B);"0(3(ED)!!,B6PD&$F)]4F)U$B)S4B+7F!(A=MT(F9F9I!F9I!(
+M.2MS#4PY8PAV![@`````ZRM(BUL82(7;=>6^`````.C[^___2(G#3(D@2(EH
+M"$B)QDR)[^AF____2(G82(L<)$B+;"0(3(MD)!!,BVPD&$B#Q"##9F9FD&9F
+M9I!F9I`/MD=-#[;02(T4DDB-5-=0_\"(1TU(B3K&0@D`QD(*`,9""P!(B=##
+M9F9FD&9F9I!!5%5328G\2(GSO0`````/ME9*C4+]T?@!PH/Z`'XED$ACQ4B+
+MM,.@"@``3(GGZ*W]____Q0^V4TJ-0OW1^`'".>I_W+H'````2(G>3(GGZ%S[
+M__];74%<PV9F9I!F9I!!5T%6055!5%532(/L&$F)_4B)5"00B?-!B<Z+M]A9
+M``#!Y@,/M\D/MM.-0OW1^`'"#Z_*.<X/@]H```!$#[;[1(E\)`Q!B<Q)C850
+M60``23F%4%D```^$V@```$F+K5!9``!(BU4(2(M%`$B)4`A(B0)(B6T`2(EM
+M"$B#?2``="9(@WTH`'0,2(M5*$B+12!(B4(@2(M5($B+12A(B0)(QT4@````
+M`&9$.75(=4HX74IU14B);0!(B6T(2(M$)!!(B45`0;P`````08/_`'X@BUPD
+M#$EC_$C!YP1(C;PO<`L``.AH_?__0?_$1#GC?^1(B>CI(@$``$B)[DR)[^B=
+M_O__08N%V%D``,'@`T0YX`^",O___[X'````3(GOZ`WZ__](B<6X`````$B%
+M[0^$Y@```.L*N`````#IV@```$B);0!(B6T(2(M$)!!(B45`9D2)=4B(74I(
+MQT4P`````$&\``````^VTXU"_='X`<*#^@`/CIT```!%B?=F0<'O`P^VTXU"
+M_='X1(TT`F:026/<00^W]TR)[^AQ^___2(F$W:`*``!(A<!U/$'_S$&#_/]T
+M'$ECQ$B+M,6@"@``3(GOZ+C[__]!_\Q!@_S_=>2Z!P```$B)[DR)[^AO^?__
+MN`````#K,DECQ$C!X`1(C1PH2(V[\`H``.A1_/__2(V[<`L``.A%_/__0?_$
+M13GF#X]Y____2(GH2(/$&%M=05Q!74%>05_#9F9FD&9FD$%455-(B?5(B=.)
+MS@^V37J)T-/H)?\!``!(BU583(TDPDV+!"1-A<!T+V9FD&9FD$DY6$!U&DF+
+M4`A)BP!(B5`(2(D"38D`38E`"$R)P.MS38M`*$V%P'77#[?.#[9U>$B)VNAH
+M_?__28G`2(7`=%!(B6@X28L$)$F)0"A(A<!T#$F+%"1)C4`H2(E"($V)8"!-
+MB00D#[9U>$&(<$H/MDUZ2(G82-/H2`^V57A(B=&Z`````$CW\4`HUO_.08AP
+M2TR)P%M=05S#9F9FD&9F9I!!5T%6055!5%532('L:`$``$B)?"1(2(ET)$!(
+MB50D.`^V5B")T-#H@_`!B<&#X0&)3"0T2(-^*`!T!?;"`70<3(U\)%"Y`0``
+M`$R)^DB+="1`2(M\)$C_5C#K"4B+7"1`3(M[*,=$)`P```$`9D&#?P@`=`E!
+M#[='"(E$)`Q!O0````#'1"0P`````$B+5"1`@+J```````^$=P$``$AC1"0P
+M2(M,)$!(BT3!6$B)1"0@#[:88`P``(A<)"\XF&$,```/@C(!```/MD0D+TB+
+M5"0@.$)+#X0%`0``#[;`2(G1#[>4@D`,```/MX2!0@P``&:)1"00B=!FP>@#
+M#[?`B40D"$&)UD&#Y@=!#[?&B<;!Y@FX"````&9$*?!!B<9F1#MT)!"+7"00
+M1`]'\T$/M\:)Q<'E"4@/MD0D+TB+5"0@2(N,PJ`*``"+5"0(B?!)B<1,`V31
+M"$0[;"0,=2-)@\<0QT0D#````0!F08-_"`!T"4$/MT\(B4PD#$&]`````(M<
+M)`Q$*>LYZP]'W8-\)#0`=!)$B>Y)`S>)VDR)Y^@`````ZQ!$B>])`S^)VDR)
+MYN@`````00'=*=UT!XG820'$ZY-F1"ET)!!T%/]$)`B^`````$&^"````.D_
+M_____D0D+P^V7"0O2(M$)"`XF&$,```/@\[^____1"0P2(M4)$`/MH*`````
+M.T0D,`^/B?[__[H`````2(MT)$!(BWPD2/]4)#A(@<1H`0``6UU!7$%=05Y!
+M7\-F9I!F9I!54TB#[`A(B?V%TG0$QD8A"T&[`````("^@``````/A,$```!(
+MC9U@60``2(V]4%D``$ECPTR+1,982<>`,`P```````!-B<*Y`````$&`>$H`
+M=D8/ML%(P>`$3`'02(V0\`H``$&Y`````(.X\`H```!U$H-Z!`!U#(-Z"`!U
+M!H-Z#`!T!D&Y`0```$6%R9!U)?_!03A*2G>ZN`````"%P'0=2(M#"$R)0PA)
+MB1A)B4`(3(D`ZQNX`0```.OA9I!(BT<(3(E'"$F).$F)0`A,B0!!_\,/MH:`
+M````1#G8#X]-____2(GOZ%,X``!(@\0(6UW#9F9FD&9F9I!F9F:02(/L2$B)
+M7"082(EL)"!,B60D*$R);"0P3(ET)#A,B7PD0$F)_4B)\TB+KC`,``#_C]Q9
+M``!(C4802(M("$B+5A!(B4H(2(D12(E&$$B)0`A(A>T/A*<"``#V12`@#X0M
+M`0``3(ME`$@/MT9(28G&3`-V0$0/MWY(2,>&,`P```````!(C8=060``2(M0
+M"$B)<`A(B09(B58(2(DR@+YB#````758@'T2`G5)2`^V1DM(B[S&H`H``.@3
+M]___A<!U,\:#8@P```Q!@(PDD@````I!#[>$))````#WV$B82"-#0$DYA"28
+M````=@A)B80DF````("[8@P```%T&@^V@V(,``"(12%(B>Y,B>_H+C<``.F)
+M`P``20^WA"20````2(M5"$C_PD@/K\),.?!V+4$/M\],B?),B>9,B>_HKOK_
+M_TB)J#`,``!(B4582(G&3(GOZ$@0``#I0P,``$D/MX0DD````$D#A"28````
+M3#GP=0A-B;0DF````$B)[DR)[^BZ-@``Z14#``!F9I!FD`^VA8$```#_P(B%
+M@0```#J%@`````^%E0(``/9%(`0/A-P```!!O@````"`O8``````#X:Y````
+M2(V'4%D``$B)1"002(V78%D``$B)5"0(1(GR#[;"2(M<Q5A(QX,P#```````
+M`("[8@P```%U44F)WT&\`````(![2@!V(F:01(G@#[;X2,'G!$J-O#_P"@``
+MZ.GU__]!_\1%.&=*=^!(BU0D$$B+0@A(B5H(2(D32(E#"$B)&.LH9F9FD&9F
+MD`^V@V(,``"(12%(BU0D"$B+0@A(B5H(2(D32(E#"$B)&$'_QD0XM8`````/
+MAU____](B>Y,B>_HM#4``.FO`0``0;X`````@+V``````'8J1(GR#[;"2(M<
+MQ5B`NV(,```!=`H/MH-B#```B$4A0?_&1#BU@````'?6@'TA`'472,?"````
+M`$B)[DR)[^C>^?__Z5D!``"Z`````$B)[DR)[^@)_/__Z40!``!F9F:0@+YB
+M#````0^%@P```$F)]+T`````@'Y*`'8D9F:09I")Z`^V^$C!YP1*C;PG\`H`
+M`.C:]/___\5!.&PD2G?A2(.[.`P```!T*4B+@S@,``!(B8,P#```2,>#.`P`
+M``````!(B=Y,B>_H,PX``.G.````28V%4%D``$B+4`A(B5@(2(D#2(E3"$B)
+M&NM:9F:02(V'8%D``$B+4`A(B7`(2(D&2(E6"$B),DB#OC@,````=#1(BX8X
+M#```#[:68@P``(A0(4B+EC@,``!(Q\8`````Z`````!(QX,X#````````.FY
+M````2(MS,$R)[^@]]/__2,=#,`````!(BT,X2(-X,`!T%4B-4#!(Q\8`````
+M3(GOZ`````#K)4F#O>A9````=!M)C97H60``2,?&`````$R)[^@`````9F:0
+M9I!!@[W@60```'16@+MB#````75-2(M[.$B#?T@`=##H#3(``(7`=2=(BT,X
+M2(M04$B+<$A,B>_H`````$B+0SA(QT!(`````$'_C>!9``!!@[W@60```'0(
+M3(GOZ`````!(BUPD&$B+;"0@3(MD)"A,BVPD,$R+="0X3(M\)$!(@\1(PV9F
+M9I!F9I!F9I!F9I!(@^P(2(L6@'I.`'03QD).`$C'Q@````#H`````&9FD$B#
+MQ`C#9F9FD&9F9I!F9I!!5T%6055!5%532(/L"$F)U$&)STB+1D!(BQ!(#[9`
+M$4R+M,*@"@``BT8(*T)`P>`)#[=N$,'E"8G!@>'_#P``08G%0<'M#+L`$```
+M*<LYZP]'W46%_W0,26/5B<A)`T36".L/26/%B<])`WS&".@`````28D$)&9!
+MB5PD"#GK=!AF0<=$)`H``$F#Q!!!_\4IW;D`````ZZYF0<=$)`H`@+@!````
+M2(/$"%M=05Q!74%>05_#9F9FD&9F9I!F9I`/MD<2`D<3B$<22(U7%+X!````
+MZP^`?Q(`>0:X`````,/^1Q(/MD\2@_D?=Q.)\-/@A4<4#Y7`#[;`ZU1F9F:0
+M@_D_=Q*#Z2")\-/@A4($#Y7`#[;`ZSF#^5]W%(/I0(GPT^"%0@@/E<`/ML#K
+M(F:0N`````"#^7]W%H/I8(GPT^"%0@P/E<`/ML!F9I!F9I"%P'2!QD<3`4R-
+M1Q0/ME<2O@$```#K`_Y'$P^V1Q.-#`*#^1]W#XGPT^"%1Q0/E<`/ML#K3H/Y
+M/W<6@^D@B?#3X$&%0`0/E<`/ML#K-F9FD(/Y7W<3@^E`B?#3X$&%0`@/E<`/
+MML#K&[@`````@_E_=Q&#Z6")\-/@085`#`^5P`^VP(7`=9"P`<-F9I!F9I!(
+M@^P82(D<)$B);"0(3(ED)!!(B?U(BUY`3(LC#[9&(8A#"N@`````@'L*`75R
+M2(G?Z*K^__^%P'06QD,*`$B)WDB)[^C'````ZV1F9I!FD(!["`%U2@^V0Q!!
+M.$0D2Y!T/CA#$74Y#[;P2(GP2,'@!$J-M"!P"P``2(U+%(L&"T,4B0:+1@0+
+M002)1@2+1@@+00B)1@B+1@P+00R)1@R0QD,)`DB)WDB)[^A!_?__2(L<)$B+
+M;"0(3(MD)!!(@\08PV9F9I!F9F:09F:09F:02(/L&$B)7"0(3(ED)!!)B?Q(
+MB?/&1A,`QD82`$B)]^CL_?__2(G>3(GGZ!$```!(BUPD"$R+9"002(/$&,-F
+MD$B#["A(B5PD"$B);"003(ED)!A,B6PD($F)_4B)\TB++DB+53A(#[9&$$R+
+MI,*@````Z`````!(B<9,B2"`>P@!=0V`2"`"QD`2(.L+9F:0@$@@!,9`$C!(
+M#[9#$D@#14!(B48(9@^V0Q-FB4802,=&,`````!(B5Y`2,=&.`````!,B>]!
+M_U0D:$B+7"0(2(ML)!!,BV0D&$R+;"0@2(/$*,-F9F:09F:09F:09F:00515
+M4T&Y`````$&[`````$B)_4&\`0```.L808/[?W8/1(G(Z5X!``!F9F:09F:0
+M0?_#1(G:08/['W<21(G@1(G9T^"%!P^5P`^VP.M208/[/W<3C4K@1(GCT^.%
+M700/E<`/ML#K.4&#^U]W$XU*P$2)X-/@A44(#Y7`#[;`ZR"X`````$&#^W]W
+M%8U*H$2)X]/CA5T,#Y7`#[;`9F9FD(7`#X1S____0;@!````2(GZ0;H!````
+MZPIF9F:09F:00?_`0XT,`X/Y'W<41(G0T^"%`@^5P`^VP.M59F:09I"#^3]W
+M$X/I($2)T-/@A4($#Y7`#[;`ZSB#^5]W$X/I0$2)T]/CA5H(#Y7`#[;`ZR"X
+M`````(/Y?W<6@^E@1(G0T^"%0@P/E<`/ML!F9I!FD(7`=8Q$B=E%`<-!N@@`
+M``!$B<J)R,'H`X@$EHG(@^`'B$26`42)TRC#B%R6`@^VPT0YP',*02G``<%!
+M_\'KT42)R$2(1(8"0?_!08/[?P^&K_[__T2)R%M=05S#9F9FD$B![%@!``!(
+MB9PD*`$``$B)K"0P`0``3(FD)#@!``!,B:PD0`$``$R)M"1(`0``3(F\)%`!
+M``!(B7PD"$F)]4B++DB+7A"`>QD`#X3A````2(-[$`!U"4B)<Q#I8@$``,9#
+M&0!(C70D($B)W^C__?__08G&2`^V0QA,BZ3%H`H``$B+0Q!(#[9`&$R+O,6@
+M"@``20^V11A(BZS%H`H``+L`````1#GS#XT#`0``9F:02&/3#[9$E"&)P<'A
+M"0^V1)0B08G`0<'@"40/MDR4($2)R`^VT(G(2(G'20-\UPA(B7PD$$B)QD@#
+M=-4(2(ET)!A)`T34"$@YQW4*1(G"Z`````#K($B+5"082(MT)!!$B<A$#[;(
+MB<]+`WS,"$2)P>@`````_\-$.?-\A.E_````2(UT)"!(B=_H,OW__T&)QD@/
+MMD,83(NDQ:`*``!)#[9%&$R+O,6@"@``NP````!$.?-]2TACTP^V1)0AB<'!
+MX0D/MD24(D&)P$'!X`E(#[9$E"")SDB)]TD#?,0(2(E\)!!)`W3'"$B)="08
+M1(G"Z`````#_PT0Y\WRY9F9FD+H`````3(GN2(M\)`CHC@(``$B+G"0H`0``
+M2(NL)#`!``!,BZ0D.`$``$R+K"1``0``3(NT)$@!``!,B[PD4`$``$B!Q%@!
+M``##9F:09F:02('L2`$``$B)G"08`0``2(FL)"`!``!,B:0D*`$``$R)K"0P
+M`0``3(FT)#@!``!,B;PD0`$``$F)_TB)]4B+'DB-="002(U]$.@B_/__08G$
+M2`^V12!,B[3#H`H``$@/MD4A3(NLPZ`*``"[`````$0YXWT]2&/##[9TA!'!
+MY@D/ME2$$L'B"4@/MD2$$(GV2(GW20-\Q@A(B3PD20-TQ0A(B70D".@`````
+M_\-$.>-\P[H`````2(GN3(G_Z(X!``!(BYPD&`$``$B+K"0@`0``3(ND)"@!
+M``!,BZPD,`$``$R+M"0X`0``3(N\)$`!``!(@<1(`0``PV9FD&9FD$%7059!
+M54%455-(@>R(`0``28G_2(GU3(LF2(M>$$B-M"2`````2(G?Z$+[__]!B<9(
+M#[9#&$V+K,2@"@``NP````"`?2``=B,/MDT@9F:0#[;#2`^V5"@828N4U*`*
+M``!(B53$0/_#.-EWY$&\`````$4Y]`^-K0```&9FD&:026/$#[:4A($```!!
+MB=)!P>()#[:4A((```!!B=-!P>,)#[:,A(````"[`````(!](`!V*$0/MLE%
+MB=`/MG4@D`^VPTB+5,1`2HM\R@A)C10X2(D4Q/_#0#C>=^0/MM%$B=!(B<=)
+M`WS5"$@Y/"1U$$B+="0(1(G:Z`````#K'9!(BU0D"$B+-"0/ML%$B==)`WS%
+M"$2)V>@`````0?_$13GT#XQ8____N@````!(B>Y,B?_H&````$B!Q(@!``!;
+M74%<05U!7D%?PV9FD&9FD$B#[!A(B1PD2(EL)`A,B60D$$F)_$B)\XG5QD8)
+M`H/Z`1G`@^#U@\`,B$8*Z!_V__^`>P@#=2](BT,02(MP$$B%]G0B2,=`$```
+M``#&1@D"@_T!&<"#X/6#P`R(1@I,B>?HZO7__TB+'"1(BVPD"$R+9"002(/$
+M&,-F9F:09F:0#[96"X72=$:)T$@/MH1'L`D``$B-!(!(C43'4(!X"0)T!K@`
+M````PX!X"@%T$L9&"0(/MD`*B$8*N`````##D(G0#[:41[$)``"%TG6ZN`$`
+M``##9F9FD&9F9I!F9F:005=!5D%505154TB#[`A)B?Y(B?7&1DX`@+YC#```
+M``^$BP```$&\`````$B-75"`?4T`=$Z`>PD!=3B`>P@#=2E(BT,0@'@9`'0?
+M2(-X$`!T&$C'0!``````QD,)`@^VA6(,``"(0PKK"<9%3@'IMP$``$'_Q$B#
+MPR@/MD5-1#G@?[)(B>Y,B??H$O#__^F7`0``#[9#"HB%8@P``,:%8PP```'I
+M?/___V9F9I!F9I!!OP````!!O0$```!!O`````!(C5U0@'U-``^$-`$``(![
+M"0!U>DB)WDB)[^B__O__A<!T6\9#"0%,B?=(B=X/MD,(@_@&=TB)P/\DQ0``
+M``#H2O?__^LX9F9FD&9F9I#HN_G__^LIZ+3[__]F9F:0ZQ[HJ?S__^L7QD5/
+M`<9&"0+&1@H!Z#;T__]F9I!F9I"`>PD"=1!!_\=F9F:09F:0@'L)`G0+0;T`
+M````Z8\```"`>PH!#X2%````@'U/`)`/A!W___](BT4X2`^V4Q!(BX30H```
+M`/9``01T7T&]`````,9#"0'&0PH`3(GW2(G>#[9#"(/X!G=&B<#_),4`````
+MZ)CV___K-F9FD&9FD.@+^?__ZRGH!/O__V9F9I#K'NCY^___ZQ?&14\!QD8)
+M`L9&"@'HAO/__^L$QD,*`4'_Q$B#PR@/MD5-1#G@#X_,_O__187_#X6C_O__
+M187M=!3&A6(,```!2(GN3(GWZ'SN___K!,9%3@%(@\0(6UU!7$%=05Y!7\-F
+M9F:09F:005=!5D%505154TB!['@!``!(B7PD2$B)]4B+7CA,BZ8P#```0;T`
+M````387D=2A(#[=.2$B+5D!(`=%(B=[HO>;__TB)13"X_____TB#?3``#X2N
+M(@``2(M$)$C_@-Q9``!(C4402(M4)$A(@<)P60``2(M*"$B)0@A(B5402(E(
+M"$B)`6:#NY0`````=$/V@Y(````!=0U(BU5`2#F3F````'<G0;X`````#[>#
+ME````(G"J`%U#D'_QHG01(GQT_BH`73R1(AU3.LIQD5,".LC]H.2`````G06
+M2(M]0$@YNY@```!W"0^V14N(14SK!,9%3`C&14T`QH6P"0```,9%3P#&A6(,
+M````QH5C#````$C'A3@,````````387D#X1?#P``0?9$)"`@#X2.!P``00^V
+M1"02@_@"#X1M!```@_@"?P^#^`$/A`0!``"0Z9XA``"#^`,/A94A``!(C5PD
+M8$B)W^@%Y?__28G:OP`````/MW5(NP````!!N2````!!NP$```"#_Q]W/P^V
+MPTF-%()$B<DI^3GQ#T?.@_D@=0C'`O_____K#42)V-/@_\B)^=/@"0)$B<@I
+M^#GP<Q6-=#[@OP````#K`X/O(/_#@/L#=K)(B[V@"@``Z`[D__]!OP````"`
+M?4H`#X8!(0``1(G[#[;#2,'@!$B-'"A(C;OP"@``Z&3D__](C;MP"P``Z%CD
+M__](B>_H<.7__\9`"`)$B'@0QD`1`$B+5"1@2(E0%$B+5"1H2(E0'$'_QT0X
+M?4IWJ^FG(```N`$```#K54B)[[D`````@'U*`)!V00^VP4C!X`1(`?A(C9#P
+M"@``O@````"#N/`*````=1*#>@0`=0R#>@@`=0:#>@P`=`6^`0```(7V=;#_
+MP3A/2G>_N`````"%P'073(FE.`P``$C'A3`,````````Z<(-``!(C5PD8$B)
+MW^B@X___28G:OP`````/MW5(NP````!!N2````!!NP$```"#_Q]W/P^VPTF-
+M%()$B<DI^3GQ#T?.@_D@=0C'`O_____K#42)V-/@_\B)^=/@"0)$B<@I^#GP
+M<Q6-=#[@OP````#K`X/O(/_#@/L#=K*`?4P(=0</MD5+B$5,#[9=3(B="`P`
+M`,:%"0P```%(BT0D8$B)A?`+``!(BT0D:$B)A?@+``!(QX4`#````````$B)
+M[^@,Y/__QD`(`HA8$(A8$4B+5"1@2(E0%$B+5"1H2(E0'$F)Q(!]2@(/AP`!
+M``!(C9WP"P``2(GOZ-3C___&0`@%2(E8$,9`(`!)B<5)BPPD_H&P"0``#[:Q
+ML`D``(GW2"G(2(/H4$C!^`-(NLW,S,S,S,S,2`^OPHB$>;`)``!!#[9$)`N(
+MA'FQ"0``08AT)`M!OP````"`?4H`#X;3'@``2(G31#A]3'1R2(GOZ%_C___&
+M0`@!1(AX$$2(>!%(BU0D8$B)4!1(BU0D:$B)4!Q!#[95(`^VRD:(?"D8_\)!
+MB%4@28M5`/Z"L`D```^VLK`)``")\4@IT$B#Z%!(P?@#2`^OPXB$2K`)``!!
+M#[9%"XB$2K$)``!!B'4+0?_'1#A]2@^'>____^E&'@``0;\`````@'U*``^&
+M-AX``$B-G?`+``!)OLW,S,S,S,S,1#A]3`^$O0```$B)[^BPXO__QD`(`42(
+M>!!$B'@12(M4)&!(B5`42(M4)&A(B5`<2(E$)$!(B>_HA>+__\9`"`-(B5@0
+M1(AX&$F)Q4B+$/Z"L`D```^VLK`)``")\4B+1"1`2"G02(/H4$C!^`-)#Z_&
+MB(1*L`D``$$/MD4+B(1*L0D``$&(=0M)BQ0D_H*P"0``#[:RL`D``(GQ3(GH
+M2"G02(/H4$C!^`-)#Z_&B(1*L`D``$$/MD0D"XB$2K$)``!!B'0D"T'_QT0X
+M?4H/ARS____I3!T``+@!````ZUE(B>^Y`````(!]2@!V1F9F9I`/ML%(P>`$
+M2`'X2(V0\`H``+X`````@[CP"@```'43@WH$`'4-@WH(`'4'@WH,`)!T!;X!
+M````A?9UK/_!.$]*=[ZX`````(7`=!=,B:4X#```2,>%,`P```````#I8PH`
+M`$B-7"1@2(G?Z$'@__])B=J_``````^W=4B[`````$&Y(````$&[`0```&9F
+M9I"#_Q]W/P^VPTF-%()$B<DI^3GQ#T?.@_D@=0C'`O_____K#42)V-/@_\B)
+M^=/@"0)$B<@I^#GP<Q6-=#[@OP````#K`X/O(/_#@/L#=K(/MD5+B(4(#```
+MQH4)#````$B+1"1@2(F%\`L``$B+1"1H2(F%^`L``$C'A0`,````````@'U*
+M`@^'N````$B-G?`+``!(B>_HI>#__\9`"`5(B5@0QD`@`$F)Q4&_`````(!]
+M2@`/AND;``!(N\W,S,S,S,S,2(GOZ'3@___&0`@!1(AX$$2(>!%(BU0D8$B)
+M4!1(BU0D:$B)4!Q!#[95(`^VRD:(?"D8_\)!B%4@28M5`/Z"L`D```^VLK`)
+M``")\4@IT$B#Z%!(P?@#2`^OPXB$2K`)``!!#[9%"XB$2K$)``!!B'4+0?_'
+M1#A]2G>%Z5\;```/MEU+2(GOZ/#?___&0`@!B%@0B%@12(M4)&!(B5`42(M4
+M)&A(B5`<2(E$)#A!OP````"`?4H`#X8B&P``2(V=\`L``$F\S<S,S,S,S,Q$
+M.'U+#X2]````2(GOZ)S?___&0`@!1(AX$$2(>!%(BU0D8$B)4!1(BU0D:$B)
+M4!Q(B40D0$B)[^AQW___QD`(`TB)6!!$B'@828G%2(L0_H*P"0``#[:RL`D`
+M`(GQ2(M$)$!(*=!(@^A02,'X`TD/K\2(A$JP"0``00^V10N(A$JQ"0``08AU
+M"TF+50#^@K`)```/MK*P"0``B?%(BT0D.$@IT$B#Z%!(P?@#20^OQ(B$2K`)
+M``!!#[9%"XB$2K$)``!!B'4+0?_'1#A]2@^'+/___^DX&@``9F:09I!-A>0/
+MA+P'``!!]D0D(`*0#X2O!P``1`^VO6`,``!$.+UA#```#X)5`0``9F:09I!$
+M.'U+#X0V`0``1(GX#[;82(G82,'@!$R-I`3P````3(GGZ%G=__\/MY2=0`P`
+M``^WO)U"#```NP````!!N2````!!N@$```"#^A]W1`^VPTF--(1$B<DIT3GY
+M#T?/@_D@=0W'!O_____K$F9FD&:01(G0T^#_R(G1T^`)!D2)R"G0.?AS%8U\
+M%^"Z`````.L#@^H@_\.`^P-VK42)^P^VPTB)QTC!YP1(C00O2(V0<`L``(N(
+M<`L``/?1B4PD8(M"!/?0B40D9(M""/?0B40D:(M"#/?0B40D;$B-=#QP2(V4
+M//`````C"HD.BT($(T0D9(E&!(M"""-$)&B)1@B+0@PC1"1LB48,1#A]3'4Q
+MN@````"#/@!U$H-^!`!U#(-^"`!U!H-^#`!T!;H!````A=*X`0```$0/1>AF
+M9I!FD$'_QT0XO6$,```/@[#^__]%A.T/A)H%``!(C7PD8.@=W/__0;\`````
+M@'U*`'99D$0X?4QT240X?4MT0T2)^`^VT$C!X@1(`>I(C8KP"@``BT0D8`N"
+M\`H``(E$)&"+1"1D"T$$B40D9(M$)&@+00B)1"1HBT0D;`M!#(E$)&Q!_\=$
+M.'U*=ZA,#[9%3$G!X`1*C70$<(M\)&`C/HE\)%"+3"1D(TX$B4PD5(M4)&@C
+M5@B)5"18BT0D;"-&#(E$)%SWUXE\)&#WT8E,)&3WTHE4)&CWT(E$)&PC/HE\
+M)&`C3@2)3"1D(U8(B50D:"-&#(E$)&Q(B>_H9-S__\9`"`!(B40D*+@`````
+M@WPD8`!U%8-\)&0`=0Z#?"1H`'4'@WPD;`!T!;@!````A<`/A*8!```/MD5,
+MB(4(#```2(M$)&!(B87P"P``2(M$)&A(B87X"P``QH4)#````4C'A0`,````
+M````0;\`````@'U*``^&`P(``$R-I?`+``!$.'U,#X0_`0``1#B]8`P``'</
+M1#B]80P``'(&1#A]2W4M2(GOZ+C;___&0`@!1(AX$$2(>!%(BU0D8$B)4!1(
+MBU0D:$B)4!Q(B40D0.M21(GZ#[;"2,'@!$B-7`1PBT0D8`D#BT0D9`E#!(M$
+M)&@)0PB+1"1L"4,,2(GOZ&';___&0`@!1(AX$$2(>!%(BQ-(B5`42(M3"$B)
+M4!Q(B40D0$B)[^@YV___QD`(`TR)8!!$B'@828G%2(L0_H*P"0``#[:RL`D`
+M`(GQ2(M$)$!(*=!(@^A02,'X`TB_S<S,S,S,S,Q(#Z_'B(1*L`D``$$/MD4+
+MB(1*L0D``$&(=0M$.'U+=41(BTPD*$B+$?Z"L`D```^VLK`)``")\4R)Z$@I
+MT$B#Z%!(P?@#2`^OQXB$2K`)``!(BUPD*`^V0PN(A$JQ"0``0(AS"T'_QT0X
+M?4H/AZK^___IH0```$&_`````(!]2@`/AI$```!$.+U@#```=WM$.+UA#```
+M<G)$.'U+=&Q$.'U,=&9$B?H/ML)(P>`$2(U$!'"Z`````(,X`'42@W@$`'4,
+M@W@(`'4&@W@,`'0%N@$```"%TG0R1(GY#[;!2,'@!$B-7`1P2(GOZ`[:___&
+M0`@!1(AX$$2(>!%(BQ-(B5`42(M3"$B)4!Q!_\=$.'U*#X=O____N`````"#
+M?"10`'45@WPD5`!U#H-\)%@`=0>#?"1<`'0%N`$```"%P`^$&Q4``$0/MF5,
+M1(BE*`P``$B+1"102(F%$`P``$B+1"182(F%&`P``,:%*0P```!(QX4@#```
+M``````^V74M(B>_H=MG__\9`"`&(6!!$B&`12(M4)%!(B5`42(M4)%A(B5`<
+M2(E$)#A(QT0D,`````!!OP````"`?4H`#X:>%```3(VE$`P``$F^S<S,S,S,
+MS,Q$.'U,#X1A`0``1#A]2P^$5P$```^V74M(B>_H"MG__\9`"`%$B'@0B%@1
+M2(M4)%!(B5`42(M4)%A(B5`<2(E$)$!(BQ#^@K`)```/MK*P"0``B?%(BT0D
+M*$@IT$B#Z%!(P?@#2+_-S,S,S,S,S$@/K\>(A$JP"0``2(M<)$`/MD,+B(1*
+ML0D``$"(<PM(@WPD,`!T/$B+$_Z"L`D```^VLK`)``")\4B+1"0P2"G02(/H
+M4$C!^`-(#Z_'B(1*L`D```^V0PN(A$JQ"0``0(AS"P^V74M(B>_H3=C__\9`
+M"`-,B6`0B%@828G%2(E$)#!(BQ#^@K`)```/MK*P"0``B?%(BT0D0$@IT$B#
+MZ%!(P?@#20^OQHB$2K`)``!!#[9%"XB$2K$)``!!B'4+28M5`/Z"L`D```^V
+MLK`)``")\4B+1"0X2"G02(/H4$C!^`-)#Z_&B(1*L`D``$$/MD4+B(1*L0D`
+M`$&(=0M!_\=$.'U*#X>(_O__Z1`3``!$#[:]8`P``$0XO6$,```/@OL2``!$
+M.'U+=')$B?H/ML)(P>`$2(U$!'"Z`````(,X`'49@W@$`'43@W@(`'4-@W@,
+M`'0,9F9FD&9FD+H!````A=)T-T2)^0^VP4C!X`1(C5P$<$B)[^A`U___QD`(
+M`42(>!!$B'@12(L32(E0%$B+4PA(B5`<9F:09I!!_\=$.+UA#```#X-X____
+MZ6X2``"`?4P'#X;3"0``2(U\)&#HU]7__\9$)"<`0;\`````@'U*``^&H0``
+M`&:01#A]2P^$B````$2)^P^VPTC!X`1(`>A(C9#P"@``N0````"#N/`*````
+M=1:#>@0`=1"#>@@`=0J#>@P`9F9FD'0%N0$```"%R71'1(GX#[;02,'B!$@!
+MZDB-BO`*``"+1"1@"X+P"@``B40D8(M$)&0+002)1"1DBT0D:`M!"(E$)&B+
+M1"1L"T$,B40D;/Y$)"=!_\=$.'U*#X=A____#[94)"</MD5*@^@#T?@YP@^.
+M5`4```^V74N(G0@,``!(BT0D8$B)A?`+``!(BT0D:$B)A?@+``#&A0D,```!
+M2,>%``P```````!(B>_H[]7__\9`"`*(6!"(6!%(BU0D8$B)4!1(BU0D:$B)
+M4!Q)B<1(B>_HR-7__\9`"`9(B40D*$F+%"3^@K`)```/MK*P"0``B?%(BT0D
+M*$@IT$B#Z%!(P?@#2;[-S,S,S,S,S$D/K\:(A$JP"0``00^V1"0+B(1*L0D`
+M`$&(="0+@'U*`@^'20(``$B-G?`+``!(B>_H7-7__\9`"`5(B5@0QD`@`$F)
+MQ4B+3"0H2(L1_H*P"0``#[:RL`D``(GQ2"G02(/H4$C!^`-)#Z_&B(1*L`D`
+M`$B+7"0H#[9#"XB$2K$)``!`B',+0;\`````@'U*``^&7Q```$0X?4L/A+X!
+M``!!#[9%(`^VT$:(?"H8_\!!B$4@1(GX#[;P2,'F!$B-!"Y(C8AP"P``BY!P
+M"P``]]*)5"10BT$$]]")1"14BT$(]]")1"18BT$,]]")1"1<2(U,-'`C5"1@
+MB1&+1"1D(T0D5(E!!(M$)&@C1"18B4$(BT0D;"-$)%R)00RX`````(,Y`'42
+M@WD$`'4,@WD(`'4&@WD,`'0%N`$```"%P'1K1(GZ#[;"2,'@!$B-7`1P2(GO
+MZ#O4___&0`@!1(AX$$2(>!%(BQ-(B5`42(M3"$B)4!Q)BU4`_H*P"0``#[:R
+ML`D``(GQ2"G02(/H4$C!^`-)#Z_&B(1*L`D``$$/MD4+B(1*L0D``$&(=0M$
+MB?D/ML%(P>`$2(V$*/`*``"Z`````(,X`'42@W@$`'4,@W@(`'4&@W@,`'0%
+MN@$```"%TF:0='I$B?@/MMA(B=A(P>`$2(V<*/`*``!(B>_HD=/__\9`"`)$
+MB'@01(AX$4B+$TB)4!1(BU,(2(E0'$F)Q$B+$/Z"L`D```^VLK`)``")\4B+
+M1"0H2"G02(/H4$C!^`-)#Z_&B(1*L`D``$$/MD0D"XB$2K$)``!!B'0D"T'_
+MQT0X?4H/ARO^___IA0X``$&_`````(!]2@`/AG4.``!(C97P"P``2(E4)!A)
+MOLW,S,S,S,S,1#A]2P^$"0(``$2)^0^V\4C!Y@1(C00N2(V(<`L``(N0<`L`
+M`/?2B50D4(M!!/?0B40D5(M!"/?0B40D6(M!#/?0B40D7$B-7#1P(U0D8(D3
+MBT0D9"-$)%2)0P2+1"1H(T0D6(E#"(M$)&PC1"1<B4,,2(GOZ(+2___&0`@#
+M2(M\)!A(B7@01(AX&$F)Q4B+1"0H2(L0_H*P"0``#[:RL`D``(GQ3(GH2"G0
+M2(/H4$C!^`-)#Z_&B(1*L`D``$B+?"0H#[9'"XB$2K$)``!`B'<+N`````"#
 M.P!U$H-[!`!U#(-["`!U!H-[#`!T!;@!````A<!T:T2)^@^VPDC!X`1(C5P$
-M<$B)[^BLR/__QD`(`42(>!!$B'@12(L32(E0%$B+4PA(B5`<28M5`/Z"L`D`
-M``^VLK`)``")\4@IT$B#Z%!(P?@#20^OQ(B$2K`)``!!#[9%"XB$2K$)``!!
-MB'4+0?_'1#A]1@^'2_[__[@`````@WPD4`!U%8-\)%0`=0Z#?"18`'4'@WPD
-M7`!T!;@!````A<`/A#`#```/MEU'2(GOZ!'(___&0`@!B%@0B%@12(M4)%!(
-MB5`42(M4)%A(B5`<2(E$)#A(QT0D,`````!!OP````"`?48`#X;J`@``9F9F
-MD&9FD$0X?4@/A,P"``!$.'U'#X3"`@``1(GY#[;!2,'@!$B-=`1P2`'H2(V(
-M\`H``(M4)%`CD/`*``")%HM$)%0C002)1@2+1"18(T$(B48(BT0D7"-!#(E&
-M#+@`````@SX`=1*#?@0`=0R#?@@`=0:#?@P`=`6X`0```(7`#X16`@``1`^V
-M94A$B?L/ML-(P>`$2(U<!'!(B>_H-,?__\9`"`%$B'@01(A@$4B+$TB)4!1(
-MBU,(2(E0'$B)1"1`2(-\)#``=$M(BQ#^@K`)```/MK*P"0``B?%(BT0D,$@I
-MT$B#Z%!(P?@#2+_-S,S,S,S,S$@/K\>(A$JP"0``2(M<)$`/MD,+B(1*L0D`
-M`$"(<PM$#[9E1P^V?4A`B'PD#T2)^$0/MO!)P>8$2HU<-'!(B>_HF\;__\9`
-M"`1$B&`@#[94)`^(4"%(BQ-(B5`02(M3"$B)4!A)B<5(B40D,$B+$/Z"L`D`
-M``^VLK`)``")\4B+1"0X2"G02(/H4$C!^`-(N\W,S,S,S,S,2`^OPXB$2K`)
-M``!!#[9%"XB$2K$)``!!B'4+28M5`/Z"L`D```^VLK`)``")\4B+1"1`2"G0
-M2(/H4$C!^`-(#Z_#B(1*L`D``$$/MD4+B(1*L0D``$&(=0M(BWPD*$B+%_Z"
-ML`D```^VLK`)``")\4R)Z$@IT$B#Z%!(P?@#2`^OPXB$2K`)```/MD<+B(1*
-ML0D``$"(=PL/MEU'3HUD-'!(B>_HG\7__\9`"`2(6"!$B'@A28L4)$B)4!!)
-MBU0D"$B)4!A)B<5(BQ#^@K`)```/MK*P"0``B?%(BT0D.$@IT$B#Z%!(P?@#
-M2+O-S,S,S,S,S$@/K\.(A$JP"0``00^V10N(A$JQ"0``08AU"TB+?"0H2(L7
-M_H*P"0``#[:RL`D``(GQ3(GH2"G02(/H4$C!^`-(#Z_#B(1*L`D```^V1PN(
-MA$JQ"0``0(AW"T'_QT0X?48/AQW]__](B>Y(BWPD2.C`VO__N`````!(@<1X
-M`0``6UU!7$%=05Y!7\-F9I!F9I!F9I!!5T%6055!5%532(/L&$F)_8._/%@`
-M``,/AQ(!``!(C8?`5P``2#F'P%<```^$_@```$F)QTR+M\!7``!(C8>P5P``
-M2(E$)!!,B7PD"&9FD$R)\TV+-DB+4PA)B58(3(DR2(D;2(E;"$B+0SCV0`$$
-M=7Y)B=R]`````(![1@!V'XGH#[;X2,'G!$J-O"?P"@``Z`K#____Q4$X;"1&
-M=^%)BX6P5P``2(E8"$B)`TB+1"002(E#"$F)G;!7``!(BT,X2(-X0`!T6$B+
-M4$A(BW!`3(GOZ`````!(BT,X2,=`0`````!!_XU`6```ZS-F9I!(B=Y,B>_H
-MY=O__X7`=#%)BX7`5P``2(E8"$B)`TB+1"0(2(E#"$F)G<!7``!F9I!-.?X/
-MA2?___]F9F:09F:02(/$&%M=05Q!74%>05_#D%532(/L"$B+7Q"#NSQ8````
-M=1!(C8/`5P``2#F#P%<``'0W2(VKP%<``&9FD&9FD$B)W^@`````2(G?Z```
-M``!(B=_H`````(.[/%@```!UWT@YJ\!7``!UUK@`````2(/$"%M=PV9F9I!F
-M9I!F9I!33(M/$$F+B<!7``!)C8'`5P``2#G(#X1^````28G"D$@Y>3AU;$F)
-MR+L`````@'E&`'9"#[;#2,'@!$P!P$B-D/`*``"^`````(.X\`H```!U$H-Z
-M!`!U#(-Z"`!U!H-Z#`!T!;X!````A?9U&__#03A81G>^N`````"%P'03N`$`
-M``#IN````+@!````9I#KZ4B+"4DYRG6&28N)T%<``$F-@=!7``!(.<@/A(L`
-M``!)B<%(C4'P2#EX.'5R2(G&NP````"`>$8`=DAF9F:0#[;#2,'@!$@!\$B-
-MD/`*``!!N`````"#N/`*````=1*#>@0`=0R#>@@`=0:#>@P`=`9!N`$```!%
-MA<!U%__#.%Y&=[RX`````(7`=!.X`0```.L=N`$```#K[F9FD&:02(L)23G)
-M#X5X____N`````!;PV9F9I!F9I!F9I!F9I!(B5PDX$B);"3H3(ED)/!,B6PD
-M^$B#["A(B?M)B?5)B=1(BV\02(-_0`!U">A__O__A<!U#4R)YDB)[T'_U6:0
-MZQ9,B6M`3(EC2/^%0%@``$B)[^@`````2(M<)`A(BVPD$$R+9"083(ML)"!(
-M@\0HPV9F9I!F9F:09F9FD&9FD%532(/L"$B)_4B)\X!^&0!U!,9&&0$/MD,9
-M/`$/E<(\!@^5P`^VTH70=!6`>QD#=`_V0Q@@=0G&0QD+9F:09I!(BW-(2(GO
-MZ/2___](BW,P2(G:2(GOZ`````!(BP-(@W@H`'0;2(U0*$C'Q@````!(B>_H
-M`````.M<9F:09F:02(.]2%@```!T*$B-A;!7``!(.86P5P``=!A(C95(6```
-M2,?&`````$B)[^@`````ZR2!O3A8``#_````=QA(C86P5P``2#F%L%<``'4(
-M2(GOZ`````!(@\0(6UW#9F:005=!5D%505154TB![$@!``!(B70D*$F)U(E,
-M)"1,BT8X28L`2(D$)`^V4'#_RHE4)!1!#[=`#&:)1"0B08N0@````(E4)!Q%
-M#[9P?D$/MD!_B40D$$4/M[B&````00^WF(0```!)@W@@`'0-9D$/MD`8@^`!
-M.<AU,KH`````28-X*``/A!0"``!,C6PD,(M,)"1,B>I,B<9!_U`HN@````"%
-MP`^$]`$``.L$38MH(+T`````2(M4)"A(BP(/MD`#B40D#+D`````9F9FD&9F
-MD`^WPP-$)!R)1"081#MT)`P/A04!``"%R711*<B)P\'C"71(@WPD)`!T"4B+
-M!0````#K#$B+/0````#H`````$F)!"1F0<=$)`H``('[`!````^&L0```&9!
-MQT0D"``028/$$('K`!```'6X00^WQXG#P>,)00^W50B%TK@```$`#T30B=`I
-MZ#G8<BV)Z$D#10!)B00D9D&)7"0(9D''1"0*``!)@\00`=TYU74]O0````!)
-M@\40ZS*)Z$D#10!)B00DB=!F*>AF08E$)`AF0<=$)`H``$F#Q!")T"GH*<-)
-M@\40O0````#KBD$/M\\#3"082(M4)"@/MT(,`T((.<@/A+H```#K2&9!B5PD
-M"$F#Q!#I5/___P'=ZS5!#[??P>,)00^W50B%TK@```$`#T30B=`IZ#G8=]Z)
-MT"GH*<-)@\40O0````#KV&9FD&9FD&9$*7PD(G1HNP````!!_\9$.W0D$`^4
-MP`^VP$$!QDB+%"0/MD)P1#GP=26+1"00_\B#?"00``]$1"04B40D$(7`#Y3`
-M1`^V\`^W0GP!1"0<2(L$)$0/MWA\9D0[?"0B#X9+_O__1`^W?"0BZ4#^__]F
-M0<=$)/H`@+H!````B=!(@<1(`0``6UU!7$%=05Y!7\-F9F:09F9FD&9F9I!F
-M9I!(B5PD\$B);"3X2(/L&$B)_4B+7CB`?AD!=`</MD89B$,92(GOZ``````/
-MMD-]_\"(0WTZ0WQU.(![&0!U!,9#&0$/MD,9/`$/E<(\!@^5P`^VTH70=`J`
-M>QD#=`3&0QD+2(G>2(GO_U,P9F:09F:02(M<)`A(BVPD$$B#Q!C#D$%7059!
-M54%455-(@^P82(E\)!!)B?5(BP9(B40D"/9&&`0/A-4!``!!O@````"`?G@`
-M#X18`0``26/&38M$Q5!!#[:H8`P``$$XJ&$,```/@BH!``!!.&A'#X01`0``
-MB>H/ML)(B<)(P>($3HV4`O`*``!!#[>4@$`,``!!#[>\@$(,``"[`````$&[
-M(````$&\`0```(/Z'W=&#[;#28TT@D2)V2G1.?D/1\^#^2!U#\<&_____^L4
-M9F9FD&9FD$2)X-/@_\B)T=/@"09$B=@IT#GX<Q6-?!?@N@````#K`X/J(/_#
-M@/L#=JN)ZP^VPTB)PDC!X@1.C90"<`L``$$/MY2`0`P``$$/M[R`0@P``+L`
-M````0;L@````0;P!````@_H?=S\/ML-)C32"1(G9*=$Y^0]'SX/Y('4(QP;_
-M____ZPU$B>#3X/_(B='3X`D&1(G8*=`Y^',5C7P7X+H`````ZP.#ZB#_PX#[
-M`W:R_\5!.*AA#```#X/6_O__0?_&00^V17A$.?`/CZC^__](BUPD$(.[J%<`
-M```/A&(#``!!O@````!!@'UX`'0\2(G:2('"P%<``$ECQDV+1,502<>`,`P`
-M``````!(BT((3(E""$F)$$F)0`A,B0!!_\9!#[9%>$0Y\'_.3(GN2(M\)!#H
-MZ/G__^DS`P``]D88`@^$_0(``$B+1"002`506```2(D$)$R+?"0028''<%@`
-M`$B+5"0(]D)^`P^%TP(``+H0````O@````!,B?_H`````$&^`````$&`?7@`
-M#X3_````26/&38M$Q5!,B<>Y`````$&`>$8`=DP/ML%(P>`$2`'X2(V0\`H`
-M`+X`````@[CP"@```'42@WH$`'4,@WH(`'4&@WH,`'0%O@$```"%]G0+N`$`
-M``#K$&9F9I#_P3A/1G>TN`````"%P&:0#X4^`@``00^VJ&`,``!!.*AA#```
-M<FM!.&A'=%J)Z0^VT69!@SQ7`'4D00^WA)!`#```00-`0$B+'"2)!)-!#[>$
-MD$(,``!F08D$5^LIB>@/MM!!#[>$D$`,``!F00-`0&9!`X200@P``$B+#"1F
-M*P219D&)!%?_Q4$XJ&$,``!SE4'_QD$/MD5X1#GP#X\!____28MU2$B+?"00
-MZ+^X__]!O@````!!@'UX``^$X````$B+3"002('!L%<``$ECQDV+1,50187V
-M=5Y!BT!`08F%@````$$/MH!@#```08A%?D$/MD!'08A%?TD/MH!@#```00^W
-MA(!`#```9D&)A80```!)#[:`8`P``$$/MX2`0@P``&9!B86&````0<9%?`!!
-MQD5]`.M&08M00$$[E8````!V.4B+7"0(#[=#?/?8(<)!(X6`````.<)U(4D/
-MMI!@#```00^WA88```!F00.$D$(,``!F08F%A@```$B+00A,B4$(28D(28E`
-M"$R)`$'_QD$/MD5X1#GP#X\L____2(M$)!!(@[A(6````'092(G"2('"2%@`
-M`$C'Q@````!(B<?H`````+T`````2(M4)`B`>G``#X:B````B>E$#[;A9D.#
-M/&<`=%E(BT0D"$J+G."(````2(M\)!#H`````$B)QD'^17S&0`X@0P^W!&=F
-MB48,2(L4)$*+!**)1@B`3A@"2,=&*`````!(QT8P`````$B)'DR);CA(BWPD
-M$/]38/_%2(M,)`A`.&EP=XSK+$&^`````$&`?7@`=!])8\9)BW3%4$B+?"00
-MZ!W0__]!_\9!#[9%>$0Y\'_A2(/$&%M=05Q!74%>05_#D$%7059!54%455-(
-M@^Q82(E\)$A)B?=(B50D0$B+!DB)1"0XO@`````/MU!\9H'Z@0"X@`````]#
-MT&:)5"0@0?9'&"!T0`^WRDB+7"1`BU,(2(MT)#CH1KK__TB)QKC_____2(7V
-M#X1K`P``0<9'>`%)B7=03(F^,`P``+@`````Z5$#``!(BT0D0`^V4!)(B<&+
-M`(G3N@````#W\T&)UDB+1"0X#[90<(M!!(G1N@````#W\4B+7"1`#[9#$BC0
-MB$0D'T$XQD&`WO](BT0D.`^W4'R-0O\/MTL,(<%!B=1F02G,9D0Y8Q!S!40/
-MMV,00<9'>`!!QD=Y`,=$)!@!````#[=$)"#WV(E$)!0/M\%!B<5(BU0D0$0#
-M:@A(BTPD2$B!P<!7``!(B4PD"$B+7"1(2('#L%<``$B)'"1F9F:0BT0D(/_(
-M1"'HBUPD(&8IPV9$.>-!#T?<BVPD%$0A[8-\)!@`#X1>`0``QT0D&`````"+
-M5"0@2(M$)#AF.5!\=BBZ`````$$/MD=XB<&#^`!^%TACPDF+=,=0.6Y`#X0E
-M`0``_\(YT7_I#[=,)"")ZDB+="0X2(M\)$CHX;C__TB)QDB%P`^%U0```$'^
-M3WA!@']X_P^$M0```$R+5"0(3(L,)$D/MD=X28MTQU!(QX8P#````````$F)
-M\+D`````@'Y&`'9$#[;!2,'@!$P!P$B-D/`*``"_`````(.X\`H```!U$H-Z
-M!`!U#(-Z"`!U!H-Z#`!T!;\!````A?]FD'4E_\%!.$A&=[RX`````(7`=!M)
-MBT((28ER"$R)%DB)1@A(B3#K&;@!````Z^%)BT$(28EQ"$R)#DB)1@A(B3!!
-M_D]X08!_>/\/A53___^X_____^D]`0``9F9FD&9FD$$/MD=X#[;028ETUU#_
-MP$&(1WA,B;XP#```QH9@#```",:&80P```!$B?$/MM%$B>AF*>AFB8260`P`
-M`&:)G)9"#```1#BV8`P``'8'1(BV8`P``$0XMF$,``!S!T2(MF$,``!(BT0D
-M0&8I6!!F02G<=!,/M\-!`<7'1"08`0```.D:_O__2(M4)$!F@WH0``^$FP``
-M`+D`````0?_&1#IT)!\/E,!!`<9(BUPD.$0X<W!U/H!\)!\`=`;^3"0?ZPU(
-MBT0D0`^V0!*(1"0?@'PD'P!!#Y3&2(M4)#@/MT)\2(M<)$`!0PC'1"08`0``
-M`.L;BU0D($B+1"0X9CE0?+@!````#T9$)!B)1"082(M<)$!$#[=C$$B+1"0X
-M9D0[8'P/AD7]__]$#[=@?.D[_?__N`````!(@\186UU!7$%=05Y!7\-F9F:0
-M9F9FD&9FD$B#[`B%TG08QD89"TB)\DC'Q@````#H`````.L(9F:0Z*OV__](
-M@\0(PV9FD&9FD$%455-(@^P@28G\2(GU2(L>]D,!!'49QD89`DB+=C!(B>KH
-M`````.DT`@``9F9FD$B#>U``=2F^!P```&9F9I#H.Z___TB)QTB)0U"Z`!``
-M`+X`````Z`````!F9I!FD`^V11BH(`^%N````*@&=1C&11D!2(MU,$B)ZDR)
-MY^@`````Z=@!``#V11@$="=(@WM``'0@2(M32$B+<T!,B>?H`````$C'0T``
-M````0?^,)$!8``"+?0B)?"0,#[=%#&:)1"00#[9#</_(B$0D$@^V2W*)^-/H
-MB00D#[9T)!*Z`````/?VB40D!`^V2W+3X(E$)`@/MTM\#Z_.#[=$)!"-/#@/
-MKW0D""GWC7P/_XGXN@````#W\8G!#[=#?(G/#Z_XZRSV0WX!=!C&11D&2(MU
-M,$B)ZDR)Y^@`````Z1X!```/MWM\B?@/KT4(B40D"(M4)`B-#!=(B=Y,B>?H
-MSK'__TB)14A(A<!U&4C'14@`````2(U[*$B)[N@`````Z=P```!(B>)(B>Y,
-MB>?H3OK__X7`#X2&````2(MU2$R)Y^@:L?__08N,))A$``!!.8PDG$0``'1&
-M2&/!2,'@!$J!O""((@```````'1M_\%(8\%(:<#APW_P2,'H((T$`8G&P?X)
-MB<B9B?`IT&G`(0(``"G!03F,))Q$``!UNDC'14@`````28V\)$A8``!(B>[H
-M`````$R)Y^@`````ZT#V11@$="E(Q\(`````2(GN3(GGZ!BU___K)L9%&05(
-MBW4P2(GJ3(GGZ`````#K$4B)[DR)Y^A&]/__9F:09F:02(/$(%M=05S#9F9F
-MD&9FD%532(/L"$B)_4B)\TB#?E``=%%(B??H`````+D`````2&/12(M#4$B+
-M!-!(A<!T$4C'0"``````2(M`*$B%P'7O_\&!^?\!``!VU4B+<U"Z!P```$B)
-M[^C4K/__2,=#4`````!(@\0(6UW#9F:09I!!5%532(G[QX>H5P```````$B-
-MA[!7``!(B8>P5P``2(F'N%<``$B-A\!7``!(B8?`5P``2(F'R%<``$B-A]!7
-M``!(B8?05P``2(F'V%<``(L%`````,'@#(G"N*NJJJI(#Z_"2,'H($&)Q$'!
-M[`]$B>*^:`P``.ADK/__B<5!C50D(+X8````2(G?Z%"L__\!Q8L5`````,'B
-M#,'J#KZ(````2(G?Z#6L__\!Q;H(````O@`0``!(B=_H(:S__P'%2(G?Z```
-M``!(B04`````_\4[+0````!S-V9FD&9FD$B)W^@`````2(G"2(7`="%(BX,P
-M6```2(D"2(F3,%@``/^#.%@``/_%.RT`````<L];74%<PV9F9I!F9I!F9I!5
-M4TB#[`A(B?5(BUX(#[9.`[@!````B<+3XH!F`?O'@X0`````````]D-^`G02
-M9H.[@`````!T&^L09F:09F:09H.[@`````!T-F:%DX````!U+68)DX````"`
-M8P'[2(-[:`!T"TB)WO]3:&9FD&:0@&-^'^G'````9F9FD&9FD`^V0WYFB9.`
-M````_DMQQX.$`````````(/(!8A#?H3`>0Z#X'^(0W[IDP```&9FD(!C?I](
-MB=[H`````$B)PDB%P'1:Q@`##[9%`XA"`TB)6@B`2@$(2(L%`````$B)0F!(
-MBP4`````2(E":$C'10@`````2`^V0@-(B93#B`````^V0WZ#X/Z#R`J(0W[^
-M0W%(B=:_!P```.@`````2(-["`!T"TB+6PA(@WL(`'7UZ`````")0W1(B=_H
-M`````$B#Q`A;7<-F9I!F9I!F9I"Z`````/_.@_[_=!-F9F:0#[8'`<)(_\?_
-MSH/^_W7Q#[;"PV9F9I!F9I!F9I!F9I!54TB#[`A(B?N`/P-V6+T`````@']P
-M`'1'9F9FD&9FD$AC]4B#O/.(`````'0H2(N,\X@````/ME,!@^(!#[9!`8/@
-M_@G0B$$!2(N\\X@```#H`````/_%#[9#<#GH?\"`8W[[ZPKH%0```&9FD&:0
-M2(/$"%M=PV9FD&9FD&9FD$%505154TB![`@"``!)B?Q!O0````!(B>.`?W``
-M#X1)`@``2(MO"$B%[70.2(-]"`!T!TF)[4B+;0BZ``(``+X`````2(G?Z```
-M``#'`_,6>%JX`````$B%[70#BT5TB4,$08!\)'@`=`V`2PD"00^V1"1[B$,*
-M00^V5"1TO@$````AU@^V@Y,```"#X/R)T8/A`@GP"<B^!````"'6@^#SB=&#
-MX0@)\`G(OA`````AUH/@SXG1@^$@"?`)R(G1@^%`@^`_@^*`"<@)T(B#DP``
-M`$'V1"0!`70$@$L)`;@`````2(7M=`E)@_T!&<"#P`*(0PM(A>T/A+P```"+
-M11B)0PS'@Y0`````````#[9%`(A#$`^V17"(0Q%-A>UT!T$/MD4#ZPE!#[9$
-M)`-F9I"(0Q(/MD5RB$,3]D5^`G0$@$L4`0^WA8````!FB4,6BX6$````B4,8
-MQX.8`````````$B-M<@```!(C7LLNA````#H`````$B-M=@```!(C7L\N@0`
-M``#H`````$B-M1P!``!(C;N`````NA````#H`````$B-M=P```!(C7M`ND``
-M``#H`````$V%[71=08M%&(E#',>#G`````````!!#[9%`(A#($$/MD5PB$,A
-M00^V1"0#B$,B00^V17*(0R-!]D5^`G0$@$LD`4$/MX6`````9HE#)D&+A80`
-M``")0RC'@Z``````````2(7M=0A!#[8$)(A#$+Z0````2(G?Z`````#WV(A#
-M"+X``@``2(G?Z`````#WV(B#D````$&+M"2(````@^X*28U\)'!(B=FZ,```
-M`.@`````2('$"`(``%M=05Q!7<-F9I!F9I!(B?B`/P-V.+H`````@']P`'08
-M#[9/<$ACPDB#O,>(`````'4,_\(YT7_LN`````##2(N\QX@```#H`````&:0
-M\\-F9F:09F9FD&9FD&9FD%532(G[@#\#=C:]`````(!_<`!T+TACQ4B#O,.(
-M`````'0-2(N\PX@```#H`````/_%#[9#<#GH?]OK"&9F9I"`9P'^6UW#9F:0
-M9F:09F:02(M/$+H`````9F9FD&9FD(G02#D\P74)2,<$P0````##_\*#^@=V
-MZ//#9F:09F:02(GW@&8!^^@`````\\-FD$%505154TB![`@$``!)B?Q(C9PD
-M``(``$B)_0^V1P&H!`^$+P$``(`_`@^&)@$``(/@_8A'`;H``@``O@````!(
-MB=_H`````$&`/"0##X2+````2(G@2(NMB````$&`/"0(=7!(A>UT#&9!@[PD
-M@`````%U7TB)Q4&]`0```$$/MD0D<(/X`7YK26/%28N\Q(@```!(@\=P2(GI
-MNB````"^`````.@`````N0````"02&/1BP23,T25`(D$D__!@_E_?NQ!_\5!
-M#[9$)'!$.>A_M^L@@'T``P^%>/___TB-?7!(B=FZ(````+X`````Z`````!F
-M@;O^`0``5:IU5X.[R@$```!T'8%[!DQ)3$]T#(&[=@$``$=254)U"$&`3"0!
-M`NLQ0;T`````26/%2,'@!$@!V("XO@$``(!U#X.XR@$```!T!D&`3"0!`D'_
-MQ4&#_0-^U4B!Q`@$``!;74%<05W#9F9FD&9F9I!F9F:01(N'A"(``$6-2`%)
-M8\%(:<#APW_P2,'H($&-!`'!^`E$B<G!^1\IR&G`(0(``$$IP4UCP$G!X`1)
-MB70X<$ACAX0B``!(P>`$2(E4.'A$B8^$(@``PV9F9I!F9F:09F:04TB#[!!(
-MB?N+AX`B```[AX0B``!T8HN+@"(``$ACP4C!X`1(`=A,BT!P3(D$)$B+<'A(
-MB70D"/_!2&/!2&G`X<-_\$C!Z""-!`&)Q\'_"8G(F8GX*=!IP"$"```IP8F+
-M@"(``$B)WT'_T(N#@"(``#N#A"(``'6>2(/$$%O#9I!$BX><1```18U(`4EC
-MP4AIP.'#?_!(P>@@08T$`<'X"42)R<'Y'RG(:<`A`@``02G!36/`2<'@!$F)
-MM#B((@``2&.'G$0``$C!X`1(B90XD"(``$2)CYQ$``##9F:09I!32(/L$$B)
-M^XN'F$0``#N'G$0```^$@P```&9FD&9FD(N+F$0``$ACP4C!X`1(C808@"(`
-M`$B+4`A(B10D2(M`$$B)1"0(_\%(8\%(:<#APW_P2,'H((T$`8G&P?X)B<B9
-MB?`IT&G`(0(``"G!B8N81```@[L\6````'0(2(G?Z`````!(BW0D"$B)W_\4
-M)(N#F$0``#N#G$0``'6#2(/$$%O#9F9FD&9FD&9FD&9FD%.X`````$B#?V@`
-M="!(BU]H2(L#2(E':+J(````O@````!(B=_H`````$B)V%O#D$B+1VA(B09(
-MB7=HPV9F9I!(@S\`=!I(BP=(BT!`2(E&0$B+!TB)<$#K"F9FD&9FD$B)=D!(
-MB3?#9F9FD&9F9I!(B5PDX$B);"3H3(ED)/!,B6PD^$B#["A)B?U(B?5(@SX`
-M=#U,BR9(BT4`2(M80$@YV'4,2,=%``````#K#F:02(M5`$B+0T!(B4)`2(G>
-M3(GO_U-(3#GC=`=(@WT``'7&2(M<)`A(BVPD$$R+9"083(ML)"!(@\0HPV9F
-MD&9FD$B#[`B`/PAU!^@`````9I!(@\0(PV9F9I!F9F:09F:02(/L"$B)\4B+
-M1Q"`/PAU!^@`````ZPM(B=9(B<=F9I#_T4B#Q`C#9F:09F:09F:02(E<)-!(
-MB6PDV$R)9"3@3(EL).A,B70D\$R)?"3X2(/L:$&)]$F)ST&)U4B+;S!$#[8W
-M00^VWHG>2(GOZ`````"Z`````(G>2(GOZ`````#'!"0`````0;D`````0;@`
-M````N0````"Z`P```(G>2(GOZ`````"Z`````(7`#X2U````0??$````\`^5
-MP`^VR$&`_3`/E,!$#[;00?_"00^V]D$/ML6%R70108#](+HT````N"0````/
-M1<*)1"0PN$````"%R74)1(G@P>@8@\A`B40D*$2)Y\'O$$`/ML>)1"0@3(GB
-M#[;&B40D&$$/MM0/M\*%R70)B?@E`/\```G0B40D$,=$)`@!````QP0D````
-M`$&Y``$``$V)^$2)TDB)[^@`````B<-!#[;V2(GOZ`````")VHG02(M<)#A(
-MBVPD0$R+9"1(3(ML)%!,BW0D6$R+?"1@2(/$:,-FD(GX@^`#P>`-!0`@``!`
-M]L<$=`T%```#`,-F9F:09F:0!0```@##9F9FD&9FD&9FD$%455-)B?R)];\@
-MH0<`Z`````"[(*$'`$`/MOWHK____XVH'`$``$F+?"00B>[H`````(3`>`FX
-M`0```.L?9I"_$"<``.@`````@<,0)P``@?L_2TP`=L^X`````%M=05S#9F9F
-MD&9F9I!F9I!F9I!(B5PD\$R)9"3X2(/L&$F)_$`/MO[H1O___XG#C7,(28M\
-M)!"Z`````.@`````C7,,28M\)!"Z&`$``.@`````2(M<)`A,BV0D$$B#Q!C#
-MD%-(B?M`#[;^Z`/___^-<`Q(BWL0N@````#H`````%O#B?!FA<EU#O?"````
-M\'4&9JD`_W0+N`$```##9F:09I!FA?8/E,"#_P$/E,(/ML"%P@^5P`^VP,-F
-M9F:09F9FD$F)T;H`````3(U'"HM!$(D'BT$4B4<$@SD`N`$````/1-!!#[9!
-M!`'`"<)FB5<(BX9$`@``A<`/E,*#^`(/E,`)T*@!#X1F`0``@WD$`0^%W```
-M`&8/MD$/@,P19D&)`$F#P`)F#[9!#H#,$69!B0!)@\`"00^V003!X`-F)?@`
-M@,P29D&)`$F#P`)F#[9!"X#,$V9!B0!)@\`"9@^V00B`S!-F08D`28/``F8/
-MMD$,@,P49D&)`$F#P`)F#[9!"8#,%&9!B0!)@\`"9@^V00V`S!5F08D`28/`
-M`F8/MD$*@,P59D&)`$F#P`)F0<<`0!9)@\`"@SD`=1F#OD0"```"NB8```"X
-M8`````]%PNF@`0``@[Y$`@```KHV````N&$````/1<+IAP$``&9F9I!F#[9!
-M#H#,$69!B0!)@\`"00^V003!X`-F)?@`@,P29D&)`$F#P`)F#[9!"(#,$V9!
-MB0!)@\`"9@^V00F`S!1F08D`28/``F8/MD$*@,P59D&)`$F#P`*+00@E````
-M#\'H&&8-0!9F08D`28/``H,Y`1G`@^#[@^@TZ04!``!FD(-Y!`$/A9<```!F
-M#[9!#X#,$F9!B0!)@\`"9@^V00Z`S!)F08D`28/``F8/MD$+@,P39D&)`$F#
-MP`)F#[9!"(#,$V9!B0!)@\`"9@^V00R`S!1F08D`28/``F8/MD$)@,P49D&)
-M`$F#P`)F#[9!#8#,%69!B0!)@\`"9@^V00J`S!5F08D`28/``F9!QP!`%DF#
-MP`*#.0$9P(/@\(/`->MB9@^V00Z`S!)F08D`28/``F8/MD$(@,P39D&)`$F#
-MP`)F#[9!"8#,%&9!B0!)@\`"9@^V00J`S!5F08D`28/``HM!""4````/P>@8
-M9@U`%F9!B0!)@\`"@SD!&<"#X/Z#Z#9F#[;`9@T`EV9!B0##2(E<)-!(B6PD
-MV$R)9"3@3(EL).A,B70D\$R)?"3X2(/L2(GPB50D%$B)S4R+;Q`/ML!(BT3'
-M,$2+<#Q!C;8$`0``3(GOZ`````"(10!!C88(`0``B40D$(G&3(GOZ`````!F
-M#[;`9HE%`D&-G@P!``")WDR)[^@`````9@^VP&:)101%C:80`0``1(GF3(GO
-MZ`````!F#[;`9HE%!D6-OA0!``!$B?Y,B>_H`````&8/ML!FB44(@WPD%`$/
-MA88```!!C88@`0``B40D"(G&3(GOZ`````"(1"0/B<*#RH`/MM*+="0(3(GO
-MZ`````"+="003(GOZ`````#!X`AF"44"B=Y,B>_H`````,'@"&8)101$B>9,
-MB>_H`````,'@"&8)109$B?Y,B>_H`````,'@"&8)10@/ME0D#XMT)`A,B>_H
-M`````$&-MA@!``!,B>_H`````(A%"D&-MAP!``!,B>_H`````(A%"TB+7"08
-M2(ML)"!,BV0D*$R+;"0P3(MT)#A,BWPD0$B#Q$C#D$B)7"302(EL)-A,B60D
-MX$R);"3H3(ET)/!,B7PD^$B#[&A)B?^)\V:)5"0>#[;#3(MTQS!!BW8\@\8P
-M2(M_$.@`````2(G%@^4?2(U$K0!(P>`$2HVL,%`"``"#?0`!=%U(C44@2(E$
-M)!!,C60D((M0!$0/MNM,B>%$B>Y,B?_H[/W__T'^CF4,``!(BUPD$$B+2R!)
-MBWXP3(DD)$&Y`````$0/MT0D'KH!````1(GN_U,82(GN3(GWZ!(3``!(BUPD
-M.$B+;"1`3(MD)$A,BVPD4$R+="183(M\)&!(@\1HPV9F9I!F9F:09F9FD&9F
-MD$B)7"302(EL)-A,B60DX$R);"3H3(ET)/!,B7PD^$B#[&A)B?]!B?7'1"04
-M`````(GQ#[;!2(MLQS!!O@`````/MP)FB40D&$B-3"08#[="`F:)00*+0@2)
-M000/MT0D&$B)PX/C'TB-!)M(P>`$2(V<*%`"```/MT$"A,!T*L=$)!0!````
-M#[=!`D0/MO!F08'.``''A6`,````````QX5``@```````$R-8R!(C40D&`^W
-M0`*H!'072(U,)"!!BU0D!$$/MO5,B?_HL?S__Y"#.P$/A)<```#^C64,``!(
-MC40D&$2+2`1)BTPD($4/MNU(BWTP2(U$)"!(B00D10^WQHM4)!1$B>Y!_U0D
-M&$B)WDB)[^C'$0``@[U@#````'1.2(.]4`P```!T1$B+A5`,``"#>!@!=3=$
-MB>Y,B?_HW!,``$B+M5`,``!(B>_H#14``(7`=1E(B[50#```N@````!(B>_H
-M]08``&9FD&:02(M<)#A(BVPD0$R+9"1(3(ML)%!,BW0D6$R+?"1@2(/$:,-F
-M9F:09F:09F:09F:02(E<).!(B6PDZ$R)9"3P3(EL)/A(@^PH2(G[B?5!B=5(
-MBW\0OBP-``#H`````$&)Q(G"@\H!2(M[$+XL#0``Z`````!(BWL0OBP-``#H
-M`````$B+>Q"^\`0!`.@`````B>KWTB'01`GH2(M[$(G"OO`$`0#H`````$B+
-M>Q"^\`0!`.@`````2(M[$$2)XKXL#0``Z`````!(BUPD"$B+;"003(MD)!A,
-MBVPD($B#Q"C#9F9FD&9F9I!F9F:09F:02(E<)-!(B6PDV$R)9"3@3(EL).A,
-MB70D\$R)?"3X2(/L2$F)_D&)S$2)3"042(M'$$B)1"0(QT0D!`````!$C3RU
-M`````$$)UT2)^@^VPDR+;,<P187`=')-A>T/A-<```!!BYWP#```0HT4Y0``
-M``!!"U4L08MU/(/&)$B+?"0(Z`````!$.>-T(T$/MN^0B=I)BT482(T4T(GN
-M3(GWZ/S\____PX/C'T0YXW7B08.]8`P```"X`0````]%1"0$B40D!$&)G?`,
-M``"#?"04``^5PH-\)`0`#Y7`"="H`0^$^P$``$$/MO_H(O;__T2-8`A$B>9(
-MBWPD".@`````B<4/M]B)VO?21(GF2(M\)`CH`````/;#"`^$(0$``$V%[706
-M0<>%8`P```````!!QX5``@```````$&#OI`````!=`I!@[Z4`````744O@`!
-M``!$B?G3YHGR3(GWZ-_]__]!@[Z,`````70408.^D`````%T"D&#OI0````!
-M=0Y!#[;W3(GWZ*4)``#K?T&#OH@````!=`I!@[Z,`````75K00^VWXG?Z&3U
-M__]$C6`HN@0```!$B>9(BWPD".@`````1(GF2(M\)`CH`````+\9````Z```
-M``"Z`````$2)YDB+?"0(Z`````!$B>9(BWPD".@`````O]`'``#H`````(G>
-M3(GWZ.0'``!!#[;?B=_H^?3__XVP'`$``$B+?"0(Z`````")V;H`````O@$`
-M``!,B?=!_U8HZ:$```!`]L40=%)!#[;W3(GW9F9FD.B;!P``08.^D`````%T
-M"D&#OI0````!=1>^``$``$2)^=/FN@````!,B??HP/S__T$/ML^Z`0```+X!
-M````3(GW0?]6*.M)#[?5]L8!=$%!@[U``@```74B0<>%8`P```````!!QX5`
-M`@```````$$/MO=,B??HY?G__X,]``````%U#$R)[^C4$```9F9FD$B+7"08
-M2(ML)"!,BV0D*$R+;"0P3(MT)#A,BWPD0$B#Q$C#9F9FD&9FD&9FD&9FD$B#
-M[`A)B?B-#+4`````"=$/ML%(BWS',$B%_W43#[;Q3(G'Z'@.``#K2V9FD&9F
-MD$B+MU`,``!(A?9U%`^V\4R)Q^A9#@``ZRQF9F:09F:0@SX!=`V#?A@!9F9F
-MD&9FD'0.#[;Q3(G'Z#,.``#K!I#H"P```$B#Q`C#9F:09F:02(E<)-!(B6PD
-MV$R)9"3@3(EL).A,B70D\$R)?"3X2(/L.$F)_DF)]TB+1S!,BV`0BV\\0;T#
-M````1"(O#[8?@^,$P>L"C;4@`0``3(GGZ`````"-M1P!``!,B>?H`````(G%
-MN@`!``!$B>G3XO?2@/L!&?9FO@``@<84``,`3(GGZ`````!`A.T/B/$```!`
-M]L4!=!NZ`0```$R)_DR)]^C+`0``Z=8```!F9I!F9I!!BT<@@_@!="R#^`%F
-M9F:0<@Z#^`)T<NFT````9F9FD+H`````3(G^3(GWZ)`!``#IFP```$#VQ4!T
-M/$F-=R!,B??HN0```(/X`74908-_,`!U?;H`````3(G^3(GWZ%T!``#K:[H!
-M````3(G^3(GWZ$L!``#K6;H!````3(G^3(GWZ#D!``#K1T`/ML6H0'0_J"!U
-M.T&#?S``=1*Z`````$R)_DR)]^@4`0``ZR))C7<@3(GWZ$8```"%P'42N@$`
-M``!,B?Y,B??H\@```&:02(M<)`A(BVPD$$R+9"083(ML)"!,BW0D*$R+?"0P
-M2(/$.,-F9F:09F:09F:09F:005154TF)_$B)\[@`````@7X0_P````^&GP``
-M`(L6@_H!=%BX`````(/Z`0^"B@```(/Z`@^%?````+T`````D$B+0P@/MQ!(
-M@\`"2(E#"$&+="0\@<8``0``28M$)#!(BW@0Z`````#_Q8']_P```';.@6L0
-M``$``.L[O0````!!BW0D/('&``$``$F+1"0P2(MX$.@`````2(M3"&:)`DB#
-M0P@"_\6!_?\```!VT8%K$``!``"X`0```%M=05S#2(E<)-A(B6PDX$R)9"3H
-M3(EL)/!,B70D^$B#[$A(B?U)B?1!B=5!O@````!(C5X@2(U,)!"+4P0/MC=(
-MBW\PZ-ST__]!@_T!=0U!M@''A6`,````````2(M+*`^V=0!(BWTP2(U$)!!(
-MB00D0;D`````0;@`````1(GR_U,@3(GF2(GOZ/<)``!!@_T!=1&#/0`````!
-M=0A(B>_H$`T``$B#O5`,````#X2C````2(NU4`P``(-^&`%U)DB)[^@]#0``
-MA<`/A88```!(B[50#```N@````!(B>_H(?___^MP@[U$`@```'4J#[9U`$B+
-M?3#HN0H``(MU/(/&*$B+13!(BW@0Z`````"_`0```.@`````#[9U`$B+?3#H
-MOPH``$B+G5`,``!(A=MT)(-[&`!U'DB-4R!(B=Y(B>_H_@L``$B+6PA(A=MT
-M!H-[&`!TXDB+7"0@2(ML)"A,BV0D,$R+;"0X3(MT)$!(@\1(PY!!54%455-(
-M@^P(2(G[2(M',$R+8!!$BV\\08UM*(GN3(GGZ`````"H!'0QN@(```")[DR)
-MY^@`````B>Y,B>?H`````+_0!P``Z``````/MC-(BWLPZ"8"``#K$4&-=2BZ
-M`@```$R)Y^@`````QH-(`@```,:#9`P```#&@V4,````QX/L#````````,>#
-M\`P```````#'@T`"````````QX/H#```(````+H`````D$ACPHF4@V@,``#_
-MPH/Z'W[ON@````!(8\)(C02`2,'@!,>$&%`"```!````_\*#^A]^XTC'@U`,
-M````````2,>#6`P```````#'@V`,````````08UU"+H`````3(GGZ`````!!
-MC74,NA@!``!,B>?H`````(M3($&-=1!,B>?H`````(M3)('B`/S__T&-=11,
-MB>?H`````$&-=1BZ`````$R)Y^@`````BU,H08UU'$R)Y^@`````08UU(+H`
-M````3(GGZ`````"+4RRR`$&-=21,B>?H`````+@!````2(/$"%M=05Q!7<-F
-M9I!FD$%7059!54%455-(@^PX2(G[B50D'$&)SX7V#X6B````O0````!,C70D
-M(&9FD&9FD$ACQ4B-!(!(P>`$2`'8@[A0`@```'5P2(V08`(``(-Z"`%U$XM2
-M%$R+H)@"``!,BZB0`@``ZR)(8\5(C02`2,'@!$@!V(N0=`(``$R+H)`"``!,
-MBZB(`@``#[8S2(M[,$R)\>AX\?__#[8S2(M[,$R)-"1!N0````!%#[?'3(GA
-MBU0D'$'_U?_%@_T?#XYN____2(/$.%M=05Q!74%>05_#9F9FD&9F9I!F9F:0
-M9F:02(E<).!(B6PDZ$R)9"3P3(EL)/A(@^PH2(G[B?6)Z(/@!$&)Q4'![0)!
-MO`,```!!(>R#OX@````!=`F#OXP````!=5I!#[;TP>8(@<8,`0``C88```,`
-M@<8```(`183M#T7P2(M[$.@`````B<&#X?R#R0%!#[;TP>8(@<8,`0``C88`
-M``,`@<8```(`183M#T7P2(M[$(G*Z`````!!#[;TP>8(@<9T`0``C88```,`
-M@<8```(`183M#T7P2(M[$.@`````B<&`X1^)Z`^VT`^VA!JF````P>`%)>``
-M```)P8#EYP^VA!JN````P>`+)0`8```)P4$/MO3!Y@B!QG0!``"-A@```P"!
-MQ@```@!%A.T/1?!(BWL0B<KH`````$B+7"0(2(ML)!!,BV0D&$R+;"0@2(/$
-M*,-F9I!F9I!(B5PDZ$R)9"3P3(EL)/A(@^P82(G[1`^V[D2)[^B^Z___1(U@
-M*$2)[DB)W^B?[/__2(M[$+H$````1(GFZ`````!(BWL01(GFZ`````"_&0``
-M`.@`````2(M[$+H`````1(GFZ`````!(BWL01(GFZ`````!$B>Y(B=_H0/[_
-M_T2)[DB)W^CUZ___2(L<)$R+9"0(3(ML)!!(@\08PV:005=!5D%505154TB#
-M[`A(B?W&1"0'`$&^``````^V1"0'1(T\A0````!F9F:09F:00XT$/D0/MNA$
-MB>_H`.O__T&)Q$&-7"0H2(M]$+H"````B=[H`````$B+?1")WN@`````1(GN
-M2(GOZ/+^__](BWT0N@````")WN@`````2(M]$+H?`0``1(GFZ`````!!C70D
-M!$B+?1"Z`````.@`````08UT)`A(BWT0N@````#H`````$&-="0,2(M]$+H`
-M````Z`````!!C70D$$B+?1"Z`````.@`````08UT)!1(BWT0N@````#H````
-M`$&-="082(M]$+H`````Z`````!!C70D'$B+?1"Z`````.@`````08UT)"1(
-MBWT0N@````#H`````$&-="0@2(M]$+H`````Z`````!!C70D+$B+?1"Z````
-M`.@`````08UT)#1(BWT0NKP```#H`````$'_QD&`_@,/AL/^__^`?"0'`1G;
-M9KL``('#```#`(US#$B+?1"Z`````.@`````C7,02(M]$+H`````Z`````"-
-M<Q1(BWT0N@````#H`````(US&$B+?1"Z`````.@`````@\,@2(M]$(G>Z```
-M```E'!P<'`T#`P,#2(M]$(G"B=[H`````/Y$)`>`?"0'`0^&(/[__TB#Q`A;
-M74%<05U!7D%?PV9F9I!F9F:09F:02(/L"$B+?Q"Z___/#[YL!`$`Z`````!(
-M@\0(PV9F9I!32(G[BT<$L``]``"`4'0A2(M_$+XL#0``Z`````"#R`%(BWL0
-MB<*^+`T``.@`````2(M[$+X`#0``Z``````E__\`_TB+>Q")PKX`#0``Z```
-M``!(BWL0N@````"^!`T``.@`````2(M[$+H`````OC@,``#H`````$B+>Q"Z
-M_P`!`+X$'0``Z`````!(BWL0N@````"^9!T``.@`````2(M[$+H`````OB@,
-M``#H`````$B+>Q"Z`````+Y8'0``Z`````!(BWL0N@````"^7!T``.@`````
-M2(M[$+H`````OD`=``#H`````$B+>Q"Z`````+Y$'0``Z`````!(BWL0N@``
-M``"^2!T``.@`````2(M[$+H`````OE`=``#H`````%O#9F:09F:09F:02,=&
-M"`````!(BX=8#```2(E&$$B#OU@,````=`M(BX=8#```2(EP"$B)MU@,``!(
-M@[]0#````'4'2(FW4`P``//#9F9FD&9F9I!F9I!F9I!(@WX(`'4:2(M&$$B)
-MAU@,``!(A<!T%DC'0`@`````ZPQ(BU8(2(M&$$B)0A!(@WX0`'4<2(M&"$B)
-MAU`,``!(A<!T&$C'0!``````ZPYFD$B+5A!(BT8(2(E""$C'1@@`````2,=&
-M$`````##9F:02(E<).A(B6PD\$R)9"3X2(/L&$F)_$B)]4B)TX,Z`'442(UR
-M"$B-?2"Z*````.@`````ZQI(C7((2(U](+HP````Z`````!!_H0D9`P``(L#
-MB4482(GN3(GGZ.'^___'10``````0?Z$)$@"``!(BQPD2(ML)`A,BV0D$$B#
-MQ!C#9F9FD&9F9I!F9F:09F:02(E<)/!(B6PD^$B#[!!(B?M(B?6#?A@`=`;^
-MCV0,``!(B>Y(B=_HU?[__XN#Z`P``(G!#[95!(F4BV@,``#_P(F#Z`P``,=%
-M``$```#^BT@"``!(BQPD2(ML)`A(@\00PV9F9I!F9F:09F9FD&9FD$"`_P-V
-M$4`/ML^-3`D"N`$```!(T^##0`^VSXU,"0&X`0```$C3X,-F9F:09F9FD%53
-M2(/L"$B)^T`/MNZ)[^B\____"8.`````B>_HS^7__XVP'`$``$B+>Q#H````
-M`(GI@^$#N@`!``#3XO?2B>B#X`3!^`*#^`$9]F:^``"!QA0``P!(BWL0Z```
-M``"+DX````!(BWL0OF0=``#H`````$B+>Q"^9!T``.@`````2(/$"%M=PV9F
-MD&9FD&9FD%-(B?M`#[;^Z#/____WT(G"(Y.`````B9.`````2(M[$+YD'0``
-MZ`````!;PV9FD$B)7"382(EL).!,B60DZ$R);"3P3(ET)/A(@^PH28G^B?-,
-MBV\0#[;#3(MDQS!!BVPD/$''A"1``@```0```(G8@^`$P>@"B=F#X0.Z`0$`
-M`-/B]](\`1GV9KX``('&%``#`$R)[^@`````08MT)#R#Q@BZ`````$R)[^@`
-M````#[;S3(GWZ$7___^-M1P!``!,B>_H`````$''A"1,`@```0```(UU*+H!
-M````3(GOZ`````!(BQPD2(ML)`A,BV0D$$R+;"083(MT)"!(@\0HPV9F9I!F
-M9F:09F:09F:02(E<).!(B6PDZ$R)9"3P3(EL)/A(@^PH2(G[08GU3(MG$(GR
-M#[;"2(M$QS"+4#S'@$`"````````C6HHN@(```")[DR)Y^@`````B>Y,B>?H
-M`````$$/MO5(B=_H__W__TB+7"0(2(ML)!!,BV0D&$R+;"0@2(/$*,-F9I!F
-M9I!!5%-(@^P(2(G[2(G12(M',$R+8!"+O^P,``!(P><%2`-[$$B)\DB)WNCR
-MY/__BX/L#```_\"#X!^)@^P,``#^@V4,``"+4R2!X@#\___!X`4)PHMS/(/&
-M%$R)Y^@`````2(/$"%M!7,-F9F:09F:04TB)^[D`````N@(```"^`````.A(
-M]?__2(G?Z*#S__](B=_H`````(/X`746QX-@#````0````^V,TB+>S#H+?W_
-M_UO#9F9FD&9F9I!F9I!(B5PDX$B);"3H3(ED)/!,B6PD^$B#["A,C68@2(M'
-M,$B+:!"+7SR-LQP!``!(B>_H`````"7!````N@````"#^$`/A<@!``!!@WPD
-M!`%U9D$/ME0D%8VS!`$``$B)[^@`````00^V5"07C;,(`0``2(GOZ`````!!
-M#[94)!F-LPP!``!(B>_H`````$$/ME0D&XVS$`$``$B)[^@`````00^V5"0=
-MC;,4`0``2(GOZ`````#K-[@`____9D&%1"04=2!F085$)!9U&&9!A40D&'40
-M9D&%1"0:=0AF085$)!QT"KH`````Z2,!``!!#[94)!2-LP0!``!(B>_H````
-M`$$/ME0D%HVS"`$``$B)[^@`````00^V5"08C;,,`0``2(GOZ`````!!#[94
-M)!J-LQ`!``!(B>_H`````$$/ME0D'(VS%`$``$B)[^@`````00^V5"0>C;,8
-M`0``2(GOZ`````!!#[94)!^-LQP!``!(B>_H`````$&#/"0"#X6$````C;,@
-M`0``2(GOZ`````"Y9````+H*````B=Y(B>_H`````+H`````A<!T88VS'`$`
-M`$B)[^@`````)8D```"Z`````(/X"'5$0;T`````@<,``0``28M$)`@/MQ!(
-M@\`"28E$)`B)WDB)[^@`````0?_%08']_P```';908%L)!```0``N@$```!F
-M9I")T$B+7"0(2(ML)!!,BV0D&$R+;"0@2(/$*,-F9I!FD$B)7"382(EL).!,
-MB60DZ$R);"3P3(ET)/A(@^PH2(G[QX>``````````$B+?Q"Z`````+YD'0``
-MZ`````#'@X@`````````QX.,`````````,>#D`````````#'@Y0`````````
-MQX.8`````````,>#G`````````#'@Z``````````QH.D`````;@`````2(-[
-M*``/A*8"``"X`````&:!>P:!4`^%E0(```^V4P6#^@$/A(@"``"#^@%_!X72
-MD'00ZRJ#^@)T&69F9I!F9I#K',>#C`````$```!F9F:0ZQ;'@Y`````!````
-MZPK'@Y0````!````O0````!F9I!`#[;5B="#X`/!X`@%=`$``/;"!'0*C;``
-M``,`ZPAFD(VP```"`$B+>Q#H`````(GJ#[;*B<!(B<*!XN````!(P>H%B)09
-MI@```"4`&```2,'H"XB$&:X```#_Q4"`_0=VGDB)W^@V]/__2(G?Z#[V__](
-MB=_H5O;__TB+>Q"Z`````+[P!`$`Z`````!(BWL0OBP-``#H`````$&)Q$&#
-MY/Y(BWL01(GBOBP-``#H`````$B+>Q"^``T``.@`````J#!T18M#<*G@`P"`
-M=`@E'_S_?XE#<(.[B`````%T&X.[C`````%T$H.[D`````%T"8.[E`````%U
-M$XM#<*@0=`R#X.^)0W#K!(-C<,^]`````$`/MO5(B=_HH?'____%0(#]!W;L
-MBU-P2(M[$+X`#```Z`````"+4W1(BWL0OB@,``#H`````(M3>$B+>Q"^7!T`
-M`.@`````O0````")Z0^VP8M4@QA`@/T!&?9FO@``@<8,``,`2(M[$.@`````
-MB>H/ML*+5(,@0(#]`1GV9KX``('&$``#`$B+>Q#H`````/_%0(#]`7:TQX.`
-M````5:L&`+T`````9F:09F:00`^V]4B)W^BTWO___\5`@/T'=NR_4,,``.@`
-M````@[N0`````70)@[N4`````75'0;P`````O0````!!O@`!``!$#[;M1(GN
-M2(G?Z`````"%P'4+1(GP1(GIT^!!"<3_Q4"`_0=VVD2)XKX`_P``2(G?Z-GE
-M__^X`0```&9F9I!(BQPD2(ML)`A,BV0D$$R+;"083(MT)"!(@\0HPV9FD$B#
-M[`A(BW\0N@````"^9!T``.@`````N`$```!(@\0(PV9F9I!F9F:09F9FD&9F
-MD,<%``````$```##9F:09I#'!0``````````PV9FD&:02(E<)/!(B6PD^$B#
-M[!A(B?V)\@^VPDB+7,<PN`````!(A=MT+$`/MO[H`MW__XE#/$B):S#'@V`,
-M````````QX-``@```````$B)W^@_[?__2(M<)`A(BVPD$$B#Q!C#B?`/ML!(
-MBU3',+@`````2(72=`V#NF`,```!#Y7`#[;`\\-F9F:09F:09F:09F:055-(
-M@^P(2(G]B?"#X`/!X`@%``$``$#VQ@1T"(V8```#`.L&C9@```(`2(M]$(G>
-MZ`````"#X`^#^`%T$X/X`7(^@_@#=#'K-V9F9I!F9I!(BWT0B=[H`````+]`
-M#0,`Z`````!(BWT0B=[H`````(/@#X/X`W4(N`$```#K!I"X`````$B#Q`A;
-M7<-F9F:005=!5D%505154TB#[#A(B?U!B?:)\(/@`XA$)#-$B?"#X`1!B<5!
-MP>T"B?(/ML)(BU3',$R+?Q"X`````$B%T@^$W@$``(M"/(E$)#2X`````(.Z
-M8`P```$/A,4!``!$#[;F1(GFZ)7<__^+7"0T@\,HN@0```")WDR)_^@`````
-MB=Y,B?_H`````+\9````Z`````"Z`````(G>3(G_Z`````")WDR)_^@`````
-M1(GF2(GOZ#?N__]!O``````/MD0D,\'@"`4(`0``B<)(B50D*$B)5"0@2(E4
-M)!A(B50D$(G`2(E$)`A!#[;&B40D!$B+5"0HC8(```,`C;(```(`183M#T7P
-M2(M]$.@`````B<.#X_"#RP%(BW0D((V&```#`(VV```"`$6$[0]%\$B+?1")
-MVN@`````2(MT)!B-A@```P"-M@```@!%A.T/1?!(BWT0Z`````"_&0```.@`
-M````@^/P2(MT)!"-A@```P"-M@```@!%A.T/1?!(BWT0B=KH`````$B+="0(
-MC88```,`C;8```(`183M#T7P2(M]$.@`````O]`'``#H`````(M<)`2)WDB)
-M[^@`````A<!U'D'_Q$&#_`,/A1[___^)WDB)[^C3VO__N`````#K0(MT)#2!
-MQAP!``!,B?_H`````"7`````@_A`=!-!#[;V2(GOZ*3:__^X`````.L100^V
-M]DB)[^B1VO__N`$```!(@\0X6UU!7$%=05Y!7\-F9F:09F:09F:09F:02(E<
-M)-A(B6PDX$R)9"3H3(EL)/!,B70D^$B#["B)\(G508G-#[;`2(M<QS"Z````
-M`$B%VP^$BP```$R+=Q!$BV,\N@````"#NV`,```!='5$B>9,B??H`````(G&
-MA>UU+T&-1?^Z`````#P?=U?'@T0"````````@^;`00^VQ?_("<:!S@`"``#K
-M%69FD&:0QX-$`@```0```('FP/W__XGR@,X!B?"`Y/Z#>PP!B=8/1?")\D2)
-MYDR)]^@`````N@$```")T$B+'"1(BVPD"$R+9"003(ML)!A,BW0D($B#Q"C#
-M9F9FD&9F9I!F9F:02(/L"(GQ#[;!2(M4QS"X`````$B%TG0FN`````"#NF`,
-M```!=!C'@F`,```!````0`^V]NAW\___N`$```!(@\0(PV9F9I!F9I!F9I!F
-M9I!(@^P(B?(/ML*Y`````$B#?,<P`'0@#[;"2(M$QS#'@&`,````````0`^V
-M]N@/]/__N0$```")R$B#Q`C#9F:04XGPB=8/ML!(BUS',+@`````2(7;="VX
-M`````(.[8`P```%T'[D`````N@(```!(B=_H&>K__TB)W^AQZ/__N`$```!;
-MPV9F9I!F9I!F9I")\`^VP$B+5,<PN/\```!(A=)T!P^V@D@"``#SPV9FD$B)
-M7"3H2(EL)/!,B60D^$B#[!B)\$&)U$B+;Q`/ML!(BU3',+D`````2(72=%*+
-M6CR-<RQ(B>_H`````$&#_`%U'KD`````J`)U-8US++H"````2(GOZ`````#K
-M'&9FD+D`````J`)T%XUS++H`````2(GOZ`````"Y`0```&:0B<A(BQPD2(ML
-M)`A,BV0D$$B#Q!C#9F9FD&9F9I!F9I!(B5PDV$B);"3@3(ED).A,B6PD\$R)
-M="3X2(/L*$F)_8GSB=5!B<R)\H/B`XG8@^`$B<'!Z0)`]L7X=09!]L3\=`ZX
-M`````.MZ9F9FD&9FD`^VPL'@"(V0=`$``(V"```#`('"```"`(3)08G&1`]$
-M\DB+?Q!$B?;H`````"0?B>K!X@6!XN`````)T(#DYT2)XL'B"X'B`!@```G0
-M28M]$(G"1(GVZ``````/ML-&B*0HK@```$*(K"BF````N`$```!(BQPD2(ML
-M)`A,BV0D$$R+;"083(MT)"!(@\0HPTB)7"3H2(EL)/!,B60D^$B#[!A(B?N)
-M\$&\`P```$$AQ(/@!(G%P>T"0(#]`1GV9KX``('&&``#`$B+?Q#H`````$$/
-MMM2-2ABZ`0```$C3X@G00(#]`1GV9KX``('&&``#`$B+>Q")PN@`````0(#]
-M`1GV9KX``('&&``#`$B+>Q#H`````+@!````2(L<)$B+;"0(3(MD)!!(@\08
-MPY!!54%455-(@^P(28G\08GUO0,```!$(>U`#[;VB?.#XP3!ZP+H=^C__X#[
-M`1GV9KX``('&&``#`$F+?"00Z`````!`#[;-@\$82,?"_O___TC3PB'0@/L!
-M&?9FO@``@<88``,`28M\)!")PN@`````@/L!&?9FO@``@<88``,`28M\)!#H
-M`````$`/MO7!Y@B!Q@@!``"-A@```P"!Q@```@"$VP]%\$F+?"00Z`````"#
-MX/"#R`%`#[;5P>((C8H(`0``C9$```,`@<$```(`A-N)U@]$\4F+?"00B<+H
-M`````$$/MO5,B>?HL>?__[@!````2(/$"%M=05Q!7<.005=!5D%505154TB#
-M[`A)B?Q(B=-!B?5,BW\0B?(/ML)(BVS',$`/MM:)T(/@`\'@"`5T`0``]L($
-M=`B-L````P#K!HVP```"`$F+?"00Z`````")0UA(A>UU$,<#`````+@!````
-MZ=,```!$BW4\00^V]4R)Y^@`````B0.X`0```(,[``^$LP```(N%8`P``(E#
-M+(N%1`(``(E#,`^VA4@"``"(0S2Y`````&9F9I")R@^V1"IVB$0:!/_!@_DG
-M=NY$B?9,B?_H`````(E#.$&-=A!,B?_H`````(E#/$&-=A1,B?_H`````(E#
-M0$&-=AA,B?_H`````(E#1$&-=AQ,B?_H`````(E#2$&-=B!,B?_H`````(E#
-M3$&-=B1,B?_H`````(E#4$&-=BA,B?_H`````(E#5+@!````2(/$"%M=05Q!
-M74%>05_#9F:02(/L2$B-1"0(QP0D`````(D0B4@(9D2)0`QF1(E(#HM4)%")
-M4!"+5"18B5`42(M4)&!(B5`82(M4)&A(B5`@2(GB0`^V]N@`````2(/$2,-(
-MB5PDT$B);"383(ED).!,B6PDZ$R)="3P3(E\)/A(@^PX2(D\)$F)UD&)]XGR
-M#[;"2(MLQS"X!````$B%[0^$80$``+@!````@[U@#`````^$3P$``+@"````
-M@+U(`@``'P^$/0$``/^-Z`P``(N%Z`P``$2+K(5H#```18GL2XT$I$C!X`1,
-MC:0H4`(``$6(;"0$08,^``^%M````$F-7@B#O40"```"=0G'0P0!````ZT,/
-MMTL,#[=S#HM3"(M]".AIT___B4,$@_@!=2B#?0@`=2*+A>@,``")PD2)K)5H
-M#```_\")A>@,``"X`P```.FN````@+UD#````'5"@[U``@```'4:00^VWXG>
-M2(L\).BH[/__B=Y(BSPDZ,WL__],B?),B>9(B>_HS^K__TF-5@A,B>9(B>_H
-M$.[__^M>3(GR3(GF2(GOZ+#J___K3DR)\DR)YDB)[^B@ZO__@+U(`@```74W
-M@[U``@```74-00^V]TB+/"3H4>W__TR)YDB)[^B&[O__A<!U$DR)YDB)[^CW
-MZO__N`$```#K!;@`````2(M<)`A(BVPD$$R+9"083(ML)"!,BW0D*$R+?"0P
-M2(/$.,-F9F:09F9FD$B)7"302(EL)-A,B60DX$R);"3H3(ET)/!,B7PD^$B#
-M[%A(B7PD($R+?Q"^8!T``$R)_^@`````B<5(BT0D((N`@````(E$)!RX````
-M`(5L)!P/A&$!``#WQ0``!`!T-+Y8'0``3(G_Z`````")PXG"]]*^6!T``$R)
-M_^@`````N0````")VKX`````2(M\)"#_5RC&1"0;`/?%_P$```^$^P```(!\
-M)!L!&=MFNP``@<,4``,`NN____^)WDR)_^@`````B=Y,B?_H`````$&)QHG"
-M]]*#RA")WDR)_^@`````@'PD&P$9]F:^``"!Q@@``P!,B?_H`````$&)Q$&]
-M``````^V3"0;B4PD%(E,)!!!#[;5N``!``")T=/@B<-$(?.X`0```-/@1"'P
-M08GI08/A`8G!1`G)=!M$B>&#X1]$B30D08G`BW0D%$B+?"0@Z.G8__^%VW0L
-MBUPD$(T$G0````!$">@/MOCHO^G__X5$)!QT$$$/MM6)WDB+?"0@Z,G;___!
-M[0)!P>P(0?_%08#]`P^&=____^L&9F:0P>T(T>W^1"0;@'PD&P$/AN7^__^X
-M`0```$B+7"0H2(ML)#!,BV0D.$R+;"1`3(MT)$A,BWPD4$B#Q%C#9F9FD$B#
-M[`A(BW\0N@````"^9!T``.@`````N`$```!(@\0(PV9F9I!F9F:09F9FD&9F
-MD$B#[`B+EX````!(BW\0OF0=``#H`````+@!````2(/$",-F9F:09F9FD&9F
-MD&9FD$B)7"382(EL).!,B60DZ$R);"3P3(ET)/A(@^PH28G^08G408G-NP``
-M```YTW,PC:X<`0``B>Y,B??H`````(3`>`VH`0^4P`^VP.L79F:01(GOZ```
-M``#_PT0YXW+6N`````!(BQPD2(ML)`A,BV0D$$R+;"083(MT)"!(@\0HPV9F
-M9I!F9F:09F9FD&9FD%-(B?N+=SR!QB`!``!(BT<P2(MX$+H`````Z`````"+
-M<SR!QB`!``!(BT,P2(MX$.@`````6\-F9I!F9I!F9I!32(G[BW<\@<8@`0``
-M2(M',$B+>!"Z`@```.@`````BW,\@<8<`0``2(M#,$B+>!#H`````%O#9F:0
-M9F:09F:0055!5%532(/L"$B)^TB+1S!,BV@01(MG/+T`````N`````"#OT`"
-M```!#X25````08VT)"`!``!,B>_H`````*B!=`R_,@```.@`````9I!!C;0D
-M(`$``$R)[^@`````J(%T+HGH_\5F/;<+=QP/MC-(BWLPZ`````"%P'1'O^@#
-M``#H`````.O&N`````"0ZS.Y9````+H*````1(GF3(GOZ`````!!C;0D'`$`
-M`$R)[^@`````)<$```"#^$`/E,`/ML!(@\0(6UU!7$%=PTB)7"3H2(EL)/!,
-MB60D^$B#[!B)\`^VP$B+7,<PN`````!(A=MT:4B+0S!,BV`0BVL\N`````"#
-MNV`,```!=%!(B=_H`````(VU'`$``+KA````3(GGZ`````"Y9````+H0)P``
-MB>Y,B>?H`````(7`=0](B=_H`````+@`````ZPY(B=_H`````+@!````D$B+
-M'"1(BVPD"$R+9"002(/$&,-F9F:09F:09F:09F:04TB#[$")\@^VPDB+1,<P
-MO@````!(A<`/A*$```!(C5A`#[;RQT0D,.P```#'1"0H`````,=$)"``````
-MQT0D&`````#'1"00`````,=$)`@`````QP0D`````$&Y``$``$F)V+D`````
-MN@$```#H`````+X`````A<!T0O:#H````.!T-+D`````28G8O@$```"`N_X!
-M``"E=2.Z``````^WPD("#`#_PF:!^O\!=O"^`````(3)=06^`0```(GP2(/$
-M0%O#9F9FD&9FD&9FD$B#[`B)\`^VP$B+?,<PN`````!(A?]T$[@`````@[]@
-M#````70%Z`````!(@\0(PTB#[#A`#[;VQT0D,.\```#'1"0H``````^V1"1`
-MB40D($4/MLE$B4PD&$4/ML!$B40D$`^VR8E,)`@/MM*)%"1!N0````!!N```
-M``"Y`````+H`````Z`````!(@\0XPTB)7"302(EL)-A,B60DX$R);"3H3(ET
-M)/!,B7PD^$B#[&A(B?M!B==!B<Y-B<5%B<R)\HML)'!$BTPD>$2+A"2`````
-MB[PDB````(N,))````!$#[:<))@```!$#[:4)*`````/ML*^`````$B#?,,P
-M`'1-#[;R00^VPHE$)#!!#[;#B40D*`^WP8E$)"`/M\>)1"0800^WP(E$)!!!
-M#[?!B40D"`^WQ8D$)$6)X4V)Z$2)\42)^DB)W^@`````B<:)\$B+7"0X2(ML
-M)$!,BV0D2$R+;"103(MT)%A,BWPD8$B#Q&C#9F9FD&9FD&9FD$B)7"302(EL
-M)-A,B60DX$R);"3H3(ET)/!,B7PD^$B![)@```")\(E4)%R)S4R)1"101(E,
-M)$Q$B[0DH````$2+O"2H````BY0DL````&:)5"0PBXPDN````&:)3"0@BY0D
-MP````&:)5"00#[:,),@```"(3"0/#[:4)-````"(5"0.#[;`2(M<QS!,BV<0
-M1(MK/$B)W^B5^___N@````"%P`^$B`(``$B)W^@`````@_T!=6Y,B?$/MM5!
-MC;4$`0``3(GGZ`````!,B?@/MM1!C;4(`0``3(GGZ`````"+3"0P#[;508VU
-M#`$``$R)Y^@`````BT0D(`^VU$&-M1`!``!,B>?H`````(M,)!`/MM5!C;44
-M`0``3(GGZ`````#K-[@`____9D2%\'4B9D2%^'4<9H5$)#!U%6:%1"0@=0YF
-MA40D$'019F9FD&9FD+H`````Z=8!``!!#[;608VU!`$``$R)Y^@`````00^V
-MUT&-M0@!``!,B>?H``````^V5"0P08VU#`$``$R)Y^@`````#[94)"!!C;40
-M`0``3(GGZ``````/ME0D$$&-M10!``!,B>?H``````^V5"0/08VU&`$``$R)
-MY^@`````#[94)`Y!C;4<`0``3(GGZ`````"#?"1<`'4]N60```"Z\+H$`$2)
-M[DR)Y^@`````A<!U$DB)W^@`````N@````#I&@$``$B)W^@`````N@$```#I
-M"`$``+ED````NO"Z!`!$B>Y,B>?H`````(7`=11(B=_H`````+H`````9I#I
-MVP```$&-M1P!``!,B>?H`````*@(=1)(B=_H`````+H`````Z;8```"]````
-M`#ML)$QS8D6-M0`!``!`A.UU&;ED````NE##``!$B>Y,B>?H`````(7`=&6#
-M?"1<`7481(GV3(GGZ`````")ZDB+3"109HD$4>L6B>A(BTPD4`^W%$%$B?9,
-MB>?H`````/_%.VPD3'*EN60```"Z4,,``$2)[DR)Y^@`````A<!U'DB)W^@`
-M````N@````#K(4B)W^@`````N@````#K$DB)W^@`````N@$```!F9I!FD(G0
-M2(M<)&A(BVPD<$R+9"1X3(NL)(````!,B[0DB````$R+O"20````2('$F```
-M`,-F9F:09F9FD&9F9I!F9I!!5%532(G[2(M',$B+:!!$BV<\08VT)"`!``"Z
-M!@```$B)[^@`````BW,\@<8@`0``2(GOZ`````"_"@```.@`````2(G?Z```
-M``"_]`$``.@`````O_0!``#H`````+_T`0``Z`````"_]`$``.@`````NP``
-M``!!@<0<`0``D$2)YDB)[^@`````A,!X![@!````ZQZ_]`$``&9FD&:0Z```
-M``#_PX'[#R<``';1N`````!;74%<PV9F9I!F9I!32(G[Z#?___^Z`0```(/X
-M`704O_0!``#H`````$B)W^@;____B<*)T%O#9F:09I!54TB#[`A(B?U(BP4`
-M````BW`8@<8```0`NJ````#H`````+L`````2(L%`````(MP$('&```$`$B)
-M[^@`````_\.#^P1^X;L`````9F9FD$B+!0````"+<!2!Q@``!`!(B>_H````
-M`/_#@_L%?N&[`````$B+!0````"+<!"!Q@``!`!(B>_H`````/_#@_L$?N%(
-M@\0(6UW#9F:09F:04TB)^TB+!0````"+<`R!Q@``!`"Z@````.@`````2(L%
-M`````(MP&('&```$`+JP````2(G?Z`````!;PV9FD$B)7"3P3(ED)/A(@^P8
-M28G\Z`K___](BP4`````BW`(@<8```0`N@````!,B>?H`````$B+!0````"+
-M,('&```$`$R)Y^@`````B<-,B>?H;/___P^VVXG82(M<)`A,BV0D$$B#Q!C#
-M9F9FD&9F9I!!5%532(G[OBP-``#H`````$&)Q(G"@\H!OBP-``!(B=_H````
-M`+[P!`$`2(G?Z`````")Q8/@8H/X8G05B>J#RF(/MM*^\`0!`$B)W^@`````
-MO@``!`!(B=_H`````$B)W^A)_O__2(L%`````(MP"('&```$`+H'````2(G?
-MZ`````!(BP4`````BS"!Q@``!`"Z$0```$B)W^@`````2(L%`````(MP"('&
-M```$`+H%````2(G?Z`````!(BP4`````BS"!Q@``!`"Z$0```$B)W^@`````
-M2(G?Z&_^__^)ZK[P!`$`2(G?Z`````"^\`0!`$B)W^@`````1(GBOBP-``!(
-MB=_H`````%M=05S#9F9FD&9F9I!F9I!F9I!!5%532(G[OBP-``#H`````$&)
-MQ(G"@\H!OBP-``!(B=_H`````+[P!`$`2(G?Z`````")Q8/@8H/X8G05B>J#
-MRF(/MM*^\`0!`$B)W^@`````O@``!`!(B=_H`````$B)W^@I_?__2(L%````
-M`(MP"('&```$`+H'````2(G?Z`````!(BP4`````BS"!Q@``!`"Z`````$B)
-MW^@`````2(L%`````(MP"('&```$`+H%````2(G?Z`````!(BP4`````BS"!
-MQ@``!`"Z`````$B)W^@`````2(G?Z$_]__^)ZK[P!`$`2(G?Z`````"^\`0!
-M`$B)W^@`````1(GBOBP-``!(B=_H`````%M=05S#9F9FD&9F9I!F9I!F9I!3
-M2(G[Z&?\__](BP4`````BW`(@<8```0`N@H```!(B=_H`````$B+!0````"+
-M,('&```$`+H`````2(G?Z`````!(B=_HQOS__[X!``0`2(G?Z`````"Z____
-M_Z@"=4U(B=_H"/S__TB+!0````"+,('&```$`+H!````2(G?Z`````!(B=_H
-MA/S__[]D````Z`````"^`0`$`$B)W^@`````T>B#X`&#^`$9THG06\-F9F:0
-M9F9FD&9F9I!F9I!(B5PDV$B);"3@3(ED).A,B6PD\$R)="3X2(/L*$F)_$&)
-M]DB+7Q"^+`T``$B)W^@`````08G%B<*#R@&^+`T``$B)W^@`````OO`$`0!(
-MB=_H`````(G%@^!B@_AB=!6)ZH/*8@^VTK[P!`$`2(G?Z`````"^```$`$B)
-MW^@`````18BT)*4```!!#[;6O@#@!0!(B=_H`````(GJOO`$`0!(B=_H````
-M`+[P!`$`2(G?Z`````!$B>J^+`T``$B)W^@`````2(L<)$B+;"0(3(MD)!!,
-MBVPD&$R+="0@2(/$*,-F9F:09F9FD&9F9I!(@^P(B?&$TG06N`$```#3X`B'
-MI0```.L49F9FD&9FD+C^____T\`@AZ4````/MK>E````Z`````!(@\0(PV:0
-M05=!5D%505154TB#[`A(B?U!OP````"^+`T``.@`````08G&B<*#R@&^+`T`
-M`$B)[^@`````OO`$`0!(B>_H`````$&)Q8/@8H/X8G061(GJ@\IB#[;2OO`$
-M`0!(B>_H`````+X```0`2(GOZ`````"^```$`$B)[^@`````9CU5J@^%[P``
-M`$&\`````&:0NP````!$.>-_*F9FD&9FD$B+!0````"+<!B!Q@``!`"ZH```
-M`$B)[^@`````_\-$.>-^W$&!_,@```!^"K\!````Z`````"^```$`$B)[^@`
-M````B<-(BP4`````BW`8@<8```0`NK````!(B>_H`````&:!^U6J=1A!_\1!
-M@?S'````#XYV____9H'[5:J0=%5(BP4`````BW`8@<8```0`NK````!(B>_H
-M`````+X```0`2(GOZ`````!F/56J=25!OP$```!(B>_H`/K__SP^=1-(B>_H
-MI/S__X7`N`(```!$#T3X2(L%`````(MP&('&```$`+JP````2(GOZ`````!$
-MB>J^\`0!`$B)[^@`````OO`$`0!(B>_H`````$2)\KXL#0``2(GOZ`````!!
-M#[;'2(/$"%M=05Q!74%>05_#9F9FD&9F9I!F9F:055-(@^P(2(G]NP````!F
-MD$C'!0``````````2(GOZ/W]__^$P'4A2,<%``````````!(B>_HYOW__X3`
-M=0K_PX'[YP,``'[(#[;`2(/$"%M=PP````````````````````0`````````
+M<$B)[^CNT?__QD`(`42(>!!$B'@12(L32(E0%$B+4PA(B5`<28M5`/Z"L`D`
+M``^VLK`)``")\4@IT$B#Z%!(P?@#20^OQHB$2K`)``!!#[9%"XB$2K$)``!!
+MB'4+1(GY#[;!2,'@!$B-A"CP"@``N@````"#.`!U$H-X!`!U#(-X"`!U!H-X
+M#`!T!;H!````A=)T>D2)^`^VV$B)V$C!X`1(C9PH\`H``$B)[^A&T?__QD`(
+M`D2(>!!$B'@12(L32(E0%$B+4PA(B5`<28G$2(L0_H*P"0``#[:RL`D``(GQ
+M2(M$)"A(*=!(@^A02,'X`TD/K\:(A$JP"0``00^V1"0+B(1*L0D``$&(="0+
+M0?_'1#A]2@^'X/W__^DZ#```#[9=2TB)[^C+T/__QD`(`8A8$(A8$4B+5"1@
+M2(E0%$B+5"1H2(E0'$B)1"0X#[9=2TB)[^B>T/__QD`(`HA8$(A8$4B+5"1@
+M2(E0%$B+5"1H2(E0'$F)Q$B)[^AWT/__QD`(!DB)1"0H28L,)/Z!L`D```^V
+MN;`)``")_DB+1"0H2"G(2(/H4$C!^`-(NLW,S,S,S,S,2`^OPHB$<;`)``!!
+M#[9$)`N(A'&Q"0``08A\)`M!OP````"`?4H`#X9W"P``9F9FD$0X?4L/A,8"
+M``!$B?H/ML)(P>`$2`'H2(V0\`H``+D`````@[CP"@```'46@WH$`'40@WH(
+M`'4*@WH,`&9F9I!T!;D!````A<D/A($"``#^3"0G#[9,)"<"34I!B<Y$B?M$
+M#[;C2<'D!$D![$F-O"3P"@``2(E\)!!(B>_HC\___\9`"`%$B'@01(AP$4F+
+ME"3P"@``2(E0%$B+3"002(M1"$B)4!Q(B40D0`^V74M(B>_H6<___\9`"`2(
+M6"!$B'`A28N4)/`*``!(B5`02(M<)!!(BU,(2(E0&$F)Q4B+$/Z"L`D```^V
+MLK`)``")\4B+1"0X2"G02(/H4$C!^`-(O\W,S,S,S,S,2`^OQXB$2K`)``!!
+M#[9%"XB$2K$)``!!B'4+28M5`/Z"L`D```^VLK`)``")\4B+1"1`2"G02(/H
+M4$C!^`-(#Z_'B(1*L`D``$$/MD4+B(1*L0D``$&(=0M(BT0D*$B+$/Z"L`D`
+M``^VLK`)``")\4R)Z$@IT$B#Z%!(P?@#2`^OQXB$2K`)``!(BUPD*`^V0PN(
+MA$JQ"0``0(AS"P^V74M(B>_H7<[__\9`"`2(6"!$B'@A28N4)/`*``!(B5`0
+M2(M\)!!(BU<(2(E0&$F)Q4B+$/Z"L`D```^VLK`)``")\4B+1"0X2"G02(/H
+M4$C!^`-(N\W,S,S,S,S,2`^OPXB$2K`)``!!#[9%"XB$2K$)``!!B'4+2(M\
+M)"A(BQ?^@K`)```/MK*P"0``B?%,B>A(*=!(@^A02,'X`T@/K\.(A$JP"0``
+M#[9'"XB$2K$)``!`B'<+2(GOZ*C-___&0`@"1(AX$$2(>!%)BY0D\`H``$B)
+M4!1(BTPD$$B+40A(B5`<28G$2(L0_H*P"0``#[:RL`D``(GQ2(M$)"A(*=!(
+M@^A02,'X`T@/K\.(A$JP"0``00^V1"0+B(1*L0D``$&(="0+D$'_QT0X?4H/
+MAR/]___ID0@```^V14PZ14L/A:@```!!OP````"`?4H`#X9T"```D$0X?4MT
+M?T2)^P^VPTC!X`1(`>A(C9#P"@``N0````"#N/`*````=1B#>@0`=1*#>@@`
+M=0R#>@P`=`MF9I!F9I"Y`0```(7)=#Q$B?@/MMA(P>,$2`'K3(VC\`H``$B)
+M[^BKS/__QD`(`D2(>!!$B'@12(N3\`H``$B)4!1)BU0D"$B)4!Q!_\=$.'U*
+M#X=N____Z=P'``!(C7PD8.A/R___2(GOZ&?,___&0`@&2(E$)"A!OP````"`
+M?4H`#X8;`0``1#A]2P^$!`$``$2)^@^VPDC!X`1(`>A(C9#P"@``N0````"#
+MN/`*````=12#>@0`=0Z#>@@`=0B#>@P`9I!T!;D!````A<D/A,$```!$B?D/
+MML%(P>`$3(TD*$F-G"3P"@``BT0D8$$+A"3P"@``B40D8(M$)&0+0P2)1"1D
+MBT0D:`M#"(E$)&B+1"1L"T,,B40D;$0X?4QT=$B)[^BLR___QD`(`D2(>!!$
+MB'@128N4)/`*``!(B5`42(M3"$B)4!Q)B<1(BPC^@;`)```/MKFP"0``B?Y(
+MBT0D*$@IR$B#Z%!(P?@#2+K-S,S,S,S,S$@/K\*(A'&P"0``00^V1"0+B(1Q
+ML0D``$&(?"0+0?_'1#A]2@^'Y?[__P^V74M(B>_H)\O__\9`"`*(6!"(6!%(
+MBU0D8$B)4!1(BU0D:$B)4!Q)B<1(BPC^@;`)```/MKFP"0``B?Y(BT0D*$@I
+MR$B#Z%!(P?@#2+K-S,S,S,S,S$@/K\*(A'&P"0``00^V1"0+B(1QL0D``$&(
+M?"0+2`^V=4Q(B?=(B?!(P>`$2(VT*'`+``"+!O?0B40D4(M6!/?2B50D5(M.
+M"/?1B4PD6(MV#/?61(M,)&!$(<B)1"101(M$)&1$(<*)5"14BU0D:"'1B4PD
+M6(M$)&PAQHET)%Q(P><$2(V\+W`+``!$(P]$B4PD8$0C1P1$B40D9"-7"(E4
+M)&@C1PR)1"1LN`````!%A<EU$$6%P'4+A=)U!X-\)&P`=`6X`0```(7`D`^$
+M#P(```^V14N(A0@,``!(BT0D8$B)A?`+``!(BT0D:$B)A?@+``#&A0D,```!
+M2,>%``P```````!!OP````"`?4H`#X;+`0``3(VU\`L``$F\S<S,S,S,S,QF
+M9I!FD$0X?4QU6DB)[^BBR?__QD`(`TR)<!!$B'@82(M<)"A(BQ/^@K`)```/
+MMK*P"0``B?%(*=!(@^A02,'X`TD/K\2(A$JP"0``#[9#"XB$2K$)``!`B',+
+MZ4T!``!F9I!FD$0X?4L/A#X!``!$B?H/ML)(P>`$2(U<!'!(`>A(C;AP"P``
+MB[!P"P``]]:),XM/!/?1B4L$BU<(]]*)4PB+1PSWT(E##"-T)&"),R-,)&2)
+M2P0C5"1HB5,((T0D;(E##$B)[^CHR/__QD`(`TR)<!!$B'@828G%2(M,)"A(
+MBQ'^@K`)```/MK*P"0``B?%(*=!(@^A02,'X`TD/K\2(A$JP"0``2(M\)"@/
+MMD<+B(1*L0D``$"(=PNX`````(,[`'42@WL$`'4,@WL(`'4&@WL,`'0%N`$`
+M``"%P'1K1(GZ#[;"2,'@!$B-7`1P2(GOZ%S(___&0`@!1(AX$$2(>!%(BQ-(
+MB5`42(M3"$B)4!Q)BU4`_H*P"0``#[:RL`D``(GQ2"G02(/H4$C!^`-)#Z_$
+MB(1*L`D``$$/MD4+B(1*L0D``$&(=0M!_\=$.'U*#X=+_O__N`````"#?"10
+M`'45@WPD5`!U#H-\)%@`=0>#?"1<`'0%N`$```"%P`^$,`,```^V74M(B>_H
+MP<?__\9`"`&(6!"(6!%(BU0D4$B)4!1(BU0D6$B)4!Q(B40D.$C'1"0P````
+M`$&_`````(!]2@`/ANH"``!F9F:09F:01#A]3`^$S`(``$0X?4L/A,("``!$
+MB?D/ML%(P>`$2(UT!'!(`>A(C8CP"@``BU0D4".0\`H``(D6BT0D5"-!!(E&
+M!(M$)%@C00B)1@B+1"1<(T$,B48,N`````"#/@!U$H-^!`!U#(-^"`!U!H-^
+M#`!T!;@!````A<`/A%8"``!$#[9E3$2)^P^VPTC!X`1(C5P$<$B)[^CDQO__
+MQD`(`42(>!!$B&`12(L32(E0%$B+4PA(B5`<2(E$)$!(@WPD,`!T2TB+$/Z"
+ML`D```^VLK`)``")\4B+1"0P2"G02(/H4$C!^`-(O\W,S,S,S,S,2`^OQXB$
+M2K`)``!(BUPD0`^V0PN(A$JQ"0``0(AS"T0/MF5+#[9]3$"(?"0/1(GX1`^V
+M\$G!Y@1*C5PT<$B)[^A+QO__QD`(!$2(8"`/ME0D#XA0(4B+$TB)4!!(BU,(
+M2(E0&$F)Q4B)1"0P2(L0_H*P"0``#[:RL`D``(GQ2(M$)#A(*=!(@^A02,'X
+M`TB[S<S,S,S,S,Q(#Z_#B(1*L`D``$$/MD4+B(1*L0D``$&(=0M)BU4`_H*P
+M"0``#[:RL`D``(GQ2(M$)$!(*=!(@^A02,'X`T@/K\.(A$JP"0``00^V10N(
+MA$JQ"0``08AU"TB+?"0H2(L7_H*P"0``#[:RL`D``(GQ3(GH2"G02(/H4$C!
+M^`-(#Z_#B(1*L`D```^V1PN(A$JQ"0``0(AW"P^V74M.C60T<$B)[^A/Q?__
+MQD`(!(A8($2(>"%)BQ0D2(E0$$F+5"0(2(E0&$F)Q4B+$/Z"L`D```^VLK`)
+M``")\4B+1"0X2"G02(/H4$C!^`-(N\W,S,S,S,S,2`^OPXB$2K`)``!!#[9%
+M"XB$2K$)``!!B'4+2(M\)"A(BQ?^@K`)```/MK*P"0``B?%,B>A(*=!(@^A0
+M2,'X`T@/K\.(A$JP"0``#[9'"XB$2K$)``!`B'<+0?_'1#A]2@^''?W__TB)
+M[DB+?"1(Z,#:__^X`````$B!Q'@!``!;74%<05U!7D%?PV9FD&9FD&9FD$%7
+M059!54%455-(@^P828G]@[_<60```P^'$@$``$B-AV!9``!(.8=@60``#X3^
+M````28G'3(NW8%D``$B-AU!9``!(B40D$$R)?"0(9F:03(GS38LV2(M3"$F)
+M5@A,B3)(B1M(B5L(2(M#./9``01U?DF)W+T`````@'M*`'8?B>@/MOA(P><$
+M2HV\)_`*``#HNL+____%03AL)$IWX4F+A5!9``!(B5@(2(D#2(M$)!!(B4,(
+M28F=4%D``$B+0SA(@WA(`'182(M04$B+<$A,B>_H`````$B+0SA(QT!(````
+M`$'_C>!9``#K,V9FD$B)WDR)[^CEV___A<!T,4F+A6!9``!(B5@(2(D#2(M$
+M)`A(B4,(28F=8%D``&9FD$TY_@^%)____V9F9I!F9I!(@\086UU!7$%=05Y!
+M7\.055-(@^P(2(M?$(.[W%D```!U$$B-@V!9``!(.8-@60``=#=(C:M@60``
+M9F:09F:02(G?Z`````!(B=_H`````$B)W^@`````@[O<60```'7?2#FK8%D`
+M`'76N`````!(@\0(6UW#9F9FD&9FD&9FD%-,BT\028N)8%D``$F-@6!9``!(
+M.<@/A'X```!)B<*02#EY.'5L28G(NP````"`>4H`=D(/ML-(P>`$3`'`2(V0
+M\`H``+X`````@[CP"@```'42@WH$`'4,@WH(`'4&@WH,`'0%O@$```"%]G4;
+M_\-!.%A*=[ZX`````(7`=!.X`0```.FX````N`$```!FD.OI2(L)23G*=89)
+MBXEP60``28V!<%D``$@YR`^$BP```$F)P4B-0?!(.7@X=7)(B<:[`````(!X
+M2@!V2&9F9I`/ML-(P>`$2`'P2(V0\`H``$&X`````(.X\`H```!U$H-Z!`!U
+M#(-Z"`!U!H-Z#`!T!D&X`0```$6%P'47_\,X7DIWO+@`````A<!T$[@!````
+MZQVX`0```.ON9F:09I!(BPE).<D/A7C___^X`````%O#9F9FD&9FD&9FD&9F
+MD$B#["A(B5PD"$B);"003(ED)!A,B6PD($B)^TF)]4F)U$B+;Q!(@W](`'4)
+MZ'_^__^%P'4-3(GF2(GO0?_59I#K%DR):TA,B6-0_X7@60``2(GOZ`````!(
+MBUPD"$B+;"003(MD)!A,BVPD($B#Q"C#9F9FD&9F9I!F9F:09F:055-(@^P(
+M2(G]2(GS@'XA`'4$QD8A`0^V0R$\`0^5PCP&#Y7`#[;2A=!T%8![(0-T#_9#
+M("!U"<9#(0MF9I!FD$B+<U!(B>_HI+___TB+<SA(B=I(B>_H`````$B+`TB#
+M>#``=!M(C5`P2,?&`````$B)[^@`````ZUQF9I!F9I!(@[WH60```'0H2(V%
+M4%D``$@YA5!9``!T&$B-E>A9``!(Q\8`````2(GOZ`````#K)(&]V%D``/\`
+M``!W&$B-A5!9``!(.85060``=0A(B>_H`````$B#Q`A;7<-F9I!!5T%6055!
+M5%532('L6`$``$B)="0X28G4B4PD-$R+1D!)BP!(B40D"`^V4'C_RHE4)!Q!
+M#[=`$&:)1"0R28N0D````$B)5"0H10^VL(H```!!#[:`BP```(E$)!A%#[>X
+MF@```$$/MYB8````28-X*`!T#69!#[9`((/@`3G(=3*Z`````$F#>#``#X0F
+M`@``3(UL)$"+3"0T3(GJ3(G&0?]0,+H`````A<`/A`8"``#K!$V+:"B]````
+M`$B+5"0X2(L"#[9``XE$)!2Y``````^WPT@#1"0H2(E$)"!$.W0D%`^%$@$`
+M`$B%R715BT0D("G(B</!XPET2(-\)#0`=`E(BP4`````ZPQ(BST`````Z```
+M``!)B00D9D''1"0*``"!^P`0```/AKD```!F0<=$)`@`$$F#Q!"!ZP`0``!U
+MN$$/M\>)P\'C"69F9I!!#[=5"(72N````0`/1-")T"GH.=AR+8GH20-%`$F)
+M!"1F08E<)`AF0<=$)`H``$F#Q!`!W3G5=3V]`````$F#Q1#K,HGH20-%`$F)
+M!"2)T&8IZ&9!B40D"&9!QT0D"@``28/$$(G0*>@IPTF#Q1"]`````.N*00^W
+MST@#3"0@2(M4)#A(#[="$$@#0@A(.<@/A,0```#K2&9!B5PD"$F#Q!#I3/__
+M_P'=ZS5!#[??P>,)00^W50B%TK@```$`#T30B=`IZ#G8=]Z)T"GH*<-)@\40
+MO0````#KV&9FD&9FD&9$*7PD,G1RNP````!!_\9$.W0D&`^4P`^VP$$!QDB+
+M5"0(#[9">$0Y\'4JBT0D&/_(@WPD&``/1$0D'(E$)!B%P`^4P$0/MO!(#[>"
+MD````$@!1"0H2(M$)`A$#[>XD````&9$.WPD,@^&,O[__T0/MWPD,NDG_O__
+M9D''1"3Z`("Z`0```(G02('$6`$``%M=05Q!74%>05_#9F:09I!(@^P82(E<
+M)`A(B6PD$$B)_4B+7D"`?B$!=`</MD8AB$,A2(GOZ``````/MH.)````_\"(
+M@XD````Z@X@```!U,H![(0!U!,9#(0$/MD,A/`$/E<(\!@^5P`^VTH70=`J`
+M>R$#=`3&0R$+2(G>2(GO_U,X2(M<)`A(BVPD$$B#Q!C#9F9FD&9F9I!F9I!F
+M9I!!5T%6055!5%532(/L&$B)?"0028GU2(L&2(E$)`CV1B`$#X3H`0``0;X`
+M````@+Z```````^$8@$``$ECQDV+1,5800^VJ&`,``!!.*AA#```#X(Q`0``
+M03AH2P^$&`$``(GJ#[;"2(G"2,'B!$Z-E`+P"@``00^WE(!`#```00^WO(!"
+M#```NP````!!NR````!!O`$```!F9F:09F:0@_H?=S\/ML-)C32"1(G9*=$Y
+M^0]'SX/Y('4(QP;_____ZPU$B>#3X/_(B='3X`D&1(G8*=`Y^',5C7P7X+H`
+M````ZP.#ZB#_PX#[`W:RB>L/ML-(B<)(P>($3HV4`G`+``!!#[>4@$`,``!!
+M#[>\@$(,``"[`````$&[(````$&\`0```(/Z'W=&#[;#28TT@D2)V2G1.?D/
+M1\^#^2!U",<&_____^L-1(G@T^#_R(G1T^`)!D2)V"G0.?AS'(U\%^"Z````
+M`.L*9F9FD&9FD(/J(/_#@/L#=JO_Q4$XJ&$,```/@\_^__]!_\9!#[:%@```
+M`$0Y\`^/GO[__TB+7"00@[M(60````^$B0,``$&^`````$&`O8``````=#](
+MB=I(@<)@60``26/&38M$Q5A)QX`P#````````$B+0@A,B4((28D028E`"$R)
+M`$'_QD$/MH6`````1#GP?\M,B>Y(BWPD$.BU^?__Z5H#``#V1B`"#X0>`P``
+M2(M$)!!(!?!9``!(B00D3(M\)!!)@<<P6@``2(M4)`CV@I(````##X7Q`@``
+MNA````"^`````$R)_^@`````0;X`````08"]@``````/A/T```!)8\9-BT3%
+M6$R)Q[D`````08!X2@!V2`^VP4C!X`1(`?A(C9#P"@``O@````"#N/`*````
+M=1*#>@0`=0R#>@@`=0:#>@P`=`6^`0```(7V=`>X`0```.L,_\$X3TIWN+@`
+M````A<`/A5\"``!!#[:H8`P``$$XJ&$,``!R;$$X:$MT6XGI#[;19D&#/%<`
+M=25)#[>$D$`,``!)`T!`2(L<)$B)!--!#[>$D$(,``!F08D$5^LIB>@/MM!!
+M#[>$D$`,``!F00-`0&9!`X200@P``$B+#"1F*P319D&)!%?_Q4$XJ&$,``!S
+ME$'_QD$/MH6`````1#GP#X\#____28MU4$B+?"00Z#BX__]!O@````!!@+V`
+M``````^$]@```$B+3"002('!4%D``$ECQDV+1,58187V=6I)BT!`28F%D```
+M`$$/MH!@#```08B%B@```$$/MD!+08B%BP```$D/MH!@#```00^WA(!`#```
+M9D&)A9@```!)#[:`8`P``$$/MX2`0@P``&9!B86:````0<:%B`````!!QH6)
+M`````.M-28M00$D[E9````!V0$B+7"0(#[>#D````/?82)A((<))(X60````
+M2#G"=2%)#[:08`P``$$/MX6:````9D$#A)!"#```9D&)A9H```!(BT$(3(E!
+M"$F)"$F)0`A,B0!!_\9!#[:%@````$0Y\`^/%O___TB+1"002(.XZ%D```!T
+M&4B)PDB!PNA9``!(Q\8`````2(G'Z`````"]`````$B+5"0(@'IX``^&K```
+M`(GI1`^VX69#@SQG`'1=2(M$)`A*BYS@H````$B+?"00Z`````!(B<9!_H6(
+M````QD`2($,/MP1G9HE&$$B+%"1*BP3B2(E&"(!.(`)(QT8P`````$C'1C@`
+M````2(D>3(EN0$B+?"00_U-H_\5(BTPD"$`X:7AWB.LR0;X`````08"]@```
+M``!T(DECQDF+=,582(M\)!#HQL___T'_QD$/MH6`````1#GP?]Y(@\086UU!
+M7$%=05Y!7\-F9F:09F:005=!5D%505154TB#[%A(B7PD2$F)]TB)5"1`2(L&
+M2(E$)#B^``````^WD)````!F@?J!`+B`````#T/09HE4)"!!]D<@('1$#[?*
+M2(M<)$!(BU,02(MT)#CHDKG__TB)QKC_____2(7V#X2C`P``0<:'@`````%)
+MB7=83(F^,`P``+@`````Z88#``!(BT0D0$@/ME`B2(G!2(L`2(G3N@````!(
+M]_-!B=9(BT0D.$@/ME!X2(M!"$B)T;H`````2/?Q2(M<)$`/MD,B*-"(1"0?
+M03C&08#>_TB+1"0X#[>0D````(U"_P^W2Q@AP4&)U&9!*<QF1#EC(',%1`^W
+M8R!!QH>``````$'&AX$`````QT0D&`$````/MT0D(/?82)A(B40D$`^WP4F)
+MQ4B+5"1`3`-J$$B+3"1(2('!8%D``$B)3"0(2(M<)$A(@<-060``2(D<)(M$
+M)"#_R$0AZ(M<)"!F*<-F1#GC00]'W$B+;"003"'M@WPD&``/A'0!``#'1"08
+M`````(M4)"!(BT0D.&8YD)````!V++H`````00^VAX````")P8/X`'X82&/"
+M28MTQUA(.6Y`#X0T`0``_\(YT7_H#[=,)"!(B>I(BW0D.$B+?"1(Z!&X__](
+MB<9(A<`/A=T```!!_H^`````08"_@````/\/A+X```!,BU0D"$R+#"1)#[:'
+M@````$F+=,=82,>&,`P```````!)B?"Y`````(!^2@!V1)`/ML%(P>`$3`'`
+M2(V0\`H``+\`````@[CP"@```'43@WH$`'4-@WH(`'4'@WH,`)!T!;\!````
+MA?]U)?_!03A(2G>]N`````"%P'0;28M""$F)<@A,B19(B48(2(DPZQFX`0``
+M`.OA28M!"$F)<0A,B0Y(B48(2(DP0?Z/@````$&`OX````#_#X5+____N/__
+M___I2@$``$$/MH>`````#[;028ETUUC_P$&(AX````!,B;XP#```QH9@#```
+M",:&80P```!$B?$/MM%$B>AF*>AFB8260`P``&:)G)9"#```1#BV8`P``'8'
+M1(BV8`P``$0XMF$,``!S!T2(MF$,``!(BT0D0&8I6"!F02G<=!,/M\-)`<7'
+M1"08`0```.D#_O__2(M4)$!F@WH@``^$J0```+D`````0?_&1#IT)!\/E,!!
+M`<9(BUPD.$0X<WAU0X!\)!\`=`;^3"0?ZPU(BT0D0`^V0"*(1"0?@'PD'P!!
+M#Y3&2(M4)#A(#[>"D````$B+7"1`2`%#$,=$)!@!````ZQZ+5"0@2(M$)#AF
+M.9"0````N`$````/1D0D&(E$)!A(BUPD0$0/MV,@2(M$)#AF1#N@D`````^&
+M)_W__T0/MZ"0````Z1K]__^X`````$B#Q%A;74%<05U!7D%?PV9F9I!F9F:0
+M9F9FD&9FD$B#[`B%TG08QD8A"TB)\DC'Q@````#H`````.L(9F:0Z"OV__](
+M@\0(PV9FD&9FD$%455-(@^PP28G\2(GU2(L>]D,!!'49QD8A`DB+=CA(B>KH
+M`````.ED`@``9F9FD$B#>U@`=2F^!P```&9F9I#H2Z[__TB)QTB)0UBZ`!``
+M`+X`````Z`````!F9I!FD`^V12"H(`^%U````*@&=1C&12$!2(MU.$B)ZDR)
+MY^@`````Z0@"``#V12`$="=(@WM(`'0@2(M34$B+<TA,B>?H`````$C'0T@`
+M````0?^,).!9``!(BWT(2(E\)!@/MT409HE$)"`/MD-X_\B(1"0B#[9+>DB)
+M^$C3Z$B)!"1(#[9T)"*Z`````$CW]DB)1"0(#[9+>DC3X$B)1"00#[>3D```
+M``^V3"0B2`^W1"0@2`'X2`^O="002"GP#Z_12(U$$/](B=&Z`````$CW\4@/
+MMY.0````#Z_0ZSAF9F:0]H.2`````708QD4A!DB+=3A(B>I,B>?H`````.DO
+M`0``#[>3D````(G02`^O10A(B40D$(G12(M4)!!(`=%(B=Y,B>?HM[#__TB)
+M15!(A<!U'DC'15``````2(U[,$B)[N@`````Z>4```!F9I!FD$B)XDB)[DR)
+MY^CB^?__A<`/A(H```!(BW503(GGZ/ZO__]!BXPDN$0``$$YC"2\1```=$9(
+M8\%(P>`$2H&\()@B````````=''_P4ACP4AIP/$`#_!(P>@@C00!B<;!_@F)
+MR)F)\"G0:<`B`@``*<%!.8PDO$0``'6Z2,=%4`````!)C;PDZ%D``$B)[N@`
+M````3(GGZ`````#K1&9F9I#V12`$="E(Q\(`````2(GN3(GGZ/BS___K)L9%
+M(05(BW4X2(GJ3(GGZ`````#K$4B)[DR)Y^B6\___9F:09F:02(/$,%M=05S#
+M9F9FD&9FD%532(/L"$B)_4B)\TB#?E@`=%%(B??H`````+D`````2&/12(M#
+M6$B+!-!(A<!T$4C'0"``````2(M`*$B%P'7O_\&!^?\!``!VU4B+<UBZ!P``
+M`$B)[^BTJ___2,=#6`````!(@\0(6UW#9F:09I!!5%532(G[QX=(60``````
+M`$B-AU!9``!(B8=060``2(F'6%D``$B-AV!9``!(B8=@60``2(F':%D``$B-
+MAW!9``!(B8=P60``2(F'>%D``(L%`````,'@#(G"N*NJJJI(#Z_"2,'H($&)
+MQ$'![`]$B>*^:`P``.A$J___B<5!C50D(+X@````2(G?Z#"K__\!Q8L5````
+M`,'B#,'J#KZ(````2(G?Z!6K__\!Q;H(````O@`0``!(B=_H`:O__P'%2(G?
+MZ`````!(B04`````_\4[+0````!S-V9FD&9FD$B)W^@`````2(G"2(7`="%(
+MBX/060``2(D"2(F3T%D``/^#V%D``/_%.RT`````<L];74%<PV9F9I!F9I!F
+M9I!54TB#[`A(B?5(BUX(#[9.`[@!````B<+3XH!F`?M(QX.8`````````$BX
+M```"`/__``!((X.0````2#T```(`=!-F@[N4`````'0T9H63E````'4K9@F3
+ME````(!C`?M(@WMP`'0)2(G>_U-P9F:0@*.2````'^G4````9F9FD`^V@Y(`
+M``!FB9.4````_DMY2,>#F`````````"#R`6(@Y(```"$P'D.@^!_B(.2````
+MZ9D```"`HY(```"?2(G>Z`````!(B<)(A<!T8,8``P^V10.(0@-(B5H(@$H!
+M"$B+!0````!(B4)H2(L%`````$B)0G!(QT4(`````$@/MD(#2(F4PZ`````/
+MMH.2````@^#^@\@*B(.2````_D-Y2(G6OP<```#H`````$B#>P@`=`M(BUL(
+M2(-["`!U]>@`````B4-\2(G?Z`````!(@\0(6UW#9F:09F:09F:0N@````#_
+MSH/^_W039F9FD`^V!P'"2/_'_\Z#_O]U\0^VPL-F9F:09F:09F:09F:055-(
+M@^P(2(G[@#\#=EN]`````(!_>`!T1V9F9I!F9I!(8_5(@[SSH`````!T*$B+
+MC/.@````#[93`8/B`0^V00&#X/X)T(A!`4B+O/.@````Z`````#_Q0^V0W@Y
+MZ'_`@*.2````^^L'Z!(```!FD$B#Q`A;7<-F9I!F9I!F9I!!54%455-(@>P(
+M`@``28G\0;T`````2(GE@']X``^$6`(``$B+7PA(A=MT#DB#>P@`=`=)B=U(
+MBUL(N@`"``"^`````$B)[^@`````QT4`\Q9X6K@`````2(7;=`.+0WR)101!
+M@+PD@`````!T$(!-"0)!#[:$)(,```"(10I!#[94)'R^`0```"'6#[:%DP``
+M`(/@_(G1@^$""?`)R+X$````(=:#X/.)T8/A"`GP"<B^$````"'6@^#/B=&#
+MX2`)\`G(B=&#X4"#X#^#XH`)R`G0B(63````0?9$)`$!=`2`30D!N`````!(
+MA=MT"4F#_0$9P(/``HA%"TB%VP^$O@```$B+0QB)10R+0QR)A90````/M@.(
+M11`/MD-XB$41387M=`=!#[9%`^L&00^V1"0#B$42#[9#>HA%$_:#D@````)T
+M!(!-%`$/MX.4````9HE%%DB+@Y@```")11B+@YP```")A9@```!(C;/@````
+M2(U]++H0````Z`````!(C;/P````2(U]/+H$````Z`````!(C;,T`0``2(V]
+M@````+H0````Z`````!(C;/T````2(U]0+I`````Z`````!-A>UT8TF+11B)
+M11Q!BT4<B86<````00^V10"(12!!#[9%>(A%(4$/MD0D`XA%(D$/MD5ZB$4C
+M0?:%D@````)T!(!-)`%!#[>%E````&:)129)BX68````B44H08N%G````(F%
+MH````$B%VW4(00^V!"2(11"^D````$B)[^@`````]]B(10B^``(``$B)[^@`
+M````]]B(A9````!!B[0DD````(/N"DF-?"1X2(GINC````#H`````$B!Q`@"
+M``!;74%<05W#9F9FD&9FD$B)^(`_`W8XN@````"`?W@`=!@/MD]X2&/"2(.\
+MQZ``````=0S_PCG1?^RX`````,-(B[S'H````.@`````9I#SPV9F9I!F9F:0
+M9F:09F:055-(B?N`/P-V-KT`````@']X`'0O2&/%2(.\PZ``````=`U(B[S#
+MH````.@`````_\4/MD-X.>A_V^L(9F9FD(!G`?Y;7<-F9I!F9I!F9I!(BT\0
+MN@````!F9F:09F:0B=!(.3S!=0E(QP3!`````,/_PH/Z!W;H\\-F9I!F9I!(
+MB?>`9@'[Z`````#SPV:0055!5%532('L"`0``$F)_$B-G"0``@``2(G]#[9'
+M`:@$#X0O`0``@#\"#X8F`0``@^#]B$<!N@`"``"^`````$B)W^@`````08`\
+M)`,/A(L```!(B>!(BZV@````08`\)`AU<$B%[70,9D&#O"24`````75?2(G%
+M0;T!````00^V1"1X@_@!?FM)8\5)B[S$H````$B#QWA(B>FZ(````+X`````
+MZ`````"Y`````)!(8]&+!),S1)4`B023_\&#^7]^[$'_Q4$/MD0D>$0YZ'^W
+MZR"`?0`##X5X____2(U]>$B)V;H@````O@````#H`````&:!N_X!``!5JG57
+M@[O*`0```'0=@7L&3$E,3W0,@;MV`0``1U)50G4(08!,)`$"ZS%!O0````!)
+M8\5(P>`$2`'8@+B^`0``@'4/@[C*`0```'0&08!,)`$"0?_%08/]`W[52('$
+M"`0``%M=05Q!7<-F9F:09F9FD&9F9I!$BX>4(@``18U(`4ECP4AIP/$`#_!(
+MP>@@08T$`<'X"42)R<'Y'RG(:<`B`@``02G!36/`2<'@!$F)=#AP2&.'E"(`
+M`$C!X`1(B50X>$2)CY0B``##9F9FD&9F9I!F9I!32(/L$$B)^XN'D"(``#N'
+ME"(``'1BBXN0(@``2&/!2,'@!$@!V$R+0'!,B00D2(MP>$B)="0(_\%(8\%(
+M:<#Q``_P2,'H((T$`8G'P?\)B<B9B?@IT&G`(@(``"G!B8N0(@``2(G?0?_0
+MBX.0(@``.X.4(@``=9Y(@\006\-FD$2+A[Q$``!%C4@!26/!2&G`\0`/\$C!
+MZ"!!C00!P?@)1(G)P?D?*<AIP"("``!!*<%-8\!)P>`$28FT.)@B``!(8X>\
+M1```2,'@!$B)E#B@(@``1(F/O$0``,-F9I!FD%-(@^P02(G[BX>X1```.X>\
+M1```#X2#````9F:09F:0BXNX1```2&/!2,'@!$B-A!B0(@``2(M0"$B)%"1(
+MBT`02(E$)`C_P4ACP4AIP/$`#_!(P>@@C00!B<;!_@F)R)F)\"G0:<`B`@``
+M*<&)B[A$``"#N]Q9````=`A(B=_H`````$B+="0(2(G?_Q0DBX.X1```.X.\
+M1```=8-(@\006\-F9F:09F:09F:09F:04[@`````2(-_:`!T($B+7VA(BP-(
+MB4=HNJ````"^`````$B)W^@`````2(G86\.02(M':$B)!DB)=VC#9F9FD$B#
+M/P!T&DB+!TB+0$A(B49(2(L'2(EP2.L*9F:09F:02(EV2$B)-\-F9F:09F9F
+MD$B#["A(B5PD"$B);"003(ED)!A,B6PD($F)_4B)]4B#/@!T/4R+)DB+10!(
+MBUA(2#G8=0Q(QT4``````.L.9I!(BU4`2(M#2$B)0DA(B=Y,B>__4U!,.>-T
+M!TB#?0``=<9(BUPD"$B+;"003(MD)!A,BVPD($B#Q"C#9F:09F:02(/L"(`_
+M"'4'Z`````!FD$B#Q`C#9F9FD&9F9I!F9I!(@^P(2(GQ2(M'$(`_"'4'Z```
+M``#K"TB)UDB)QV9FD/_12(/$",-F9I!F9I!F9I!(@^QH2(E<)#A(B6PD0$R)
+M9"1(3(EL)%!,B70D6$R)?"1@28GT28G/08G52(MO,$0/MC=!#[;>B=Y(B>_H
+M`````+H`````B=Y(B>_H`````,<$)`````!!N0````!!N`````"Y`````+H#
+M````B=Y(B>_H`````+H`````A<`/A+T```!!]\0```#P#Y7`#[;(08#],`^4
+MP$0/MM!!_\)!#[;V00^VQ87)=!%!@/T@NC0```"X)`````]%PHE$)#"X0```
+M`(7)=0E$B>#!Z!B#R$")1"0H3(G@2,'H$`^W^$`/ML>)1"0@3(G@2,'H"`^V
+MP(E$)!A!#[;4#[?"A<ET"8GX)0#_```)T(E$)!#'1"0(`0```,<$)`````!!
+MN0`!``!-B?A$B=)(B>_H`````(G#00^V]DB)[^@`````B=J)T$B+7"0X2(ML
+M)$!,BV0D2$R+;"103(MT)%A,BWPD8$B#Q&C#9F9FD&9FD&9FD(GX@^`#P>`-
+M!0`@``!`]L<$=`T%```#`,-F9F:09F:0!0```@##9F9FD&9FD&9FD$%455-)
+MB?R)];\@H0<`Z`````"[(*$'`$`/MOWHK____XVH'`$``$F+?"00B>[H````
+M`(3`>`FX`0```.L?9I"_$"<``.@`````@<,0)P``@?L_2TP`=L^X`````%M=
+M05S#9F9FD&9F9I!F9I!F9I!(@^P82(E<)`A,B60D$$F)_$`/MO[H1O___XG#
+MC7,(28M\)!"Z`````.@`````C7,,28M\)!"Z&`$``.@`````2(M<)`A,BV0D
+M$$B#Q!C#D%-(B?M`#[;^Z`/___^-<`Q(BWL0N@````#H`````%O#28G1N@``
+M``!,C4<*BT$0B0>+012)1P2#.0"X`0````]$T$$/MD$$`<`)PF:)5PB+AD0"
+M``"%P`^4PH/X`@^4P`G0J`$/A&8!``"#>00!#X7<````9@^V00^`S!%F08D`
+M28/``F8/MD$.@,P19D&)`$F#P`)!#[9!!,'@`V8E^`"`S!)F08D`28/``F8/
+MMD$+@,P39D&)`$F#P`)F#[9!"(#,$V9!B0!)@\`"9@^V00R`S!1F08D`28/`
+M`F8/MD$)@,P49D&)`$F#P`)F#[9!#8#,%69!B0!)@\`"9@^V00J`S!5F08D`
+M28/``F9!QP!`%DF#P`*#.0!U&8.^1`(```*Z)@```+A@````#T7"Z:`!``"#
+MOD0"```"NC8```"X80````]%PNF'`0``9F9FD&8/MD$.@,P19D&)`$F#P`)!
+M#[9!!,'@`V8E^`"`S!)F08D`28/``F8/MD$(@,P39D&)`$F#P`)F#[9!"8#,
+M%&9!B0!)@\`"9@^V00J`S!5F08D`28/``HM!""4````/P>@89@U`%F9!B0!)
+M@\`"@SD!&<"#X/N#Z#3I!0$``&:0@WD$`0^%EP```&8/MD$/@,P29D&)`$F#
+MP`)F#[9!#H#,$F9!B0!)@\`"9@^V00N`S!-F08D`28/``F8/MD$(@,P39D&)
+M`$F#P`)F#[9!#(#,%&9!B0!)@\`"9@^V00F`S!1F08D`28/``F8/MD$-@,P5
+M9D&)`$F#P`)F#[9!"H#,%69!B0!)@\`"9D''`$`628/``H,Y`1G`@^#P@\`U
+MZV)F#[9!#H#,$F9!B0!)@\`"9@^V00B`S!-F08D`28/``F8/MD$)@,P49D&)
+M`$F#P`)F#[9!"H#,%69!B0!)@\`"BT$()0````_!Z!AF#4`69D&)`$F#P`*#
+M.0$9P(/@_H/H-F8/ML!F#0"79D&)`,-(@^Q(2(E<)!A(B6PD($R)9"0H3(EL
+M)#!,B70D.$R)?"1`B?")5"042(G-3(MG$`^VP$B+1,<P1(MH/$&-A2`!``")
+M1"00N@````")QDR)Y^@`````08VU!`$``$R)Y^@`````B$4`08V%"`$``(E$
+M)`R)QDR)Y^@`````9@^VP&:)10)!C9T,`0``B=Y,B>?H`````&8/ML!FB44$
+M18V]$`$``$2)_DR)Y^@`````9@^VP&:)109%C;44`0``1(GV3(GGZ`````!F
+M#[;`9HE%"(-\)!0!=6JZ@````(MT)!!,B>?H`````(MT)`Q,B>?H`````,'@
+M"&8)10*)WDR)Y^@`````P>`(9@E%!$2)_DR)Y^@`````P>`(9@E%!D2)]DR)
+MY^@`````P>`(9@E%"+H`````BW0D$$R)Y^@`````08VU&`$``$R)Y^@`````
+MB$4*08VU'`$``$R)Y^@`````B$4+2(M<)!A(BVPD($R+9"0H3(ML)#!,BW0D
+M.$R+?"1`2(/$2,-F9F:09F:02(/L:$B)7"0X2(EL)$!,B60D2$R);"103(ET
+M)%A,B7PD8$F)_XGS9HE4)!X/ML-,BW3',$&+=CR#QC!(BW\0Z`````!(B<6#
+MY1](C42M`$C!X`1*C:PP4`(``(-]``%T74B-12!(B40D$$R-9"0@BU`$1`^V
+MZTR)X42)[DR)_^CL_?__0?Z.90P``$B+7"002(M+($F+?C!,B20D0;D`````
+M1`^W1"0>N@$```!$B>[_4QA(B>Y,B??H(A,``$B+7"0X2(ML)$!,BV0D2$R+
+M;"103(MT)%A,BWPD8$B#Q&C#9F9FD&9F9I!F9F:09F:02(/L:$B)7"0X2(EL
+M)$!,B60D2$R);"103(ET)%A,B7PD8$F)_T&)]<=$)!0`````B?$/ML%(BVS'
+M,$&^``````^W`F:)1"082(U,)!@/MT("9HE!`HM"!(E!!`^W1"082(G#@^,?
+M2(T$FTC!X`1(C9PH4`(```^W00*$P'0JQT0D%`$````/MT$"1`^V\&9!@<X`
+M`<>%8`P```````#'A4`"````````3(UC($B-1"08#[=``J@$=!=(C4PD($&+
+M5"0$00^V]4R)_^BQ_/__D(,[`0^$EP```/Z-90P``$B-1"081(M(!$F+3"0@
+M10^V[4B+?3!(C40D($B)!"1%#[?&BU0D%$2)[D'_5"082(G>2(GOZ-<1``"#
+MO6`,````=$Y(@[U0#````'1$2(N%4`P``(-X&`%U-T2)[DR)_^@L%```2(NU
+M4`P``$B)[^A]%0``A<!U&4B+M5`,``"Z`0```$B)[^CU!@``9F:09I!(BUPD
+M.$B+;"1`3(MD)$A,BVPD4$R+="183(M\)&!(@\1HPV9F9I!F9I!F9I!F9I!(
+M@^PH2(E<)`A(B6PD$$R)9"083(EL)"!(B?N)]4&)U4B+?Q"^+`T``.@`````
+M08G$B<*#R@%(BWL0OBP-``#H`````$B+>Q"^+`T``.@`````2(M[$+[P!`$`
+MZ`````")ZO?2(=!$">A(BWL0B<*^\`0!`.@`````2(M[$+[P!`$`Z`````!(
+MBWL01(GBOBP-``#H`````$B+7"0(2(ML)!!,BV0D&$R+;"0@2(/$*,-F9F:0
+M9F9FD&9F9I!F9I!(@^Q(2(E<)!A(B6PD($R)9"0H3(EL)#!,B70D.$R)?"1`
+M28G^08G,1(E,)!1(BT<02(E$)`C'1"0$`````$2-/+4`````00G71(GZ#[;"
+M3(MLQS!%A<!T<DV%[0^$UP```$&+G?`,``!"C13E`````$$+52Q!BW4\@\8D
+M2(M\)`CH`````$0YXW0C00^V[Y")VDF+11A(C130B>Y,B??H_/S____#@^,?
+M1#GC=>)!@[U@#````+@!````#T5$)`2)1"0$08F=\`P``(-\)!0`#Y7"@WPD
+M!``/E<`)T*@!#X3P`0``00^V_^AB]O__1(U@"$2)YDB+?"0(Z`````")Q0^W
+MV(G:]])$B>9(BWPD".@`````]L,(#X0<`0``387M=!9!QX5@#````````$''
+MA4`"````````08.^D`````%T"D&#OI0````!=12^``$``$2)^=/FB?),B??H
+MW_W__T&#OHP````!=!1!@[Z0`````70*08.^E`````%U#D$/MO=,B??HQ0D`
+M`.M_08.^B`````%T"D&#OHP````!=6M!#[;?B=_HI/7__T2-8"BZ!````$2)
+MYDB+?"0(Z`````!$B>9(BWPD".@`````OQD```#H`````+H`````1(GF2(M\
+M)`CH`````$2)YDB+?"0(Z`````"_T`<``.@`````B=Y,B??H!`@``$$/MM^)
+MWDR)]^@`````A<`/A;,```")V;H`````O@$```!,B?=!_U8HZ9L```!`]L40
+M=$Y!#[;W3(GWZ,0'``!!@[Z0`````70*08.^E`````%U%[X``0``1(GYT^:Z
+M`````$R)]^C)_/__00^VS[H!````O@$```!,B?=!_U8HZT</M]7VQ@%T/T&#
+MO4`"```!9I!U(D''A6`,````````0<>%0`(```````!!#[;W3(GWZ.SY__^#
+M/0`````!=0A,B>_H2Q$``$B+7"082(ML)"!,BV0D*$R+;"0P3(MT)#A,BWPD
+M0$B#Q$C#9F9FD&9F9I!(@^P(28GXC0RU``````G1#[;!2(M\QS!(A?]U$P^V
+M\4R)Q^BH#@``ZTMF9I!F9I!(B[=0#```2(7V=10/MO%,B<?HB0X``.LL9F9F
+MD&9FD(,^`70-@WX8`69F9I!F9I!T#@^V\4R)Q^AC#@``ZP:0Z`L```!(@\0(
+MPV9FD&9FD$B#[#A(B5PD"$B);"003(ED)!A,B6PD($R)="0H3(E\)#!)B?Y)
+MB?=(BT<P3(M@$(MO/$&]`P```$0B+P^V'X/C!,'K`HVU(`$``$R)Y^@`````
+MC;4<`0``3(GGZ`````")Q;H``0``1(GIT^+WTH#[`1GV9KX``('&%``#`$R)
+MY^@`````0(3M#X@'`0``0/;%`70;N@$```!,B?Y,B??HVP$``.GL````9F:0
+M9F:008M'((/X`70L@_@!9F9FD'(.@_@"=';IR@```&9F9I"Z`````$R)_DR)
+M]^B@`0``Z;$```!`]L5`=$!)C7<@3(GWZ,D```"#^`%U'4&#?S``#X6/````
+MN@````!,B?Y,B??H:0$``.M]N@$```!,B?Y,B??H5P$``.MKN@$```!,B?Y,
+MB??H10$``.M90`^VQ:A`=$&H('4]08-_,`!FD'42N@````!,B?Y,B??H'@$`
+M`.LR28UW($R)]^A0````A<!U(KH!````3(G^3(GWZ/P```#K$+H!````3(G^
+M3(GWZ.H```!(BUPD"$B+;"003(MD)!A,BVPD($R+="0H3(M\)#!(@\0XPV9F
+M9I!F9I!!5%5328G\2(GSN`````"!?A#_````#X:?````BQ:#^@%T6+@`````
+M@_H!#X**````@_H"#X5\````O0````"02(M#"`^W$$B#P`)(B4,(08MT)#R!
+MQ@`!``!)BT0D,$B+>!#H`````/_%@?W_````=LZ!:Q```0``ZSN]`````$&+
+M="0\@<8``0``28M$)#!(BW@0Z`````!(BU,(9HD"2(-#"`+_Q8']_P```';1
+M@6L0``$``+@!````6UU!7,-(@^Q(2(E<)"!(B6PD*$R)9"0P3(EL)#A,B70D
+M0$B)_4F)]$&)U4&^`````$B-7B!(C4PD$(M3!`^V-TB+?S#HW/3__T&#_0%U
+M#4&V`<>%8`P```````!(BTLH#[9U`$B+?3!(C40D$$B)!"1!N0````!!N```
+M``!$B?+_4R!,B>9(B>_H!PH``$&#_0%U$8,]``````%U"$B)[^B`#0``2(.]
+M4`P````/A*,```!(B[50#```@WX8`74F2(GOZ*T-``"%P`^%A@```$B+M5`,
+M``"Z`0```$B)[^@A____ZW"#O40"````=2H/MG4`2(M],.C9"@``BW4\@\8H
+M2(M%,$B+>!#H`````+\!````Z``````/MG4`2(M],.@/"P``2(N=4`P``$B%
+MVW0D@WL8`'4>2(U3($B)WDB)[^AN#```2(M;"$B%VW0&@WL8`'3B2(M<)"!(
+MBVPD*$R+9"0P3(ML)#A,BW0D0$B#Q$C#D$%505154TB#[`A(B?M(BT<P3(M@
+M$$2+;SQ!C6THB>Y,B>?H`````*@$=#&Z`@```(GN3(GGZ`````")[DR)Y^@`
+M````O]`'``#H``````^V,TB+>S#H1@(``.L108UU*+H"````3(GGZ`````#&
+M@T@"````QH-D#````,:#90P```#'@^P,````````QX/P#````````,>#0`(`
+M``````#'@^@,```@````N@````"02&/"B92#:`P``/_"@_H??N^Z`````$AC
+MPDB-!(!(P>`$QX084`(```$```#_PH/Z'W[C2,>#4`P```````!(QX-8#```
+M`````,>#8`P```````!!C74(N@````!,B>?H`````$&-=0RZ&`$``$R)Y^@`
+M````BU,@08UU$$R)Y^@`````BU,D@>(`_/__08UU%$R)Y^@`````2(M#,+H`
+M````@[BD`````'0#BU,D08UU&$R)Y^@`````BU,H08UU'$R)Y^@`````2(M#
+M,+H`````@[BD`````'0#BU,D08UU($R)Y^@`````BU,LL@!!C74D3(GGZ```
+M``"X`0```$B#Q`A;74%<05W#9F:09I!!5T%6055!5%532(/L.$B)^XE4)!Q!
+MB<^%]@^%H@```+T`````3(UT)"!F9I!F9I!(8\5(C02`2,'@!$@!V(.X4`(`
+M``!U<$B-D&`"``"#>@@!=1.+4A1,BZ"8`@``3(NHD`(``.LB2&/%2(T$@$C!
+MX`1(`=B+D'0"``!,BZ"0`@``3(NHB`(```^V,TB+>S!,B?'H6/'__P^V,TB+
+M>S!,B30D0;D`````10^WQTR)X8M4)!Q!_]7_Q8/]'P^.;O___TB#Q#A;74%<
+M05U!7D%?PV9F9I!F9F:09F9FD&9FD$B#["A(B5PD"$B);"003(ED)!A,B6PD
+M($B)^XGUB>B#X`1!B<5!P>T"0;P#````02'L@[^(`````70)@[^,`````75:
+M00^V],'F"('&#`$``(V&```#`('&```"`$6$[0]%\$B+>Q#H`````(G!@^'\
+M@\D!00^V],'F"('&#`$``(V&```#`('&```"`$6$[0]%\$B+>Q")RN@`````
+M00^V],'F"('&=`$``(V&```#`('&```"`$6$[0]%\$B+>Q#H`````(G!@.$?
+MB>@/MM`/MH0:M0```,'@!27@````"<&`Y></MH0:O0```,'@"R4`&```"<%!
+M#[;TP>8(@<9T`0``C88```,`@<8```(`183M#T7P2(M[$(G*Z`````!(BUPD
+M"$B+;"003(MD)!A,BVPD($B#Q"C#9F:09F:02(/L&$B)'"1,B60D"$R);"00
+M2(G[1`^V[D2)[^C?Z___1(U@*$2)[DB)W^C`[/__2(M[$+H$````1(GFZ```
+M``!(BWL01(GFZ`````"_&0```.@`````2(M[$+H`````1(GFZ`````!(BWL0
+M1(GFZ`````!$B>Y(B=_H0?[__T2)[DB)W^@6[/__2(L<)$R+9"0(3(ML)!!(
+M@\08PV9FD$%7059!54%455-(@^P(2(G]QD0D!P!!O@`````/MD0D!T2-/(4`
+M````9F9FD&9FD$.-!#Y$#[;H1(GOZ"#K__]!B<1!C5PD*$B+?1"Z`@```(G>
+MZ`````!(BWT0B=[H`````$2)[DB)[^CR_O__2(M]$+H`````B=[H`````$B+
+M?1"Z'P$``$2)YN@`````08UT)`1(BWT0N@````#H`````$&-="0(2(M]$+H`
+M````Z`````!!C70D#$B+?1"Z`````.@`````08UT)!!(BWT0N@````#H````
+M`$&-="042(M]$+H`````Z`````!!C70D&$B+?1"Z`````.@`````08UT)!Q(
+MBWT0N@````#H`````$&-="0D2(M]$+H`````Z`````!!C70D($B+?1"Z````
+M`.@`````08UT)"Q(BWT0N@````#H`````$&-="0T2(M]$+J\````Z`````!!
+M_\9!@/X##X;#_O__@'PD!P$9VV:[``"!PP```P"-<PQ(BWT0N@````#H````
+M`(US$$B+?1"Z`````.@`````C7,42(M]$+H`````Z`````"-<QA(BWT0N@``
+M``#H`````(/#($B+?1")WN@`````)1P<'!P-`P,#`TB+?1")PHG>Z`````#^
+M1"0'@'PD!P$/AB#^__](@\0(6UU!7$%=05Y!7\-F9F:09F9FD&9FD$B#[`A(
+MBW\0NO__SP^^;`0!`.@`````2(/$",-F9F:04TB)^XM'!+``/0``@%!T(4B+
+M?Q"^+`T``.@`````@\@!2(M[$(G"OBP-``#H`````$B+>Q"^``T``.@`````
+M)?__`/](BWL0B<*^``T``.@`````2(M[$+H`````O@0-``#H`````$B+>Q"Z
+M`````+XX#```Z`````!(BWL0NO\``0"^!!T``.@`````2(M[$+H`````OF0=
+M``#H`````$B+>Q"Z`````+XH#```Z`````!(BWL0N@````"^6!T``.@`````
+M2(M[$+H`````OEP=``#H`````$B+>Q"Z`````+Y`'0``Z`````!(BWL0N@``
+M``"^1!T``.@`````2(M[$+H`````OD@=``#H`````$B+>Q"Z`````+Y0'0``
+MZ`````!;PV9FD&9FD&9FD$C'1@@`````2(N'6`P``$B)1A!(@[]8#````'0+
+M2(N'6`P``$B)<`A(B;=8#```2(._4`P```!U!TB)MU`,``#SPV9F9I!F9F:0
+M9F:09F:02(-^"`!U&DB+1A!(B8=8#```2(7`=!9(QT`(`````.L,2(M6"$B+
+M1A!(B4(02(-^$`!U'$B+1@A(B8=0#```2(7`=!A(QT`0`````.L.9I!(BU80
+M2(M&"$B)0@A(QT8(`````$C'1A``````PV9FD$B#[!A(B1PD2(EL)`A,B60D
+M$$F)_$B)]4B)TX,Z`'442(UR"$B-?2"Z*````.@`````ZQI(C7((2(U](+HP
+M````Z`````!!_H0D9`P``(L#B4482(GN3(GGZ.+^___'10``````0?Z$)$@"
+M``!(BQPD2(ML)`A,BV0D$$B#Q!C#2(/L$$B)'"1(B6PD"$B)^TB)]8-^&`!T
+M!OZ/9`P``$B)[DB)W^CF_O__BX/H#```B<$/ME4$B92+:`P``/_`B8/H#```
+MQT4``0```/Z+2`(``$B+'"1(BVPD"$B#Q!##0(#_`W810`^VSXU,"0*X`0``
+M`$C3X,-`#[;/C4P)`;@!````2-/@PV9F9I!F9F:005154TF)_(N?@````$`/
+MMNZ)[^BX____"=A!B80D@````(GOZ`?F__^-L!P!``!)BWPD$.@`````B>F#
+MX0.Z``$``-/B]]*)Z(/@!,'X`H/X`1GV9KX``('&%``#`$F+?"00Z`````!)
+MBWPD$+YD'0``Z``````YV'4F08N4)(````!)BWPD$+YD'0``Z`````!)BWPD
+M$+YD'0``Z`````!;74%<PV9F9I!F9I!F9I!(@^P82(E<)`A(B6PD$$B)_8N?
+M@````$`/MO[H`/____?0(=B)A8````!(BWT0OF0=``#H`````#G8=12+E8``
+M``!(BWT0OF0=``#H`````$B+7"0(2(ML)!!(@\08PY!(@^PH2(D<)$B);"0(
+M3(ED)!!,B6PD&$R)="0@28G^B?-,BV\0#[;#3(MDQS!!BVPD/$''A"1``@``
+M`0```(G8@^`$P>@"B=F#X0.Z`0$``-/B]](\`1GV9KX``('&%``#`$R)[^@`
+M````08MT)#R#Q@BZ`````$R)[^@`````#[;S3(GWZ!;___^-M1P!``!,B>_H
+M`````$''A"1,`@```0```(UU*+H!````3(GOZ`````!(BQPD2(ML)`A,BV0D
+M$$R+;"083(MT)"!(@\0HPV9F9I!F9F:09F9FD&9FD$%6055!5%5328G]08GV
+MO0````!,BV<0B?(/ML)(BT3',(M8/,>`0`(```````"-<RBZ`@```$R)Y^@`
+M````@\,HB=Y,B>?H`````*@!=!:_Z`,``.@`````_\6!_><#``!VWNL-@?WG
+M`P``9F:09I!V#$$/MO9,B>_HPO?__T$/MO9,B>_HAOW__UM=05Q!74%>PV9F
+M9I!F9I!F9I!F9I!!5%-(@^P(2(G[2(G12(M',$R+8!"+O^P,``!(P><%2`-[
+M$$B)\DB)WNB"Y/__BX/L#```_\"#X!^)@^P,``#^@V4,``"+4R2!X@#\___!
+MX`4)PHMS/(/&%$R)Y^@`````2(/$"%M!7,-F9F:09F:04TB)^[D`````N@(`
+M``"^`````.CX]/__2(G?Z##S__](B=_H`````(/X`746QX-@#````0````^V
+M,TB+>S#HO?S__UO#9F9FD&9F9I!F9I!(@^PH2(E<)`A(B6PD$$R)9"083(EL
+M)"!)B?U,C68@2(M',$B+:!"+7SR-LQP!``!(B>_H`````"7!````N@````"#
+M^$`/A=H!``!!@WPD!`%U;4$/ME0D%8VS!`$``$B)[^@`````00^V5"07C;,(
+M`0``2(GOZ`````!!#[94)!F-LPP!``!(B>_H`````$$/ME0D&XVS$`$``$B)
+M[^@`````00^V5"0=C;,4`0``2(GOZ`````#K0V9F9I!F9I"X`/___V9!A40D
+M%'4E9D&%1"06=1UF085$)!AU%69!A40D&G4-9D&%1"0<9F:09I!T"KH`````
+MZ2D!``!!#[94)!2-LP0!``!(B>_H`````$$/ME0D%HVS"`$``$B)[^@`````
+M00^V5"08C;,,`0``2(GOZ`````!!#[94)!J-LQ`!``!(B>_H`````$$/ME0D
+M'(VS%`$``$B)[^@`````00^V5"0>C;,8`0``2(GOZ`````!!#[94)!^-LQP!
+M``!(B>_H`````$&#/"0"#X6-````C;,@`0``2(GOZ`````!!N,@```"Y$"<`
+M`+H!````O@````!,B>_H`````+H`````A<!T7HVS'`$``$B)[^@`````)8D`
+M``"Z`````(/X"'5!0;T`````@<,``0``28M$)`@/MQ!(@\`"28E$)`B)WDB)
+M[^@`````0?_%08']_P```';908%L)!```0``N@$```")T$B+7"0(2(ML)!!,
+MBV0D&$R+;"0@2(/$*,-(@^PH2(D<)$B);"0(3(ED)!!,B6PD&$R)="0@2(G[
+MQX>``````````$B+?Q"Z`````+YD'0``Z`````!(BWL0Z`````"(0P2X````
+M`(![!``/A`P#``#'@X@`````````QX.,`````````,>#D`````````#'@Y0`
+M````````QX.8`````````,>#G`````````#'@Z``````````QH.L`````;@`
+M````2(-[*``/A*\"``"X`````&:!>P:!4`^%G@(```^V4P6#^@$/A)$"``"#
+M^@%_"X72=!=F9I!FD.LL@_H"=!MF9I!F9I!F9I#K',>#C`````$```!F9F:0
+MZQ;'@Y`````!````ZPK'@Y0````!````O0````!F9I!`#[;5B="#X`/!X`@%
+M=`$``/;"!'0*C;````,`ZPAFD(VP```"`$B+>Q#H`````(GJ#[;*B<!(B<*!
+MXN````!(P>H%B)09M0```"4`&```2,'H"XB$&;T```#_Q4"`_0=VGDB)W^BV
+M\___2(G?Z+[U__](B=_HUO7__TB+>Q"Z`````+[P!`$`Z`````!(BWL0OBP-
+M``#H`````$&)Q4&#Y?Y(BWL01(GJOBP-``#H`````(.[L`````!U6TB+>Q"^
+M``T``.@`````J#!T18M#<*G@`P"`=`@E'_S_?XE#<(.[B`````%T&X.[C```
+M``%T$H.[D`````%T"8.[E`````%U$XM#<*@0=`R#X.^)0W#K!(-C<,^]````
+M`&9F9I!`#[;U2(G?Z!3Q____Q4"`_0=V[(M3<$B+>Q"^``P``.@`````BU-T
+M2(M[$+XH#```Z`````"+4WA(BWL0OEP=``#H`````+T`````B>D/ML&+5(,8
+M0(#]`1GV9KX``('&#``#`$B+>Q#H`````(GJ#[;"BU2#($"`_0$9]F:^``"!
+MQA```P!(BWL0Z`````#_Q4"`_0%VM,>#@````%6K!@"]`````$`/MO5(B=_H
+M3=[____%0(#]!W;LOU##``#H`````(.[D`````%T"8.[E`````%U1T&]````
+M`+T`````0;X``0``1`^VY42)YDB)W^@`````A<!U"T2)\$2)X=/@00G%_\5`
+M@/T'=MI$B>J^`/\``$B)W^@RY?__N`$```!(BQPD2(ML)`A,BV0D$$R+;"08
+M3(MT)"!(@\0HPTB#[`A(BW\0N@````"^9!T``.@`````N`$```!(@\0(PV9F
+M9I!F9F:09F9FD&9FD,<%``````$```##9F:09I#'!0``````````PV9FD&:0
+M2(/L&$B)7"0(2(EL)!!(B?V)\@^VPDB+7,<PN`````!(A=MT+$`/MO[HHMS_
+M_XE#/$B):S#'@V`,````````QX-``@```````$B)W^B?[/__2(M<)`A(BVPD
+M$$B#Q!C#B?`/ML!(BU3',+@`````2(72=`V#NF`,```!#Y7`#[;`\\-F9F:0
+M9F:09F:09F:055-(@^P(2(G]B?"#X`/!X`@%``$``$#VQ@1T"(V8```#`.L&
+MC9@```(`2(M]$(G>Z`````"#X`^#^`%T$X/X`7(^@_@#=#'K-V9F9I!F9I!(
+MBWT0B=[H`````+]`#0,`Z`````!(BWT0B=[H`````(/@#X/X`W4(N`$```#K
+M!I"X`````$B#Q`A;7<-F9F:005=!5D%505154TB#[#A(B?U!B?>)\@^VPDB+
+M5,<P2(M'$$B)1"0PN`````!(A=(/A,L!``"+0CR)1"0LN`````"#NF`,```!
+M#X2R`0``QT0D*`````#_1"0H@WPD*`IV#[@`````Z94!``!F9I!FD$$/MM^)
+MWDB)[^@R[___B=Y(B>_H&-S__X/C!$&)W$'![`)!O0````!$B?B#X`/!X`@%
+M"`$``$&)QHG"2(E4)"!(B50D&$B)5"00B<!(B40D"$$/ML>)1"0$08V&```#
+M`$&-M@```@!%A.0/1?!(BWT0Z`````")PX/C\(/+`4B+="0@C88```,`C;8`
+M``(`183D#T7P2(M]$(G:Z`````!(BW0D&(V&```#`(VV```"`$6$Y`]%\$B+
+M?1#H`````+\9````Z`````"#X_!(BW0D$(V&```#`(VV```"`$6$Y`]%\$B+
+M?1")VN@`````2(MT)`B-A@```P"-M@```@!%A.0/1?!(BWT0Z`````"_T`<`
+M`.@`````BUPD!(G>2(GOZ`````"%P'4<0?_%08/]`@^&(?___XG>2(GOZ)[:
+M___IG_[__XMT)"R!QAP!``!(BWPD,.@`````)<````"#^$!T$T$/MO=(B>_H
+M;]K__[@`````ZQA!#[;W2(GOZ%S:__^X`0```&9F9I!F9I!(@\0X6UU!7$%=
+M05Y!7\.02(/L*$B)'"1(B6PD"$R)9"003(EL)!A,B70D((GPB=5!B<X/ML!(
+MBUS',+D`````2(7;='%,BV\01(MC/+D`````@[M@#````71;1(GF3(GOZ```
+M``")PH7M=2=!C4;_N0`````\'W<]QX-$`@```````(/BP$$/ML;_R`G"@,X"
+MZQ#'@T0"```!````@>+`_?__@,X!1(GF3(GOZ`````"Y`0```(G(2(L<)$B+
+M;"0(3(MD)!!,BVPD&$R+="0@2(/$*,-F9F:09F9FD&9FD$B#[`B)\0^VP4B+
+M5,<PN`````!(A=)T)K@`````@[I@#````708QX)@#````0```$`/MO;H=_/_
+M_[@!````2(/$",-F9F:09F:09F:09F:02(/L"(GR#[;"N0````!(@WS',`!T
+M(`^VPDB+1,<PQX!@#````````$`/MO;H#_3__[D!````B<A(@\0(PV9FD%.)
+M\(G6#[;`2(M<QS"X`````$B%VW0MN`````"#NV`,```!=!^Y`````+H"````
+M2(G?Z.GI__](B=_H(>C__[@!````6\-F9F:09F:09F:0B?`/ML!(BU3',+C_
+M````2(72=`</MH)(`@``\\-F9I!(@^PH2(D<)$B);"0(3(ED)!!,B6PD&$R)
+M="0@28G]B?.)U4&)S(GR@^(#B=B#X`2)P<'I`D#VQ?AU!D'VQ/QT![@`````
+MZW@/ML+!X`B-D'0!``"-@@```P"!P@```@"$R4&)QD0/1/)(BW\01(GVZ```
+M```D'XGJP>(%@>+@````"="`Y.=$B>+!X@N!X@`8```)T$F+?1")PD2)]N@`
+M````#[;#1HBD*+T```!"B*PHM0```+@!````9F:09I!(BQPD2(ML)`A,BV0D
+M$$R+;"083(MT)"!(@\0HPV9FD$B#[!A(B1PD2(EL)`A,B60D$$B)^XGP0;P#
+M````02'$@^`$B<7![0)`@/T!&?9FO@``@<88``,`2(M_$.@`````00^VU(U*
+M&+H!````2-/B"=!`@/T!&?9FO@``@<88``,`2(M[$(G"Z`````!`@/T!&?9F
+MO@``@<88``,`2(M[$.@`````N`$```!(BQPD2(ML)`A,BV0D$$B#Q!C#9I!!
+M54%455-(@^P(28G\08GUO0,```!$(>U`#[;VB?.#XP3!ZP+HY^C__X#[`1GV
+M9KX``('&&``#`$F+?"00Z`````!`#[;-@\$82,?"_O___TC3PB'0@/L!&?9F
+MO@``@<88``,`28M\)!")PN@`````@/L!&?9FO@``@<88``,`28M\)!#H````
+M`$`/MO7!Y@B!Q@@!``"-A@```P"!Q@```@"$VP]%\$F+?"00Z`````"#X/"#
+MR`%`#[;5P>((C8H(`0``C9$```,`@<$```(`A-N)U@]$\4F+?"00B<+H````
+M`$$/MO5,B>?H(>C__[@!````2(/$"%M=05Q!7<.005=!5D%505154TB#[`A)
+MB?Q(B=-!B?5,BW\0B?(/ML)(BVS',$`/MM:)T(/@`\'@"`5T`0``]L($=`B-
+ML````P#K!HVP```"`$F+?"00Z`````")0UA(A>UU$,<#`````+@!````Z=,`
+M``!$BW4\00^V]4R)Y^@`````B0.X`0```(,[``^$LP```(N%8`P``(E#+(N%
+M1`(``(E#,`^VA4@"``"(0S2Y`````&9F9I")R@^V1"IVB$0:!/_!@_DG=NY$
+MB?9,B?_H`````(E#.$&-=A!,B?_H`````(E#/$&-=A1,B?_H`````(E#0$&-
+M=AA,B?_H`````(E#1$&-=AQ,B?_H`````(E#2$&-=B!,B?_H`````(E#3$&-
+M=B1,B?_H`````(E#4$&-=BA,B?_H`````(E#5+@!````2(/$"%M=05Q!74%>
+M05_#9F:02(/L*$B)'"1(B6PD"$R)9"003(EL)!A,B70D($F)_DF)U8GS#[;#
+M2(MLQS"X`0```(.]8`P````/A$<!``"X`@```("]2`(``!\/A#4!``#_C>@,
+M``"+A>@,``"+C(5H#```08G,2XT$I$C!X`1,C:0H4`(``$&(3"0$@SH`#X6N
+M````2(U""(.]1`(```)U#\=`!`$```#K-F9FD&9FD(-X!`%U*H-]"`!U)(N%
+MZ`P``(G"B8R5:`P``/_`B87H#```N`,```#IN````&9FD("]9`P```!U1X.]
+M0`(```!U'@^VVXG>3(GWZ)'M__^)WDR)]^CG[?__9F9FD&9FD$R)ZDR)YDB)
+M[^BRZ___28U5"$R)YDB)[^A#[___ZV&03(GJ3(GF2(GOZ)+K__]FD.M.3(GF
+M2(GOZ(/K__^`O4@"```!=3J#O4`"```!=0L/MO-,B??H9N[__TR)YDB)[^B[
+M[___A<!U%TR)YDB)[^C,Z___N`$```#K"F9FD&:0N`````!(BQPD2(ML)`A,
+MBV0D$$R+;"083(MT)"!(@\0HPV9F9I!F9F:09F:09F:02(/L6$B)7"0H2(EL
+M)#!,B60D.$R);"1`3(ET)$A,B7PD4$F)_TB+1Q!(B40D&+Y@'0``2(G'Z```
+M``")Q4&+CX````")3"0DN`````"%S0^$BP$``/?%```$`'1908._L`````$9
+M]H'F6`0``('&`!D``$B+?"08Z`````")PXG"]])!@[^P`````1GV@>98!```
+M@<8`&0``2(M\)!CH`````+D`````B=J^`````$R)_T'_5RC&1"0C`/?%_P$`
+M``^$``$``(!\)",!&=MFNP``@<,4``,`NN____^)WDB+?"08Z`````")WDB+
+M?"08Z`````!!B<:)PO?2@\H0B=Y(BWPD&.@`````@'PD(P$9]F:^``"!Q@@`
+M`P!(BWPD&.@`````08G$0;T`````#[9$)".)1"04B40D$&9F9I!!#[;5N``!
+M``")T=/@B<-$(?.X`0```-/@1"'P08GI08/A`8G!1`G)=!E$B>&#X1]$B30D
+M08G`BW0D%$R)_^A]V?__A=MT*HM<)!"-!)T`````1`GH#[;XZ%/J__^%1"0D
+M=`Y!#[;5B=Y,B?_H3]S__\'M`D'![`A!_\5!@/T##X9[____ZP/![0C1[?Y$
+M)".`?"0C`0^&X/[__[@!````2(M<)"A(BVPD,$R+9"0X3(ML)$!,BW0D2$R+
+M?"102(/$6,-F9F:09F:09F:09F:02(/L"$B+?Q"Z`````+YD'0``Z`````"X
+M`0```$B#Q`C#9F9FD&9F9I!F9F:09F:02(/L"(N7@````$B+?Q"^9!T``.@`
+M````N`$```!(@\0(PV9F9I!F9F:09F:09F:02(/L.$B)7"0(2(EL)!!,B60D
+M&$R);"0@3(ET)"A,B7PD,$B)^T&)S$6)Q4&)]HG0O0`````YS7-.1`GP08G'
+MD(MS/('&'`$``$B+0S!(BW@0Z`````"$P'@.1"'X1#CP#Y3`#[;`ZR1$B>_H
+M``````^V,TB+>S#H`````(7`=`S_Q40YY7*YN`````!(BUPD"$B+;"003(MD
+M)!A,BVPD($R+="0H3(M\)#!(@\0XPY!32(G[BW<\@<8@`0``2(M',$B+>!"Z
+M`````.@`````BW,\@<8@`0``2(M#,$B+>!#H`````%O#9F:09F:09F:04TB)
+M^XMW/('&(`$``$B+1S!(BW@0N@(```#H`````(MS/('&'`$``$B+0S!(BW@0
+MZ`````!;PV9FD&9FD&9FD$B#[!A(B1PD2(EL)`A,B60D$(GP#[;`2(M<QS"X
+M`````$B%VW1M3(MG$(MK/+@`````@[M@#````7182(G?Z`````"-M1P!``"Z
+MX0```$R)Y^@`````0;AD````N1`G``"Z`0```+X`````2(G?Z`````"%P'4/
+M2(G?Z`````"X`````.L-2(G?Z`````"X`0```$B+'"1(BVPD"$R+9"002(/$
+M&,-F9F:09F:09F:0055!5%532(/L.$F)_(GUB?(/ML),BVS',+@`````387M
+M#X3M````28U=0&9FD&:00`^V]<=$)##L````QT0D*`````#'1"0@`````,=$
+M)!@`````QT0D$`````#'1"0(`````,<$)`````!!N0`!``!-C45`N0````"Z
+M`0```$R)Y^@`````A<`/A(4```#V`P1T/V:!>P3(-W4W0`^V]<<$)`````!!
+MN0````!!N`````"Y`````+H'````3(GGZ`````"%P'1+9L<#``#I7?____:#
+MH````.!T,[D`````2(G>N`$```"`N_X!``"E=2*Z``````^WP@(,,/_"9H'Z
+M_P%V\;@`````A,EU!;@!````2(/$.%M=05Q!7<.02(/L"(GP#[;`2(M\QS"X
+M`````$B%_W03N`````"#OV`,```!=`7H`````$B#Q`C#2(/L.$`/MO;'1"0P
+M[P```,=$)"@`````#[9$)$")1"0@10^VR42)3"0810^VP$2)1"00#[;)B4PD
+M"`^VTHD4)$&Y`````$&X`````+D`````N@````#H`````$B#Q#C#2(/L:$B)
+M7"0X2(EL)$!,B60D2$R);"103(ET)%A,B7PD8$B)^T&)UT&)SDV)Q46)S(GR
+MBVPD<$2+3"1X1(N$)(````"+O"2(````BXPDD````$0/MIPDF````$0/MI0D
+MH`````^VPKX`````2(-\PS``=$T/MO)!#[;"B40D,$$/ML.)1"0H#[?!B40D
+M(`^WQXE$)!A!#[?`B40D$$$/M\&)1"0(#[?%B00D18GA38GH1(GQ1(GZ2(G?
+MZ#$```")QHGP2(M<)#A(BVPD0$R+9"1(3(ML)%!,BW0D6$R+?"1@2(/$:,-F
+M9F:09F:09F:02('LF````$B)7"1H2(EL)'!,B60D>$R)K"2`````3(FT)(@`
+M``!,B;PDD````(GPB50D7(G-3(E$)%!$B4PD3$2+K"2@````1(N\)*@```"+
+ME"2P````9HE4)#"+C"2X````9HE,)""+E"3`````9HE4)!`/MHPDR````(A,
+M)`\/MI0DT````(A4)`X/ML!(BUS',$R+9Q!$BW,\@[M``@```70)@[M@#```
+M`74*N@````#IV`(``$&XZ`,``+FX"P``N@````"^0````$B)W^@`````N@``
+M``"%P`^$K@(``$B)W^@`````@_T!=6Y,B>D/MM5!C;8$`0``3(GGZ`````!,
+MB?@/MM1!C;8(`0``3(GGZ`````"+3"0P#[;508VV#`$``$R)Y^@`````BT0D
+M(`^VU$&-MA`!``!,B>?H`````(M,)!`/MM5!C;84`0``3(GGZ`````#K/K@`
+M____9D2%Z'4A9D2%^&9F9I!U%V:%1"0P=1!FA40D(&:0=0=FA40D$'022(G?
+MZ`````"Z`````.GU`0``00^VU4&-M@0!``!,B>?H`````$$/MM=!C;8(`0``
+M3(GGZ``````/ME0D,$&-M@P!``!,B>?H``````^V5"0@08VV$`$``$R)Y^@`
+M````#[94)!!!C;84`0``3(GGZ``````/ME0D#T&-MA@!``!,B>?H``````^V
+M5"0.08VV'`$``$R)Y^@`````@WPD7`!U14&X$"<``+D<#```N@$```"^````
+M`$B)W^@`````A<!U$DB)W^@`````N@````#I,0$``$B)W^@`````N@$```#I
+M'P$``$&X$"<``+D<#```N@$```"^`````$B)W^@`````A<!U$TB)W^@`````
+MN@````"0Z>L```!!C;8<`0``3(GGZ`````"H"'422(G?Z`````"Z`````.G&
+M````O0`````[;"1,<VI%C:X``0``0(3M=2%!N&0```"Y4,,``+H!````O@``
+M``!(B=_H`````(7`=&V#?"1<`7481(GN3(GGZ`````")ZDB+3"109HD$4>L6
+MB>A(BTPD4`^W%$%$B>Y,B>?H`````/_%.VPD3'*=0;AD````N5##``"Z`0``
+M`+X`````2(G?Z`````"%P'4>2(G?Z`````"Z`````.LA2(G?Z`````"Z````
+M`.L22(G?Z`````"Z`0```&9FD&:0B=!(BUPD:$B+;"1P3(MD)'A,BZPD@```
+M`$R+M"2(````3(N\))````!(@<28````PV9F9I!F9F:09F9FD&9FD$%455-(
+MB?M(BT<P2(MH$$2+9SQ!C;0D(`$``+H&````2(GOZ`````"+<SR!QB`!``!(
+MB>_H`````+\*````Z`````!(B=_H`````+_T`0``Z`````"_]`$``.@`````
+MO_0!``#H`````+_T`0``Z`````"[`````$&!Q!P!``"01(GF2(GOZ`````"$
+MP'@'N`$```#K'K_T`0``9F:09I#H`````/_#@?L/)P``=M&X`````%M=05S#
+M9F9FD&9FD%-(B?OH-____[H!````@_@!=!2_]`$``.@`````2(G?Z!O___^)
+MPHG06\-F9I!FD%532(/L"$B)_4B+!0````"+<!B!Q@``!`"ZH````.@`````
+MNP````!(BP4`````BW`0@<8```0`2(GOZ`````#_PX/[!'[ANP````!F9F:0
+M2(L%`````(MP%('&```$`$B)[^@`````_\.#^P5^X;L`````2(L%`````(MP
+M$('&```$`$B)[^@`````_\.#^P1^X4B#Q`A;7<-F9I!F9I!32(G[2(L%````
+M`(MP#('&```$`+J`````Z`````!(BP4`````BW`8@<8```0`NK````!(B=_H
+M`````%O#9F:02(/L&$B)7"0(3(ED)!!)B?SH"O___TB+!0````"+<`B!Q@``
+M!`"Z`````$R)Y^@`````2(L%`````(LP@<8```0`3(GGZ`````")PTR)Y^AL
+M____#[;;B=A(BUPD"$R+9"002(/$&,-F9F:09F9FD$B#[!A(B1PD2(EL)`A,
+MB60D$$B)^X,]```````/A08!``"^+`T``.@`````08G$B<*#R@&^+`T``$B)
+MW^@`````OO`$`0!(B=_H`````(G%@^!B@_AB=!6)ZH/*8@^VTK[P!`$`2(G?
+MZ`````"^```$`$B)W^@`````2(G?Z"[^__](BP4`````BW`(@<8```0`N@<`
+M``!(B=_H`````$B+!0````"+,('&```$`+H1````2(G?Z`````!(BP4`````
+MBW`(@<8```0`N@4```!(B=_H`````$B+!0````"+,('&```$`+H1````2(G?
+MZ`````!(B=_H5/[__XGJOO`$`0!(B=_H`````+[P!`$`2(G?Z`````!$B>*^
+M+`T``$B)W^@`````2(L<)$B+;"0(3(MD)!!(@\08PV9FD&:005154TB)^[XL
+M#0``Z`````!!B<2)PH/*`;XL#0``2(G?Z`````"^\`0!`$B)W^@`````B<6#
+MX&*#^&)T%8GJ@\IB#[;2OO`$`0!(B=_H`````+X```0`2(G?Z`````!(B=_H
+M"?W__TB+!0````"+<`B!Q@``!`"Z!P```$B)W^@`````2(L%`````(LP@<8`
+M``0`N@````!(B=_H`````$B+!0````"+<`B!Q@``!`"Z!0```$B)W^@`````
+M2(L%`````(LP@<8```0`N@````!(B=_H`````$B)W^@O_?__B>J^\`0!`$B)
+MW^@`````OO`$`0!(B=_H`````$2)XKXL#0``2(G?Z`````!;74%<PV9F9I!F
+M9F:09F:09F:04TB)^^A'_/__2(L%`````(MP"('&```$`+H*````2(G?Z```
+M``!(BP4`````BS"!Q@``!`"Z`````$B)W^@`````2(G?Z*;\__^^`0`$`$B)
+MW^@`````NO____^H`G5-2(G?Z.C[__](BP4`````BS"!Q@``!`"Z`0```$B)
+MW^@`````2(G?Z&3\__^_9````.@`````O@$`!`!(B=_H`````-'H@^`!@_@!
+M&=*)T%O#9F9FD&9F9I!F9F:09F:02(/L*$B)'"1(B6PD"$R)9"003(EL)!A,
+MB70D($F)_$&)]DB+7Q"^+`T``$B)W^@`````08G%B<*#R@&^+`T``$B)W^@`
+M````OO`$`0!(B=_H`````(G%@^!B@_AB=!6)ZH/*8@^VTK[P!`$`2(G?Z```
+M``"^```$`$B)W^@`````18BT)+0```!!#[;62(L%`````(MP'('&```$`$B)
+MW^@`````B>J^\`0!`$B)W^@`````OO`$`0!(B=_H`````$2)ZKXL#0``2(G?
+MZ`````!(BQPD2(ML)`A,BV0D$$R+;"083(MT)"!(@\0HPV:02(/L"(GQA-)T
+M%K@!````T^`(A[0```#K%&9F9I!F9I"X_O___]/`((>T````#[:WM````.@`
+M````2(/$",-FD$%7059!54%455-(@^P(2(G]0;\`````OBP-``#H`````$&)
+MQHG"@\H!OBP-``!(B>_H`````+[P!`$`2(GOZ`````!!B<6#X&*#^&)T%D2)
+MZH/*8@^VTK[P!`$`2(GOZ`````"^```$`$B)[^@`````O@``!`!(B>_H````
+M`&8]5:H/A>\```!!O`````!FD+L`````1#GC?RIF9I!F9I!(BP4`````BW`8
+M@<8```0`NJ````!(B>_H`````/_#1#GC?MQ!@?S(````?@J_`0```.@`````
+MO@``!`!(B>_H`````(G#2(L%`````(MP&('&```$`+JP````2(GOZ`````!F
+M@?M5JG480?_$08'\QP````^.=O___V:!^U6JD'152(L%`````(MP&('&```$
+M`+JP````2(GOZ`````"^```$`$B)[^@`````9CU5JG4E0;\!````2(GOZ.#Y
+M__\\/G432(GOZ*3\__^%P+@"````1`]$^$B+!0````"+<!B!Q@``!`"ZL```
+M`$B)[^@`````1(GJOO`$`0!(B>_H`````+[P!`$`2(GOZ`````!$B?*^+`T`
+M`$B)[^@`````00^VQTB#Q`A;74%<05U!7D%?PV9F9I!F9F:09F9FD%532(/L
+M"$B)_;L`````9I!(QP4``````````$B)[^C]_?__A,!U(4C'!0``````````
+M2(GOZ.;]__^$P'4*_\.!^^<#``!^R`^VP$B#Q`A;7<-F9F:09F9FD&9FD&9F
+MD(!^`0)U&H!^`@%U%`^V1@.)!0````"X`````,-F9F:0N/_____#9F9FD&9F
+MD&9FD$B#[#A(B5PD"$B);"003(ED)!A,B6PD($R)="0H3(E\)#!!B?=!B<U$
+MB<-(BV\028G4OBP-``!(B>_H`````(E$)`2)PH/*`;XL#0``2(GOZ`````"^
+M\`0!`$B)[^@`````08G&@^!B@_AB=!9$B?*#RF(/MM*^\`0!`$B)[^@`````
+MO@``!`!(B>_H`````(7;="R[`````$0YZWU29I!!C;0?```$`$B)[^@`````
+M08@$)/_#2?_$1#GK?.+K,+L`````1#GK?29F9I!F9I!!#[84)$&-M!\```0`
+M2(GOZ`````#_PTG_Q$0YZWSAD$2)\K[P!`$`2(GOZ`````"^\`0!`$B)[^@`
+M````BU0D!+XL#0``2(GOZ`````"X`0```$B+7"0(2(ML)!!,BV0D&$R+;"0@
+M3(MT)"A,BWPD,$B#Q#C#```````````````$````````````````````````
 M````````````````````````````````````````````````````````````
 M````````````````````````````````````````````````````````````
 M````````````````````````````````````````````````````````````
 M````````````````````````````````````````````````````````````
-M```````````````!`````````````0`!``$``@`!``,``0`$``$`!0`!``8`
-M`0`````````!```@`0``0`$``&`!``"``0``H`$``,`!````````````````
+M`0`````````````````````````````````!``$``0`"``$``P`!``0``0`%
+M``$`!@`!``<``0````$``"`!``!``0``8`$``(`!``"@`0``P`$``.`!````
 M````````````````````````````````````````````````````````````
 M````````````````````````````````````````````````````````````
-M````````````````````%``````````!``%X$`P'")`!````````)````!P`
-M`````````````)P%````````90[0!(\"C@.-!(P%A@:#!R0```!$````````
-M``````#L`````````$$.$$$.&$0.((,#A@(````D````;```````````````
-M"0(```````!8#C"-`HP#A@2#!0``````)````)0``````````````%0`````
-M````0@X000X800X@@P2&`XP"`#P```"\``````````````!*`0```````$(.
-M$$(.&$(.($(.*$$.,$$..$<.P`2#!X8&C`6-!(X#CP(````````T````_```
-M````````````]`````````!"#A!"#AA"#B!!#BA!#C!'#K`$@P:&!8P$C0..
-M`@```!0````T`0````````````!Q`````````!P```!,`0````````````#[
-M`````````$$.$(,"````)````&P!`````````````%L`````````00X000X8
-M1`X@@P.&`@```"0```"4`0`````````````T`0```````&(.0(\"C@.-!(P%
-MA@:#!P`<````O`$`````````````.P````````!$#A```````!P```#<`0``
-M``````````!L`````````$0.$```````)````/P!`````````````&T`````
-M````00X000X8@P.&`@```````"0````D`@````````````"2`0```````%T.
-M,(X"C0.,!(8%@P8````<````3`(`````````````@P````````!$#A``````
-M`"0```!L`@````````````"V`````````%,.((P"A@.#!``````````D````
-ME`(`````````````[@````````!3#B",`H8#@P0`````````)````+P"````
-M`````````%$!````````0@X000X800X@@P2&`XP"`"0```#D`@``````````
-M``#^`````````%,.((P"A@.#!``````````D````#`,`````````````QP$`
-M``````!8#C"-`HP#A@2#!0``````)````#0#`````````````$,"````````
-M8@Y`CP*.`XT$C`6&!H,'`"0```!<`P````````````!/`0```````$$.$$$.
-M&$0.((,#A@(````D````A`,`````````````?P$```````!@#K`$C@*-`XP$
-MA@6#!@``%````*P#`````````````"@`````````%````,0#````````````
-M`*0`````````)````-P#`````````````%L`````````00X000X81`X@@P.&
-M`@```"0````$!``````````````T`0```````&(.0(\"C@.-!(P%A@:#!P`<
-M````+`0`````````````*0````````!$#A```````!0```!,!```````````
-M```?`````````!0```!D!``````````````7`````````#P```!\!```````
-M``````"9`0```````$(.$$(.&$(.($(.*$$.,$$..$0.0(,'A@:,!8T$C@./
-M`@`````````D````O`0`````````````:@````````!!#A!!#AB#`X8"````
-M````%````.0$`````````````$(`````````)````/P$`````````````#<`
-M````````00X000X81`X@@P.&`@```!0````D!0`````````````]````````
-M`!P````\!0`````````````8`````````$0.$```````%````%P%````````
-M`````#(`````````%````'0%`````````````"4`````````)````(P%````
-M`````````(0`````````6`XHC0*,`X8$@P4``````!0```"T!0``````````
-M```H`````````"0```#,!0````````````!9`````````$(.$$$.&$$.((,$
-MA@.,`@`\````]`4`````````````*0(```````!"#A!"#AA"#B!"#BA!#C!!
-M#CA$#D"#!X8&C`6-!(X#CP(`````````)````#0&`````````````,(`````
-M````0@X000X800X@@P2&`XP"`#P```!<!@`````````````U`@```````$(.
-M$$(.&$(.($(.*$$.,$$..$<.H`.#!X8&C`6-!(X#CP(````````D````G`8`
-M````````````\0````````!!#A!!#AA$#B"#`X8"````)````,0&````````
-M`````*,$````````8@Y0CP*.`XT$C`6&!H,'`!P```#L!@`````````````E
-M`````````$0.$```````/`````P'`````````````+4`````````0@X00@X8
-M0@X@0@XH00XP00XX1`Y`@P>&!HP%C02.`X\"`````````!0```!,!P``````
-M```````:`0```````"0```!D!P````````````#"`````````%,.((P"A@.#
-M!``````````<````C`<`````````````/@````````!.#B",`H,#`"0```"L
-M!P````````````"@`````````%@.,(T"C`.&!(,%```````D````U`<`````
-M````````C`$```````!"#A!!#AA!#B"#!(8#C`(`)````/P'````````````
-M`.H!````````90[@`H\"C@.-!(P%A@:#!R0````D"`````````````#H````
-M`````&4.T`*/`HX#C02,!88&@P<\````3`@`````````````2@$```````!"
-M#A!"#AA"#B!"#BA!#C!!#CA'#L`#@P>&!HP%C02.`X\"````````)````(P(
-M`````````````'H`````````4PX@C`*&`X,$`````````!0```"T"```````
-M``````!4`````````#P```#,"``````````````Y`@```````$(.$$(.&$(.
-M($(.*$$.,$$..$0.0(,'A@:,!8T$C@./`@`````````\````#`D`````````
-M````%R,```````!"#A!"#AA"#B!"#BA!#C!!#CA'#K`#@P>&!HP%C02.`X\"
-M````````/````$P)`````````````#\!````````0@X00@X80@X@0@XH00XP
-M00XX1`Y0@P>&!HP%C02.`X\"`````````"0```","0````````````!F````
-M`````$$.$$$.&$0.((,#A@(````<````M`D`````````````0P$```````!!
-M#A"#`@```"0```#4"0````````````!Q`````````%@.,(T"C`.&!(,%````
-M```D````_`D`````````````W0````````!!#A!!#AA$#B"#`X8"````/```
-M`"0*`````````````+$"````````0@X00@X80@X@0@XH00XP00XX1PZ``X,'
-MA@:,!8T$C@./`@```````!P```!D"@````````````!_`````````$X.((8"
-M@P,`/````(0*`````````````#\%````````0@X00@X80@X@0@XH00XP00XX
-M1`Y0@P>&!HP%C02.`X\"`````````#P```#$"@````````````#E`P``````
-M`$(.$$(.&$(.($(.*$$.,$$..$0.D`&#!X8&C`6-!(X#CP(````````<````
-M!`L`````````````*@````````!$#A```````"P````D"P````````````!I
-M`@```````$(.$$$.&$$.($0.0(,$A@.,`@```````"0```!4"P``````````
-M``!K`````````$$.$$$.&$0.((,#A@(````D````?`L`````````````%@$`
-M``````!"#A!!#AA!#B"#!(8#C`(`)````*0+`````````````$<!````````
-M00X000X81`X@@P.&`@```!0```#,"P`````````````C`````````"0```#D
-M"P````````````!W`````````$$.$$$.&$0.((,#A@(````L````#`P`````
-M````````>@(```````!"#A!"#AA!#B!!#BA'#K`$@P6&!(P#C0(4````/`P`
-M````````````0@`````````D````5`P`````````````1P````````!!#A!!
-M#AB#`X8"````````%````'P,`````````````"H`````````%````)0,````
-M``````````X`````````+````*P,`````````````&0!````````0@X00@X8
-M00X@00XH1PZP"(,%A@2,`XT"%````-P,`````````````%4`````````'```
-M`/0,`````````````'X`````````00X01`X@@P(4````%`T`````````````
-M6P`````````<````+`T`````````````HP````````!!#A!$#B"#`AP```!,
-M#0`````````````O`````````$$.$(,"````%````&P-``````````````P`
-M````````%````(0-`````````````"@`````````)````)P-````````````
-M`'H`````````6`XPC0*,`X8$@P4``````!P```#$#0`````````````5````
-M`````$0.$```````'````.0-`````````````"<`````````1`X0```````D
-M````!`X`````````````7@$```````!B#G"/`HX#C02,!88&@P<`%````"P.
-M`````````````"8`````````)````$0.`````````````&(`````````0@X0
-M00X800X@@P2&`XP"`!P```!L#@````````````!/`````````$X.((P"@P,`
-M'````(P.`````````````"``````````00X0@P(````4````K`X`````````
-M````.``````````4````Q`X`````````````P`(````````D````W`X`````
-M````````CP$```````!B#E"/`HX#C02,!88&@P<`)`````0/````````````
-M`.$`````````8@YPCP*.`XT$C`6&!H,'`"0````L#P````````````"3`0``
-M`````&(.<(\"C@.-!(P%A@:#!P`D````5`\`````````````L0````````!8
-M#C"-`HP#A@2#!0``````)````'P/``````````````,#````````8@Y0CP*.
-M`XT$C`6&!H,'`!P```"D#P````````````!Z`````````$0.$```````)```
-M`,0/`````````````*,!````````8@Y`CP*.`XT$C`6&!H,'`"0```#L#P``
-M``````````#``````````$(.$$$.&$$.((,$A@.,`@`D````%!``````````
-M````;P$```````!=#E".`HT#C`2&!8,&````+````#P0`````````````)L!
-M````````0@X00@X800X@00XH1`XP@P6&!(P#C0(`/````&P0````````````
-M`-$`````````0@X00@X80@X@0@XH00XP00XX1`YP@P>&!HP%C02.`X\"````
-M`````"0```"L$``````````````Z`0```````%@.,(T"C`.&!(,%```````D
-M````U!``````````````G@````````!3#B"-`HP#@P0`````````/````/P0
-M``````````````4"````````0@X00@X80@X@0@XH00XP00XX1`Y`@P>&!HP%
-MC02.`X\"`````````!P````\$0`````````````<`````````$0.$```````
-M'````%P1`````````````"<!````````00X0@P(````4````?!$`````````
-M````0@`````````4````E!$`````````````;0`````````D````K!$`````
-M````````@0````````!3#B",`H8#@P0`````````'````-01````````````
-M`&$`````````3@X8A@*#`P`4````]!$`````````````*``````````D````
-M#!(`````````````AP````````!!#A!!#AA$#B"#`X8"````'````#02````
-M`````````"T`````````00X0@P(````D````5!(`````````````T@``````
-M``!=#C".`HT#C`2&!8,&````)````'P2`````````````'H`````````6`XP
-MC0*,`X8$@P4``````"0```"D$@````````````!I`````````$(.$$$.&$0.
-M((,#C`(````<````S!(`````````````10````````!!#A"#`@```"0```#L
-M$@`````````````K`@```````%@.,(T"C`.&!(,%```````D````%!,`````
-M````````70,```````!=#C".`HT#C`2&!8,&````'````#P3````````````
-M`"$`````````1`X0```````4````7!,`````````````"P`````````4````
-M=!,`````````````"P`````````<````C!,`````````````8`````````!.
-M#B"&`H,#`!0```"L$P`````````````C`````````"0```#$$P``````````
-M``",`````````$$.$$$.&$0.((,#A@(````\````[!,`````````````,P(`
-M``````!"#A!"#AA"#B!"#BA!#C!!#CA$#G"#!X8&C`6-!(X#CP(`````````
-M)````"P4`````````````.0`````````70XPC@*-`XP$A@6#!@```!P```!4
-M%`````````````!#`````````$0.$```````'````'04`````````````#T`
-M````````1`X0```````<````E!0`````````````1@````````!!#A"#`@``
-M`!0```"T%``````````````=`````````"0```#,%`````````````"5````
-M`````%,.((P"A@.#!``````````D````]!0`````````````X`````````!=
-M#C".`HT#C`2&!8,&````)````!P5`````````````)\`````````4PX@C`*&
-M`X,$`````````"P```!$%0````````````#_`````````$(.$$(.&$$.($$.
-M*$0.,(,%A@2,`XT"`#P```!T%0````````````!-`0```````$(.$$(.&$(.
-M($(.*$$.,$$..$0.0(,'A@:,!8T$C@./`@`````````<````M!4`````````
-M````4`````````!$#E```````"0```#4%0````````````#(`0```````&(.
-M0(\"C@.-!(P%A@:#!P`D````_!4`````````````W`$```````!B#F"/`HX#
-MC02,!88&@P<`'````"06`````````````"$`````````1`X0```````<````
-M1!8`````````````(@````````!$#A```````"0```!D%@````````````"!
-M`````````%T.,(X"C0.,!(8%@P8````<````C!8`````````````-P``````
-M``!!#A"#`@```!P```"L%@`````````````W`````````$$.$(,"````+```
-M`,P6`````````````-``````````0@X00@X800X@00XH1`XP@P6&!(P#C0(`
-M)````/P6`````````````*,`````````4PX@C`*&`X,$`````````!P````D
-M%P````````````#&`````````$$.$$0.4(,"'````$07`````````````#``
-M````````1`X0```````<````9!<`````````````8`````````!$#D``````
-M`"0```"$%P````````````#F`````````&(.<(\"C@.-!(P%A@:#!P`D````
-MK!<`````````````80,```````!E#J`!CP*.`XT$C`6&!H,')````-07````
-M`````````+D`````````0@X000X800X@@P2&`XP"`!P```#\%P``````````
-M```K`````````$$.$(,"````)````!P8`````````````)H`````````00X0
-M00X81`X@@P.&`@```!P```!$&``````````````]`````````$$.$(,"````
-M'````&08`````````````&@`````````3@X@C`*#`P`D````A!@`````````
-M````$@$```````!"#A!!#AA!#B"#!(8#C`(`)````*P8`````````````!(!
-M````````0@X000X800X@@P2&`XP"`!P```#4&`````````````"Q````````
-M`$$.$(,"````)````/08`````````````.0`````````70XPC@*-`XP$A@6#
-M!@```!P````<&0`````````````^`````````$0.$```````/````#P9````
-M`````````-0!````````0@X00@X80@X@0@XH00XP00XX1`Y`@P>&!HP%C02.
-M`X\"`````````"0```!\&0````````````!2`````````$$.$$$.&$0.((,#
-MA@(`````1T-#.B`H1TY5*2`S+C0N,B!;1G)E94)31%T@,C`P-#`W,C@``"YS
-M>6UT86(`+G-T<G1A8@`N<VAS=')T86(`+G)E;&$N=&5X=``N<F5L82YD871A
-M`"YB<W,`+G)E;&$N<F]D871A`"YR96QA+F5H7V9R86UE`"YC;VUM96YT````
+M`````````````````````````````````````!0``````````0`!>!`,!PB0
+M`0```````"P````<``````````````!L!@```````$<.T`1PCP*.`XT$C`6&
+M!H,'`````````"0```!,``````````````#W`````````$$.$$$.&$0.((,#
+MA@(````D````=```````````````*P(```````!$#C!4C0*,`X8$@P4`````
+M)````)P``````````````%0`````````0@X000X800X@@P2&`XP"`#P```#$
+M``````````````!*`0```````$(.$$(.&$(.($(.*$$.,$$..$<.P`2#!X8&
+MC`6-!(X#CP(````````T````!`$`````````````]P````````!"#A!"#AA"
+M#B!!#BA!#C!'#K`$@P:&!8P$C0..`@```!0````\`0````````````")````
+M`````!P```!4`0`````````````1`0```````$$.$(,"````)````'0!````
+M`````````%L`````````00X000X81`X@@P.&`@```"0```"<`0``````````
+M```T`0```````$0.0%Z/`HX#C02,!88&@P<<````Q`$`````````````10``
+M``````!$#A```````!P```#D`0````````````!L`````````$0.$```````
+M)`````0"`````````````&T`````````00X000X8@P.&`@```````"0````L
+M`@````````````"2`0```````$0.,%B.`HT#C`2&!8,&```<````5`(`````
+M````````@P````````!$#A```````"0```!T`@````````````"Z````````
+M`$0.($Z,`H8#@P0````````D````G`(`````````````\@````````!$#B!.
+MC`*&`X,$````````)````,0"`````````````%$!````````0@X000X800X@
+M@P2&`XP"`"0```#L`@`````````````"`0```````$0.($Z,`H8#@P0`````
+M```D````%`,`````````````WP$```````!$#C!4C0*,`X8$@P4`````)```
+M`#P#`````````````$,"````````1`Y`7H\"C@.-!(P%A@:#!R0```!D`P``
+M``````````"6`0```````$$.$$$.&$0.((,#A@(````D````C`,`````````
+M````CP$```````!'#K`$:(X"C0.,!(8%@P8`%````+0#`````````````#$`
+M````````%````,P#`````````````*4`````````)````.0#````````````
+M`%L`````````00X000X81`X@@P.&`@```"0````,!``````````````T`0``
+M`````$0.0%Z/`HX#C02,!88&@P<<````-`0`````````````+`````````!$
+M#A```````!0```!4!``````````````?`````````!0```!L!```````````
+M```7`````````#P```"$!`````````````"9`0```````$(.$$(.&$(.($(.
+M*$$.,$$..$0.0(,'A@:,!8T$C@./`@`````````D````Q`0`````````````
+M:@````````!!#A!!#AB#`X8"````````%````.P$`````````````$(`````
+M````)`````0%`````````````#<`````````00X000X81`X@@P.&`@```!0`
+M```L!0`````````````]`````````!P```!$!0`````````````8````````
+M`$0.$```````%````&0%`````````````#(`````````%````'P%````````
+M`````"4`````````)````)0%`````````````(4`````````1`XH4XT"C`.&
+M!(,%`````!0```"\!0`````````````H`````````"0```#4!0``````````
+M``!9`````````$(.$$$.&$$.((,$A@.,`@`\````_`4`````````````*0(`
+M``````!"#A!"#AA"#B!"#BA!#C!!#CA$#E"#!X8&C`6-!(X#CP(`````````
+M)````#P&`````````````,@`````````0@X000X800X@@P2&`XP"`#P```!D
+M!@`````````````Z`@```````$(.$$(.&$(.($(.*$$.,$$..$<.H`.#!X8&
+MC`6-!(X#CP(````````D````I`8`````````````]`````````!!#A!!#AA$
+M#B"#`X8"````)````,P&`````````````,,$````````1`Y07H\"C@.-!(P%
+MA@:#!QP```#T!@`````````````E`````````$0.$```````/````!0'````
+M`````````+4`````````0@X00@X80@X@0@XH00XP00XX1`Y`@P>&!HP%C02.
+M`X\"`````````!0```!4!P`````````````:`0```````"0```!L!P``````
+M``````#"`````````$0.($Z,`H8#@P0````````<````E`<`````````````
+M/@````````!$#B!*C`*#`R0```"T!P````````````"C`````````$0.,%2-
+M`HP#A@2#!0`````D````W`<`````````````C`$```````!"#A!!#AA!#B"#
+M!(8#C`(`+`````0(`````````````/H!````````1P[@`G"/`HX#C02,!88&
+M@P<`````````+````#0(`````````````/H`````````1P[0`G"/`HX#C02,
+M!88&@P<`````````/````&0(`````````````$H!````````0@X00@X80@X@
+M0@XH00XP00XX1P[``X,'A@:,!8T$C@./`@```````"0```"D"```````````
+M``!Y`````````$0.($Z,`H8#@P0````````4````S`@`````````````5```
+M```````\````Y`@`````````````.0(```````!"#A!"#AA"#B!"#BA!#C!!
+M#CA$#D"#!X8&C`6-!(X#CP(`````````/````"0)`````````````!<C````
+M````0@X00@X80@X@0@XH00XP00XX1PZP`X,'A@:,!8T$C@./`@```````#P`
+M``!D"0`````````````_`0```````$(.$$(.&$(.($(.*$$.,$$..$0.4(,'
+MA@:,!8T$C@./`@`````````D````I`D`````````````9@````````!!#A!!
+M#AA$#B"#`X8"````'````,P)`````````````$,!````````00X0@P(````D
+M````[`D`````````````<0````````!$#C!4C0*,`X8$@P4`````)````!0*
+M`````````````-T`````````00X000X81`X@@P.&`@```#P````\"@``````
+M``````#+`@```````$(.$$(.&$(.($(.*$$.,$$..$<.D`.#!X8&C`6-!(X#
+MCP(````````<````?`H`````````````@@````````!$#B!*A@*#`SP```"<
+M"@````````````!Y!0```````$(.$$(.&$(.($(.*$$.,$$..$0.4(,'A@:,
+M!8T$C@./`@`````````\````W`H`````````````(00```````!"#A!"#AA"
+M#B!"#BA!#C!!#CA$#I`!@P>&!HP%C02.`X\"````````'````!P+````````
+M`````"H`````````1`X0```````L````/`L`````````````F0(```````!"
+M#A!!#AA!#B!$#E"#!(8#C`(````````D````;`L`````````````:P``````
+M``!!#A!!#AA$#B"#`X8"````)````)0+`````````````!8!````````0@X0
+M00X800X@@P2&`XP"`"0```"\"P````````````!7`0```````$$.$$$.&$0.
+M((,#A@(````4````Y`L`````````````(P`````````D````_`L`````````
+M````=P````````!!#A!!#AA$#B"#`X8"````+````"0,`````````````(D"
+M````````0@X00@X800X@00XH1PZP!(,%A@2,`XT"%````%0,````````````
+M`$(`````````)````&P,`````````````$<`````````00X000X8@P.&`@``
+M`````!0```"4#``````````````J`````````!0```"L#``````````````.
+M`````````"P```#$#`````````````!D`0```````$(.$$(.&$$.($$.*$<.
+ML`B#!88$C`.-`A0```#T#`````````````!5`````````!P````,#0``````
+M``````!^`````````$$.$$0.((,"%````"P-`````````````%L`````````
+M'````$0-`````````````*,`````````00X01`X@@P(<````9`T`````````
+M````+P````````!!#A"#`@```!0```"$#0`````````````,`````````!0`
+M``"<#0`````````````H`````````"0```"T#0````````````!Z````````
+M`$0.,%2-`HP#A@2#!0`````<````W`T`````````````%0````````!$#A``
+M`````!P```#\#0`````````````G`````````$0.$```````)````!P.````
+M`````````&8!````````1`YP7H\"C@.-!(P%A@:#!Q0```!$#@``````````
+M```F`````````"0```!<#@````````````!B`````````$(.$$$.&$$.((,$
+MA@.,`@`<````A`X`````````````3P````````!$#B!*C`*#`QP```"D#@``
+M```````````@`````````$$.$(,"````%````,0.`````````````,`"````
+M````)````-P.`````````````(D!````````1`Y07H\"C@.-!(P%A@:#!R0`
+M```$#P````````````#A`````````$0.<%Z/`HX#C02,!88&@P<D````+`\`
+M````````````DP$```````!$#G!>CP*.`XT$C`6&!H,')````%0/````````
+M`````+$`````````1`XP5(T"C`.&!(,%`````"0```!\#P````````````#X
+M`@```````$0.4%Z/`HX#C02,!88&@P<<````I`\`````````````>@``````
+M``!$#A```````"0```#$#P````````````"Y`0```````$0.0%Z/`HX#C02,
+M!88&@P<D````[`\`````````````P`````````!"#A!!#AA!#B"#!(8#C`(`
+M)````!00`````````````&\!````````1`Y068X"C0.,!(8%@P8``"P````\
+M$`````````````"[`0```````$(.$$(.&$$.($$.*$0.,(,%A@2,`XT"`#P`
+M``!L$`````````````#1`````````$(.$$(.&$(.($(.*$$.,$$..$0.<(,'
+MA@:,!8T$C@./`@`````````D````K!``````````````.@$```````!$#C!4
+MC0*,`X8$@P4`````)````-00`````````````)T`````````1`X@3HT"C`.#
+M!````````#P```#\$``````````````%`@```````$(.$$(.&$(.($(.*$$.
+M,$$..$0.0(,'A@:,!8T$C@./`@`````````<````/!$`````````````'```
+M``````!$#A```````!P```!<$0`````````````G`0```````$$.$(,"````
+M%````'P1`````````````$(`````````%````)01`````````````&T`````
+M````)````*P1`````````````(``````````1`X@3HP"A@.#!````````!P`
+M``#4$0````````````!@`````````$0.&$F&`H,#%````/01````````````
+M`"@`````````)`````P2`````````````*8`````````0@X000X800X@@P2&
+M`XP"`!P````T$@````````````!?`````````$0.($J&`H,#)````%02````
+M`````````-$`````````1`XP6(X"C0.,!(8%@P8``#0```!\$@``````````
+M``"3`````````$(.$$(.&$(.($$.*$$.,(,&A@6,!(T#C@(`````````)```
+M`+02`````````````&D`````````0@X000X81`X@@P.,`@```!P```#<$@``
+M``````````!%`````````$$.$(,"````)````/P2`````````````$`"````
+M````1`XP5(T"C`.&!(,%`````"0````D$P````````````"``P```````$0.
+M,%B.`HT#C`2&!8,&```<````3!,`````````````(0````````!$#A``````
+M`!0```!L$P`````````````+`````````!0```"$$P`````````````+````
+M`````!P```"<$P````````````!@`````````$0.($J&`H,#%````+P3````
+M`````````",`````````)````-03`````````````(P`````````00X000X8
+M1`X@@P.&`@```#P```#\$P`````````````/`@```````$(.$$(.&$(.($(.
+M*$$.,$$..$0.<(,'A@:,!8T$C@./`@`````````D````/!0`````````````
+MQ0````````!$#C!8C@*-`XP$A@6#!@``'````&04`````````````$,`````
+M````1`X0```````<````A!0`````````````/0````````!$#A```````!P`
+M``"D%`````````````!&`````````$$.$(,"````%````,04````````````
+M`!T`````````)````-P4`````````````-T`````````1`XP6(X"C0.,!(8%
+M@P8``"0````$%0````````````">`````````$0.($Z,`H8#@P0````````L
+M````+!4`````````````_P````````!"#A!"#AA!#B!!#BA$#C"#!88$C`.-
+M`@`\````7!4`````````````30$```````!"#A!"#AA"#B!"#BA!#C!!#CA$
+M#D"#!X8&C`6-!(X#CP(`````````)````)P5`````````````*(!````````
+M1`XP6(X"C0.,!(8%@P8``"0```#$%0`````````````#`@```````$0.8%Z/
+M`HX#C02,!88&@P<<````[!4`````````````(0````````!$#A```````!P`
+M```,%@`````````````B`````````$0.$```````)````"P6````````````
+M`*\`````````1`Y`7H\"C@.-!(P%A@:#!QP```!4%@`````````````W````
+M`````$$.$(,"````'````'06`````````````#<`````````00X0@P(````D
+M````E!8`````````````I@````````!$#B!.C`*&`X,$````````+````+P6
+M`````````````!\!````````0@X00@X800X@00XH1`Y@@P6&!(P#C0(`'```
+M`.P6`````````````#``````````1`X0```````<````#!<`````````````
+M8`````````!$#D```````"0````L%P````````````#F`````````$0.<%Z/
+M`HX#C02,!88&@P<L````5!<`````````````P0,```````!'#J`!9X\"C@.-
+M!(P%A@:#!P`````````D````A!<`````````````N0````````!"#A!!#AA!
+M#B"#!(8#C`(`'````*P7`````````````"L`````````00X0@P(````D````
+MS!<`````````````F@````````!!#A!!#AA$#B"#`X8"````'````/07````
+M`````````#T`````````00X0@P(````<````%!@`````````````:```````
+M``!$#B!*C`*#`R0````T&``````````````[`0```````$0.($Z,`H8#@P0`
+M```````D````7!@`````````````$@$```````!"#A!!#AA!#B"#!(8#C`(`
+M'````(08`````````````+$`````````00X0@P(````D````I!@`````````
+M````[@````````!$#C!8C@*-`XP$A@6#!@``'````,P8`````````````#X`
+M````````1`X0```````\````[!@`````````````U`$```````!"#A!"#AA"
+M#B!"#BA!#C!!#CA$#D"#!X8&C`6-!(X#CP(`````````)````"P9````````
+M`````%(`````````00X000X81`X@@P.&`@```!0```!4&0`````````````F
+M`````````"0```!L&0````````````!&`0```````$0.0%Z/`HX#C02,!88&
+M@P<`1T-#.B`H1TY5*2`S+C0N,B!;1G)E94)31%T@,C`P-#`W,C@``"YS>6UT
+M86(`+G-T<G1A8@`N<VAS=')T86(`+G)E;&$N=&5X=``N<F5L82YD871A`"YB
+M<W,`+G)E;&$N<F]D871A`"YR96QA+F5H7V9R86UE`"YC;VUM96YT````````
 M````````````````````````````````````````````````````````````
-M```````````````````````@`````0````8```````````````````!`````
-M`````#+!```````````````````0````````````````````&P````0`````
-M````````````````````T`$!``````#X-`````````P````!````"```````
-M```8`````````"L````!`````P```````````````````(#!````````#`$`
-M`````````````````"`````````````````````F````!```````````````
-M``````````#(-@$``````(`!````````#`````,````(`````````!@`````
-M````,0````@````#````````````````````D,(````````0````````````
-M````````"````````````````````#L````!`````@``````````````````
-M`)#"````````<`````````````````````@````````````````````V````
-M!`````````````````````````!(.`$``````%`!````````#`````8````(
-M`````````!@`````````2`````$````"`````````````````````,,`````
-M``"@&0``````````````````"````````````````````$,````$````````
-M`````````````````)@Y`0``````(!`````````,````"`````@`````````
-M&`````````!2`````0````````````````````````"@W````````"4`````
-M```````````````!````````````````````$0````,`````````````````
-M````````Q=P```````!;`````````````````````0``````````````````
-M``$````"`````````````````````````*#@````````L!,````````-````
-M8@````@`````````&``````````)`````P````````````````````````!0
-M]````````'P-```````````````````!````````````````````````````
-M`````````````````````````0````0`\?\`````````````````````````
-M``,``0````````````````````````````,``P``````````````````````
-M``````,`!0``````````````````````"`````(``0!0"P```````'$`````
-M````&0````(``0#0"P```````/L`````````+`````(``0`@#P```````&T`
-M````````.0````(``0#0%0```````,<!````````1@````(``0#`'```````
-M`"@`````````50````(``0#P'````````*0`````````9@````(``0!P'P``
-M`````!\`````````<@````(``0"0'P```````!<`````````?0````$`!0``
-M``````````@`````````B@````(``0"P'P```````)D!````````F@````(`
-M`0!0(0```````&H`````````HP````(``0#`(0```````$(`````````JP``
-M``(``0`0(@```````#<`````````LP````(``0!0(@```````#T`````````
-MO@````(``0"0(@```````!@`````````Q@````(``0"P(@```````#(`````
-M````V`````(``0#P(@```````"4`````````YP````(``0`@(P```````(0`
-M````````\@````(``0"P(P```````"@`````````_P````(``0#@(P``````
-M`%D`````````"P$```(``0!`)````````"D"````````&`$```(``0!P)@``
-M`````,(`````````)0$```(``0!`)P```````#4"````````+P$```(``0"`
-M*0```````/$`````````/@$```(``0!P8@```````-T`````````20$```(`
-M`0"`*@```````*,$````````50$```(``0#0.P```````!<C````````8P$`
-M``(``0"@8````````$,!````````;P$```(``0`P+P```````"4`````````
-M>0$```(``0"0.0```````#D"````````BP$```(``0!@+P```````+4`````
-M````G`$```(``0`@,````````!H!````````J0$```(``0!`,0```````,(`
-M````````M@$```(``0!0,@```````*``````````PP$```(``0`0,@``````
-M`#X`````````S@$```(``0#P,@```````(P!````````V0$```(``0"`-```
-M`````.H!````````Y0$```(``0"P.````````'H`````````\P$```(``0!P
-M-@```````.@```````````(```(``0!@-P```````$H!````````#0(```(`
-M`0`P.0```````%0```````````````,`!@``````````````````````'@(`
-M``(``0!08P```````+$"````````+@(```(``0`09@```````'\`````````
-M10(```(``0"09@```````#\%````````4`(```(``0#0:P```````.4#````
-M````7`(```(``0#`;P```````"H`````````;`(```(``0#P=0```````'H"
-M````````>0(```(``0!0?P```````"8`````````B@(```(``0#P?P``````
-M`$\`````````GP(```(``0!`@````````"``````````L@(```(``0!@@```
-M`````#@`````````OP(```(``0"@@````````,`"````````U0(```(``0!@
-M@P```````(\!````````[`(```(``0#PA````````.$`````````!`,```(`
-M`0#`F````````&$`````````$@,```(``0#@A0```````),!````````)0,`
-M``(``0``FP```````'H`````````-`,```(``0!`G````````"L"````````
-M2`,```(``0!`C@```````&\!````````6P,```(``0"`AP```````+$`````
-M````:@,```(``0!`B`````````,#````````?@,```(``0!PDP```````)X`
-M````````D`,```(``0`PD@```````#H!````````G@,```(``0#PFP``````
-M`$4`````````JP,```(``0!0BP```````'H`````````P0,```(``0#PF0``
-M`````"T`````````V`,```(``0#0BP```````*,!````````ZP,```(``0"`
-MC0```````,``````````^P,```(``0`@F@```````-(`````````"`0```(`
-M`0"`FP```````&D`````````&P0```(``0"PCP```````)L!````````+`0`
-M``(``0!0D0```````-$`````````.@0```(``0`0E`````````4"````````
-M2P0```(``0`@E@```````!P`````````9`0```(``0!`E@```````"<!````
-M````>P0```(``0!PEP```````$(`````````D`0```(``0#`EP```````&T`
-M````````I`0```(``0`PF````````($`````````KP0```(``0`PF0``````
-M`"@`````````P00```(``0!@F0```````(<`````````UP0```(``0!0L0``
-M`````-``````````[P0```(``0"0N````````+D`````````_`0```$``P#0
-M`````````!P`````````!@4```$``P#P`````````!P`````````$`4```(`
-M`0"`N0```````)H`````````&04```$`!0`(``````````@`````````(04`
-M``(``0`@N@```````#T`````````*04```(``0!@N@```````&@`````````
-M-`4```(``0`0O0```````+$`````````/`4```(``0``OP```````-0!````
-M``````````,`"`````````````````````````````,`"@``````````````
-M````````5`4``!(``0```````````)P%````````8`4``!$``P`@````````
-M`$@`````````;P4``!``````````````````````````?`4``!$``P"`````
-M`````$@`````````C`4``!(``0!`=0```````",`````````E04``!``````
-M````````````````````HP4``!``````````````````````````K04``!``
-M````````````````````````MP4``!(``0"@!0```````.P`````````P04`
-M`!``````````````````````````S@4``!``````````````````````````
-MX`4``!(``0"0!@````````D"````````\`4``!``````````````````````
-M````^04``!(``0!P=0```````'<`````````!P8``!(``0"@"````````%0`
-M````````&@8``!(``0!`>0````````X`````````*`8``!(``0``"0``````
-M`$H!````````-08``!(``0`0>0```````"H`````````1P8``!(``0!@<@``
-M`````&L`````````4@8``!(``0!0"@```````/0`````````908``!(``0#0
-M#````````%L`````````=@8``!(``0#@?`````````P`````````@@8``!(`
-M`0#`>@```````%4`````````D@8``!(``0`P#0```````#0!````````I`8`
-M`!(``0"P?````````"\`````````M`8``!(``0!P#@```````#L`````````
-MQP8``!(``0"P#@```````&P`````````V08``!(``0"0#P```````)(!````
-M````[P8``!(``0`P$0```````(,`````````#`<``!(``0#`$0```````+8`
-M````````(`<``!(``0`@?0```````'H`````````+@<``!(``0"`$@``````
-M`.X`````````10<``!(``0!P$P```````%$!````````70<``!(``0#0%```
-M`````/X`````````<P<``!``````````````````````````?0<``!(``0"@
-M%P```````$,"````````CP<``!(``0#P?````````"@`````````H`<``!(`
-M`0#P&0```````$\!````````LP<``!(``0!`&P```````'\!````````Q`<`
-M`!(``0"@'0```````%L`````````U`<``!(``0``'@```````#0!````````
-MY0<``!(``0!`'P```````"D`````````]P<``!$``P````````````0`````
-M````!P@``!``````````````````````````%0@``!(``0#P7@```````#\!
-M````````*`@``!``````````````````````````.P@``!``````````````
-M````````````0@@``!``````````````````````````20@``!(``0`P8```
-M`````&8`````````50@``!``````````````````````````=P@``!(``0``
-M?````````*,`````````A0@``!(``0#P80```````'$`````````EP@``!(`
-M`0#P;P```````&D"````````J0@``!(``0#P<P```````$<!````````O`@`
-M`!(``0#0<@```````!8!````````S@@``!``````````````````````````
-MX`@``!``````````````````````````\P@``!(``0!P>````````$(`````
-M`````@D``!(``0#`>````````$<`````````$0D``!(``0!0>0```````&0!
-M````````(`D``!(``0`@>P```````'X`````````,0D``!(``0"@>P``````
-M`%L`````````/@D``!(``0"@?0```````!4`````````20D``!(``0#`?0``
-M`````"<`````````60D``!(``0#P?0```````%X!````````90D``!(``0#`
-MI@```````#T`````````?0D``!(``0``IP```````$8`````````D0D``!(`
-M`0#0LP```````&``````````K`D``!(``0`PM````````.8`````````T0D`
-M`!(``0!PI@```````$,`````````Z`D``!(``0"`?P```````&(`````````
-M_PD``!``````````````````````````$PH``!``````````````````````
-M````)`H``!``````````````````````````-PH``!$``P#(``````````0`
-M````````2@H``!``````````````````````````7`H``!``````````````
-M````````````;@H``!``````````````````````````@`H``!``````````
-M````````````````D0H``!(``0!0N0```````"L`````````H`H``!(``0!`
-ML````````($`````````N@H``!(``0!PG@```````%T#````````S`H``!(`
-M`0"PH@```````(P`````````ZPH``!(``0#0H0```````"$``````````0L`
-M`!(``0``H@````````L`````````$PL``!(``0`0H@````````L`````````
-M)@L``!(``0`@H@```````&``````````/0L``!(``0"`H@```````",`````
-M````40L``!(``0!`HP```````#,"````````:`L``!(``0"`I0```````.0`
-M````````?0L``!(``0!0IP```````!T`````````E`L``!(``0!PIP``````
-M`)4`````````M0L``!(``0`0J````````.``````````SPL``!(``0#PJ```
-M`````)\`````````Z`L``!(``0"0J0```````/\```````````P``!(``0"0
-MJ@```````$T!````````%PP``!(``0#@JP```````%``````````+@P``!(`
-M`0`PK````````,@!````````00P``!(``0``K@```````-P!````````7PP`
-M`!(``0#@KP```````"$`````````>@P``!(``0`0L````````"(`````````
-MEPP``!(``0#0L````````#<`````````L0P``!(``0`0L0```````#<`````
-M````S`P``!(``0`@L@```````*,`````````Z0P``!(``0#0L@```````,8`
-M````````!PT``!(``0"@LP```````#``````````)@T``!(``0`@M0``````
-M`&$#````````/`T``!(``0#0N@```````!(!````````0PT``!(``0#PNP``
-M`````!(!````````2PT``!(``0#0O0```````.0`````````60T``!(``0#`
-MO@```````#X`````````9@T``!(``0#@P````````%(``````````')A:60N
-M8P!3=')I<&5'971#;VUM86YD`%-T<FEP95-P;&ET0V]M;6%N9`!)<U9$979/
-M;FQI;F4`1&]#;VYT<F]L0VUD`$I"3T1'971#;VUM86YD`$I"3T13<&QI=$-O
-M;6UA;F0`7U]H<'1?86QL;V,`7U]H<'1?9G)E90!G87)B86=E7V%D9'(`:6YI
-M=%]H96%P7V)L;V-K`&-B7V%L;&]C`&-B7V9R964`8V)?>F5R;P!C8E]I<U]Z
-M97)O`&-F7VEN:70`<F5M;W9E7W)A;F=E7VQO8VL`861D7W)A;F=E7VQO8VL`
-M;&]C:U]R86YG90!?7V%L;&]C7W1A<VL`9G)E95]S=')I<&4`86QL;V-?<W1R
-M:7!E`%]?9V5T7W-T<FEP90!D871A7WAF97(`<F5A9%]X9F5R7V1O;F4`<F%I
-M9#5?9&]N90!S=')I<&5?9&]N90!H86YD;&5?<W1R:7!E`')A:60U7V1I<G1Y
-M`'1A<VM?9&]N90!S=')I<&5?8VAE8VM?=&%S:P!R=U]T87-K7V)U:6QD<V=L
-M`')W7V=E=%]R86YG90!R=U]T87-K7V1O;F4`7U]D;U]R=U]T87-K`&1O7W)W
-M7W1A<VL`9V5T7W-L:6-E<P!D;U]X;W)?=&%S:P!X;W)?=&%S:U]D;VYE`&1O
-M7WAO<C)?=&%S:P!D;U]X;W)N7W1A<VL`8VAE8VM?9&5P96YD96YC>0!D:7)E
-M8W1?<F5A9%]S9VP`9&ER96-T7W)E861?8V]M<&QE=&EO;@!H86YD;&5?8VUD
-M`&=E=%]S=')I<&5S`'=R:71E7WAF97)?9&]N90!3>6YC1&ES:TEN9F\`9V5T
-M161M85)E9T]F9G-E=`!U;FUA<VM%9&UA26YT97)R=7!T<P!M87-K161M84EN
-M=&5R<G5P=',`:7-%>'1#;VUM86YD`'=R:71E161M85)E<75E<W1%;G1R>0!D
-M=6UP071A1&5V:6-E4F5G:7-T97)S`&AA;F1L945D;6%&86EL961#;VUM86YD
-M`')E;6]V94-O;6UA;F0`:&%N9&QE161M85)E<W!O;G-E`&1E86-T:79A=&5%
-M9&UA`'-E;F1.;VYE561M84-O;6UA;F0`8V]M<&QE=&5024]#;VUM86YD`%-E
-M=$=024]#=')L4F5G`&AA;F1L945D;6%);G1E<G)U<'0`7V-H86YN96Q(87)D
-M4F5S970`7V9I>%!H>5!A<F%M<P!?9&]!=71O1FQU<V@`:&%N9&QE1&5V:6-E
-M26YT97)R=7!T`&1I<V%B;&53841E=DEN=&5R<G5T<',`:&%N9&QE4$E/26YT
-M97)R=7!T`'1R86YS9F5R4$E/1&%T80!A8W1I=F%T945D;6$`161M85)E<5%U
-M975E26YS97)T`')E<V5T161M84-H86YN96P`9FQU<VA$;6%1=65U90!R979E
-M<G13871A2$-296=S`')E=F5R=$9L87-H26YT97)F86-E4F5G<P!R979E<G10
-M0TE);G1E<F9A8V5296=S`&-O;6UA;F1S475E=65!9&1486EL`&-O;6UA;F1S
-M475E=65296UO=F4`861D0V]M;6%N9`!3841E=DEN=&5R<G5T<$)I=`!E;F%B
-M;&53841E=DEN=&5R<G5T<',`:7-3=&]R86=E1&5V4F5A9'E&;W)024\`7V1O
-M4V]F=%)E<V5T`%]A9&1R36%P,0!?861D<DUA<#(`16YT97(V,#$`861D<DUA
-M<`!%>&ET-C`Q`&=E=%\V,#%?:60`<V5T7V-T;`!?7V-H96-K7W!R;W1E8W1?
-M8VER8W5I=`!F0VAE8VM!<G)A>0!P9FY396YD0V]M;6%N9`!F1&5296%D5W)I
-M=&4`<&9N1&5V:6-E1F%I;&5D`$-H96-K4W5M`&9$95-E;&5C=$UO9&4`;W-?
-M;65M<V5T`&]S7VUE;6-P>0!F061D4W!A<F4`1V5T4W!A<F5$:7-K`&EO8W1L
-M7U)E<&]R=$5V96YT`$9I>'5P07)R87E3=&%T90!'9713=&%M<`!3>6YC07)R
-M87E);F9O`$-H96-K07)R87E#<FET:6-A;`!F3W-$:7-K1F%I;&5D`&9$96QE
-M=&5!<G)A>0!5;G)E9VES=&5R5D1E=FEC90!R86ED-5]F<F5E`%-T<FEP94)U
-M:6QD4V=486)L90!F4D%)1#!-96UB97)$;VYE`$9R965#;VUM86YD`$-A;&Q!
-M9G1E<E)E='5R;@!F4D%)1#!396YD0V]M;6%N9`!!;&QO8V%T94-O;6UA;F0`
-M9E)!240P365M8F5R1F%I;&5D`%)!240Q0G5I;&139U1A8FQE`%)!240Q365M
-M8F5R0V]M<&QE=&EO;@!204E$,4EN:71296)U:6QD0G5I;&139U1A8FQE`%)!
-M240Q26YI=$-O;7!L971I;VX`1&]786ET:6YG3&ES=`!204E$,5)E8G5I;&1#
-M;VUP;&5T:6]N`%)!240Q5F5R:69Y0G5I;&139U1A8FQE`%)!240Q5F5R:69Y
-M0V]M<&QE=&EO;@!O<U]M96UC;7``9E)!240Q4V5N9$-O;6UA;F0`061D5&]7
-M86ET:6YG3&ES=`!F4D%)1#%-96UB97)&86EL960`2D)/1$)U:6QD4V=486)L
-M90!F2D)/1$UE;6)E<D1O;F4`9DI"3T1396YD0V]M;6%N9`!F2D)/1$UE;6)E
-M<D9A:6QE9`!N=6U?<F%I9#5?<&%G97,`;W-?86QL;V-?<&%G90!F;'5S:%]S
-M=')I<&5?8V%C:&4`9D]S4&AY<VEC86Q!9&1R97-S`$1O6&]R,@!$;UAO<C$`
-M9FQU<VA?<F%I9#4`9D-O;7!L971E06QL0V]M;6%N9'-3>6YC:')O;F]U<VQY
-M`$-H96-K261L94-A;&P`9FQU<VA?<F%I9#5?87-Y;F,`9E)!240U4V5N9$-O
-M;6UA;F0`9E)!240U365M8F5R1F%I;&5D`&EN:71?<F%I9#5?;65M;W)Y`&]S
-M7V%L;&]C7V1M85]P86=E`&9$979I8V5396YD0V]M;6%N9`!F1V5T1FER<W1#
-M:&EL9`!F4F5S971";V]T36%R:P!F0VAE8VM";V]T86)L90!#:&5C:U!E;F1I
-M;F=#86QL`$-A;&Q7:&5N261L90!F1FQU<VA61&5V`&9&;'5S:%9$979!<WEN
-M8P!M=E)E8617<FET90!M=E-A=&%$:7-A8FQE0VAA;FYE;$1M80!M=E-A=&%&
-M;'5S:$1M85%U975E`&UV4W1O<F%G941E=D%405-E=$9E871U<F5S`&UV4W1O
-M<F%G941E=D%4045X96-U=&5.;VY51$U!0V]M;6%N9`!M=E-A=&%%;F%B;&5#
-M:&%N;F5L1&UA`'=A:71&;W)"=7-Y069T97)(4F5S970`;79-:6-R;U-E8V]N
-M9'-$96QA>0!-5E]214=?4D5!1%]"651%`$U67U)%1U]74DE415]$5T]21`!M
-M=D%U=&]&;'5S:$]N17)R;W(`359?4D5'7U=2251%7T)95$4`359?4D5'7U)%
-M041?1%=/4D0`359?4D5'7U=2251%7U=/4D0`359?4D5'7U)%041?5T]21`!?
-M9&]-=E-O9G1297-E=`!W86ET5VAI;&53=&]R86=E1&5V27-"=7-Y`&UV4V%T
-M84EN:71!9&%P=&5R`&UV4V%T84ES4W1O<F%G941E=FEC94-O;FYE8W1E9`!M
-M=E-A=&%3:'5T9&]W;D%D87!T97(`;79%;F%B;&5!=71O1FQU<V@`;79$:7-A
-M8FQE075T;T9L=7-H`&UV4V%T84-O;F9I9W5R94-H86YN96P`;793871A4F5M
-M;W9E0VAA;FYE;`!M=E-A=&%#:&%N;F5L2&%R9%)E<V5T`&UV4V%T84-O;F9I
-M9T5D;6%-;V1E`&UV4V%T84YU;4]F1&UA0V]M;6%N9',`;793871A0VAA;FYE
-M;%-E=$5D;6%,;V]P0F%C:TUO9&4`;793871A4V5T0VAA;FYE;%!H>5!A<F%M
-M<P!M=E-A=&%#:&%N;F5L4&AY4VAU=&1O=VX`;793871A0VAA;FYE;%!H>5!O
-M=V5R3VX`;793871A1V5T0VAA;FYE;%-T871U<P!M=E-A=&%1=65U955$;6%#
-M;VUM86YD`&UV4V%T85%U975E0V]M;6%N9`!M=E-A=&%);G1E<G)U<'1397)V
-M:6-E4F]U=&EN90!M=E-A=&%-87-K061A<'1E<DEN=&5R<G5P=`!M=E-A=&%5
-M;FUA<VM!9&%P=&5R26YT97)R=7!T`&5N86)L95-T;W)A9V5$979);G1E<G)U
-M<'0`9&ES86)L95-T;W)A9V5$979);G1E<G)U<'0`;793=&]R86=E1&5V051!
-M261L94EM;65D:6%T90!M=E-T;W)A9V5$979!5$%)9&5N=&EF>41E=FEC90!M
-M=E-T;W)A9V5$979!5$%3;V9T4F5S971$979I8V4`97AE8W5T94YO;E5$34%#
-M;VUM86YD`$)E97!/;@!"965P3V9F`'-E=%]F86EL7VQE9',`<V5T7V9A:6Q?
-M;&5D`&-H96-K7W!R;W1E8W1?8VER8W5I=```````<0`````````+````8P``
-M````````````EP`````````"````9````/S_________K@`````````+````
-M90``````````````R``````````"````9@```/S_________\0`````````"
-M````9P```/S__________@`````````"````9@```/S_________&0$`````
-M```"````:````/S_________I0$````````"````90```/S_________TP$`
-M```````+````8P``````````````8@(````````"````:````/S_________
-MYP(````````+````8P```````````````0,````````"````:0```/S_____
-M____%P,````````"````:0```/S_________,`,````````"````:0```/S_
-M________1@,````````"````:0```/S_________OP,````````"````:```
-M`/S_________2P0````````+````8P``````````````6P0````````+````
-M90``````````````B@0````````"````:0```/S_________H`0````````"
-M````:0```/S_________N00````````"````:0```/S_________SP0`````
-M```"````:0```/S_________$@4````````+````90``````````````7`4`
-M```````+````90``````````````[04````````"````:P```/S_________
-M(`8````````"````8P```!0`````````,`8````````+````90``````````
-M````?08````````"````;````/S_________+0<````````"````;0```/S_
-M________HP<````````"````:@```/S_________0@@````````"````:@``
-M`/S_________;@@````````"````;@```/S_________>0@````````"````
-M;P```/S_________U@@````````"````;0```/S_________Y`@````````+
-M````<0``````````````(0D````````"````<P```/S_________,@D`````
-M```"````=````/S_________;0D````````"````<@```/S_________GPD`
-M```````+````<0``````````````PPD````````"````:````/S_________
-MV0D````````"````9````/S_________]@D````````"````;P```/S_____
-M____&PH````````"````:````/S_________``T````````"````=P```/S_
-M________(`T````````"````>````/S_________;@T````````"````>```
-M`/S_________E0T````````"````>````/S_________P@T````````"````
-M>@```/S_________%@X````````+````=@``````````````'@X````````+
-M````=0``````````````'A`````````"````:````/S_________<1``````
-M```"````:````/S_________KA`````````"````=P```/S_________WA``
-M```````"````>````/S_________`Q(````````"````=P```/S_________
-M/A(````````"````>````/S_________5Q(````````+````@```````````
-M````7Q(````````"````>````/S_________UQ(````````"````:````/S_
-M________)Q,````````"````=P```/S_________-A,````````"````>```
-M`/S_________3Q,````````+````@```````````````5Q,````````"````
-M>````/S_________*A4````````"````:````/S_________414````````"
-M````A````/S_________AQ4````````"````=P```/S_________EA4`````
-M```"````>````/S_________KQ4````````+````@```````````````MQ4`
-M```````"````>````/S_________318````````"````:````/S_________
-M818````````"````>@```/S_________AA8````````+````?P``````````
-M````CA8````````+````?@``````````````PQ8````````"````>@```/S_
-M________Z!8````````+````@0``````````````\!8````````+````?@``
-M````````````%1<````````"````>@```/S_________.A<````````+````
-M@P``````````````0A<````````+````@@``````````````9Q<````````"
-M````>````/S_________<A<````````+````@```````````````>A<`````
-M```"````>````/S_________W1<````````"````>````/S_________\!<`
-M```````+`````@```-`5````````11@````````+````A0``````````````
-M41@````````"````A@```/S_________7AD````````"````>@```/S_____
-M____<1D````````"````:0```/S_________C1D````````+````?0``````
-M````````E1D````````+````?```````````````HAH````````"````:P``
-M`/S_________O!H````````"````8P```!0`````````QQH````````"````
-M90```"P`````````"AL````````"````;````/S_________*1L````````"
-M````;@```/S_________-!L````````"````;P```/S_________AAP`````
-M```"````:0```/S_________T!T````````"````=P```/S_________\!T`
-M```````"````>````/S_________/AX````````"````>````/S_________
-M91X````````"````>````/S_________DAX````````"````>@```/S_____
-M____YAX````````+````B0``````````````[AX````````+````B```````
-M````````=B`````````"````C0```/S_________-2(````````"````:```
-M`/S_________GR(````````"````:````/S_________WR@````````"````
-M:0```/S_________\2@````````"````:0```/S_________,"T````````+
-M`````@```(`I````````+BX````````+`````@```'!B````````,RX`````
-M```"````>````/S_________;2X````````+````@```````````````=2X`
-M```````"````>````/S_________CRX````````+````@```````````````
-MERX````````"````>````/S_________URX````````"````>````/S_____
-M_____"X````````"````C@```/S_________1"\````````+`````@```)`Y
-M````````22\````````"````>````/S_________SR\````````"````CP``
-M`/S_________93$````````"````=P```/S_________@S(````````"````
-M>@```/S_________OS(````````+`````@```&`O````````RS(````````+
-M`````@```$`Q````````;S4````````"````D````/S_________D34`````
-M```"````D0```/S_________$38````````"````D````/S_________!3<`
-M```````"````D````/S_________63@````````"````D````/S_________
-M>#@````````"````D0```/S_________C3H````````+````+@``````````
-M````/SL````````+````+@```#@`````````Q%\````````"````>````/S_
-M________9&`````````"````C@```/S_________;&`````````"````DP``
-M`/S_________=&`````````"````E````/S_________1&(````````"````
-MC@```/S_________QV(````````"````>````/S_________W&(````````+
-M````@```````````````Y&(````````"````>````/S_________%&,`````
-M```+````@```````````````'&,````````"````>````/S_________0F,`
-M```````"````C@```/S_________360````````"````!````/S_________
-M5F0````````"````!````/S_________6V0````````"````CP```/S_____
-M____-F8````````"````=P```/S_________SV@````````"````:````/S_
-M_________FH````````+````@```````````````!FL````````"````>```
-M`/S_________/VL````````"````>@```/S_________:FL````````+````
-M`@```%!C````````<FL````````+`````@```!!F````````TF\````````+
-M`````@```'!B````````UV\````````"````>````/S_________$W``````
-M```"````>````/S_________1W`````````"````:````/S_________;W``
-M```````"````>````/S_________D7`````````"````>````/S_________
-M*7$````````"````>````/S_________7W$````````+````E@``````````
-M````:W$````````"````A@```/S_________MW$````````+````EP``````
-M````````\G$````````+````E@```````````````G(````````"````A@``
-M`/S_________"G(````````"````C@```/S_________&7(````````+````
-M`@```,!O````````.7(````````"````>````/S_________=W(````````"
-M````D@```/S_________(G,````````"````C````/S_________9',`````
-M```"````C````/S_________E7,````````"````F0```/S_________G',`
-M```````"````!````/S_________I',````````"````C````/S_________
-MM',````````"````F0```/S_________VW,````````"````C````/S_____
-M____J'0````````"````:P```/S_________R70````````"````8P```!0`
-M````````U'0````````"````90```#P`````````"G4````````"````;```
-M`/S_________(74````````"````;@```/S_________+'4````````"````
-M;P```/S_________PG4````````"````;P```/S_________.'8````````"
-M````:````/S_________8W<````````"````:0```/S_________>'<`````
-M```"````:0```/S_________D'<````````"````:0```/S_________I7<`
-M```````"````:0```/S_________(7@````````"````9@```/S_________
-M,W@````````"````9@```/S_________6'@````````"````9````/S_____
-M____JG@````````"````FP```/S_________['@````````"````G````/S_
-M________2'D````````"````<P```/S_________E'D````````"````:```
-M`/S_________]GD````````"````9````/S_________0'H````````"````
-M9````/S_________@'P````````"````DP```/S_________UGP````````"
-M````:````/S_________JGT````````"````D@```/S_________T7T`````
-M```"````E0```/S_________+7X````````"````HP```/S_________/'X`
-M```````"````I````/S_________8WX````````"````I0```/S_________
-M%7\````````"````I@```/S_________(W\````````"````IP```/S_____
-M____CW\````````"````J0```/S_________KW\````````"````J@```/S_
-M________QG\````````"````J0```/S_________&H`````````"````JP``
-M`/S_________+(`````````"````JP```/S_________6H`````````"````
-MJP```/S_________IH,````````"````J@```/S_________OH,````````"
-M````J@```/S_________UX,````````"````J@```/S_________\8,`````
-M```"````J@```/S_________"X0````````"````J@```/S_________,X0`
-M```````"````J@```/S_________2X0````````"````K0```/S_________
-M5X0````````"````J@```/S_________:(0````````"````J@```/S_____
-M____>H0````````"````J@```/S_________C(0````````"````J@```/S_
-M________I(0````````"````K0```/S_________LX0````````"````J@``
-M`/S_________Q80````````"````J@```/S_________,(4````````"````
-MK@```/S_________JH<````````"````K@```/S_________P(<````````"
-M````JP```/S_________SH<````````"````K@```/S_________W(<`````
-M```"````K@```/S_________]8<````````"````JP```/S_________`X@`
-M```````"````K@```/S_________%(@````````"````JP```/S_________
-MPH@````````"````JP```/S_________.XD````````"````K@```/S_____
-M____48D````````"````JP```/S__________HD````````"````JP```/S_
-M________"XH````````"````K@```/S_________%8H````````"````J0``
-M`/S_________)XH````````"````JP```/S_________-(H````````"````
-MK@```/S_________/HH````````"````J0```/S_________8XH````````"
-M````J@```/S_________#8L````````"````K````/O_________'XP`````
-M```"````J@```/S_________+8P````````"````J@```/S_________4HP`
-M```````"````JP```/S_________Y(T````````"````KP```/S_________
-M%8X````````"````L````/S_________T8X````````"````K````/O_____
-M____1H\````````"````K@```/S_________4(\````````"````J0```/S_
-M________TX\````````"````K@```/S_________YH\````````"````JP``
-M`/S_________\(\````````"````K@```/S_________^H\````````"````
-MJ0```/S_________&9`````````"````JP```/S_________P)`````````"
-M````JP```/S_________T9`````````"````JP```/S_________X)``````
-M```"````JP```/S_________]9`````````"````JP```/S_________!I$`
-M```````"````JP```/S_________%9$````````"````JP```/S_________
-M)I$````````"````JP```/S_________-Y$````````"````JP```/S_____
-M____F)(````````"````K@```/S_________RI(````````"````JP```/S_
-M________\I(````````"````K@```/S_________39,````````"````JP``
-M`/S_________KI,````````"````JP```/S_________NI,````````"````
-MK@```/S_________Q),````````"````J0```/S_________U9,````````"
-M````JP```/S_________X9,````````"````K@```/S_________9)0`````
-M```"````JP```/S_________;Y0````````"````K@```/S_________BI0`
-M```````"````JP```/S_________FY0````````"````JP```/S_________
-MKI0````````"````JP```/S_________P90````````"````JP```/S_____
-M____U)0````````"````JP```/S_________YY0````````"````JP```/S_
-M________^I0````````"````JP```/S_________#94````````"````JP``
-M`/S_________()4````````"````JP```/S_________,Y4````````"````
-MJP```/S_________1I4````````"````JP```/S_________694````````"
-M````JP```/S_________;)4````````"````JP```/S_________FY4`````
-M```"````JP```/S_________K)4````````"````JP```/S_________O94`
-M```````"````JP```/S_________SI4````````"````JP```/S_________
-MW)4````````"````K@```/S_________\Y4````````"````JP```/S_____
-M____,Y8````````"````JP```/S_________6I8````````"````K@```/S_
-M________;98````````"````JP```/S_________>Y8````````"````K@``
-M`/S_________D)8````````"````JP```/S_________HY8````````"````
-MJP```/S_________MI8````````"````JP```/S_________R98````````"
-M````JP```/S_________W)8````````"````JP```/S_________[Y8`````
-M```"````JP```/S_________`I<````````"````JP```/S_________%9<`
-M```````"````JP```/S_________*)<````````"````JP```/S_________
-M.Y<````````"````JP```/S_________3I<````````"````JP```/S_____
-M____89<````````"````JP```/S_________7Y@````````"````:0```/S_
-M________<Y@````````"````:0```/S_________C)D````````"````J@``
-M`/S_________NID````````"````JP```/S_________SID````````"````
-MJP```/S_________W)D````````"````K@```/S_________%YH````````"
-M````JP```/S_________AYH````````"````JP```/S_________G)H`````
-M```"````JP```/S_________M9H````````"````J@```/S_________T9H`
-M```````"````JP```/S_________1YL````````"````JP```/S_________
-M49L````````"````K@```/S_________W9L````````"````JP```/S_____
-M____%)P````````"````L0```/S_________<9P````````"````J@```/S_
-M________H)P````````"````K0```/S_________M)P````````"````K0``
-M`/S_________R)P````````"````K0```/S_________W)P````````"````
-MK0```/S_________\)P````````"````K0```/S_________/9T````````"
-M````K0```/S_________49T````````"````K0```/S_________99T`````
-M```"````K0```/S_________>9T````````"````K0```/S_________C9T`
-M```````"````K0```/S_________H9T````````"````K0```/S_________
-MM9T````````"````K0```/S_________SIT````````"````J@```/S_____
-M____XIT````````"````L@```/S_________^9T````````"````J@```/S_
-M________+YX````````"````KP```/S_________J9X````````"````JP``
-M`/S_________FY\````````"````K@```/S_________^9\````````"````
-MJP```/S_________!Z`````````"````K@```/S_________'Z`````````"
-M````JP```/S_________+:`````````"````K@```/S_________I*``````
-M```"````JP```/S_________M:`````````"````JP```/S_________QJ``
-M```````"````JP```/S_________[:`````````"````JP```/S_________
-M#Z$````````"````JP```/S_________2J$````````"````J0```/S_____
-M____?*$````````"````M````/S_________XZ$````````"````JP```/S_
-M________`J(````````"````K````/C_________$J(````````"````K```
-M`/C_________X:(````````"````K@```/S_________!Z,````````"````
-MK@```/S_________$:,````````"````J0```/S_________'*,````````"
-M````K@```/S_________O:,````````"````JP```/S_________QZ,`````
-M```"````K@```/S_________T:,````````"````J0```/S_________X*,`
-M```````"````JP```/S_________ZJ,````````"````K@```/S_________
-M3:0````````"````K@```/S_________=Z0````````"````JP```/S_____
-M____EZ0````````"````K@```/S_________H:0````````"````J0```/S_
-M________QJ0````````"````JP```/S_________YJ0````````"````K@``
-M`/S_________\*0````````"````J0```/S__________J0````````"````
-MJ````/S_________,J4````````"````J@```/S_________UZ4````````"
-M````K@```/S_________/*8````````"````JP```/S_________J*<`````
-M```"````K@```/S_________QZ<````````"````JP```/S_________Y:<`
-M```````"````JP```/S_________B:@````````"````K@```/S_________
-MMZ@````````"````JP```/S_________+JD````````"````K@```/S_____
-M____6JD````````"````JP```/S_________<ZD````````"````K@```/S_
-M________SJD````````"````K@```/S__________*D````````"````JP``
-M`/S_________%:H````````"````K@```/S_________/:H````````"````
-MK@```/S_________;ZH````````"````JP```/S_________WZH````````"
-M````K@```/S_________!ZL````````"````M````/S_________6:L`````
-M```"````K@```/S_________:*L````````"````K@```/S_________=ZL`
-M```````"````K@```/S_________AJL````````"````K@```/S_________
-ME:L````````"````K@```/S_________I*L````````"````K@```/S_____
-M____LZL````````"````K@```/S_________PJL````````"````K@```/S_
-M________)ZP````````"````PP```/S_________-*X````````"````K@``
-M`/S_________::X````````"````K@```/S_________?*X````````"````
-MJP```/S_________P:X````````"````JP```/S_________RZX````````"
-M````K@```/S_________WZX````````"````JP```/S_________^*X`````
-M```"````K@```/S_________\Z\````````"````JP```/S_________)+``
-M```````"````JP```/S_________>[`````````"````J@```/S_________
-ME+`````````"````J0```/S_________Z[`````````"````K0```/S_____
-M____`;$````````"````J@```/S_________*[$````````"````K0```/S_
-M________0;$````````"````J@```/S_________C+$````````"````J@``
-M`/S_________FK$````````"````J0```/S_________K+$````````"````
-MJ@```/S_________QK$````````"````M````/S_________U+$````````"
-M````J0```/S_________\[$````````"````L@```/S_________`[(`````
-M```"````J@```/S_________9+(````````"````R````/S_________=[(`
-M```````"````K0```/S_________B[(````````"````L@```/S_________
-ME[(````````"````QP```/S_________IK(````````"````QP```/S_____
-M____/[,````````"````I@```/S_________Q[,````````"````L0```/S_
-M________)[0````````"````I@```/S_________Z[0````````"````S```
-M`/S_________S+4````````"````R````/S_________YK4````````"````
-MK0```/S_________^[4````````"````K0```/S_________$;8````````"
-M````K0```/S_________)[8````````"````K0```/S_________/;8`````
-M```"````K0```/S_________B;8````````"````K0```/S_________G+8`
-M```````"````K0```/S_________L+8````````"````K0```/S_________
-MQ+8````````"````K0```/S_________V+8````````"````K0```/S_____
-M____[+8````````"````K0```/S_________`+<````````"````K0```/S_
-M________'+<````````"````L@```/S_________*+<````````"````QP``
-M`/S_________.K<````````"````QP```/S_________6;<````````"````
-ML@```/S_________9;<````````"````QP```/S_________@+<````````"
-M````J@```/S_________C+<````````"````QP```/S_________PK<`````
-M```"````L@```/S_________V+<````````"````L````/S_________^[<`
-M```````"````KP```/S_________&+@````````"````L@```/S_________
-M)+@````````"````QP```/S_________,[@````````"````QP```/S_____
-M____0K@````````"````QP```/S_________M+@````````"````K0```/S_
-M________Q;@````````"````J@```/S_________S[@````````"````J0``
-M`/S_________U[@````````"````QP```/S_________X;@````````"````
-MJ0```/S_________Z[@````````"````J0```/S_________];@````````"
-M````J0```/S__________[@````````"````J0```/S_________%[D`````
-M```"````J@```/S_________,;D````````"````J0```/S_________:;D`
-M```````"````J0```/S_________C+D````````"````!`````0`````````
-MG[D````````"````K0```/S_________J[D````````"````!`````0`````
-M````O+D````````"````J@```/S_________T[D````````"````!`````0`
-M````````Y+D````````"````J@```/S_________][D````````"````!```
-M``0`````````"+H````````"````J@```/S_________)[H````````"````
-M!`````0`````````.KH````````"````K0```/S_________0;H````````"
-M````!`````0`````````5[H````````"````K0```/S_________>;H`````
-M```"````!`````0`````````C[H````````"````K0```/S_________EKH`
-M```````"````!`````0`````````IKH````````"````J@```/S_________
-MW;H````````"````K@```/S_________\KH````````"````JP```/S_____
-M_____[H````````"````K@```/S_________'KL````````"````KP```/S_
-M________*[L````````"````K@```/S_________.KL````````"````!```
-M``0`````````4+L````````"````K0```/S_________5[L````````"````
-M!`````0`````````;+L````````"````K0```/S_________<[L````````"
-M````!`````0`````````B;L````````"````K0```/S_________D+L`````
-M```"````!`````0`````````I;L````````"````K0```/S_________O+L`
-M```````"````JP```/S_________R;L````````"````K@```/S_________
-MV;L````````"````JP```/S__________;L````````"````K@```/S_____
-M____$KP````````"````JP```/S_________'[P````````"````K@```/S_
-M________/KP````````"````KP```/S_________2[P````````"````K@``
-M`/S_________6KP````````"````!`````0`````````<+P````````"````
-MK0```/S_________=[P````````"````!`````0`````````C+P````````"
-M````K0```/S_________D[P````````"````!`````0`````````J;P`````
-M```"````K0```/S_________L+P````````"````!`````0`````````Q;P`
-M```````"````K0```/S_________W+P````````"````JP```/S_________
-MZ;P````````"````K@```/S_________^;P````````"````JP```/S_____
-M____'+T````````"````!`````0`````````,KT````````"````K0```/S_
-M________.;T````````"````!`````0`````````3KT````````"````K0``
-M`/S_________8[T````````"````J@```/S_________>[T````````"````
-M!`````0`````````D+T````````"````K0```/S_________HKT````````"
-M````J0```/S_________K[T````````"````J@```/S_________`+X`````
-M```"````K@```/S_________%;X````````"````JP```/S_________(KX`
-M```````"````K@```/S_________0;X````````"````KP```/S_________
-M3KX````````"````K@```/S_________9[X````````"````K0```/S_____
-M____=KX````````"````JP```/S_________@[X````````"````K@```/S_
-M________D[X````````"````JP```/S_________];X````````"````SP``
-M`/S_________';\````````"````K@```/S_________,K\````````"````
-MJP```/S_________/[\````````"````K@```/S_________8+\````````"
-M````KP```/S_________;;\````````"````K@```/S_________>K\`````
-M```"````L````/S_________H[\````````"````!`````0`````````N;\`
-M```````"````K0```/S_________T[\````````"````J0```/S_________
-MX+\````````"````L````/S_________Z;\````````"````!`````0`````
-M````_[\````````"````K0```/S_________)<`````````"````!`````0`
-M````````.\`````````"````K0```/S_________2,`````````"````L```
-M`/S_________>L`````````"````!`````0`````````D,`````````"````
-MK0```/S_________H,`````````"````JP```/S_________K<`````````"
-M````K@```/S_________O<`````````"````JP```/S_________\\``````
-M```"````!```````````````]\`````````+`````P```-``````````"L$`
-M```````"````!```````````````#L$````````+`````P```/``````````
-M(``````````!````F@``````````````*``````````!````F@``````````
-M````,``````````!````F@``````````````.``````````!````F@``````
-M````````0``````````!````B@``````````````2``````````!````>0``
-M````````````4``````````!````A0``````````````8``````````!````
-ME@``````````````@``````````!````<0``````````````B``````````!
-M````<0``````````````D``````````!````<0``````````````F```````
-M```!````<0``````````````H``````````!````BP``````````````J```
-M```````!````>P``````````````L``````````!````AP``````````````
-MP``````````!````EP`````````````````````````!`````@```+TZ````
-M````"``````````!`````@```)$Z````````$``````````!`````@```)$Z
-M````````&``````````!`````@```)@Z````````(``````````!`````@``
-M`*<Z````````*``````````!`````@```+(Z````````,``````````!````
-M`@```+DZ````````.``````````!`````@```&T[````````0``````````!
-M`````@```$,[````````2``````````!`````@```$,[````````4```````
-M```!`````@```$H[````````6``````````!`````@```%<[````````8```
-M```````!`````@```&([````````:``````````!`````@```&D[````````
-M(``````````!`````@``````````````2``````````!`````@```*`%````
-M````<``````````!`````@```)`&````````F``````````!`````@```*`(
-M````````P``````````!`````@`````)``````````$````````!`````@``
-M`%`*````````.`$````````!`````@```%`+````````4`$````````!````
-M`@```-`+````````<`$````````!`````@```-`,````````F`$````````!
-M`````@```#`-````````P`$````````!`````@```'`.````````X`$`````
-M```!`````@```+`.``````````(````````!`````@```"`/````````*`(`
-M```````!`````@```)`/````````4`(````````!`````@```#`1````````
-M<`(````````!`````@```,`1````````F`(````````!`````@```(`2````
-M````P`(````````!`````@```'`3````````Z`(````````!`````@```-`4
-M````````$`,````````!`````@```-`5````````.`,````````!`````@``
-M`*`7````````8`,````````!`````@```/`9````````B`,````````!````
-M`@```$`;````````L`,````````!`````@```,`<````````R`,````````!
-M`````@```/`<````````X`,````````!`````@```*`=````````"`0`````
-M```!`````@`````>````````,`0````````!`````@```$`?````````4`0`
-M```````!`````@```'`?````````:`0````````!`````@```)`?````````
-M@`0````````!`````@```+`?````````P`0````````!`````@```%`A````
-M````Z`0````````!`````@```,`A``````````4````````!`````@```!`B
-M````````*`4````````!`````@```%`B````````0`4````````!`````@``
-M`)`B````````8`4````````!`````@```+`B````````>`4````````!````
-M`@```/`B````````D`4````````!`````@```"`C````````N`4````````!
-M`````@```+`C````````T`4````````!`````@```.`C````````^`4`````
-M```!`````@```$`D````````.`8````````!`````@```'`F````````8`8`
-M```````!`````@```$`G````````H`8````````!`````@```(`I````````
-MR`8````````!`````@```(`J````````\`8````````!`````@```#`O````
-M````$`<````````!`````@```&`O````````4`<````````!`````@```"`P
-M````````:`<````````!`````@```$`Q````````D`<````````!`````@``
-M`!`R````````L`<````````!`````@```%`R````````V`<````````!````
-M`@```/`R``````````@````````!`````@```(`T````````*`@````````!
-M`````@```'`V````````4`@````````!`````@```&`W````````D`@`````
-M```!`````@```+`X````````N`@````````!`````@```#`Y````````T`@`
-M```````!`````@```)`Y````````$`D````````!`````@```-`[````````
-M4`D````````!`````@```/!>````````D`D````````!`````@```#!@````
-M````N`D````````!`````@```*!@````````V`D````````!`````@```/!A
-M``````````H````````!`````@```'!B````````*`H````````!`````@``
-M`%!C````````:`H````````!`````@```!!F````````B`H````````!````
-M`@```)!F````````R`H````````!`````@```-!K````````"`L````````!
-M`````@```,!O````````*`L````````!`````@```/!O````````6`L`````
-M```!`````@```&!R````````@`L````````!`````@```-!R````````J`L`
-M```````!`````@```/!S````````T`L````````!`````@```$!U````````
-MZ`L````````!`````@```'!U````````$`P````````!`````@```/!U````
-M````0`P````````!`````@```'!X````````6`P````````!`````@```,!X
-M````````@`P````````!`````@```!!Y````````F`P````````!`````@``
-M`$!Y````````L`P````````!`````@```%!Y````````X`P````````!````
-M`@```,!Z````````^`P````````!`````@```"![````````&`T````````!
-M`````@```*![````````,`T````````!`````@````!\````````4`T`````
-M```!`````@```+!\````````<`T````````!`````@```.!\````````B`T`
-M```````!`````@```/!\````````H`T````````!`````@```"!]````````
-MR`T````````!`````@```*!]````````Z`T````````!`````@```,!]````
-M````"`X````````!`````@```/!]````````,`X````````!`````@```%!_
-M````````2`X````````!`````@```(!_````````<`X````````!`````@``
-M`/!_````````D`X````````!`````@```$"`````````L`X````````!````
-M`@```&"`````````R`X````````!`````@```*"`````````X`X````````!
-M`````@```&"#````````"`\````````!`````@```/"$````````,`\`````
-M```!`````@```."%````````6`\````````!`````@```("'````````@`\`
-M```````!`````@```$"(````````J`\````````!`````@```%"+````````
-MR`\````````!`````@```-"+````````\`\````````!`````@```("-````
-M````&!`````````!`````@```$".````````0!`````````!`````@```+"/
-M````````<!`````````!`````@```%"1````````L!`````````!`````@``
-M`#"2````````V!`````````!`````@```'"3`````````!$````````!````
-M`@```!"4````````0!$````````!`````@```""6````````8!$````````!
-M`````@```$"6````````@!$````````!`````@```'"7````````F!$`````
-M```!`````@```,"7````````L!$````````!`````@```#"8````````V!$`
-M```````!`````@```,"8````````^!$````````!`````@```#"9````````
-M$!(````````!`````@```&"9````````.!(````````!`````@```/"9````
-M````6!(````````!`````@```"":````````@!(````````!`````@````";
-M````````J!(````````!`````@```(";````````T!(````````!`````@``
-M`/";````````\!(````````!`````@```$"<````````&!,````````!````
-M`@```'">````````0!,````````!`````@```-"A````````8!,````````!
-M`````@````"B````````>!,````````!`````@```!"B````````D!,`````
-M```!`````@```""B````````L!,````````!`````@```("B````````R!,`
-M```````!`````@```+"B````````\!,````````!`````@```$"C````````
-M,!0````````!`````@```("E````````6!0````````!`````@```'"F````
-M````>!0````````!`````@```,"F````````F!0````````!`````@````"G
-M````````N!0````````!`````@```%"G````````T!0````````!`````@``
-M`'"G````````^!0````````!`````@```!"H````````(!4````````!````
-M`@```/"H````````2!4````````!`````@```)"I````````>!4````````!
-M`````@```)"J````````N!4````````!`````@```."K````````V!4`````
-M```!`````@```#"L`````````!8````````!`````@````"N````````*!8`
-M```````!`````@```."O````````2!8````````!`````@```!"P````````
-M:!8````````!`````@```$"P````````D!8````````!`````@```-"P````
-M````L!8````````!`````@```!"Q````````T!8````````!`````@```%"Q
-M`````````!<````````!`````@```""R````````*!<````````!`````@``
-M`-"R````````2!<````````!`````@```*"S````````:!<````````!````
-M`@```-"S````````B!<````````!`````@```#"T````````L!<````````!
-M`````@```""U````````V!<````````!`````@```)"X`````````!@`````
-M```!`````@```%"Y````````(!@````````!`````@```("Y````````2!@`
-M```````!`````@```""Z````````:!@````````!`````@```&"Z````````
-MB!@````````!`````@```-"Z````````L!@````````!`````@```/"[````
-M````V!@````````!`````@```!"]````````^!@````````!`````@```-"]
-M````````(!D````````!`````@```,"^````````0!D````````!`````@``
-A``"_````````@!D````````!`````@```.#`````````
+M```````````````````@`````0````8```````````````````!`````````
+M`#;%```````````````````0````````````````````&P````0`````````
+M````````````````(`8!``````"X-0````````P````!````"``````````8
+M`````````"L````!`````P```````````````````(#%````````*`$`````
+M`````````````"`````````````````````F````!```````````````````
+M``````#8.P$``````)@!````````#`````,````(`````````!@`````````
+M,0````@````#````````````````````J,8````````,````````````````
+M````"````````````````````#L````!`````@```````````````````*C&
+M````````<`````````````````````@````````````````````V````!```
+M``````````````````````!P/0$``````%`!````````#`````8````(````
+M`````!@`````````2`````$````"````````````````````&,<```````"0
+M&0``````````````````"````````````````````$,````$````````````
+M`````````````,`^`0``````\`\````````,````"`````@`````````&```
+M``````!2`````0````````````````````````"HX````````"4`````````
+M```````````!````````````````````$0````,`````````````````````
+M````S>````````!;`````````````````````0````````````````````$`
+M```"`````````````````````````*CD````````^!,````````-````8@``
+M``@`````````&``````````)`````P````````````````````````"@^```
+M`````'\-```````````````````!````````````````````````````````
+M`````````````````````0````0`\?\```````````````````````````,`
+M`0````````````````````````````,``P``````````````````````````
+M``,`!0``````````````````````"`````(``0!0#````````(D`````````
+M&0````(``0#@#````````!$!````````+`````(``0!@$````````&T`````
+M````.0````(``0`P%P```````-\!````````1@````(``0"0'@```````#$`
+M````````50````(``0#0'@```````*4`````````9@````(``0!0(0``````
+M`!\`````````<@````(``0!P(0```````!<`````````?0````$`!0``````
+M``````@`````````B@````(``0"0(0```````)D!````````F@````(``0`P
+M(P```````&H`````````HP````(``0"@(P```````$(`````````JP````(`
+M`0#P(P```````#<`````````LP````(``0`P)````````#T`````````O@``
+M``(``0!P)````````!@`````````Q@````(``0"0)````````#(`````````
+MV`````(``0#0)````````"4`````````YP````(``0``)0```````(4`````
+M````\@````(``0"0)0```````"@`````````_P````(``0#`)0```````%D`
+M````````"P$```(``0`@)@```````"D"````````&`$```(``0!0*```````
+M`,@`````````)0$```(``0`@*0```````#H"````````+P$```(``0!@*P``
+M`````/0`````````/@$```(``0"@9````````-T`````````20$```(``0!@
+M+````````,,$````````50$```(``0``/@```````!<C````````8P$```(`
+M`0#08@```````$,!````````;P$```(``0`P,0```````"4`````````>0$`
+M``(``0#`.P```````#D"````````BP$```(``0!@,0```````+4`````````
+MG`$```(``0`@,@```````!H!````````J0$```(``0!`,P```````,(`````
+M````M@$```(``0!0-````````*,`````````PP$```(``0`0-````````#X`
+M````````S@$```(``0``-0```````(P!````````V0$```(``0"0-@``````
+M`/H!````````Y0$```(``0#@.@```````'D`````````\P$```(``0"0.```
+M`````/H```````````(```(``0"0.0```````$H!````````#0(```(``0!@
+M.P```````%0```````````````,`!@``````````````````````'@(```(`
+M`0"`90```````,L"````````+@(```(``0!0:````````((`````````10(`
+M``(``0#@:````````'D%````````4`(```(``0!@;@```````"$$````````
+M7`(```(``0"0<@```````"H`````````;`(```(``0``>0```````(D"````
+M````>0(```(``0"`@@```````"8`````````B@(```(``0`@@P```````$\`
+M````````GP(```(``0!P@P```````"``````````L@(```(``0"0@P``````
+M`,`"````````R`(```(``0!0A@```````(D!````````WP(```(``0#@AP``
+M`````.$`````````]P(```(``0#`FP```````&``````````!0,```(``0#0
+MB````````),!````````&`,```(``0!`G@```````),`````````)P,```(`
+M`0"@GP```````$`"````````.P,```(``0`PD0```````&\!````````3@,`
+M``(``0!PB@```````+$`````````70,```(``0`PBP```````/@"````````
+M<0,```(``0"`E@```````)T`````````@P,```(``0!`E0```````#H!````
+M````D0,```(``0!0GP```````$4`````````G@,```(``0`PC@```````'H`
+M````````M`,```(``0``G0```````%\`````````RP,```(``0"PC@``````
+M`+D!````````W@,```(``0!PD````````,``````````[@,```(``0!@G0``
+M`````-$`````````^P,```(``0#@G@```````&D`````````#@0```(``0"@
+MD@```````+L!````````'P0```(``0!@E````````-$`````````+00```(`
+M`0`@EP````````4"````````/@0```(``0`PF0```````!P`````````5P0`
+M``(``0!0F0```````"<!````````;@0```(``0"`F@```````$(`````````
+M@P0```(``0#0F@```````&T`````````EP0```(``0!`FP```````(``````
+M````H@0```(``0`@G````````"@`````````M`0```(``0!0G````````*8`
+M````````R@0```(``0`@MP```````,$#````````X`0```(``0#PN@``````
+M`+D`````````[00```$``P#@`````````"``````````]P0```$``P```0``
+M`````"```````````04```$``P`@`0````````@`````````"04```$`!0`(
+M``````````0`````````&`4```(``0#@NP```````)H`````````(04```(`
+M`0"`O````````#T`````````*04```(``0#`O````````&@`````````-`4`
+M``(``0"0OP```````+$`````````/`4```(``0"`P0```````-0!````````
+M``````,`"`````````````````````````````,`"@``````````````````
+M````5`4``!(``0```````````&P&````````8`4``!$``P`@`````````$@`
+M````````;P4``!``````````````````````````?`4``!$``P"`````````
+M`$@`````````C`4``!(``0!0>````````",`````````E04``!``````````
+M````````````````HP4``!``````````````````````````K04``!``````
+M````````````````````MP4``!``````````````````````````P04``!``
+M````````````````````````T@4``!``````````````````````````X@4`
+M`!``````````````````````````[`4``!(``0!P!@```````/<`````````
+M]@4``!```````````````````````````P8``!``````````````````````
+M````%08``!(``0!P!P```````"L"````````)08``!``````````````````
+M````````+@8``!(``0"`>````````'<`````````/`8``!(``0"@"0``````
+M`%0`````````3P8``!(``0!@?`````````X`````````708``!(``0``"@``
+M`````$H!````````:@8``!(``0`P?````````"H`````````?`8``!(``0!@
+M=0```````&L`````````AP8``!(``0!0"P```````/<`````````F@8``!(`
+M`0``#@```````%L`````````JP8``!(``0``@`````````P`````````MP8`
+M`!(``0#@?0```````%4`````````QP8``!(``0!@#@```````#0!````````
+MV08``!(``0#0?P```````"\`````````Z08``!(``0"@#P```````$4`````
+M````_`8``!(``0#P#P```````&P`````````#@<``!(``0#0$````````)(!
+M````````)`<``!(``0!P$@```````(,`````````00<``!(``0``$P``````
+M`+H`````````50<``!(``0!`@````````'H`````````8P<``!(``0#`$P``
+M`````/(`````````>@<``!(``0#`%````````%$!````````D@<``!(``0`@
+M%@````````(!````````J`<``!``````````````````````````L@<``!(`
+M`0`0&0```````$,"````````Q`<``!(``0`0@````````"@`````````U0<`
+M`!(``0!@&P```````)8!````````Z`<``!(``0``'0```````(\!````````
+M^0<``!(``0"`'P```````%L`````````"0@``!(``0#@'P```````#0!````
+M````&@@``!(``0`@(0```````"P`````````+`@``!$``P````````````0`
+M````````/`@``!``````````````````````````2@@``!(``0`@80``````
+M`#\!````````70@``!``````````````````````````<`@``!``````````
+M````````````````=P@``!``````````````````````````?@@``!(``0!@
+M8@```````&8`````````B@@``!``````````````````````````K`@``!(`
+M`0`@?P```````*,`````````N@@``!(``0`@9````````'$`````````S`@`
+M`!(``0#`<@```````)D"````````W@@``!(``0#P=@```````%<!````````
+M\0@``!(``0#0=0```````!8!`````````PD``!``````````````````````
+M````%0D``!``````````````````````````*`D``!(``0"0>P```````$(`
+M````````-PD``!(``0#@>P```````$<`````````1@D``!(``0!P?```````
+M`&0!````````50D``!(``0!`?@```````'X`````````9@D``!(``0#`?@``
+M`````%L`````````<PD``!(``0#`@````````!4`````````?@D``!(``0#@
+M@````````"<`````````C@D``!(``0`0@0```````&8!````````F@D``!(`
+M`0``J@```````#T`````````L@D``!(``0!`J@```````$8`````````Q@D`
+M`!(``0#0M0```````&``````````X0D``!(``0`PM@```````.8`````````
+M!@H``!(``0"PJ0```````$,`````````'0H``!(``0"P@@```````&(`````
+M````-`H``!``````````````````````````2`H``!``````````````````
+M````````60H``!``````````````````````````;`H``!$``P#(````````
+M``0`````````?PH``!``````````````````````````D0H``!``````````
+M````````````````HPH``!(``0!`I@```````(P`````````P@H``!``````
+M````````````````````U`H``!``````````````````````````Y0H``!(`
+M`0"PNP```````"L`````````]`H``!(``0"@L@```````*\`````````"0L`
+M`!(``0#@H0```````(`#````````&PL``!(``0!@PP```````%(`````````
+M,0L``!(``0!@I0```````"$`````````1PL``!(``0"0I0````````L`````
+M````60L``!(``0"@I0````````L`````````;`L``!(``0"PI0```````&``
+M````````@PL``!(``0`0I@```````",`````````EPL``!(``0#0I@``````
+M``\"````````K@L``!(``0#@J````````,4`````````PPL``!(``0"0J@``
+M`````!T`````````V@L``!(``0"PJ@```````-T`````````]`L``!(``0"0
+MJP```````)X`````````#0P``!(``0`PK````````/\`````````)0P``!(`
+M`0`PK0```````$T!````````/`P``!(``0"`K@```````*(!````````3PP`
+M`!(``0`PL`````````,"````````;0P``!(``0!`L@```````"$`````````
+MB`P``!(``0!PL@```````"(`````````I0P``!(``0!0LP```````#<`````
+M````OPP``!(``0"0LP```````#<`````````V@P``!(``0#0LP```````*8`
+M````````]PP``!(``0"`M````````!\!````````%0T``!(``0"@M0``````
+M`#``````````-`T``!(``0`PO0```````#L!````````.PT``!(``0!PO@``
+M`````!(!````````0PT``!(``0!0P````````.X`````````40T``!(``0!`
+MP0```````#X`````````7@T``!(``0#`PP```````"8`````````:PT``!(`
+M`0#PPP```````$8!`````````')A:60N8P!3=')I<&5'971#;VUM86YD`%-T
+M<FEP95-P;&ET0V]M;6%N9`!)<U9$979/;FQI;F4`1&]#;VYT<F]L0VUD`$I"
+M3T1'971#;VUM86YD`$I"3T13<&QI=$-O;6UA;F0`7U]H<'1?86QL;V,`7U]H
+M<'1?9G)E90!G87)B86=E7V%D9'(`:6YI=%]H96%P7V)L;V-K`&-B7V%L;&]C
+M`&-B7V9R964`8V)?>F5R;P!C8E]I<U]Z97)O`&-F7VEN:70`<F5M;W9E7W)A
+M;F=E7VQO8VL`861D7W)A;F=E7VQO8VL`;&]C:U]R86YG90!?7V%L;&]C7W1A
+M<VL`9G)E95]S=')I<&4`86QL;V-?<W1R:7!E`%]?9V5T7W-T<FEP90!D871A
+M7WAF97(`<F5A9%]X9F5R7V1O;F4`<F%I9#5?9&]N90!S=')I<&5?9&]N90!H
+M86YD;&5?<W1R:7!E`')A:60U7V1I<G1Y`'1A<VM?9&]N90!S=')I<&5?8VAE
+M8VM?=&%S:P!R=U]T87-K7V)U:6QD<V=L`')W7V=E=%]R86YG90!R=U]T87-K
+M7V1O;F4`7U]D;U]R=U]T87-K`&1O7W)W7W1A<VL`9V5T7W-L:6-E<P!D;U]X
+M;W)?=&%S:P!X;W)?=&%S:U]D;VYE`&1O7WAO<C)?=&%S:P!D;U]X;W)N7W1A
+M<VL`8VAE8VM?9&5P96YD96YC>0!D:7)E8W1?<F5A9%]S9VP`9&ER96-T7W)E
+M861?8V]M<&QE=&EO;@!H86YD;&5?8VUD`&=E=%]S=')I<&5S`'=R:71E7WAF
+M97)?9&]N90!3>6YC1&ES:TEN9F\`9V5T161M85)E9T]F9G-E=`!U;FUA<VM%
+M9&UA26YT97)R=7!T<P!M87-K161M84EN=&5R<G5P=',`=W)I=&5%9&UA4F5Q
+M=65S=$5N=')Y`&1U;7!!=&%$979I8V5296=I<W1E<G,`:&%N9&QE161M849A
+M:6QE9$-O;6UA;F0`<F5M;W9E0V]M;6%N9`!H86YD;&5%9&UA4F5S<&]N<V4`
+M9&5A8W1I=F%T945D;6$`<V5N9$YO;F559&UA0V]M;6%N9`!C;VUP;&5T95!)
+M3T-O;6UA;F0`4V5T1U!)3T-T<FQ296<`:&%N9&QE161M84EN=&5R<G5P=`!?
+M8VAA;FYE;$AA<F1297-E=`!?9FEX4&AY4&%R86US`%]D;T%U=&]&;'5S:`!H
+M86YD;&5$979I8V5);G1E<G)U<'0`9&ES86)L95-A1&5V26YT97)R=71P<P!H
+M86YD;&5024]);G1E<G)U<'0`=')A;G-F97)024]$871A`&%C=&EV871E161M
+M80!%9&UA4F5Q475E=65);G-E<G0`<F5S971%9&UA0VAA;FYE;`!F;'5S:$1M
+M85%U975E`')E=F5R=%-A=&%(0U)E9W,`<F5V97)T1FQA<VA);G1E<F9A8V52
+M96=S`')E=F5R=%!#24EN=&5R9F%C95)E9W,`8V]M;6%N9'-1=65U94%D9%1A
+M:6P`8V]M;6%N9'-1=65U95)E;6]V90!A9&1#;VUM86YD`%-A1&5V26YT97)R
+M=71P0FET`&5N86)L95-A1&5V26YT97)R=71P<P!E>&5C=71E3F]N541-04-O
+M;6UA;F0`7V1O4V]F=%)E<V5T`%]A9&1R36%P,0!?861D<DUA<#(`861D<DUA
+M<`!D:7-A8FQE7V)E97!E<@!%;G1E<C8P,0!%>&ET-C`Q`&=E=%\V,#%?:60`
+M<V5T7V-T;`!?7V-H96-K7W!R;W1E8W1?8VER8W5I=`!F0VAE8VM!<G)A>0!P
+M9FY396YD0V]M;6%N9`!F1&5296%D5W)I=&4`<&9N1&5V:6-E1F%I;&5D`$-H
+M96-K4W5M`&9$95-E;&5C=$UO9&4`;W-?;65M<V5T`&9$95-E=%1#40!F1&53
+M971.0U$`9D1E4V5T5W)I=&5#86-H90!F1&53971296%D06AE860`;W-?;65M
+M8W!Y`&9!9&13<&%R90!'9713<&%R941I<VL`:6]C=&Q?4F5P;W)T179E;G0`
+M1FEX=7!!<G)A>5-T871E`$=E=%-T86UP`%-Y;F-!<G)A>4EN9F\`0VAE8VM!
+M<G)A>4-R:71I8V%L`&9/<T1I<VM&86EL960`9D1E;&5T94%R<F%Y`%5N<F5G
+M:7-T97)61&5V:6-E`')A:60U7V9R964`4W1R:7!E0G5I;&139U1A8FQE`&92
+M04E$,$UE;6)E<D1O;F4`1G)E94-O;6UA;F0`0V%L;$%F=&5R4F5T=7)N`&92
+M04E$,%-E;F1#;VUM86YD`$%L;&]C871E0V]M;6%N9`!F4D%)1#!-96UB97)&
+M86EL960`4D%)1#%"=6EL9%-G5&%B;&4`4D%)1#%-96UB97)#;VUP;&5T:6]N
+M`%)!240Q26YI=%)E8G5I;&1"=6EL9%-G5&%B;&4`4D%)1#%);FET0V]M<&QE
+M=&EO;@!$;U=A:71I;F=,:7-T`%)!240Q4F5B=6EL9$-O;7!L971I;VX`4D%)
+M1#%697)I9GE"=6EL9%-G5&%B;&4`4D%)1#%697)I9GE#;VUP;&5T:6]N`&]S
+M7VUE;6-M<`!F4D%)1#%396YD0V]M;6%N9`!!9&14;U=A:71I;F=,:7-T`&92
+M04E$,4UE;6)E<D9A:6QE9`!*0D]$0G5I;&139U1A8FQE`&9*0D]$365M8F5R
+M1&]N90!F2D)/1%-E;F1#;VUM86YD`&9*0D]$365M8F5R1F%I;&5D`&YU;5]R
+M86ED-5]P86=E<P!O<U]A;&QO8U]P86=E`&9L=7-H7W-T<FEP95]C86-H90!F
+M3W-0:'ES:6-A;$%D9')E<W,`1&]8;W(R`$1O6&]R,0!F;'5S:%]R86ED-0!F
+M0V]M<&QE=&5!;&Q#;VUM86YD<U-Y;F-H<F]N;W5S;'D`0VAE8VM)9&QE0V%L
+M;`!F;'5S:%]R86ED-5]A<WEN8P!F4D%)1#5396YD0V]M;6%N9`!F4D%)1#5-
+M96UB97)&86EL960`:6YI=%]R86ED-5]M96UO<GD`;W-?86QL;V-?9&UA7W!A
+M9V4`9D1E=FEC95-E;F1#;VUM86YD`&9'971&:7)S=$-H:6QD`&9297-E=$)O
+M;W1-87)K`&9#:&5C:T)O;W1A8FQE`$-H96-K4&5N9&EN9T-A;&P`0V%L;%=H
+M96Y)9&QE`&9&;'5S:%9$978`9D9L=7-H5D1E=D%S>6YC`&UV4F5A9%=R:71E
+M`&UV4V%T841I<V%B;&5#:&%N;F5L1&UA`&UV4V%T849L=7-H1&UA475E=64`
+M;793=&]R86=E1&5V051!4V5T1F5A='5R97,`;793=&]R86=E1&5V051!17AE
+M8W5T94YO;E5$34%#;VUM86YD`&UV4V%T845N86)L94-H86YN96Q$;6$`=V%I
+M=$9O<D)U<WE!9G1E<DA297-E=`!M=DUI8W)O4V5C;VYD<T1E;&%Y`$U67U)%
+M1U]214%$7T)95$4`359?4D5'7U=2251%7T173U)$`&UV075T;T9L=7-H3VY%
+M<G)O<@!-5E]214=?5U))5$5?0EE410!-5E]214=?4D5!1%]$5T]21`!M=E-A
+M=&%)<U-T;W)A9V5$979I8V5#;VYN96-T960`359?4D5'7U=2251%7U=/4D0`
+M359?4D5'7U)%041?5T]21`!?9&]-=E-O9G1297-E=`!M=E-T;W)A9V5$9797
+M86ET4W1A=`!M=E-A=&%);FET061A<'1E<@!C:&5C:U]P<F]T96-T7V-I<F-U
+M:70`;793871A4VAU=&1O=VY!9&%P=&5R`&UV16YA8FQE075T;T9L=7-H`&UV
+M1&ES86)L94%U=&]&;'5S:`!M=E-A=&%#;VYF:6=U<F5#:&%N;F5L`&UV4V%T
+M85)E;6]V94-H86YN96P`;793871A0VAA;FYE;$AA<F1297-E=`!M=E-A=&%#
+M;VYF:6=%9&UA36]D90!M=E-A=&%.=6U/9D1M84-O;6UA;F1S`&UV4V%T85-E
+M=$-H86YN96Q0:'E087)A;7,`;793871A0VAA;FYE;%!H>5-H=71D;W=N`&UV
+M4V%T84-H86YN96Q0:'E0;W=E<D]N`&UV4V%T84=E=$-H86YN96Q3=&%T=7,`
+M;793871A475E=65#;VUM86YD`&UV4V%T84EN=&5R<G5P=%-E<G9I8V52;W5T
+M:6YE`&UV4V%T84UA<VM!9&%P=&5R26YT97)R=7!T`&UV4V%T855N;6%S:T%D
+M87!T97));G1E<G)U<'0`96YA8FQE4W1O<F%G941E=DEN=&5R<G5P=`!D:7-A
+M8FQE4W1O<F%G941E=DEN=&5R<G5P=`!M=E-T;W)A9V5$979!5$%)9&QE26UM
+M961I871E`&UV4W1O<F%G941E=D%404ED96YT:69Y1&5V:6-E`&UV4W1O<F%G
+M941E=D%405-O9G1297-E=$1E=FEC90!"965P3VX`0F5E<$]F9@!S971?9F%I
+M;%]L961S`'-E=%]F86EL7VQE9`!S>#4P.'A?:6]C=&P`<W@U,#AX7V9L87-H
+M7V%C8V5S<P``A``````````+````8P``````````````K``````````"````
+M9````/S_________R0`````````+````90``````````````XP`````````"
+M````9@```/S_________#`$````````"````9P```/S_________&0$`````
+M```"````9@```/S_________-`$````````"````:````/S_________I`$`
+M```````"````:0```/S_________P0$````````"````:@```/S_________
+MV0$````````"````:P```/S_________[P$````````"````;````/S_____
+M____(P(````````"````90```/S_________40(````````+````8P``````
+M````````ZP(````````"````:````/S_________C0,````````+````8P``
+M````````````IP,````````"````;0```/S_________O0,````````"````
+M;0```/S_________U@,````````"````;0```/S_________[`,````````"
+M````;0```/S_________;P0````````"````:````/S_________%@4`````
+M```+````8P``````````````)@4````````+````90``````````````6@4`
+M```````"````;0```/S_________<`4````````"````;0```/S_________
+MB04````````"````;0```/S_________GP4````````"````;0```/S_____
+M____X@4````````+````90``````````````+`8````````+````90``````
+M````````O08````````"````;P```/S_________\`8````````"````8P``
+M`!0```````````<````````+````90``````````````50<````````"````
+M<````/S_________#@@````````"````<0```/S_________D0@````````"
+M````;@```/S_________00D````````"````;@```/S_________<PD`````
+M```"````<@```/S_________?@D````````"````<P```/S_________U@D`
+M```````"````<0```/S_________Y`D````````+````=0``````````````
+M(0H````````"````=P```/S_________,@H````````"````>````/S_____
+M____;0H````````"````=@```/S_________GPH````````+````=0``````
+M````````PPH````````"````:````/S_________V0H````````"````9```
+M`/S_________]@H````````"````<P```/S_________&PL````````"````
+M:````/S_________,`X````````"````>P```/S_________4`X````````"
+M````?````/S_________G@X````````"````?````/S_________Q0X`````
+M```"````?````/S_________\@X````````"````?@```/S_________1@\`
+M```````+````>@``````````````3@\````````+````>0``````````````
+M8!$````````"````:````/S_________L1$````````"````:````/S_____
+M____[A$````````"````>P```/S_________'A(````````"````?````/S_
+M________0A,````````"````>P```/S_________@A,````````"````?```
+M`/S_________FQ,````````+````A```````````````HQ,````````"````
+M?````/S_________%Q0````````"````:````/S_________:Q0````````"
+M````>P```/S_________>A0````````"````?````/S_________DQ0`````
+M```+````A```````````````FQ0````````"````?````/S_________>A8`
+M```````"````:````/S_________H18````````"````B````/S_________
+MVQ8````````"````>P```/S_________ZA8````````"````?````/S_____
+M____`Q<````````+````A```````````````"Q<````````"````?````/S_
+M________MQ<````````"````:````/S_________RQ<````````"````?@``
+M`/S_________\A<````````+````@P``````````````^A<````````+````
+M@@``````````````,1@````````"````?@```/S_________6!@````````+
+M````A0``````````````8!@````````+````@@``````````````AQ@`````
+M```"````?@```/S_________KA@````````+````AP``````````````MA@`
+M```````+````A@``````````````WQ@````````"````?````/S_________
+MZA@````````+````A```````````````\A@````````"````?````/S_____
+M____31D````````"````?````/S_________8!D````````+`````@```#`7
+M````````M1D````````+````B0``````````````P1D````````"````B@``
+M`/S_________SAH````````"````?@```/S_________X1H````````"````
+M;0```/S__________1H````````+````@0``````````````!1L````````+
+M````@```````````````(AP````````"````;P```/S_________/!P`````
+M```"````8P```!0`````````1QP````````"````90```"P`````````D1P`
+M```````"````<````/S_________V1P````````"````<@```/S_________
+MZQP````````"````<P```/S_________5AX````````"````;0```/S_____
+M____L!\````````"````>P```/S_________T!\````````"````?````/S_
+M________'B`````````"````?````/S_________12`````````"````?```
+M`/S_________<B`````````"````?@```/S_________QB`````````+````
+MC0``````````````SB`````````+````C```````````````5B(````````"
+M````D0```/S_________%20````````"````:````/S_________?R0`````
+M```"````:````/S_________P2H````````"````;0```/S_________TRH`
+M```````"````;0```/S_________,R\````````+`````@```&`K````````
+M+C`````````+`````@```*!D````````,S`````````"````?````/S_____
+M____;3`````````+````A```````````````=3`````````"````?````/S_
+M________CS`````````+````A```````````````ES`````````"````?```
+M`/S_________US`````````"````?````/S__________#`````````"````
+MD@```/S_________1#$````````+`````@```,`[````````23$````````"
+M````?````/S_________SS$````````"````DP```/S_________9#,`````
+M```"````>P```/S_________@S0````````"````?@```/S_________PC0`
+M```````+`````@```&`Q````````SC0````````+`````@```$`S````````
+MCS<````````"````E````/S_________L3<````````"````E0```/S_____
+M____,3@````````"````E````/S_________-SD````````"````E````/S_
+M________B3H````````"````E````/S_________J#H````````"````E0``
+M`/S_________O3P````````+````+@``````````````;ST````````+````
+M+@```#@`````````]&$````````"````?````/S_________E&(````````"
+M````D@```/S_________G&(````````"````EP```/S_________I&(`````
+M```"````F````/S_________=&0````````"````D@```/S_________]V0`
+M```````"````?````/S_________#&4````````+````A```````````````
+M%&4````````"````?````/S_________1&4````````+````A```````````
+M````3&4````````"````?````/S_________<F4````````"````D@```/S_
+M________A68````````"````!````/S_________CF8````````"````!```
+M`/S_________DV8````````"````DP```/S_________=F@````````"````
+M>P```/S_________-6L````````"````:````/S_________?FT````````+
+M````A```````````````AFT````````"````?````/S_________OVT`````
+M```"````?@```/S_________[FT````````+`````@```(!E````````]FT`
+M```````+`````@```%!H````````HG(````````+`````@```*!D````````
+MIW(````````"````?````/S_________XW(````````"````?````/S_____
+M____%W,````````"````:````/S_________/W,````````"````?````/S_
+M________87,````````"````?````/S_________&'0````````"````?```
+M`/S_________5G0````````+````F@``````````````8G0````````"````
+MB@```/S_________LW0````````+````FP``````````````[G0````````+
+M````F@``````````````_G0````````"````B@```/S_________!G4`````
+M```"````D@```/S_________&74````````+`````@```)!R````````.74`
+M```````"````?````/S_________=W4````````"````E@```/S_________
+M(G8````````"````D````/S_________9'8````````"````D````/S_____
+M____E78````````"````G0```/S_________G'8````````"````!````/S_
+M________I'8````````"````D````/S_________M'8````````"````G0``
+M`/S_________VW8````````"````D````/S_________LG<````````"````
+M;P```/S_________TW<````````"````8P```!0`````````WG<````````"
+M````90```#P`````````&G@````````"````<````/S_________,7@`````
+M```"````<@```/S_________/'@````````"````<P```/S_________TG@`
+M```````"````<P```/S_________2'D````````"````:````/S_________
+M?'H````````"````;0```/S_________D7H````````"````;0```/S_____
+M____J7H````````"````;0```/S_________OGH````````"````;0```/S_
+M________0'L````````"````9@```/S_________4GL````````"````9@``
+M`/S_________=WL````````"````9````/S_________RGL````````"````
+MGP```/S_________#'P````````"````H````/S_________:'P````````"
+M````=P```/S_________M'P````````"````:````/S_________%GT`````
+M```"````9````/S_________8'T````````"````9````/S_________H'\`
+M```````"````EP```/S_________]G\````````"````:````/S_________
+MRH`````````"````E@```/S_________\8`````````"````F0```/S_____
+M____38$````````"````IP```/S_________7($````````"````J````/S_
+M________@X$````````"````J0```/S_________/8(````````"````J@``
+M`/S_________2X(````````"````JP```/S_________OX(````````"````
+MK0```/S_________WX(````````"````K@```/S_________]H(````````"
+M````K0```/S_________2H,````````"````KP```/S_________7(,`````
+M```"````KP```/S_________BH,````````"````KP```/S_________H88`
+M```````"````L0```/S_________L(8````````"````K@```/S_________
+MR(8````````"````K@```/S_________X88````````"````K@```/S_____
+M____^X8````````"````K@```/S_________%8<````````"````K@```/S_
+M________-8<````````"````L0```/S_________08<````````"````K@``
+M`/S_________4H<````````"````K@```/S_________9(<````````"````
+MK@```/S_________=H<````````"````K@```/S_________CH<````````"
+M````L0```/S_________G8<````````"````K@```/S_________KX<`````
+M```"````K@```/S_________((@````````"````L@```/S_________FHH`
+M```````"````L@```/S_________L(H````````"````KP```/S_________
+MOHH````````"````L@```/S_________S(H````````"````L@```/S_____
+M____Y8H````````"````KP```/S_________\XH````````"````L@```/S_
+M________!(L````````"````KP```/S_________LHL````````"````KP``
+M`/S_________*XP````````"````L@```/S_________08P````````"````
+MKP```/S_________[HP````````"````KP```/S_________^XP````````"
+M````L@```/S_________!8T````````"````K0```/S_________%XT`````
+M```"````KP```/S_________)(T````````"````L@```/S_________+HT`
+M```````"````K0```/S_________1HT````````"````LP```/S_________
+M]HT````````"````L````/O__________XX````````"````K@```/S_____
+M____#8\````````"````K@```/S_________,H\````````"````KP```/S_
+M________U)`````````"````M````/S_________!9$````````"````M0``
+M`/S_________P9$````````"````L````/O_________-I(````````"````
+ML@```/S_________0)(````````"````K0```/S_________PY(````````"
+M````L@```/S_________UI(````````"````KP```/S_________X)(`````
+M```"````L@```/S_________ZI(````````"````K0```/S_________"9,`
+M```````"````KP```/S_________L),````````"````KP```/S_________
+MP9,````````"````KP```/S_________T),````````"````KP```/S_____
+M____Y9,````````"````KP```/S_________!I0````````"````KP```/S_
+M________%90````````"````KP```/S_________-I0````````"````KP``
+M`/S_________1Y0````````"````KP```/S_________J)4````````"````
+ML@```/S_________VI4````````"````KP```/S_________`I8````````"
+M````L@```/S_________798````````"````KP```/S_________O98`````
+M```"````KP```/S_________R98````````"````L@```/S_________TY8`
+M```````"````K0```/S_________Y)8````````"````KP```/S_________
+M\)8````````"````L@```/S_________=)<````````"````KP```/S_____
+M____?Y<````````"````L@```/S_________FI<````````"````KP```/S_
+M________JY<````````"````KP```/S_________OI<````````"````KP``
+M`/S_________T9<````````"````KP```/S_________Y)<````````"````
+MKP```/S_________]Y<````````"````KP```/S_________"I@````````"
+M````KP```/S_________'9@````````"````KP```/S_________,)@`````
+M```"````KP```/S_________0Y@````````"````KP```/S_________5I@`
+M```````"````KP```/S_________:9@````````"````KP```/S_________
+M?)@````````"````KP```/S_________JY@````````"````KP```/S_____
+M____O)@````````"````KP```/S_________S9@````````"````KP```/S_
+M________WI@````````"````KP```/S_________[)@````````"````L@``
+M`/S_________`YD````````"````KP```/S_________0YD````````"````
+MKP```/S_________:ID````````"````L@```/S_________?9D````````"
+M````KP```/S_________BYD````````"````L@```/S_________H)D`````
+M```"````KP```/S_________LYD````````"````KP```/S_________QID`
+M```````"````KP```/S_________V9D````````"````KP```/S_________
+M[)D````````"````KP```/S__________YD````````"````KP```/S_____
+M____$IH````````"````KP```/S_________)9H````````"````KP```/S_
+M________.)H````````"````KP```/S_________2YH````````"````KP``
+M`/S_________7IH````````"````KP```/S_________<9H````````"````
+MKP```/S_________;IL````````"````;0```/S_________@IL````````"
+M````;0```/S_________A9P````````"````K@```/S_________M)P`````
+M```"````KP```/S_________PYP````````"````L@```/S_________WIP`
+M```````"````KP```/S_________[9P````````"````L@```/S_________
+M-)T````````"````L@```/S_________3)T````````"````KP```/S_____
+M____QIT````````"````KP```/S_________VYT````````"````KP```/S_
+M________])T````````"````K@```/S_________$)X````````"````KP``
+M`/S_________>IX````````"````KP```/S_________AYX````````"````
+ML@```/S_________E9X````````"````K0```/S_________/9\````````"
+M````KP```/S_________=)\````````"````M@```/S_________U)\`````
+M```"````K@```/S_________`Z`````````"````L0```/S_________%Z``
+M```````"````L0```/S_________*Z`````````"````L0```/S_________
+M/Z`````````"````L0```/S_________4Z`````````"````L0```/S_____
+M____K*`````````"````L0```/S_________P*`````````"````L0```/S_
+M________U*`````````"````L0```/S_________Z*`````````"````L0``
+M`/S__________*`````````"````L0```/S_________$*$````````"````
+ML0```/S_________)*$````````"````L0```/S_________/:$````````"
+M````K@```/S_________6J$````````"````MP```/S_________<:$`````
+M```"````K@```/S_________IZ$````````"````M````/S_________&*(`
+M```````"````KP```/S_________(:(````````"````N0```/S_________
+M*Z,````````"````L@```/S_________B:,````````"````KP```/S_____
+M____EZ,````````"````L@```/S_________KZ,````````"````KP```/S_
+M________QJ,````````"````L@```/S_________0:0````````"````KP``
+M`/S_________4J0````````"````KP```/S_________8Z0````````"````
+MKP```/S_________BJ0````````"````KP```/S_________K*0````````"
+M````KP```/S_________X:0````````"````K0```/S_________$Z4`````
+M```"````LP```/S_________<Z4````````"````KP```/S_________DJ4`
+M```````"````L````/C_________HJ4````````"````L````/C_________
+M<:8````````"````L@```/S_________EZ8````````"````L@```/S_____
+M____H:8````````"````K0```/S_________K*8````````"````L@```/S_
+M________LJ<````````"````L@```/S_________W*<````````"````KP``
+M`/S__________*<````````"````L@```/S_________!J@````````"````
+MK0```/S_________*Z@````````"````KP```/S_________2Z@````````"
+M````L@```/S_________5:@````````"````K0```/S_________8Z@`````
+M```"````K````/S_________EZ@````````"````K@```/S_________,JD`
+M```````"````L@```/S_________?:D````````"````KP```/S_________
+M(:L````````"````L@```/S_________3ZL````````"````KP```/S_____
+M____S:L````````"````L@```/S_________^:L````````"````KP```/S_
+M________$JP````````"````L@```/S_________;JP````````"````L@``
+M`/S_________G*P````````"````KP```/S_________M:P````````"````
+ML@```/S_________W:P````````"````L@```/S_________#ZT````````"
+M````KP```/S_________?ZT````````"````L@```/S_________IZT`````
+M```"````LP```/S_________^:T````````"````L@```/S_________"*X`
+M```````"````L@```/S_________%ZX````````"````L@```/S_________
+M)JX````````"````L@```/S_________-:X````````"````L@```/S_____
+M____1*X````````"````L@```/S_________4ZX````````"````L@```/S_
+M________8JX````````"````L@```/S_________9[`````````"````L@``
+M`/S_________J;`````````"````L@```/S_________S[`````````"````
+MKP```/S_________%;$````````"````KP```/S_________(;$````````"
+M````L@```/S_________-[$````````"````KP```/S_________4K$`````
+M```"````L@```/S_________4[(````````"````KP```/S_________A+(`
+M```````"````KP```/S_________\K(````````"````K@```/S_________
+M#+,````````"````K0```/S_________&+,````````"````LP```/S_____
+M____:[,````````"````L0```/S_________@;,````````"````K@```/S_
+M________J[,````````"````L0```/S_________P;,````````"````K@``
+M`/S_________#[0````````"````RP```/S_________(K0````````"````
+ML0```/S_________/[0````````"````MP```/S_________2[0````````"
+M````R@```/S_________6K0````````"````R@```/S_________`[4`````
+M```"````J@```/S_________0;4````````"````J0```/S_________Q[4`
+M```````"````M@```/S_________)[8````````"````J@```/S_________
+M\;<````````"````MP```/S_________!K@````````"````RP```/S_____
+M____(+@````````"````L0```/S_________-;@````````"````L0```/S_
+M________2[@````````"````L0```/S_________8;@````````"````L0``
+M`/S_________=[@````````"````L0```/S_________K;@````````"````
+MR@```/S_________RK@````````"````L0```/S_________W;@````````"
+M````L0```/S_________\;@````````"````L0```/S_________!;D`````
+M```"````L0```/S_________&;D````````"````L0```/S_________+;D`
+M```````"````L0```/S_________0;D````````"````L0```/S_________
+M9;D````````"````MP```/S_________<;D````````"````R@```/S_____
+M____@[D````````"````R@```/S_________JKD````````"````MP```/S_
+M________MKD````````"````R@```/S_________T+D````````"````K@``
+M`/S_________W+D````````"````R@```/S_________&KH````````"````
+MMP```/S_________,+H````````"````M0```/S_________4[H````````"
+M````M````/S_________>+H````````"````MP```/S_________A+H`````
+M```"````R@```/S_________D[H````````"````R@```/S_________HKH`
+M```````"````R@```/S_________%+L````````"````L0```/S_________
+M);L````````"````K@```/S_________+[L````````"````K0```/S_____
+M____-[L````````"````R@```/S_________0;L````````"````K0```/S_
+M________2[L````````"````K0```/S_________5;L````````"````K0``
+M`/S_________7[L````````"````K0```/S_________=[L````````"````
+MK@```/S_________D;L````````"````K0```/S_________R;L````````"
+M````K0```/S_________[+L````````"`````P```!P!````````_[L`````
+M```"````L0```/S_________"[P````````"`````P```!P!````````'+P`
+M```````"````K@```/S_________,[P````````"`````P```!P!````````
+M1+P````````"````K@```/S_________5[P````````"`````P```!P!````
+M````:+P````````"````K@```/S_________A[P````````"`````P```!P!
+M````````FKP````````"````L0```/S_________H;P````````"`````P``
+M`!P!````````M[P````````"````L0```/S_________V;P````````"````
+M`P```!P!````````[[P````````"````L0```/S_________]KP````````"
+M`````P```!P!````````!KT````````"````K@```/S_________1[T`````
+M```"````!`````,`````````6+T````````"````L@```/S_________;;T`
+M```````"````KP```/S_________>KT````````"````L@```/S_________
+MF;T````````"````M````/S_________IKT````````"````L@```/S_____
+M____M;T````````"`````P```!P!````````R[T````````"````L0```/S_
+M________TKT````````"`````P```!P!````````Y[T````````"````L0``
+M`/S_________[KT````````"`````P```!P!````````!+X````````"````
+ML0```/S_________"[X````````"`````P```!P!````````(+X````````"
+M````L0```/S_________-[X````````"````KP```/S_________1+X`````
+M```"````L@```/S_________5+X````````"````KP```/S_________?;X`
+M```````"````L@```/S_________DKX````````"````KP```/S_________
+MG[X````````"````L@```/S_________OKX````````"````M````/S_____
+M____R[X````````"````L@```/S_________VKX````````"`````P```!P!
+M````````\+X````````"````L0```/S_________][X````````"`````P``
+M`!P!````````#+\````````"````L0```/S_________$[\````````"````
+M`P```!P!````````*;\````````"````L0```/S_________,+\````````"
+M`````P```!P!````````1;\````````"````L0```/S_________7+\`````
+M```"````KP```/S_________:;\````````"````L@```/S_________>;\`
+M```````"````KP```/S_________G+\````````"`````P```!P!````````
+MLK\````````"````L0```/S_________N;\````````"`````P```!P!````
+M````SK\````````"````L0```/S_________X[\````````"````K@```/S_
+M________^[\````````"`````P```!P!````````$,`````````"````L0``
+M`/S_________(L`````````"````K0```/S_________+\`````````"````
+MK@```/S_________?\`````````"````L@```/S_________E,`````````"
+M````KP```/S_________H<`````````"````L@```/S_________P,``````
+M```"````M````/S_________S<`````````"````L@```/S_________X,``
+M```````"`````P```!P!````````\<`````````"````L0```/S_________
+M`,$````````"````KP```/S_________#<$````````"````L@```/S_____
+M____'<$````````"````KP```/S_________=<$````````"````T0```/S_
+M________G<$````````"````L@```/S_________LL$````````"````KP``
+M`/S_________O\$````````"````L@```/S_________X,$````````"````
+MM````/S_________[<$````````"````L@```/S_________^L$````````"
+M````M0```/S_________(\(````````"`````P```!P!````````.<(`````
+M```"````L0```/S_________4\(````````"````K0```/S_________8,(`
+M```````"````M0```/S_________:<(````````"`````P```!P!````````
+M?\(````````"````L0```/S_________I<(````````"`````P```!P!````
+M````N\(````````"````L0```/S_________R,(````````"````M0```/S_
+M________^L(````````"`````P```!P!````````$,,````````"````L0``
+M`/S_________(,,````````"````KP```/S_________+<,````````"````
+ML@```/S_________/<,````````"````KP```/S_________<\,````````"
+M`````P```!@!````````=\,````````+`````P```.``````````BL,`````
+M```"`````P```!@!````````CL,````````+`````P`````!````````TL,`
+M```````"````!`````0`````````*\0````````"````L@```/S_________
+M0<0````````"````KP```/S_________3L0````````"````L@```/S_____
+M____;\0````````"````M````/S_________?,0````````"````L@```/S_
+M________G,0````````"````K@```/S_________T<0````````"````L0``
+M`/S_________[,0````````"````KP```/S_________^<0````````"````
+ML@```/S_________"L4````````"````KP```/S_________(``````````!
+M````G@``````````````*``````````!````G@``````````````,```````
+M```!````G@``````````````.``````````!````G@``````````````0```
+M```````!````C@``````````````2``````````!````?0``````````````
+M4``````````!````B0``````````````8``````````!````F@``````````
+M````@``````````!````=0``````````````B``````````!````=0``````
+M````````D``````````!````=0``````````````F``````````!````=0``
+M````````````H``````````!````CP``````````````J``````````!````
+M?P``````````````L``````````!````BP``````````````P``````````!
+M````FP``````````````(`$````````!`````P`````!````````````````
+M```!`````@```.T\````````"``````````!`````@```,$\````````$```
+M```````!`````@```,$\````````&``````````!`````@```,@\````````
+M(``````````!`````@```-<\````````*``````````!`````@```.(\````
+M````,``````````!`````@```.D\````````.``````````!`````@```)T]
+M````````0``````````!`````@```',]````````2``````````!`````@``
+M`',]````````4``````````!`````@```'H]````````6``````````!````
+M`@```(<]````````8``````````!`````@```)(]````````:``````````!
+M`````@```)D]````````(``````````!`````@``````````````4```````
+M```!`````@```'`&````````>``````````!`````@```'`'````````H```
+M```````!`````@```*`)````````R``````````!`````@`````*````````
+M"`$````````!`````@```%`+````````0`$````````!`````@```%`,````
+M````6`$````````!`````@```.`,````````>`$````````!`````@`````.
+M````````H`$````````!`````@```&`.````````R`$````````!`````@``
+M`*`/````````Z`$````````!`````@```/`/````````"`(````````!````
+M`@```&`0````````,`(````````!`````@```-`0````````6`(````````!
+M`````@```'`2````````>`(````````!`````@`````3````````H`(`````
+M```!`````@```,`3````````R`(````````!`````@```,`4````````\`(`
+M```````!`````@```"`6````````&`,````````!`````@```#`7````````
+M0`,````````!`````@```!`9````````:`,````````!`````@```&`;````
+M````D`,````````!`````@`````=````````N`,````````!`````@```)`>
+M````````T`,````````!`````@```-`>````````Z`,````````!`````@``
+M`(`?````````$`0````````!`````@```.`?````````.`0````````!````
+M`@```"`A````````6`0````````!`````@```%`A````````<`0````````!
+M`````@```'`A````````B`0````````!`````@```)`A````````R`0`````
+M```!`````@```#`C````````\`0````````!`````@```*`C````````"`4`
+M```````!`````@```/`C````````,`4````````!`````@```#`D````````
+M2`4````````!`````@```'`D````````:`4````````!`````@```)`D````
+M````@`4````````!`````@```-`D````````F`4````````!`````@`````E
+M````````P`4````````!`````@```)`E````````V`4````````!`````@``
+M`,`E``````````8````````!`````@```"`F````````0`8````````!````
+M`@```%`H````````:`8````````!`````@```"`I````````J`8````````!
+M`````@```&`K````````T`8````````!`````@```&`L````````^`8`````
+M```!`````@```#`Q````````&`<````````!`````@```&`Q````````6`<`
+M```````!`````@```"`R````````<`<````````!`````@```$`S````````
+MF`<````````!`````@```!`T````````N`<````````!`````@```%`T````
+M````X`<````````!`````@`````U````````"`@````````!`````@```)`V
+M````````.`@````````!`````@```)`X````````:`@````````!`````@``
+M`)`Y````````J`@````````!`````@```.`Z````````T`@````````!````
+M`@```&`[````````Z`@````````!`````@```,`[````````*`D````````!
+M`````@`````^````````:`D````````!`````@```"!A````````J`D`````
+M```!`````@```&!B````````T`D````````!`````@```-!B````````\`D`
+M```````!`````@```"!D````````&`H````````!`````@```*!D````````
+M0`H````````!`````@```(!E````````@`H````````!`````@```%!H````
+M````H`H````````!`````@```.!H````````X`H````````!`````@```&!N
+M````````(`L````````!`````@```)!R````````0`L````````!`````@``
+M`,!R````````<`L````````!`````@```&!U````````F`L````````!````
+M`@```-!U````````P`L````````!`````@```/!V````````Z`L````````!
+M`````@```%!X``````````P````````!`````@```(!X````````*`P`````
+M```!`````@````!Y````````6`P````````!`````@```)![````````<`P`
+M```````!`````@```.![````````F`P````````!`````@```#!\````````
+ML`P````````!`````@```&!\````````R`P````````!`````@```'!\````
+M````^`P````````!`````@```.!]````````$`T````````!`````@```$!^
+M````````,`T````````!`````@```,!^````````2`T````````!`````@``
+M`"!_````````:`T````````!`````@```-!_````````B`T````````!````
+M`@````"`````````H`T````````!`````@```!"`````````N`T````````!
+M`````@```$"`````````X`T````````!`````@```,"```````````X`````
+M```!`````@```."`````````(`X````````!`````@```!"!````````2`X`
+M```````!`````@```(""````````8`X````````!`````@```+""````````
+MB`X````````!`````@```""#````````J`X````````!`````@```'"#````
+M````R`X````````!`````@```)"#````````X`X````````!`````@```%"&
+M````````"`\````````!`````@```."'````````,`\````````!`````@``
+M`-"(````````6`\````````!`````@```'"*````````@`\````````!````
+M`@```#"+````````J`\````````!`````@```#".````````R`\````````!
+M`````@```+".````````\`\````````!`````@```'"0````````&!``````
+M```!`````@```#"1````````0!`````````!`````@```*"2````````<!``
+M```````!`````@```&"4````````L!`````````!`````@```$"5````````
+MV!`````````!`````@```("6`````````!$````````!`````@```""7````
+M````0!$````````!`````@```#"9````````8!$````````!`````@```%"9
+M````````@!$````````!`````@```(":````````F!$````````!`````@``
+M`-":````````L!$````````!`````@```$";````````V!$````````!````
+M`@```,";````````^!$````````!`````@```""<````````$!(````````!
+M`````@```%"<````````.!(````````!`````@````"=````````6!(`````
+M```!`````@```&"=````````@!(````````!`````@```$">````````N!(`
+M```````!`````@```.">````````X!(````````!`````@```%"?````````
+M`!,````````!`````@```*"?````````*!,````````!`````@```."A````
+M````4!,````````!`````@```&"E````````<!,````````!`````@```)"E
+M````````B!,````````!`````@```*"E````````H!,````````!`````@``
+M`+"E````````P!,````````!`````@```!"F````````V!,````````!````
+M`@```$"F`````````!0````````!`````@```-"F````````0!0````````!
+M`````@```."H````````:!0````````!`````@```+"I````````B!0`````
+M```!`````@````"J````````J!0````````!`````@```$"J````````R!0`
+M```````!`````@```)"J````````X!0````````!`````@```+"J````````
+M"!4````````!`````@```)"K````````,!4````````!`````@```#"L````
+M````8!4````````!`````@```#"M````````H!4````````!`````@```("N
+M````````R!4````````!`````@```#"P````````\!4````````!`````@``
+M`$"R````````$!8````````!`````@```'"R````````,!8````````!````
+M`@```*"R````````6!8````````!`````@```%"S````````>!8````````!
+M`````@```)"S````````F!8````````!`````@```-"S````````P!8`````
+M```!`````@```("T````````\!8````````!`````@```*"U````````$!<`
+M```````!`````@```-"U````````,!<````````!`````@```#"V````````
+M6!<````````!`````@```""W````````B!<````````!`````@```/"Z````
+M````L!<````````!`````@```+"[````````T!<````````!`````@```."[
+M````````^!<````````!`````@```("\````````&!@````````!`````@``
+M`,"\````````.!@````````!`````@```#"]````````8!@````````!````
+M`@```'"^````````B!@````````!`````@```)"_````````J!@````````!
+M`````@```%#`````````T!@````````!`````@```$#!````````\!@`````
+M```!`````@```(#!````````,!D````````!`````@```&##````````6!D`
+M```````!`````@```,##````````<!D````````!`````@```/##````````
 `
 end
diff --git a/sys/dev/hptmv/array.h b/sys/dev/hptmv/array.h
index 14d627a..7b50774 100644
--- a/sys/dev/hptmv/array.h
+++ b/sys/dev/hptmv/array.h
@@ -62,7 +62,7 @@ typedef struct _RaidArray
     UCHAR   reserve1;
 
     ULONG   dArStamp;          /* array ID. all disks in a array has same ID */
-	ULONG   failedStamp;       /* stamp for failed member */
+	ULONG   failedStamps[4];   /* stamp for failed members */
     USHORT  bStripeWitch;      /* = (1 << BlockSizeShift) */
 
 	USHORT	rf_broken: 1;
@@ -84,7 +84,7 @@ typedef struct _RaidArray
 
 	USHORT  CriticalMembers;   /* tell which member is critial */
 	UCHAR   last_read;       /* for RAID 1 load banlancing */
-	UCHAR   PrivateFlag1;
+	UCHAR   alreadyBroken;
 
 	LBA_T   RebuildSectors;  /* how many sectors is OK (LBA on member disk) */
 
@@ -152,8 +152,8 @@ typedef struct _ArrayDescript
 #define ArrayDescript_3_1_size 512
 
 	UCHAR   bCheckSum31;        /* new check sum */
-	UCHAR   PrivateFlag1;       /* private */
-	UCHAR   reserve1;
+	UCHAR   PrivateFlag1;       /* private */ 
+	UCHAR   alreadyBroken;      /* last stamp has been saved to failedStamps */
 	
 #ifdef __BIG_ENDIAN_BITFIELD
     UCHAR   df_read_ahead: 1;   /* enable read ahead */
@@ -181,7 +181,7 @@ typedef struct _ArrayDescript
     }
     levelex[2];
 
-	ULONG failedStamp; /* array stamp for failed memebr */
+	ULONG failedStamps[4]; /* failed memebrs's stamps */
 
 } ArrayDescript;
 
diff --git a/sys/dev/hptmv/command.h b/sys/dev/hptmv/command.h
index 12eb694..3b6a53b 100644
--- a/sys/dev/hptmv/command.h
+++ b/sys/dev/hptmv/command.h
@@ -42,9 +42,9 @@ typedef struct _AtaCommand
 typedef struct _PassthroughCmd {
 	BYTE     bFeaturesReg;     /* feature register */
 	BYTE     bSectorCountReg;  /* IDE sector count register. */
-	BYTE     bSectorNumberReg; /* IDE sector number register. */
-	BYTE     bCylLowReg;       /* IDE low order cylinder value. */
-	BYTE     bCylHighReg;      /* IDE high order cylinder value. */
+	BYTE     bLbaLowReg; /* IDE sector number register. */
+	BYTE     bLbaMidReg;       /* IDE low order cylinder value. */
+	BYTE     bLbaHighReg;      /* IDE high order cylinder value. */
 	BYTE     bDriveHeadReg;    /* IDE drive/head register. */
 	BYTE     bCommandReg;      /* Actual IDE command. Checked for validity by driver. */
 	BYTE     nSectors;         /* data transfer */
@@ -247,7 +247,7 @@ DPC_ROUTINE;
  * IdleRoutines[] size:
  *   Each command may invoke CallWhenIdle once.
  */
-#define MAX_COMMAND_BLOCKS_FOR_EACH_VBUS (MAX_QUEUE_COMM * (1+MAX_MEMBERS*2))
+#define MAX_COMMAND_BLOCKS_FOR_EACH_VBUS (MAX_QUEUE_COMM * (1+MAX_MEMBERS*2) + 1)
 #define MAX_PENDING_ROUTINES  (MAX_COMMAND_BLOCKS_FOR_EACH_VBUS+1)
 #define MAX_IDLE_ROUTINES     (MAX_COMMAND_BLOCKS_FOR_EACH_VBUS+1)
 
diff --git a/sys/dev/hptmv/entry.c b/sys/dev/hptmv/entry.c
index 33c3066..843281a 100644
--- a/sys/dev/hptmv/entry.c
+++ b/sys/dev/hptmv/entry.c
@@ -93,7 +93,7 @@ static device_method_t driver_methods[] = {
 	DEVMETHOD(device_attach,	hpt_attach),
 	DEVMETHOD(device_detach,	hpt_detach),
 
-/*	DEVMETHOD(device_shutdown,	hpt_shutdown), */
+	DEVMETHOD(device_shutdown,	hpt_shutdown),
 	{ 0, 0 }
 };
 
@@ -158,7 +158,7 @@ static ST_HPT_DPC DpcQueue[MAX_DPC];
 static int DpcQueue_First=0;
 static int DpcQueue_Last = 0;
 
-char DRIVER_VERSION[] = "v1.12";
+char DRIVER_VERSION[] = "v1.16";
 
 #if (__FreeBSD_version >= 500000)
 static struct mtx driver_lock;
@@ -204,7 +204,7 @@ void unlock_driver(intrmask_t spl)
  *
  *	Description:	free allocated queues for the given channel
  *
- *	Parameters:    	pMvSataAdapter - pointer to the RR182x controler this 
+ *	Parameters:    	pMvSataAdapter - pointer to the RR18xx controler this 
  * 					channel connected to. 
  *			channelNum - channel number. 
  *     
@@ -238,7 +238,6 @@ static void failDevice(PVDevice pVDev)
 }
 
 int MvSataResetChannel(MV_SATA_ADAPTER *pMvSataAdapter, MV_U8 channel);
-/*void fDeviceSendCommand(_VBUS_ARG PCommand pCmd); */
 
 static void
 handleEdmaError(_VBUS_ARG PCommand pCmd)
@@ -292,7 +291,7 @@ hptmv_init_channel(IAL_ADAPTER_T *pAdapter, MV_U8 channelNum)
 
 	if (channelNum >= MV_SATA_CHANNELS_NUM)
 	{
-		MV_ERROR("RR182x[%d]: Bad channelNum=%d",
+		MV_ERROR("RR18xx[%d]: Bad channelNum=%d",
 				 pAdapter->mvSataAdapter.adapterId, channelNum);
 		return -1;
 	}
@@ -313,14 +312,14 @@ hptmv_init_channel(IAL_ADAPTER_T *pAdapter, MV_U8 channelNum)
 	/* check the 1K alignment of the request queue*/
 	if (req_dma_addr & 0x3ff)
 	{
-		MV_ERROR("RR182x[%d]: request queue allocated isn't 1 K aligned,"
+		MV_ERROR("RR18xx[%d]: request queue allocated isn't 1 K aligned,"
 				 " dma_addr=%llx channel=%d\n", pAdapter->mvSataAdapter.adapterId,
 				 (HPT_U64)(ULONG_PTR)req_dma_addr, channelNum);
 		return -1;
 	}
 	pMvSataChannel->requestQueuePciLowAddress = req_dma_addr;
 	pMvSataChannel->requestQueuePciHiAddress = 0;
-	KdPrint(("RR182x[%d,%d]: request queue allocated: 0x%p",
+	KdPrint(("RR18xx[%d,%d]: request queue allocated: 0x%p",
 			  pAdapter->mvSataAdapter.adapterId, channelNum,
 			  pMvSataChannel->requestQueue));
 	pMvSataChannel->responseQueue = (struct mvDmaResponseQueueEntry *)
@@ -330,14 +329,14 @@ hptmv_init_channel(IAL_ADAPTER_T *pAdapter, MV_U8 channelNum)
 	/* check the 256 alignment of the response queue*/
 	if (rsp_dma_addr & 0xff)
 	{
-		MV_ERROR("RR182x[%d,%d]: response queue allocated isn't 256 byte "
+		MV_ERROR("RR18xx[%d,%d]: response queue allocated isn't 256 byte "
 				 "aligned, dma_addr=%llx\n",
 				 pAdapter->mvSataAdapter.adapterId, channelNum, (HPT_U64)(ULONG_PTR)rsp_dma_addr);
 		return -1;
 	}
 	pMvSataChannel->responseQueuePciLowAddress = rsp_dma_addr;
 	pMvSataChannel->responseQueuePciHiAddress = 0;
-	KdPrint(("RR182x[%d,%d]: response queue allocated: 0x%p",
+	KdPrint(("RR18xx[%d,%d]: response queue allocated: 0x%p",
 			  pAdapter->mvSataAdapter.adapterId, channelNum,
 			  pMvSataChannel->responseQueue));
 
@@ -349,7 +348,7 @@ hptmv_init_channel(IAL_ADAPTER_T *pAdapter, MV_U8 channelNum)
  *	Name: hptmv_parse_identify_results
  *
  *	Description:	this functions parses the identify command results, checks
- *					that the connected deives can be accesed by RR182x EDMA,
+ *					that the connected deives can be accesed by RR18xx EDMA,
  *					and updates the channel stucture accordingly.
  *
  *	Parameters:     pMvSataChannel, pointer to the channel data structure.
@@ -507,14 +506,14 @@ start_channel(IAL_ADAPTER_T *pAdapter, MV_U8 channelNum)
 	MV_CHANNEL		*pChannelInfo = &(pAdapter->mvChannel[channelNum]);
 	MV_U32          udmaMode,pioMode;
 
-	KdPrint(("RR182x [%d]: start channel (%d)", pMvSataAdapter->adapterId, 
+	KdPrint(("RR18xx [%d]: start channel (%d)", pMvSataAdapter->adapterId, 
 			 channelNum));
 
 
 	/* Software reset channel */
 	if (mvStorageDevATASoftResetDevice(pMvSataAdapter, channelNum) == MV_FALSE)
 	{
-		MV_ERROR("RR182x [%d,%d]: failed to perform Software reset\n",
+		MV_ERROR("RR18xx [%d,%d]: failed to perform Software reset\n",
 				 pMvSataAdapter->adapterId, channelNum);
 		return -1;
 	}
@@ -527,7 +526,7 @@ start_channel(IAL_ADAPTER_T *pAdapter, MV_U8 channelNum)
 		StallExec(5000000); /* wait 5 sec before trying again */
 		if (mvSataChannelHardReset(pMvSataAdapter, channelNum) == MV_FALSE)
 		{
-			MV_ERROR("RR182x [%d,%d]: failed to perform Hard reset\n",
+			MV_ERROR("RR18xx [%d,%d]: failed to perform Hard reset\n",
 					 pMvSataAdapter->adapterId, channelNum);
 			return -1;
 		}
@@ -536,13 +535,13 @@ start_channel(IAL_ADAPTER_T *pAdapter, MV_U8 channelNum)
 	/* identify device*/
 	if (mvStorageDevATAIdentifyDevice(pMvSataAdapter, channelNum) == MV_FALSE)
 	{
-		MV_ERROR("RR182x [%d,%d]: failed to perform ATA Identify command\n"
+		MV_ERROR("RR18xx [%d,%d]: failed to perform ATA Identify command\n"
 				 , pMvSataAdapter->adapterId, channelNum);
 		return -1;
 	}
 	if (hptmv_parse_identify_results(pMvSataChannel))
 	{
-		MV_ERROR("RR182x [%d,%d]: Error in parsing ATA Identify message\n"
+		MV_ERROR("RR18xx [%d,%d]: Error in parsing ATA Identify message\n"
 				 , pMvSataAdapter->adapterId, channelNum);
 		return -1;
 	}
@@ -551,13 +550,13 @@ start_channel(IAL_ADAPTER_T *pAdapter, MV_U8 channelNum)
 	/* Disable 8 bit PIO in case CFA enabled */
 	if (pMvSataChannel->identifyDevice[86] & 4)
 	{
-		KdPrint(("RR182x [%d]: Disable 8 bit PIO (CFA enabled) \n",
+		KdPrint(("RR18xx [%d]: Disable 8 bit PIO (CFA enabled) \n",
 				  pMvSataAdapter->adapterId));
 		if (mvStorageDevATASetFeatures(pMvSataAdapter, channelNum,
 									   MV_ATA_SET_FEATURES_DISABLE_8_BIT_PIO, 0,
 									   0, 0, 0) == MV_FALSE)
 		{
-			MV_ERROR("RR182x [%d]: channel %d: mvStorageDevATASetFeatures"
+			MV_ERROR("RR18xx [%d]: channel %d: mvStorageDevATASetFeatures"
 					 " failed\n", pMvSataAdapter->adapterId, channelNum); 
 			return -1;
 		}
@@ -572,48 +571,48 @@ start_channel(IAL_ADAPTER_T *pAdapter, MV_U8 channelNum)
 										   MV_ATA_SET_FEATURES_ENABLE_WCACHE, 0,
 										   0, 0, 0) == MV_FALSE)
 			{
-				MV_ERROR("RR182x [%d]: channel %d: mvStorageDevATASetFeatures failed\n",
+				MV_ERROR("RR18xx [%d]: channel %d: mvStorageDevATASetFeatures failed\n",
 						 pMvSataAdapter->adapterId, channelNum); 
 				return -1;
 			}
 		}
-		KdPrint(("RR182x [%d]: channel %d, write cache enabled\n",
+		KdPrint(("RR18xx [%d]: channel %d, write cache enabled\n",
 				  pMvSataAdapter->adapterId, channelNum));
 	}
 	else
 	{
-		KdPrint(("RR182x [%d]: channel %d, write cache not supported\n",
+		KdPrint(("RR18xx [%d]: channel %d, write cache not supported\n",
 				  pMvSataAdapter->adapterId, channelNum));
 	}
 #else /* disable write cache */
 	{
 		if (pMvSataChannel->identifyDevice[85] & 0x20)
 		{
-			KdPrint(("RR182x [%d]: channel =%d, disable write cache\n",
+			KdPrint(("RR18xx [%d]: channel =%d, disable write cache\n",
 					  pMvSataAdapter->adapterId, channelNum));
 			if (mvStorageDevATASetFeatures(pMvSataAdapter, channelNum,
 										   MV_ATA_SET_FEATURES_DISABLE_WCACHE, 0,
 										   0, 0, 0) == MV_FALSE)
 			{
-				MV_ERROR("RR182x [%d]: channel %d: mvStorageDevATASetFeatures failed\n",
+				MV_ERROR("RR18xx [%d]: channel %d: mvStorageDevATASetFeatures failed\n",
 						 pMvSataAdapter->adapterId, channelNum); 
 				return -1;
 			}
 		}
-		KdPrint(("RR182x [%d]: channel=%d, write cache disabled\n",
+		KdPrint(("RR18xx [%d]: channel=%d, write cache disabled\n",
 				  pMvSataAdapter->adapterId, channelNum));
 	}
 #endif
 
 	/* Set transfer mode */
-	KdPrint(("RR182x [%d] Set transfer mode XFER_PIO_SLOW\n",
+	KdPrint(("RR18xx [%d] Set transfer mode XFER_PIO_SLOW\n",
 			  pMvSataAdapter->adapterId));
 	if (mvStorageDevATASetFeatures(pMvSataAdapter, channelNum,
 								   MV_ATA_SET_FEATURES_TRANSFER,
 								   MV_ATA_TRANSFER_PIO_SLOW, 0, 0, 0) == 
 		MV_FALSE)
 	{
-		MV_ERROR("RR182x [%d] channel %d: Set Features failed\n",
+		MV_ERROR("RR18xx [%d] channel %d: Set Features failed\n",
 				 pMvSataAdapter->adapterId, channelNum); 
 		return -1;
 	}
@@ -632,14 +631,14 @@ start_channel(IAL_ADAPTER_T *pAdapter, MV_U8 channelNum)
 		pioMode = MV_ATA_TRANSFER_PIO_SLOW;
 	}
 
-	KdPrint(("RR182x [%d] Set transfer mode XFER_PIO_4\n",
+	KdPrint(("RR18xx [%d] Set transfer mode XFER_PIO_4\n",
 			  pMvSataAdapter->adapterId));
 	pAdapter->mvChannel[channelNum].maxPioModeSupported = pioMode;
 	if (mvStorageDevATASetFeatures(pMvSataAdapter, channelNum,
 								   MV_ATA_SET_FEATURES_TRANSFER,
 								   pioMode, 0, 0, 0) == MV_FALSE)
 	{
-		MV_ERROR("RR182x [%d] channel %d: Set Features failed\n",
+		MV_ERROR("RR18xx [%d] channel %d: Set Features failed\n",
 				 pMvSataAdapter->adapterId, channelNum); 
 		return -1;
 	}
@@ -666,7 +665,7 @@ start_channel(IAL_ADAPTER_T *pAdapter, MV_U8 channelNum)
 		udmaMode =  MV_ATA_TRANSFER_UDMA_2;
 	}
 
-	KdPrint(("RR182x [%d] Set transfer mode XFER_UDMA_%d\n",
+	KdPrint(("RR18xx [%d] Set transfer mode XFER_UDMA_%d\n",
 			  pMvSataAdapter->adapterId, udmaMode & 0xf));
 	pChannelInfo->maxUltraDmaModeSupported = udmaMode;
 
@@ -674,7 +673,7 @@ start_channel(IAL_ADAPTER_T *pAdapter, MV_U8 channelNum)
 								   MV_ATA_SET_FEATURES_TRANSFER, udmaMode,
 								   0, 0, 0) == MV_FALSE)
 	{
-		MV_ERROR("RR182x [%d] channel %d: Set Features failed\n",
+		MV_ERROR("RR18xx [%d] channel %d: Set Features failed\n",
 				 pMvSataAdapter->adapterId, channelNum); 
 		return -1;
 	}*/
@@ -721,48 +720,48 @@ start_channel(IAL_ADAPTER_T *pAdapter, MV_U8 channelNum)
 										   MV_ATA_SET_FEATURES_ENABLE_RLA, 0, 0,
 										   0, 0) == MV_FALSE)
 			{
-				MV_ERROR("RR182x [%d] channel %d: Set Features failed\n",
+				MV_ERROR("RR18xx [%d] channel %d: Set Features failed\n",
 						 pMvSataAdapter->adapterId, channelNum); 
 				return -1;
 			}
 		}
-		KdPrint(("RR182x [%d]: channel=%d, read look ahead enabled\n", 
+		KdPrint(("RR18xx [%d]: channel=%d, read look ahead enabled\n", 
 				  pMvSataAdapter->adapterId, channelNum));
 	}
 	else
 	{
-		KdPrint(("RR182x [%d]: channel %d, Read Look Ahead not supported\n",
+		KdPrint(("RR18xx [%d]: channel %d, Read Look Ahead not supported\n",
 				  pMvSataAdapter->adapterId, channelNum));
 	}
 #else 
 	{
 		if (pMvSataChannel->identifyDevice[86] & 0x20)
 		{
-			KdPrint(("RR182x [%d]:channel %d, disable read look ahead\n",
+			KdPrint(("RR18xx [%d]:channel %d, disable read look ahead\n",
 					  pMvSataAdapter->adapterId, channelNum));
 			if (mvStorageDevATASetFeatures(pMvSataAdapter, channelNum,
 										   MV_ATA_SET_FEATURES_DISABLE_RLA, 0, 0,
 										   0, 0) == MV_FALSE)
 			{
-				MV_ERROR("RR182x [%d]:channel %d:  ATA Set Features failed\n",
+				MV_ERROR("RR18xx [%d]:channel %d:  ATA Set Features failed\n",
 						 pMvSataAdapter->adapterId, channelNum); 
 				return -1;
 			}
 		}
-		KdPrint(("RR182x [%d]:channel %d, read look ahead disabled\n",
+		KdPrint(("RR18xx [%d]:channel %d, read look ahead disabled\n",
 				  pMvSataAdapter->adapterId, channelNum));
 	}    
 #endif
 
 
 	{
-		KdPrint(("RR182x [%d]: channel %d config EDMA, Non Queued Mode\n",
+		KdPrint(("RR18xx [%d]: channel %d config EDMA, Non Queued Mode\n",
 				  pMvSataAdapter->adapterId, 
 				  channelNum));
 		if (mvSataConfigEdmaMode(pMvSataAdapter, channelNum,
 								 MV_EDMA_MODE_NOT_QUEUED, 0) == MV_FALSE)
 		{
-			MV_ERROR("RR182x [%d] channel %d Error: mvSataConfigEdmaMode failed\n",
+			MV_ERROR("RR18xx [%d] channel %d Error: mvSataConfigEdmaMode failed\n",
 					 pMvSataAdapter->adapterId, channelNum);
 			return -1;
 		}
@@ -770,11 +769,11 @@ start_channel(IAL_ADAPTER_T *pAdapter, MV_U8 channelNum)
 	/* Enable EDMA */
 	if (mvSataEnableChannelDma(pMvSataAdapter, channelNum) == MV_FALSE)
 	{
-		MV_ERROR("RR182x [%d] Failed to enable DMA, channel=%d\n",
+		MV_ERROR("RR18xx [%d] Failed to enable DMA, channel=%d\n",
 				 pMvSataAdapter->adapterId, channelNum);
 		return -1;
 	}
-	MV_ERROR("RR182x [%d,%d]: channel started successfully\n",
+	MV_ERROR("RR18xx [%d,%d]: channel started successfully\n",
 			 pMvSataAdapter->adapterId, channelNum);
 
 #ifndef FOR_DEMO
@@ -799,12 +798,12 @@ hptmv_handle_event(void * data, int flag)
 				/* Handle only connects */
 				if (flag == 1)
 					break;
-				KdPrint(("RR182x [%d,%d]: new device connected\n",
+				KdPrint(("RR18xx [%d,%d]: new device connected\n",
 						 pMvSataAdapter->adapterId, channelIndex));
 				hptmv_init_channel(pAdapter, channelIndex);
 				if (mvSataConfigureChannel( pMvSataAdapter, channelIndex) == MV_FALSE)
 				{
-					MV_ERROR("RR182x [%d,%d] Failed to configure\n",
+					MV_ERROR("RR18xx [%d,%d] Failed to configure\n",
 							 pMvSataAdapter->adapterId, channelIndex);
 					hptmv_free_channel(pAdapter, channelIndex);
 				}
@@ -813,7 +812,7 @@ hptmv_handle_event(void * data, int flag)
 					/*mvSataChannelHardReset(pMvSataAdapter, channel);*/
 					if (start_channel( pAdapter, channelIndex))
 					{
-						MV_ERROR("RR182x [%d,%d]Failed to start channel\n",
+						MV_ERROR("RR18xx [%d,%d]Failed to start channel\n",
 								 pMvSataAdapter->adapterId, channelIndex);
 						hptmv_free_channel(pAdapter, channelIndex);
 					}
@@ -829,7 +828,7 @@ hptmv_handle_event(void * data, int flag)
 				/* Handle only disconnects */
 				if (flag == 0)
 					break;
-				KdPrint(("RR182x [%d,%d]: device disconnected\n",
+				KdPrint(("RR18xx [%d,%d]: device disconnected\n",
 						 pMvSataAdapter->adapterId, channelIndex));
 					/* Flush pending commands */
 				if(pMvSataAdapter->sataChannel[channelIndex])
@@ -841,14 +840,14 @@ hptmv_handle_event(void * data, int flag)
 					mvSataRemoveChannel(pMvSataAdapter,channelIndex);
 					hptmv_free_channel(pAdapter, channelIndex);
 					pMvSataAdapter->sataChannel[channelIndex] = NULL;
-					KdPrint(("RR182x [%d,%d]: channel removed\n",
+					KdPrint(("RR18xx [%d,%d]: channel removed\n",
 						 pMvSataAdapter->adapterId, channelIndex));
 					if (pAdapter->outstandingCommands==0 && DPC_Request_Nums==0)
 						Check_Idle_Call(pAdapter);
 				}
 				else
 				{
-					KdPrint(("RR182x [%d,%d]: channel already removed!!\n",
+					KdPrint(("RR18xx [%d,%d]: channel already removed!!\n",
 							 pMvSataAdapter->adapterId, channelIndex));
 				}
 				pAdapter->sataEvents[channelIndex] = SATA_EVENT_NO_CHANGE;
@@ -894,7 +893,7 @@ hptmv_event_notify(MV_SATA_ADAPTER *pMvSataAdapter, MV_EVENT_TYPE eventType,
 				if (param1 == EVENT_CONNECT)
 				{
 					pAdapter->sataEvents[channel] = SATA_EVENT_CHANNEL_CONNECTED;
-					KdPrint(("RR182x [%d,%d]: device connected event received\n",
+					KdPrint(("RR18xx [%d,%d]: device connected event received\n",
 							 pMvSataAdapter->adapterId, channel));
 					/* Delete previous timers (if multiple drives connected in the same time */
 					pAdapter->event_timer_connect = timeout(hptmv_handle_event_connect, pAdapter, 10*hz);
@@ -902,7 +901,7 @@ hptmv_event_notify(MV_SATA_ADAPTER *pMvSataAdapter, MV_EVENT_TYPE eventType,
 				else if (param1 == EVENT_DISCONNECT)
 				{
 					pAdapter->sataEvents[channel] = SATA_EVENT_CHANNEL_DISCONNECTED;
-					KdPrint(("RR182x [%d,%d]: device disconnected event received \n",
+					KdPrint(("RR18xx [%d,%d]: device disconnected event received \n",
 							 pMvSataAdapter->adapterId, channel));
 					device_change(pAdapter, channel, FALSE);
 					/* Delete previous timers (if multiple drives disconnected in the same time */
@@ -913,7 +912,7 @@ hptmv_event_notify(MV_SATA_ADAPTER *pMvSataAdapter, MV_EVENT_TYPE eventType,
 				else
 				{
 
-					MV_ERROR("RR182x: illigal value for param1(%d) at "
+					MV_ERROR("RR18xx: illigal value for param1(%d) at "
 							 "connect/disconect event, host=%d\n", param1,
 							 pMvSataAdapter->adapterId );
 
@@ -921,11 +920,11 @@ hptmv_event_notify(MV_SATA_ADAPTER *pMvSataAdapter, MV_EVENT_TYPE eventType,
 			}
 			break;
 		case MV_EVENT_TYPE_ADAPTER_ERROR:
-			KdPrint(("RR182x: DEVICE error event received, pci cause "
+			KdPrint(("RR18xx: DEVICE error event received, pci cause "
 					  "reg=%x,  don't how to handle this\n", param1));
 			return MV_TRUE;
 		default:
-			MV_ERROR("RR182x[%d]: unknown event type (%d)\n",
+			MV_ERROR("RR18xx[%d]: unknown event type (%d)\n",
 					 pMvSataAdapter->adapterId, eventType);
 			return MV_FALSE;
 	}
@@ -939,7 +938,7 @@ hptmv_allocate_edma_queues(IAL_ADAPTER_T *pAdapter)
 			M_DEVBUF, M_NOWAIT, 0, 0xffffffff, PAGE_SIZE, 0ul);
 	if (pAdapter->requestsArrayBaseAddr == NULL)
 	{
-		MV_ERROR("RR182x[%d]: Failed to allocate memory for EDMA request"
+		MV_ERROR("RR18xx[%d]: Failed to allocate memory for EDMA request"
 				 " queues\n", pAdapter->mvSataAdapter.adapterId);
 		return -1;
 	}
@@ -955,7 +954,7 @@ hptmv_allocate_edma_queues(IAL_ADAPTER_T *pAdapter)
 	if ((pAdapter->requestsArrayBaseDmaAlignedAddr - pAdapter->requestsArrayBaseDmaAddr) != 
 		(pAdapter->requestsArrayBaseAlignedAddr - pAdapter->requestsArrayBaseAddr))
 	{
-		MV_ERROR("RR182x[%d]: Error in Request Quueues Alignment\n",
+		MV_ERROR("RR18xx[%d]: Error in Request Quueues Alignment\n",
 				 pAdapter->mvSataAdapter.adapterId);
 		contigfree(pAdapter->requestsArrayBaseAddr, REQUESTS_ARRAY_SIZE, M_DEVBUF);
 		return -1;
@@ -965,7 +964,7 @@ hptmv_allocate_edma_queues(IAL_ADAPTER_T *pAdapter)
 			M_DEVBUF, M_NOWAIT, 0, 0xffffffff, PAGE_SIZE, 0ul);
 	if (pAdapter->responsesArrayBaseAddr == NULL)
 	{
-		MV_ERROR("RR182x[%d]: Failed to allocate memory for EDMA response"
+		MV_ERROR("RR18xx[%d]: Failed to allocate memory for EDMA response"
 				 " queues\n", pAdapter->mvSataAdapter.adapterId);
 		contigfree(pAdapter->requestsArrayBaseAddr, RESPONSES_ARRAY_SIZE, M_DEVBUF);
 		return -1;
@@ -982,7 +981,7 @@ hptmv_allocate_edma_queues(IAL_ADAPTER_T *pAdapter)
 	if ((pAdapter->responsesArrayBaseDmaAlignedAddr - pAdapter->responsesArrayBaseDmaAddr) != 
 		(pAdapter->responsesArrayBaseAlignedAddr - pAdapter->responsesArrayBaseAddr))
 	{
-		MV_ERROR("RR182x[%d]: Error in Response Quueues Alignment\n",
+		MV_ERROR("RR18xx[%d]: Error in Response Quueues Alignment\n",
 				 pAdapter->mvSataAdapter.adapterId);
 		contigfree(pAdapter->requestsArrayBaseAddr, REQUESTS_ARRAY_SIZE, M_DEVBUF);
 		contigfree(pAdapter->responsesArrayBaseAddr, RESPONSES_ARRAY_SIZE, M_DEVBUF);
@@ -1065,8 +1064,8 @@ PVDevice
 GetSpareDisk(_VBUS_ARG PVDevice pArray)
 {
 	IAL_ADAPTER_T *pAdapter = (IAL_ADAPTER_T *)pArray->pVBus->OsExt;
-	ULONG capacity = LongDiv(pArray->VDeviceCapacity, pArray->u.array.bArnMember-1);
-	ULONG thiscap, maxcap = MAX_LBA_T;
+	LBA_T capacity = LongDiv(pArray->VDeviceCapacity, pArray->u.array.bArnMember-1);
+	LBA_T thiscap, maxcap = MAX_LBA_T;
 	PVDevice pVDevice, pFind = NULL;
 	int i;
 
@@ -1131,6 +1130,127 @@ void HPTLIBAPI fDeSelectMode(PDevice pDev, UCHAR NewMode)
 		KdPrint(("Failed to enable DMA, channel=%d", channelIndex));
 }
 
+int HPTLIBAPI fDeSetTCQ(PDevice pDev, int enable, int depth)
+{
+	MV_SATA_CHANNEL *pSataChannel = pDev->mv;
+	MV_SATA_ADAPTER *pSataAdapter = pSataChannel->mvSataAdapter;
+	MV_U8 channelIndex = pSataChannel->channelNumber;
+	IAL_ADAPTER_T *pAdapter = pSataAdapter->IALData;
+	MV_CHANNEL		*channelInfo = &(pAdapter->mvChannel[channelIndex]);
+	int dmaActive = pSataChannel->queueCommandsEnabled;
+	int ret = 0;
+
+	if (dmaActive) {
+		mvSataDisableChannelDma(pSataAdapter, channelIndex);
+		mvSataFlushDmaQueue(pSataAdapter,channelIndex,MV_FLUSH_TYPE_CALLBACK);
+	}
+
+	if (enable) {
+		if (pSataChannel->queuedDMA == MV_EDMA_MODE_NOT_QUEUED &&
+			(pSataChannel->identifyDevice[IDEN_SUPPORTED_COMMANDS2] & (0x2))) {
+			UCHAR depth = ((pSataChannel->identifyDevice[IDEN_QUEUE_DEPTH]) & 0x1f) + 1;
+			channelInfo->queueDepth = (depth==32)? 31 : depth;
+			mvSataConfigEdmaMode(pSataAdapter, channelIndex, MV_EDMA_MODE_QUEUED, depth);
+			ret = 1;
+		}
+	}
+	else
+	{
+		if (pSataChannel->queuedDMA != MV_EDMA_MODE_NOT_QUEUED) {
+			channelInfo->queueDepth = 2;
+			mvSataConfigEdmaMode(pSataAdapter, channelIndex, MV_EDMA_MODE_NOT_QUEUED, 0);
+			ret = 1;
+		}
+	}
+
+	if (dmaActive)
+		mvSataEnableChannelDma(pSataAdapter,channelIndex);
+	return ret;
+}
+
+int HPTLIBAPI fDeSetNCQ(PDevice pDev, int enable, int depth)
+{
+	return 0;
+}
+
+int HPTLIBAPI fDeSetWriteCache(PDevice pDev, int enable)
+{
+	MV_SATA_CHANNEL *pSataChannel = pDev->mv;
+	MV_SATA_ADAPTER *pSataAdapter = pSataChannel->mvSataAdapter;
+	MV_U8 channelIndex = pSataChannel->channelNumber;
+	IAL_ADAPTER_T *pAdapter = pSataAdapter->IALData;
+	MV_CHANNEL		*channelInfo = &(pAdapter->mvChannel[channelIndex]);
+	int dmaActive = pSataChannel->queueCommandsEnabled;
+	int ret = 0;
+
+	if (dmaActive) {
+		mvSataDisableChannelDma(pSataAdapter, channelIndex);
+		mvSataFlushDmaQueue(pSataAdapter,channelIndex,MV_FLUSH_TYPE_CALLBACK);
+	}
+
+	if ((pSataChannel->identifyDevice[82] & (0x20))) {
+		if (enable) {
+			if (mvStorageDevATASetFeatures(pSataAdapter, channelIndex,
+				MV_ATA_SET_FEATURES_ENABLE_WCACHE, 0, 0, 0, 0))
+			{
+				channelInfo->writeCacheEnabled = MV_TRUE;
+				ret = 1;
+			}
+		}
+		else {
+			if (mvStorageDevATASetFeatures(pSataAdapter, channelIndex,
+				MV_ATA_SET_FEATURES_DISABLE_WCACHE, 0, 0, 0, 0))
+			{
+				channelInfo->writeCacheEnabled = MV_FALSE;
+				ret = 1;
+			}
+		}
+	}
+
+	if (dmaActive)
+		mvSataEnableChannelDma(pSataAdapter,channelIndex);
+	return ret;
+}
+
+int HPTLIBAPI fDeSetReadAhead(PDevice pDev, int enable)
+{
+	MV_SATA_CHANNEL *pSataChannel = pDev->mv;
+	MV_SATA_ADAPTER *pSataAdapter = pSataChannel->mvSataAdapter;
+	MV_U8 channelIndex = pSataChannel->channelNumber;
+	IAL_ADAPTER_T *pAdapter = pSataAdapter->IALData;
+	MV_CHANNEL		*channelInfo = &(pAdapter->mvChannel[channelIndex]);
+	int dmaActive = pSataChannel->queueCommandsEnabled;
+	int ret = 0;
+
+	if (dmaActive) {
+		mvSataDisableChannelDma(pSataAdapter, channelIndex);
+		mvSataFlushDmaQueue(pSataAdapter,channelIndex,MV_FLUSH_TYPE_CALLBACK);
+	}
+
+	if ((pSataChannel->identifyDevice[82] & (0x40))) {
+		if (enable) {
+			if (mvStorageDevATASetFeatures(pSataAdapter, channelIndex,
+				MV_ATA_SET_FEATURES_ENABLE_RLA, 0, 0, 0, 0))
+			{
+				channelInfo->readAheadEnabled = MV_TRUE;
+				ret = 1;
+			}
+		}
+		else {
+			if (mvStorageDevATASetFeatures(pSataAdapter, channelIndex,
+				MV_ATA_SET_FEATURES_DISABLE_RLA, 0, 0, 0, 0))
+			{
+				channelInfo->readAheadEnabled = MV_FALSE;
+				ret = 1;
+			}
+		}
+	}
+
+	if (dmaActive)
+		mvSataEnableChannelDma(pSataAdapter,channelIndex);
+	return ret;
+}
+
 #ifdef SUPPORT_ARRAY
 #define IdeRegisterVDevice  fCheckArray
 #else 
@@ -1166,9 +1286,12 @@ dmamap_put(PBUS_DMAMAP p)
 
 /*Since mtx not provide the initialize when declare, so we Final init here to initialize the global mtx*/
 #if __FreeBSD_version >= 500000
+#define override_kernel_driver()
+
 static void hpt_init(void *dummy)
 {
-	mtx_init(&driver_lock, "hptlock", NULL, MTX_DEF);
+	override_kernel_driver();	
+	mtx_init(&driver_lock, "hptsleeplock", NULL, MTX_DEF);
 }
 SYSINIT(hptinit, SI_SUB_CONFIGURE, SI_ORDER_FIRST, hpt_init, NULL);
 #endif
@@ -1183,6 +1306,8 @@ init_adapter(IAL_ADAPTER_T *pAdapter)
 
 	PVDevice pVDev;
 
+	intrmask_t oldspl = lock_driver();
+
 	pAdapter->next = 0;
 
 	if(gIal_Adapter == 0){
@@ -1222,7 +1347,8 @@ init_adapter(IAL_ADAPTER_T *pAdapter)
 
 	if (hptmv_allocate_edma_queues(pAdapter))
 	{
-		MV_ERROR("RR182x: Failed to allocate memory for EDMA queues\n");
+		MV_ERROR("RR18xx: Failed to allocate memory for EDMA queues\n");
+		unlock_driver(oldspl);
 		return ENOMEM;
 	}
 
@@ -1233,13 +1359,14 @@ init_adapter(IAL_ADAPTER_T *pAdapter)
 		||
 		!(pMvSataAdapter->adapterIoBaseAddress = rman_get_virtual(pAdapter->mem_res)))
 	{
-		MV_ERROR("RR182x: Failed to remap memory space\n");
+		MV_ERROR("RR18xx: Failed to remap memory space\n");
 		hptmv_free_edma_queues(pAdapter);
+		unlock_driver(oldspl);
 		return ENXIO;
 	}
 	else
 	{
-		KdPrint(("RR182x: io base address 0x%p\n", pMvSataAdapter->adapterIoBaseAddress));
+		KdPrint(("RR18xx: io base address 0x%p\n", pMvSataAdapter->adapterIoBaseAddress));
 	}
 
 	pMvSataAdapter->adapterId = num_adapters++;
@@ -1247,7 +1374,7 @@ init_adapter(IAL_ADAPTER_T *pAdapter)
 	pMvSataAdapter->pciConfigRevisionId = pci_read_config(pAdapter->hpt_dev, PCIR_REVID, 1);
 	pMvSataAdapter->pciConfigDeviceId = pci_get_device(pAdapter->hpt_dev);
 	
-	/* init RR182x */
+	/* init RR18xx */
 	pMvSataAdapter->intCoalThre[0]= 1;
 	pMvSataAdapter->intCoalThre[1]= 1;
 	pMvSataAdapter->intTimeThre[0] = 1;
@@ -1259,11 +1386,12 @@ init_adapter(IAL_ADAPTER_T *pAdapter)
 
 	if (mvSataInitAdapter(pMvSataAdapter) == MV_FALSE)
 	{
-		MV_ERROR("RR182x[%d]: core failed to initialize the adapter\n",
+		MV_ERROR("RR18xx[%d]: core failed to initialize the adapter\n",
 				 pMvSataAdapter->adapterId);
 unregister:
 		bus_release_resource(pAdapter->hpt_dev, SYS_RES_MEMORY, rid, pAdapter->mem_res);
 		hptmv_free_edma_queues(pAdapter);
+		unlock_driver(oldspl);
 		return ENXIO;
 	}
 	pAdapter->ver_601 = pMvSataAdapter->pcbVersion;
@@ -1341,14 +1469,14 @@ unregister:
 		if (mvSataIsStorageDeviceConnected(pMvSataAdapter, channel)
 			== MV_TRUE)
 		{
-			KdPrint(("RR182x[%d]: channel %d is connected\n",
+			KdPrint(("RR18xx[%d]: channel %d is connected\n",
 					  pMvSataAdapter->adapterId, channel));
 
 			if (hptmv_init_channel(pAdapter, channel) == 0)
 			{
 				if (mvSataConfigureChannel(pMvSataAdapter, channel) == MV_FALSE)
 				{
-					MV_ERROR("RR182x[%d]: Failed to configure channel"
+					MV_ERROR("RR18xx[%d]: Failed to configure channel"
 							 " %d\n",pMvSataAdapter->adapterId, channel);
 					hptmv_free_channel(pAdapter, channel);
 				}
@@ -1356,7 +1484,7 @@ unregister:
 				{
 					if (start_channel(pAdapter, channel))
 					{
-						MV_ERROR("RR182x[%d]: Failed to start channel,"
+						MV_ERROR("RR18xx[%d]: Failed to start channel,"
 								 " channel=%d\n",pMvSataAdapter->adapterId,
 								 channel);
 						hptmv_free_channel(pAdapter, channel);
@@ -1402,10 +1530,11 @@ unregister:
 #if defined(SUPPORT_ARRAY) && defined(_RAID5N_)
 	init_raid5_memory(_VBUS_P0);
 	_vbus_(r5).enable_write_back = 1;
-	printf("RR182x: RAID5 write-back %s\n", _vbus_(r5).enable_write_back? "enabled" : "disabled");
+	printf("RR18xx: RAID5 write-back %s\n", _vbus_(r5).enable_write_back? "enabled" : "disabled");
 #endif
 
 	mvSataUnmaskAdapterInterrupt(pMvSataAdapter);
+	unlock_driver(oldspl);
 	return 0;
 }
 
@@ -1421,7 +1550,7 @@ MvSataResetChannel(MV_SATA_ADAPTER *pMvSataAdapter, MV_U8 channel)
 	/* Software reset channel */
 	if (mvStorageDevATASoftResetDevice(pMvSataAdapter, channel) == MV_FALSE)
 	{
-		MV_ERROR("RR182x [%d,%d]: failed to perform Software reset\n",
+		MV_ERROR("RR18xx [%d,%d]: failed to perform Software reset\n",
 				 pMvSataAdapter->adapterId, channel);
 		hptmv_free_channel(pAdapter, channel);
 		return -1;
@@ -1430,7 +1559,7 @@ MvSataResetChannel(MV_SATA_ADAPTER *pMvSataAdapter, MV_U8 channel)
 	/* Hardware reset channel */
 	if (mvSataChannelHardReset(pMvSataAdapter, channel)== MV_FALSE)
 	{
-		MV_ERROR("RR182x [%d,%d] Failed to Hard reser the SATA channel\n",
+		MV_ERROR("RR18xx [%d,%d] Failed to Hard reser the SATA channel\n",
 				 pMvSataAdapter->adapterId, channel);
 		hptmv_free_channel(pAdapter, channel);
 		return -1;
@@ -1438,7 +1567,7 @@ MvSataResetChannel(MV_SATA_ADAPTER *pMvSataAdapter, MV_U8 channel)
 
 	if (mvSataIsStorageDeviceConnected(pMvSataAdapter, channel) == MV_FALSE)
 	{
-		 MV_ERROR("RR182x [%d,%d] Failed to Connect Device\n",
+		 MV_ERROR("RR18xx [%d,%d] Failed to Connect Device\n",
 				 pMvSataAdapter->adapterId, channel);
 		hptmv_free_channel(pAdapter, channel);
 		return -1;
@@ -1632,7 +1761,7 @@ fDeviceSendCommand(_VBUS_ARG PCommand pCmd)
 	MV_SATA_CHANNEL *pMvSataChannel;
 	PVDevice pVDevice = pCmd->pVDevice;
 	PDevice  pDevice = &pVDevice->u.disk;
-	ULONG    Lba = pCmd->uCmd.Ide.Lba;
+	LBA_T    Lba = pCmd->uCmd.Ide.Lba;
 	USHORT   nSector = pCmd->uCmd.Ide.nSectors;
 
 	MV_QUEUE_COMMAND_RESULT result;
@@ -1640,7 +1769,7 @@ fDeviceSendCommand(_VBUS_ARG PCommand pCmd)
 	MV_UDMA_COMMAND_PARAMS  *pUdmaParams = &commandInfo.commandParams.udmaCommand;
 	MV_NONE_UDMA_COMMAND_PARAMS *pNoUdmaParams = &commandInfo.commandParams.NoneUdmaCommand;
 
-	MV_BOOLEAN is48bit = MV_FALSE;
+	MV_BOOLEAN is48bit;
 	MV_U8      channel;
 	int        i=0;
 	
@@ -1667,9 +1796,12 @@ fDeviceSendCommand(_VBUS_ARG PCommand pCmd)
 		return;
 	}
 	
-	if(Lba & 0xF0000000){
-		is48bit = MV_TRUE;
-	}
+	/*
+	 * always use 48bit LBA if drive supports it.
+	 * Some Seagate drives report error if you use a 28-bit command
+	 * to access sector 0xfffffff.
+	 */
+	is48bit = pMvSataChannel->lba48Address;
 
 	switch (pCmd->uCmd.Ide.Command)
 	{
@@ -1929,7 +2061,13 @@ hpt_attach(device_t dev)
 		return(ENXIO);
 	}
 
-	if(bus_setup_intr(pAdapter->hpt_dev, pAdapter->hpt_irq, INTR_TYPE_CAM, NULL, hpt_intr, pAdapter, &pAdapter->hpt_intr))
+#if __FreeBSD_version <700000
+	if (bus_setup_intr(pAdapter->hpt_dev, pAdapter->hpt_irq, INTR_TYPE_CAM,
+				hpt_intr, pAdapter, &pAdapter->hpt_intr))
+#else 
+	if (bus_setup_intr(pAdapter->hpt_dev, pAdapter->hpt_irq, INTR_TYPE_CAM,
+				NULL, hpt_intr, pAdapter, &pAdapter->hpt_intr))
+#endif
 	{
 		hpt_printk(("can't set up interrupt\n"));
 		free(pAdapter, M_DEVBUF);
@@ -1959,14 +2097,23 @@ hpt_attach(device_t dev)
 	/*
 	 * Construct our SIM entry
 	 */
-	if ((hpt_vsim = cam_sim_alloc(hpt_action, hpt_poll, __str(PROC_DIR_NAME),
-			pAdapter, device_get_unit(pAdapter->hpt_dev),
-			&Giant, /*untagged*/1, /*tagged*/8,  devq)) == NULL)	{
+#if __FreeBSD_version <700000
+	hpt_vsim = cam_sim_alloc(hpt_action, hpt_poll, __str(PROC_DIR_NAME),
+			pAdapter, device_get_unit(pAdapter->hpt_dev), 1, 8, devq);
+#else 
+	hpt_vsim = cam_sim_alloc(hpt_action, hpt_poll, __str(PROC_DIR_NAME),
+			pAdapter, device_get_unit(pAdapter->hpt_dev), &Giant, 1, 8, devq);
+#endif
+	if (hpt_vsim == NULL) {
 		cam_simq_free(devq);
 		return ENOMEM;
 	}
 
-	if(xpt_bus_register(hpt_vsim, dev, 0) != CAM_SUCCESS)
+#if __FreeBSD_version <700000
+	if (xpt_bus_register(hpt_vsim, 0) != CAM_SUCCESS)
+#else 
+	if (xpt_bus_register(hpt_vsim, dev, 0) != CAM_SUCCESS)
+#endif
 	{
 		cam_sim_free(hpt_vsim, /*free devq*/ TRUE);
 		hpt_vsim = NULL;
@@ -1991,14 +2138,9 @@ hpt_attach(device_t dev)
 	xpt_action((union ccb *)ccb);
 	free(ccb, M_DEVBUF);
 
-	/* Register shutdown handler, and start the work thread. */
 	if (device_get_unit(dev) == 0) {
-		pAdapter->eh = EVENTHANDLER_REGISTER(shutdown_final, 
-			hpt_shutdown, dev, SHUTDOWN_PRI_DEFAULT);
-		if (pAdapter->eh)
-			launch_worker_thread();
-		else
-			hpt_printk(("shutdown event registration failed\n"));
+		/* Start the work thread.  XXX */
+		launch_worker_thread();
 	}
 
 	return 0;
@@ -2358,7 +2500,11 @@ static void hpt_worker_thread(void)
 #if (__FreeBSD_version < 500000)
 			YIELD_THREAD;
 #else 
+#if (__FreeBSD_version > 700033)
 			pause("sched", 1);
+#else
+			tsleep((caddr_t)hpt_worker_thread, PPAUSE, "sched", 1); 
+#endif
 #endif
 			if (SIGISMEMBER(curproc->p_siglist, SIGSTOP)) {
 				/* abort rebuilding process. */
@@ -2392,15 +2538,25 @@ static void hpt_worker_thread(void)
 /*
 #ifdef DEBUG
 		if (SIGISMEMBER(curproc->p_siglist, SIGSTOP))
+#if (__FreeBSD_version > 700033)
 			pause("hptrdy", 2*hz);
+#else
+			tsleep((caddr_t)hpt_worker_thread, PPAUSE, "hptrdy", 2*hz);
+#endif
 #endif
 */
-	#if (__FreeBSD_version >= 500043)
+	#if (__FreeBSD_version >= 800002)
 		kproc_suspend_check(curproc);
+	#elif (__FreeBSD_version >= 500043)
+		kthread_suspend_check(curproc);
 	#else 
 		kproc_suspend_loop(curproc);
 	#endif
+#if (__FreeBSD_version > 700033)
 		pause("hptrdy", 2*hz);  /* wait for something to do */
+#else
+		tsleep((caddr_t)hpt_worker_thread, PPAUSE, "hptrdy", 2*hz);  /* wait for something to do */
+#endif
 	}
 }
 
@@ -2537,7 +2693,7 @@ SetInquiryData(PINQUIRYDATA inquiryData, PVDevice pVDev)
 		}
 		break;
 	default:
-		memcpy(&inquiryData->VendorId, "RR182x  ", 8);
+		memcpy(&inquiryData->VendorId, "RR18xx  ", 8);
 #ifdef SUPPORT_ARRAY
 		switch(pVDev->VDeviceType){
 		case VD_RAID_0:
@@ -2659,20 +2815,57 @@ OsSendCommand(_VBUS_ARG union ccb *ccb)
 
 		case READ_CAPACITY:
 		{		
-			UCHAR swip[4];
+			UCHAR *rbuf=csio->data_ptr;
+			unsigned int cap;
+
+			if (pVDev->VDeviceCapacity > 0xfffffffful) {
+				cap = 0xfffffffful;
+			} else {
+				cap = pVDev->VDeviceCapacity - 1;
+			}
+
+			rbuf[0] = (UCHAR)(cap>>24);
+			rbuf[1] = (UCHAR)(cap>>16);
+			rbuf[2] = (UCHAR)(cap>>8);
+			rbuf[3] = (UCHAR)cap;
 			/* Claim 512 byte blocks (big-endian). */
-			((PREAD_CAPACITY_DATA)csio->data_ptr)->BytesPerBlock = 0x20000;
-			*(ULONG*)swip = pVDev->VDeviceCapacity - 1;
-			((PREAD_CAPACITY_DATA)csio->data_ptr)->LogicalBlockAddress =
-				(swip[0] << 24) |  (swip[1] << 16) | (swip[2] << 8) | swip[3];
+			rbuf[4] = 0;
+			rbuf[5] = 0;
+			rbuf[6] = 2;
+			rbuf[7] = 0;
+			
 			ccb_h->status = CAM_REQ_CMP;
 			break;
 		}
 
+		case 0x9e: /*SERVICE_ACTION_IN*/ 
+		{
+			UCHAR *rbuf = csio->data_ptr;
+			LBA_T cap = pVDev->VDeviceCapacity - 1;
+			
+			rbuf[0] = (UCHAR)(cap>>56);
+			rbuf[1] = (UCHAR)(cap>>48);
+			rbuf[2] = (UCHAR)(cap>>40);
+			rbuf[3] = (UCHAR)(cap>>32);
+			rbuf[4] = (UCHAR)(cap>>24);
+			rbuf[5] = (UCHAR)(cap>>16);
+			rbuf[6] = (UCHAR)(cap>>8);
+			rbuf[7] = (UCHAR)cap;
+			rbuf[8] = 0;
+			rbuf[9] = 0;
+			rbuf[10] = 2;
+			rbuf[11] = 0;
+			
+			ccb_h->status = CAM_REQ_CMP;
+			break;	
+		}
+
 		case READ_6:
 		case WRITE_6:
 		case READ_10:
 		case WRITE_10:
+		case 0x88: /* READ_16 */
+		case 0x8a: /* WRITE_16 */
 		case 0x13:
 		case 0x2f:
 		{
@@ -2716,6 +2909,20 @@ OsSendCommand(_VBUS_ARG union ccb *ccb)
 					pCmd->uCmd.Ide.nSectors = (USHORT) Cdb[4];
 					break;
 
+				case 0x88: /* READ_16 */
+				case 0x8a: /* WRITE_16 */
+					pCmd->uCmd.Ide.Lba = 
+						(HPT_U64)Cdb[2] << 56 |
+						(HPT_U64)Cdb[3] << 48 |
+						(HPT_U64)Cdb[4] << 40 |
+						(HPT_U64)Cdb[5] << 32 |
+						(HPT_U64)Cdb[6] << 24 |
+						(HPT_U64)Cdb[7] << 16 |
+						(HPT_U64)Cdb[8] << 8 |
+						(HPT_U64)Cdb[9];
+					pCmd->uCmd.Ide.nSectors = (USHORT)Cdb[12] << 8 | (USHORT)Cdb[13];
+					break;
+					
 				default:
 					pCmd->uCmd.Ide.Lba = (ULONG)Cdb[5] | ((ULONG)Cdb[4] << 8) | ((ULONG)Cdb[3] << 16) | ((ULONG)Cdb[2] << 24);
 					pCmd->uCmd.Ide.nSectors = (USHORT) Cdb[8] | ((USHORT)Cdb[7]<<8);
@@ -2726,12 +2933,14 @@ OsSendCommand(_VBUS_ARG union ccb *ccb)
 			{
 				case READ_6:
 				case READ_10:
+				case 0x88: /* READ_16 */
 					pCmd->uCmd.Ide.Command = IDE_COMMAND_READ;
 					pCmd->cf_data_in = 1;
 					break;
 
 				case WRITE_6:
 				case WRITE_10:
+				case 0x8a: /* WRITE_16 */
 					pCmd->uCmd.Ide.Command = IDE_COMMAND_WRITE;
 					pCmd->cf_data_out = 1;
 					break;
diff --git a/sys/dev/hptmv/global.h b/sys/dev/hptmv/global.h
index 4e6a0d0..c6f6982 100644
--- a/sys/dev/hptmv/global.h
+++ b/sys/dev/hptmv/global.h
@@ -33,12 +33,12 @@
 #include <dev/hptmv/mvStorageDev.h>
 
 #define COMPANY      "HighPoint Technologies, Inc."
-#define COPYRIGHT    "(c) 2000-2004. HighPoint Technologies, Inc."
-#define DRIVER_NAME		"RocketRAID 182x SATA Controller driver"
-#define CONTROLLER_NAME	"RocketRAID 182x SATA Controller"
+#define COPYRIGHT    "(c) 2000-2007. HighPoint Technologies, Inc."
+#define DRIVER_NAME		"RocketRAID 18xx SATA Controller driver"
+#define CONTROLLER_NAME	"RocketRAID 18xx SATA Controller"
 #define PROC_DIR_NAME hptmv
 
-#define HPT_INTERFACE_VERSION 0x01000003
+#define HPT_INTERFACE_VERSION 0x01010000
 #define SUPPORT_48BIT_LBA
 #define SUPPORT_ARRAY
 #define SUPPORT_RAID5 1
@@ -193,6 +193,10 @@ extern void HPTLIBAPI ioctl_ReportEvent(UCHAR event, PVOID param);
 
 int HPTLIBAPI fDeReadWrite(PDevice pDev, ULONG Lba, UCHAR Cmd, void *tmpBuffer);
 void HPTLIBAPI fDeSelectMode(PDevice pDev, UCHAR NewMode);
+int HPTLIBAPI fDeSetTCQ(PDevice pDev, int enable, int depth);
+int HPTLIBAPI fDeSetNCQ(PDevice pDev, int enable, int depth);
+int HPTLIBAPI fDeSetWriteCache(PDevice pDev, int enable);
+int HPTLIBAPI fDeSetReadAhead(PDevice pDev, int enable);
 
 #include <dev/hptmv/atapi.h>
 #include <dev/hptmv/command.h>
diff --git a/sys/dev/hptmv/gui_lib.c b/sys/dev/hptmv/gui_lib.c
index 13def6b..a9feb4a 100644
--- a/sys/dev/hptmv/gui_lib.c
+++ b/sys/dev/hptmv/gui_lib.c
@@ -57,11 +57,14 @@ static int hpt_get_controller_info(int id, PCONTROLLER_INFO pInfo);
 static int hpt_get_channel_info(int id, int bus, PCHANNEL_INFO pInfo);
 static int hpt_get_logical_devices(DEVICEID * pIds, int nMaxCount);
 static int hpt_get_device_info(DEVICEID id, PLOGICAL_DEVICE_INFO pInfo);
+static int hpt_get_device_info_v2(DEVICEID id, PLOGICAL_DEVICE_INFO_V2 pInfo);
 static DEVICEID hpt_create_array(_VBUS_ARG PCREATE_ARRAY_PARAMS pParam);
+static DEVICEID hpt_create_array_v2(_VBUS_ARG PCREATE_ARRAY_PARAMS_V2 pParam);
 static int hpt_add_spare_disk(_VBUS_ARG DEVICEID idDisk);
 static int hpt_remove_spare_disk(_VBUS_ARG DEVICEID idDisk);
 static int hpt_set_array_info(_VBUS_ARG DEVICEID idArray, PALTERABLE_ARRAY_INFO pInfo);
 static int hpt_set_device_info(_VBUS_ARG DEVICEID idDisk, PALTERABLE_DEVICE_INFO pInfo);
+static int hpt_set_device_info_v2(_VBUS_ARG DEVICEID idDisk, PALTERABLE_DEVICE_INFO_V2 pInfo);
 
 int
 check_VDevice_valid(PVDevice p)
@@ -97,82 +100,65 @@ check_VDevice_valid(PVDevice p)
 }
 
 #ifdef SUPPORT_ARRAY
-static void get_array_info(PVDevice pVDevice, PLOGICAL_DEVICE_INFO pInfo)
-{
-	int	i;
-
-	pInfo->Type = LDT_ARRAY;
-	pInfo->Capacity = pVDevice->VDeviceCapacity;
-	pInfo->ParentArray = VDEV_TO_ID(pVDevice->pParent);
 
-	memcpy(pInfo->u.array.Name, pVDevice->u.array.ArrayName, MAX_ARRAY_NAME);
-
-	switch( pVDevice->VDeviceType )
+static UCHAR get_vdev_type(PVDevice pVDevice)
 	{
-		case VD_RAID_0:
-			pInfo->u.array.ArrayType = AT_RAID0;
-			break;
-		case VD_RAID_1:
-			pInfo->u.array.ArrayType = AT_RAID1;
-			break;
-		case VD_JBOD:
-			pInfo->u.array.ArrayType = AT_JBOD;
-			break;
-		case VD_RAID_5:
-			pInfo->u.array.ArrayType = AT_RAID5;
-			break;
-		default:
-			pInfo->u.array.ArrayType = AT_UNKNOWN;
+	switch (pVDevice->VDeviceType) {
+		case VD_RAID_0: return AT_RAID0;
+		case VD_RAID_1: return AT_RAID1;
+		case VD_JBOD:   return AT_JBOD;
+		case VD_RAID_5: return AT_RAID5;
+		default:        return AT_UNKNOWN;
+	}
 	}
 
-	pInfo->u.array.BlockSizeShift = pVDevice->u.array.bArBlockSizeShift;
-
-	pInfo->u.array.RebuiltSectors = pVDevice->u.array.RebuildSectors;	
+static DWORD get_array_flag(PVDevice pVDevice)
+{
+	int i;
+	DWORD f = 0;
 
 	/* The array is disabled */
 	if(!pVDevice->vf_online)	{
-		pInfo->u.array.Flags |= ARRAY_FLAG_DISABLED;
-		goto ignore_info;
+		f |= ARRAY_FLAG_DISABLED;
+		/* Ignore other info */
+		return f;
 	}
 
 	/* array need synchronizing */
 	if(pVDevice->u.array.rf_need_rebuild && !pVDevice->u.array.rf_duplicate_and_create)
-		pInfo->u.array.Flags |= ARRAY_FLAG_NEEDBUILDING;
-
-	pInfo->u.array.RebuildingProgress = ((pVDevice->u.array.RebuildSectors>>11)*1000 / 
-		(pVDevice->VDeviceCapacity>>11) * (pVDevice->u.array.bArnMember-1)) * 10;
+		f |= ARRAY_FLAG_NEEDBUILDING;
 
 	/* array is in rebuilding process */
 	if(pVDevice->u.array.rf_rebuilding)
-		pInfo->u.array.Flags |= ARRAY_FLAG_REBUILDING;
+		f |= ARRAY_FLAG_REBUILDING;
 
 	/* array is being verified */
 	if(pVDevice->u.array.rf_verifying)
-		pInfo->u.array.Flags |= ARRAY_FLAG_VERIFYING;
+		f |= ARRAY_FLAG_VERIFYING;
 
 	/* array is being initialized */
 	if(pVDevice->u.array.rf_initializing)
-		pInfo->u.array.Flags |= ARRAY_FLAG_INITIALIZING;
+		f |= ARRAY_FLAG_INITIALIZING;
 
 	/* broken but may still working */
 	if(pVDevice->u.array.rf_broken)
-		pInfo->u.array.Flags |= ARRAY_FLAG_BROKEN;
+		f |= ARRAY_FLAG_BROKEN;
 
 	/* array has a active partition */
 	if(pVDevice->vf_bootable)
-		pInfo->u.array.Flags |= ARRAY_FLAG_BOOTDISK;
+		f |= ARRAY_FLAG_BOOTDISK;
 
 	/* a newly created array */
 	if(pVDevice->u.array.rf_newly_created)
-		pInfo->u.array.Flags |= ARRAY_FLAG_NEWLY_CREATED;
+		f |= ARRAY_FLAG_NEWLY_CREATED;
 
 	/* array has boot mark set */
 	if(pVDevice->vf_bootmark)
-		pInfo->u.array.Flags |= ARRAY_FLAG_BOOTMARK;
+		f |= ARRAY_FLAG_BOOTMARK;
 
 	/* auto-rebuild should start */
 	if(pVDevice->u.array.rf_auto_rebuild)
-		pInfo->u.array.Flags |= ARRAY_FLAG_NEED_AUTOREBUILD;
+		f |= ARRAY_FLAG_NEED_AUTOREBUILD;
 
 	for(i = 0; i < pVDevice->u.array.bArnMember; i++)
 	{
@@ -183,94 +169,128 @@ static void get_array_info(PVDevice pVDevice, PLOGICAL_DEVICE_INFO pInfo)
 		/* array need synchronizing */
 		if(pMember->u.array.rf_need_rebuild && 
 		   !pMember->u.array.rf_duplicate_and_create)
-			pInfo->u.array.Flags |= ARRAY_FLAG_NEEDBUILDING;
+			f |= ARRAY_FLAG_NEEDBUILDING;
 
 		/* array is in rebuilding process */
 		if(pMember->u.array.rf_rebuilding)
-			pInfo->u.array.Flags |= ARRAY_FLAG_REBUILDING;
+			f |= ARRAY_FLAG_REBUILDING;
 		
 		/* array is being verified */	
 		if(pMember->u.array.rf_verifying)
-			pInfo->u.array.Flags |= ARRAY_FLAG_VERIFYING;
+			f |= ARRAY_FLAG_VERIFYING;
 			
 		/* array is being initialized */
 		if(pMember->u.array.rf_initializing)
-			pInfo->u.array.Flags |= ARRAY_FLAG_INITIALIZING;
+			f |= ARRAY_FLAG_INITIALIZING;
 
 		/* broken but may still working */
 		if(pMember->u.array.rf_broken)
-			pInfo->u.array.Flags |= ARRAY_FLAG_BROKEN;
+			f |= ARRAY_FLAG_BROKEN;
 
 		/* a newly created array */
 		if(pMember->u.array.rf_newly_created)
-			pInfo->u.array.Flags |= ARRAY_FLAG_NEWLY_CREATED;
+			f |= ARRAY_FLAG_NEWLY_CREATED;
 
 		/* auto-rebuild should start */
 		if(pMember->u.array.rf_auto_rebuild)
-			pInfo->u.array.Flags |= ARRAY_FLAG_NEED_AUTOREBUILD;
+			f |= ARRAY_FLAG_NEED_AUTOREBUILD;
+	}
+
+	return f;
+}
+
+static DWORD calc_rebuild_progress(PVDevice pVDevice)
+{
+	int i;
+	DWORD result = ((ULONG)(pVDevice->u.array.RebuildSectors>>11)*1000 /
+		(ULONG)(pVDevice->VDeviceCapacity>>11) * (pVDevice->u.array.bArnMember-1)) * 10;
+
+	for(i = 0; i < pVDevice->u.array.bArnMember; i++)
+	{
+		PVDevice pMember = pVDevice->u.array.pMember[i];
+		if (!pMember || !pMember->vf_online || (pMember->VDeviceType==VD_SINGLE_DISK))
+			continue;
 
 		/* for RAID1/0 case */
 		if (pMember->u.array.rf_rebuilding || 
 			pMember->u.array.rf_verifying ||
 			pMember->u.array.rf_initializing)
 		{
-			DWORD percent = ((pMember->u.array.RebuildSectors>>11)*1000 / 
-				(pMember->VDeviceCapacity>>11) * (pMember->u.array.bArnMember-1)) * 10;
-			if (pInfo->u.array.RebuildingProgress==0 || 
-				pInfo->u.array.RebuildingProgress>percent)
-				pInfo->u.array.RebuildingProgress = percent;
+			DWORD percent = ((ULONG)(pMember->u.array.RebuildSectors>>11)*1000 /
+				(ULONG)(pMember->VDeviceCapacity>>11) * (pMember->u.array.bArnMember-1)) * 10;
+			if (result==0 || result>percent)
+				result = percent;
 		}
+		}
+
+	if (result>10000) result = 10000;
+	return result;
 	}
 	
-	if (pInfo->u.array.RebuildingProgress>10000)
-		pInfo->u.array.RebuildingProgress = 10000;
+static void get_array_info(PVDevice pVDevice, PHPT_ARRAY_INFO pArrayInfo)
+{
+	int	i;
 
-ignore_info:
+	memcpy(pArrayInfo->Name, pVDevice->u.array.ArrayName, MAX_ARRAY_NAME);
+	pArrayInfo->ArrayType = get_vdev_type(pVDevice);
+	pArrayInfo->BlockSizeShift = pVDevice->u.array.bArBlockSizeShift;
+	pArrayInfo->RebuiltSectors = pVDevice->u.array.RebuildSectors;
+	pArrayInfo->Flags = get_array_flag(pVDevice);
+	pArrayInfo->RebuildingProgress = calc_rebuild_progress(pVDevice);
 
-	pInfo->u.array.nDisk = 0;
-	for(i=0; i<MAX_ARRAY_MEMBERS; i++)
-		pInfo->u.array.Members[i] = INVALID_DEVICEID;
+	pArrayInfo->nDisk = 0;
 
 	for(i = 0; i < pVDevice->u.array.bArnMember; i++)
-	{
-		if(pVDevice->u.array.pMember[i] != 0)
-		{
-			pInfo->u.array.Members[pInfo->u.array.nDisk] = VDEV_TO_ID(pVDevice->u.array.pMember[i]);
-			pInfo->u.array.nDisk++;
-		}
+		if(pVDevice->u.array.pMember[i] != NULL)
+			pArrayInfo->Members[pArrayInfo->nDisk++] = VDEV_TO_ID(pVDevice->u.array.pMember[i]);
+
+	for(i=pArrayInfo->nDisk; i<MAX_ARRAY_MEMBERS; i++)
+		pArrayInfo->Members[i] = INVALID_DEVICEID;
 	}
+
+static void get_array_info_v2(PVDevice pVDevice, PHPT_ARRAY_INFO_V2 pArrayInfo)
+{
+	int	i;
+
+	memcpy(pArrayInfo->Name, pVDevice->u.array.ArrayName, MAX_ARRAYNAME_LEN);
+	pArrayInfo->ArrayType = get_vdev_type(pVDevice);
+	pArrayInfo->BlockSizeShift = pVDevice->u.array.bArBlockSizeShift;
+	pArrayInfo->RebuiltSectors.lo32 = pVDevice->u.array.RebuildSectors;
+	pArrayInfo->RebuiltSectors.hi32 = sizeof(LBA_T)>4? (pVDevice->u.array.RebuildSectors>>32) : 0;
+	pArrayInfo->Flags = get_array_flag(pVDevice);
+	pArrayInfo->RebuildingProgress = calc_rebuild_progress(pVDevice);
+
+	pArrayInfo->nDisk = 0;
+
+	for(i = 0; i < pVDevice->u.array.bArnMember; i++)
+		if(pVDevice->u.array.pMember[i] != NULL)
+			pArrayInfo->Members[pArrayInfo->nDisk++] = VDEV_TO_ID(pVDevice->u.array.pMember[i]);
+
+	for(i=pArrayInfo->nDisk; i<MAX_ARRAY_MEMBERS_V2; i++)
+		pArrayInfo->Members[i] = INVALID_DEVICEID;
 }
 #endif
 
-static int get_disk_info(PVDevice pVDevice, PLOGICAL_DEVICE_INFO pInfo)
+static int get_disk_info(PVDevice pVDevice, PDEVICE_INFO pDiskInfo)
 {
 	MV_SATA_ADAPTER *pSataAdapter;
 	MV_SATA_CHANNEL *pSataChannel;
 	IAL_ADAPTER_T   *pAdapter;
+	MV_CHANNEL		*channelInfo;
 	char *p;
 	int i;
 
-	pInfo->Type = LDT_DEVICE;
-
-	if (pVDevice->pParent)
-		pInfo->ParentArray = VDEV_TO_ID(pVDevice->pParent);
-	else
-		pInfo->ParentArray = INVALID_DEVICEID;
-
-	/* report real capacity to be compatible with old arrays */
-	pInfo->Capacity = pVDevice->u.disk.dDeRealCapacity;
-
 	/* device location */
 	pSataChannel = pVDevice->u.disk.mv;
 	if(pSataChannel == NULL)	return -1;	
-	pInfo->u.device.TargetId = 0;
+	pDiskInfo->TargetId = 0;
 	pSataAdapter = pSataChannel->mvSataAdapter;
 	if(pSataAdapter == NULL)	return -1;
 
 	pAdapter = pSataAdapter->IALData;
 
-	pInfo->u.device.PathId = pSataChannel->channelNumber;
-	pInfo->u.device.ControllerId = (UCHAR)pSataAdapter->adapterId;
+	pDiskInfo->PathId = pSataChannel->channelNumber;
+	pDiskInfo->ControllerId = (UCHAR)pSataAdapter->adapterId;
 
 /*GUI uses DeviceModeSetting to display to users
 (1) if users select a mode, GUI/BIOS should display that mode.
@@ -278,48 +298,58 @@ static int get_disk_info(PVDevice pVDevice, PLOGICAL_DEVICE_INFO pInfo)
 (3) display real mode if case (1)&&(2) not satisfied.
 */
 	if (pVDevice->u.disk.df_user_mode_set)
-		pInfo->u.device.DeviceModeSetting = pVDevice->u.disk.bDeUserSelectMode;
-	else if ((((PIDENTIFY_DATA)pVDevice->u.disk.mv->identifyDevice)->SataCapability & 3)==2) 
-		pInfo->u.device.DeviceModeSetting = 15;
+		pDiskInfo->DeviceModeSetting = pVDevice->u.disk.bDeUserSelectMode;
+	else if (((((PIDENTIFY_DATA)pVDevice->u.disk.mv->identifyDevice)->SataCapability) & 3)==2)
+		pDiskInfo->DeviceModeSetting = 15;
 	else {
 		p = (char *)&((PIDENTIFY_DATA)pVDevice->u.disk.mv->identifyDevice)->ModelNumber;
-		if (*(WORD*)p==0x5354 /*'ST'*/ &&
-			(*(WORD*)(p+8)==0x4153/*'AS'*/ || (p[8]=='A' && p[11]=='S')))
-			pInfo->u.device.DeviceModeSetting = 15;
+		if (*(WORD*)p==(0x5354) /*'ST'*/ &&
+			(*(WORD*)(p+8)==(0x4153)/*'AS'*/ || (p[8]=='A' && p[11]=='S')))
+			pDiskInfo->DeviceModeSetting = 15;
 		else
-			pInfo->u.device.DeviceModeSetting = pVDevice->u.disk.bDeModeSetting;
+			pDiskInfo->DeviceModeSetting = pVDevice->u.disk.bDeModeSetting;
 	}
 		
-	pInfo->u.device.UsableMode = pVDevice->u.disk.bDeUsable_Mode;
+	pDiskInfo->UsableMode = pVDevice->u.disk.bDeUsable_Mode;
 
-	pInfo->u.device.DeviceType = PDT_HARDDISK;
+	pDiskInfo->DeviceType = PDT_HARDDISK;
 
-	pInfo->u.device.Flags = 0x0;
+	pDiskInfo->Flags = 0x0;
 
 	/* device is disabled */
 	if(!pVDevice->u.disk.df_on_line)
-		pInfo->u.device.Flags |= DEVICE_FLAG_DISABLED;
+		pDiskInfo->Flags |= DEVICE_FLAG_DISABLED;
 
 	/* disk has a active partition */
 	if(pVDevice->vf_bootable)
-		pInfo->u.device.Flags |= DEVICE_FLAG_BOOTDISK;
+		pDiskInfo->Flags |= DEVICE_FLAG_BOOTDISK;
 
 	/* disk has boot mark set */
 	if(pVDevice->vf_bootmark)
-		pInfo->u.device.Flags |= DEVICE_FLAG_BOOTMARK;
+		pDiskInfo->Flags |= DEVICE_FLAG_BOOTMARK;
 
-	pInfo->u.device.Flags |= DEVICE_FLAG_SATA;
+	pDiskInfo->Flags |= DEVICE_FLAG_SATA;
 
 	/* is a spare disk */
 	if(pVDevice->VDeviceType == VD_SPARE)
-		pInfo->u.device.Flags |= DEVICE_FLAG_IS_SPARE;
+		pDiskInfo->Flags |= DEVICE_FLAG_IS_SPARE;
 
-	memcpy(&(pInfo->u.device.IdentifyData), (pSataChannel->identifyDevice), sizeof(IDENTIFY_DATA2));
-	p = (char *)&pInfo->u.device.IdentifyData.ModelNumber;
+	memcpy(&(pDiskInfo->IdentifyData), (pSataChannel->identifyDevice), sizeof(IDENTIFY_DATA2));
+	p = (char *)&pDiskInfo->IdentifyData.ModelNumber;
 	for (i = 0; i < 20; i++)
 		((WORD*)p)[i] = shortswap(pSataChannel->identifyDevice[IDEN_MODEL_OFFSET+i]);
 	p[39] = '\0';
 
+	channelInfo = &pAdapter->mvChannel[pSataChannel->channelNumber];
+	pDiskInfo->ReadAheadSupported = channelInfo->readAheadSupported;
+	pDiskInfo->ReadAheadEnabled = channelInfo->readAheadEnabled;
+	pDiskInfo->WriteCacheSupported = channelInfo->writeCacheSupported;
+	pDiskInfo->WriteCacheEnabled = channelInfo->writeCacheEnabled;
+	pDiskInfo->TCQSupported = (pSataChannel->identifyDevice[IDEN_SUPPORTED_COMMANDS2] & (0x2))!=0;
+	pDiskInfo->TCQEnabled = pSataChannel->queuedDMA==MV_EDMA_MODE_QUEUED;
+	pDiskInfo->NCQSupported = MV_SATA_GEN_2(pSataAdapter) &&
+		(pSataChannel->identifyDevice[IDEN_SATA_CAPABILITIES] & (0x0100));
+	pDiskInfo->NCQEnabled = pSataChannel->queuedDMA==MV_EDMA_MODE_NATIVE_QUEUING;
 	return 0;
 }
 
@@ -403,7 +433,7 @@ int hpt_get_controller_info(int id, PCONTROLLER_INFO pInfo)
 #else 
 #define _set_product_id(x) strcpy(pInfo->szProductID, x)
 #endif
-			_set_product_id("RocketRAID 182x SATA Controller");			
+			_set_product_id("RocketRAID 18xx SATA Controller");			
 			pInfo->NumBuses = 8;
 			pInfo->ChipFlags |= CHIP_SUPPORT_ULTRA_133|CHIP_SUPPORT_ULTRA_150;
 			return 0;
@@ -487,24 +517,60 @@ int hpt_get_device_info(DEVICEID id, PLOGICAL_DEVICE_INFO pInfo)
 		return -1;
 
 #ifdef SUPPORT_ARRAY
-	if (mIsArray(pVDevice))
-		get_array_info(pVDevice, pInfo);
-	else
+	if (mIsArray(pVDevice)) {
+		pInfo->Type = LDT_ARRAY;
+		pInfo->Capacity = pVDevice->VDeviceCapacity;
+		pInfo->ParentArray = VDEV_TO_ID(pVDevice->pParent);
+		get_array_info(pVDevice, &pInfo->u.array);
+		return 0;
+	}
 #endif
-	return	get_disk_info(pVDevice, pInfo);
 
+	pInfo->Type = LDT_DEVICE;
+	pInfo->ParentArray = pVDevice->pParent? VDEV_TO_ID(pVDevice->pParent) : INVALID_DEVICEID;
+	/* report real capacity to be compatible with old arrays */
+	pInfo->Capacity = pVDevice->u.disk.dDeRealCapacity;
+	return get_disk_info(pVDevice, &pInfo->u.device);
+}
+
+int hpt_get_device_info_v2(DEVICEID id, PLOGICAL_DEVICE_INFO_V2 pInfo)
+{
+	PVDevice pVDevice = ID_TO_VDEV(id);
+
+	if((id == 0) || check_VDevice_valid(pVDevice))
+		return -1;
+
+#ifdef SUPPORT_ARRAY
+	if (mIsArray(pVDevice)) {
+		pInfo->Type = LDT_ARRAY;
+		pInfo->Capacity.lo32 = pVDevice->VDeviceCapacity;
+		pInfo->Capacity.hi32 = sizeof(LBA_T)>4? (pVDevice->VDeviceCapacity>>32) : 0;
+		pInfo->ParentArray = VDEV_TO_ID(pVDevice->pParent);
+		get_array_info_v2(pVDevice, &pInfo->u.array);
 	return 0;
 }
+#endif
+
+	pInfo->Type = LDT_DEVICE;
+	pInfo->ParentArray = pVDevice->pParent? VDEV_TO_ID(pVDevice->pParent) : INVALID_DEVICEID;
+	/* report real capacity to be compatible with old arrays */
+	pInfo->Capacity.lo32 = pVDevice->u.disk.dDeRealCapacity;
+	pInfo->Capacity.hi32 = 0;
+	return get_disk_info(pVDevice, &pInfo->u.device);
+}
 
 #ifdef SUPPORT_ARRAY
-DEVICEID hpt_create_array(_VBUS_ARG PCREATE_ARRAY_PARAMS pParam)
+DEVICEID hpt_create_array_v2(_VBUS_ARG PCREATE_ARRAY_PARAMS_V2 pParam)
 {
 	ULONG Stamp = GetStamp();
 	int	i,j;
-	ULONG  capacity = MAX_LBA_T;
+	LBA_T  capacity = MAX_LBA_T;
 	PVDevice pArray,pChild;
 	int		Loca = -1;
 
+	if (pParam->nDisk > MAX_MEMBERS)
+		return INVALID_DEVICEID;
+/* check in verify_vd
 	for(i = 0; i < pParam->nDisk; i++)
 	{
 		PVDevice pVDev = ID_TO_VDEV(pParam->Members[i]);
@@ -516,6 +582,8 @@ DEVICEID hpt_create_array(_VBUS_ARG PCREATE_ARRAY_PARAMS pParam)
 		else if (_vbus_p != pVDev->u.disk.pVBus)
 			return INVALID_DEVICEID;
 	}
+*/
+	_vbus_p = (ID_TO_VDEV(pParam->Members[0]))->u.disk.pVBus;
 	if (!_vbus_p) return INVALID_DEVICEID;
 
 	mArGetArrayTable(pArray);
@@ -742,6 +810,22 @@ error:
 	return INVALID_DEVICEID;
 }
 
+DEVICEID hpt_create_array(_VBUS_ARG PCREATE_ARRAY_PARAMS pParam)
+{
+	CREATE_ARRAY_PARAMS_V2 param2;
+	param2.ArrayType = pParam->ArrayType;
+	param2.nDisk = pParam->nDisk;
+	param2.BlockSizeShift = pParam->BlockSizeShift;
+	param2.CreateFlags = pParam->CreateFlags;
+	param2.CreateTime = pParam->CreateTime;
+	memcpy(param2.ArrayName, pParam->ArrayName, sizeof(param2.ArrayName));
+	memcpy(param2.Description, pParam->Description, sizeof(param2.Description));
+	memcpy(param2.CreateManager, pParam->CreateManager, sizeof(param2.CreateManager));
+	param2.Capacity.lo32 = param2.Capacity.hi32 = 0;
+	memcpy(param2.Members, pParam->Members, sizeof(pParam->Members));
+	return hpt_create_array_v2(_VBUS_P &param2);
+}
+
 #ifdef SUPPORT_OLD_ARRAY
 /* this is only for old RAID 0/1 */
 int old_add_disk_to_raid01(_VBUS_ARG DEVICEID idArray, DEVICEID idDisk)
@@ -822,7 +906,7 @@ int hpt_add_disk_to_array(_VBUS_ARG DEVICEID idArray, DEVICEID idDisk)
 {
 	int	i;
 
-	ULONG	Capacity;
+	LBA_T Capacity;
 	PVDevice pArray = ID_TO_VDEV(idArray);
 	PVDevice pDisk	= ID_TO_VDEV(idDisk);
 
@@ -859,7 +943,7 @@ int hpt_add_disk_to_array(_VBUS_ARG DEVICEID idArray, DEVICEID idDisk)
 	if (pArray->pVBus!=_vbus_p) { HPT_ASSERT(0); return -1;}
 
 	for(i = 0; i < pArray->u.array.bArnMember; i++)
-		if((pArray->u.array.pMember[i] == NULL) || !pArray->u.array.pMember[i]->vf_online)
+		if((pArray->u.array.pMember[i] == 0) || !pArray->u.array.pMember[i]->vf_online)
 		{
 			if(pArray->u.array.pMember[i] != NULL)
 				pArray->u.array.pMember[i]->pParent = NULL;
@@ -961,124 +1045,76 @@ int hpt_set_array_info(_VBUS_ARG DEVICEID idArray, PALTERABLE_ARRAY_INFO pInfo)
 	return 0;
 }
 
-int hpt_set_device_info(_VBUS_ARG DEVICEID idDisk, PALTERABLE_DEVICE_INFO pInfo)
+static int hpt_set_device_info(_VBUS_ARG DEVICEID idDisk, PALTERABLE_DEVICE_INFO pInfo)
 {
 	PVDevice pVDevice = ID_TO_VDEV(idDisk);
 
-	/* stop buzzer. */
-	if(idDisk == 0) {
-#ifndef FOR_DEMO
-		IAL_ADAPTER_T *pAdapter;
-		for (pAdapter=gIal_Adapter; pAdapter; pAdapter=pAdapter->next) {
-			if (pAdapter->beeping) {
-				pAdapter->beeping = 0;
-				BeepOff(pAdapter->mvSataAdapter.adapterIoBaseAddress);
-			}
-		}
-#endif
+	if(idDisk == 0 || check_VDevice_valid(pVDevice)) return -1;
+	if (mIsArray(pVDevice))
+		return -1;
+
+	if (pVDevice->u.disk.pVBus!=_vbus_p) return -1;
+
+	/* TODO */
 		return 0;
 	}
 	
-	if (check_VDevice_valid(pVDevice)) return -1;
+static int hpt_set_device_info_v2(_VBUS_ARG DEVICEID idDisk, PALTERABLE_DEVICE_INFO_V2 pInfo)
+{
+	PVDevice pVDevice = ID_TO_VDEV(idDisk);
+	int sync = 0;
+
+	if(idDisk==0 || check_VDevice_valid(pVDevice)) return -1;
 	if (mIsArray(pVDevice))
 		return -1;
 
 	if (pVDevice->u.disk.pVBus!=_vbus_p) return -1;
 
-/*	if (pInfo->ValidFields & ADIF_MODE) {
+	if (pInfo->ValidFields & ADIF_MODE) {
 		pVDevice->u.disk.bDeModeSetting = pInfo->DeviceModeSetting;
 		pVDevice->u.disk.bDeUserSelectMode = pInfo->DeviceModeSetting;
 		pVDevice->u.disk.df_user_mode_set = 1;
 		fDeSelectMode((PDevice)&(pVDevice->u.disk), (UCHAR)pInfo->DeviceModeSetting);
-		SyncArrayInfo(pVDevice);
-	}*/
-	return 0;
+		sync = 1;
 }
-#endif
-
-#ifdef SUPPORT_HPT601
-int hpt_get_601_info(DEVICEID idDisk, PHPT601_INFO pInfo)
-{
-	PVDevice pVDevice = ID_TO_VDEV(idDisk);
-	PChannel          pChan = pVDevice->u.disk.pChannel;
-	PIDE_REGISTERS_1  IoPort = pChan->BaseIoAddress1;
-
-	if(!pVDevice->u.disk.df_with_601) return -1;
-
-	mSelectUnit(IoPort, pVDevice->u.disk.bDeUnitId);
-	pChan->pChipInstance->ftbl.pfnWaitOnBusy(pChan, pVDevice->u.disk.bDeUnitId, 1);
-	
-	BeginAccess601(IoPort);
-
-	mSetBlockCount(IoPort, 0);
-	pInfo->DeviceId = InWord(&IoPort->Data);
-
-	mSetBlockCount(IoPort, 0x14);
-	pInfo->Temperature = InWord(&IoPort->Data);
-
-	mSetBlockCount(IoPort, 0xA);
-	pInfo->FanStatus = InWord(&IoPort->Data);
 
-	mSetBlockCount(IoPort, 7);
-	pInfo->BeeperControl = InWord(&IoPort->Data);
-
-	mSetBlockCount(IoPort, 3);
-	pInfo->LED1Control = InWord(&IoPort->Data);
-
-	mSetBlockCount(IoPort, 5);
-	pInfo->LED2Control = InWord(&IoPort->Data);
-
-	mSetBlockCount(IoPort, 0x18);
-	pInfo->PowerStatus = InWord(&IoPort->Data);
-		
-	EndAccess601(IoPort);
-	pInfo->ValidFields = 0x7F;
-	/*DEVICEID|TEMPERATURE|FANSTATUS|BEEPERCONTROL|LED1CONTROL|LED2CONTROL|POWERSTATUS*/
-	return 0;
+	if (pInfo->ValidFields & ADIF_TCQ) {
+		if (fDeSetTCQ(&pVDevice->u.disk, pInfo->TCQEnabled, 0)) {
+			pVDevice->u.disk.df_tcq_set = 1;
+			pVDevice->u.disk.df_tcq = pInfo->TCQEnabled!=0;
+			sync = 1;
 }
-
-int hpt_set_601_info(DEVICEID idDisk, PHPT601_INFO pInfo)
-{
-	PVDevice pVDevice = ID_TO_VDEV(idDisk);
-	PChannel          pChan = pVDevice->u.disk.pChannel;
-	PIDE_REGISTERS_1  IoPort = pChan->BaseIoAddress1;
-
-	if(!pVDevice->u.disk.df_with_601) return -1;
-
-	mSelectUnit(IoPort, pVDevice->u.disk.bDeUnitId);
-	pChan->pChipInstance->ftbl.pfnWaitOnBusy(pChan, pVDevice->u.disk.bDeUnitId, 1);
-	
-	BeginAccess601(IoPort);
-
-	if (pInfo->ValidFields & HPT601_INFO_TEMPERATURE) {
-		mSetBlockCount(IoPort, 1);
-		OutWord(&IoPort->Data, pInfo->Temperature);
 	}
 
-	if (pInfo->ValidFields & HPT601_INFO_FANSTATUS) {
-		mSetBlockCount(IoPort, 0xA);
-		OutWord(&IoPort->Data, pInfo->FanStatus);
+	if (pInfo->ValidFields & ADIF_NCQ) {
+		if (fDeSetNCQ(&pVDevice->u.disk, pInfo->NCQEnabled, 0)) {
+			pVDevice->u.disk.df_ncq_set = 1;
+			pVDevice->u.disk.df_ncq = pInfo->NCQEnabled!=0;
+			sync = 1;
 	}
-
-	if (pInfo->ValidFields & HPT601_INFO_BEEPERCONTROL) {
-		mSetBlockCount(IoPort, 7);
-		OutWord(&IoPort->Data, pInfo->BeeperControl);
 	}
 
-	if (pInfo->ValidFields & HPT601_INFO_LED1CONTROL) {
-		mSetBlockCount(IoPort, 3);
-		OutWord(&IoPort->Data, pInfo->LED1Control);
+	if (pInfo->ValidFields & ADIF_WRITE_CACHE) {
+		if (fDeSetWriteCache(&pVDevice->u.disk, pInfo->WriteCacheEnabled)) {
+			pVDevice->u.disk.df_write_cache_set = 1;
+			pVDevice->u.disk.df_write_cache = pInfo->WriteCacheEnabled!=0;
+			sync = 1;
 	}
-
-	if (pInfo->ValidFields & HPT601_INFO_LED2CONTROL) {
-		mSetBlockCount(IoPort, 5);
-		OutWord(&IoPort->Data, pInfo->LED2Control);
 	}
 		
-	EndAccess601(IoPort);
+	if (pInfo->ValidFields & ADIF_READ_AHEAD) {
+		if (fDeSetReadAhead(&pVDevice->u.disk, pInfo->ReadAheadEnabled)) {
+			pVDevice->u.disk.df_read_ahead_set = 1;
+			pVDevice->u.disk.df_read_ahead = pInfo->ReadAheadEnabled!=0;
+			sync = 1;
+		}
+	}
 
+	if (sync)
+		SyncArrayInfo(pVDevice);
 	return 0;
 }
+
 #endif
 
 /* hpt_default_ioctl()
@@ -1175,21 +1211,46 @@ int hpt_default_ioctl(_VBUS_ARG
 		}
 		break;
 
-#ifdef SUPPORT_ARRAY
-	case HPT_IOCTL_CREATE_ARRAY:
+	case HPT_IOCTL_GET_DEVICE_INFO_V2:
 		{
-			CREATE_ARRAY_PARAMS *pParam;
 			DEVICEID id;
+			PLOGICAL_DEVICE_INFO_V2 pInfo;
+
+			if (nInBufferSize!=sizeof(DEVICEID)) return -1;
+			if (nOutBufferSize!=sizeof(LOGICAL_DEVICE_INFO_V2)) return -1;
 
+			id = *(DWORD *)lpInBuffer;
+			if (id == INVALID_DEVICEID)	return -1;
+
+			pInfo = (PLOGICAL_DEVICE_INFO_V2)lpOutBuffer;
+			memset(pInfo, 0, sizeof(LOGICAL_DEVICE_INFO_V2));
+
+			if (hpt_get_device_info_v2(id, pInfo)!=0)
+				return -1;
+		}
+		break;
+
+#ifdef SUPPORT_ARRAY
+	case HPT_IOCTL_CREATE_ARRAY:
+		{
 			if (nInBufferSize!=sizeof(CREATE_ARRAY_PARAMS)) return -1;
 			if (nOutBufferSize!=sizeof(DEVICEID)) return -1;
 
-			pParam = (PCREATE_ARRAY_PARAMS)lpInBuffer;
+			*(DEVICEID *)lpOutBuffer = hpt_create_array(_VBUS_P (PCREATE_ARRAY_PARAMS)lpInBuffer);
 
-			id = hpt_create_array(_VBUS_P pParam);
-			*(DEVICEID *)lpOutBuffer = id;
+			if(*(DEVICEID *)lpOutBuffer == INVALID_DEVICEID)
+				return -1;
+		}
+		break;
+
+	case HPT_IOCTL_CREATE_ARRAY_V2:
+		{
+			if (nInBufferSize!=sizeof(CREATE_ARRAY_PARAMS_V2)) return -1;
+			if (nOutBufferSize!=sizeof(DEVICEID)) return -1;
+
+			*(DEVICEID *)lpOutBuffer = hpt_create_array_v2(_VBUS_P (PCREATE_ARRAY_PARAMS_V2)lpInBuffer);
 
-			if(id == (DEVICEID)INVALID_DEVICEID)
+			if (*(DEVICEID *)lpOutBuffer == INVALID_DEVICEID)
 				return -1;
 		}
 		break;
@@ -1225,6 +1286,21 @@ int hpt_default_ioctl(_VBUS_ARG
 		}
 		break;
 
+	case HPT_IOCTL_SET_DEVICE_INFO_V2:
+		{
+			DEVICEID idDisk;
+			PALTERABLE_DEVICE_INFO_V2 pInfo;
+
+			if (nInBufferSize < sizeof(HPT_SET_DEVICE_INFO_V2)) return -1;
+			if (nOutBufferSize!=0) return -1;
+
+			idDisk = ((PHPT_SET_DEVICE_INFO_V2)lpInBuffer)->idDisk;
+			pInfo = &((PHPT_SET_DEVICE_INFO_V2)lpInBuffer)->Info;
+			if(hpt_set_device_info_v2(_VBUS_P idDisk, pInfo) != 0)
+				return -1;
+		}
+		break;
+
 	case HPT_IOCTL_SET_BOOT_MARK:
 		{
 			DEVICEID id;
@@ -1238,7 +1314,7 @@ int hpt_default_ioctl(_VBUS_ARG
 			while(pAdapter != 0)
 			{
 				pVBus = &pAdapter->VBus;
-				for(i = 0; i < MAX_ARRAY_PER_VBUS; i++)
+				for(i = 0; i < MAX_VDEVICE_PER_VBUS; i++)
 				{
 					if(!(pTop = pVBus->pVDevice[i])) continue;
 					if (pTop->pVBus!=_vbus_p) return -1;
@@ -1256,16 +1332,7 @@ int hpt_default_ioctl(_VBUS_ARG
 			}
 		}
 		break;
-#endif
-	case HPT_IOCTL_RESCAN_DEVICES:
-		{
-			if (nInBufferSize!=0) return -1;
-			if (nOutBufferSize!=0) return -1;
-			fRescanAllDevice(_VBUS_P0);
-		}
-		break;
 
-#ifdef SUPPORT_ARRAY
 	case HPT_IOCTL_ADD_SPARE_DISK:
 		{
 			DEVICEID id;
@@ -1319,41 +1386,57 @@ int hpt_default_ioctl(_VBUS_ARG
 		}
 		break;
 
-#ifdef SUPPORT_HPT601	
-	case HPT_IOCTL_GET_601_INFO:
+	case HPT_IOCTL_GET_CONTROLLER_VENID:
 		{
-			DEVICEID id;
-			PHPT601_INFO pInfo;
-
-			if (nInBufferSize!=sizeof(DEVICEID)) return -1;
-			if (nOutBufferSize!=sizeof(HPT601_INFO)) return -1;
+			DWORD id = ((DWORD*)lpInBuffer)[0];
+			IAL_ADAPTER_T *pAdapTemp;
+			int iControllerCount = 0;
 
-			id = *(DWORD *)lpInBuffer;
-			if (id == INVALID_DEVICEID)	return -1;
-
-			pInfo = (PHPT601_INFO)lpOutBuffer;
-			memset(pInfo, 0, sizeof(HPT601_INFO));
-
-			if (hpt_get_601_info(id, pInfo)!=0)
+			for (pAdapTemp = gIal_Adapter; pAdapTemp; pAdapTemp = pAdapTemp->next)
+				if (iControllerCount++==id)
+					break;
+			
+			if (!pAdapTemp)
 				return -1;
+				
+			if (nOutBufferSize < 4)
+				return -1;
+				
+			*(DWORD*)lpOutBuffer = ((DWORD)pAdapTemp->mvSataAdapter.pciConfigDeviceId << 16) | 0x11AB;
+			return 0;
 		}
-		break;
 
-	case HPT_IOCTL_SET_601_INFO:
+	case HPT_IOCTL_EPROM_IO:
 		{
-			DEVICEID id;
-			PHPT601_INFO pInfo;
+			DWORD id           = ((DWORD*)lpInBuffer)[0];
+			DWORD offset	   = ((DWORD*)lpInBuffer)[1];
+			DWORD direction    = ((DWORD*)lpInBuffer)[2];
+			DWORD length	   = ((DWORD*)lpInBuffer)[3];
+			IAL_ADAPTER_T *pAdapTemp;
+			int iControllerCount = 0;
+
+			for (pAdapTemp = gIal_Adapter; pAdapTemp; pAdapTemp = pAdapTemp->next)
+				if (iControllerCount++==id)
+					break;
+			
+			if (!pAdapTemp)
+				return -1;
+				
+			if (nInBufferSize < sizeof(DWORD) * 4 + (direction? length : 0) ||
+				nOutBufferSize < (direction? 0 : length))
+				return -1;
 
-			if (nInBufferSize!=sizeof(HPT_SET_601_INFO)) return -1;
-			if (nOutBufferSize!=0) return -1;
+			if (direction == 0) /* read */
+				sx508x_flash_access(&pAdapTemp->mvSataAdapter,
+					offset, lpOutBuffer, length, 1);
+			else
+				sx508x_flash_access(&pAdapTemp->mvSataAdapter,
+					offset, (char *)lpInBuffer + 16, length, 0);
 
-			id = ((PHPT_SET_601_INFO)lpInBuffer)->idDisk;
-			pInfo = &((PHPT_SET_601_INFO)lpInBuffer)->Info;
-			if(hpt_set_601_info(id, pInfo) != 0)
-				return -1;
+			return 0;
 		}
 		break;
-#endif
+
 	default:
 		return -1;
 	}
diff --git a/sys/dev/hptmv/hptintf.h b/sys/dev/hptmv/hptintf.h
index 34477a2..09d638f 100644
--- a/sys/dev/hptmv/hptintf.h
+++ b/sys/dev/hptmv/hptintf.h
@@ -162,7 +162,7 @@ typedef DWORD DEVICEID;
  * It would be better if ioctl code are the same on different platforms,
  * but we must not conflict with system defined ioctl code.
  ************************************************************************/
-#if defined(LINUX) || defined(__FreeBSD_version) || defined(_MACOSX_)
+#if defined(LINUX) || defined(__FreeBSD_version)
 #define HPT_CTL_CODE(x) (x+0xFF00)
 #elif defined(_MS_WIN32_) || defined(WIN32)
 
@@ -221,6 +221,11 @@ typedef DWORD DEVICEID;
 #define HPT_IOCTL_DEVICE_IO_V2              HPT_CTL_CODE(39)
 #define HPT_IOCTL_DEVICE_IO_EX_V2           HPT_CTL_CODE(40)
 
+#define HPT_IOCTL_I2C_TRANSACTION           HPT_CTL_CODE(48)
+#define HPT_IOCTL_GET_PARAMETER_LIST        HPT_CTL_CODE(49)
+#define HPT_IOCTL_GET_PARAMETER             HPT_CTL_CODE(50)
+#define HPT_IOCTL_SET_PARAMETER             HPT_CTL_CODE(51)
+
 /* Windows only */
 #define HPT_IOCTL_GET_CONTROLLER_IDS        HPT_CTL_CODE(100)
 #define HPT_IOCTL_GET_DCB                   HPT_CTL_CODE(101)
@@ -368,8 +373,13 @@ typedef struct _HPT_ARRAY_INFO {
 
 #if HPT_INTERFACE_VERSION>=0x01010000
 typedef struct _LBA64 {
+#ifdef __BIG_ENDIAN_BITFIELD
+	DWORD hi32;
+	DWORD lo32;
+#else 
 	DWORD lo32;
 	DWORD hi32;
+#endif
 }
 LBA64;
 typedef struct _HPT_ARRAY_INFO_V2 {
@@ -687,16 +697,15 @@ typedef struct _IDE_PASS_THROUGH_HEADER {
 	DEVICEID idDisk;           /* disk ID */
 	BYTE     bFeaturesReg;     /* feature register */
 	BYTE     bSectorCountReg;  /* IDE sector count register. */
-	BYTE     bSectorNumberReg; /* IDE sector number register. */
-	BYTE     bCylLowReg;       /* IDE low order cylinder value. */
-	BYTE     bCylHighReg;      /* IDE high order cylinder value. */
+	BYTE     bLbaLowReg; /* IDE sector number register. */
+	BYTE     bLbaMidReg;       /* IDE low order cylinder value. */
+	BYTE     bLbaHighReg;      /* IDE high order cylinder value. */
 	BYTE     bDriveHeadReg;    /* IDE drive/head register. */
 	BYTE     bCommandReg;      /* Actual IDE command. Checked for validity by driver. */
-	BYTE     reserve1;
-	DWORD    DataSize;         /* data size in bytes, if the command has data transfer */
-#ifdef _MSC_VER
-	BYTE     DataBuffer[0];    /* data buffer */
-#endif
+	BYTE     nSectors;		/* data sze in sectors, if the command has data transfer */
+	BYTE     protocol;            /* IO_COMMAND_(READ,WRITE) or zero for non-DATA */
+	BYTE     reserve[3];
+#define IDE_PASS_THROUGH_buffer(p) ((unsigned char *)(p) + sizeof(IDE_PASS_THROUGH_HEADER))	
 }
 IDE_PASS_THROUGH_HEADER, *PIDE_PASS_THROUGH_HEADER;
 
@@ -729,7 +738,8 @@ DEVICE_IO_EX_PARAMS, *PDEVICE_IO_EX_PARAMS;
 /*
  * ioctl structure
  */
-#define HPT_IOCTL_MAGIC 0x1A2B3C4D
+#define HPT_IOCTL_MAGIC32 0x1A2B3C4D
+#define HPT_IOCTL_MAGIC   0xA1B2C3D4
 
 typedef struct _HPT_IOCTL_PARAM {
 	DWORD   Magic;                 /* used to check if it's a valid ioctl packet */
@@ -742,7 +752,7 @@ typedef struct _HPT_IOCTL_PARAM {
 }
 HPT_IOCTL_PARAM, *PHPT_IOCTL_PARAM;
 
-/* for 64-bit system */
+/* for 32-bit app running on 64-bit system */
 typedef struct _HPT_IOCTL_PARAM32 {
 	DWORD   Magic;
 	DWORD   dwIoControlCode;
@@ -1206,6 +1216,18 @@ int hpt_query_remove(DWORD ndev, DEVICEID *pIds);
 int hpt_remove_devices(DWORD ndev, DEVICEID *pIds);
 /*-------------------------------------------------------------------------- */
 
+/* hpt_ide_pass_through
+ *  directly access controller's command and control registers.
+ *  Can only call it on physical devices.
+ * Version compatibility: v1.0.0.3 or later
+ * Parameters:
+ *   p - IDE_PASS_THROUGH header pointer
+ * Returns:
+ *   0  Success
+ */
+int hpt_ide_pass_through(PIDE_PASS_THROUGH_HEADER p);
+/*-------------------------------------------------------------------------- */
+
 #endif
 
 #pragma pack()
diff --git a/sys/dev/hptmv/hptproc.c b/sys/dev/hptmv/hptproc.c
index b38ef38..b93ab22 100644
--- a/sys/dev/hptmv/hptproc.c
+++ b/sys/dev/hptmv/hptproc.c
@@ -278,7 +278,8 @@ hpt_set_info(int length)
 		}
 #ifdef SUPPORT_IOCTL	
 		piop = (PHPT_IOCTL_PARAM)buffer;
-		if (piop->Magic == HPT_IOCTL_MAGIC) 	{
+		if (piop->Magic == HPT_IOCTL_MAGIC || 
+			piop->Magic == HPT_IOCTL_MAGIC32) 	{
 			KdPrintE(("ioctl=%d in=%p len=%d out=%p len=%d\n", 
 				piop->dwIoControlCode,
         			piop->lpInBuffer,
@@ -450,14 +451,14 @@ hpt_copy_array_info(HPT_GET_INFO *pinfo, int nld, PVDevice pArray)
 			
 			if (pTmpArray->u.array.rf_rebuilding) {
 #ifdef DEBUG
-				sprintf(buf, "Rebuilding %dMB", (pTmpArray->u.array.RebuildSectors>>11));
+				sprintf(buf, "Rebuilding %lldMB", (pTmpArray->u.array.RebuildSectors>>11));
 #else 
-				sprintf(buf, "Rebuilding %d%%", (pTmpArray->u.array.RebuildSectors>>11)*100/((pTmpArray->VDeviceCapacity/(pTmpArray->u.array.bArnMember-1))>>11));
+				sprintf(buf, "Rebuilding %d%%", (UINT)((pTmpArray->u.array.RebuildSectors>>11)*100/((pTmpArray->VDeviceCapacity/(pTmpArray->u.array.bArnMember-1))>>11)));
 #endif
 				sStatus = buf;
 			}
 			else if (pTmpArray->u.array.rf_verifying) {
-				sprintf(buf, "Verifying %d%%", (pTmpArray->u.array.RebuildSectors>>11)*100/((pTmpArray->VDeviceCapacity/(pTmpArray->u.array.bArnMember-1))>>11));
+				sprintf(buf, "Verifying %d%%", (UINT)((pTmpArray->u.array.RebuildSectors>>11)*100/((pTmpArray->VDeviceCapacity/(pTmpArray->u.array.bArnMember-1))>>11)));
 				sStatus = buf;
 			}
 			else if (pTmpArray->u.array.rf_need_rebuild)
@@ -472,7 +473,7 @@ hpt_copy_array_info(HPT_GET_INFO *pinfo, int nld, PVDevice pArray)
 	}
 out:	
 	if (!sStatus) sStatus = "Normal";
-	hpt_copy_info(pinfo, "%2d  %11s  %-20s  %5dMB  %-16s", nld, sType, pArray->u.array.ArrayName, pArray->VDeviceCapacity>>11, sStatus);
+	hpt_copy_info(pinfo, "%2d  %11s  %-20s  %5lldMB  %-16s", nld, sType, pArray->u.array.ArrayName, pArray->VDeviceCapacity>>11, sStatus);
 }
 #endif
 
diff --git a/sys/dev/hptmv/i386-elf.raid.o.uu b/sys/dev/hptmv/i386-elf.raid.o.uu
index a7462b8..6e1b8ca 100644
--- a/sys/dev/hptmv/i386-elf.raid.o.uu
+++ b/sys/dev/hptmv/i386-elf.raid.o.uu
@@ -26,1358 +26,1415 @@
  * $FreeBSD$
  */
 begin 644 hptmvraid.o
-M?T5,1@$!`0D```````````$``P`!``````````````#(O@```````#0`````
-M`"@`#``)`%6)Y5=64X'L'`(``(M]"(M'#(F%Y/W__XUWQ+`"@'\!`'4*@'\"
-M`1G`@^`"0(@&QT8$`````(!.`02+1PR)1@B+5Q")5@P/M@:+!(4`````B48T
-M@#X#=2R-0O:)1@R-E>C]__]2:B"+1Q"#Z`I05^C\____@\00@;WH_?__\Q9X
-M6G07D`^V!HL$A0````")1CB`3@$$Z;8$``"#[`AHD````(V-Z/W__U'H_/__
-M_X/$$(3`=<_VA?']__\"="+&1P@!BH7R_?__B$<+.D<)<Q"#[`@/ML!05^C\
-M____@\00@^P(:``"``"-A>C]__]0Z/S___^#Q!"$P'0>@^P$:'`!``!J`(V%
-MZ/W__P60````4.C\____@\00BI5[_O__B-.#XP&*1P2#X/R(T8/A`@G8"<B(
-MTX/C!(/@\XC1@^$("=@)R(C3@^,0@^#/B-&#X2`)V`G(B-&#X4"#X#^#XH`)
-MR`G0B$<$BI7Q_?__@^(!BD8!@^#^"="(1@&`O?C]__\##X3\_O__@+WX_?__
-M`'40Q@8`H0````")1CCIKP,``(.][/W__P`/A-;^__^Z``````^VA?/]__^)
-MP8/X`'XONP````")T,'@!(V]Z/W__P'X@'@0"`^'I_[__P^V0!"#/(,`#X29
-M_O__0CG1?]:+A0#^__^)A=S]__^Z`````(T$4HT$P(N-Y/W__XV<P6`B``"#
-M>P0`=0^+0T`[A>S]__\/A"$!``!"@_H/?M*+O>3]__^+GUPB``"%VW0?BT,(
-MB8=<(@``@^P$:-@```!J`%/H_/___XE["(/$$(N%[/W__XE#0(I32H/*`8A3
-M2F8/MH7\_?__@^`!T>"#XOT)PHA32F:+A?[]__]FB4-,BX7<_?__B4-0BH7X
-M_?__B`.*A?O]__^(0SX/MLBX`0```-/@9HE#2,=#!`````"+A?3]__^)0PR*
-MA?G]__^(0SP/M@.+!(4`````B4,T@^P$:A"-A>C]__^#P"Q0C4-T4.C\____
-M@\0,:@2-A>C]__^#P#Q0C8.$````4.C\____@\0,:A"-A>C]__^#Z(!0C8/(
-M````4.C\____@\0,:D"-A>C]__^#P$!0C8.(````4.C\____@\00BY7<_?__
-M.5-0=@.)4U#VA?']__\!=`2`2P$!BHWZ_?__B(WC_?__@+WS_?__`0^&HP$`
-M``^VP8M\@U2+A1#^__^)A=S]__^%_P^%/0$``(N5Y/W__XNZ7"(``(7_="6+
-M1PB)@EPB``"#[`1HV````&H`5^C\____BXWD_?__B4\(@\00BX7L_?__B4=`
-MBE=*@\H!B%=*9@^VA0S^__^#X`'1X(/B_0G"B%=*9HN%#O[__V:)1TR+A=S]
-M__^)1U"*A0G^__^(1SR*A0C^__^(!XE?!(N%!/[__XE'#(J%"_[__XA'/@^V
-MR+@!````T^!FB4=(#[8'BP2%`````(E'-`^V`XL$A0````")1SB*E>/]__^(
-M5P,/ML*)?(-4_D,]@^P$:A"-A>C]__^#P"Q0C4=T4.C\____@\0,:@2-A>C]
-M__^#P#Q0C8>$````4.C\____@\0,:A"-A>C]__^#Z(!0C8?(````4.C\____
-M@\0,:D"-A>C]__^#P$!0C8>(````4.C\____@\00]H7Q_?__`70$@$\!`0^V
-MA0K^__^#?(=4`'5?B72'5(J%"O[__XA&`_Y'/8E^!`^V!XL$A0````")1CB+
-MC=S]__\Y3U!V-(E/4.LO#[:%X_W__X-\@U0`=2&)=(-4B5X$_D,]BH7C_?__
-MB$8##[8#BP2%`````(E&.)"-9?1;7E_)PU6)Y593BW4,NP`````/MD8\B<*#
-M^``/CJ@```"#?)Y4`'0*0SG:?_3IEP```+@!````B-G3X&8)1DR#[`A6_W4(
-MZ/S___^)PH/$$(7`='7&``.(6`.)<`2`/@@/E,#!X`.*2@&#X?<)P8A*`:$,
-M````B4(T#[8&BP2%`````(E"./9&`1!T#HC(@\@0B$(!BT),B4(,B52>5(I&
-M2H/@_H/("HA&2L=&4`````#^1CV#[`A2:@?H_/___X!.2@2#Q!"-=@"-9?A;
-M7LG#D%6)Y5=64X/L#(MU#`^V!H/X!G1\@_@&?PN#^`0/C%(!``#K$(/X"`^$
-MNP```.E"`0``B?:*1CTZ1CP/A30!``"`3@$$@&9*_K\`````@'X\``^$'0$`
-M`)"+7+Y4A=MT((`[`W8;@^P(4_]U".C\____@\00]D-*!'0&@$Y*!(GV1P^V
-M1CPY^'_/Z>8```")]H!.`02`?CT"=0J`9DK^Z=$```"0@&9*_8-^5`!U$8M&
-M6,9``P")1E3'1E@`````BT94@#@#=1&#[`A6_W4(Z/S___^#Q!")]H!.2@3'
-M1E``````Z8P```"*1CTZ1CQU'(!.`02*1DJ#X/Z(1DIF@WY,`'1D@\@"B$9*
-MZUP/MD8]0`^V5CPYT'5/9H-^3`!T(;D`````#[=&3(G"J`%U"4&)T-/XJ`%T
-M]X-\CE0`=1_K!O9&2@)U%X!.`02`3DH$@^P(5O]U".C\____@\00QT90````
-M`(UV``^W1DCWV"%&4(UV`(-^!`!U&O9&2@1T%.C\____B49`@^P,5NC\____
-M@\00C67T6UY?R<-5B>575E.#[`R+?0B^`````(UV`(T$=HT$P(V<QV`B``"#
-M>T``="&#>P0`=1N#[`A35^C\____@\00]D,!!'0(QT,X`````)!&@_X/?L:-
-M9?1;7E_)PXGV58GE5U93@>PH`@``BWT,BD40B(7G_?__5^C\____@\00@#\(
-M=1*#[`A7_W4(Z/S___^#Q!"-=@"^`````(!_/``/AL4```"0@WRW5``/A*T`
-M``"+1+=4@#@#=B"#[`0/MH7G_?__4/]TMU3_=0CH_/___X/$$.F%````D(M<
-MMU3&0P,`QT,$`````,=#4`````"`8P'^QT,X`````("]Y_W__P!U#H`_!G4)
-M@W\$`'0RC78`@^P$:``"``!J`(V5Z/W__U+H_/___XV%Z/W__U!J,&H`C4,\
-M4.C\____@\0@ZQ&*5P&#X@&*0P&#X/X)T(A#`8/L#%/H_/___X/$$$:)\CA7
-M/`^'//___X/L!&C8````:@!7Z/S___^+50B+@EPB``")1PB)NEPB``#'1T``
-M````@\00C67T6UY?R<-5B>575E.![!P"``"+50R+?1"+312+`HM8!(F=Y/W_
-M_XM:((M3%(F5X/W__P^V0`.)A=S]__^-E>C]__^#O>#]__\`=`QF#[9#$(/@
-M`3G(=2:)E>#]__^#>Q@`=!!14E/_=0C_4QB#Q!"%P'4*N`````#IE@````^W
-M0RS!X`D/MG,HB?:+E>#]__^+&KD```$`9H-Z!`!T!`^W2@2)PCG(=@*)RCNU
-MW/W__W409HE7!&;'1P8``(D?@\<(D"G1`=,IT'4A1HN5Y/W__P^V0CPYQG4%
-MO@````"+E>3]__\/MT)(P>`)A<EULXN=X/W__V:!>P8`@'0+@\,(B9W@_?__
-MZX)FQT?^`("X`0```(UE]%M>7\G#C78`58GE5U93BT4(BTT,BW@$BW$DBE`#
-M9HM9,#A1*'4*#[=!+@'&ZPV)]CA1*'8&#[='2`'&.%$H=01F`UDL.%$I=0=F
-M`UDJC78`@'DT`'0..%$H<QTX42EW$NL6B?8X42AS!&8#7T@X42EV!F8#7TB)
-M]HM%$&:)6`B)<`1;7E_)PY!5B>575E.#[`B+?0QFBW<(9L=%\@``BT4(#[9(
-M/HM?!-/K9L=',```QD<T`0^V2#R)V+H`````]_&)1>R+50@/MDH^T^")1R0/
-MMDH\B=BZ`````/?QB%<HB-&+50AFBT)(2&8C1P1FB4<NBUT(9HM32&8IPF8Y
-M\G(+9HEW++X`````ZU1FB5<L9BG69L=%\@$`N@$```")]HG0T^!F"4<R08M%
-M"#A(/'4?L0"`?S0`=`?&1S0`ZQ&09HM',(M="&8#0TAFB4<PD(M%"&8Y<$AS
-M!V8K<$CKP9"X`0```-/@9@E',HA/*6:)=RH/MT7R@\0(6UY?R<.)]E6)Y593
-MBW4(BU4,BUH@BP(/MD@#N/[____3P&8A0S*`>A$!=`:*0A&(0Q&#[`A25NC\
-M____@\009H-[,@!U&H![$0!U!,9#$0&#[`13_W,<5NC\____@\00C67X6U[)
-MPU6)Y5=64X/L#(M]#(L'B47P]D`!!'49QD<1`H/L!%?_=QS_=0CH_/___^G)
-M````D%?_=?#H;_[__X/$"&:#?S(`=1G&1Q$"@^P$5_]W'/]U".C\____Z9T`
-M``"0O@````"+5?"`>CP`#X2*````B?8/MT<RB?'3^*@!=&R#[`S_=0CH_/__
-M_XG#BT7PBT2P5(E%[(D#B7L@BE<0@^("BD,0@^#]"="(0Q"*5Q"#X@2#X/L)
-MT(A#$(I'"HA#"E-7_W7LZ&']___'0QP`````QT,8`````(/$%%/_=0B+5>S_
-M4C2#Q!"-=@!&BTWP#[9!/#GP#X]X____C67T6UY?R<-5B>6#[`B+50R+2@2*
-M00'`Z`*#\`&#X`&`8@'[@&$!^_Y)/8!)2@&#>3@`=!.%P'4/@^P(4?]U"/]1
-M.(/$$(GVR<.)]E6)Y593BUT0BW44BT4,BU`@BTH4A<ET#&8/MD(0@^`!.?!U
-M)(-Z&`!T%5934O]U"/]2&(/$$+H!````A<!U)[H`````ZR")]HL!BU$$B0.)
-M4P2#PPB-00:#P0AF@S@`>>>Z`0```(G0C67X6U[)PY!5B>564XMU"`^V!H/H
-M!(/X`7=!NP````"`?CP`="^)]H-\GE0`=!.#[`S_=)Y4Z,W___^#Q!"%P'4*
-MN`````#K'(UV`$,/MD8\.=A_T[@!````ZPF*1@'`Z`*#X`&-9?A;7LG#58GE
-M5U93@^P,BW4,BUX@BSN+!HE%\(!^$0$/A-````#V0Q`"=':`>Q(`#X7`````
-M]D=*`@^%M@```,9#$@&`>`,!&<"#X`2+1#A4B47PA<!T0U#H1/___X/$!(7`
-M=#:#[`1J,&H`C48D4.C\____BT7P#[9(`[@!````T^!FB4,RBT7PB0:#Q`A6
-M_W4(_U`TZ<<```"+!HE%\.M4]D,0!'1._W7PZ/+^__^#Q`2%P'0B@^P$:C!J
-M`(U&)%#H_/___X/$"%;_=0B+1?#_4#3IB0```(![$0%T'0^W4S*+1?`/MD@#
-MN`$```#3X#G"=0>0BD81B$,1@^P(5O]U".C\____BT7P#[9(`[C^____T\!F
-M(4,R@\009H-[,@!U/H![$0!U!,9#$0&#[`13_W,<_W4(Z/S___]F_T\6@\00
-M9H-_%@!U%XM''(7`=!"#[`C_=R#_=0C_T(/$$(GVC67T6UY?R<-5B>53@^P$
-MBU4,BUT0BTT4BT(@A<ET&8M`#(D#9HM"",'@"6:)0P1FQT,&`(#K19"#>!0`
-M=";V0!`!=""+2!20BP&+402)`XE3!(/#"(U!!H/!"&:#.`!YY^L9D+H`````
-M@W@8`'0245-0_W4(_U`8B<+K!;H!````B="+7?S)PU6)Y5=64X/L#(M]"(MU
-M#(M>((L&#[9(`[C^____T\!F(4,R@'X1`70&BD81B$,1@^P(5E?H_/___X/$
-M$&:#>S(`=4F`>Q$`=1?&0Q$!BPN+45`[5@1U"0^W1@@!T(E!4(/L!%/_<QQ7
-MZ/S___^+`V;'0!0``(/$#(L#@\`84&@`````5^C\____@\00C67T6UY?R<-5
-MB>575E.#[`R+?0B+=0R+7B"`?A$!=`B*1A&(0Q'K7(!^"B!U/L9&$0"+`XM`
-M6(D&@^P$:C!J`(U&)%#H_/___\9&$@#&1@HPBD80@^#]@\@$B$80@\0(BP96
-M5_]0-.M.C78`QD,1`8L+BU%0.U8$=0H/MT8(`=")05"0@^P(5E?H_/___X/$
-M#%/_<QQ7Z/S___^+`V;'0!0``(/$#(L#@\`84&@`````5^C\____C67T6UY?
-MR<.058GE5U93@^P,BWT0BTT4BU4,BT(@A<ET0XM0#(D7BTT,9HM!",'@"6:)
-M1P1FQT<&`("+`8!X`P`/A/(```"-@@```0!F@W\$`'0'#[='!(T$`HD'Z=<`
-M``"-=@")^XM5#`^W<@C!Y@F#>!0`=";V0!`!=""+2!20BP&+402)`XE3!(/#
-M"(U!!H/!"&:#.`!YY^LUD+H`````@W@8``^$D@```%%74/]U"/]0&(/$$+H`
-M````A<!T?>L.9HEQ!&;'008`@.L]B?:)^8GVN@```0!F@WD$`'0$#[=1!(M=
-M#(L#@'@#`'01.?)V$0$QB=!F*?!FB4$$ZPLY\G.^@\$(*=9UR8M5#(L"@'@#
-M`'0B.<]T'HGVBP&+402)!XE7!(/'"(U!!H/!"&:#.`!YYXUV`+H!````B="-
-M9?1;7E_)PY!5B>575E.#[`R+?0B+=0R+7B"`?A$!=`B*1A&(0Q'K<(L&@'@#
-M`'4LQD81`(L#BT!8B0:#[`1J,&H`C48D4.C\____QD82`(/$"(L&5E?_4#3K
-M<)"#[`0/MT8(P>`)4`-##%#_<PSH_/___X/$$(7`=`;&0Q$,ZQC&0Q$!BPN+
-M45`[5@1U"@^W1@@!T(E!4)"#[`A65^C\____@\0,4_]S'%?H_/___XL#9L=`
-M%```@\0,BP.#P!A0:`````!7Z/S___^-9?1;7E_)PY!5B>575E.#[`R+?0B+
-M70R+,\=&'`````#V1DH!#X5:`0``#[9#"H/X`@^$^0```(/X`G\0@_@!#X27
-M````Z3H!``")]H/X`P^%+P$``(M#!(E&$&:+0PAFB484@^P$#[=#",'@"5!J
-M`/]S#.C\____9L=#,@,`QT7P`````(/$$(/L#%?H_/___XG"QD`*,(M#!(E"
-M!&:+0PAFB4((@$H0!(E:(,="'`````#'0A@`````BTWPBT2.5(D"@\0(4E?_
-M4#2#Q!#_1?"#??`!?K'IO````(M#!(E&$&:+0PAFB484@^P,5^C\____B<+&
-M0`H@BT,$B4($9HM#"&:)0@B`2A`"B5H@QT(<`````,="&`````"+1E2)`H/$
-M"%)7_U`T@\00ZVJ)]HM#!(E&$&:+0PAFB484@^P,5^C\____B<+&0`H@BT,$
-MB4($9HM#"&:)0@B`2A`"B5H@QT(<`````,="&`````"+1E2)`H/$"%)7_U`T
-M@\00ZQ:)]L9#$0:#[`13_W,<5^C\____@\00@^P$C4884&@`````5^C\____
-MC67T6UY?R<.)]E6)Y5=64X/L'(M]#(LW]D8!!'48QD<1`H/L!%?_=QS_=0CH
-M_/___^E\`0``]D<0('0JQT8<G!,``(E^(&:#?A8`#X5A`0``@^P(5_]U".@!
-M_O__@\00Z4T!``"0@WX<`'4B9H-^%`!T-P^W1A0#1A`[1P1V*P^W1P@#1P0Y
-M1A!S'XUV`,='*`````"#[`A7C4884.C\____Z0L!``"-=@!F_T86]D9*`W5Z
-M]D<0`G1TBU94BTY8@#H#=5F`.0-U5(M'!(G#*UE4B5WDBUI8B5WHBTE8*T)4
-M>0+WV(-]Y`!Y`_==Y#M%Y'T4QD7S`(7)=3>#?>@%=C'&1?,!ZRO&1?,`@WWH
-M`'4%@_D%=QS&1?,!ZQ:)]H!^3@`/E$7SBD7SB$9.ZP3&1?,`9L=',@``LP`/
-MML.#?(94`'1FBT2&5/9``01T7/9'$`1U!3A=\W51@^P,_W4(Z/S___^)1>R#
-MQ`QJ)%=0Z/S___\/MLNX`0```-/@9@E',HM5[(EZ(,="'`````#'0A@`````
-MBT2.5(D"@\0(4O]U"/]0-(/$$(GV0X#[`7:*B?:-9?1;7E_)PU6)Y593BTT(
-MBW4,BUX$#[96`X!F`?N`2TH!QT-0`````/Y+/8![/0!U!H!C`?OK&(I#2J@"
-M=!&#X-6(0TJ%TG4'@&,!^XUV`/9#`01U&H-[.``/A,P```"#[`A34?]3.(/$
-M$.F\````A=)U%(M35,9"`P&+0UC&0`,`B4-4B5-8@WL$`'0)BT,$]D`!!'1M
-MBT-4@#@#=66#[`A34>C\____B<*#Q!"%P'12Q@`#QD`#`8E8!*$,````B4(T
-MH1@```")0CCV0P$0=`J`2@$0BT),B4(,QT8$`````(E36(I#2H/@_H/("HA#
-M2OY#/8/L"%)J!^C\____@\00D(-[!`!T"XGVBUL$@WL$`'7WBT-`B4-$Z/S_
-M__^)0T"#[`Q3Z/S___^-9?A;7LG#D%6)Y5=64X'L+`(``(M%#(M-%(L0B97D
-M_?__BT`@B874_?__BU@4QX7@_?__$0```(V5Z/W__X7;=`QF#[9`$(/@`3G(
-M=2J)TXNUU/W__X-^&`!T$%%25O]U"/]6&(/$$(7`=0RX`````.D``0``B?:+
-MO=3]__^`?RG_#X33````#[=/+,'A";X```$`9H-[!`!T!`^W<P0Y\79(D(N5
-MY/W__XI"`XN]U/W__SI'*'44BP.+4P2+?1")!XE7!(/'"(E]$)`I\8/#"+X`
-M``$`9H-[!`!T!`^W<P3_C>#]__\Y\7>YBY7D_?__BD(#B[W4_?__.D<H=1F+
-M`XM5$(D"9HE*!&;'0@8`@+@!````ZU^0.?%S-HG(`P.+?1")!XGP9BG(9HE'
-M!&:!>P8`@'4-9L='!@"`N`$```#K,XM%$&;'0`8``(/`"(E%$(/#"/^-X/W_
-M_X/L!(N%X/W__\'@`U!3_W40Z/S___^X`0```(UE]%M>7\G#B?95B>6+50R+
-M31"+10B*0`,Z0BAU$&:+0BQFB4$(BT(DB4$$ZP]FBT(J9HE!",=!!`````#)
-MPXUV`%6)Y5=64X/L"(M]"(MU#(M>!&:+1@AFB47RQT7L`````+D`````@'\\
-M`'1\BU2/5#E:#'9GB5XDB$XHN`$```#3X&:+?C()QV:)?C(/MT7R`=@[0@QV
-M-<=%[`$```"-00&(1BE!BT7LT^")P8GX"<AFB48RBT(,9BG89HE&+&:+5?)F
-M*<)FB58JZQV0QD8I_V:+1?)FB48LZPZ)]BM:#$$/MD<\.<AWA(M%[(/$"%M>
-M7\G#D%6)Y593BW4(BU4,BUH@BP(/MD@#N/[____3P&8A0S*`>A$!=`:*0A&(
-M0Q&#[`A25NC\____@\009H-[,@!U&H![$0!U!,9#$0&#[`13_W,<5NC\____
-M@\00C67X6U[)PU6)Y5=64X/L#(M]#(L'B47P]D`!!'49QD<1`H/L!%?_=QS_
-M=0CH_/___^G)````D%?_=?#HL_[__X/$"&:#?S(`=1G&1Q$"@^P$5_]W'/]U
-M".C\____Z9T```"0O@````"+5?"`>CP`#X2*````B?8/MT<RB?'3^*@!=&R#
-M[`S_=0CH_/___XG#BT7PBT2P5(E%[(D#B7L@BE<0@^("BD,0@^#]"="(0Q"*
-M5Q"#X@2#X/L)T(A#$(I'"HA#"E-7_W7LZ.W]___'0QP`````QT,8`````(/$
-M%%/_=0B+5>S_4C2#Q!"-=@!&BTWP#[9!/#GP#X]X____C67T6UY?R<-5B>6#
-M[`B+10R+4`2`8`'[@&(!^_Y*/8!*2@&#>C@`=`^#[`A2_W4(_U(X@\00B?;)
-MPXGV58GE4XM="(M-#(N4B_PO``"%TG0)BP*)A(O\+P``B=!;R<.058GE4XM=
-M"(M-#(M5$(N$D_PO``")`8F,D_PO``!;R<-5B>575E.#[`R+10R[`(```#T`
-M0```=V%FNP!`/0`@``!W5F:[`"`]`!```'=+9KL`$#T`"```=T!FNP`(/0`$
-M``!W-6:[``0]``(``'<J9KL``CT``0``=Q]FNP`!/8````!W%&:[@`"#^$!W
-M"X/X(1G;@^/@@\-`BT40#Z_#!?\/``#!Z`R)1?#'1>P`````N`````"!^P`0
-M```/A]8```#_3?"#??#_#X3&````D(/L#/]U".C\____B<;_1>R_`````(/$
-M$+@`$```N@````#W\X/X``^&BP```(UV`+@*````@?L`0```=U>P"8'[`"``
-M`'=-L`B!^P`0``!W0[`'@?L`"```=SFP!H'[``0``'<OL`6!^P`"``!W);`$
-M@?L``0``=QNP`X'[@````'<1L`*#^T!W"H/[(`^7P`^VP)!05O]U".B2_O__
-M`=Z#Q`Q'N``0``"Z`````/?S.?@/AWC_____3?"#??#_#X4[____BT7LC67T
-M6UY?R<-5B>564XMU"(M=#+@`````.9XH,```<DIJ`E;H'/[__XG!@\0(A<!T
-M-XD82X/[_W0OC78`N@````"#OB0P````=!2+EB0P``"+`HF&)#```/^.*#``
-M`(E4F01+@_O_==2)R(UE^%M>R<.-=@!5B>564XM="(MU#(L.28/Y_W0?D(M4
-MC@2+@R0P``")`HF3)#```/^#*#```$F#^?]UXFH"5E/HN?W__XUE^%M>R<.)
-M]E6)Y593BW4(BQY+@_O_=!R#[`1H`!```&H`_W2>!.C\____@\002X/[_W7D
-MC67X6U[)PY!5B>53BUT(BPM)@_G_=">+1(L$N@````"#.`!T![@`````ZQ="
-M@\`$@?K_`P``=NA)@_G_==FX`0```%O)PXGV58GE@^P,:A!J`/]U".C\____
-MR<-5B>6+30R#>0P`=`F+40R+00B)0@B+40B+00R)`L=!"`````!J`%'_=0CH
-M^?S__\G#C78`58GE4XM5"(M-#(U:,(M",(E!#(7`=`F+4C"-00R)0@B)60B)
-M"UO)PU6)Y5=64XM]$(MU%(M%#(M8,(7;=!F)]CDS<PPY>P1V![@`````ZR.+
-M6PR%VW7I:@#_=0CH;_S__XG#B3B)<`10_W4,Z(____^)V(UE]%M>7\G#D%6)
-MY5.+30B*42D/MMJ-!-T`````*=B-7($L0HA1*8D+QD,%`,9#!@#&0P<`B=A;
-MR<.-=@!5B>575E.+?0B+=0R[``````^V5B:-0OW1^`'"@_H`?B.-=@#_M)ZL
-M!P``5^@?_O__@\0(0P^V5B:-0OW1^`'".=I_X&H'5E?H]_O__XUE]%M>7\G#
-MC78`58GE5U93@^P<BWT0BD4,B$7SBU449HE5\(M%"(N8*#```,'C`P^WR@^V
-M5?.-0OW1^`'"#Z_*.<L/@]H````/ME7SB57DB=")5>`/MU7PC4#]T?@#1>2)
-MTP^OV)"+10@%Y"\``(M5"#F"Y"\```^$PP```(NRY"\``(M6!(L&B5`$B0*)
-M-HEV!(-^$`!T'H-^%`!T"8M6%(M&$(E"$(M6$(M&%(D"QT80`````(M%\&8Y
-M1B1U0XI5\SA6)G4[B3:)=@2)?B"_`````(-]Y`!^(8M=X)"#[`R)^,'@!(V$
-M,%0(``!0Z,?]__^#Q!!'.?M_XXGPZ1,!``!6_W4(Z*?^__^#Q`B+50B+@B@P
-M``#!X`,YV`^"1/___VH'_W4(Z*;Z__^)QH/$"+@`````A?8/A-<```#K"K@`
-M````Z<L```")-HEV!(E^((M%\&:)1B2*5?.(5B;'1A@`````OP`````/ME7S
-MC4+]T?@!PH/Z``^.E````(M%\&;!Z`,/M\")1>@/ME7SC4+]T?@!PHE5[/]U
-MZ/]U".CQ^___B82^K`<``(/$"(7`=3-/@___=!F0_[2^K`<``/]U".@Y_/__
-M@\0(3X/__W7H:@=6_W4(Z!GZ__^X`````.LQB?:#[`R)^\'C!`'SC8/4!P``
-M4.B^_/__@<-4"```B1PDZ+#\__^#Q!!'.7WL?X>)\(UE]%M>7\G#D%6)Y5=6
-M4X/L#(M]$(M%%&:)1?*+50P/MDH^B?C3Z"7_`P``BU(LC1R"BS.%]G0@D#E^
-M('43BU8$BP:)4`2)`HDVB78$B?#K:XMV%(7V=>$/MT7R4%>+50P/MD(\4/]U
-M".B'_?__B<:#Q!"%P'1#BTT,B4@<BP.)1A2%P'0(BQ.-1A2)0A")7A"),XM%
-M#(I8/(A>)@^V2#Z)^-/HBTT,#[91/(G1N@````#W\2C32XA>)XGPC67T6UY?
-MR<.058GE5U93@>S,````BT4,BE`0B-#0Z(/P`8G!@^$!B8U4____BT4,@W@4
-M`'0%]L(!=""-E5C___^)E5#___]J`5+_=0S_=0B+30S_41B#Q!#K#(M%#(M`
-M%(F%4/___\>%./___P```0"+E5#___]F@WH$`'0*#[=*!(F-./___\>%0/__
-M_P````#'A4S___\`````BT4,@'A$``^$W`$``(N53/___XM-#(M4D3")E43_
-M__^*BBP)``"(C4O___\XBBT)```/@I@!``"*A4O___^+E43___\X0B</A&4!
-M```/ML")T6:+E((,"0``9HN$@0X)``!FB84^____B=!FP>@##[?`B84T____
-M@^('9HF5//___P^WPHG"P>()N`@```!F*X4\____9HF%//___V:+A3[___]F
-M.84\____=@=FB84\____#[>%//___XG&P>8)#[:%2____XN-1/___XN$@:P'
-M``")UXN--/___P-\B`20BX4X____.85`____=3*#A5#___\(QX4X____```!
-M`(N54/___V:#>@0`=`H/MTH$B8TX____QX5`____`````(N=./___RN=0/__
-M_SGS=@*)\X.]5/___P!T(8/L!%.+A4#___^+E5#___\#`E!7Z/S___^#Q!#K
-M'XUV`(/L!%-7BX5`____BXU0____`P%0Z/S___^#Q!`!G4#___\IWG0'`=_I
-M5____XN%//___V8IA3[___]T&?^%-/___[H`````9L>%//___P@`Z>O^___^
-MA4O___^*E4O___^+C43___\XD2T)```/@VC^____A4S___^+50P/MD)$.X5,
-M____#X\D_O__@^P$:@#_=0S_=0C_51"-9?1;7E_)PXGV58GE5U93@^P,@WT0
-M`'0'BT4,QD`1"\=%\`````"+50R`>D0`#X3!````BT4(!>PO``")1>R+50B!
-MPN0O``")5>B-=@"+1?"+50R+7((PQX,$"0```````(G?L0"`>R8`=D&)]@^V
-MP<'@!`'XC9#4!P``O@````"#N-0'````=12#>@0`=0Z#>@@`=0B#>@P`=`>)
-M]KX!````A?9U(4$X3R9WP;@`````A<!T'(M5[(M"!(E:!(D3B4,$B1CK&K@!
-M````Z^.-=@"+5>B+0@2)6@2)$XE#!(D8_T7PBU4,#[9"1#M%\`^/6?___X/L
-M"/]U#/]U".AC/P``C67T6UY?R<.-=@!5B>575E.#[!R+70R+LP0)``"+10C_
-MB"PP``"-0PB+2`2+4PB)2@2)$8E#"(E`!(7V#X11`@``]D80(`^$^P```(L^
-M#[=#)`-#((E%[&:+4R1FB57JQX,$"0```````(M%"`7D+P``BU`$B5@$B0.)
-M4P2)&H"[+@D```%U1(!^"@)U-0^V0R?_M(.L!P``Z'7W__^#Q`2%P'4>QH,N
-M"0``#(!/2@H/MT=(]]@C0R`Y1U!V!8E'4(GV@+LN"0```70:BH,N"0``B$81
-M@^P(5O]U".B"/@``Z5D#```/MT=(BU8$0@^OPCM%['8K#[=%ZE#_=>Q7_W4(
-MZ+#Z__^)L`0)``")1C"#Q`A0_W4(Z*L/``#I'@,```^W1T@#1U`[1>QU!HM%
-M[(E'4(/L"%;_=0CH)#X``.G[`@``C78`BD9%0(A&13I&1`^%=`(``/9&$`0/
-MA,H```#&1?,`@'Y$``^&J````(M5"('"Y"\``(E5X(M%"`7L+P``B47<D`^V
-M1?.+7(8PQX,$"0```````("[+@D```%U38E=Y+\`````@'LF`'8LC78`@^P,
-MB?H/ML+!X`2+5>2-A!#4!P``4.B%]O__@\001XGXBU7D.$(F=]>+5>"+0@2)
-M6@2)$XE#!(D8ZQJ0BH,N"0``B$81BU7<BT($B5H$B1.)0P2)&/Y%\XI%\SA&
-M1`^'</___X/L"%;_=0CH0#T``(/$$.F@`0``QD7S`(!^1`!V)XGV#[9%\XM<
-MAC"`NRX)```!=`F*@RX)``"(1A'^1?.*5?,X5D1WVX!^$0!U&8/L!&BX)0``
-M5O]U".@`^O__@\00Z5`!``"#[`1J`%;_=0CH?OS__X/$$.DZ`0``B?:`NRX)
-M```!=7^)W[X`````@'LF`'8EB?:#[`R)\@^VPL'@!(V$.-0'``!0Z)#U__^#
-MQ!!&B?`X1R9WW8.["`D```!T*HN#"`D``(F#!`D``,>#"`D```````"#[`A3
-M_W4(Z,X-``"#Q!#IR@```(M%"`7D+P``BU`$B5@$B0.)4P2)&NM7C78`BT4(
-M!>PO``"+4`2)6`2)`XE3!(D:@[L("0```'0VBY,("0``BH,N"0``B$(1@^P$
-M_[,("0``:!1F``#_=0CH_/___\>#"`D```````#IT@```(GV_W,8_W4(Z.WT
-M___'0Q@`````BT,<@\0(@W@8`'09@^P$@\`84&@`````_W4(Z/S___^#Q!#K
-M)8M5"(.Z-#````!T&8/L!(G0!30P``!0:`````!2Z/S___^#Q!"+10B#N#`P
-M````=&B`NRX)```!=5^+0QR#>"0`=#V#[`Q0Z-TY``"#Q!"%P'4M@^P$BT,<
-M_W`HBT,<_W`D_W4(Z/S___^+0QS'0"0`````BU4(_XHP,```@\00BT4(@[@P
-M,````'0-@^P,4.C\____@\00D(UE]%M>7\G#58GE@^P(BT4,BP"`>"H`=!O&
-M0"H`@^P$4&@4-0``_W4(Z/S___^#Q!"-=@#)PXGV58GE5U93@^P,BU4,BWT0
-MBT(@BP@/MD`)BX2!K`<``(E%\(M"!"M!(,'@"0^W<@C!Y@F)PH'B_P\``,'H
-M#(E%[+@`$```B<,ITSGS=@*)\X-]%`!T#XM%[(M-\`-4@03K&XUV`(/L#(M%
-M[(M-\`-4@012Z/S___^#Q!")PHD79HE?!#GS=!5FQT<&``"#QPC_1>PIWKH`
-M````ZZ5FQT<&`("X`0```(UE]%M>7\G#B?95B>575E.+50B*0@H"0@N(0@J-
-M6@R^`0```.L3@'H*`'D*N`````#I\@```/Y""@^V2@J#^1]W$(GPT^"%0@P/
-ME<`/ML#K39"#^3]W$X/I((GPT^"%0P0/E<`/ML#K-9"#^5]W$X/I0(GPT^"%
-M0P@/E<`/ML#K'9"X`````(/Y?W<2@^E@B?#3X(5##`^5P`^VP(GVA<!TA,9"
-M"P&->@P/MEH*O@$```#K!8GV_D(+#[9""XT,`X/Y'W<1B?#3X(5"#`^5P`^V
-MP.M.B?:#^3]W$X/I((GPT^"%1P0/E<`/ML#K-9"#^5]W$H/I0(GPT^"%1P@/
-ME<`/ML#K';@`````@_E_=Q.#Z6")\-/@A4<,#Y7`#[;`C78`A<!UD+`!6UY?
-MR<.058GE5U93@^P4BWT(BU4,BUH@BS.*0A&(0P925^C\____@\00@'L&`75F
-M4^BL_O__@\0$A<!T$<9#!@"#[`A35^B/````ZU>0@'L$`75"BD,(.$8G=#HX
-M0PEU-0^VR(G(P>`$C8PP5`@``(US#(L!"T,,B0&+000+1@2)002+00@+1@B)
-M00B+00P+1@R)00R0QD,%`H/L"%-7Z%[]__^-9?1;7E_)PXGV58GE4X/L!(M=
-M#,9#"P#&0PH`4^@<_O__@^P$4_]U".@(````BUW\R<.-=@!5B>575E.#[!B+
-M70R+,XM6'`^V0PB+?()4_W4(Z/S___^)PHDX@\00@'L$`74*@$@0`L9`"B#K
-M"(!($`3&0`HP#[9#"@-&((E"!&8/MD,+9HE"",="&"@K``")6B#'0AS\+```
-M@^P(4O]U"/]7-(UE]%M>7\G#B?95B>575E.#[`B+?0R^`````,=%\`````"+
-M10B)1>SK$X-]\']V"HGPZ4<!``"-=@#_1?"+5?"#^A]W%[@!````BDWPT^"+
-M50B%`@^5P`^VP.M:@_H_=QB-2N"X`0```-/@BU7LA4($#Y7`#[;`ZSV#^E]W
-M&(U*P+@!````T^"+5>R%0@@/E<`/ML#K(+@`````@_I_=Q:-2J"P`=/@BU7L
-MA4(,#Y7`#[;`C78`A<`/A&S___^[`0```(M5".L#B?9#BTWP`=F#^1]W$;@!
-M````T^"%`@^5P`^VP.M4@_D_=Q>#Z2"X`0```-/@A4($#Y7`#[;`ZSJ)]H/Y
-M7W<7@^E`N`$```#3X(5""`^5P`^VP.L>B?:X`````(/Y?W<2@^E@L`'3X(5"
-M#`^5P`^VP(GVA<!UC(M5\`%=\(GVB=#!Z`.(!+>(T(/@!XA$MP&Q""C!B$RW
-M`@^VP3G8<P<IPP'"1NO9B%RW`D:#??!_#X;!_O__B?"#Q`A;7E_)PU6)Y5=6
-M4X'L3`$``(M%#(LPBU@(@'L5``^$.@$``(-[$`!U"(E#$.GF`0``QD,5`(V%
-MZ/[__U!3Z$?^__^)A<S^__\/MD,4BX2&K`<``(F%W/[__XM#$`^V0`R+A(:L
-M!P``B878_O__BU4,#[9"#(N$AJP'``")A=3^__^[`````(/$"#N=S/[__P^-
-M<@$```^VA)WI_O__B<'!X0D/MH2=ZO[__XG&P>8)BH2=Z/[__XB%T_[__P^V
-MT(G(B[W8_O__`T27!(F%X/[__XN%U/[__XM\D`0!SXF]M/[__XF]Y/[__XN%
-MW/[__XM\D`0!SSF]X/[__W4D@^P$5O^UM/[__XG(B[7<_O__`T26!%#H_/__
-M_X/$$.LJC78`5O^UY/[___^UX/[__P^VA=/^__^+O=S^__\#3(<$4>C\____
-M@\000SN=S/[__P^,0/___^FM````C78`C87H_O__4%/H'_W__XF%S/[__P^V
-M0Q2+A(:L!P``B87<_O__BU4,#[9"#(N$AJP'``")A=C^__^[`````(/$"#N=
-MS/[__WUB#[:$G>G^__^)P<'A"0^VA)WJ_O__B<;!Y@D/MI2=Z/[__XG(B[W<
-M_O__`T27!(F%X/[__XG(B[W8_O__`T27!(F%Y/[__X/L!%90_[7@_O__Z/S_
-M__^#Q!!#.YW,_O__?)Z#[`1J`/]U#/]U".AJ`@``C67T6UY?R<.)]E6)Y5=6
-M4X'L+`$``(M%#(L8C87H_O__4(M%#(/`"%#H1/S__XF%V/[__XM5#`^V0AB+
-MA(.L!P``B87<_O__#[9"&8N$@ZP'``")A=3^__^^`````(/$"#NUV/[__WU=
-MD`^VA+7I_O__P>`)#[:,M>K^___!X0D/MIRUZ/[__XG"B[W<_O__`U2?!(F5
-MX/[__XN5U/[__P-$F@2)A>3^__^#[`114/^UX/[__^C\____@\001CNUV/[_
-M_WRD@^P$:@#_=0S_=0CHE`$``(UE]%M>7\G#58GE5U93@>Q\`0``BT4,BS"+
-M6`B-A>C^__]04^AS^___B864_O__#[9#%(N$AJP'``")A:3^__^Q`(/$"(M5
-M#(!Z%`!V)(I:%(UV``^VP8M]#`^V5`<,BY26K`<``(F4A<C^__]!.,MWXL>%
-MF/[__P````"+A93^__\YA9C^__\/C>X```")]HN5F/[__P^VA)7I_O__B<?!
-MYPD/MH25ZO[__\'@"8F%H/[__XJ,E>C^__^(C9_^__^Q`(M=#(![%`!V,0^V
-MM9_^__^*0Q2(A8?^__\/ML&+E(7(_O__BURR!(T4'XF4A:C^__]!.(V'_O__
-M=]\/MI6?_O__B?B+C:3^__\#1)$$.86H_O__=2"#[`3_M:#^____M:S^__^)
-M^`-$D010Z/S___^#Q!#K+/^UH/[___^UK/[___^UJ/[__P^VA9_^__^+G:3^
-M__\#?(,$5^C\____@\00_X68_O__B[V4_O__.;V8_O__#XP4____@^P$:@#_
-M=0S_=0CH"````(UE]%M>7\G#58GE5U93@^P4BWT(BUT,BW40QD,%`H/^`1G`
-M@^#U@\`,B$,&4U?HC?;__X/$$(![!`-U,(M#"(M0$(72=";'0!``````QD(%
-M`H/^`1G`@^#U@\`,B$(&@^P(4E?H5_;__X/$$(UE]%M>7\G#58GE5E.+70B+
-M=0P/MDX'A<ET29`/MI1+O`8``(T$U0`````IT(U$@RR`>`4"=`FX`````.LK
-MB?:`>`8!=!+&1@4"BD`&B$8&N`````#K$I`/MHQ+O08``(7)=;BX`0```%M>
-MR<.-=@!5B>575E.#[!R+?0S&1RH`@+\O"0```'1[O@````"-7RR`?RD`=$F0
-M@'L%`74V@'L$`W4EBT,(@'@5`'0<@W@0`'06QT`0`````,9#!0**ARX)``"(
-M0P;K"\9'*@'I]P$``(GV1H/#'`^V1RDY\'^X@^P(5_]U".@L\?__Z=@!``"*
-M0P:(ARX)``#&AR\)```!ZX:0QT7P`````,=%[`$```"^`````(U?+(!_*0`/
-MA'P!``"`>P4`#X66````4U?HT_[__X/$"(7`='S&0P4!BT4(B47HB=H/MD,$
-M@_@&=V?_)(4`````@^P(4_]UZ.B<]___@\00ZT^#[`A3_W7HZ*OY__^#Q!#K
-M/H/L"%/_=>CHMOO__X/$$.LM@^P(4_]UZ.AY_/__@\00ZQS&1RL!QD(%`L9"
-M!@&#[`A2_W7HZ*#T__^#Q!"0@'L%`G4,_T7PC78`@'L%`G0.QT7L`````.FZ
-M````B?:`>P8!#X2N````@'\K``^$!?___XM''`^V4PB+1)!4]D`!!`^$B```
-M`,=%[`````#&0P4!QD,&`(M%"(E%Y(G:#[9#!(/X!G=O_R2%'````(/L"%/_
-M=>3HS/;__X/$$.M7@^P(4_]UY.C;^/__@\00ZT:#[`A3_W7DZ.;Z__^#Q!#K
-M-8/L"%/_=>3HJ?O__X/$$.LDQD<K`<9"!0+&0@8!@^P(4O]UY.C0\___@\00
-MZP?&0P8!C78`1H/#'`^V1RDY\`^/A/[__X-]\``/A5K^__^#?>P`=!7&ARX)
-M```!@^P(5_]U".A5[___ZP3&1RH!C67T6UY?R<.-=@!5B>575E.![&P!``"+
-M10R+6!R+N`0)``#&A;'^__\`A?]U,0^W0"2+30R+42`!T%!24_]U".C"Y___
-MBW4,B488@\00N/____^#?A@`#X09*P``B?:+10C_@"PP``"+10R#P`B+50B!
-MPO0O``"+2@2)0@2+=0R)5@B)2`2)`6:#>TP`=$#V0TH!=0B+0U`[1B!W)[X`
-M````#[=#3(G"J`%U#HUV`$:)T(GQT_BH`73UB?.+10R(6"CK+HM5#,9"*`CK
-M)8GV]D-*`G06BT-0BTT,.T$@=PN*02>(02CK"HUV`(M=#,9#*`B+=0S&1BD`
-MQH:\!@```,9&*P#&ABX)````QH8O"0```,>&"`D```````"%_P^$7Q,``/9'
-M$"`/A&D)```/MD<*@_@"#X2&!0``@_@"?PZ#^`$/A$0!``#I%2H``(/X`P^%
-M#"H``(/L#(V=V/[__U/H,>;__X/$$+\`````BT4,#[=P)+,`@_\?=U(/ML.-
-ME(78_O__B;VL_O__N2`````I^3GQ=@*)\8/Y('4(QP+_____ZQ*X`0```-/@
-M2(J-K/[__]/@"0*X(````"GX.?!S%(UT/N"_`````.L#@^\@0X#[`W:@@^P,
-MBUT,_[.L!P``Z#OE___&A;/^__\`@\00@'LF``^&9RD``(/L#`^VG;/^___!
-MXP0#70R-@]0'``!0Z'_E__^!PU0(``")'"3H<>7__X/$$/]U#.@JYO__@\0$
-MQD`$`HJ5L_[__XA0",9`"0"+E=C^__^)4`R+E=S^__^)4!"+E>#^__^)4!2+
-ME>3^__^)4!C^A;/^__^*C;/^__^+70PX2R8/AWO____IW2@``+@!````ZTV+
-M=0RQ`(!^)@!V/0^VP<'@!`'PC9#4!P``NP````"#N-0'````=1*#>@0`=0R#
-M>@@`=0:#>@P`=`6[`0```(7;=;=!.$XF=\.X`````(7`=!B+=0R)O@@)``#'
-MA@0)````````Z8T1``"#[`R-G=C^__]3Z)+D__^#Q!"_`````(M%#`^W<"2S
-M`(/_'W=2#[;#C92%V/[__XF]J/[__[D@````*?DY\78"B?&#^2!U",<"____
-M_^L2N`$```#3X$B*C:C^___3X`D"N"`````I^#GP<Q2-=#[@OP````#K`X/O
-M($.`^P-VH(M=#(![*`AU!HI#)XA#*(MU#(I>*(B>Z`@``,:&Z0@```&+A=C^
-M__^)AM0(``"+A=S^__^)AM@(``"+A>#^__^)AMP(``"+A>3^__^)AN`(``#'
-MAN0(````````5NB#Y/__@\0$QD`$`HA8"(A8"8N5V/[__XE0#(N5W/[__XE0
-M$(N5X/[__XE0%(N5Y/[__XE0&(G'@'XF`@^'5`$``(GS@</4"```5N@XY/__
-M@\0$QD`$!8E8",9`%`")A<#^__^+'_Z#O`8```^VL[P&``")PBG:@^HLP?H"
-MC032C03"C03"C03"B<'!X0\!R(T$PO?8B(1SO`8``(I'!XB$<[T&``")\(A'
-M!\:%L_[__P"+50R`>B8`#X;1)@``BHVS_O__BUT,.$LH#X2H````4^BQX___
-M@\0$QD`$`8J5L_[__XA0"(A0"8N5V/[__XE0#(N5W/[__XE0$(N5X/[__XE0
-M%(N5Y/[__XE0&(N-P/[__XI1%`^VRHJ=L_[__XNUP/[__XA<#@Q"B%84BQ[^
-M@[P&```/MK.\!@``*=B#Z"S!^`*-%,"-%-"-%-"-%-")T<'A#P'*C130]]J(
-ME'.\!@``B[W`_O__BD<'B(1SO08``(GPB$<'_H6S_O__BI6S_O__BTT,.%$F
-M#X<N____Z?HE``#&A;/^__\`BUT,@'LF``^&YB4``('#U`@``(F=I/[__XJ%
-ML_[__XM5##A"*`^$`@$``%+HNN+__X/$!,9`!`&*C;/^__^(2`B(2`F+E=C^
-M__^)4`R+E=S^__^)4!"+E>#^__^)4!2+E>3^__^)4!B)A<3^____=0SH=>+_
-M_X/$!,9`!`.+G:3^__^)6`B*E;/^__^(4`R)A<#^__^+&/Z#O`8```^VL[P&
-M``"+E<3^__\IVH/J+,'Z`HT$THT$PHT$PHT$PHG!P>$/`<B-!,+WV(B$<[P&
-M``"+C<#^__^*00>(A'.]!@``B?.(60>+'_Z#O`8```^VL[P&``")RBG:@^HL
-MP?H"C032C03"C03"C03"B<'!X0\!R(T$PO?8B(1SO`8``(I'!XB$<[T&``")
-M\(A'!_Z%L_[__XJ5L_[__XM-##A1)@^'U/[__^FI)```N`$```#K4HMU#+$`
-M@'XF`'9"C78`#[;!P>`$`?"-D-0'``"[`````(.XU`<```!U%(-Z!`!U#H-Z
-M"`!U"(-Z#`!T!XGVNP$```"%VW6R03A.)G?!N`````"%P'08BUT,B;L("0``
-MQX,$"0```````.E4#0``@^P,C9W8_O__4^A9X/__@\00OP````"+10P/MW`D
-MLP"#_Q]W4@^VPXV4A=C^__^)O:#^__^Y(````"GY.?%V`HGQ@_D@=0C'`O__
-M___K$K@!````T^!(BHV@_O__T^`)`K@@````*?@Y\',4C70^X+\`````ZP.#
-M[R!#@/L#=J"+70R*0R>(@^@(``#&@^D(````BX78_O__B8/4"```BX7<_O__
-MB8/8"```BX7@_O__B8/<"```BX7D_O__B8/@"```QX/D"````````(![)@(/
-MAP`!``"+70R!P]0(``#_=0SH1.#__X/$!,9`!`6)6`C&0!0`B87`_O__QH6S
-M_O__`(MU#(!^)@`/AB0C``#_=0SH%.#__X/$!,9`!`&*E;/^__^(4`B(4`F+
-ME=C^__^)4`R+E=S^__^)4!"+E>#^__^)4!2+E>3^__^)4!B+C<#^__^*410/
-MMLJ*G;/^__^+M<#^__^(7`X,0HA6%(L>_H.\!@``#[:SO`8``"G8@^@LP?@"
-MC13`C130C130C130B='!X0\!RHT4T/?:B)1SO`8``(N]P/[__XI'!XB$<[T&
-M``")\(A'!_Z%L_[__XJ5L_[__XM-##A1)@^'/O___^E=(@``BW4,BEXG5NA)
-MW___@\0$QD`$`8A8"(A8"8N5V/[__XE0#(N5W/[__XE0$(N5X/[__XE0%(N5
-MY/[__XE0&(F%O/[__\:%L_[__P"`?B8`#X8)(@``B?>!Q]0(``"*A;/^__^+
-M50PX0B</A`8!``!2Z.'>__^#Q`3&0`0!BHVS_O__B$@(B$@)BY78_O__B5`,
-MBY7<_O__B5`0BY7@_O__B5`4BY7D_O__B5`8B87$_O___W4,Z)S>__^#Q`3&
-M0`0#B7@(BIVS_O__B%@,B87`_O__BQC^@[P&```/MK.\!@``BY7$_O__*=J#
-MZBS!^@*-!-*-!,*-!,*-!,*)P<'A#P'(C03"]]B(A'.\!@``BY7`_O__BD('
-MB(1SO08``(GQB$H'BQK^@[P&```/MK.\!@``BY6\_O__*=J#ZBS!^@*-!-*-
-M!,*-!,*-!,*)P<'A#P'(C03"]]B(A'.\!@``BY7`_O__BD('B(1SO08``(GQ
-MB$H'_H6S_O__BIVS_O__BW4,.%XF#X?0_O__Z<P@``"%_P^$Y`D``/9'$`(/
-MA-H)``"+10R*@"P)``"(A;/^__^+50PX@BT)```/@H<!``"*C;/^__^+70PX
-M2R</A%H!``"#[`P/MMF)V,'@!(VT!6C___]6Z*;<__^#Q!")M9S^__^+=0P/
-MMY2>#`D```^WO)X."0``O@````"0@_H?=U>)\0^VP8N-G/[__XT<@8F5F/[_
-M_[D@````*=$Y^78"B?F#^2!U"<<#_____^L3D+@!````T^!(BHV8_O__T^`)
-M`[@@````*=`Y^',6C7P7X+H`````ZP.#ZB!&B?.`^P-VF0^VG;/^__^)VHG8
-MP>`$BW4,C9PP5`@``(L+]]&)C=C^__^+0P3WT(F%W/[__XM#"/?0B87@_O__
-MBT,,]]")A>3^__^)UL'F!(V<->C^__^-E#5H____(PJ)"XM"!".%W/[__XE#
-M!(M""".%X/[__XE#"(M"#".%Y/[__XE##(J%L_[__XM5##A"*'4MN@````"#
-M.P!U$H-[!`!U#(-["`!U!H-[#`!T!;H!````A=)T",:%L?[__P&0_H6S_O__
-MBHVS_O__BUT,.(LM"0``#X-Y_O__@+VQ_O__``^$9`<``(/L#(V%V/[__U#H
-M+=O__\:%L_[__P"#Q!"+=0R`?B8`=G>)]HJ%L_[__XM5##A"*'13.$(G=$X/
-MMM#!X@0#50R-BM0'``"+A=C^__\+@M0'``")A=C^__^+A=S^__\+002)A=S^
-M__^+A>#^__\+00B)A>#^__^+A>3^__\+00R)A>3^___^A;/^__^*C;/^__^+
-M70PX2R9WBXMU#`^V?BC!YP2-G#WH_O__B[78_O__(S.)M<C^__^+C=S^__\C
-M2P2)C<S^__^+E>#^__\C4PB)E=#^__^+A>3^__\C0PR)A=3^___WUHFUV/[_
-M__?1B8W<_O__]]*)E>#^___WT(F%Y/[__R,SB;78_O__(TL$B8W<_O__(U,(
-MB97@_O__(T,,B87D_O___W4,Z-#:__^#Q`3&0`0`B86T_O__N`````"#O=C^
-M__\`=1N#O=S^__\`=1*#O>#^__\`=0F#O>3^__\`=`6X`0```(7`#X0\`@``
-MBWT,BD<HB(?H"```BX78_O__B8?4"```BX7<_O__B8?8"```BX7@_O__B8?<
-M"```BX7D_O__B8?@"```QH?I"````<>'Y`@```````#&A;/^__\`@'\F``^&
-MI@(``(J%L_[__XM5##A"*`^$KP$``#B"+`D``'<-.((M"0``<@4X0B=U1_]U
-M#.@`VO__@\0$QD`$`8J-L_[__XA("(A("8N5V/[__XE0#(N5W/[__XE0$(N5
-MX/[__XE0%(N5Y/[__XE0&(F%Q/[__^MW#[:-L_[__\'A!(V<#>C^__^+A=C^
-M__\+`XD#BX7<_O__"T,$B4,$BX7@_O__"T,(B4,(BX7D_O__"T,,B4,,_W4,
-MZ'K9__^#Q`3&0`0!BI6S_O__B%`(B%`)BQ.)4`R+4P2)4!"+4PB)4!2+4PR)
-M4!B)A<3^__^+70R!P]0(``#_=0SH.=G__X/$!,9`!`.)6`B*C;/^__^(2`R)
-MA<#^__^+&/Z#O`8```^VL[P&``"+E<3^__\IVH/J+,'Z`HT$THT$PHT$PHT$
-MPHG!P>$/`<B-!,+WV(B$<[P&``"+O<#^__^*1P>(A'.]!@``B?"(1P>*E;/^
-M__^+30PX42=U4XNUM/[__XL>_H.\!@``#[:SO`8``(GZ*=J#ZBS!^@*-!-*-
-M!,*-!,*-!,*)P<'A#P'(C03"]]B(A'.\!@``B[VT_O__BD<'B(1SO08``(GP
-MB$<'_H6S_O__BI6S_O__BTT,.%$F#X<G_O__Z<@```#&A;/^__\`BUT,@'LF
-M``^&M````(J%L_[__XM5##B"+`D```^'AP```#B"+0D``')_.$(G='HX0BAT
-M=0^VP,'@!(V$!>C^__^Z`````(,X`'42@W@$`'4,@W@(`'4&@W@,`'0%N@$`
-M``"%TG1##[:%L_[__\'@!(V<!>C^____=0SHR]?__X/$!,9`!`&*C;/^__^(
-M2`B(2`F+$XE0#(M3!(E0$(M3"(E0%(M3#(E0&/Z%L_[__XJ=L_[__XMU##A>
-M)@^'3/___[@`````@[W(_O__`'4;@[W,_O__`'42@[W0_O__`'4)@[W4_O__
-M`'0%N`$```"%P`^$6QH``(M%#(I`*(B%E_[__XM5#(B"``D``(N%R/[__XF"
-M[`@``(N%S/[__XF"\`@``(N%T/[__XF"]`@``(N%U/[__XF"^`@``,:"`0D`
-M``#'@OP(````````BEHG4NCTUO__@\0$QD`$`8A8"(J-E_[__XA("8N5R/[_
-M_XE0#(N5S/[__XE0$(N5T/[__XE0%(N5U/[__XE0&(F%O/[__\>%N/[__P``
-M``#&A;/^__\`BUT,@'LF``^&H1D``(J%L_[__XM5##A"*`^$S`$``#A")P^$
-MPP$``(I:)U+H==;__X/$!,9`!`&*C;/^__^(2`B(6`F+E<C^__^)4`R+E<S^
-M__^)4!"+E=#^__^)4!2+E=3^__^)4!B)A<3^__^+&/Z#O`8```^VL[P&``"+
-ME;3^__\IVH/J+,'Z`HT$THT$PHT$PHT$PHG!P>$/`<B-!,+WV(B$<[P&``"+
-MO<3^__^*1P>(A'.]!@``B?"(1P>#O;C^__\`=$N+'_Z#O`8```^VL[P&``"+
-ME;C^__\IVH/J+,'Z`HT$THT$PHT$PHT$PHG!P>$/`<B-!,+WV(B$<[P&``"*
-M1P>(A'.]!@``B?*(5P>+70R!P^P(``"+30R*22>(C9;^____=0SH=M7__X/$
-M!,9`!`.)6`B*G9;^__^(6`R)A<#^__^)A;C^__^+&/Z#O`8```^VL[P&``"+
-ME<3^__\IVH/J+,'Z`HT$THT$PHT$PHT$PHG!P>$/`<B-!,+WV(B$<[P&``"+
-MO<#^__^*1P>(A'.]!@``B?"(1P>+'_Z#O`8```^VL[P&``"+E;S^__\IVH/J
-M+,'Z`HT$THT$PHT$PHT$PHG!P>$/`<B-!,+WV(B$<[P&``"*1P>(A'.]!@``
-MB?*(5P?^A;/^__^*C;/^__^+70PX2R8/APK^___IIA<``(GVBT4,BH`L"0``
-MB(6S_O__BU4,.((M"0``#X*&%P``B?:*C;/^__^+70PX2R=T=@^VP<'@!(V$
-M!>C^__^Z`````(,X`'43@W@$`'4-@W@(`'4'@W@,`'0&D+H!````A=)T0P^V
-MA;/^___!X`2-G`7H_O___W4,Z"+4__^#Q`3&0`0!BI6S_O__B%`(B%`)BQ.)
-M4`R+4P2)4!"+4PB)4!2+4PR)4!C^A;/^__^*C;/^__^+70PXBRT)```/@V'_
-M___IX!8``(MU#(!^*`</AA\,``"#[`R-A=C^__]0Z/C2___&A;+^__\`QH6S
-M_O__`(/$$(!^)@`/AKD```"0BH6S_O__BU4,.$(G#X2.````#[;`P>`$`="-
-MD-0'``"Y`````(.XU`<```!U$H-Z!`!U#(-Z"`!U!H-Z#`!T!;D!````A<ET
-M5P^VE;/^__^)T,'@!(M-#(V4"-0'``"+A=C^__\+`HF%V/[__XN%W/[__PM"
-M!(F%W/[__XN%X/[__PM""(F%X/[__XN%Y/[__PM"#(F%Y/[___Z%LO[___Z%
-ML_[__XJ=L_[__XMU##A>)@^'2/___P^VE;+^__^+?0P/MD<F@^@#T?@YP@^.
-ME@8``(I?)XB?Z`@``(N%V/[__XF'U`@``(N%W/[__XF'V`@``(N%X/[__XF'
-MW`@``(N%Y/[__XF'X`@``,:'Z0@```''A^0(````````5^A]TO__@\0$QD`$
-M`HA8"(A8"8N5V/[__XE0#(N5W/[__XE0$(N5X/[__XE0%(N5Y/[__XE0&(G'
-M_W4,Z$+2__^#Q`3&0`0&B86T_O__BQ_^@[P&```/MK.\!@``B<(IVH/J+,'Z
-M`HT$THT$PHT$PHT$PHG!P>$/`<B-!,+WV(B$<[P&``"*1P>(A'.]!@``B?"(
-M1P>+50R`>B8"#X?6`@``B=.!P]0(``!2Z-/1__^#Q`3&0`0%B5@(QD`4`(F%
-MP/[__XN-M/[__XL9_H.\!@``#[:SO`8``(G"*=J#ZBS!^@*-!-*-!,*-!,*-
-M!,*)P<'A#P'(C03"]]B(A'.\!@``B[VT_O__BD<'B(1SO08``(GPB$<'QH6S
-M_O__`(M5#(!Z)@`/AF`4``"*C;/^__^+70PX2R</A!X"``"+M<#^__^*1A0/
-MMM"(3!8,0(A&%`^VV<'C!(M]#(T$.XV(5`@``(N05`@``/?2B97(_O__BT$$
-M]]")A<S^__^+00CWT(F%T/[__XM!#/?0B874_O__C8P=Z/[__R.5V/[__XD1
-MBX7<_O__(X7,_O__B4$$BX7@_O__(X70_O__B4$(BX7D_O__(X74_O__B4$,
-MN@````"#.0!U$H-Y!`!U#(-Y"`!U!H-Y#`!T!;H!````A=(/A)0````/MH6S
-M_O__P>`$C9P%Z/[___]U#.AST/__@\0$QD`$`8J5L_[__XA0"(A0"8L3B5`,
-MBU,$B5`0BU,(B5`4BU,,B5`8BXW`_O__BQG^@[P&```/MK.\!@``*=B#Z"S!
-M^`*-%,"-%-"-%-"-%-")T<'A#P'*C130]]J(E'.\!@``B[W`_O__BD<'B(1S
-MO08``(GPB$<'#[:%L_[__\'@!(M5#(V$$-0'``"Z`````(,X`'42@W@$`'4,
-M@W@(`'4&@W@,`'0%N@$```"%T@^$DP````^VG;/^__^)V,'@!(M-#(V<"-0'
-M``!1Z)_/__^#Q`3&0`0"BI6S_O__B%`(B%`)BQ.)4`R+4P2)4!"+4PB)4!2+
-M4PR)4!B)QXL8_H.\!@``#[:SO`8``(N5M/[__RG:@^HLP?H"C032C03"C03"
-MC03"B<'!X0\!R(T$PO?8B(1SO`8``(I'!XB$<[T&``")\8A/!_Z%L_[__XJ=
-ML_[__XMU##A>)@^'N/W__^D3$@``QH6S_O__`(M]#(!_)@`/AO\1``"-=@"*
-MA;/^__^+50PX0B</A(X"```/MOC!YP2-!!>-B%0(``"+D%0(``#WTHF5R/[_
-M_XM!!/?0B87,_O__BT$(]]")A=#^__^+00SWT(F%U/[__XV,/>C^__\CE=C^
-M__^)$8N%W/[__R.%S/[__XE!!(N%X/[__R.%T/[__XE!"(N%Y/[__R.%U/[_
-M_XE!#(M=#('#U`@``/]U#.A7SO__@\0$QD`$`XE8"(J-L_[__XA(#(F%P/[_
-M_XNUM/[__XL>_H.\!@``#[:SO`8``(G"*=J#ZBS!^@*-!-*-!,*-!,*-!,*)
-MP<'A#P'(C03"]]B(A'.\!@``BY6T_O__BD('B(1SO08``(GQB$H'C80]Z/[_
-M_[H`````@S@`=1*#>`0`=0R#>`@`=0:#>`P`=`6Z`0```(72#X24````#[:%
-ML_[__\'@!(V<!>C^____=0SHHLW__X/$!,9`!`&*E;/^__^(4`B(4`F+$XE0
-M#(M3!(E0$(M3"(E0%(M3#(E0&(N-P/[__XL9_H.\!@``#[:SO`8``"G8@^@L
-MP?@"C13`C130C130C130B='!X0\!RHT4T/?:B)1SO`8``(N]P/[__XI'!XB$
-M<[T&``")\(A'!P^VA;/^___!X`2+50R-A!#4!P``N@````"#.`!U$H-X!`!U
-M#(-X"`!U!H-X#`!T!;H!````A=(/A)8````/MIVS_O__B=C!X`2+30R-G`C4
-M!P``4>C.S/__@\0$QD`$`HJ5L_[__XA0"(A0"8L3B5`,BU,$B5`0BU,(B5`4
-MBU,,B5`8B<>+&/Z#O`8```^VL[P&``"+E;3^__\IVH/J+,'Z`HT$THT$PHT$
-MPHT$PHG!P>$/`<B-!,+WV(B$<[P&``"*1P>(A'.]!@``B?&(3P>-=@#^A;/^
-M__^*G;/^__^+=0PX7B8/ATC]___I/P\``(M]#(I?)U?H*\S__\9`!`&(6`B(
-M6`F+E=C^__^)4`R+E=S^__^)4!"+E>#^__^)4!2+E>3^__^)4!B)A;S^__^*
-M7R>)/"3H[,O__X/$!,9`!`*(6`B(6`F+E=C^__^)4`R+E=S^__^)4!"+E>#^
-M__^)4!2+E>3^__^)4!B)Q_]U#.BQR___@\0$QD`$!HF%M/[__XL?_H.\!@``
-M#[:SO`8``(G"*=J#ZBS!^@*-!-*-!,*-!,*-!,*)P<'A#P'(C03"]]B(A'.\
-M!@``BD<'B(1SO08``(GPB$<'QH6S_O__`(M5#(!Z)@`/AE$.``"0BHVS_O__
-MBUT,.$LG#X1J`P``#[;!P>`$`=B-D-0'``"Y`````(.XU`<```!U$H-Z!`!U
-M#(-Z"`!U!H-Z#`!T!;D!````A<D/A"\#``#^C;+^__^*A;+^__^+50R*4B:-
-M-!`/MIVS_O__B=^)V,'@!(M-#(V<"-0'``!1Z,C*___&0`0!BI6S_O__B%`(
-MB?&(2`F+$XE0#(M3!(E0$(M3"(E0%(M3#(E0&(F%Q/[__XM=#(I;)XB=E?[_
-M_\'G!(M%#(V<!]0'``")!"3H>,K__\9`!`2*E97^__^(4!B)\8A(&8L3B5`(
-MBU,$B5`,BU,(B5`0BU,,B5`4B87`_O__BQC^@[P&```/MK.\!@``BY6\_O__
-M*=J#ZBS!^@*-!-*-!,*-!,*-!,*)P<'A#P'(C03"]]B(A'.\!@``BY7`_O__
-MBD('B(1SO08``(GQB$H'BQK^@[P&```/MK.\!@``BY7$_O__*=J#ZBS!^@*-
-M!-*-!,*-!,*-!,*)P<'A#P'(C03"]]B(A'.\!@``BY7`_O__BD('B(1SO08`
-M`(GQB$H'B[6T_O__BQ[^@[P&```/MK.\!@``BY7`_O__*=J#ZBS!^@*-!-*-
-M!,*-!,*-!,*)P<'A#P'(C03"]]B(A'.\!@``BY6T_O__BD('B(1SO08``(GQ
-MB$H'BUT,BELGB)V4_O__BW4,C9PWU`<``(DT).@RR?__QD`$!(J5E/[__XA0
-M&(J-L_[__XA(&8L3B5`(BU,$B5`,BU,(B5`0BU,,B5`4B87`_O__BQC^@[P&
-M```/MK.\!@``BY6\_O__*=J#ZBS!^@*-!-*-!,*-!,*-!,*)P<'A#P'(C03"
-M]]B(A'.\!@``BY7`_O__BD('B(1SO08``(GQB$H'B[6T_O__BQ[^@[P&```/
-MMK.\!@``BY7`_O__*=J#ZBS!^@*-!-*-!,*-!,*-!,*)P<'A#P'(C03"]]B(
-MA'.\!@``BY6T_O__BD('B(1SO08``(GQB$H'BUT,C;P?U`<``(D<).A%R/__
-M@\0$QD`$`HJ5L_[__XA0"(A0"8L7B5`,BU<$B5`0BU<(B5`4BU<,B5`8B<>+
-M&/Z#O`8```^VL[P&``"+E;3^__\IVH/J+,'Z`HT$THT$PHT$PHT$PHG!P>$/
-M`<B-!,+WV(B$<[P&``"*1P>(A'.]!@``B?&(3P>)]OZ%L_[__XJ=L_[__XMU
-M##A>)@^';/S__^FW"@``C78`BWT,BD<H.D<G#X6Z````QH6S_O__`(!_)@`/
-MAI0*``"*A;/^__^+50PX0B=T?@^VP,'@!`'0C9#4!P``N0````"#N-0'````
-M=1*#>@0`=0R#>@@`=0:#>@P`=`6Y`0```(7)=$</MIVS_O__B=C!X`2+30R-
-MG`C4!P``4>@KQ___@\0$QD`$`HJ5L_[__XA0"(A0"8L3B5`,BU,$B5`0BU,(
-MB5`4BU,,B5`8D/Z%L_[__XJ-L_[__XM=##A+)@^'7/___^GK"0``@^P,C878
-M_O__4.@0QO__@\00_W4,Z,G&__^#Q`3&0`0&B86T_O__QH6S_O__`(MU#(!^
-M)@`/AE0!``"*A;/^__^+50PX0B</A"H!```/ML#!X`0!T(V0U`<``+D`````
-M@[C4!P```'42@WH$`'4,@WH(`'4&@WH,`'0%N0$```"%R0^$[P````^VE;/^
-M__^)T8G0P>`$BUT,C908U`<``(N%V/[__PL"B878_O__BX7<_O__"T($B87<
-M_O__BX7@_O__"T((B87@_O__BX7D_O__"T(,B87D_O__BH6S_O__.$,H#X2-
-M````B<O!XP2+50R-G!/4!P``4NCDQ?__@\0$QD`$`HJ-L_[__XA("(A("8L3
-MB5`,BU,$B5`0BU,(B5`4BU,,B5`8B<>+&/Z#O`8```^VL[P&``"+E;3^__\I
-MVH/J+,'Z`HT$THT$PHT$PHT$PHG!P>$/`<B-!,+WV(B$<[P&``"*1P>(A'.]
-M!@``B?.(7P>0_H6S_O__BH6S_O__BU4,.$(F#X>L_O__BTT,BEDG4>A(Q?__
-M@\0$QD`$`HA8"(A8"8N5V/[__XE0#(N5W/[__XE0$(N5X/[__XE0%(N5Y/[_
-M_XE0&(G'BQC^@[P&```/MK.\!@``BY6T_O__*=J#ZBS!^@*-!-*-!,*-!,*-
-M!,*)P<'A#P'(C03"]]B(A'.\!@``BD<'B(1SO08``(GSB%\'BW4,#[9>*(G>
-MB=C!X`2+?0R-G#A4"```BP/WT(F%R/[__XM3!/?2B97,_O__BTL(]]&)C=#^
-M__^+6PSWTR.%V/[__XF%R/[__XN]W/[__R'ZB97,_O__BY7@_O__(=&)C=#^
-M__^+A>3^__\APXF=U/[__\'F!(M-#(VT#E0(``"+C=C^__\C#HF-V/[__R-^
-M!(F]W/[__R-6"(F5X/[__R-&#(F%Y/[__[@`````A<EU$87_=0V%TG4)@[WD
-M_O__`'0%N`$```"%P`^$N@(``(M=#(I#)XB#Z`@``(N%V/[__XF#U`@``(N%
-MW/[__XF#V`@``(N%X/[__XF#W`@``(N%Y/[__XF#X`@``,:#Z0@```''@^0(
-M````````QH6S_O__`(![)@`/AEP"``"!P]0(``")G9#^__^*A;/^__^+50PX
-M0BAU=E+H?,/__X/$!,9`!`.+C9#^__^)2`B*G;/^__^(6`R+M;3^__^+'OZ#
-MO`8```^VL[P&```IV(/H+,'X`HT4P(T4T(T4T(T4T(G1P>$/`<J-%-#WVHB4
-M<[P&``"+O;3^__^*1P>(A'.]!@``B?"(1P?IM0$``)"*E;/^__^+30PX42</
-MA*(!```/MOK!YP2-C#WH_O__BUT,C90?5`@``(L"]]")`8M"!/?0B4$$BT((
-M]]")00B+0@SWT(E!#(L!(X78_O__B0&+000CA=S^__^)002+00@CA>#^__^)
-M00B+00PCA>3^__^)00Q3Z)'"__^#Q`3&0`0#B[60_O__B7`(BI6S_O__B%`,
-MB87`_O__BXVT_O__BQG^@[P&```/MK.\!@``B<(IVH/J+,'Z`HT$THT$PHT$
-MPHT$PHG!P>$/`<B-!,+WV(B$<[P&``"+E;3^__^*0@>(A'.]!@``B?&(2@>-
-MA#WH_O__N@````"#.`!U$H-X!`!U#(-X"`!U!H-X#`!T!;H!````A=(/A)<`
-M```/MH6S_O__P>`$C9P%Z/[___]U#.C6P?__@\0$QD`$`8J5L_[__XA0"(A0
-M"8L3B5`,BU,$B5`0BU,(B5`4BU,,B5`8BXW`_O__BQG^@[P&```/MK.\!@``
-M*=B#Z"S!^`*-%,"-%-"-%-"-%-")T<'A#P'*C130]]J(E'.\!@``B[W`_O__
-MBD<'B(1SO08``(GPB$<'C78`_H6S_O__BI6S_O__BTT,.%$F#X>P_?__N```
-M``"#O<C^__\`=1N#O<S^__\`=1*#O=#^__\`=0F#O=3^__\`=`6X`0```(7`
-M#X02!```BW4,BEXG5NC^P/__@\0$QD`$`8A8"(A8"8N5R/[__XE0#(N5S/[_
-M_XE0$(N5T/[__XE0%(N5U/[__XE0&(F%O/[__\>%N/[__P````#&A;/^__\`
-M@'XF``^&M`,``(J%L_[__XM5##A"*`^$B@,``#A")P^$@0,```^VP,'@!(V<
-M!>C^__\!T(V(U`<``(N5R/[__R.0U`<``(D3BX7,_O__(T$$B4,$BX70_O__
-M(T$(B4,(BX74_O__(T$,B4,,N`````"#.P!U$H-[!`!U#(-["`!U!H-[#`!T
-M!;@!````A<`/A!$#``"+30R*22B(C8_^__\/MH6S_O__P>`$C9P%Z/[___]U
-M#.C\O___@\0$QD`$`8J5L_[__XA0"(J-C_[__XA("8L3B5`,BU,$B5`0BU,(
-MB5`4BU,,B5`8B87$_O__@[VX_O__`'11BQC^@[P&```/MK.\!@``BY6X_O__
-M*=J#ZBS!^@*-!-*-!,*-!,*-!,*)P<'A#P'(C03"]]B(A'.\!@``B[W$_O__
-MBD<'B(1SO08``(GPB$<'BU4,BE(GB)6._O__BTT,BDDHB(V-_O__#[:]L_[_
-M_\'G!(V</>C^____=0SH.[___X/$!,9`!`2*E8[^__^(4!B*C8W^__^(2!F+
-M$XE0"(M3!(E0#(M3"(E0$(M3#(E0%(F%P/[__XF%N/[__XL8_H.\!@``#[:S
-MO`8``(N5O/[__RG:@^HLP?H"C032C03"C03"C03"B<'!X0\!R(T$PO?8B(1S
-MO`8``(N5P/[__XI"!XB$<[T&``")\8A*!XL:_H.\!@``#[:SO`8``(N5Q/[_
-M_RG:@^HLP?H"C032C03"C03"C03"B<'!X0\!R(T$PO?8B(1SO`8``(N5P/[_
-M_XI"!XB$<[T&``")\8A*!XNUM/[__XL>_H.\!@``#[:SO`8``(N5P/[__RG:
-M@^HLP?H"C032C03"C03"C03"B<'!X0\!R(T$PO?8B(1SO`8``(N5M/[__XI"
-M!XB$<[T&``")\8A*!XM=#(I;)XB=C/[__XV</>C^____=0SHZ[W__X/$!,9`
-M!`2*E8S^__^(4!B*C;/^__^(2!F+$XE0"(M3!(E0#(M3"(E0$(M3#(E0%(F%
-MP/[__XL8_H.\!@``#[:SO`8``(N5O/[__RG:@^HLP?H"C032C03"C03"C03"
-MB<'!X0\!R(T$PO?8B(1SO`8``(N]P/[__XI'!XB$<[T&``")\(A'!XN5M/[_
-M_XL:_H.\!@``#[:SO`8``(GZ*=J#ZBS!^@*-!-*-!,*-!,*-!,*)P<'A#P'(
-MC03"]]B(A'.\!@``BXVT_O__BD$'B(1SO08``(GSB%D'D/Z%L_[__XJ%L_[_
-M_XM5##A")@^'3/S__X/L"/]U#/]U".@VTO__N`````"-9?1;7E_)PY!5B>57
-M5E.#[`R+10B#N"PP```##X</`0``!>PO``"+50@Y@NPO```/A/L```")1>R+
-MBNPO``")3?"-=@"+7?"+`XE%\(M3!(L#B5`$B0*)&XE;!(M#'/9``00/A8@`
-M``")W[X`````@'LF`'8FC78`@^P,B?(/ML+!X`2-A#C4!P``4.B0N___@\00
-M1HGQ.$\F=]V+50B!PN0O``"+30B+@>0O``")6`2)`XE3!(F9Y"\``(M#'(-X
-M)`!T8(/L!/]P*(M#'/]P)%'H_/___XM#',=`)`````"+10C_B#`P``"#Q!#K
-M-HGV@^P(4_]U".B8T___@\00A<!T+8M5"('"["\``(M-"(N!["\``(E8!(D#
-MB5,$B9GL+P``D(M%[#E%\`^%%/___XUE]%M>7\G#58GE4X/L!(M%"(M8"(.[
-M+#````!U#HV#["\``#F#["\``'0S@^P,4^C\____B1PDZ/S___^)'"3H_/__
-M_X/$$(.[+#````!UVXV#["\``#F#["\``'7-N`````"+7?S)PXUV`%6)Y5=6
-M4X/L!(M%"(M`"(E%\(N(["\```7L+P``.<@/A($```"-=@"+50@Y41QU:(G/
-MLP"`>28`=D&)]@^VP\'@!`'XC9#4!P``O@````"#N-0'````=12#>@0`=0Z#
-M>@@`=0B#>@P`=`>)]KX!````A?9U&4,X7R9WP;@`````A<!T%+@!````Z:T`
-M``"X`0```.OKC78`BPF+1?`%["\``#G(=8*+1?"+B/0O```%]"\``#G(='Z-
-M0?B+50@Y4!QU98G#O@````"`>"8`=D&)\@^VPL'@!`'8C9#4!P``OP````"#
-MN-0'````=1*#>@0`=0R#>@@`=0:#>@P`=`6_`0```(7_=1A&B?`X0R9WO[@`
-M````A<!T#K@!````ZQJX`0```.ONBPF+1?`%]"\``#G(=8*X`````(/$!%M>
-M7\G#D%6)Y5=64X/L#(M="(M]$(MS"(-[)`!U#5/HGO[__X/$!(7`=0N#[`A7
-M5O]5#.L9D(M%#(E#)(E[*/^&,#```(/L#%;H_/___XUE]%M>7\G#58GE5E.+
-M=0B+70R`>Q$`=03&0Q$!@'L1`709@'L1!G03@'L1`W0-]D,0('4'QD,1"XUV
-M`/]S+%;HQ[C__XD<)/]S'%;H_/___XL#@\00@W@8`'08@^P$@\`84&@`````
-M5NC\____@\00ZUV0@[XT,````'0KC8;D+P``.8;D+P``=!V#[`2-AC0P``!0
-M:`````!6Z/S___^#Q!#K*XUV`(&^*#```/\```!W'(V&Y"\``#F&Y"\``'4.
-M@^P,5NC\____@\00B?:-9?A;7LG#D%6)Y5=64X'LS````(M]$(M%#(M0((L*
-MB8TT____#[9!/$B)A4C___]FBTH(9HF-5O___XM"3(F%4/___P^V2DJ)C43_
-M__\/MD)+B85`____9HM*4F:)C53___]FBUI0@WH4`'0-9@^V0A"#X`$[111U
-M.+D`````@WH8``^$?P(``(V%6/___XF%./____]U%%!2_W4(_U(8@\00N0``
-M``"%P`^$6`(``.L)BU(4B94X____O@````"+50R+`@^V0`.)A3S___^Y````
-M``^WPP.%4/___XF%3/___XN%//___SF%1/___P^%%P$``(7)=%*+A4S___\I
-MR(G#P>,)=$.#?10`=`>A`````.L1@^P,_S4`````Z/S___^#Q!")!V;'1P8`
-M`('[`!````^&OP```&;'1P0`$(/'"('K`!```'6]#[>=5/___\'C"8UV`(N-
-M./___P^W402%TG4%N@```0")T"GP.=AR,8GPBXTX____`P&)!V:)7P1FQT<&
-M``"#QP@!WCG6=4:^`````(/!"(F-./___^LVB?:)\(N-./___P,!B0>)T&8I
-M\&:)1P1FQT<&``"#QPB)T"GP*<.#P0B)C3C___^^`````.N`#[>-5/___P.-
-M3/___XM5#`^W0@@#0@0YR`^$!@$``.M/9HE?!(/'".E&____C78``=[K/`^W
-MG53____!XPF)]HN%./___P^W4`2%TG4%N@```0")T"GP.=AWU8G0*?`IPX.%
-M./___PB^`````.O1D(N55/___V8IE5;___\/A*(```"[`````/^%1/___XN%
-M0/___SF%1/___W4(_X5$____B?:+E33___\/MD(\.X5$____=4*#O4#___\`
-M=`C_C4#____K#(N%2/___XF%0/___X.]0/___P`/E,`/ML")A43___^+E33_
-M__\/MT)(`850____B?:+A33___]FBT!(9HF%5/___V:+E5;___]F.=`/AN/]
-M__]FB954____Z=?]__]FQT?^`("Y`0```(G(C67T6UY?R<.)]E6)Y593BW4(
-MBU4,BUH@@'H1`70&BD(1B$,1@^P(4E;H_/___XI#24"(0TF#Q!`Z0TAU+8![
-M$0!U!,9#$0&`>Q$!=!"`>Q$&=`J`>Q$#=`3&0Q$+@^P(4U;_4QR#Q!")]HUE
-M^%M>R<.058GE5U93@^PLBT4,BP")1>R+50SV0A`$#X3T`0``QT7P`````(!Z
-M1``/A'L!``"+3?"+70R+3(LPB4WHBIDL"0``B%WG.)DM"0``#X)&`0``C78`
-MBD7GBU7H.$(G#X0?`0``#[;`B<+!X@2+3>B-E`K4!P``B57@#[>4@0P)```/
-MM[2!#@D``+\`````C78`@_H?=U.)^P^VPXM-X(T<@8E5W+D@````*=$Y\78"
-MB?&#^2!U"\<#_____^L2C78`N`$```#3X$B*3=S3X`D#N"`````IT#GP<QF-
-M=!;@N@````#K!HUV`(/J($>)^X#[`W:=#[9%YXG"P>`$BTWHC80(5`@``(E%
-MV`^WG)$,"0``#[>TD0X)``"_`````(GV@_L?=U.)^@^VPHM-V(T4@8E=U+D@
-M````*=DY\78"B?&#^2!U"\<"_____^L2C78`N`$```#3X$B*3=33X`D"N"``
-M```IV#GP<QB-=![@NP````#K!HUV`(/K($>)^#P#=I[^1>>*5>>+3>@XD2T)
-M```/@[W^____1?"+70P/MD-$.T7P#X^%_O__BT4(@[C@+P````^$/@,``,=%
-M\`````"+50R`>D0`=#6)PH'"["\``(M-\(M=#(M,BS#'@00)````````BT($
-MB4H$B1&)002)"/]%\`^V0T0[1?!_TX/L"/]U#/]U".BO^?__Z1T#``"+70SV
-M0Q`"#X39`@``BT4(!3@P``")1="+?0B!QU@P``"+5>SV0DH##X6X`@``@^P$
-M:A!J`%?H_/___\=%\`````"#Q!"`>T0`#X3_````BTWPBUT,BTR+,(E-Z(G.
-ML0"`?B8`=D8/ML'!X`0!\(V0U`<``+L`````@[C4!P```'42@WH$`'4,@WH(
-M`'4&@WH,`'0%NP$```"%VW0)N`$```#K#8GV03A.)G>ZN`````"%P`^%+0(`
-M`(M%Z(J`+`D``(A%YXM5Z#B"+0D``')KBDWGBUWH.$LG=$\/MM%F@SQ7`'4?
-M#[>$DPP)```#0R"+3=")!)%FBX23#@D``&:)!%?K)@^V5>>+7>AFBX23#`D`
-M`&8#0R!F`X23#@D``(M-T&8K!)%FB017_D7GBEWGBT7H.)@M"0``<Y7_1?"+
-M50P/MD)$.T7P#X\!____BTT,_W$L_W4(Z%*Q___'1?``````@\0(BUT,@'M$
-M``^$S````(M-"('!Y"\``(M%\(M5#(M$@C")1>B#??``=4J)PHM`((M=#(E#
-M3(J"+`D``(A#2HI")XA#2P^V@BP)``!FBX2"#`D``&:)0U`/MH(L"0``9HN$
-M@@X)``!FB4-2QD-(`,9#20#K0XM%Z(M0((M=##M33'8UBUWL#[=#2/?8(<*+
-M70PC0TPYPG4@BT7H#[:0+`D``&:+0U*+7>AF`X23#@D``(M5#&:)0E*+002+
-M7>B)602)"XE#!(D8_T7PBU4,#[9"1#M%\`^//?___XM-"(.Y-#````!T&8/L
-M!(G(!30P``!0:`````!1Z/S___^#Q!#&1><`BUWL@'L\``^&J`````^V=>=F
-M@SQW`'14BT7LBURP5(/L#/]U".C\____BU4,_D)(QD`*(&:+%'=FB5`(BTW0
-MBQ2QB5`$@$@0`L=`&.1F``#'0!SP:0``B1B+50R)4""#Q`A0_W4(_U,T@\00
-M_D7GBDWGBUWL.$L\=Y/K.8GVQT7P`````(M%#(!X1`!T)X/L"(M5\(M-#/]T
-MD3#_=0CH`\C__X/$$/]%\(M=#`^V0T0[1?!_V8UE]%M>7\G#D%6)Y5=64X/L
-M/(M%#(L`B47PQT7D`````&:+4$AFB57J9H'Z@`!V!F;'1>J``(M-#/9!$"!T
-M2`^W1>I0BUT0_W,(_W7P_W4(Z(RR__^)1>2#Q!"X_____X-]Y``/A#,#``"+
-M10S&0$0!BU7DB5`PB8($"0``N`````#I%@,``(M-$`^V41*+`8G3N@````#W
-M\XA5XXM%\`^V4#R+002)T;H`````]_&+71"*0Q(HT(A%XCA%XX!=X_^+1?!F
-MBU!(C4+_9HM+#"'!B==F*<]F.7L0<P1FBWL0BU4,QD)$`,9"10#'1=P!````
-M#[==ZHE=V/?;B5W4#[?!BU40`T((B47LBTT(@<'L+P``B4W,BUT(@</D+P``
-MB5W(C78`9HM%ZD@C1>QFBUWJ9BG#9CG[=@*)^XMUU"-U[(-]W``/A%X!``#'
-M1=P`````9HM5ZHM%\&8Y4$AV+KH`````BTT,#[9!1(G!@_@`?AN-=@"+10R+
-M1)`PB47D.7`@#X0A`0``0CG1?^C_==A6_W7P_W4(Z$6Q__^)1>2#Q!"%P`^%
-MRP```(M5#/Y*1(!Z1/\/A*\```"+3<R)3="+?<B+70P/MD-$BT2#,(E%Y,>`
-M!`D```````")QK$`@'@F`'8_#[;!P>`$`?"-D-0'``"[`````(.XU`<```!U
-M%(-Z!`!U#H-Z"`!U"(-Z#`!T!XGVNP$```"%VW4D03A.)G?!N`````"%P'0<
-MBU70BT($BTWDB4H$B1&)002)".L7N`$```#KX(M'!(M=Y(E?!(D[B4,$B1B+
-M10S^2$2`>$3_#X5:____N/_____I*@$``(GVBU4,BE)$B%7##[;*BU7DBT4,
-MB52(,(I%PT"+30R(042)B@0)``#&@BP)```(QH(M"0````^V5>.+1>QF*?"+
-M3>1FB821#`D``&:)G)$."0``BD7C.($L"0``=@:(@2P)``"*5>.+3>0XD2T)
-M``!S!HB1+0D``(M%$&8I6!!F*=]T$@^WPP%%[,=%W`$```#I'/[__XM5$&:#
-M>A``#X2!````N0````#^1>.*7>(X7>-U`_Y%XXI%XXM5\#A"/'4R@'WB`'0%
-M_DWBZPF+71"*6Q*(7>*`?>(`#Y1%XXM5\`^W0DB+71`!0PC'1=P!````ZQ1F
-MBU7JBT7P9CE02'8'QT7<`0```(M=$&:+>Q"+1?!F.WA(#X9O_?__9HMX2.EF
-M_?__N`````"-9?1;7E_)PU6)Y8/L"(M5"(M%#(-]$`!T&L9`$0N#[`10:!1F
-M``!2Z/S___^#Q!#K#XGV@^P(4%+H]O;__X/$$,G#D%6)Y5=64X/L3(MU#(L^
-M]D<!!'48QD81`H/L!%;_=AS_=0CH_/___^D0`@``@W\L`'4B:@?_=0CH>*C_
-M_XE'+,<$)``0``!J`%#H_/___X/$$(UV`(I&$*@@#X6U````J`9U&<9&$0&#
-M[`16_W8<_W4(Z/S____IP0$``)#V1A`$="J#?R0`="2#[`3_=RC_=R3_=0CH
-M_/___\=')`````"+10C_B#`P``"#Q!"+5@2)5=1FBT8(9HE%V(I'/$B(1=H/
-MMD\^B=#3Z(E%R`^V7=JZ`````/?SB47,#[9//M/@B470#[='2`^OPXE%M`^W
-M3=@#3@0/KUW0*=F-3`'_B<BZ`````/=UM(G!#[='2`^OR.LND/9'2@%T&L9&
-M$0:#[`16_W8<_W4(Z/S____I"@$``(GV#[=/2(G(#Z]&!(E%T(M%T(T4`5)0
-M5_]U".BZJO__B48L@\00A<!U',=&*`````"#[`A6C4<84.C\____Z<<```"-
-M=@"#[`2-1<A05O]U".B@^O__@\00A<!T;?]V+/]U".@6JO__BT4(BY!4(@``
-M@\0(.9!8(@``=">Y(0(``)"+10B!O-!,$0```````'140HG0F??YBT4(.9!8
-M(@``==_'1B@`````@^P(5HM%"`4T,```4.C\____@\0$_W4(Z/S____K09#V
-M1A`$="N#[`1H+',``%;_=0CHJ:W__X/$$.LDQD81!8/L!%;_=AS_=0CH_/__
-M_^L/@^P(5O]U".BW]/__@\00C67T6UY?R<-5B>53@^P$BUT,@WLL`'1.@^P,
-M4^C\____N@````"#Q!"-=@"+0RR+!)"%P'00B?;'0!``````BT`4A<!U\D*!
-M^O\#``!VW6H'_W,L_W4(Z$"F___'0RP`````@\0,BUW\R<.058GE5U93@^P0
-MBWT(QX?@+P```````(V'Y"\``(F'Y"\``(F'Z"\``(V'["\``(F'["\``(F'
-M\"\``(V']"\``(F']"\``(F'^"\``*$`````P>`,NJNJJJKWXHG3P>L/4V@P
-M"0``5^CDI?__B<:#Q`R#PR!3:A!7Z-.E__\!QH/$#*$`````P>`,P>@.4&I$
-M5^BZI?__`<:#Q`QJ"&@`$```5^BHI?__`<:)/"3H_/___Z,`````@\001CLU
-M`````',QD(/L#%?H_/___XG"@\00A<!T'HN')#```(D"B9<D,```_X<H,```
-M1CLU`````'+1D(UE]%M>7\G#58GE5U93@^P,BWT(BW4,BUX$#[9.`[@!````
-MB<+3XH!F`?O'0U``````]D-*`G0,9H-[3`!T$NL*C78`9H-[3`!T+6:%4TQU
-M)V8)4TR`8P'[@WLX`'0-@^P(4U?_4SB#Q!")]H!C2A_ILP```(UV`(I#2F:)
-M4TS^2SW'0U``````@\@%B$-*A,!Y#8/@?XA#2NF*````B?:`8TJ?@^P(4U?H
-M_/___XG"@\00A<!T3,8``XI&`XA"`XE:!(!*`0BA#````(E"-*$@````B4(X
-MQT8$``````^V0@.)5(-4BD-*@^#^@\@*B$-*_D,]@^P(4FH'Z/S___^#Q!"#
-M>P0`=`R-=@"+6P2#>P0`=??H_/___XE#0(/L#%/H_/___XGVC67T6UY?R<-5
-MB>53BTT(NP````"+50Q*@_K_=`^-=@`/M@$!PT%*@_K_=?0/ML-;R<.)]E6)
-MY593BW4(@#X#=D:[`````(!^/`!T-8-\GE0`="6+3)Y4BE8!@^(!BD$!@^#^
-M"="(00&#[`S_=)Y4Z/S___^#Q!"00P^V1CPYV'_+@&9*^^L)@^P,5N@(````
-MC67X6U[)PY!5B>575E.![!P"``#'A>3]__\`````C;7H_?__BT4(@'@\``^$
-M.@(``(MX!(7_=`^#?P0`=`F)O>3]__^+?P2#[`1H``(``&H`5NC\____QP;S
-M%GA:@\00N`````"%_W0#BT=`B48$BU4(@'I$`'0*@$X)`HI"1XA&"HM%"(I0
-M0(C3@^,!BH:3````@^#\B-&#X0()V`G(B-.#XP2#X/.(T8/A"`G8"<B(TX/C
-M$(/@SXC1@^$@"=@)R(C1@^%`@^`_@^*`"<@)T(B&DP```(M5"/9"`0%T!(!.
-M"0&P`(7_=`R#O>3]__\!&<"#P`*(1@N%_P^$N@```(M'#(E&#,>&E```````
-M``"*!XA&$(I'/(A&$8.]Y/W__P!T"XN5Y/W__XI"`^L)BU4(BD(#C78`B$82
-MBD<^B$83]D=*`G0$@$X4`6:+1TQFB486BT=0B488QX:8`````````(/L!&H0
-MC4=T4(U&+%#H_/___X/$#&H$C8>$````4(U&/%#H_/___X/$#&H0C8?(````
-M4(V&@````%#H_/___X/$#&I`C8>(````4(U&0%#H_/___X/$$(.]Y/W__P!T
-M7HN5Y/W__XM"#(E&',>&G`````````"*`HA&((I"/(A&(8M5"(I"`XA&(HN5
-MY/W__XI"/HA&(_9"2@)T!(!.)`&+E>3]__]FBT),9HE&)HM"4(E&*,>&H```
-M``````"%_W4+BU4(B@*(1A"-=@!HD````%;H_/____?8B$8(:``"``!6Z/S_
-M___WV(B&D````%9J,(M5"(M"3(/H"E")T(/`/%#H_/___XUE]%M>7\G#D%6)
-MY8M-"(G(@#D#=B^X`````(!Y/`!T$`^V43R#?(%4`'4,0#G"?_2X`````.L-
-M@^P,_W2!5.C\____D,G#B?95B>564XMU"(`^`W8ONP````"`?CP`="B#?)Y4
-M`'01@^P,_W2>5.C\____@\00B?9##[9&/#G8?]_K!9"`9@'^C67X6U[)PY!5
-MB>6+30B+40BX`````(GV.0R"=0O'!((`````ZPB)]D"#^`=VZLG#58GEBT4,
-M@&`!^U#H_/___\G#B?95B>575E.![!P$``"+?0B-M>C]__^)^XI'`:@$#X09
-M`0``@#\"#X80`0``@^#]B$<!@^P$:``"``!J`%;H_/___X/$$(`_`W1XC87H
-M^___D(M;5(`_"'5DA=MT!V:#?TP!=5F)P\>%Y/O__P$````/MD<\@_@!?EI3
-M:B!J`(N5Y/O__XM$EU2#P#Q0Z/S___^Z`````(/$$(L$EC,$DXD$ED*#^G]^
-M\?^%Y/O__P^V1SP[A>3[__]_O^L7D(`[`W6/5FH@:@"-0SQ0Z/S___^#Q!!F
-M@;[^`0``5:IU7H.^R@$```!T&X%^!DQ)3$]T#(&^=@$``$=254)U!H!/`0+K
-M.L>%Y/O__P````"+A>3[___!X`0!\("XO@$``(!U#8.XR@$```!T!(!/`0+_
-MA>3[__^#O>3[__\#?M"-9?1;7E_)PY!5B>564XM-"(N92!$``(U#`;XA`@``
-MF??^B=:+10R)1-D\BY%($0``BT40B4310(FQ2!$``%M>R<.-=@!5B>575E.#
-M[`R+=0B+AD01```[AD@1``!T-K\A`@``D(N&1!$``(M,QCR+7,9`0)GW_XF6
-M1!$``(/L"%-6_]&#Q!"+AD01```[AD@1``!UT(UE]%M>7\G#58GE5E.+30B+
-MF5@B``"-0P&^(0(``)GW_HG6BT4,B8393!$``(N16"(``(M%$(F$T5`1``")
-ML5@B``!;7LG#D%6)Y5=64X/L#(M="(N#5"(``#N#6"(``'16B?:+@U0B``"+
-MM,-,$0``B[S#4!$``$"Z(0(``(G1F??YB9-4(@``@[LL,````'0.@^P,4^C\
-M____@\00B?:#[`A74__6@\00BX-4(@``.X-8(@``=:R-9?1;7E_)PU6)Y5.#
-M[`2+50BX`````(-Z.`!T%XM:.(L#B4(X@^P$:E1J`%/H_/___XG8BUW\R<.-
-M=@!5B>6+30B+50R+03B)`HE1.,G#D%6)Y8M-"(M5#(,Y`'02BP&+0"2)0B2+
-M`8E0).L&C78`B5(DB1')PY!5B>575E.#[`R+=0R#/@!T-8L^D(L&BU@D.=AU
-M"\<&`````.L+C78`BQ:+0R2)0B2#[`A3_W4(_U,H@\00.?MT!8,^`'7.C67T
-M6UY?R<.)]E6)Y8/L"(M%"(`X"'4.@^P,4.C\____@\00B?;)PXGV58GE4X/L
-M!(M%"(M=#(M-$(M0"(`X"'40@^P$45-0Z/S____K"HUV`(/L"%%2_].+7?S)
-MPU6)Y5=64X/L%(M%"(I5$(A5\XMX*(H`B$7R#[;84U?H_/___X/$#&H`4U?H
-M_/___X/$#&H`:@!J`&H`:@-35^C\____@\0@N@````"%P`^$F````/=%#```
-M`/`/E<`/MLB#[`R%R704N"0```"`??,@=`VX-````.L&B?8/MD7S4+A`````
-MA<EU"8M%#,'H&(/(0%"+70S!ZQ`/ML-0BU4,#[;&4`^V50P/M\*%R70)B=@E
-M`/\```G04&H!:@!H``$``/]U%%&`??,P#Y3`#[;`0%`/MEWR4U?H_/___XG&
-M@\0X4U?H_/___XGRB="-9?1;7E_)PXGV58GEBD4(B<*#X@.)T<'A#8V1`"``
-M`*@$=`J-@0`@`P#K"(GVC8(```(`R<-5B>564X/L'(MU"(I%#(A%]V@@H0<`
-MZ/S___^[(*$'`(/$$)"#[`@/MD7W4.BC____!1P!``")!"3_=@SH_/___X/$
-M$(3`>`BX`0```.LDD(/L#&@0)P``Z/S___^#Q!"!PQ`G``"!^S]+3`!VMK@`
-M````C67X6U[)PXGV58GE5E.+70B#[`1J``^V10Q0Z$'___^)QHU`"(D$)/]S
-M#.C\____@\0,:!@!``"-1@Q0_W,,Z/S___^#Q!"-9?A;7LG#C78`58GE@^P,
-M:@`/MD4,4.C^_O__@\`,B00DBT4(_W`,Z/S____)PXUV`%6)Y8M%#&:#?10`
-M=1/W11````#P=0JI`/\``'0+C78`N`$```#K&9!FA<!U"[@!````@WT(`70(
-MN`````"-=@#)PXGV58GE5U93BUT(BWT0BW44N0````"-4PJ+1A")`XM&%(E#
-M!(,^`'4"L0$/MD<$T>`)P6:)2PB+10R#N#@"````=`^#N#@"```"#X5&`0``
-MB?:#?@0!#X7&````9@^V1@^`S!%FB0*#P@)F#[9&#H#,$6:)`H/"`@^V1P3!
-MX`,E^````(#,$F:)`H/"`F8/MD8+@,P39HD"@\("9@^V1@B`S!-FB0*#P@)F
-M#[9&#(#,%&:)`H/"`F8/MD8)@,P49HD"@\("9@^V1@V`S!5FB0*#P@)F#[9&
-M"H#,%6:)`H/"`F;'`D`6@\("@SX`=1FP8(M-#(.Y.`(```(/A((!``"P)NE[
-M`0``L&&+30R#N3@"```"#X1I`0``L#;I8@$``(GV9@^V1@Z`S!%FB0*#P@(/
-MMD<$P>`#)?@```"`S!)FB0*#P@)F#[9&"(#,$V:)`H/"`F8/MD8)@,P49HD"
-M@\("9@^V1@J`S!5FB0*#P@*+1@@E````#\'H&`U`%@``9HD"@\("@SX!&<"#
-MX/N#Z#3I[0```)"#?@0!#X6&````9@^V1@^`S!)FB0*#P@)F#[9&#H#,$F:)
-M`H/"`F8/MD8+@,P39HD"@\("9@^V1@B`S!-FB0*#P@)F#[9&#(#,%&:)`H/"
-M`F8/MD8)@,P49HD"@\("9@^V1@V`S!5FB0*#P@)F#[9&"H#,%6:)`H/"`F;'
-M`D`6@\("@SX!&<"#X/"#P#7K79!F#[9&#H#,$F:)`H/"`F8/MD8(@,P39HD"
-M@\("9@^V1@F`S!1FB0*#P@)F#[9&"H#,%6:)`H/"`HM&""4````/P>@8#4`6
-M``!FB0*#P@*#/@$9P(/@_H/H-HUV`&8/ML`-`)?__V:)`EM>7\G#C78`58GE
-M5U93@^PDBU4(BUT4BW(,#[9%#(M$@B2+>#"-AP0!``!05NC\____B`.#Q`B-
-MAP@!``")1?!05NC\____9@^VP&:)0P*#Q`B-APP!``")1>Q05NC\____9@^V
-MP&:)0P2#Q`B-AQ`!``")1>A05NC\____9@^VP&:)0P:#Q`B-AQ0!``")1>10
-M5NC\____9@^VP&:)0PB#Q!"#?1`!#X6+````@^P(C8<@`0``B47<4%;H_/__
-M_XA%XX/$#(/(@`^VP%#_==Q6Z/S___^#Q`C_=?!6Z/S____!X`AF"4,"@\0(
-M_W7L5NC\____P>`(9@E#!(/$"/]UZ%;H_/___\'@"&8)0P:#Q`C_=>16Z/S_
-M___!X`AF"4,(@\0,#[9%XU#_==Q6Z/S___^#Q!")]H/L"(V'&`$``%!6Z/S_
-M__^(0PJ#Q`B-AQP!``!05NC\____B$,+C67T6UY?R<.)]E6)Y5=64X/L-(M%
-M$&:)1=8/ME4,B570BT4(BWR0)(M',(/`,%"+50C_<@SH_/___X/@'XT4Q0``
-M```IPHVTUT0"``"#Q!"#/@%T08U>%(U%V%#_<P3_==#_=0CH0/[___Z/40D`
-M`(/$#(U5V%)J``^W1=90_W,<:@'_==#_=RC_4QB#Q!A65^B,$0``C67T6UY?
-MR<-5B>575E.#[#R+51"*10R(1<_'1<@`````#[;`BTT(BW2!)&;'1<8``&:+
-M`F:)1="-3=!FBT("9HE!`HM"!(E!!&:+1=")PX/C'XT$W0`````IV(V<QD0"
-M``!FBT$"A,!T+L=%R`$```!FBT$")?\```!FB47&9H%-Q@`!QX9,"0``````
-M`,>&-`(```````"->Q2-1=!FBT`"J`1T&HU%V%#_=P0/MD7/4/]U".A:_?__
-M@\00C78`@SL!#X23````_HY1"0``@^P$C4784(U%T(M`!%`/MT7&4/]W'/]U
-MR`^V1<^)1<!0_W8H_U<8@\084U;HC!```(/$$(.^3`D```!T4(.^1`D```!T
-M1XN&1`D``(-X$`%U.X/L"/]UP/]U".A!$@``@\0(_[9$"0``5N@Z$P``@\00
-MA<!U%X/L!&H`_[9$"0``5N@R!@``@\00C78`C67T6UY?R<-5B>564XM="(/L
-M"&@L#0``_W,,Z/S___^)QH/$#(/(`5!H+`T``/]S#.C\____@\0(:"P-``#_
-M<PSH_/___X/$"&CP!`$`_W,,Z/S___^+50SWTB'0"T40@\0,4&CP!`$`_W,,
-MZ/S___^#Q`AH\`0!`/]S#.C\____@\0,5F@L#0``_W,,Z/S___^-9?A;7LG#
-MC78`58GE5U93@^P<BT4(BT`,B47LQT7H``````^V10R-#(4`````B$WSBD40
-M"$7S#[9%\XM5"(MT@B2#?1@`=&N%]@^$S0```(N>W`D``(/L!(M%%,'@`PM&
-M)%"+1C"#P"10_W7LZ/S___^#Q!`[711T(P^V??.#[`2+1A2-!-A05_]U".B)
-M_?__0X/C'X/$$#M=%'7A@[Y,"0```'4'QT7H`0```(F>W`D``(-]'`!U"H-]
-MZ``/A`,"``"#[`@/MD7S4.AF]___@\`(B47DB00D_W7LZ/S___^)QX/$#`^W
-MV(G8]]!0_W7D_W7LZ/S___^#Q!#VPP@/A!@!``"%]G04QX9,"0```````,>&
-M-`(```````"+30B#>60!=`:#>6@!=1J#[`2X``$``(I-\]/@4%#_=0CH,_[_
-M_X/$$(M%"(-X8`%T#(-X9`%T!H-X:`%U%8/L"`^V1?-0_W4(Z#,)``"#Q!#K
-M>HM5"(-Z7`%T!H-Z8`%U:X/L!&H$#[9=\U/HJ/;__XUP*(DT)/]U[.C\____
-M@\0(5O]U[.C\____QP0D&0```.C\____@\0,:@!6_W7LZ/S___^#Q`A6_W7L
-MZ/S____'!"30!P``Z/S___^#Q`A3_W4(Z(`'``"#Q!"0@^P(#[9=\U/H/_;_
-M_P4<`0``B00D_W7LZ/S___^#Q!!3:@!J`?]U"(M-"/]1(.FI````]\<0````
-M=%&#[`@/MD7S4/]U".@Q!P``@\00BT4(@WAD`70&@WAH`74;@^P$:@"X``$`
-M`(I-\]/@4/]U".@7_?__@\00#[9%\U!J`6H!_W4(BU4(_U(@ZU`/M\?VQ`%T
-M2(.^-`(```%U*,>&3`D```````#'AC0"````````@^P$4`^V1?-0_W4(Z-KZ
-M__^#Q!"#/0`````!=0Z#[`Q6Z'4/``"#Q!")]HUE]%M>7\G#58GE4X/L!(M=
-M"`^V10S!X`(+11`/MM"+3),DA<EU#8/L"%)3Z*<-``#K19"+D40)``"%TG42
-M@^P(#[;`4%/HC0T``.LKC78`@SH!=`:#>A`!=!&#[`@/ML!04^AP#0``ZPZ)
-M]H/L"%)1Z`H```")]HM=_,G#C78`58GE5U93@^PDBU4(BT(HBW`,BUHPB@J#
-MX0.)SP^V`H/@!,'H`HA%V(V#(`$``%!6Z/S___^#Q`B!PQP!``!35NC\____
-MB,.#Q`RX``$``(GYT^#WT%"`?=@!&<`E``#__P44``,`4%;H_/___X/$$(3;
-M#X@0`0``]L,!=!>#[`1J`?]U#/]U".C'`0``Z?8```")]HM5#(M"%(/X`70M
-M@_@!<A"#^`(/A(4```#IU@```(GV@^P$:@#_=0S_=0CHD`$``(/$$.F\````
-M]L-`=$^#[`B+10R#P!10_W4(Z*T```"#Q!"#^`%U((M-#(-Y(``/A9````"#
-M[`1J`%'_=0CH3@$``(/$$.M]@^P$:@'_=0S_=0CH.0$``(/$$.MH@^P$:@'_
-M=0S_=0CH)`$``.M6#[;#J$!T3Z@@=4N+10R#>"``=1.#[`1J`%#_=0CH``$`
-M`(/$$.LO@^P(BT4,@\`44/]U".@E````@\00A<!U%H/L!&H!_W4,_W4(Z-(`
-M``"#Q!"-=@"-9?1;7E_)PU6)Y5=64X/L#(M]"(MU#+@`````@7X,_P````^&
-MF````(L6@_H!=%&X`````(/Z`0^"@P```(/Z`G5YNP````"#[`2+1@@/MQ!2
-M@\`"B48(BT<P!0`!``!0BT<H_W`,Z/S___^#Q!!#@?O_````=M"!;@P``0``
-MZSN[`````(GV@^P(BT<P!0`!``!0BT<H_W`,Z/S___^+5@AFB0*#1@@"@\00
-M0X'[_P```';3@6X,``$``+@!````C67T6UY?R<.-=@!5B>575E.#[!R+=0B_
-M`````(M=#(/#%(U%V%#_<P0/M@90_W8HZ&+V__^#Q!"#?1`!=1%FOP$`QX9,
-M"0```````(UV`(/L!(U%V%!J`&H`_W,@5P^V!E#_=BC_4QR#Q!C_=0Q6Z)L)
-M``"#Q!"#?1`!=1:#/0`````!=0V#[`Q6Z!@,``"#Q!"0@[Y$"0````^$NP``
-M`(N&1`D``(-X$`%U+X/L"%!6Z$4,``"#Q!"%P`^%F@```(/L!&H`_[9$"0``
-M5N@Y____@\00Z8$```"0@[XX`@```'4S@^P(#[8&4/]V*.@<"@``@\0(BT8P
-M@\`H4(M&*/]P#.C\____QP0D`0```.C\____@\00@^P(#[8&4/]V*.@9"@``
-MBYY$"0``@\00A=MT*(-[$`!U(HGV@^P$C4,44%-6Z/H*``"+6PB#Q!"%VW0(
-M@WL0`'3BB?:-9?1;7E_)PU6)Y5=64X/L%(M="(M#*(MX#(M#,(E%\(G&@\8H
-M5E?H_/___X/$$*@$=#:#[`1J`E97Z/S___^#Q`A65^C\____QP0DT`<``.C\
-M____@\0(#[8#4/]S*.@8`@``@\00ZQ6#[`1J`HM%\(/`*%!7Z/S___^#Q!#&
-M@SP"````QH-0"0```,:#40D```#'@]@)````````QX/<"0```````,>#-`(`
-M``````#'@]0)```@````N@````")E)-4"0``0H/Z'W[SN@````")]HT$U0``
-M```IT,>$PT0"```!````0H/Z'W[FQX-$"0```````,>#2`D```````#'@TP)
-M````````@^P$:@"+1?"#P`A05^C\____@\0,:!@!``"+1?"#P`Q05^C\____
-M@\0,_W,8BT7P@\`04%?H_/___X/$#(M#'"4`_/__4(M%\(/`%%!7Z/S___^#
-MQ`QJ`(M%\(/`&%!7Z/S___^#Q`S_<R"+1?"#P!Q05^C\____@\0,:@"+1?"#
-MP"!05^C\____@\0,BT,DL`!0BT7P@\`D4%?H_/___[@!````C67T6UY?R<.-
-M=@!5B>575E.#["R+=0B+111FB476@WT,``^%F@```+L`````B?:-!-T`````
-M*=B-!,:#N$0"````=72#N%0"```!=1J+D%P"``"+N'@"``"+@'0"``")1=#K
-M)(UV`(T$W0`````IV(T$QHN07`(``(NX=`(``(N`<`(``(E%T(U%V%!2#[8&
-M4/]V*.CR\O__@\0,C4784&H`#[=%UE!7_W40#[8&4/]V*/]5T(/$($.#^Q\/
-MCFW___^-9?1;7E_)PY!5B>575E.#[!R+70@/MG4,B?"#X`3!Z`*(1=B_`P``
-M`"'W@WM<`70&@WM@`75K@^P(B?H/ML*)PL'B"(V"#`$``(!]V`!T"8V"#`$#
-M`.L&D`4```(`4/]S#.C\____B<&#X?R#R0&#Q`Q1B?H/ML*)PL'B"(V"#`$`
-M`(!]V`!T"(V"#`$#`.L%!0```@!0_W,,Z/S___^#Q!"#[`B)^@^VPHG"P>((
-MC8)T`0``@'W8`'0*C8)T`0,`ZP>)]@4```(`4/]S#.C\____B<&`X1^)\@^V
-MP@^V5!AZP>(%@>+@````"=&`Y></MH08@@```,'@"R4`&```"<&#Q`Q1B?H/
-MML*)PL'B"(V"=`$``(!]V`!T"8V"=`$#`.L&D`4```(`4/]S#.C\____@\00
-MC67T6UY?R<.-=@!5B>575E.#[`R+70@/MG4,5NB"[?__@^P$C7@H5E/H9>[_
-M_X/$#&H$5_]S#.C\____@\0(5_]S#.C\____QP0D&0```.C\____@\0,:@!7
-M_W,,Z/S___^#Q`A7_W,,Z/S___^#Q`A64^A;_O__@\0(5E/HR>W__XUE]%M>
-M7\G#D%6)Y5=64X/L#,9%\@#&1?,`C78`BD7RBE7SC1R"#[;;4^CR[/__B<=J
-M`HUP*%:+10C_<`SH_/___X/$"%:+50C_<@SH_/___X/$"%/_=0CH+/___X/$
-M#&H`5HM%"/]P#.C\____@\0,:!\!``!7BU4(_W(,Z/S___^#Q`QJ`(U'!%"+
-M10C_<`SH_/___X/$#&H`C4<(4(M5"/]R#.C\____@\0,:@"-1PQ0BT4(_W`,
-MZ/S___^#Q`QJ`(U'$%"+50C_<@SH_/___X/$#&H`C4<44(M%"/]P#.C\____
-M@\0,:@"-1QA0BU4(_W(,Z/S___^#Q`QJ`(U''%"+10C_<`SH_/___X/$#&H`
-MC4<D4(M5"/]R#.C\____@\0,:@"-1R!0BT4(_W`,Z/S___^#Q`QJ`(U'+%"+
-M50C_<@SH_/___X/$#&B\````C4<T4(M%"/]P#.C\____@\00_D7S@'WS`P^&
-MK/[__X!]\@$9VX'C``#__X'#```#`(/L!&H`C4,,4(M5"/]R#.C\____@\0,
-M:@"-0Q!0BT4(_W`,Z/S___^#Q`QJ`(U#%%"+50C_<@SH_/___X/$#&H`C4,8
-M4(M%"/]P#.C\____@\0(@\,@4XM5"/]R#.C\____)1P<'!P-`P,#`X/$#%!3
-MBT4(_W`,Z/S___^#Q!#^1?*`??(!#X8'_O__C67T6UY?R<.)]E6)Y8/L#&C_
-M_\\/:&P$`0"+10C_<`SH_/___\G#C78`58GE4X/L!(M="(M#!+``/0``@%!T
-M)X/L"&@L#0``_W,,Z/S___^#R`&#Q`Q0:"P-``#_<PSH_/___X/$$(/L"&@`
-M#0``_W,,Z/S___\E__\`_X/$#%!H``T``/]S#.C\____@\0,:@!H!`T``/]S
-M#.C\____@\0,:@!H.`P``/]S#.C\____@\0,:/\``0!H!!T``/]S#.C\____
-M@\0,:@!H9!T``/]S#.C\____@\0,:@!H*`P``/]S#.C\____@\0,:@!H6!T`
-M`/]S#.C\____@\0,:@!H7!T``/]S#.C\____@\0,:@!H0!T``/]S#.C\____
-M@\0,:@!H1!T``/]S#.C\____@\0,:@!H2!T``/]S#.C\____@\0,:@!H4!T`
-M`/]S#.C\____BUW\R<.-=@!5B>6+50B+30S'00@`````BX)("0``B4$,@[I(
-M"0```'0)BX)("0``B4@(B8I("0``@[I$"0```'4&B8I$"0``R<.)]E6)Y5.+
-M70B+30R#>0@`=1B+00R)@T@)``"%P'04QT`(`````.L+B?:+40B+00R)0@R#
-M>0P`=1F+00B)@T0)``"%P'05QT`,`````.L,C78`BU$,BT$(B4((QT$(````
-M`,=!#`````!;R<.)]E6)Y5=64X/L#(M]"(MU#(M=$(,[`'49@^P$:B"-0P10
-MC4844.C\____@\00ZQV)]H/L!&HDC4,$4(U&%%#H_/____Z'4`D``(/$$(L#
-MB4805E?H^?[__\<&`````/Z'/`(``(UE]%M>7\G#D%6)Y593BUT(BW4,@WX0
-M`'0&_HM0"0``5E/H"O___XN#U`D```^V5@2)E(-4"0``0(F#U`D``,<&`0``
-M`/Z+/`(``(UE^%M>R<.-=@!5B>6*10@\`W82#[;(C4P)`K@!````T^#K$(GV
-M#[;(C4P)`;@!````T^#)PU6)Y593BW4(#[9=#%/HPO___X/L!`E&5%/H`NC_
-M_P4<`0``B00D_W8,Z/S___^#Q`R)V8/A`[@``0``T^#WT%"#XP3!^P*#^P$9
-MP"4``/__!10``P!0_W8,Z/S___^#Q`S_=E1H9!T``/]V#.C\____@\0(:&0=
-M``#_=@SH_/___XUE^%M>R<-5B>53@^P$BUT(#[9%#%#H./____?0(T-4B4-4
-M4&AD'0``_W,,Z/S___^+7?S)PY!5B>575E.#[!"*70R+10B+0`R)1?`/MM.+
-M10B+=)`DBWXPQX8T`@```0```(/B!,'J`HG9@^$#N`$!``#3X/?04(#Z`1G`
-M)0``__\%%``#`%#_=?#H_/___X/$#&H`BT8P@\`(4/]U\.C\____@\0(#[;;
-M4_]U".A4____@\0(C8<<`0``4/]U\.C\____QX9``@```0```(/$#&H!@\<H
-M5_]U\.C\____C67T6UY?R<.-=@!5B>575E.#[!"+10B+>`P/MG4,BT2P)(M8
-M,,>`-`(```````!J`H/#*%-7Z/S___^#Q`A35^C\____@\0(5O]U".A._O__
-MC67T6UY?R<.)]E6)Y593BUT(BT,HBW`,_W40_W4,4XN#V`D``,'@!0-#$%#H
-MF>?__XN3V`D``$*#XA^)D]@)``#^@U$)``"#Q`R+0QPE`/S__\'B!0G04(M#
-M,(/`%%!6Z/S___^-9?A;7LG#B?95B>53@^P$BUT(:@!J`FH`4^A:]O__B1PD
-MZ)[T__^)'"3H_/___X/$$(/X`74>QX-,"0```0```(/L"`^V`U#_<RCHF?W_
-M_X/$$(GVBUW\R<.-=@!5B>575E.#[!2+50B+?0R#QQ2+0BB+<`R+6C"-@QP!
-M``!05NC\____)<$```"#Q!"Z`````(/X0`^%T@$``(-_!`%U<(/L!`^V1Q%0
-MC8,$`0``4%;H_/___X/$#`^V1Q-0C8,(`0``4%;H_/___X/$#`^V1Q50C8,,
-M`0``4%;H_/___X/$#`^V1Q=0C8,0`0``4%;H_/___X/$#`^V1QE0C8,4`0``
-M4%;H_/___X/$$.LTB?9F]T<0`/]U(&;W1Q(`_W489O='%`#_=1!F]T<6`/]U
-M"&;W1Q@`_W0*N@````#I*@$``(/L!`^V1Q!0C8,$`0``4%;H_/___X/$#`^V
-M1Q)0C8,(`0``4%;H_/___X/$#`^V1Q10C8,,`0``4%;H_/___X/$#`^V1Q90
-MC8,0`0``4%;H_/___X/$#`^V1QA0C8,4`0``4%;H_/___X/$#`^V1QI0C8,8
-M`0``4%;H_/___X/$#`^V1QM0C8,<`0``4%;H_/___X/$$(,_`@^%A````(/L
-M"(V#(`$``%!6Z/S___]J9&H*4U;H_/___X/$(+H`````A<!T9(/L"(V#'`$`
-M`%!6Z/S___\EB0```(/$$+H`````@_@(=4+'1?``````@<,``0``D(/L!(M'
-M"`^W$%*#P`*)1PA35NC\____@\00_T7P@7WP_P```';:@6\,``$``+H!````
-MB?:)T(UE]%M>7\G#B?95B>575E.#[!"+=0C'1E0`````:@!H9!T``/]V#.C\
-M____QT9<`````,=&8`````#'1F0`````QT9H`````,=&;`````#'1G``````
-MQT9T`````,9&>`&#Q!"X`````(-^(``/A',"``"X`````&:!?@:!4`^%8@(`
-M``^V5@6#^@$/A%4"``"#^@%_!H72=`GK&8/Z`G0+ZQ+'1F`!````ZQ#'1F0!
-M````ZP?'1F@!````LP"#[`@/MM.)T(/@`\'@"(V(=`$``/;"!'0'!70!`P#K
-M!HV!```"`%#_=@SH_/___P^VRXG"@>+@````P>H%B%0Q>B4`&```P>@+B(0Q
-M@@```(/$$$.`^P=VIX/L#%;H8O7__XDT).AJ]___B30DZ(+W__^#Q`QJ`&CP
-M!`$`_W8,Z/S___^#Q`AH+`T``/]V#.C\____B<>#Y_Z#Q`Q7:"P-``#_=@SH
-M_/___X/$"&@`#0``_W8,Z/S___^#Q!"H,'0YBT9$J>`#`(!T""4?_/]_B49$
-M@WY<`702@WY@`70,@WYD`70&@WYH`743BT9$J!!T#(/@[XE&1.L$@V9$S[,`
-M@^P(#[;#4%;H__+__X/$$$.`^P=VZH/L!/]V1&@`#```_W8,Z/S___^#Q`S_
-M=DAH*`P``/]V#.C\____@\0,_W9,:%P=``#_=@SH_/___[,`@\00@^P$#[;#
-M_W2&$(#[`1G`)0``__\%#``#`%#_=@SH_/___X/$#`^VP_]TAAB`^P$9P"4`
-M`/__!1```P!0_W8,Z/S___^#Q!!#@/L!=K/'1E15JP8`LP")]H/L"`^VPU!6
-MZ,OA__^#Q!!#@/L'=NJ#[`QH4,,``.C\____@\00@WYD`70&@WYH`75$OP``
-M``"S`(UV`(/L"`^VPXE%\%!6Z/S___^#Q!"%P'4+9K@``8I-\-/@"<=#@/L'
-M=MB#[`17:`#_``!6Z/WG__^#Q!"X`0```)"-9?1;7E_)PU6)Y8/L#&H`:&0=
-M``"+10C_<`SH_/___[@!````R<.058GEQP4``````0```,G#D%6)Y<<%````
-M``````#)PY!5B>564XMU"(I5#`^VPHM<AB2X`````(7;="P/ML)0Z$C@__^#
-M[`B)0S")<RC'@TP)````````QX,T`@```````%/HX>[__XUE^%M>R<.)]E6)
-MY0^V50R+10B+5)`DN`````"%TG0-@[I,"0```0^5P`^VP,G#B?95B>564XMU
-M"(I%#(G"@^(#B='!X0B-D0`!``"H!'0)C9D``0,`ZP>0C9H```(`@^P(4_]V
-M#.C\____@^`/@\00@_@!=`^#^`%R0H/X`W0UZSN-=@"#[`A3_W8,Z/S____'
-M!"1`#0,`Z/S___^#Q`A3_W8,Z/S___^#X`^#Q!"#^`-U"9"X`0```.L&D+@`
-M````C67X6U[)PU6)Y5=64X/L+(M]"(I%#(A%\X/@`XA%YP^V5?.)T(/@!(G&
-MP>X"BU27)(M/#(E-[+@`````A=(/A!\"``"+0C")1>BX`````(.Z3`D```$/
-MA`<"``"#[`@/ME7SB57<4E?HYM___X/$#&H$BUWH@\,H4_]U[.C\____@\0(
-M4_]U[.C\____QP0D&0```.C\____@\0,:@!3_W7LZ/S___^#Q`A3_W7LZ/S_
-M__^#Q`C_==Q7Z-3O___'1>``````@\00#[9%YXG!P>$(B4W4@<$(`0``B4W8
-MP>`(B470@^P(BU74C8((`0,`B?&$R74)BU78C8(```(`4/]W#.C\____B<.#
-MX_"#RP&#Q`Q3BT70C9`(`0``C8`(`0,`B?&$R74&C8(```(`4/]W#.C\____
-M@\0(#[9%YXG"P>((C8((`0``B?&$R70+C8((`0,`ZPB-=@`%```"`%#_=PSH
-M_/___\<$)!D```#H_/___X/C\(/$#%,/MD7GB<+!X@B-@@@!``")\83)=`J-
-M@@@!`P#K!XGV!0```@!0_W<,Z/S___^#Q`@/MD7GB<+!X@B-@@@!``")\83)
-M=`J-@@@!`P#K!XGV!0```@!0_W<,Z/S____'!"30!P``Z/S___^#Q`@/MEWS
-M4U?H_/___X/$$(7`=1[_1>"#?>`##X7=_O__@^P(4U?H^MW__[@`````ZTN#
-M[`B+1>@%'`$``%#_=>SH_/___R7`````@\00@_A`=!6#[`@/MD7S4%?HQ-W_
-M_[@`````ZQ6#[`@/MD7S4%?HK]W__[@!````B?:-9?1;7E_)PU6)Y5=64X/L
-M#(M5"`^V=10/MD4,BUR")+D`````A=L/A(L```"+4@R)5?"+>S"Y`````(.[
-M3`D```%T=(/L"%?_=?#H_/___XG"@\00@WT0`'4IC4;_N0`````\'W=1QX,X
-M`@```````(/BP(GQ#[;!2`G"@,X"ZQ.-=@#'@S@"```!````@>+`_?__@WL,
-M`74&@,X!ZP20@.;^@^P$4E?_=?#H_/___[D!````C78`B<B-9?1;7E_)PXGV
-M58GE4X/L!(M="(I-#`^VP8M4@R2X`````(72="JX`````(.Z3`D```%T',>"
-M3`D```$```"#[`@/ML%04^B2]/__N`$```"+7?S)PU6)Y5.#[`2+70B*50P/
-MML*Y`````(-\@R0`="`/MM*+1),DQX!,"0```````(/L"%)3Z`7U__^Y`0``
-M`(G(BUW\R<.058GE4X/L!`^V50R+10B+7)`DN`````"%VW0HN`````"#NTP)
-M```!=!IJ`&H"_W404^CFZ___B1PDZ"KJ__^X`0```(M=_,G#58GE#[95#(M%
-M"(M4D"2X_P```(72=`</MH(\`@``R<-5B>564XM5"(MR#`^V10R+1((DN@``
-M``"%P'18BU@P@^P(C4,L4%;H_/___X/$$(-]$`%U'[H`````J`)U-H/L!&H"
-MC4,L4%;H_/___X/$$.L=B?:Z`````*@"=!>#[`1J`(U#+%!6Z/S___^#Q!"Z
-M`0```(G0C67X6U[)PXUV`%6)Y5=64X/L#(I=#`^V=1`/MGT4B-J#X@.)V(/@
-M!,'H`O?&^````'4(]\?\````=`NX`````.F)````D`^VTHG1P>$(C9%T`0``
-MA,!T#H'!=`$#`(E-\.L,C78`@<(```(`B57P@^P(_W7PBT4(_W`,Z/S___\D
-M'XGRP>(%@>+@````"="`Y.>)^L'B"X'B`!@```G0@\0,4/]U\(M5"/]R#.C\
-M____#[;#B?F+50B(C!""````B?&(3!!Z@\00N`$```"-9?1;7E_)PU6)Y5=6
-M4X/L%(MU"(I=#+\#````(=^#XP3!ZP*`^P$9P"4``/__!1@``P!0_W8,Z/S_
-M__^)^0^VT8U*&+H!````T^()T(/$#%"`^P$9P"4``/__!1@``P!0_W8,Z/S_
-M__^#Q`B`^P$9P"4``/__!1@``P!0_W8,Z/S___^#Q!"X`0```(UE]%M>7\G#
-M58GE5U93@^P4BW4(#[9]#(GX@^`#B$7SB?@/MM")T(/@!(G#P>L"4E;H;NK_
-M_X/$"(#[`1G`)0``__\%&``#`%#_=@SH_/___P^V3?.#P1BZ_O___]/"(="#
-MQ`Q0@/L!&<`E``#__P48``,`4/]V#.C\____@\0(@/L!&<`E``#__P48``,`
-M4/]V#.C\____@\0(#[9%\XG"P>((C8((`0``A-MT"HV""`$#`.L'B?8%```"
-M`%#_=@SH_/___X/@\(/(`8/$#%`/MD7SB<+!X@B-@@@!``"$VW0)C8((`0,`
-MZP:0!0```@!0_W8,Z/S___^#Q`B)^@^VPE!6Z)OI__^#Q!"X`0```(UE]%M>
-M7\G#C78`58GE5U93@^P4BWT(BW40BD4,B$7SBT<,B47L#[9%\XM<AR2)PH/B
-M`XG1P>$(C9%T`0``J`1T"(V!=`$#`.L&C8(```(`4/]W#.C\____B498@\00
-MA=MU$,<&`````+@!````Z?\```"+0S")1>B#[`@/MD7S4%?H_/___XD&@\00
-MN`$```"#/@`/A-@```"+@TP)``")1BR+@S@"``")1C"*@SP"``"(1C2Z````
-M`)"*1!-JB$06!$*#^B=V\H/L"/]UZ/]U[.C\____B48X@\0(BT7H@\`04/]U
-M[.C\____B48\@\0(BT7H@\`44/]U[.C\____B49`@\0(BT7H@\`84/]U[.C\
-M____B49$@\0(BT7H@\`<4/]U[.C\____B49(@\0(BT7H@\`@4/]U[.C\____
-MB49,@\0(BT7H@\`D4/]U[.C\____B490@\0(BT7H@\`H4/]U[.C\____B494
-MN`$```"-9?1;7E_)PY!5B>6#[#S'1<@`````BT40B47,BT44B474BT489HE%
-MV(M%'&:)1=J+12")1=R+122)1>"+12B)1>2+12R)1>B-1<A0#[9%#%#_=0CH
-M_/___\G#B?95B>575E.#[`R*10R(1?,/ML"+50B+=((DN`0```"%]@^$BP$`
-M`+@!````@[Y,"0````^$>0$``+@"````@+X\`@``'P^$9P$``(N&U`D``$B)
-MAM0)``"+O(94"0``C03]`````"GXC83&1`(``(E%[(GZB%`$BT40@S@`#X7.
-M````B<.#PP2#OC@"```"=0S'0P0!````ZT>-=@`/MT,,#[=3#E#_<PA2_W8(
-MZ/O6__^#Q!")0P2#^`%U)(-^"`!U'HN&U`D``(F\AE0)``!`B8;4"0``N`,`
-M``#IT0```("^4`D```!U4X.^-`(```!U(H/L"`^V7?-3_W4(Z/KM__^#Q`A3
-M_W4(Z![N__^#Q!"-=@"#[`3_=1#_=>Q6Z&WL__^#Q`R+11"#P`10_W7L5NCZ
-M[O__@\00ZW&0@^P$_W40_W7L5NA%[/__@\00ZUR#[`3_=1#_=>Q6Z#'L__^#
-MQ!"`OCP"```!=4&#OC0"```!=1.#[`@/MD7S4/]U".A<[O__@\00@^P(_W7L
-M5NA5[___@\00A<!U$O]U[%;H6>S__[@!````ZP>)]K@`````C67T6UY?R<.-
-M=@!5B>575E.#["2+10B+0`R)1>1H8!T``%#H_/___XG&BU4(BU)4B57P@\00
-MN`````"%U@^$9P$``/?&```$`'0X@^P(:%@=``#_=>3H_/___XG#@\0,]]!0
-M:%@=``#_=>3H_/___VH`4VH`_W4(BU4(_U(@@\0@B?;&1>L`]\;_`0``#X0`
-M`0``@'WK`1G;@>,``/__@<,4``,`@^P$:N]3_W7DZ/S___^#Q`A3_W7DZ/S_
-M__^)QX/$#/?0@\@04%/_=>3H_/___X/$"(!]ZP$9P"4``/__!0@``P!0_W7D
-MZ/S___^)1>"S`(/$$`^V1>N)1=B0#[;+N``!``#3X"'XB47LN`$```#3X(GR
-M@^(!(?AU!(72="&#[`174E"+1>"#X!]0#[;#4`^V1>M0_W4(Z'?;__^#Q""#
-M?>P`=#:+5=B)5=R)T,'@`@G8#[;`4.@XZ___@\0$A47P=!B#[`0/ML-0_W7<
-M_W4(Z`+>__^#Q!"-=@#![@+!;>`(0X#[`P^&;____^L$D,'N"-'N_D7K@'WK
-M`0^&XO[__[@!````C67T6UY?R<.058GE@^P,:@!H9!T``(M%"/]P#.C\____
-MN`$```#)PY!5B>6#[`R+10C_<%1H9!T``/]P#.C\____N`$```#)PU6)Y5=6
-M4X/L#(MU$(M]%+L`````.?-S.8/L"(M%#`4<`0``4/]U".C\____@\00A,!X
-M#:@!#Y3`#[;`ZQF-=@"#[`Q7Z/S___^#Q!!#.?-RQ[@`````C67T6UY?R<.)
-M]E6)Y5.#[`B+70AJ`(M#,`4@`0``4(M#*/]P#.C\____@\0(BT,P!2`!``!0
-MBT,H_W`,Z/S___^+7?S)PU6)Y5.#[`B+70AJ`HM#,`4@`0``4(M#*/]P#.C\
-M____@\0(BT,P!1P!``!0BT,H_W`,Z/S___^+7?S)PU6)Y5=64X/L#(MU"(M&
-M*(M`#(E%\(M^,+L`````N`````"#OC0"```!#X2M````@^P(C8<@`0``4/]U
-M\.C\____@\00J(%T$(/L#&HRZ/S___^#Q!"-=@"#[`B-AR`!``!0_W7PZ/S_
-M__^#Q!"H@70XB=A#9CVW"W<H@^P(#[8&4/]V*.C\____@\00A<!T3(/L#&CH
-M`P``Z/S___^#Q!#KMK@`````ZS-J9&H*5_]U\.C\____@\0(C8<<`0``4/]U
-M\.C\____)<$```"#Q!"#^$`/E,`/ML"-=@"-9?1;7E_)PU6)Y5=64X/L#`^V
-M50R+10B+7)`DN`````"%VW1KBT,HBW@,BW,PN`````"#NTP)```!=%2#[`Q3
-MZ/S___^#Q`QHX0```(V&'`$``%!7Z/S___]J9&@0)P``5E?H_/___X/$((7`
-M=1&#[`Q3Z/S___^X`````.L1D(/L#%/H_/___[@!````B?:-9?1;7E_)PU6)
-MY593BTT(BE4,#[;"BT2!)+X`````A<!T<8U8-(/L#&CL````:@!J`&H`:@!J
-M`&H`:``!``!3:@!J`0^VPE!1Z/S___^#Q$"^`````(7`=#KV@Z````#@="RQ
-M`+X!````@+O^`0``I74AN@`````/M\("#!A"9H'Z_P%V\KX`````A,EU!;X!
-M````B?"-9?A;7LG#C78`58GE@^P(#[95#(M%"(M4D"2X`````(72=!>X````
-M`(.Z3`D```%T"8/L#%+H_/___\G#D%6)Y8/L%&CO````:@`/MD4@4`^V11Q0
-M#[9%&%`/MD444`^V11!0:@!J`&H`:@`/MD4,4/]U".C\____R<.-=@!5B>57
-M5E.#[`R*10R(1?.+32!FB4WPBT4D9HE%[HM-*&:)3>R+?2R+=3"*732*13B(
-M1>L/MD7SN@````"+30B#?($D`'0^@^P,#[9%ZU`/ML-0#[?&4`^WQU`/MT7L
-M4`^W1>Y0#[=%\%#_=1S_=1C_=13_=1`/MD7S4/]U".C\____B<*)T(UE]%M>
-M7\G#B?95B>575E.#['B+10B+52!FB578BTTD9HE-R(M5*&:)5;B+32QFB4VH
-MBU4P9HE5F(I--(A-EXI5.(A5E@^V50R+7)`DBW`,BWLP4^BA_/__@\00N@``
-M``"%P`^$K0(``(/L#%/H_/___X/$$(-]%`%U>X/L!(M-V`^VQ5"-AP0!``!0
-M5NC\____@\0,BU7(#[;&4(V'"`$``%!6Z/S___^#Q`R+3;@/ML50C8<,`0``
-M4%;H_/___X/$#(M5J`^VQE"-AQ`!``!05NC\____@\0,BTV8#[;%4(V'%`$`
-M`%!6Z/S___^#Q!#K/8UV`/=%V`#_``!U)_=%R`#_``!U'O=%N`#_``!U%?=%
-MJ`#_``!U#/=%F`#_``!T#8UV`+H`````Z>8!``"#[`0/MD784(V'!`$``%!6
-MZ/S___^#Q`P/MD7(4(V'"`$``%!6Z/S___^#Q`P/MD6X4(V'#`$``%!6Z/S_
-M__^#Q`P/MD6H4(V'$`$``%!6Z/S___^#Q`P/MD684(V'%`$``%!6Z/S___^#
-MQ`P/MD674(V'&`$``%!6Z/S___^#Q`P/MD664(V''`$``%!6Z/S___^#Q!"#
-M?1``=3MJ9&CPN@0`5U;H_/___X/$$(7`=1.#[`Q3Z/S___^Z`````.DB`0``
-M@^P,4^C\____N@$```#I#P$``&ID:/"Z!`!75NC\____@\00A<!U$X/L#%/H
-M_/___[H`````Z><```"#[`B-AQP!``!05NC\____@\00J`AU$X/L#%/H_/__
-M_[H`````Z;T```#'19``````BT4<.460<VJ-EP`!``")58R`?9``=15J9&A0
-MPP``5U;H_/___X/$$(7`=&N#?1`!=1N#[`C_=8Q6Z/S___^+39"+51AFB01*
-M@\00ZQJ#[`2+39"+51@/MP1*4/]UC%;H_/___X/$$/]%D(M-'#E-D'*?:F1H
-M4,,``%=6Z/S___^#Q!"%P'4@@^P,4^C\____N@````#K'X/L#%/H_/___[H`
-M````ZP^#[`Q3Z/S___^Z`0```)")T(UE]%M>7\G#B?95B>575E.#[!"+70B+
-M0RB+<`R+>S!J!HV'(`$``%!6Z/S___^#Q`B+0S`%(`$``%!6Z/S____'!"0*
-M````Z/S___^)'"3H_/___\<$)/0!``#H_/___\<$)/0!``#H_/___\<$)/0!
-M``#H_/___\<$)/0!``#H_/___[L`````@\00B?:#[`B-AQP!``!05NC\____
-M@\00A,!X";@!````ZR")]H/L#&CT`0``Z/S___^#Q!!#@?L/)P``=L>X````
-M`(UE]%M>7\G#B?95B>53@^P0BUT(4^@D____@\00N@$```"#^`%T%X/L#&CT
-M`0``Z/S___^)'"3H`O___XG"B="+7?S)PY!5B>564XMU"(/L!&B@````H00`
-M``"+0!@%```$`%!6Z/S___^[`````(/$$(/L"*$$````BT`0!0``!`!05NC\
-M____@\000X/[!'[@NP````"-=@"#[`BA!````(M`%`4```0`4%;H_/___X/$
-M$$.#^P5^X+L`````C78`@^P(H00```"+0!`%```$`%!6Z/S___^#Q!!#@_L$
-M?N"-9?A;7LG#D%6)Y5.#[`B+70AH@````*$$````BT`,!0``!`!04^C\____
-M@\0,:+````"A!````(M`&`4```0`4%/H_/___XM=_,G#58GE5E.+=0B#[`Q6
-MZ`?___^#Q`QJ`*$$````BT`(!0``!`!05NC\____@\0(H00```"+``4```0`
-M4%;H_/___XC#B30DZ'+___\/MMN)V(UE^%M>R<.)]E6)Y5=64X/L%(M="&@L
-M#0``4^C\____B<>#Q`R#R`%0:"P-``!3Z/S___^#Q`AH\`0!`%/H_/___XG&
-M@^!B@\00@_AB=!J#[`2)\(/(8@^VP%!H\`0!`%/H_/___X/$$(/L"&@```0`
-M4^C\____B1PDZ$K^__^#Q`QJ!Z$$````BT`(!0``!`!04^C\____@\0,:A&A
-M!````(L`!0``!`!04^C\____@\0,:@6A!````(M`"`4```0`4%/H_/___X/$
-M#&H1H00```"+``4```0`4%/H_/___XD<).B$_O__@\0,5FCP!`$`4^C\____
-M@\0(:/`$`0!3Z/S___^#Q`Q7:"P-``!3Z/S___^#Q!"-9?1;7E_)PY!5B>57
-M5E.#[!2+70AH+`T``%/H_/___XG'@\0,@\@!4&@L#0``4^C\____@\0(:/`$
-M`0!3Z/S___^)QH/@8H/$$(/X8G0:@^P$B?"#R&(/ML!0:/`$`0!3Z/S___^#
-MQ!"#[`AH```$`%/H_/___XD<).@R_?__@\0,:@>A!````(M`"`4```0`4%/H
-M_/___X/$#&H`H00```"+``4```0`4%/H_/___X/$#&H%H00```"+0`@%```$
-M`%!3Z/S___^#Q`QJ`*$$````BP`%```$`%!3Z/S___^)'"3H;/W__X/$#%9H
-M\`0!`%/H_/___X/$"&CP!`$`4^C\____@\0,5V@L#0``4^C\____@\00C67T
-M6UY?R<.058GE4X/L$(M="%/H@/S__X/$#&H*H00```"+0`@%```$`%!3Z/S_
-M__^#Q`QJ`*$$````BP`%```$`%!3Z/S___^)'"3HZ_S__X/$"&@!``0`4^C\
-M____@\00NO____^H`G50@^P,4^@D_/__@\0,:@&A!````(L`!0``!`!04^C\
-M____B1PDZ*C\___'!"1D````Z/S___^#Q`AH`0`$`%/H_/___]'H@^`!@\00
-M@_@!&=*)T(M=_,G#B?95B>575E.#[!2+?0B*10R(1?.+7PQH+`T``%/H_/__
-M_XE%[(/$#(/(`5!H+`T``%/H_/___X/$"&CP!`$`4^C\____B<:#X&*#Q!"#
-M^&)T&H/L!(GP@\AB#[;`4&CP!`$`4^C\____@\00@^P(:```!`!3Z/S___^#
-MQ`R*1?.(1WD/ML!0:`#@!0!3Z/S___^#Q`Q6:/`$`0!3Z/S___^#Q`AH\`0!
-M`%/H_/___X/$#/]U[&@L#0``4^C\____@\00C67T6UY?R<.-=@!5B>6#[`B+
-M50B*30R`?1``=`ZX`0```-/@"$)YZPR)]KC^____T\`@0GF#[`@/MD)Y4%+H
-M_/___\G#B?95B>575E.#[!2+?0C&1?,`:"P-``!7Z/S___^)1>B#Q`R#R`%0
-M:"P-``!7Z/S___^#Q`AH\`0!`%?H_/___XE%[(/@8H/$$(/X8G0;@^P$BD7L
-M@\AB#[;`4&CP!`$`5^C\____@\00@^P(:```!`!7Z/S___^#Q`AH```$`%?H
-M_/___X/$$&8]5:H/A?4```"^`````(UV`+L`````.?-_)XUV`(/L!&B@````
-MH00```"+0!@%```$`%!7Z/S___^#Q!!#.?-^W('^R````'X0@^P,:@'H_/__
-M_X/$$(UV`(/L"&@```0`5^C\____B<.#Q`QHL````*$$````BT`8!0``!`!0
-M5^C\____@\009H'[5:IU%$:!_L<````/CG7___]F@?M5JG1;@^P$:+````"A
-M!````(M`&`4```0`4%?H_/___X/$"&@```0`5^C\____@\009CU5JG4HQD7S
-M`8/L#%?H2OK__X/$$#P^=12#[`Q7Z,+\__^#Q!"%P'4$QD7S`H/L!&BP````
-MH00```"+0!@%```$`%!7Z/S___^#Q`S_=>QH\`0!`%?H_/___X/$"&CP!`$`
-M5^C\____@\0,_W7H:"P-``!7Z/S___^#Q!`/MD7SC67T6UY?R<-5B>564XMU
-M"+L`````C78`QP4$````B````(/L#%;H_?W__X/$$(3`=2/'!00```"D````
-M@^P,5NCC_?__@\00A,!U"4.!^^<#``!^PP^VP(UE^%M>R<,`````````````
-M``````````````````````0`````````````````````````````````````
+M?T5,1@$!`0D```````````$``P`!``````````````#LQP```````#0`````
+M`"@`#``)`%6)Y5=64X'L+`(``(M]"(M'#(F%Y/W__XUWO+`"@'\!`'4*@'\"
+M`1G`@^`"0(@&QT8$`````(!.`02+1PR)1@B+5Q"Y`````(E6#(E.$`^V!HL$
+MA0````")1CR`/@-U.(G0B<J#P/:#TO^)1@R)5A"-E>C]__]2:B"+1Q"#Z`I0
+M5^C\____@\00@;WH_?__\Q9X6G09C78`#[8&BP2%`````(E&0(!.`03I>@8`
+M`(/L"&B0````C8WH_?__4>C\____@\00A,!US_:%\?W__P)T(L9'"`&*A?+]
+M__^(1PLZ1PES$(/L"`^VP%!7Z/S___^#Q!"#[`AH``(``(V%Z/W__U#H_/__
+M_X/$$(3`=!Z#[`1H<`$``&H`C87H_?__!9````!0Z/S___^#Q!"*E7O^__^(
+MTX/C`8I'!(/@_(C1@^$""=@)R(C3@^,$@^#SB-&#X0@)V`G(B-.#XQ"#X,^(
+MT8/A(`G8"<B(T8/A0(/@/X/B@`G("="(1P2H`704@^P$:@#0Z(/@`5!7Z/S_
+M__^#Q!"*1P2H!'05@^P$:@#`Z`.#X`%05^C\____@\00BD<$J!!T$X/L",#H
+M!8/@`5!7Z/S___^#Q!"*1P2H0'03@^P(P.@'#[;`4%?H_/___X/$$(J5\?W_
+M_X/B`8I&`8/@_@G0B$8!@+WX_?__`P^$E/[__X"]^/W__P!U$,8&`*$`````
+MB49`Z0L%``"#O>S]__\`#X1N_O__N@`````/MH7S_?__B<&#^`!^+[L`````
+MB=#!X`2-O>C]__\!^(!X$`@/AS_^__\/MD`0@SR#``^$,?[__T(YT7_6BX4`
+M_O__N@````")A=C]__^)E=S]__^+C8#^__^)R[D`````BX78_?__"<B+E=S]
+M__\)VHF%V/W__XF5W/W__[H`````C012C02`P>`$BXWD_?__C;P(<"(``(-_
+M!`!U#XM'2#N%[/W__P^$60$``$*#^@]^SXN%Y/W__XNX;"(``(7_="N+1PB+
+ME>3]__^)@FPB``"#[`1H\````&H`5^C\____BXWD_?__B4\(@\00BX7L_?__
+MB4=(BE=>@\H!B%=>9@^VA?S]__^#X`'1X(/B_0G"B%=>9HN%_OW__V:)1V"+
+MA=C]__^+E=S]__^)1V2)5VB*A?C]__^(!XJ%^_W__XA'1@^VR+@!````T^!F
+MB4=<QT<$`````(N5]/W__XE7#,='$`````"+A7S^__^)PK@`````"T<,"U<0
+MB4<,B5<0BH7Y_?__B$=$#[8'BP2%`````(E'/(/L!&H0C87H_?__@\`L4(V'
+MC````%#H_/___X/$#&H$C87H_?__@\`\4(V'G````%#H_/___X/$#&H0C87H
+M_?__@^B`4(V'X````%#H_/___X/$#&I`C87H_?__@\!`4(V'H````%#H_/__
+M_X/$$(N-W/W__SE/:'(?=PN+E=C]__\Y5V1V$HN%V/W__XN5W/W__XE'9(E7
+M:/:%\?W__P%T!(!/`0&*E?K]__^(E>/]__^`O?/]__\!#X9U`@``#[;"BT2'
+M;(F%U/W__XN%$/[__[H`````B878_?__B97<_?__BXV(_O__B<NY`````(N%
+MV/W__PG(BY7<_?__"=J)A=C]__^)E=S]__^#O=3]__\`#X6>`0``BY7D_?__
+MBY)L(@``B974_?__A=)T,8M""(N-Y/W__XF!;"(``(/L!&CP````:@!2Z/S_
+M__^+E>3]__^+A=3]__^)4`B#Q!"+A>S]__^+C=3]__^)04B*45Z#R@&(45YF
+M#[:%#/[__X/@`='@@^+]"<*(45YFBX4._O__9HE!8(N%V/W__XN5W/W__XN-
+MU/W__XE!9(E1:(J%"?[__XA!1(J%"/[__X@!B7D$BX4$_O__B4$,QT$0````
+M`(N%A/[__XG"N``````+00P+41")00R)41"*A0O^__^(04:+E=3]__\/MDI&
+MN`$```#3X&:)0EP/M@*+!(4`````B4(\#[8'BP2%`````(E"0(J-X_W__XA*
+M`P^VP8E4AVS^1T6#[`1J$(V%Z/W__X/`+%")T`6,````4.C\____@\0,:@2-
+MA>C]__^#P#Q0BX74_?__!9P```!0Z/S___^#Q`QJ$(V%Z/W__X/H@%"+A=3]
+M__\%X````%#H_/___X/$#&I`C87H_?__@\!`4(N%U/W__P6@````4.C\____
+M@\00]H7Q_?__`70*B[W4_?__@$\!`0^VA0K^__^+E=3]__^#?()L``^%@```
+M`(ET@FR*A0K^__^(1@/^0D6)5@0/M@*+!(4`````B49`BXW<_?__.4IH<E5W
+M"XN%V/W__SE"9'9(BY78_?__BXW<_?__B[W4_?__B5=DB4]HZRX/MH7C_?__
+M@WR';`!U((ETAVR)?@3^1T6*A>/]__^(1@,/M@>+!(4`````B49`C67T6UY?
+MR<-5B>564XMU#+L`````#[9&1(G"@_@`#XZX````@WR>;`!T"D,YVG_TZ:<`
+M``"X`0```(C9T^!F"49@@^P(5O]U".C\____B<*#Q!"%P`^$@0```,8``XA8
+M`XEP!(`^"`^4P,'@`XI*`8/A]PG!B$H!H0P```")0CP/M@:+!(4`````B4)`
+M]D8!$'05B,B#R!"(0@&+0E2)0@S'0A``````B52>;(I&7H/@_H/("HA&7L=&
+M9`````#'1F@`````_D9%@^P(4FH'Z/S___^`3EX$@\00D(UE^%M>R<.058GE
+M5U93@^P,BW4,#[8&@_@&='R#^`9_"X/X!`^,<@$``.L0@_@(#X3#````Z6(!
+M``")]HI&13I&1`^%5`$``(!.`02`9E[^OP````"`?D0`#X0]`0``D(M<OFR%
+MVW0@@#L#=AN#[`A3_W4(Z/S___^#Q!#V0UX$=`:`3EX$B?9'#[9&1#GX?\_I
+M!@$``(GV@$X!!(!^10)U"H!F7O[I\0```)"`9E[]@WYL`'41BT9PQD`#`(E&
+M;,=&<`````"+1FR`.`-U$8/L"%;_=0CH_/___X/$$(GV@$Y>!,=&9`````#'
+M1F@`````Z:4```"0BD9%.D9$=1R`3@$$BD9>@^#^B$9>9H-^8`!T:(/(`HA&
+M7NM@#[9&14`/ME9$.=!U4V:#?F``="&Y``````^W1F")PJ@!=0E!B=#3^*@!
+M=/>#?(YL`'4?ZP;V1EX"=1>`3@$$@$Y>!(/L"%;_=0CH_/___X/$$,=&9```
+M``#'1F@`````#[=&7+H`````@\#_@]+_]]#WTB-&9"-6:(E&9(E6:)"#?@0`
+M=1KV1EX$=!3H_/___XE&2(/L#%;H_/___X/$$(UE]%M>7\G#58GE5U93@^P,
+MBWT(O@````"-=@"-!':-!(#!X`2-G#AP(@``@WM(`'0B@WL$`'4<@^P(4U?H
+M_/___X/$$/9#`01T"<=#0`````")]D:#_@]^PHUE]%M>7\G#B?95B>575E.!
+M["@"``"+?0R*11"(A>?]__]7Z/S___^#Q!"`/PAU$H/L"%?_=0CH_/___X/$
+M$(UV`+X`````@']$``^&Q0```)"#?+=L``^$K0```(M$MVR`.`-V((/L!`^V
+MA>?]__]0_W2W;/]U".C\____@\00Z84```"0BURW;,9#`P#'0P0`````QT-8
+M`````(!C`?['0T``````@+WG_?__`'4.@#\&=0F#?P0`=#*-=@"#[`1H``(`
+M`&H`C97H_?__4NC\____C87H_?__4&HP:@"-0T10Z/S___^#Q"#K$8I7`8/B
+M`8I#`8/@_@G0B$,!@^P,4^C\____@\001HGR.%=$#X<\____@^P$:/````!J
+M`%?H_/___XM5"(N";"(``(E'"(FZ;"(``,='2`````"#Q!"-9?1;7E_)PU6)
+MY5=64X'L'`(``(M5#(M]$(M-%(L"BU@$B9WD_?__BUHDBU,8B97@_?__#[9`
+M`XF%W/W__XV5Z/W__X.]X/W__P!T#&8/MD,4@^`!.<AU)HF5X/W__X-['`!T
+M$%%24_]U"/]3'(/$$(7`=0JX`````.F6````#[=#-,'@"0^V<S")]HN5X/W_
+M_XL:N0```0!F@WH$`'0$#[=*!(G".<AV`HG*.[7<_?__=1!FB5<$9L='!@``
+MB1^#QPB0*=$!TRG0=2%&BY7D_?__#[9"1#G&=06^`````(N5Y/W__P^W0ES!
+MX`F%R76SBYW@_?__9H%[!@"`=`N#PPB)G>#]___K@F;'1_X`@+@!````C67T
+M6UY?R<.-=@!5B>575E.#[`R+10B+70R+>`2+4RB+2RR)5>B)3>R*2`-FBW,X
+M.$LP=1(/MT,VN@`````!1>@15>SK%9`X2S!V#P^W1URZ``````%%Z!%5[#A+
+M,'4'9@-S-(UV`#A+,74'9@-S,HUV`(![/`!T#CA+,',=.$LQ=Q+K%HGV.$LP
+M<P1F`W=<.$LQ=@9F`W=<B?:+31!FB7$,BT7HBU7LBTT0B4$$B5$(@\0,6UY?
+MR<.)]E6)Y5=64X/L'(M="(MU#&:+?@QFQT7F```/MDM&BT8$BU8(#ZW0T^KV
+MP2!T!XG0N@````")1>B)5>QFQT8X``#&1CP!#[9#1+H`````4E#_=>S_=>CH
+M_/___X/$$`^V2T8/I<+3X/;!('0'B<*X`````(E&*(E6+`^V0T2Z`````%)0
+M_W7L_W7HZ/S___^#Q!"(1C"(P6:+0UQ(9B-&!&:)1C9FBU-<9BG"9CGZ<@MF
+MB7XTOP````#K1F:)5C1F*==FQT7F`0"Z`0```(G0T^!F"48Z03A+1'4:L0"`
+M?CP`=`;&1CP`ZPQFBT8X9@-#7&:)1CAF.7M<<P9F*WM<Z\RX`0```-/@9@E&
+M.HA.,6:)?C(/MT7FC67T6UY?R<.)]E6)Y593BW4(BU4,BUHDBP(/MD@#N/[_
+M___3P&8A0SJ`>A4!=`:*0A6(0Q6#[`A25NC\____@\009H-[.@!U&H![%0!U
+M!,9#%0&#[`13_W,@5NC\____@\00C67X6U[)PU6)Y5=64X/L#(M]#(L'B47P
+M]D`!!'49QD<5`H/L!%?_=R#_=0CH_/___^G-````D(/L"%?_=?#H1/[__X/$
+M$&:#?SH`=1K&1Q4"@^P$5_]W(/]U".C\____Z9X```")]KX`````BU7P@'I$
+M``^$B@```(GV#[='.HGQT_BH`71L@^P,_W4(Z/S___^)PXM%\(M$L&R)1>R)
+M`XE[)(I7%(/B`HI#%(/@_0G0B$,4BE<4@^($@^#["="(0Q2*1PZ(0PY35_]U
+M[.@%_?__QT,@`````,=#'`````"#Q!13_W4(BU7L_U(\@\00C78`1HM-\`^V
+M040Y\`^/>/___XUE]%M>7\G#58GE@^P(BU4,BTH$BD$!P.@"@_`!@^`!@&(!
+M^X!A`?O^246`25X!@WE``'03A<!U#X/L"%'_=0C_44"#Q!")]LG#B?95B>56
+M4XM=$(MU%(M%#(M0)(M*&(7)=`QF#[9"%(/@`3GP=22#>AP`=!564U+_=0C_
+M4AR#Q!"Z`0```(7`=2>Z`````.L@B?:+`8M1!(D#B5,$@\,(C4$&@\$(9H,X
+M`'GGN@$```")T(UE^%M>R<.058GE5E.+=0@/M@:#Z`2#^`%W0;L`````@'Y$
+M`'0OB?:#?)YL`'03@^P,_W2>;.C-____@\00A<!U"K@`````ZQR-=@!##[9&
+M1#G8?].X`0```.L)BD8!P.@"@^`!C67X6U[)PU6)Y5=64X/L#(MU#(M>)(L[
+MBP:)1?"`?A4!#X30````]D,4`G1V@'L6``^%P````/9'7@(/A;8```#&0Q8!
+M@'@#`1G`@^`$BT0X;(E%\(7`=$-0Z$3___^#Q`2%P'0V@^P$:C1J`(U&*%#H
+M_/___XM%\`^V2`.X`0```-/@9HE#.HM%\(D&@\0(5O]U"/]0/.G'````BP:)
+M1?#K5/9#%`1T3O]U\.CR_O__@\0$A<!T(H/L!&HT:@"-1BA0Z/S___^#Q`A6
+M_W4(BT7P_U`\Z8D```"`>Q4!=!T/MU,ZBT7P#[9(`[@!````T^`YPG4'D(I&
+M%8A#%8/L"%;_=0CH_/___XM%\`^V2`.X_O___]/`9B%#.H/$$&:#>SH`=3Z`
+M>Q4`=03&0Q4!@^P$4_]S(/]U".C\____9O]/'H/$$&:#?QX`=1>+1R2%P'00
+M@^P(_W<H_W4(_]"#Q!")]HUE]%M>7\G#58GE4X/L!(M5#(M=$(M-%(M")(7)
+M=!F+0!")`V:+0@S!X`EFB4,$9L=#!@"`ZT60@W@8`'0F]D`4`70@BT@8D(L!
+MBU$$B0.)4P2#PPB-00:#P0AF@S@`>>?K&9"Z`````(-X'`!T$E%34/]U"/]0
+M'(G"ZP6Z`0```(G0BUW\R<-5B>575E.#[`R+10R+6"2)PHL`#[9(`[C^____
+MT\!F(4,Z@'H5`70&BD(5B$,5@^P(_W4,_W4(Z/S___^#Q!!F@WLZ`'5N@'L5
+M`'4XQD,5`8L+B4WPBW%DBWEHBTT,BU$(,?J+000Q\`G"=1@/MT$,N@`````!
+M\!'ZBTWPB4%DB5%HB?:#[`13_W,@_W4(Z/S___^+`V;'0!P``(/$#(L#@\`@
+M4&@`````_W4(Z/S___^#Q!"-9?1;7E_)PU6)Y5=64X/L#(M]#(MW)(!_%0%T
+M"XI'%8A&%>M[C78`@'\.('4^QD<5`(L&BT!PB0>#[`1J-&H`C4<H4.C\____
+MQD<6`,9'#C"*1Q2#X/V#R`2(1Q2#Q`B+!U?_=0C_4#SK;I#&1A4!BP:)1?"+
+M2&2+6&B+5P@QVHM'!#'("<)U%P^W1PRZ``````'($=J+3?")062)46B0@^P(
+M5_]U".C\____@\0,5O]V(/]U".C\____BP9FQT`<``"#Q`R+!H/`(%!H````
+M`/]U".C\____C67T6UY?R<.-=@!5B>575E.#[`R+?1"+312+50R+0B2%R71#
+MBU`0B1>+30QFBT$,P>`)9HE'!&;'1P8`@(L!@'@#``^$\@```(V"```!`&:#
+M?P0`=`</MT<$C00"B0?IUP```(UV`(G[BU4,#[=R#,'F"8-X&`!T)O9`%`%T
+M((M(&)"+`8M1!(D#B5,$@\,(C4$&@\$(9H,X`'GGZS60N@````"#>!P`#X22
+M````45=0_W4(_U`<@\00N@````"%P'1]ZPYFB7$$9L=!!@"`ZSV)]HGYB?:Z
+M```!`&:#>00`=`0/MU$$BUT,BP.`>`,`=!$Y\G81`3&)T&8I\&:)003K"SGR
+M<[Z#P0@IUG7)BU4,BP*`>`,`="(YSW0>B?:+`8M1!(D'B5<$@\<(C4$&@\$(
+M9H,X`'GGC78`N@$```")T(UE]%M>7\G#D%6)Y5=64X/L#(M]"(MU#(M>)(!^
+M%0%T#(I&%8A#%>F5````D(L&@'@#`'4PQD85`(L#BT!PB0:#[`1J-&H`C48H
+M4.C\____QD86`(/$"(L&5E?_4#SID0```(GV@^P$#[=&#,'@"5`#0Q!0_W,0
+MZ/S___^#Q!"%P'0&QD,5#.LXQD,5`8L+BT%DBU%HB47HB57LBU7L,U8(BT7H
+M,T8$"<)U%@^W1@RZ``````-%Z!-5[(E!9(E1:)"#[`A65^C\____@\0,4_]S
+M(%?H_/___XL#9L=`'```@\0,BP.#P"!0:`````!7Z/S___^-9?1;7E_)PY!5
+MB>575E.#[`R+?0B+70R+,\=&)`````#V1EX!#X5^`0``#[9##H/X`@^$$0$`
+M`(/X`G\0@_@!#X2C````Z5X!``")]H/X`P^%4P$``(M#!(M3"(E&%(E6&&:+
+M0PQFB48<@^P$#[=##,'@"5!J`/]S$.C\____9L=#.@,`QT7P`````(/$$(/L
+M#%?H_/___XG!QD`.,(M#!(M3"(E!!(E1"&:+0PQFB4$,@$D4!(E9),=!(```
+M``#'01P`````BU7PBT26;(D!@\0(45?_4#R#Q!#_1?"#??`!?JOIU````(M#
+M!(M3"(E&%(E6&&:+0PQFB48<@^P,5^C\____B<'&0`X@BT,$BU,(B4$$B5$(
+M9HM##&:)00R`210"B5DDQT$@`````,=!'`````"+1FR)`8/$"%%7_U`\@\00
+MZW:)]HM#!(M3"(E&%(E6&&:+0PQFB48<@^P,5^C\____B<'&0`X@BT,$BU,(
+MB4$$B5$(9HM##&:)00R`210"B5DDQT$@`````,=!'`````"+1FR)`8/$"%%7
+M_U`\@\00ZQ:)]L9#%0:#[`13_W,@5^C\____@\00@^P$C48@4&@`````5^C\
+M____C67T6UY?R<.)]E6)Y5=64X/L'(M]#(LW]D8!!'48QD<5`H/L!%?_=R#_
+M=0CH_/___^F8`0``]D<4('0JQT8D>!8``(E^*&:#?AX`#X5]`0``@^P(5_]U
+M".C=_?__@\00Z6D!``"0@WXD`'4^9H-^'`!T4P^W1ARZ``````-&%!-6&#M7
+M"'(_=P4[1P1V.`^W1PRZ``````-'!!-7"#E6&'((=R(Y1A1S'9#'1RP`````
+M@^P(5XU&(%#H_/___^D+`0``C78`9O]&'O9&7@-U>O9'%`)T=(M&;(M6<(`X
+M`W59@#H#=52+7P2)7>2+2EPI3>2+2&")3>B+4F`K6%QY`O?;@WWD`'D#]UWD
+M.UWD?13&1?,`A=)U-H-]Z`5V,,9%\P'K*L9%\P"#?>@`=06#^@5W&\9%\P'K
+M%9"`?F(`#Y1%\XI%\XA&8NL$QD7S`&;'1SH``+,`#[;#@WR&;`!T9HM$AFSV
+M0`$$=%SV1Q0$=04X7?-U48/L#/]U".C\____B47L@\0,:BA74.C\____#[;+
+MN`$```#3X&8)1SJ+5>R)>B3'0B``````QT(<`````(M$CFR)`H/$"%+_=0C_
+M4#R#Q!")]D.`^P%VBHGVC67T6UY?R<-5B>564XM-"(MU#(M>!`^V5@.`9@'[
+M@$M>`<=#9`````#'0V@`````_DM%@'M%`'4'@&,!^^L9D(I#7J@"=!&#X->(
+M0UZ%TG4'@&,!^XUV`/9#`01U&H-[0``/A.H```"#[`A34?]30(/$$.G:````
+MA=)U%(M3;,9"`P&+0W#&0`,`B4-LB5-P@WL$`'0)BT,$]D`!!'1UBT-L@#@#
+M=6V#[`A34>C\____B<*#Q!"%P'1:Q@`#QD`#`8E8!*$,````B4(\H1@```")
+M0D#V0P$0=!&`2@$0BT)4B4(,QT(0`````,=&!`````")4W"*0UZ#X/Z#R`J(
+M0U[^0T6#[`A2:@?H_/___X/$$(GV@WL$`'0+B?:+6P2#>P0`=?>+0U2)0UB+
+M0U")0U2+0TR)0U"+0TB)0TSH_/___XE#2,9#8P&#[`Q3Z/S___^-9?A;7LG#
+MC78`58GE5U93@>PL`@``BT4,BTT4BQ")E>3]__^+0"2)A=3]__^+6!C'A>#]
+M__\1````C97H_?__A=MT#&8/MD`4@^`!.<AU*HG3B[74_?__@WX<`'0045)6
+M_W4(_U8<@\00A<!U#+@`````Z0`!``")]HN]U/W__X!_,?\/A-,````/MT\T
+MP>$)O@```0!F@WL$`'0$#[=S!#GQ=DB0BY7D_?__BD(#B[W4_?__.D<P=12+
+M`XM3!(M]$(D'B5<$@\<(B7T0D"GQ@\,(O@```0!F@WL$`'0$#[=S!/^-X/W_
+M_SGQ=[F+E>3]__^*0@.+O=3]__\Z1S!U&8L#BU40B0)FB4H$9L="!@"`N`$`
+M``#K7Y`Y\7,VB<@#`XM]$(D'B?!F*<AFB4<$9H%[!@"`=0UFQT<&`("X`0``
+M`.LSBT409L=`!@``@\`(B440@\,(_XW@_?__@^P$BX7@_?__P>`#4%/_=1#H
+M_/___[@!````C67T6UY?R<.)]E6)Y8M5#(M-$(M%"(I``SI",'489HM"-&:)
+M00R+0BB+4BR)002)40CK&(GV9HM",F:)00S'000`````QT$(`````,G#58GE
+M5U93@^P(BU4(BT4,BU@$BW`(9HM`#&:)1?+'1>P`````N0````"`>D0`#X2<
+M````D(M\BFPY=Q!W!W)].5\,=GB+50R)6BB)<BR(2C"X`0```-/@9@E".@^W
+M1?*Z``````'8$?([5Q!R/G<%.T<,=C?'1>P!````C4$!BU4,B$(Q08M%[-/@
+M9@E".F:+1PQF*=AFB4(T9HM5\F8IPHG0BU4,9HE",NLDBT4,QD`Q_V:+5?)F
+MB5`TZQ,K7PP;=Q!!#[9"1#G(#X=E____BT7L@\0(6UY?R<.)]E6)Y593BW4(
+MBU4,BUHDBP(/MD@#N/[____3P&8A0SJ`>A4!=`:*0A6(0Q6#[`A25NC\____
+M@\009H-[.@!U&H![%0!U!,9#%0&#[`13_W,@5NC\____@\00C67X6U[)PU6)
+MY5=64X/L#(M]#(L'B47P]D`!!'49QD<5`H/L!%?_=R#_=0CH_/___^G)````
+MD%?_=?#HB_[__X/$"&:#?SH`=1G&1Q4"@^P$5_]W(/]U".C\____Z9T```"0
+MO@````"+5?"`>D0`#X2*````B?8/MT<ZB?'3^*@!=&R#[`S_=0CH_/___XG#
+MBT7PBT2P;(E%[(D#B7LDBE<4@^("BD,4@^#]"="(0Q2*5Q2#X@2#X/L)T(A#
+M%(I'#HA##E-7_W7LZ+G]___'0R``````QT,<`````(/$%%/_=0B+5>S_4CR#
+MQ!"-=@!&BTWP#[9!1#GP#X]X____C67T6UY?R<-5B>6#[`B+10R+4`2`8`'[
+M@&(!^_Y*18!*7@&#>D``=`^#[`A2_W4(_U)`@\00B?;)PXGV58GE4XM="(M-
+M#(N4BXPQ``"%TG0)BP*)A(N,,0``B=!;R<.058GE4XM="(M-#(M5$(N$DXPQ
+M``")`8F,DXPQ``!;R<-5B>575E.#[`R+10R[`(```#T`0```=V%FNP!`/0`@
+M``!W5F:[`"`]`!```'=+9KL`$#T`"```=T!FNP`(/0`$``!W-6:[``0]``(`
+M`'<J9KL``CT``0``=Q]FNP`!/8````!W%&:[@`"#^$!W"X/X(1G;@^/@@\-`
+MBT40#Z_#!?\/``#!Z`R)1?#'1>P`````N`````"!^P`0```/A]8```#_3?"#
+M??#_#X3&````D(/L#/]U".C\____B<;_1>R_`````(/$$+@`$```N@````#W
+M\X/X``^&BP```(UV`+@*````@?L`0```=U>P"8'[`"```'=-L`B!^P`0``!W
+M0[`'@?L`"```=SFP!H'[``0``'<OL`6!^P`"``!W);`$@?L``0``=QNP`X'[
+M@````'<1L`*#^T!W"H/[(`^7P`^VP)!05O]U".B2_O__`=Z#Q`Q'N``0``"Z
+M`````/?S.?@/AWC_____3?"#??#_#X4[____BT7LC67T6UY?R<-5B>564XMU
+M"(M=#+@`````.9ZX,0``<DIJ`E;H'/[__XG!@\0(A<!T-XD82X/[_W0OC78`
+MN@````"#OK0Q````=!2+EK0Q``"+`HF&M#$``/^.N#$``(E4F01+@_O_==2)
+MR(UE^%M>R<.-=@!5B>564XM="(MU#(L.28/Y_W0?D(M4C@2+@[0Q``")`HF3
+MM#$``/^#N#$``$F#^?]UXFH"5E/HN?W__XUE^%M>R<.)]E6)Y593BW4(BQY+
+M@_O_=!R#[`1H`!```&H`_W2>!.C\____@\002X/[_W7DC67X6U[)PY!5B>53
+MBUT(BPM)@_G_=">+1(L$N@````"#.`!T![@`````ZQ="@\`$@?K_`P``=NA)
+M@_G_==FX`0```%O)PXGV58GE@^P,:A!J`/]U".C\____R<-5B>6+30R#>10`
+M=`F+412+01")0A"+41"+012)`L=!$`````!J`%'_=0CH^?S__\G#C78`58GE
+M4XM5"(M-#(U:.(M".(E!%(7`=`F+4CB-012)0A")61")"UO)PU6)Y5=64XMU
+M&(M]'(M%#(M8.(7;="V)]CE[!'<?<@0Y,W,9BU44.5,,<A%W"(M5$#E3"'8'
+MN`````#K+XM;%(7;==5J`/]U".A;_/__B<.+11"+512)`XE3!(ES"(E[#%/_
+M=0SH;____XG8C67T6UY?R<.058GE4XM-"(I1+0^VVHT$W0`````IV(U<@3!"
+MB%$MB0O&0P4`QD,&`,9#!P")V%O)PXUV`%6)Y5=64XM]"(MU#+L`````#[96
+M*HU"_='X`<*#^@!^(XUV`/^TGK`'``!7Z/_]__^#Q`A##[96*HU"_='X`<(Y
+MVG_@:@=65^C7^___C67T6UY?R<.-=@!5B>575E.#[!R*10R(1?.+51AFB57P
+MBT4(BYBX,0``P>,##[?*#[95\XU"_='X`<(/K\HYRP^#X0````^V5?.)5>2)
+MT(E5X`^W5?"-0/W1^`-%Y(G3#Z_8BT4(!70Q``"+50@Y@G0Q```/A,L```"+
+MLG0Q``"+5@2+!HE0!(D"B3:)=@2#?A``=!Z#?A0`=`F+5A2+1A")0A"+5A"+
+M1A2)`L=&$`````"+1?!F.48H=4N*5?,X5BIU0XDVB78$BT40BU44B48@B58D
+MOP````"#?>0`?B"+7>"#[`R)^,'@!(V$,%@(``!0Z*/]__^#Q!!'.?M_XXGP
+MZ1L!``!6_W4(Z*/^__^#Q`B+50B+@K@Q``#!X`,YV`^"//___VH'_W4(Z(+Z
+M__^)QH/$"+@`````A?8/A-\```#K"K@`````Z=,```")-HEV!(M%$(M5%(E&
+M((E6)(M5\&:)5BB*1?.(1BK'1A@`````OP`````/MM"-0OW1^`'"@_H`#XZ4
+M````BT7P9L'H`P^WP(E%Z`^V5?.-0OW1^`'"B57L_W7H_W4(Z,7[__^)A+ZP
+M!P``@\0(A<!U,T^#__]T&9#_M+ZP!P``_W4(Z`W\__^#Q`A/@___=>AJ!U;_
+M=0CH[?G__[@`````ZS&)]H/L#(G[P>,$`?.-@]@'``!0Z)+\__^!PU@(``")
+M'"3HA/S__X/$$$<Y?>Q_AXGPC67T6UY?R<.058GE5U93@^P,BW48BT4,#[9(
+M1HM%$-/H)?\#``"+30R+432-'(*+.X7_="^+510S5R2+11`S1R`)PG48BU<$
+MBP>)4`2)`HD_B7\$B?CIF0```(GVBW\4A?]UT8/L#`^WQE#_=13_=1"+50P/
+MMD)$4/]U".AH_?__B<>#Q""%P'1HBTT,B4@<BP.)1Q2%P'0(BQ.-1Q2)0A")
+M7Q").XM%#(I81(A?*@^V2$:+11"+510/K=#3ZO;!('0'B="Z`````(E%Z(E5
+M[(M-#`^V042Z`````%)0_W7L_W7HZ/S___^#Q!`HPTN(7RN)^(UE]%M>7\G#
+MD%6)Y5=64X'LS````(M%#(I0%(C0T.B#\`&)P8/A`8F-5/___XM%#(-X&`!T
+M!?;"`70@C958____B950____:@%2_W4,_W4(BTT,_U$<@\00ZPR+10R+0!B)
+MA5#____'A3C___\```$`BY50____9H-Z!`!T"@^W2@2)C3C____'A4#___\`
+M````QX5,____`````(M%#(!X2``/A-P!``"+E4S___^+30R+5)$TB95$____
+MBHHP"0``B(U+____.(HQ"0``#X*8`0``BH5+____BY5$____.$(K#X1E`0``
+M#[;`B=%FBY2"$`D``&:+A($2"0``9HF%/O___XG09L'H`P^WP(F%-/___X/B
+M!V:)E3S___\/M\*)PL'B";@(````9BN%//___V:)A3S___]FBX4^____9CF%
+M//___W8'9HF%//___P^WA3S___^)QL'F"0^VA4O___^+C43___^+A(&P!P``
+MB=>+C33___\#?(@$D(N%./___SF%0/___W4R@X50____",>%./___P```0"+
+ME5#___]F@WH$`'0*#[=*!(F-./___\>%0/___P````"+G3C___\KG4#___\Y
+M\W8"B?.#O53___\`="&#[`13BX5`____BY50____`P)05^C\____@\00ZQ^-
+M=@"#[`135XN%0/___XN-4/___P,!4.C\____@\00`9U`____*=YT!P'?Z5?_
+M__^+A3S___]F*84^____=!G_A33___^Z`````&;'A3S___\(`.GK_O___H5+
+M____BI5+____BXU$____.)$Q"0``#X-H_O___X5,____BU4,#[9"2#N%3/__
+M_P^/)/[__X/L!&H`_W4,_W4(_U40C67T6UY?R<.)]E6)Y5=64X/L#(-]$`!T
+M!XM%#,9`%0O'1?``````BU4,@'I(``^$P0```(M%"`5\,0``B47LBU4(@<)T
+M,0``B57HC78`BT7PBU4,BUR"-,>#"`D```````")W[$`@'LJ`'9!B?8/ML'!
+MX`0!^(V0V`<``+X`````@[C8!P```'44@WH$`'4.@WH(`'4(@WH,`'0'B?:^
+M`0```(7V=2%!.$\J=\&X`````(7`=!R+5>R+0@2)6@2)$XE#!(D8ZQJX`0``
+M`.OCC78`BU7HBT($B5H$B1.)0P2)&/]%\(M5#`^V0D@[1?`/CUG___^#[`C_
+M=0S_=0CH0T```(UE]%M>7\G#C78`58GE5U93@^P\BW4,B[X("0``BT4(_XB\
+M,0``C48(BT@$BU8(B4H$B1&)1@B)0`2%_P^$Y0(``/9'%"`/A)<!``"+%XE5
+MZ`^W1BBZ``````-&(!-6)(E%X(E5Y&:+3BAFB4W>QX8("0```````(M%"`5T
+M,0``BU`$B7`$B0:)5@2),H"^,@D```%U<H!_#@)U8P^V1BO_M(:P!P``Z`/W
+M__^#Q`2%P'5,QH8R"0``#(M%Z(!(7@J+5>@/MT)<]]B9BTX@(<&+7B0ATXM%
+MZ#E8:'(B=P4Y2&1V&XM5Z`^W0ESWV)DC1B`C5B2+3>B)062)46B)]H"^,@D`
+M``%T&HJ&,@D``(A'%8/L"%?_=0CH)C\``.FI`P``BT7H#[=`7(E%N,=%O```
+M``"+3P2+7PB#P0&#TP"+1;CWX8E%T(E5U(M5N`^OTXMUU`'6BT6\#Z_!C00&
+MB474BTWD.<AR/7<(BU7@.570=C.#[`P/MT7>4/]UY/]UX/]UZ/]U".CT^?__
+MB;@("0``B4<T@\084/]U".B_#P``Z28#``"+3>@/MT%<N@`````#060346B+
+M3>0QT3-%X`G!=0^+1>"+5>2+3>B)062)46B#[`A7_W4(Z&4^``#IZ`(``(I'
+M24"(1TDZ1T@/A60"``#V1Q0$#X3"````QD7O`(!_2``/AJ````"+10@%=#$`
+M`(E%Q(M5"('"?#$``(E5P)`/MD7OBW2'-,>&"`D```````"`OC()```!=46)
+M=<BS`(!^*@!V)XGV@^P,#[;#P>`$BTW(C80(V`<``%#HB_7__X/$$$.+1<@X
+M6"IWVXM5Q(M"!(ER!(D6B48$B3#K&I"*AC()``"(1Q6+3<"+002)<02)#HE&
+M!(DP_D7OBD7O.$=(#X=X____@^P(5_]U".B,/0``@\00Z9@!``#&1>\`@'](
+M`'8GB?8/MD7OBW2'-("^,@D```%T"8J&,@D``(A'%?Y%[XI5[SA72'?;@'\5
+M`'49@^P$:)0I``!7_W4(Z&SY__^#Q!#I2`$``(/L!&H`5_]U".CJ^___@\00
+MZ3(!``")]H"^,@D```%U=XGWLP"`?BH`=B"0@^P,#[;#P>`$C80XV`<``%#H
+MGO3__X/$$$,X7RIWX8.^#`D```!T*HN&#`D``(F&"`D``,>&#`D```````"#
+M[`A6_W4(Z-H-``"#Q!#IR@```(M%"`5T,0``BU`$B7`$B0:)5@2),NM7C78`
+MBT4(!7PQ``"+4`2)<`2)!HE6!(DR@[X,"0```'0VBY8,"0``BH8R"0``B$(5
+M@^P$_[8,"0``:-!J``#_=0CH_/___\>&#`D```````#IT@```(GV_W88_W4(
+MZ/WS___'1A@`````BT8<@\0(@W@@`'09@^P$@\`@4&@`````_W4(Z/S___^#
+MQ!#K)8M-"(.YQ#$```!T&8/L!(G(!<0Q``!0:`````!1Z/S___^#Q!"+10B#
+MN,`Q````=&B`OC()```!=5^+1AR#>"P`=#V#[`Q0Z#$Z``"#Q!"%P'4M@^P$
+MBT8<_W`PBT8<_W`L_W4(Z/S___^+1AS'0"P`````BU4(_XK`,0``@\00BTT(
+M@[G`,0```'0-@^P,4>C\____@\00D(UE]%M>7\G#58GE@^P(BT4,BP"`>"X`
+M=!O&0"X`@^P$4&B(.0``_W4(Z/S___^#Q!"-=@#)PXGV58GE5U93@^P,BU4,
+MBWT0BT(DBP@/MD`)BX2!L`<``(E%\(M"!"M!(,'@"0^W<@S!Y@F)PH'B_P\`
+M`,'H#(E%[+@`$```B<,ITSGS=@*)\X-]%`!T#XM%[(M-\`-4@03K&XUV`(/L
+M#(M%[(M-\`-4@012Z/S___^#Q!")PHD79HE?!#GS=!5FQT<&``"#QPC_1>PI
+MWKH`````ZZ5FQT<&`("X`0```(UE]%M>7\G#B?95B>575E.+50B*0@H"0@N(
+M0@J-6@R^`0```.L3@'H*`'D*N`````#I\@```/Y""@^V2@J#^1]W$(GPT^"%
+M0@P/E<`/ML#K39"#^3]W$X/I((GPT^"%0P0/E<`/ML#K-9"#^5]W$X/I0(GP
+MT^"%0P@/E<`/ML#K'9"X`````(/Y?W<2@^E@B?#3X(5##`^5P`^VP(GVA<!T
+MA,9""P&->@P/MEH*O@$```#K!8GV_D(+#[9""XT,`X/Y'W<1B?#3X(5"#`^5
+MP`^VP.M.B?:#^3]W$X/I((GPT^"%1P0/E<`/ML#K-9"#^5]W$H/I0(GPT^"%
+M1P@/E<`/ML#K';@`````@_E_=Q.#Z6")\-/@A4<,#Y7`#[;`C78`A<!UD+`!
+M6UY?R<.058GE5U93@^P4BWT(BU4,BUHDBS.*0A6(0P925^C\____@\00@'L&
+M`75F4^BL_O__@\0$A<!T$<9#!@"#[`A35^B/````ZU>0@'L$`75"BD,(.$8K
+M=#HX0PEU-0^VR(G(P>`$C8PP6`@``(US#(L!"T,,B0&+000+1@2)002+00@+
+M1@B)00B+00P+1@R)00R0QD,%`H/L"%-7Z%[]__^-9?1;7E_)PXGV58GE4X/L
+M!(M=#,9#"P#&0PH`4^@<_O__@^P$4_]U".@(````BUW\R<.-=@!5B>575E.#
+M[!B+70R+,XM6'`^V0PB+?()L_W4(Z/S___^)P8DX@\00@'L$`74*@$@4`L9`
+M#B#K"(!(%`3&0`XP#[9#"KH``````T8@$U8DB4$$B5$(9@^V0PMFB4$,QT$<
+MD"\``(E9),=!(&0Q``"#[`A1_W4(_U<\C67T6UY?R<.-=@!5B>575E.#[`B+
+M?0R^`````,=%\`````"+10B)1>SK$X-]\']V"HGPZ4<!``"-=@#_1?"+5?"#
+M^A]W%[@!````BDWPT^"+50B%`@^5P`^VP.M:@_H_=QB-2N"X`0```-/@BU7L
+MA4($#Y7`#[;`ZSV#^E]W&(U*P+@!````T^"+5>R%0@@/E<`/ML#K(+@`````
+M@_I_=Q:-2J"P`=/@BU7LA4(,#Y7`#[;`C78`A<`/A&S___^[`0```(M5".L#
+MB?9#BTWP`=F#^1]W$;@!````T^"%`@^5P`^VP.M4@_D_=Q>#Z2"X`0```-/@
+MA4($#Y7`#[;`ZSJ)]H/Y7W<7@^E`N`$```#3X(5""`^5P`^VP.L>B?:X````
+M`(/Y?W<2@^E@L`'3X(5"#`^5P`^VP(GVA<!UC(M5\`%=\(GVB=#!Z`.(!+>(
+MT(/@!XA$MP&Q""C!B$RW`@^VP3G8<P<IPP'"1NO9B%RW`D:#??!_#X;!_O__
+MB?"#Q`A;7E_)PU6)Y5=64X'L3`$``(M%#(LPBU@(@'L5``^$.@$``(-[$`!U
+M"(E#$.GF`0``QD,5`(V%Z/[__U!3Z$?^__^)A<S^__\/MD,4BX2&L`<``(F%
+MW/[__XM#$`^V0`R+A(:P!P``B878_O__BU4,#[9"#(N$AK`'``")A=3^__^[
+M`````(/$"#N=S/[__P^-<@$```^VA)WI_O__B<'!X0D/MH2=ZO[__XG&P>8)
+MBH2=Z/[__XB%T_[__P^VT(G(B[W8_O__`T27!(F%X/[__XN%U/[__XM\D`0!
+MSXF]M/[__XF]Y/[__XN%W/[__XM\D`0!SSF]X/[__W4D@^P$5O^UM/[__XG(
+MB[7<_O__`T26!%#H_/___X/$$.LJC78`5O^UY/[___^UX/[__P^VA=/^__^+
+MO=S^__\#3(<$4>C\____@\000SN=S/[__P^,0/___^FM````C78`C87H_O__
+M4%/H'_W__XF%S/[__P^V0Q2+A(:P!P``B87<_O__BU4,#[9"#(N$AK`'``")
+MA=C^__^[`````(/$"#N=S/[__WUB#[:$G>G^__^)P<'A"0^VA)WJ_O__B<;!
+MY@D/MI2=Z/[__XG(B[W<_O__`T27!(F%X/[__XG(B[W8_O__`T27!(F%Y/[_
+M_X/L!%90_[7@_O__Z/S___^#Q!!#.YW,_O__?)Z#[`1J`/]U#/]U".AJ`@``
+MC67T6UY?R<.)]E6)Y5=64X'L+`$``(M%#(L8C87H_O__4(M%#(/`"%#H1/S_
+M_XF%V/[__XM5#`^V0AB+A(.P!P``B87<_O__#[9"&8N$@[`'``")A=3^__^^
+M`````(/$"#NUV/[__WU=D`^VA+7I_O__P>`)#[:,M>K^___!X0D/MIRUZ/[_
+M_XG"B[W<_O__`U2?!(F5X/[__XN5U/[__P-$F@2)A>3^__^#[`114/^UX/[_
+M_^C\____@\001CNUV/[__WRD@^P$:@#_=0S_=0CHE`$``(UE]%M>7\G#58GE
+M5U93@>Q\`0``BT4,BS"+6`B-A>C^__]04^AS^___B864_O__#[9#%(N$AK`'
+M``")A:3^__^Q`(/$"(M5#(!Z%`!V)(I:%(UV``^VP8M]#`^V5`<,BY26L`<`
+M`(F4A<C^__]!.,MWXL>%F/[__P````"+A93^__\YA9C^__\/C>X```")]HN5
+MF/[__P^VA)7I_O__B<?!YPD/MH25ZO[__\'@"8F%H/[__XJ,E>C^__^(C9_^
+M__^Q`(M=#(![%`!V,0^VM9_^__^*0Q2(A8?^__\/ML&+E(7(_O__BURR!(T4
+M'XF4A:C^__]!.(V'_O__=]\/MI6?_O__B?B+C:3^__\#1)$$.86H_O__=2"#
+M[`3_M:#^____M:S^__^)^`-$D010Z/S___^#Q!#K+/^UH/[___^UK/[___^U
+MJ/[__P^VA9_^__^+G:3^__\#?(,$5^C\____@\00_X68_O__B[V4_O__.;V8
+M_O__#XP4____@^P$:@#_=0S_=0CH"````(UE]%M>7\G#58GE5U93@^P4BWT(
+MBUT,BW40QD,%`H/^`1G`@^#U@\`,B$,&4U?H@?;__X/$$(![!`-U,(M#"(M0
+M$(72=";'0!``````QD(%`H/^`1G`@^#U@\`,B$(&@^P(4E?H2_;__X/$$(UE
+M]%M>7\G#58GE5E.+70B+=0P/MDX'A<ET29`/MI1+P`8``(T$U0`````IT(U$
+M@S"`>`4"=`FX`````.LKB?:`>`8!=!+&1@4"BD`&B$8&N`````#K$I`/MHQ+
+MP08``(7)=;BX`0```%M>R<.-=@!5B>575E.#[!R+?0S&1RX`@+\S"0```'1[
+MO@````"-7S"`?RT`=$F0@'L%`74V@'L$`W4EBT,(@'@5`'0<@W@0`'06QT`0
+M`````,9#!0**AS()``"(0P;K"\9'+@'I]P$``(GV1H/#'`^V1RTY\'^X@^P(
+M5_]U".B4\/__Z=@!``"*0P:(AS()``#&AS,)```!ZX:0QT7P`````,=%[`$`
+M``"^`````(U?,(!_+0`/A'P!``"`>P4`#X66````4U?HT_[__X/$"(7`='S&
+M0P4!BT4(B47HB=H/MD,$@_@&=V?_)(4`````@^P(4_]UZ.B0]___@\00ZT^#
+M[`A3_W7HZ*OY__^#Q!#K/H/L"%/_=>CHMOO__X/$$.LM@^P(4_]UZ.AY_/__
+M@\00ZQS&1R\!QD(%`L9"!@&#[`A2_W7HZ)3T__^#Q!"0@'L%`G4,_T7PC78`
+M@'L%`G0.QT7L`````.FZ````B?:`>P8!#X2N````@'\O``^$!?___XM''`^V
+M4PB+1)!L]D`!!`^$B````,=%[`````#&0P4!QD,&`(M%"(E%Y(G:#[9#!(/X
+M!G=O_R2%'````(/L"%/_=>3HP/;__X/$$.M7@^P(4_]UY.C;^/__@\00ZT:#
+M[`A3_W7DZ.;Z__^#Q!#K-8/L"%/_=>3HJ?O__X/$$.LDQD<O`<9"!0+&0@8!
+M@^P(4O]UY.C$\___@\00ZP?&0P8!C78`1H/#'`^V1RTY\`^/A/[__X-]\``/
+MA5K^__^#?>P`=!7&AS()```!@^P(5_]U".B][O__ZP3&1RX!C67T6UY?R<.-
+M=@!5B>575E.![&P!``"+10R+>!R+D`@)``")E<3^___&A:W^__\`A=)U.XG!
+M#[=`*+H`````B<Z+22"+7B0!R!':4E!345?_=0CHL^;__XE&&(/$&+C_____
+M@WX8``^$42L``(GVBT4(_X"\,0``BT4,@\`(BU4(@<*$,0``BTH$B4($BUT,
+MB5,(B4@$B0%F@W]@`'1(]D=>`742BT=DBU=H.U,D<@=W*CM#('<EO@`````/
+MMT=@B<*H`74,D$:)T(GQT_BH`73UB?.+?0R(7RSK.HM%#,9`+`CK,8GV]D=>
+M`G0BBT=DBU=HBTT,.U$D=Q1R!3M!('<-BUT,BD,KB$,LZPF)]HMU#,9&+`B+
+M?0S&1RT`QH?`!@```,9'+P#&AS()````QH<S"0```,>'#`D```````"#O<3^
+M__\`#X1^$P``BX7$_O__]D`4(`^$=@D```^V0`Z#^`(/A(P%``"#^`)_#H/X
+M`0^$1`$``.DN*@``@_@##X4E*@``@^P,C9W8_O__4^@&Y?__@\00OP````"+
+M50P/MW(HLP"#_Q]W4@^VPXV4A=C^__^)O:C^__^Y(````"GY.?%V`HGQ@_D@
+M=0C'`O_____K$K@!````T^!(BHVH_O__T^`)`K@@````*?@Y\',4C70^X+\`
+M````ZP.#[R!#@/L#=J"#[`R+70S_L[`'``#H$.3__\:%K_[__P"#Q!"`>RH`
+M#X:`*0``@^P,#[:=K_[__\'C!`-=#(V#V`<``%#H5.3__X'#6`@``(D<).A&
+MY/__@\00_W4,Z!_E__^#Q`3&0`0"BI6O_O__B%`(QD`)`(N5V/[__XE0#(N5
+MW/[__XE0$(N5X/[__XE0%(N5Y/[__XE0&/Z%K_[__XJ-K_[__XM=##A+*@^'
+M>____^GV*```N`$```#K38MU#+$`@'XJ`'8]#[;!P>`$`?"-D-@'``"[````
+M`(.XV`<```!U$H-Z!`!U#(-Z"`!U!H-Z#`!T!;L!````A=MUMT$X3BIWP[@`
+M````A<!T'HN]Q/[__XMU#(F^#`D``,>&"`D```````#IH!$``(/L#(V=V/[_
+M_U/H8>/__X/$$+\`````BT4,#[=P*+,`@_\?=U(/ML.-E(78_O__B;VD_O__
+MN2`````I^3GQ=@*)\8/Y('4(QP+_____ZQ*X`0```-/@2(J-I/[__]/@"0*X
+M(````"GX.?!S%(UT/N"_`````.L#@^\@0X#[`W:@BUT,@'LL"'4&BD,KB$,L
+MBW4,BEXLB)[L"```QH;M"````8N%V/[__XF&V`@``(N%W/[__XF&W`@``(N%
+MX/[__XF&X`@``(N%Y/[__XF&Y`@``,>&Z`@```````!6Z'+C__^#Q`3&0`0"
+MB%@(B%@)BY78_O__B5`,BY7<_O__B5`0BY7@_O__B5`4BY7D_O__B5`8B<>`
+M?BH"#X=4`0``B?.!P]@(``!6Z"?C__^#Q`3&0`0%B5@(QD`4`(F%O/[__XL?
+M_H/`!@``#[:SP`8``(G"*=J#ZC#!^@*-!-*-!,*-!,*-!,*)P<'A#P'(C03"
+M]]B(A'/`!@``BD<'B(1SP08``(GPB$<'QH6O_O__`(M5#(!Z*@`/AN0F``"*
+MC:_^__^+70PX2RP/A*@```!3Z*#B__^#Q`3&0`0!BI6O_O__B%`(B%`)BY78
+M_O__B5`,BY7<_O__B5`0BY7@_O__B5`4BY7D_O__B5`8BXV\_O__BE$4#[;*
+MBIVO_O__B[6\_O__B%P.#$*(5A2+'OZ#P`8```^VL\`&```IV(/H,,'X`HT4
+MP(T4T(T4T(T4T(G1P>$/`<J-%-#WVHB4<\`&``"+O;S^__^*1P>(A'/!!@``
+MB?"(1P?^A:_^__^*E:_^__^+30PX42H/AR[____I#28``,:%K_[__P"+70R`
+M>RH`#X;Y)0``@</8"```B9V@_O__BH6O_O__BU4,.$(L#X0"`0``4NBIX?__
+M@\0$QD`$`8J-K_[__XA("(A("8N5V/[__XE0#(N5W/[__XE0$(N5X/[__XE0
+M%(N5Y/[__XE0&(F%P/[___]U#.ADX?__@\0$QD`$`XN=H/[__XE8"(J5K_[_
+M_XA0#(F%O/[__XL8_H/`!@``#[:SP`8``(N5P/[__RG:@^HPP?H"C032C03"
+MC03"C03"B<'!X0\!R(T$PO?8B(1SP`8``(N-O/[__XI!!XB$<\$&``")\XA9
+M!XL?_H/`!@``#[:SP`8``(G**=J#ZC#!^@*-!-*-!,*-!,*-!,*)P<'A#P'(
+MC03"]]B(A'/`!@``BD<'B(1SP08``(GPB$<'_H6O_O__BI6O_O__BTT,.%$J
+M#X?4_O__Z;PD``"X`0```.M1BW4,L0"`?BH`=D&)]@^VP<'@!`'PC9#8!P``
+MNP````"#N-@'````=12#>@0`=0Z#>@@`=0B#>@P`=`>)]KL!````A=MULT$X
+M3BIWP;@`````A<!T'HNUQ/[__XM=#(FS#`D``,>#"`D```````#I8@T``(/L
+M#(V=V/[__U/H(]___X/$$+\`````BT4,#[=P*+,`@_\?=U(/ML.-E(78_O__
+MB;V<_O__N2`````I^3GQ=@*)\8/Y('4(QP+_____ZQ*X`0```-/@2(J-G/[_
+M_]/@"0*X(````"GX.?!S%(UT/N"_`````.L#@^\@0X#[`W:@BUT,BD,KB(/L
+M"```QH/M"````(N%V/[__XF#V`@``(N%W/[__XF#W`@``(N%X/[__XF#X`@`
+M`(N%Y/[__XF#Y`@``,>#Z`@```````"`>RH"#X<``0``BUT,@</8"```_W4,
+MZ"[?__^#Q`3&0`0%B5@(QD`4`(F%O/[__\:%K_[__P"+=0R`?BH`#X8R(P``
+M_W4,Z/[>__^#Q`3&0`0!BI6O_O__B%`(B%`)BY78_O__B5`,BY7<_O__B5`0
+MBY7@_O__B5`4BY7D_O__B5`8BXV\_O__BE$4#[;*BIVO_O__B[6\_O__B%P.
+M#$*(5A2+'OZ#P`8```^VL\`&```IV(/H,,'X`HT4P(T4T(T4T(T4T(G1P>$/
+M`<J-%-#WVHB4<\`&``"+O;S^__^*1P>(A'/!!@``B?"(1P?^A:_^__^*E:_^
+M__^+30PX42H/AS[____I:R(``(MU#(I>*U;H,][__X/$!,9`!`&(6`B(6`F+
+ME=C^__^)4`R+E=S^__^)4!"+E>#^__^)4!2+E>3^__^)4!B)A;C^___&A:_^
+M__\`@'XJ``^&%R(``(GW@<?8"```BH6O_O__BU4,.$(K#X0&`0``4NC+W?__
+M@\0$QD`$`8J-K_[__XA("(A("8N5V/[__XE0#(N5W/[__XE0$(N5X/[__XE0
+M%(N5Y/[__XE0&(F%P/[___]U#.B&W?__@\0$QD`$`XEX"(J=K_[__XA8#(F%
+MO/[__XL8_H/`!@``#[:SP`8``(N5P/[__RG:@^HPP?H"C032C03"C03"C03"
+MB<'!X0\!R(T$PO?8B(1SP`8``(N5O/[__XI"!XB$<\$&``")\8A*!XL:_H/`
+M!@``#[:SP`8``(N5N/[__RG:@^HPP?H"C032C03"C03"C03"B<'!X0\!R(T$
+MPO?8B(1SP`8``(N5O/[__XI"!XB$<\$&``")\8A*!_Z%K_[__XJ=K_[__XMU
+M##A>*@^'T/[__^G:(```B?:#O<3^__\`#X3K"0``B[W$_O__]D<4`@^$VPD`
+M`(M%#(J`,`D``(B%K_[__XM5##B",0D```^"B`$``)"*C:_^__^+70PX2RL/
+MA%H!``"#[`P/MMF)V,'@!(VT!6C___]6Z&+;__^#Q!")M9C^__^+=0P/MY2>
+M$`D```^WO)X2"0``O@````"0@_H?=U>)\0^VP8N-F/[__XT<@8F5E/[__[D@
+M````*=$Y^78"B?F#^2!U"<<#_____^L3D+@!````T^!(BHV4_O__T^`)`[@@
+M````*=`Y^',6C7P7X+H`````ZP.#ZB!&B?.`^P-VF0^VG:_^__^)VHG8P>`$
+MBW4,C9PP6`@``(L+]]&)C=C^__^+0P3WT(F%W/[__XM#"/?0B87@_O__BT,,
+M]]")A>3^__^)UL'F!(V<->C^__^-E#5H____(PJ)"XM"!".%W/[__XE#!(M"
+M"".%X/[__XE#"(M"#".%Y/[__XE##(J%K_[__XM5##A"+'4MN@````"#.P!U
+M$H-[!`!U#(-["`!U!H-[#`!T!;H!````A=)T",:%K?[__P&0_H6O_O__BHVO
+M_O__BUT,.(LQ"0``#X-Y_O__@+VM_O__``^$9`<``(/L#(V%V/[__U#HZ=G_
+M_\:%K_[__P"#Q!"+=0R`?BH`=G>)]HJ%K_[__XM5##A"+'13.$(K=$X/MM#!
+MX@0#50R-BM@'``"+A=C^__\+@M@'``")A=C^__^+A=S^__\+002)A=S^__^+
+MA>#^__\+00B)A>#^__^+A>3^__\+00R)A>3^___^A:_^__^*C:_^__^+70PX
+M2RIWBXMU#`^V?BS!YP2-G#WH_O__B[78_O__(S.)M<C^__^+C=S^__\C2P2)
+MC<S^__^+E>#^__\C4PB)E=#^__^+A>3^__\C0PR)A=3^___WUHFUV/[___?1
+MB8W<_O__]]*)E>#^___WT(F%Y/[__R,SB;78_O__(TL$B8W<_O__(U,(B97@
+M_O__(T,,B87D_O___W4,Z*S9__^#Q`3&0`0`B86P_O__N`````"#O=C^__\`
+M=1N#O=S^__\`=1*#O>#^__\`=0F#O>3^__\`=`6X`0```(7`#X0\`@``BWT,
+MBD<LB(?L"```BX78_O__B8?8"```BX7<_O__B8?<"```BX7@_O__B8?@"```
+MBX7D_O__B8?D"```QH?M"````<>'Z`@```````#&A:_^__\`@'\J``^&I@(`
+M`(J%K_[__XM5##A"+`^$KP$``#B",`D``'<-.((Q"0``<@4X0BMU1_]U#.C<
+MV/__@\0$QD`$`8J-K_[__XA("(A("8N5V/[__XE0#(N5W/[__XE0$(N5X/[_
+M_XE0%(N5Y/[__XE0&(F%P/[__^MW#[:-K_[__\'A!(V<#>C^__^+A=C^__\+
+M`XD#BX7<_O__"T,$B4,$BX7@_O__"T,(B4,(BX7D_O__"T,,B4,,_W4,Z%;8
+M__^#Q`3&0`0!BI6O_O__B%`(B%`)BQ.)4`R+4P2)4!"+4PB)4!2+4PR)4!B)
+MA<#^__^+70R!P]@(``#_=0SH%=C__X/$!,9`!`.)6`B*C:_^__^(2`R)A;S^
+M__^+&/Z#P`8```^VL\`&``"+E<#^__\IVH/J,,'Z`HT$THT$PHT$PHT$PHG!
+MP>$/`<B-!,+WV(B$<\`&``"+O;S^__^*1P>(A'/!!@``B?"(1P>*E:_^__^+
+M30PX42MU4XNUL/[__XL>_H/`!@``#[:SP`8``(GZ*=J#ZC#!^@*-!-*-!,*-
+M!,*-!,*)P<'A#P'(C03"]]B(A'/`!@``B[VP_O__BD<'B(1SP08``(GPB$<'
+M_H6O_O__BI6O_O__BTT,.%$J#X<G_O__Z<@```#&A:_^__\`BUT,@'LJ``^&
+MM````(J%K_[__XM5##B",`D```^'AP```#B",0D``')_.$(K='HX0BQT=0^V
+MP,'@!(V$!>C^__^Z`````(,X`'42@W@$`'4,@W@(`'4&@W@,`'0%N@$```"%
+MTG1##[:%K_[__\'@!(V<!>C^____=0SHI];__X/$!,9`!`&*C:_^__^(2`B(
+M2`F+$XE0#(M3!(E0$(M3"(E0%(M3#(E0&/Z%K_[__XJ=K_[__XMU##A>*@^'
+M3/___[@`````@[W(_O__`'4;@[W,_O__`'42@[W0_O__`'4)@[W4_O__`'0%
+MN`$```"%P`^$6QH``(M%#(I`+(B%D_[__XM5#(B"!`D``(N%R/[__XF"\`@`
+M`(N%S/[__XF"]`@``(N%T/[__XF"^`@``(N%U/[__XF"_`@``,:"!0D```#'
+M@@`)````````BEHK4NC0U?__@\0$QD`$`8A8"(J-D_[__XA("8N5R/[__XE0
+M#(N5S/[__XE0$(N5T/[__XE0%(N5U/[__XE0&(F%N/[__\>%M/[__P````#&
+MA:_^__\`BUT,@'LJ``^&H1D``(J%K_[__XM5##A"+`^$S`$``#A"*P^$PP$`
+M`(I:*U+H4=7__X/$!,9`!`&*C:_^__^(2`B(6`F+E<C^__^)4`R+E<S^__^)
+M4!"+E=#^__^)4!2+E=3^__^)4!B)A<#^__^+&/Z#P`8```^VL\`&``"+E;#^
+M__\IVH/J,,'Z`HT$THT$PHT$PHT$PHG!P>$/`<B-!,+WV(B$<\`&``"+O<#^
+M__^*1P>(A'/!!@``B?"(1P>#O;3^__\`=$N+'_Z#P`8```^VL\`&``"+E;3^
+M__\IVH/J,,'Z`HT$THT$PHT$PHT$PHG!P>$/`<B-!,+WV(B$<\`&``"*1P>(
+MA'/!!@``B?*(5P>+70R!P_`(``"+30R*22N(C9+^____=0SH4M3__X/$!,9`
+M!`.)6`B*G9+^__^(6`R)A;S^__^)A;3^__^+&/Z#P`8```^VL\`&``"+E<#^
+M__\IVH/J,,'Z`HT$THT$PHT$PHT$PHG!P>$/`<B-!,+WV(B$<\`&``"+O;S^
+M__^*1P>(A'/!!@``B?"(1P>+'_Z#P`8```^VL\`&``"+E;C^__\IVH/J,,'Z
+M`HT$THT$PHT$PHT$PHG!P>$/`<B-!,+WV(B$<\`&``"*1P>(A'/!!@``B?*(
+M5P?^A:_^__^*C:_^__^+70PX2RH/APK^___IIA<``(GVBT4,BH`P"0``B(6O
+M_O__BU4,.((Q"0``#X*&%P``B?:*C:_^__^+70PX2RMT=@^VP<'@!(V$!>C^
+M__^Z`````(,X`'43@W@$`'4-@W@(`'4'@W@,`'0&D+H!````A=)T0P^VA:_^
+M___!X`2-G`7H_O___W4,Z/[2__^#Q`3&0`0!BI6O_O__B%`(B%`)BQ.)4`R+
+M4P2)4!"+4PB)4!2+4PR)4!C^A:_^__^*C:_^__^+70PXBS$)```/@V'____I
+MX!8``(MU#(!^+`</AA\,``"#[`R-A=C^__]0Z+31___&A:[^__\`QH6O_O__
+M`(/$$(!^*@`/AKD```"0BH6O_O__BU4,.$(K#X2.````#[;`P>`$`="-D-@'
+M``"Y`````(.XV`<```!U$H-Z!`!U#(-Z"`!U!H-Z#`!T!;D!````A<ET5P^V
+ME:_^__^)T,'@!(M-#(V4"-@'``"+A=C^__\+`HF%V/[__XN%W/[__PM"!(F%
+MW/[__XN%X/[__PM""(F%X/[__XN%Y/[__PM"#(F%Y/[___Z%KO[___Z%K_[_
+M_XJ=K_[__XMU##A>*@^'2/___P^VE:[^__^+?0P/MD<J@^@#T?@YP@^.E@8`
+M`(I?*XB?[`@``(N%V/[__XF'V`@``(N%W/[__XF'W`@``(N%X/[__XF'X`@`
+M`(N%Y/[__XF'Y`@``,:'[0@```''A^@(````````5^A9T?__@\0$QD`$`HA8
+M"(A8"8N5V/[__XE0#(N5W/[__XE0$(N5X/[__XE0%(N5Y/[__XE0&(G'_W4,
+MZ![1__^#Q`3&0`0&B86P_O__BQ_^@\`&```/MK/`!@``B<(IVH/J,,'Z`HT$
+MTHT$PHT$PHT$PHG!P>$/`<B-!,+WV(B$<\`&``"*1P>(A'/!!@``B?"(1P>+
+M50R`>BH"#X?6`@``B=.!P]@(``!2Z*_0__^#Q`3&0`0%B5@(QD`4`(F%O/[_
+M_XN-L/[__XL9_H/`!@``#[:SP`8``(G"*=J#ZC#!^@*-!-*-!,*-!,*-!,*)
+MP<'A#P'(C03"]]B(A'/`!@``B[VP_O__BD<'B(1SP08``(GPB$<'QH6O_O__
+M`(M5#(!Z*@`/AF`4``"*C:_^__^+70PX2RL/A!X"``"+M;S^__^*1A0/MM"(
+M3!8,0(A&%`^VV<'C!(M]#(T$.XV(6`@``(N06`@``/?2B97(_O__BT$$]]")
+MA<S^__^+00CWT(F%T/[__XM!#/?0B874_O__C8P=Z/[__R.5V/[__XD1BX7<
+M_O__(X7,_O__B4$$BX7@_O__(X70_O__B4$(BX7D_O__(X74_O__B4$,N@``
+M``"#.0!U$H-Y!`!U#(-Y"`!U!H-Y#`!T!;H!````A=(/A)0````/MH6O_O__
+MP>`$C9P%Z/[___]U#.A/S___@\0$QD`$`8J5K_[__XA0"(A0"8L3B5`,BU,$
+MB5`0BU,(B5`4BU,,B5`8BXV\_O__BQG^@\`&```/MK/`!@``*=B#Z##!^`*-
+M%,"-%-"-%-"-%-")T<'A#P'*C130]]J(E'/`!@``B[V\_O__BD<'B(1SP08`
+M`(GPB$<'#[:%K_[__\'@!(M5#(V$$-@'``"Z`````(,X`'42@W@$`'4,@W@(
+M`'4&@W@,`'0%N@$```"%T@^$DP````^VG:_^__^)V,'@!(M-#(V<"-@'``!1
+MZ'O.__^#Q`3&0`0"BI6O_O__B%`(B%`)BQ.)4`R+4P2)4!"+4PB)4!2+4PR)
+M4!B)QXL8_H/`!@``#[:SP`8``(N5L/[__RG:@^HPP?H"C032C03"C03"C03"
+MB<'!X0\!R(T$PO?8B(1SP`8``(I'!XB$<\$&``")\8A/!_Z%K_[__XJ=K_[_
+M_XMU##A>*@^'N/W__^D3$@``QH6O_O__`(M]#(!_*@`/AO\1``"-=@"*A:_^
+M__^+50PX0BL/A(X"```/MOC!YP2-!!>-B%@(``"+D%@(``#WTHF5R/[__XM!
+M!/?0B87,_O__BT$(]]")A=#^__^+00SWT(F%U/[__XV,/>C^__\CE=C^__^)
+M$8N%W/[__R.%S/[__XE!!(N%X/[__R.%T/[__XE!"(N%Y/[__R.%U/[__XE!
+M#(M=#('#V`@``/]U#.@SS?__@\0$QD`$`XE8"(J-K_[__XA(#(F%O/[__XNU
+ML/[__XL>_H/`!@``#[:SP`8``(G"*=J#ZC#!^@*-!-*-!,*-!,*-!,*)P<'A
+M#P'(C03"]]B(A'/`!@``BY6P_O__BD('B(1SP08``(GQB$H'C80]Z/[__[H`
+M````@S@`=1*#>`0`=0R#>`@`=0:#>`P`=`6Z`0```(72#X24````#[:%K_[_
+M_\'@!(V<!>C^____=0SH?LS__X/$!,9`!`&*E:_^__^(4`B(4`F+$XE0#(M3
+M!(E0$(M3"(E0%(M3#(E0&(N-O/[__XL9_H/`!@``#[:SP`8``"G8@^@PP?@"
+MC13`C130C130C130B='!X0\!RHT4T/?:B)1SP`8``(N]O/[__XI'!XB$<\$&
+M``")\(A'!P^VA:_^___!X`2+50R-A!#8!P``N@````"#.`!U$H-X!`!U#(-X
+M"`!U!H-X#`!T!;H!````A=(/A)8````/MIVO_O__B=C!X`2+30R-G`C8!P``
+M4>BJR___@\0$QD`$`HJ5K_[__XA0"(A0"8L3B5`,BU,$B5`0BU,(B5`4BU,,
+MB5`8B<>+&/Z#P`8```^VL\`&``"+E;#^__\IVH/J,,'Z`HT$THT$PHT$PHT$
+MPHG!P>$/`<B-!,+WV(B$<\`&``"*1P>(A'/!!@``B?&(3P>-=@#^A:_^__^*
+MG:_^__^+=0PX7BH/ATC]___I/P\``(M]#(I?*U?H!\O__\9`!`&(6`B(6`F+
+ME=C^__^)4`R+E=S^__^)4!"+E>#^__^)4!2+E>3^__^)4!B)A;C^__^*7RN)
+M/"3HR,K__X/$!,9`!`*(6`B(6`F+E=C^__^)4`R+E=S^__^)4!"+E>#^__^)
+M4!2+E>3^__^)4!B)Q_]U#.B-RO__@\0$QD`$!HF%L/[__XL?_H/`!@``#[:S
+MP`8``(G"*=J#ZC#!^@*-!-*-!,*-!,*-!,*)P<'A#P'(C03"]]B(A'/`!@``
+MBD<'B(1SP08``(GPB$<'QH6O_O__`(M5#(!Z*@`/AE$.``"0BHVO_O__BUT,
+M.$LK#X1J`P``#[;!P>`$`=B-D-@'``"Y`````(.XV`<```!U$H-Z!`!U#(-Z
+M"`!U!H-Z#`!T!;D!````A<D/A"\#``#^C:[^__^*A:[^__^+50R*4BJ--!`/
+MMIVO_O__B=^)V,'@!(M-#(V<"-@'``!1Z*3)___&0`0!BI6O_O__B%`(B?&(
+M2`F+$XE0#(M3!(E0$(M3"(E0%(M3#(E0&(F%P/[__XM=#(I;*XB=D?[__\'G
+M!(M%#(V<!]@'``")!"3H5,G__\9`!`2*E9'^__^(4!B)\8A(&8L3B5`(BU,$
+MB5`,BU,(B5`0BU,,B5`4B86\_O__BQC^@\`&```/MK/`!@``BY6X_O__*=J#
+MZC#!^@*-!-*-!,*-!,*-!,*)P<'A#P'(C03"]]B(A'/`!@``BY6\_O__BD('
+MB(1SP08``(GQB$H'BQK^@\`&```/MK/`!@``BY7`_O__*=J#ZC#!^@*-!-*-
+M!,*-!,*-!,*)P<'A#P'(C03"]]B(A'/`!@``BY6\_O__BD('B(1SP08``(GQ
+MB$H'B[6P_O__BQ[^@\`&```/MK/`!@``BY6\_O__*=J#ZC#!^@*-!-*-!,*-
+M!,*-!,*)P<'A#P'(C03"]]B(A'/`!@``BY6P_O__BD('B(1SP08``(GQB$H'
+MBUT,BELKB)V0_O__BW4,C9PWV`<``(DT).@.R/__QD`$!(J5D/[__XA0&(J-
+MK_[__XA(&8L3B5`(BU,$B5`,BU,(B5`0BU,,B5`4B86\_O__BQC^@\`&```/
+MMK/`!@``BY6X_O__*=J#ZC#!^@*-!-*-!,*-!,*-!,*)P<'A#P'(C03"]]B(
+MA'/`!@``BY6\_O__BD('B(1SP08``(GQB$H'B[6P_O__BQ[^@\`&```/MK/`
+M!@``BY6\_O__*=J#ZC#!^@*-!-*-!,*-!,*-!,*)P<'A#P'(C03"]]B(A'/`
+M!@``BY6P_O__BD('B(1SP08``(GQB$H'BUT,C;P?V`<``(D<).@AQ___@\0$
+MQD`$`HJ5K_[__XA0"(A0"8L7B5`,BU<$B5`0BU<(B5`4BU<,B5`8B<>+&/Z#
+MP`8```^VL\`&``"+E;#^__\IVH/J,,'Z`HT$THT$PHT$PHT$PHG!P>$/`<B-
+M!,+WV(B$<\`&``"*1P>(A'/!!@``B?&(3P>)]OZ%K_[__XJ=K_[__XMU##A>
+M*@^';/S__^FW"@``C78`BWT,BD<L.D<K#X6Z````QH6O_O__`(!_*@`/AI0*
+M``"*A:_^__^+50PX0BMT?@^VP,'@!`'0C9#8!P``N0````"#N-@'````=1*#
+M>@0`=0R#>@@`=0:#>@P`=`6Y`0```(7)=$</MIVO_O__B=C!X`2+30R-G`C8
+M!P``4>@'QO__@\0$QD`$`HJ5K_[__XA0"(A0"8L3B5`,BU,$B5`0BU,(B5`4
+MBU,,B5`8D/Z%K_[__XJ-K_[__XM=##A+*@^'7/___^GK"0``@^P,C878_O__
+M4.C,Q/__@\00_W4,Z*7%__^#Q`3&0`0&B86P_O__QH6O_O__`(MU#(!^*@`/
+MAE0!``"*A:_^__^+50PX0BL/A"H!```/ML#!X`0!T(V0V`<``+D`````@[C8
+M!P```'42@WH$`'4,@WH(`'4&@WH,`'0%N0$```"%R0^$[P````^VE:_^__^)
+MT8G0P>`$BUT,C908V`<``(N%V/[__PL"B878_O__BX7<_O__"T($B87<_O__
+MBX7@_O__"T((B87@_O__BX7D_O__"T(,B87D_O__BH6O_O__.$,L#X2-````
+MB<O!XP2+50R-G!/8!P``4NC`Q/__@\0$QD`$`HJ-K_[__XA("(A("8L3B5`,
+MBU,$B5`0BU,(B5`4BU,,B5`8B<>+&/Z#P`8```^VL\`&``"+E;#^__\IVH/J
+M,,'Z`HT$THT$PHT$PHT$PHG!P>$/`<B-!,+WV(B$<\`&``"*1P>(A'/!!@``
+MB?.(7P>0_H6O_O__BH6O_O__BU4,.$(J#X>L_O__BTT,BEDK4>@DQ/__@\0$
+MQD`$`HA8"(A8"8N5V/[__XE0#(N5W/[__XE0$(N5X/[__XE0%(N5Y/[__XE0
+M&(G'BQC^@\`&```/MK/`!@``BY6P_O__*=J#ZC#!^@*-!-*-!,*-!,*-!,*)
+MP<'A#P'(C03"]]B(A'/`!@``BD<'B(1SP08``(GSB%\'BW4,#[9>+(G>B=C!
+MX`2+?0R-G#A8"```BP/WT(F%R/[__XM3!/?2B97,_O__BTL(]]&)C=#^__^+
+M6PSWTR.%V/[__XF%R/[__XN]W/[__R'ZB97,_O__BY7@_O__(=&)C=#^__^+
+MA>3^__\APXF=U/[__\'F!(M-#(VT#E@(``"+C=C^__\C#HF-V/[__R-^!(F]
+MW/[__R-6"(F5X/[__R-&#(F%Y/[__[@`````A<EU$87_=0V%TG4)@[WD_O__
+M`'0%N`$```"%P`^$N@(``(M=#(I#*XB#[`@``(N%V/[__XF#V`@``(N%W/[_
+M_XF#W`@``(N%X/[__XF#X`@``(N%Y/[__XF#Y`@``,:#[0@```''@^@(````
+M````QH6O_O__`(![*@`/AEP"``"!P]@(``")G8S^__^*A:_^__^+50PX0BQU
+M=E+H6,+__X/$!,9`!`.+C8S^__^)2`B*G:_^__^(6`R+M;#^__^+'OZ#P`8`
+M``^VL\`&```IV(/H,,'X`HT4P(T4T(T4T(T4T(G1P>$/`<J-%-#WVHB4<\`&
+M``"+O;#^__^*1P>(A'/!!@``B?"(1P?IM0$``)"*E:_^__^+30PX42L/A*(!
+M```/MOK!YP2-C#WH_O__BUT,C90?6`@``(L"]]")`8M"!/?0B4$$BT((]]")
+M00B+0@SWT(E!#(L!(X78_O__B0&+000CA=S^__^)002+00@CA>#^__^)00B+
+M00PCA>3^__^)00Q3Z&W!__^#Q`3&0`0#B[6,_O__B7`(BI6O_O__B%`,B86\
+M_O__BXVP_O__BQG^@\`&```/MK/`!@``B<(IVH/J,,'Z`HT$THT$PHT$PHT$
+MPHG!P>$/`<B-!,+WV(B$<\`&``"+E;#^__^*0@>(A'/!!@``B?&(2@>-A#WH
+M_O__N@````"#.`!U$H-X!`!U#(-X"`!U!H-X#`!T!;H!````A=(/A)<````/
+MMH6O_O__P>`$C9P%Z/[___]U#.BRP/__@\0$QD`$`8J5K_[__XA0"(A0"8L3
+MB5`,BU,$B5`0BU,(B5`4BU,,B5`8BXV\_O__BQG^@\`&```/MK/`!@``*=B#
+MZ##!^`*-%,"-%-"-%-"-%-")T<'A#P'*C130]]J(E'/`!@``B[V\_O__BD<'
+MB(1SP08``(GPB$<'C78`_H6O_O__BI6O_O__BTT,.%$J#X>P_?__N`````"#
+MO<C^__\`=1N#O<S^__\`=1*#O=#^__\`=0F#O=3^__\`=`6X`0```(7`#X02
+M!```BW4,BEXK5NC:O___@\0$QD`$`8A8"(A8"8N5R/[__XE0#(N5S/[__XE0
+M$(N5T/[__XE0%(N5U/[__XE0&(F%N/[__\>%M/[__P````#&A:_^__\`@'XJ
+M``^&M`,``(J%K_[__XM5##A"+`^$B@,``#A"*P^$@0,```^VP,'@!(V<!>C^
+M__\!T(V(V`<``(N5R/[__R.0V`<``(D3BX7,_O__(T$$B4,$BX70_O__(T$(
+MB4,(BX74_O__(T$,B4,,N`````"#.P!U$H-[!`!U#(-["`!U!H-[#`!T!;@!
+M````A<`/A!$#``"+30R*22R(C8O^__\/MH6O_O__P>`$C9P%Z/[___]U#.C8
+MOO__@\0$QD`$`8J5K_[__XA0"(J-B_[__XA("8L3B5`,BU,$B5`0BU,(B5`4
+MBU,,B5`8B87`_O__@[VT_O__`'11BQC^@\`&```/MK/`!@``BY6T_O__*=J#
+MZC#!^@*-!-*-!,*-!,*-!,*)P<'A#P'(C03"]]B(A'/`!@``B[W`_O__BD<'
+MB(1SP08``(GPB$<'BU4,BE(KB)6*_O__BTT,BDDLB(V)_O__#[:]K_[__\'G
+M!(V</>C^____=0SH%[[__X/$!,9`!`2*E8K^__^(4!B*C8G^__^(2!F+$XE0
+M"(M3!(E0#(M3"(E0$(M3#(E0%(F%O/[__XF%M/[__XL8_H/`!@``#[:SP`8`
+M`(N5N/[__RG:@^HPP?H"C032C03"C03"C03"B<'!X0\!R(T$PO?8B(1SP`8`
+M`(N5O/[__XI"!XB$<\$&``")\8A*!XL:_H/`!@``#[:SP`8``(N5P/[__RG:
+M@^HPP?H"C032C03"C03"C03"B<'!X0\!R(T$PO?8B(1SP`8``(N5O/[__XI"
+M!XB$<\$&``")\8A*!XNUL/[__XL>_H/`!@``#[:SP`8``(N5O/[__RG:@^HP
+MP?H"C032C03"C03"C03"B<'!X0\!R(T$PO?8B(1SP`8``(N5L/[__XI"!XB$
+M<\$&``")\8A*!XM=#(I;*XB=B/[__XV</>C^____=0SHQ[S__X/$!,9`!`2*
+ME8C^__^(4!B*C:_^__^(2!F+$XE0"(M3!(E0#(M3"(E0$(M3#(E0%(F%O/[_
+M_XL8_H/`!@``#[:SP`8``(N5N/[__RG:@^HPP?H"C032C03"C03"C03"B<'!
+MX0\!R(T$PO?8B(1SP`8``(N]O/[__XI'!XB$<\$&``")\(A'!XN5L/[__XL:
+M_H/`!@``#[:SP`8``(GZ*=J#ZC#!^@*-!-*-!,*-!,*-!,*)P<'A#P'(C03"
+M]]B(A'/`!@``BXVP_O__BD$'B(1SP08``(GSB%D'D/Z%K_[__XJ%K_[__XM5
+M##A"*@^'3/S__X/L"/]U#/]U".CNT?__N`````"-9?1;7E_)PY!5B>575E.#
+M[`R+10B#N+PQ```##X</`0``!7PQ``"+50@Y@GPQ```/A/L```")1>R+BGPQ
+M``")3?"-=@"+7?"+`XE%\(M3!(L#B5`$B0*)&XE;!(M#'/9``00/A8@```")
+MW[X`````@'LJ`'8FC78`@^P,B?(/ML+!X`2-A#C8!P``4.A,NO__@\001HGQ
+M.$\J=]V+50B!PG0Q``"+30B+@70Q``")6`2)`XE3!(F9=#$``(M#'(-X+`!T
+M8(/L!/]P,(M#'/]P+%'H_/___XM#',=`+`````"+10C_B,`Q``"#Q!#K-HGV
+M@^P(4_]U".A0T___@\00A<!T+8M5"('"?#$``(M-"(N!?#$``(E8!(D#B5,$
+MB9E\,0``D(M%[#E%\`^%%/___XUE]%M>7\G#58GE4X/L!(M%"(M8"(.[O#$`
+M``!U#HV#?#$``#F#?#$``'0S@^P,4^C\____B1PDZ/S___^)'"3H_/___X/$
+M$(.[O#$```!UVXV#?#$``#F#?#$``'7-N`````"+7?S)PXUV`%6)Y5=64X/L
+M!(M%"(M`"(E%\(N(?#$```5\,0``.<@/A($```"-=@"+50@Y41QU:(G/LP"`
+M>2H`=D&)]@^VP\'@!`'XC9#8!P``O@````"#N-@'````=12#>@0`=0Z#>@@`
+M=0B#>@P`=`>)]KX!````A?9U&4,X7RIWP;@`````A<!T%+@!````Z:T```"X
+M`0```.OKC78`BPF+1?`%?#$``#G(=8*+1?"+B(0Q```%A#$``#G(='Z-0?B+
+M50@Y4!QU98G#O@````"`>"H`=D&)\@^VPL'@!`'8C9#8!P``OP````"#N-@'
+M````=1*#>@0`=0R#>@@`=0:#>@P`=`6_`0```(7_=1A&B?`X0RIWO[@`````
+MA<!T#K@!````ZQJX`0```.ONBPF+1?`%A#$``#G(=8*X`````(/$!%M>7\G#
+MD%6)Y5=64X/L#(M="(M]$(MS"(-[+`!U#5/HGO[__X/$!(7`=0N#[`A75O]5
+M#.L9D(M%#(E#+(E[,/^&P#$``(/L#%;H_/___XUE]%M>7\G#58GE5E.+=0B+
+M70R`>Q4`=03&0Q4!@'L5`709@'L5!G03@'L5`W0-]D,4('4'QD,5"XUV`/]S
+M,%;H@[?__XD<)/]S(%;H_/___XL#@\00@W@@`'08@^P$@\`@4&@`````5NC\
+M____@\00ZUV0@[[$,0```'0KC89T,0``.89T,0``=!V#[`2-AL0Q``!0:```
+M``!6Z/S___^#Q!#K*XUV`(&^N#$``/\```!W'(V&=#$``#F&=#$``'4.@^P,
+M5NC\____@\00B?:-9?A;7LG#D%6)Y5=64X'LW````(M]$(M%#(M0)(L*B8T@
+M____#[991$N)G33___]FBT(,9HF%5O___XM*4(M:5(F-2/___XF=3/___P^V
+M6DZ)G3#___\/MD)/B84L____9HM*6F:)C53___]FBUI8@WH8`'0-9@^V0A2#
+MX`$[111U.+D`````@WH<``^$[@(``(V%6/___XF%)/____]U%%!2_W4(_U(<
+M@\00N0````"%P`^$QP(``.L)BU(8B94D____O@````"+50R+`@^V0`.)A2C_
+M___'A3C___\`````QX4\____``````^WP[H``````X5(____$Y5,____B85`
+M____B95$____BXTH____.8TP____#X5:`0``BYT\____"YTX____=%:+A4#_
+M__\KA3C___^)P\'C"71#@WT4`'0'H0````#K$8/L#/\U`````.C\____@\00
+MB0=FQT<&``"!^P`0```/AO0```!FQT<$`!"#QPB!ZP`0``!UO0^WG53____!
+MXPF+A23___\/MU`$A=)U!;H```$`B=`I\#G8<C&)\(N-)/___P,!B0=FB5\$
+M9L='!@``@\<(`=XYUG5&O@````"#P0B)C23____K-HGVB?"+C23___\#`8D'
+MB=!F*?!FB4<$9L='!@``@\<(B=`I\"G#@\$(B8TD____O@````#K@`^WG53_
+M__^)G3C____'A3S___\`````BY5`____BXU$____`94X____$8T\____BTT,
+M#[=!#+H``````T$$$U$(BXT\____,=$SA3C___\)P0^$$@$``.M/9HE?!(/'
+M".D1____C78``=[K/`^WG53____!XPF)]HN%)/___P^W4`2%TG4%N@```0")
+MT"GP.=AWU8G0*?`IPX.%)/___PB^`````.O1D(N55/___V8IE5;___\/A*X`
+M``"[`````/^%,/___XN-+/___SF-,/___W4(_X4P____B?:+E2#___\/MD)$
+M.X4P____=4Z#O2S___\`=`C_C2S____K#(N--/___XF-+/___X.]+/___P`/
+ME,`/ML")A3#___^+C2#___\/MT%<N@`````!A4C___\1E4S___^-=@"+A2#_
+M__]FBT!<9HF%5/___V:+E5;___]F.=`/AH/]__]FB954____Z7?]__]FQT?^
+M`("Y`0```(G(C67T6UY?R<.)]E6)Y593BW4(BU4,BUHD@'H5`70&BD(5B$,5
+M@^P(4E;H_/___XI#34"(0TV#Q!`Z0TQU+8![%0!U!,9#%0&`>Q4!=!"`>Q4&
+M=`J`>Q4#=`3&0Q4+@^P(4U;_4R"#Q!")]HUE^%M>R<.058GE5U93@^P\BT4,
+MBP")1>B+50SV0A0$#X3]`0``QT7L`````(!Z2``/A'L!``"+3>R+70R+3(LT
+MB4WDBIDP"0``B%WC.)DQ"0``#X)&`0``C78`BD7CBU7D.$(K#X0?`0``#[;`
+MB<+!X@2+3>2-E`K8!P``B57<#[>4@1`)```/M[2!$@D``+\`````C78`@_H?
+M=U.)^P^VPXM-W(T<@8E5V+D@````*=$Y\78"B?&#^2!U"\<#_____^L2C78`
+MN`$```#3X$B*3=C3X`D#N"`````IT#GP<QF-=!;@N@````#K!HUV`(/J($>)
+M^X#[`W:=#[9UXXGPP>`$BWWDC80X6`@``(E%U`^WG+<0"0``#[>TMQ()``"_
+M`````(GV@_L?=U.)^@^VPHM-U(T4@8E=T+D@````*=DY\78"B?&#^2!U"\<"
+M_____^L2C78`N`$```#3X$B*3=#3X`D"N"`````IV#GP<QB-=![@NP````#K
+M!HUV`(/K($>)^#P#=I[^1>.*5>.+3>0XD3$)```/@[W^____1>R+70P/MD-(
+M.T7L#X^%_O__BW4(@[YP,0````^$H@,``,=%[`````"+?0R`?T@`=#Z)\H'"
+M?#$``(M%[(M-#(M$@32)1>3'@`@)````````BT($BTWDB4H$B1&)002)"/]%
+M[(M=#`^V0T@[1>Q_RH/L"/]U#/]U".@N^?__Z7@#``"+=0SV1A0"#X0T`P``
+MBWT(@<?(,0``B7W,BT4(!0@R``")1<B+5>CV0EX##X40`P``@^P$:A!J`%#H
+M_/___\=%[`````"#Q!"`?D@`#X03`0``BTWLBUT,BTR+-(E-Y(G.L0"`?BH`
+M=D8/ML'!X`0!\(V0V`<``+L`````@[C8!P```'42@WH$`'4,@WH(`'4&@WH,
+M`'0%NP$```"%VW0)N`$```#K#8GV03A.*G>ZN`````"%P`^%A0(``(M%Y(J`
+M,`D``(A%XXM5Y#B",0D``')_BDWCBUWD.$LK=&,/MLF+=<AF@SQ.`'4M#[>$
+MBQ`)``"Z`````(G?`T,@$U,DBUW,B03+B53+!&:+A(\2"0``9HD$3NLI#[95
+MXXMUY&:+1B!F`X26$`D``&8#A)82"0``BWW,9BL$UXM-R&:)!%'^1>.*7>.+
+M=>0XGC$)``!S@?]%[(M]#`^V1T@[1>P/C^W^__^+10S_<##_=0CH=J___\=%
+M[`````"#Q`B+50R`>D@`#X0$`0``BTT(@<%T,0``B4W$BUWLBW4,BUR>-(E=
+MY(-][`!U3HM#((M3)(M]#(E'4(E75(J#,`D``(A'3HI#*XA'3P^V@S`)``!F
+MBX2#$`D``&:)1U@/MH,P"0``9HN$@Q()``!FB4=:QD=,`,9'30#K<8M-Y(M!
+M((M1)(M=##M35')@=P4[0U!V68MUZ`^W1ESWV(E%N(G'P?\?B7V\BTVXBT7D
+M(T@@BUV\(U@DBT6XBW4,(T90BU6\(U94B=XQUC'("<9U'8M]Y`^VES`)``"+
+M30QFBT%:9@.$EQ()``!FB4%:BUW$BT,$BW7DB7,$B1Z)1@2),/]%[(M]#`^V
+M1T@[1>P/CPC___^+10B#N,0Q````=!F#[`0%Q#$``%!H`````/]U".C\____
+M@\00QD7C`(M5Z(!Z1``/AK0````/MG7CBTW(9H,\<0!T7HM]Z(M<MVR#[`S_
+M=0CH_/___XM5#/Y"3,9`#B"+3<AFBQ1Q9HE0#(M]S(L4]XM,]P2)4`2)2`B`
+M2!0"QT`<H&L``,=`("1O``")&(M5#(E0)(/$"%#_=0C_4SR#Q!#^1>.*3>.+
+M7>@X2T1WANLXD,=%[`````"+=0R`?D@`=">#[`B+?>R+10S_=+@T_W4(Z-_&
+M__^#Q!!'B7WLBU4,#[9"2#GX?]F-9?1;7E_)PY!5B>575E.#[%R+10R+`(E%
+M[,=%V`````!FBU!<9HE5WF:!^H``=@9FQT7>@`"+30SV010@=$Z#[`P/MT7>
+M4(M=$/]S%/]S$/]U[/]U".B2L/__B478@\0@N/____^#?=@`#X2.`P``BT4,
+MQD!(`8M5V(E0-(F""`D``+@`````Z7$#``"+31`/MD$BN@````!24(G+_W$$
+M_S'H_/___X/$$(A%UXM-[`^V042Z`````%)0_W,,_W,(Z/S___^#Q!"*2R(H
+MP8A-UCA-UX!=U_^+1>QFBU!<C4+_9HM+&"'!9BG*9HE5W&8Y4R!S"&:+4R!F
+MB57<BUT,QD-(`,9#20#'1=`!````#[=%WHE%S/?8B47`F8E5Q(M="('#=#$`
+M`(E=L`^WP;H`````BTT0`T$0$U$4B47@B57DBUT(@<-\,0``B5VTC78`9HM%
+MWD@C1>!FBWW>9BG'9CM]W'8#BWW<BUW`(UW@BW7$(W7D@WW0``^$<@$``,=%
+MT`````!FBU7>BT7L9CE07'8^N0````"+50P/MD)(B46\@_@`?BJ)]HM%#(M$
+MB#2)1=B+4"0Q\HE5J(M0(#':BT6H"=`/A"8!``!!.4V\?]B#[`S_=<Q64_]U
+M[/]U".@)K___B478@\0@A<`/A<L```"+50S^2DB`>DC_#X2O````BTVTB4VX
+MBWVPBUT,#[9#2(M$@S2)1=C'@`@)````````B<:Q`(!X*@!V/P^VP<'@!`'P
+MC9#8!P``NP````"#N-@'````=12#>@0`=0Z#>@@`=0B#>@P`=`>)]KL!````
+MA=MU)$$X3BIWP;@`````A<!T'(M5N(M"!(M-V(E*!(D1B4$$B0CK%[@!````
+MZ^"+1P2+7=B)7P2).XE#!(D8BT4,_DA(@'A(_P^%6O___[C_____Z4,!``")
+M]HM5#(I22(A5IP^VRHM5V(M%#(E4B#2*1:=`BTT,B$%(B8H("0``QH(P"0``
+M",:",0D````/ME77BT7@9BG8BUW89HF$DQ`)``!FB;R3$@D``(I%USB#,`D`
+M`'8&B(,P"0``BE77BTW8.)$Q"0``<P:(D3$)``"+71!F*7L@9BE]W'0:#[?'
+MN@`````!1>`15>3'1=`!````Z??]__^+11!F@W@@``^$D0```+D`````_D77
+MBE76.%77=0/^1=>*7=>+1>PX6$1U.H!]U@!T!?Y-UNL)BU40BE(BB%76@'W6
+M``^41=>+7>P/MT-<N@````"+71`!0Q`14Q3'1=`!````ZQ1FBU7>BT7L9CE0
+M7'8'QT70`0```(M=$&:+6R!FB5W<BT7L9CM87`^&/_W__V:+4%QFB57<Z3+]
+M__^X`````(UE]%M>7\G#C78`58GE@^P(BU4(BT4,@WT0`'0:QD`5"X/L!%!H
+MT&H``%+H_/___X/$$.L/B?:#[`A04N@N]O__@\00R<.058GE5U93@>S,````
+MBT4,BP")1;3V0`$$=1Z+50S&0A4"@^P$4O]R(/]U".C\____Z0\$``"-=@"+
+M3;2#>30`=2=J!_]U".CEI?__BUVTB4,TQP0D`!```&H`_W,TZ/S___^#Q!"-
+M=@"+50R*0A2H(`^%$@(``*@&=1K&0A4!@^P$4O]R(/]U".C\____Z;(#``")
+M]HM-#/9!%`1T+XM=M(-[+`!T)H/L!/]S,/]S+/]U".C\____QT,L`````(M%
+M"/^(P#$``(/$$(GVBTT,BU$$BTD(B95`____B8U$____B570B4W4BUT,9HM#
+M#&:)1=B+5;2*0D1(B$7:#[9*1HN%0/___XN51/___P^MT-/J]L$@=`>)T+H`
+M````B46@B56DB46XB56\#[9%VKH`````4E#_=:3_=:#H_/___X/$$(E%P(E5
+MQ(M=M`^V2T8/I<+3X/;!('0'B<*X`````(E%R(E5S`^W0UP/ME7:B56<#Z_"
+MB46P#[=-V(E-D,=%E`````"+19"+590#A4#___\3E43___^)18B)58R+59R)
+ME3#____'A33___\`````BT7(]Z4P____B46`B56$BY4P____#Z]5S(M]A`'7
+MBT7(#Z^%-/___XT$!XE%A(MUB(M]C"MU@!M]A(M%L+H``````<81UXFU</__
+M_XF]=/___X/&_X/7_U)05U;H_/___X/$$(F%,/___XF5-/___XM-M`^W25R)
+MC6C____'A6S___\`````BX5H____]Z4P____B85@____B95D____BY5L____
+M#Z^5,/___XNU9/___P'6BX5H____#Z^%-/___XT$!HF%9/___XN-8/___XE-
+MK.F;````BUVT]D->`70;BT4,QD`5!H/L!%#_<"#_=0CH_/___^F8`0``BU6T
+M#[=27(E5K(F54/___\>%5/___P````"+30R+202)C3S___^+A5#____WX8F%
+M,/___XF5-/___XN-4/___XM5#`^O2@B+G33___\!RXN-//___P^OC53___^-
+M#`N)C33___^+A3#___^+E33___^)1<B)5<R+1:RZ`````(MUR(M]S`'P$?I2
+M4%=6_W6T_W4(Z$2F__^+50R)0C"#Q!B%P'4?QT(L`````(/L"%*+1;2#P"!0
+MZ/S____IUP```(UV`(/L!(U%N%#_=0S_=0CH0OC__X/$$(7`='.+30S_<3#_
+M=0CHE:7__XM="(N39"(``(/$"#F3:"(``'0FN2("``"+10B!O-!4$0``````
+M`'1;0HG0F??YBUT(.9-H(@``==^+10S'0"P`````@^P(4(M%"`7$,0``4.C\
+M____@\0$_W4(Z/S____K2HGVBU4,]D(4!'0N@^P$:"AY``!2_W4(Z(:I__^#
+MQ!#K*8M-#,9!%06#[`11_W$@_W4(Z/S____K$8/L"/]U#/]U".CG\?__@\00
+MC67T6UY?R<-5B>53@^P$BUT,@WLT`'1.@^P,4^C\____N@````"#Q!"-=@"+
+M0S2+!)"%P'00B?;'0!``````BT`4A<!U\D*!^O\#``!VW6H'_W,T_W4(Z+2A
+M___'0S0`````@\0,BUW\R<.058GE5U93@^P0BWT(QX=P,0```````(V'=#$`
+M`(F'=#$``(F'>#$``(V'?#$``(F'?#$``(F'@#$``(V'A#$``(F'A#$``(F'
+MB#$``*$`````P>`,NJNJJJKWXHG3P>L/4V@T"0``5^A8H?__B<:#Q`R#PR!3
+M:AA7Z$>A__\!QH/$#*$`````P>`,P>@.4&I$5^@NH?__`<:#Q`QJ"&@`$```
+M5^@<H?__`<:)/"3H_/___Z,`````@\001CLU`````',QD(/L#%?H_/___XG"
+M@\00A<!T'HN'M#$``(D"B9>T,0``_X>X,0``1CLU`````'+1D(UE]%M>7\G#
+M58GE5U93@^P,BWT(BW4,BUX$#[9.`[@!````B<+3XH!F`?O'0V0`````QT-H
+M`````/9#7@)T"6:#>V``=`_K!V:#>V``="UFA5-@=2=F"5-@@&,!^X-[0`!T
+M#8/L"%-7_U-`@\00B?:`8UX?Z;L```"-=@"*0UYFB5-@_DM%QT-D`````,=#
+M:`````"#R`6(0UZ$P'D.@^!_B$->Z8L```"-=@"`8UZ?@^P(4U?H_/___XG"
+M@\00A<!T3,8``XI&`XA"`XE:!(!*`0BA#````(E"/*$@````B4)`QT8$````
+M``^V0@.)5(-LBD->@^#^@\@*B$->_D-%@^P(4FH'Z/S___^#Q!"#>P0`=`R-
+M=@"+6P2#>P0`=??H_/___XE#2(/L#%/H_/___XGVC67T6UY?R<-5B>53BTT(
+MNP````"+50Q*@_K_=`^-=@`/M@$!PT%*@_K_=?0/ML-;R<.)]E6)Y593BW4(
+M@#X#=D:[`````(!^1`!T-8-\GFP`="6+3)YLBE8!@^(!BD$!@^#^"="(00&#
+M[`S_=)YLZ/S___^#Q!"00P^V1D0YV'_+@&9>^^L)@^P,5N@(````C67X6U[)
+MPY!5B>575E.![!P"``#'A>3]__\`````C;7H_?__BT4(@'A$``^$-@(``(MX
+M!(7_=`^#?P0`=`F)O>3]__^+?P2#[`1H``(``&H`5NC\____QP;S%GA:@\00
+MN`````"%_W0#BT=(B48$BU4(@'I,`'0*@$X)`HI"3XA&"HM-"(I12(C3@^,!
+MBH:3````@^#\B-&#X0()V`G(B-.#XP2#X/.(T8/A"`G8"<B(TX/C$(/@SXC1
+M@^$@"=@)R(C1@^%`@^`_@^*`"<@)T(B&DP```(M%"/9``0%T!(!."0&P`(7_
+M=`R#O>3]__\!&<"#P`*(1@N%_P^$N````(M'#(E&#(M'$(F&E````(H'B$80
+MBD=$B$81@[WD_?__`'0+BY7D_?__BD(#ZP:+30B*00.(1A**1T:(1A/V1UX"
+M=`2`3A0!9HM'8&:)1A:+1V2)1AB+1VB)AI@```"#[`1J$(V'C````%"-1BQ0
+MZ/S___^#Q`QJ!(V'G````%"-1CQ0Z/S___^#Q`QJ$(V'X````%"-AH````!0
+MZ/S___^#Q`QJ0(V'H````%"-1D!0Z/S___^#Q!"#O>3]__\`=%R+E>3]__^+
+M0@R)1AR+C>3]__^+01")AIP```"*`8A&((I!1(A&(8M5"(I"`XA&(HI!1HA&
+M(_9!7@)T!(!.)`&+C>3]__]FBT%@9HE&)HM!9(E&*(M!:(F&H````(7_=0N+
+M50B*`HA&$(UV`&B0````5NC\____]]B(1@AH``(``%;H_/____?8B(:0````
+M5FHPBTT(BT%4@^@*4(G(@\!$4.C\____C67T6UY?R<.058GEBTT(B<B`.0-V
+M+[@`````@'E$`'00#[911(-\@6P`=0Q`.<)_]+@`````ZPV#[`S_=(%LZ/S_
+M__^0R<.)]E6)Y593BW4(@#X#=B^[`````(!^1`!T*(-\GFP`=!&#[`S_=)YL
+MZ/S___^#Q!")]D,/MD9$.=A_W^L%D(!F`?Z-9?A;7LG#D%6)Y8M-"(M1"+@`
+M````B?8Y#()U"\<$@@````#K"(GV0(/X!W;JR<-5B>6+10R`8`'[4.C\____
+MR<.)]E6)Y5=64X'L'`0``(M]"(VUZ/W__XG[BD<!J`0/A!D!``"`/P(/AA`!
+M``"#X/V(1P&#[`1H``(``&H`5NC\____@\00@#\#='B-A>C[__^0BUML@#\(
+M=62%VW0'9H-_8`%U68G#QX7D^___`0````^V1T2#^`%^6E-J(&H`BY7D^___
+MBT27;(/`1%#H_/___[H`````@\00BP26,P23B0260H/Z?W[Q_X7D^___#[9'
+M1#N%Y/O__W^_ZQ>0@#L#=8]6:B!J`(U#1%#H_/___X/$$&:!OOX!``!5JG5>
+M@[[*`0```'0;@7X&3$E,3W0,@;YV`0``1U)50G4&@$\!`NLZQX7D^___````
+M`(N%Y/O__\'@!`'P@+B^`0``@'4-@[C*`0```'0$@$\!`O^%Y/O__X.]Y/O_
+M_P-^T(UE]%M>7\G#D%6)Y593BTT(BYE0$0``C4,!OB("``"9]_Z)UHM%#(E$
+MV3R+D5`1``"+11")1-%`B;%0$0``6U[)PXUV`%6)Y5=64X/L#(MU"(N&3!$`
+M`#N&4!$``'0VOR("``"0BX9,$0``BTS&/(M<QD!`F??_B99,$0``@^P(4U;_
+MT8/$$(N&3!$``#N&4!$``'70C67T6UY?R<-5B>564XM-"(N9:"(``(U#`;XB
+M`@``F??^B=:+10R)A-E4$0``BY%H(@``BT40B8316!$``(FQ:"(``%M>R<.0
+M58GE5U93@^P,BUT(BX-D(@``.X-H(@``=%:)]HN#9"(``(NTPU01``"+O,-8
+M$0``0+HB`@``B=&9]_F)DV0B``"#N[PQ````=`Z#[`Q3Z/S___^#Q!")]H/L
+M"%=3_]:#Q!"+@V0B```[@V@B``!UK(UE]%M>7\G#58GE4X/L!(M5"+@`````
+M@WHX`'07BUHXBP.)0CB#[`1J7&H`4^C\____B=B+7?S)PXUV`%6)Y8M-"(M5
+M#(M!.(D"B5$XR<.058GEBTT(BU4,@SD`=!*+`8M`*(E"*(L!B5`HZP:-=@")
+M4BB)$<G#D%6)Y5=64X/L#(MU#(,^`'0UBSZ0BP:+6"@YV'4+QP8`````ZPN-
+M=@"+%HM#*(E"*(/L"%/_=0C_4RR#Q!`Y^W0%@SX`=<Z-9?1;7E_)PXGV58GE
+M@^P(BT4(@#@(=0Z#[`Q0Z/S___^#Q!")]LG#B?95B>53@^P$BT4(BUT,BTT0
+MBU`(@#@(=1"#[`114U#H_/___^L*C78`@^P(45+_TXM=_,G#58GE5U93@^P4
+MBT4(BE44B%7SBW@HB@"(1?(/MMA35^C\____@\0,:@!35^C\____@\0,:@!J
+M`&H`:@!J`U-7Z/S___^#Q""Z`````(7`#X2M````]T4,````\`^5P`^VR(/L
+M#(7)=!2X)````(!]\R!T#;@T````ZP:)]@^V1?-0N$````"%R740BT4,BU40
+M#ZS0&(/(0`^VP%"+10R+51`/K-`0#[?8#[;#4(M%#(M5$`^LT`@/ML!0#[95
+M#`^WPH7)=`F)V"4`_P``"=!0:@%J`&@``0``_W4848!]\S`/E,`/ML!`4`^V
+M7?)35^C\____B<:#Q#A35^C\____B?*)T(UE]%M>7\G#D%6)Y8I%"(G"@^(#
+MB='!X0V-D0`@``"H!'0*C8$`(`,`ZPB)]HV"```"`,G#58GE5E.#[!R+=0B*
+M10R(1?=H(*$'`.C\____NR"A!P"#Q!"0@^P(#[9%]U#HH____P4<`0``B00D
+M_W8,Z/S___^#Q!"$P'@(N`$```#K))"#[`QH$"<``.C\____@\00@<,0)P``
+M@?L_2TP`=K:X`````(UE^%M>R<.)]E6)Y593BUT(@^P$:@`/MD4,4.A!____
+MB<:-0`B)!"3_<PSH_/___X/$#&@8`0``C48,4/]S#.C\____@\00C67X6U[)
+MPXUV`%6)Y8/L#&H`#[9%#%#H_O[__X/`#(D$)(M%"/]P#.C\____R<.-=@!5
+MB>575E.+70B+?1"+=12Y`````(U3"HM&$(D#BT84B4,$@SX`=0*Q`0^V1P31
+MX`G!9HE+"(M%#(.X.`(```!T#X.X.`(```(/A48!``")]H-^!`$/A<8```!F
+M#[9&#X#,$6:)`H/"`F8/MD8.@,P19HD"@\("#[9'!,'@`R7X````@,P29HD"
+M@\("9@^V1@N`S!-FB0*#P@)F#[9&"(#,$V:)`H/"`F8/MD8,@,P49HD"@\("
+M9@^V1@F`S!1FB0*#P@)F#[9&#8#,%6:)`H/"`F8/MD8*@,P59HD"@\("9L<"
+M0!:#P@*#/@!U&;!@BTT,@[DX`@```@^$@@$``+`FZ7L!``"P88M-#(.Y.`(`
+M``(/A&D!``"P-NEB`0``B?9F#[9&#H#,$6:)`H/"`@^V1P3!X`,E^````(#,
+M$F:)`H/"`F8/MD8(@,P39HD"@\("9@^V1@F`S!1FB0*#P@)F#[9&"H#,%6:)
+M`H/"`HM&""4````/P>@8#4`6``!FB0*#P@*#/@$9P(/@^X/H-.GM````D(-^
+M!`$/A88```!F#[9&#X#,$F:)`H/"`F8/MD8.@,P29HD"@\("9@^V1@N`S!-F
+MB0*#P@)F#[9&"(#,$V:)`H/"`F8/MD8,@,P49HD"@\("9@^V1@F`S!1FB0*#
+MP@)F#[9&#8#,%6:)`H/"`F8/MD8*@,P59HD"@\("9L<"0!:#P@*#/@$9P(/@
+M\(/`->M=D&8/MD8.@,P29HD"@\("9@^V1@B`S!-FB0*#P@)F#[9&"8#,%&:)
+M`H/"`F8/MD8*@,P59HD"@\("BT8()0````_!Z!@-0!8``&:)`H/"`H,^`1G`
+M@^#^@^@VC78`9@^VP`T`E___9HD"6UY?R<.-=@!5B>575E.#[""+50B+?12+
+M6@P/MD4,BT2")(MP,&H`C88@`0``B47P4%/H_/___X/$"(V&!`$``%!3Z/S_
+M__^(!X/$"(V&"`$``(E%[%!3Z/S___]F#[;`9HE'`H/$"(V&#`$``(E%Z%!3
+MZ/S___]F#[;`9HE'!(/$"(V&$`$``(E%Y%!3Z/S___]F#[;`9HE'!H/$"(V&
+M%`$``(E%X%!3Z/S___]F#[;`9HE'"(/$$(-]$`%U;H/L!&B`````_W7P4^C\
+M____@\0(_W7L4^C\____P>`(9@E'`H/$"/]UZ%/H_/___\'@"&8)1P2#Q`C_
+M=>13Z/S____!X`AF"4<&@\0(_W7@4^C\____P>`(9@E'"(/$#&H`_W7P4^C\
+M____@\00@^P(C888`0``4%/H_/___XA'"H/$"(V&'`$``%!3Z/S___^(1PN-
+M9?1;7E_)PXGV58GE5U93@^PTBT409HE%U@^V50R)5="+10B+?)`DBT<P@\`P
+M4(M5"/]R#.C\____@^`?C13%`````"G"C;371`(``(/$$(,^`71!C5X4C478
+M4/]S!/]UT/]U".A,_O___H]1"0``@\0,C5784FH`#[=%UE#_<QQJ`?]UT/]W
+M*/]3&(/$&%97Z,01``"-9?1;7E_)PU6)Y5=64X/L/(M5$(I%#(A%S\=%R```
+M```/ML"+30B+=($D9L=%Q@``9HL"9HE%T(U-T&:+0@)FB4$"BT($B4$$9HM%
+MT(G#@^,?C03=`````"G8C9S&1`(``&:+00*$P'0NQT7(`0```&:+00(E_P``
+M`&:)1<9F@4W&``''ADP)````````QX8T`@```````(U[%(U%T&:+0`*H!'0:
+MC4784/]W!`^V1<]0_W4(Z&;]__^#Q!"-=@"#.P$/A),```#^CE$)``"#[`2-
+M1=A0C470BT`$4`^W1<90_W<<_W7(#[9%SXE%P%#_=BC_5QB#Q!A35NC$$```
+M@\00@[Y,"0```'10@[Y$"0```'1'BX9$"0``@W@0`74[@^P(_W7`_W4(Z,$2
+M``"#Q`C_MD0)``!6Z`X4``"#Q!"%P'47@^P$:@'_MD0)``!6Z$8&``"#Q!"-
+M=@"-9?1;7E_)PU6)Y593BUT(@^P(:"P-``#_<PSH_/___XG&@\0,@\@!4&@L
+M#0``_W,,Z/S___^#Q`AH+`T``/]S#.C\____@\0(:/`$`0#_<PSH_/___XM5
+M#/?2(=`+11"#Q`Q0:/`$`0#_<PSH_/___X/$"&CP!`$`_W,,Z/S___^#Q`Q6
+M:"P-``#_<PSH_/___XUE^%M>R<.-=@!5B>575E.#[!R+10B+0`R)1>S'1>@`
+M````#[9%#(T,A0````"(3?.*11`(1?,/MD7SBU4(BW2")(-]&`!T:X7V#X3-
+M````BY[<"0``@^P$BT44P>`#"T8D4(M&,(/`)%#_=>SH_/___X/$$#M=%'0C
+M#[9]\X/L!(M&%(T$V%!7_W4(Z(G]__]#@^,?@\00.UT4=>&#ODP)````=0?'
+M1>@!````B9[<"0``@WT<`'4*@WWH``^$_P$``(/L"`^V1?-0Z+;W__^#P`B)
+M1>2)!"3_=>SH_/___XG'@\0,#[?8B=CWT%#_=>3_=>SH_/___X/$$/;#"`^$
+M%@$``(7V=!3'ADP)````````QX8T`@```````(M-"(-Y9`%T!H-Y:`%U&H/L
+M!+@``0``BDWST^!04/]U".@S_O__@\00BT4(@WA@`70,@WAD`70&@WAH`745
+M@^P(#[9%\U#_=0CH:PD``(/$$.MZBU4(@WI<`70&@WI@`75K@^P$:@0/MEWS
+M4^CX]O__C7`HB30D_W7LZ/S___^#Q`A6_W7LZ/S____'!"09````Z/S___^#
+MQ`QJ`%;_=>SH_/___X/$"%;_=>SH_/___\<$)-`'``#H_/___X/$"%/_=0CH
+MM`<``(/$$)"#[`@/MEWS4_]U".C\____@\00A<`/A;T```!3:@!J`?]U"(M-
+M"/]1((/$$.FG````]\<0````=%&#[`@/MD7S4/]U".AG!P``@\00BT4(@WAD
+M`70&@WAH`74;@^P$:@"X``$``(I-\]/@4/]U".@9_?__@\00#[9%\U!J`6H!
+M_W4(BU4(_U(@ZTX/M\?VQ`%T1H.^-`(```%U*,>&3`D```````#'AC0"````
+M````@^P$4`^V1?-0_W4(Z-SZ__^#Q!"#/0`````!=0R#[`Q6Z$L0``"#Q!"-
+M9?1;7E_)PU6)Y5.#[`2+70@/MD4,P>`""T40#[;0BTR3)(7)=0V#[`A24^@'
+M#@``ZT60BY%$"0``A=)U$H/L"`^VP%!3Z.T-``#K*XUV`(,Z`70&@WH0`701
+M@^P(#[;`4%/HT`T``.L.B?:#[`A24>@*````B?:+7?S)PXUV`%6)Y5=64X/L
+M)(M5"(M"*(MP#(M:,(H*@^$#B<\/M@*#X`3!Z`*(1=B-@R`!``!05NC\____
+M@\0(@<,<`0``4U;H_/___XC#@\0,N``!``")^=/@]]!0@'W8`1G`)0``__\%
+M%``#`%!6Z/S___^#Q!"$VP^(*`$``/;#`707@^P$:@'_=0S_=0CHWP$``.D.
+M`0``B?:+50R+0A2#^`%T+8/X`7(0@_@"#X2(````Z>X```")]H/L!&H`_W4,
+M_W4(Z*@!``"#Q!#IU````/;#0'12@^P(BT4,@\`44/]U".C%````@\00@_@!
+M=2.+30R#>2``#X6H````@^P$:@!1_W4(Z&8!``"#Q!#ID@```(/L!&H!_W4,
+M_W4(Z$X!``"#Q!#K?8/L!&H!_W4,_W4(Z#D!``#K:P^VPZA`=%"H('5,BT4,
+M@W@@`'43@^P$:@!0_W4(Z!4!``"#Q!#K1(/L"(M%#(/`%%#_=0CH.@```(/$
+M$(7`=2N#[`1J`?]U#/]U".CG````@\00ZQ:)]H/L!&H!_W4,_W4(Z-````"#
+MQ!"0C67T6UY?R<-5B>575E.#[`R+?0B+=0RX`````(%^#/\````/AI@```"+
+M%H/Z`711N`````"#^@$/@H,```"#^@)U>;L`````@^P$BT8(#[<04H/``HE&
+M"(M',`4``0``4(M'*/]P#.C\____@\000X'[_P```';0@6X,``$``.L[NP``
+M``")]H/L"(M',`4``0``4(M'*/]P#.C\____BU8(9HD"@T8(`H/$$$.!^_\`
+M``!VTX%N#``!``"X`0```(UE]%M>7\G#C78`58GE5U93@^P<BW4(OP````"+
+M70R#PQ2-1=A0_W,$#[8&4/]V*.A:]O__@\00@WT0`7419K\!`,>&3`D`````
+M``"-=@"#[`2-1=A0:@!J`/]S(%</M@90_W8H_U,<@\08_W4,5NB_"0``@\00
+M@WT0`746@ST``````74-@^P,5NC8#```@\00D(.^1`D````/A+L```"+AD0)
+M``"#>!`!=2^#[`A05N@%#0``@\00A<`/A9H```"#[`1J`?^V1`D``%;H.?__
+M_X/$$.F!````D(.^.`(```!U,X/L"`^V!E#_=BCH9`H``(/$"(M&,(/`*%"+
+M1BC_<`SH_/___\<$)`$```#H_/___X/$$(/L"`^V!E#_=BCHA0H``(N>1`D`
+M`(/$$(7;="B#>Q``=2*)]H/L!(U#%%!35NBZ"P``BUL(@\00A=MT"(-[$`!T
+MXHGVC67T6UY?R<-5B>575E.#[!2+70B+0RB+>`R+0S")1?")QH/&*%97Z/S_
+M__^#Q!"H!'0V@^P$:@)65^C\____@\0(5E?H_/___\<$)-`'``#H_/___X/$
+M"`^V`U#_<RCH.`(``(/$$.L5@^P$:@*+1?"#P"A05^C\____@\00QH,\`@``
+M`,:#4`D```#&@U$)````QX/8"0```````,>#W`D```````#'@S0"````````
+MQX/4"0``(````+H`````B9235`D``$*#^A]^\[H`````B?:-!-4`````*=#'
+MA,-$`@```0```$*#^A]^YL>#1`D```````#'@T@)````````QX-,"0``````
+M`(/L!&H`BT7P@\`(4%?H_/___X/$#&@8`0``BT7P@\`,4%?H_/___X/$#/]S
+M&(M%\(/`$%!7Z/S___^#Q`R+0QPE`/S__U"+1?"#P!105^C\____@\0,BT,H
+MN@````"#>'@`=`.+4QQ2BT7P@\`84%?H_/___X/$#/]S((M%\(/`'%!7Z/S_
+M__^#Q`R+0RBZ`````(-X>`!T`XM3'%*+1?"#P"!05^C\____@\0,BT,DL`!0
+MBT7P@\`D4%?H_/___[@!````C67T6UY?R<.-=@!5B>575E.#["R+=0B+111F
+MB476@WT,``^%F@```+L`````B?:-!-T`````*=B-!,:#N$0"````=72#N%0"
+M```!=1J+D%P"``"+N'@"``"+@'0"``")1=#K)(UV`(T$W0`````IV(T$QHN0
+M7`(``(NX=`(``(N`<`(``(E%T(U%V%!2#[8&4/]V*.C*\O__@\0,C4784&H`
+M#[=%UE!7_W40#[8&4/]V*/]5T(/$($.#^Q\/CFW___^-9?1;7E_)PY!5B>57
+M5E.#[!R+70@/MG4,B?"#X`3!Z`*(1=B_`P```"'W@WM<`70&@WM@`75K@^P(
+MB?H/ML*)PL'B"(V"#`$``(!]V`!T"8V"#`$#`.L&D`4```(`4/]S#.C\____
+MB<&#X?R#R0&#Q`Q1B?H/ML*)PL'B"(V"#`$``(!]V`!T"(V"#`$#`.L%!0``
+M`@!0_W,,Z/S___^#Q!"#[`B)^@^VPHG"P>((C8)T`0``@'W8`'0*C8)T`0,`
+MZP>)]@4```(`4/]S#.C\____B<&`X1^)\@^VP@^VE!B)````P>(%@>+@````
+M"=&`Y></MH08D0```,'@"R4`&```"<&#Q`Q1B?H/ML*)PL'B"(V"=`$``(!]
+MV`!T"HV"=`$#`.L'B?8%```"`%#_<PSH_/___X/$$(UE]%M>7\G#C78`58GE
+M5U93@^P,BUT(#[9U#%;HFNW__X/L!(UX*%93Z'WN__^#Q`QJ!%?_<PSH_/__
+M_X/$"%?_<PSH_/___\<$)!D```#H_/___X/$#&H`5_]S#.C\____@\0(5_]S
+M#.C\____@\0(5E/H5_[__X/$"%93Z.'M__^-9?1;7E_)PY!5B>575E.#[`S&
+M1?(`QD7S`(UV`(I%\HI5\XT<@@^VVU/H"NW__XG':@*-<"A6BT4(_W`,Z/S_
+M__^#Q`A6BU4(_W(,Z/S___^#Q`A3_W4(Z"S___^#Q`QJ`%:+10C_<`SH_/__
+M_X/$#&@?`0``5XM5"/]R#.C\____@\0,:@"-1P10BT4(_W`,Z/S___^#Q`QJ
+M`(U'"%"+50C_<@SH_/___X/$#&H`C4<,4(M%"/]P#.C\____@\0,:@"-1Q!0
+MBU4(_W(,Z/S___^#Q`QJ`(U'%%"+10C_<`SH_/___X/$#&H`C4<84(M5"/]R
+M#.C\____@\0,:@"-1QQ0BT4(_W`,Z/S___^#Q`QJ`(U')%"+50C_<@SH_/__
+M_X/$#&H`C4<@4(M%"/]P#.C\____@\0,:@"-1RQ0BU4(_W(,Z/S___^#Q`QH
+MO````(U'-%"+10C_<`SH_/___X/$$/Y%\X!]\P,/AJS^__^`??(!&=N!XP``
+M__^!PP```P"#[`1J`(U##%"+50C_<@SH_/___X/$#&H`C4,04(M%"/]P#.C\
+M____@\0,:@"-0Q10BU4(_W(,Z/S___^#Q`QJ`(U#&%"+10C_<`SH_/___X/$
+M"(/#(%.+50C_<@SH_/___R4<'!P<#0,#`P.#Q`Q04XM%"/]P#.C\____@\00
+M_D7R@'WR`0^&!_[__XUE]%M>7\G#B?95B>6#[`QH___/#VAL!`$`BT4(_W`,
+MZ/S____)PXUV`%6)Y5.#[`2+70B+0P2P`#T``(!0=">#[`AH+`T``/]S#.C\
+M____@\@!@\0,4&@L#0``_W,,Z/S___^#Q!"#[`AH``T``/]S#.C\____)?__
+M`/^#Q`Q0:``-``#_<PSH_/___X/$#&H`:`0-``#_<PSH_/___X/$#&H`:#@,
+M``#_<PSH_/___X/$#&C_``$`:`0=``#_<PSH_/___X/$#&H`:&0=``#_<PSH
+M_/___X/$#&H`:"@,``#_<PSH_/___X/$#&H`:%@=``#_<PSH_/___X/$#&H`
+M:%P=``#_<PSH_/___X/$#&H`:$`=``#_<PSH_/___X/$#&H`:$0=``#_<PSH
+M_/___X/$#&H`:$@=``#_<PSH_/___X/$#&H`:%`=``#_<PSH_/___XM=_,G#
+MC78`58GEBU4(BTT,QT$(`````(N"2`D``(E!#(.Z2`D```!T"8N"2`D``(E(
+M"(F*2`D``(.Z1`D```!U!HF*1`D``,G#B?95B>53BUT(BTT,@WD(`'48BT$,
+MB8-("0``A<!T%,=`"`````#K"XGVBU$(BT$,B4(,@WD,`'49BT$(B8-$"0``
+MA<!T%<=`#`````#K#(UV`(M1#(M!"(E"",=!"`````#'00P`````6\G#B?95
+MB>575E.#[`R+?0B+=0R+71"#.P!U&8/L!&H@C4,$4(U&%%#H_/___X/$$.L=
+MB?:#[`1J)(U#!%"-1A10Z/S____^AU`)``"#Q!"+`XE&$%97Z/G^___'!@``
+M``#^ASP"``"-9?1;7E_)PY!5B>564XM="(MU#(-^$`!T!OZ+4`D``%93Z`K_
+M__^+@]0)```/ME8$B92#5`D``$")@]0)``#'!@$```#^BSP"``"-9?A;7LG#
+MC78`58GEBD4(/`-V$@^VR(U,"0*X`0```-/@ZQ")]@^VR(U,"0&X`0```-/@
+MR<-5B>575E.#[`R+?0B+=U0/MET,4^B[____@^P$"?")1U13Z!'H__\%'`$`
+M`(D$)/]W#.C\____@\0,B=F#X0.X``$``-/@]]!0@^,$P?L"@_L!&<`E``#_
+M_P44``,`4/]W#.C\____@\0(:&0=``#_=PSH_/___X/$$#GP=2:#[`3_=U1H
+M9!T``/]W#.C\____@\0(:&0=``#_=PSH_/___X/$$(UE]%M>7\G#58GE5E.+
+M=0B+7E0/MD4,4.@3____@^P$]]`AV(E&5&AD'0``_W8,Z/S___^#Q!`YV'49
+M@^P$_W94:&0=``#_=@SH_/___X/$$(UV`(UE^%M>R<.058GE5U93@^P0BET,
+MBT4(BT`,B47P#[;3BT4(BW20)(M^,,>&-`(```$```"#X@3!Z@*)V8/A`[@!
+M`0``T^#WT%"`^@$9P"4``/__!10``P!0_W7PZ/S___^#Q`QJ`(M&,(/`"%#_
+M=?#H_/___X/$"`^VVU/_=0CH,/___X/$"(V''`$``%#_=?#H_/___\>&0`(`
+M``$```"#Q`QJ`8/'*%?_=?#H_/___XUE]%M>7\G#C78`58GE5U93@^P0BD4,
+MB$7SNP````"+50B+>@P/ML"+1((DBW`PQX`T`@```````&H"C48H4%?H_/__
+M_X/$$(UV`(/L"(U&*%!7Z/S___^#Q!"H`70<@^P,:.@#``#H_/___X/$$$.!
+M^^<#``!VT^L)D('[YP,``'83@^P(#[9%\U#_=0CH3/C__X/$$(/L"`^V1?-0
+M_W4(Z+']__^-9?1;7E_)PY!5B>564XM="(M#*(MP#/]U$/]U#%.+@]@)``#!
+MX`4#0Q!0Z-'F__^+D]@)``!"@^(?B9/8"0``_H-1"0``@\0,BT,<)0#\___!
+MX@4)T%"+0S"#P!105NC\____C67X6U[)PXGV58GE4X/L!(M="&H`:@)J`%/H
+MNO7__XD<).C>\___B1PDZ/S___^#Q!"#^`%U'L>#3`D```$```"#[`@/M@-0
+M_W,HZ/W\__^#Q!")]HM=_,G#C78`58GE5U93@^P4BWT,@\<4BU4(BT(HBW`,
+MBUHPC8,<`0``4%;H_/___R7!````@\00N@````"#^$`/A=X!``"#?P0!=7"#
+M[`0/MD<14(V#!`$``%!6Z/S___^#Q`P/MD<34(V#"`$``%!6Z/S___^#Q`P/
+MMD<54(V##`$``%!6Z/S___^#Q`P/MD<74(V#$`$``%!6Z/S___^#Q`P/MD<9
+M4(V#%`$``%!6Z/S___^#Q!#K-(GV9O='$`#_=2!F]T<2`/]U&&;W1Q0`_W40
+M9O='%@#_=0AF]T<8`/]T"KH`````Z38!``"#[`0/MD<04(V#!`$``%!6Z/S_
+M__^#Q`P/MD<24(V#"`$``%!6Z/S___^#Q`P/MD<44(V##`$``%!6Z/S___^#
+MQ`P/MD<64(V#$`$``%!6Z/S___^#Q`P/MD<84(V#%`$``%!6Z/S___^#Q`P/
+MMD<:4(V#&`$``%!6Z/S___^#Q`P/MD<;4(V#'`$``%!6Z/S___^#Q!"#/P(/
+MA9````"#[`B-@R`!``!05NC\____QP0DR````&@0)P``:@%J`/]U".C\____
+M@\0@N@````"%P'1C@^P(C8,<`0``4%;H_/___R6)````@\00N@````"#^`AU
+M0<=%\`````"!PP`!``"#[`2+1P@/MQ!2@\`"B4<(4U;H_/___X/$$/]%\(%]
+M\/\```!VVH%O#``!``"Z`0```(GVB="-9?1;7E_)PXGV58GE5U93@^P0BW4(
+MQT94`````&H`:&0=``#_=@SH_/___X/$!/]V#.C\____B$8$@\00N`````"`
+M?@0`#X32`@``QT9<`````,=&8`````#'1F0`````QT9H`````,=&;`````#'
+M1G``````QT9T`````,:&@`````&X`````(-^(``/A(L"``"X`````&:!?@:!
+M4`^%>@(```^V5@6#^@$/A&T"``"#^@%_!H72=`GK&8/Z`G0+ZQ+'1F`!````
+MZQ#'1F0!````ZP?'1F@!````LP"#[`@/MM.)T(/@`\'@"(V(=`$``/;"!'0'
+M!70!`P#K!HV!```"`%#_=@SH_/___P^VRXG"@>+@````P>H%B)0QB0```"4`
+M&```P>@+B(0QD0```(/$$$.`^P=VI(/L#%;HE_3__XDT).B?]O__B30DZ+?V
+M__^#Q`QJ`&CP!`$`_W8,Z/S___^#Q`AH+`T``/]V#.C\____B47P@V7P_H/$
+M#/]U\&@L#0``_W8,Z/S___^#Q!"#OH0`````=5>#[`AH``T``/]V#.C\____
+MB47P@\00J#!T.8M&1*G@`P"`=`@E'_S_?XE&1(-^7`%T$H-^8`%T#(-^9`%T
+M!H-^:`%U$XM&1*@0=`R#X.^)1D3K!(-F1,^S`(GV@^P(#[;#4%;H&_+__X/$
+M$$.`^P=VZH/L!/]V1&@`#```_W8,Z/S___^#Q`S_=DAH*`P``/]V#.C\____
+M@\0,_W9,:%P=``#_=@SH_/___[,`@\00@^P$#[;#_W2&$(#[`1G`)0``__\%
+M#``#`%#_=@SH_/___X/$#`^VP_]TAAB`^P$9P"4``/__!1```P!0_W8,Z/S_
+M__^#Q!!#@/L!=K/'1E15JP8`LP")]H/L"`^VPU!6Z`/A__^#Q!!#@/L'=NJ#
+M[`QH4,,``.C\____@\00@WYD`70&@WYH`75#QT7P`````+,`D(/L"`^V^U=6
+MZ/S___^#Q!"%P'4+9K@``8GYT^`)1?!#@/L'=MN#[`3_=?!H`/\``%;HYN;_
+M_X/$$+@!````B?:-9?1;7E_)PU6)Y8/L#&H`:&0=``"+10C_<`SH_/___[@!
+M````R<.058GEQP4``````0```,G#D%6)Y<<%``````````#)PY!5B>564XMU
+M"(I5#`^VPHM<AB2X`````(7;="P/ML)0Z(#?__^#[`B)0S")<RC'@TP)````
+M````QX,T`@```````%/HW>W__XUE^%M>R<.)]E6)Y0^V50R+10B+5)`DN```
+M``"%TG0-@[I,"0```0^5P`^VP,G#B?95B>564XMU"(I%#(G"@^(#B='!X0B-
+MD0`!``"H!'0)C9D``0,`ZP>0C9H```(`@^P(4_]V#.C\____@^`/@\00@_@!
+M=`^#^`%R0H/X`W0UZSN-=@"#[`A3_W8,Z/S____'!"1`#0,`Z/S___^#Q`A3
+M_W8,Z/S___^#X`^#Q!"#^`-U"9"X`0```.L&D+@`````C67X6U[)PU6)Y5=6
+M4X/L+(M]"(I%#(A%\P^VP(M4AR2+3PR)3>RX`````(72#X0``@``BT(PB47H
+MN`````"#NDP)```!#X3H`0``QT7D`````)#_1>2#?>0*=@NX`````.G-`0``
+MD(/L"`^V7?-35^AV\/__@\0(4U?H#-___XI5\X/B`XA5XX/C!(G>P>X"QT7<
+M`````(/$$`^VPHG!P>$(B4W4@<$(`0``B4W8P>`(B470@^P(BU74C8((`0,`
+MB?&$R74)BU78C8(```(`4/]W#.C\____B<.#X_"#RP&#Q`Q3BT70C9`(`0``
+MC8`(`0,`B?&$R74&C8(```(`4/]W#.C\____@\0(#[9%XXG"P>((C8((`0``
+MB?&$R70+C8((`0,`ZPB-=@`%```"`%#_=PSH_/___\<$)!D```#H_/___X/C
+M\(/$#%,/MD7CB<+!X@B-@@@!``")\83)=`J-@@@!`P#K!XGV!0```@!0_W<,
+MZ/S___^#Q`@/MD7CB<+!X@B-@@@!``")\83)=`J-@@@!`P#K!XGV!0```@!0
+M_W<,Z/S____'!"30!P``Z/S___^#Q`@/MEWS4U?H_/___X/$$(7`=1__1=R#
+M?=P"#X;=_O__@^P(4U?H8MW__X/$$.EJ_O__@^P(BT7H!1P!``!0_W7LZ/S_
+M__\EP````(/$$(/X0'05@^P(#[9%\U!7Z"O=__^X`````.L4@^P(#[9%\U!7
+MZ!;=__^X`0```)"-9?1;7E_)PU6)Y5=64X/L#(M5"(I%%(A%\P^V10R+7((D
+MN0````"%VW1UBWH,BW,PN0````"#NTP)```!=&&#[`A65^C\____B<*#Q!"#
+M?1``=2B*1?-(N0`````\'W<_QX,X`@```````(/BP`^V1?-("<*`S@+K$HGV
+MQX,X`@```0```('BP/W__X#.`8/L!%)65^C\____N0$```"0B<B-9?1;7E_)
+MPXGV58GE4X/L!(M="(I-#`^VP8M4@R2X`````(72="JX`````(.Z3`D```%T
+M',>"3`D```$```"#[`@/ML%04^A"]/__N`$```"+7?S)PU6)Y5.#[`2+70B*
+M50P/ML*Y`````(-\@R0`="`/MM*+1),DQX!,"0```````(/L"%)3Z+7T__^Y
+M`0```(G(BUW\R<.058GE4X/L!`^V50R+10B+7)`DN`````"%VW0HN`````"#
+MNTP)```!=!IJ`&H"_W404^A*Z___B1PDZ&[I__^X`0```(M=_,G#58GE#[95
+M#(M%"(M4D"2X_P```(72=`</MH(\`@``R<-5B>575E.#[`R*70P/MG40#[9]
+M%(C:@^(#B=B#X`3!Z`+WQO@```!U"/?'_````'0+N`````#IC0```)`/MM*)
+MT<'A"(V1=`$``(3`=`Z!P70!`P")3?#K#(UV`('"```"`(E5\(/L"/]U\(M%
+M"/]P#.C\____)!^)\L'B!8'BX`````G0@.3GB?K!X@N!X@`8```)T(/$#%#_
+M=?"+50C_<@SH_/___P^VPXGYBU4(B(P0D0```(GQB(P0B0```(/$$+@!````
+MD(UE]%M>7\G#58GE5U93@^P4BW4(BET,OP,````AWX/C!,'K`H#[`1G`)0``
+M__\%&``#`%#_=@SH_/___XGY#[;1C4H8N@$```#3X@G0@\0,4(#[`1G`)0``
+M__\%&``#`%#_=@SH_/___X/$"(#[`1G`)0``__\%&``#`%#_=@SH_/___X/$
+M$+@!````C67T6UY?R<-5B>575E.#[!2+=0@/MGT,B?B#X`.(1?.)^`^VT(G0
+M@^`$B</!ZP)25NA.ZO__@\0(@/L!&<`E``#__P48``,`4/]V#.C\____#[9-
+M\X/!&+K^____T\(AT(/$#%"`^P$9P"4``/__!1@``P!0_W8,Z/S___^#Q`B`
+M^P$9P"4``/__!1@``P!0_W8,Z/S___^#Q`@/MD7SB<+!X@B-@@@!``"$VW0*
+MC8((`0,`ZP>)]@4```(`4/]V#.C\____@^#P@\@!@\0,4`^V1?.)PL'B"(V"
+M"`$``(3;=`F-@@@!`P#K!I`%```"`%#_=@SH_/___X/$"(GZ#[;"4%;H>^G_
+M_X/$$+@!````C67T6UY?R<.-=@!5B>575E.#[!2+?0B+=1"*10R(1?.+1PR)
+M1>P/MD7SBUR')(G"@^(#B='!X0B-D70!``"H!'0(C8%T`0,`ZP:-@@```@!0
+M_W<,Z/S___^)1EB#Q!"%VW40QP8`````N`$```#I_P```(M#,(E%Z(/L"`^V
+M1?-05^C\____B0:#Q!"X`0```(,^``^$V````(N#3`D``(E&+(N#.`(``(E&
+M,(J#/`(``(A&-+H`````D(I$$VJ(1!8$0H/Z)W;R@^P(_W7H_W7LZ/S___^)
+M1CB#Q`B+1>B#P!!0_W7LZ/S___^)1CR#Q`B+1>B#P!10_W7LZ/S___^)1D"#
+MQ`B+1>B#P!A0_W7LZ/S___^)1D2#Q`B+1>B#P!Q0_W7LZ/S___^)1DB#Q`B+
+M1>B#P"!0_W7LZ/S___^)1DR#Q`B+1>B#P"10_W7LZ/S___^)1E"#Q`B+1>B#
+MP"A0_W7LZ/S___^)1E2X`0```(UE]%M>7\G#D%6)Y5=64X/L#(I=#`^VPXM5
+M"(MT@B2X`0```(.^3`D````/A$D!``"X`@```("^/`(``!\/A#<!``"+AM0)
+M``!(B8;4"0``BY2&5`D``(T$U0`````IT(V\QD0"``"(5P2+11"#.``/A:L`
+M``"#P`2#OC@"```"=0O'0`0!````ZRZ)]H-X!`%U)H-^"`!U((N&U`D``(F4
+MAE0)``!`B8;4"0``N`,```#IPP```(GV@+Y0"0```'5+@[XT`@```'4>@^P(
+M#[;;4_]U".B'[O__@\0(4_]U".C/[O__@\00@^P$_W405U;HV^S__X/$#(M%
+M$(/`!%!75N@&\/__@\00ZVF0@^P$_W405U;HM^S__X/$$.M6B?:#[`3_=1!7
+M5NBC[/__@\00@+X\`@```74[@[XT`@```742@^P(#[;#4/]U".@7[___@\00
+M@^P(5U;H9O#__X/$$(7`=0]75NC0[/__N`$```#K!I"X`````(UE]%M>7\G#
+MC78`58GE5U93@^PDBT4(BT`,B47D:&`=``!0Z/S___^)QHM5"(M25(E5\(/$
+M$+@`````A=8/A(L!``#WQ@``!`!T6H/L"(M%"(.XA`````$9P"58!```!0`9
+M``!0_W7DZ/S___^)PX/$#/?04(M5"(.ZA`````$9P"58!```!0`9``!0_W7D
+MZ/S___]J`%-J`/]U"(M5"/]2((/$(,9%ZP#WQO\!```/A`(!``"`?>L!&=N!
+MXP``__^!PQ0``P"#[`1J[U/_=>3H_/___X/$"%/_=>3H_/___XG'@\0,]]"#
+MR!!04_]UY.C\____@\0(@'WK`1G`)0``__\%"``#`%#_=>3H_/___XE%X+,`
+M@\00#[9%ZXE%V(UV``^VR[@``0``T^`A^(E%[+@!````T^")\H/B`2'X=02%
+MTG0A@^P$5U)0BT7@@^`?4`^VPU`/MD7K4/]U".B3V___@\0@@WWL`'0VBU78
+MB57<B=#!X`()V`^VP%#HC.O__X/$!(5%\'08@^P$#[;#4/]UW/]U".@:WO__
+M@\00C78`P>X"P6W@"$.`^P,/AF_____K!)#![@C1[OY%ZX!]ZP$/AN#^__^X
+M`0```(UE]%M>7\G#D%6)Y8/L#&H`:&0=``"+10C_<`SH_/___[@!````R<.0
+M58GE@^P,BT4(_W!4:&0=``#_<`SH_/___[@!````R<-5B>575E.#[`R+70B+
+M?12*10R(1?.*11"(1?*^`````#G^<UR#[`B+0S`%'`$``%"+0RC_<`SH_/__
+M_XC"@\00A,!X%(I%\PI%\B'0.D7S#Y3`#[;`ZRZ0@^P,_W48Z/S___^#Q`@/
+MM@-0_W,HZ/S___^#Q!"%P'0,1CG^<J:)]K@`````C67T6UY?R<.-=@!5B>53
+M@^P(BUT(:@"+0S`%(`$``%"+0RC_<`SH_/___X/$"(M#,`4@`0``4(M#*/]P
+M#.C\____BUW\R<-5B>53@^P(BUT(:@*+0S`%(`$``%"+0RC_<`SH_/___X/$
+M"(M#,`4<`0``4(M#*/]P#.C\____BUW\R<-5B>575E.#[`R+50@/MD4,BUR"
+M)+@`````A=MT;XMZ#(MS,+@`````@[M,"0```71;@^P,4^C\____@\0,:.$`
+M``"-AAP!``!05^C\____QP0D9````&@0)P``:@%J`%/H_/___X/$((7`=1"#
+M[`Q3Z/S___^X`````.L0@^P,4^C\____N`$```")]HUE]%M>7\G#58GE5U93
+M@^P,BWT(BD4,B$7S#[;`BW2')+@`````A?8/A*8```"-7C2-=@"#[`QH[```
+M`&H`:@!J`&H`:@!J`&@``0``C48T4&H`:@$/MD7S4%?H_/___X/$0(7`=&WV
+M`P1T+F:!>P3(-W4F@^P$:@!J`&H`:@!J!P^V1?-05^C\____@\0@A<!T06;'
+M`P``ZYKV@Z````#@="RQ`+@!````@+O^`0``I74AN@`````/M\("#!A"9H'Z
+M_P%V\K@`````A,EU!;@!````C67T6UY?R<-5B>6#[`@/ME4,BT4(BU20)+@`
+M````A=)T%[@`````@[I,"0```70)@^P,4NC\____R<.058GE@^P4:.\```!J
+M``^V12!0#[9%'%`/MD484`^V1110#[9%$%!J`&H`:@!J``^V10Q0_W4(Z/S_
+M___)PXUV`%6)Y5=64X/L#(I%#(A%\XM-(&:)3?"+121FB47NBTTH9HE-[(M]
+M+(MU,(I=-(I%.(A%ZP^V1?.Z`````(M-"(-\@20`=#Z#[`P/MD7K4`^VPU`/
+MM\90#[?'4`^W1>Q0#[=%[E`/MT7P4/]U'/]U&/]U%/]U$`^V1?-0_W4(Z`X`
+M``")PHG0C67T6UY?R<.)]E6)Y5=64X/L7(M5"(M]((M%)&:)1=B+12AFB47(
+MBT4L9HE%N(M%,&:)1:B*132(1:>*13B(1:8/MD4,BUR")(MR#(M3,(E5H(.[
+M-`(```%T"8.[3`D```%U"KH`````Z00#``"#[`QHZ`,``&BX"P``:@!J0%/H
+M_/___X/$(+H`````A<`/A-T"``"#[`Q3Z/S___^#Q!"#?10!#X6!````@^P$
+MB?H/ML90BT6@!00!``!05NC\____@\0,BU78#[;&4(M%H`4(`0``4%;H_/__
+M_X/$#(M5R`^VQE"+1:`%#`$``%!6Z/S___^#Q`R+5;@/ML90BT6@!1`!``!0
+M5NC\____@\0,BU6H#[;&4(M%H`44`0``4%;H_/___X/$$.L_]\<`_P``=23W
+M1=@`_P``=1OW1<@`_P``=1+W1;@`_P``=0GW1:@`_P``=!.#[`Q3Z/S___^Z
+M`````.D'`@``@^P$B?H/ML)0BT6@!00!``!05NC\____@\0,#[9%V%"+1:`%
+M"`$``%!6Z/S___^#Q`P/MD7(4(M%H`4,`0``4%;H_/___X/$#`^V1;A0BT6@
+M!1`!``!05NC\____@\0,#[9%J%"+1:`%%`$``%!6Z/S___^#Q`P/MD6G4(M%
+MH`48`0``4%;H_/___X/$#`^V1:90BT6@!1P!``!05NC\____@\00@WT0`'5$
+M@^P,:!`G``!H'`P``&H!:@!3Z/S___^#Q""%P'43@^P,4^C\____N@````#I
+M*P$``(/L#%/H_/___[H!````Z1@!``"#[`QH$"<``&@<#```:@%J`%/H_/__
+M_X/$((7`=1.#[`Q3Z/S___^Z`````.GG````@^P(BT6@!1P!``!05NC\____
+M@\00J`AU$X/L#%/H_/___[H`````Z;L```"_`````#M]''-GBT6@!0`!``")
+M19R)^H32=1N#[`QJ9&A0PP``:@%J`%/H_/___X/$((7`=&:#?1`!=1B#[`C_
+M=9Q6Z/S___^+51AFB01Z@\00ZQ>#[`2+51@/MP1Z4/]UG%;H_/___X/$$$<[
+M?1QRI(/L#&ID:%##``!J`6H`4^C\____@\0@A<!U((/L#%/H_/___[H`````
+MZQ^#[`Q3Z/S___^Z`````.L/@^P,4^C\____N@$```"0B="-9?1;7E_)PXGV
+M58GE5U93@^P0BUT(BT,HBW`,BWLP:@:-AR`!``!05NC\____@\0(BT,P!2`!
+M``!05NC\____QP0D"@```.C\____B1PDZ/S____'!"3T`0``Z/S____'!"3T
+M`0``Z/S____'!"3T`0``Z/S____'!"3T`0``Z/S___^[`````(/$$(GV@^P(
+MC8<<`0``4%;H_/___X/$$(3`>`FX`0```.L@B?:#[`QH]`$``.C\____@\00
+M0X'[#R<``';'N`````"-9?1;7E_)PXGV58GE4X/L$(M="%/H)/___X/$$+H!
+M````@_@!=!>#[`QH]`$``.C\____B1PDZ`+___^)PHG0BUW\R<.058GE5E.+
+M=0B#[`1HH````*'@````BT`8!0``!`!05NC\____NP````"#Q!"#[`BAX```
+M`(M`$`4```0`4%;H_/___X/$$$.#^P1^X+L`````C78`@^P(H>````"+0!0%
+M```$`%!6Z/S___^#Q!!#@_L%?N"[`````(UV`(/L"*'@````BT`0!0``!`!0
+M5NC\____@\000X/[!'[@C67X6U[)PY!5B>53@^P(BUT(:(````"AX````(M`
+M#`4```0`4%/H_/___X/$#&BP````H>````"+0!@%```$`%!3Z/S___^+7?S)
+MPU6)Y593BW4(@^P,5N@'____@\0,:@"AX````(M`"`4```0`4%;H_/___X/$
+M"*'@````BP`%```$`%!6Z/S___^(PXDT).AR____#[;;B=B-9?A;7LG#B?95
+MB>575E.#[`R+70B#/00`````#X4'`0``@^P(:"P-``!3Z/S___^)QX/$#(/(
+M`5!H+`T``%/H_/___X/$"&CP!`$`4^C\____B<:#X&*#Q!"#^&)T&H/L!(GP
+M@\AB#[;`4&CP!`$`4^C\____@\00@^P(:```!`!3Z/S___^)'"3H.O[__X/$
+M#&H'H>````"+0`@%```$`%!3Z/S___^#Q`QJ$:'@````BP`%```$`%!3Z/S_
+M__^#Q`QJ!:'@````BT`(!0``!`!04^C\____@\0,:A&AX````(L`!0``!`!0
+M4^C\____B1PDZ'3^__^#Q`Q6:/`$`0!3Z/S___^#Q`AH\`0!`%/H_/___X/$
+M#%=H+`T``%/H_/___X/$$)"-9?1;7E_)PU6)Y5=64X/L%(M="&@L#0``4^C\
+M____B<>#Q`R#R`%0:"P-``!3Z/S___^#Q`AH\`0!`%/H_/___XG&@^!B@\00
+M@_AB=!J#[`2)\(/(8@^VP%!H\`0!`%/H_/___X/$$(/L"&@```0`4^C\____
+MB1PDZ"+]__^#Q`QJ!Z'@````BT`(!0``!`!04^C\____@\0,:@"AX````(L`
+M!0``!`!04^C\____@\0,:@6AX````(M`"`4```0`4%/H_/___X/$#&H`H>``
+M``"+``4```0`4%/H_/___XD<).A<_?__@\0,5FCP!`$`4^C\____@\0(:/`$
+M`0!3Z/S___^#Q`Q7:"P-``!3Z/S___^#Q!"-9?1;7E_)PY!5B>53@^P0BUT(
+M4^AP_/__@\0,:@JAX````(M`"`4```0`4%/H_/___X/$#&H`H>````"+``4`
+M``0`4%/H_/___XD<).C;_/__@\0(:`$`!`!3Z/S___^#Q!"Z_____Z@"=5"#
+M[`Q3Z!3\__^#Q`QJ`:'@````BP`%```$`%!3Z/S___^)'"3HF/S__\<$)&0`
+M``#H_/___X/$"&@!``0`4^C\____T>B#X`&#Q!"#^`$9THG0BUW\R<.)]E6)
+MY5=64X/L%(M]"(I%#(A%\XM?#&@L#0``4^C\____B47L@\0,@\@!4&@L#0``
+M4^C\____@\0(:/`$`0!3Z/S___^)QH/@8H/$$(/X8G0:@^P$B?"#R&(/ML!0
+M:/`$`0!3Z/S___^#Q!"#[`AH```$`%/H_/___X/$#(I%\XB'B`````^VP%"A
+MX````(M`'`4```0`4%/H_/___X/$#%9H\`0!`%/H_/___X/$"&CP!`$`4^C\
+M____@\0,_W7L:"P-``!3Z/S___^#Q!"-9?1;7E_)PXUV`%6)Y8/L"(M5"(I-
+M#(!]$`!T$K@!````T^`(@H@```#K$(UV`+C^____T\`@@H@```"#[`@/MH*(
+M````4%+H_/___\G#58GE5U93@^P4BWT(QD7S`&@L#0``5^C\____B47H@\0,
+M@\@!4&@L#0``5^C\____@\0(:/`$`0!7Z/S___^)1>R#X&*#Q!"#^&)T&X/L
+M!(I%[(/(8@^VP%!H\`0!`%?H_/___X/$$(/L"&@```0`5^C\____@\0(:```
+M!`!7Z/S___^#Q!!F/56J#X7U````O@````"-=@"[`````#GS?R>-=@"#[`1H
+MH````*'@````BT`8!0``!`!05^C\____@\000SGS?MR!_L@```!^$(/L#&H!
+MZ/S___^#Q!"-=@"#[`AH```$`%?H_/___XG#@\0,:+````"AX````(M`&`4`
+M``0`4%?H_/___X/$$&:!^U6J=11&@?['````#XYU____9H'[5:IT6X/L!&BP
+M````H>````"+0!@%```$`%!7Z/S___^#Q`AH```$`%?H_/___X/$$&8]5:IU
+M*,9%\P&#[`Q7Z";Z__^#Q!`\/G44@^P,5^BN_/__@\00A<!U!,9%\P*#[`1H
+ML````*'@````BT`8!0``!`!05^C\____@\0,_W7L:/`$`0!7Z/S___^#Q`AH
+M\`0!`%?H_/___X/$#/]UZ&@L#0``5^C\____@\00#[9%\XUE]%M>7\G#58GE
+M5E.+=0B[`````(UV`,<%X````*````"#[`Q6Z/W]__^#Q!"$P'4CQP7@````
+MP````(/L#%;HX_W__X/$$(3`=0E#@?OG`P``?L,/ML"-9?A;7LG#D%6)Y8M%
+M#(!X`0)U&(!X`@%U$@^V0`.C!````+@`````ZP>)]KC_____R<.058GE5U93
+M@^P4BT4(BW`,BWT0:"P-``!6Z/S___^)1>R#Q`R#R`%0:"P-``!6Z/S___^#
+MQ`AH\`0!`%;H_/___XE%\(/@8H/$$(/X8G0;@^P$BD7P@\AB#[;`4&CP!`$`
+M5NC\____@\00@^P(:```!`!6Z/S___^#Q!"#?1@`=#"[`````#M=%'U6B?:#
+M[`B+50R-A!H```0`4%;H_/___X@'@\000T<[711\X.LRB?:[`````#M=%'TF
+MB?:#[`0/M@=0BU4,C80:```$`%!6Z/S___^#Q!!#1SM=%'S>B?:#[`3_=?!H
+M\`0!`%;H_/___X/$"&CP!`$`5NC\____@\0,_W7L:"P-``!6Z/S___^#Q!"X
+M`0```(UE]%M>7\G#``````````````````0`````````````````````````
 M````````````````````````````````````````````````````````````
 M````````````````````````````````````````````````````````````
-M`````````````````0```````0`!``$``@`!``,``0`$``$`!0`!``8``0``
-M``$``"`!``!``0``8`$``(`!``"@`0``P`$`1#8``/PU``#\-0``#38``!XV
-M```O-@``0#8``!0W``#,-@``S#8``-TV``#N-@``_S8``!`W````1T-#.B`H
+M`````````````````````````````0``````````````````````````````
+M`````````0`!``$``@`!``,``0`$``$`!0`!``8``0`'``$````!```@`0``
+M0`$``&`!``"``0``H`$``,`!``#@`0#`````N#H``'`Z``!P.@``@3H``)(Z
+M``"C.@``M#H``(@[``!`.P``0#L``%$[``!B.P``<SL``(0[````1T-#.B`H
 M1TY5*2`S+C0N,B!;1G)E94)31%T@,C`P-#`W,C@``"YS>6UT86(`+G-T<G1A
 M8@`N<VAS=')T86(`+G)E;"YT97AT`"YR96PN9&%T80`N8G-S`"YR96PN<F]D
 M871A`"YC;VUM96YT````````````````````````````````````````````
-M`````````````!\````!````!@`````````T````$[T`````````````!```
-M```````;````"0``````````````--L``*@1```*`````0````0````(````
-M*0````$````#`````````&"]``#````````````````@`````````"4````)
-M``````````````#<[```@`````H````#````!`````@````O````"`````,`
-M````````(+X```@```````````````0`````````.`````$````"````````
-M`""^```X```````````````$`````````#0````)``````````````!<[0``
-M<`````H````&````!`````@```!``````0``````````````6+X``"4`````
-M``````````$`````````$0````,``````````````'V^``!)````````````
-M```!``````````$````"``````````````"HP```$`T```L```!A````!```
-M`!`````)`````P``````````````N,T``'P-``````````````$`````````
+M`````````````!\````!````!@`````````T````(,8`````````````!```
+M```````;````"0``````````````P.0``"`2```*`````0````0````(````
+M*0````$````#`````````&#&``#D```````````````@`````````"4````)
+M``````````````#@]@``B`````H````#````!`````@````O````"`````,`
+M````````1,<```@```````````````0`````````.`````$````"````````
+M`$3'```X```````````````$`````````#0````)``````````````!H]P``
+M<`````H````&````!`````@```!``````0``````````````?,<``"4`````
+M``````````$`````````$0````,``````````````*''``!)````````````
+M```!``````````$````"``````````````#,R0``8`T```L```!A````!```
+M`!`````)`````P``````````````+-<``),-``````````````$`````````
 M``````````````````````$```````````````0`\?\````````````````#
-M``$``````````````````P`#``````````````````,`!0`(````<`H``'\`
-M```"``$`&0```/`*``#V`````@`!`"P```#\#0``9`````(``0`Y````G!,`
-M`*H!```"``$`1@```)P9```U`````@`!`%4```#4&0``LP````(``0!F````
-M'!P``",````"``$`<@```$`<```@`````@`!`'T`````````!`````$`!0"*
-M````8!P``(`!```"``$`F@```.`=``!I`````@`!`*,```!,'@``0@````(`
-M`0"K````D!X``#,````"``$`LP```,0>```^`````@`!`+X````$'P``%```
-M``(``0#&````&!\``#$````"``$`V````$P?```H`````@`!`.<```!T'P``
-M4P````(``0#R````R!\``#$````"``$`_P```/P?``!5`````@`!``L!``!4
-M(```$P(```(``0`8`0``:"(``+L````"``$`)0$``"0C``"2`@```@`!`"\!
-M``"X)0```0$```(``0`^`0``%&8``,\````"``$`20$``+PF```\!````@`!
-M`%4!``!X-P``<RL```(``0!C`0``@&0``$,!```"``$`;P$``/@J```N````
-M`@`!`'D!```4-0``80(```(``0"+`0``*"L``*X````"``$`G`$``-@K```C
-M`0```@`!`*D!``#\+```I@````(``0"V`0``T"T``'8````"``$`PP$``*0M
-M```I`````@`!`,X!``!(+@``?`$```(``0#9`0``Q"\``!H"```"``$`Y0$`
-M`$`T``!L`````@`!`/,!``#@,0``U`````(``0```@``M#(``(P!```"``$`
-M#0(``*PT``!E`````@`!``````````````````,`!@`>`@``Y&8```H#```"
-M``$`+@(``/!I``!G`````@`!`$4"``!8:@``-P4```(``0!0`@``D&\``)P#
-M```"``$`7`(``"QS```[`````@`!`&P"``#$>```:P(```(``0!Y`@``D(``
-M`"P````"``$`B@(``#B!``!%`````@`!`)\"``"`@0``)0````(``0"R`@``
-MJ($``$(````"``$`OP(``.R!``"1`@```@`!`-4"``"`A```7@$```(``0#L
-M`@``X(4``)0````"``$`!`,``/B7``!)`````@`!`!(#``!TA@``7`$```(`
-M`0`E`P``W)D``$X````"``$`-`,``.2:```:`@```@`!`$@#``#TC0``6`$`
-M``(``0!;`P``T(<``)$````"``$`:@,``&2(``#$`@```@`!`'X#``#XD@``
-M?P````(``0"0`P``P)$``#4!```"``$`G@,``)":``!1`````@`!`*L#```H
-MBP``=0````(``0#!`P``^)@``"\````"``$`V`,``*"+``"0`0```@`!`.L#
-M```PC0``P0````(``0#[`P``*)D``+$````"``$`"`0``"R:``!B`````@`!
-M`!L$``!,CP``L0$```(``0`L!````)$``+\````"``$`.@0``'B3```.`@``
-M`@`!`$L$``"(E0``'0````(``0!D!```J)4``#$!```"``$`>P0``-R6``!"
-M`````@`!`)`$```@EP``:@````(``0"D!```C)<``&L````"``$`KP0``$28
-M```L`````@`!`,$$``!PF```B`````(``0#7!```>*X``.0````"``$`[P0`
-M`*"T``#*`````@`!`/P$``"(````'`````$``P`&!0``I````!P````!``,`
-M$`4``*BU``"C`````@`!`!D%```$````!`````$`!0`A!0``3+8``$0````"
-M``$`*04``)"V``!6`````@`!`#0%```8N0``N@````(``0`\!0``W+H``.`!
-M```"``$``````````````````P`(`%0%````````6`4``!(``0!@!0``(```
+M``$``````````````````P`#``````````````````,`!0`(````?`P``*X`
+M```"``$`&0```"P-```>`0```@`!`"P```!D$```9`````(``0`Y````>!8`
+M`,X!```"``$`1@```.`<``!$`````@`!`%4````D'0``V@````(``0!F````
+ME!\``",````"``$`<@```+@?```@`````@`!`'T`````````!`````$`!0"*
+M````V!\``(`!```"``$`F@```%@A``!I`````@`!`*,```#$(0``0@````(`
+M`0"K````""(``#,````"``$`LP```#PB```^`````@`!`+X```!\(@``%```
+M``(``0#&````D"(``#$````"``$`V````,0B```H`````@`!`.<```#L(@``
+M<P````(``0#R````8",``#$````"``$`_P```)0C``!5`````@`!``L!``#L
+M(P``'P(```(``0`8`0``#"8``/,````"``$`)0$````G``"2`@```@`!`"\!
+M``"4*0```0$```(``0`^`0``T&H``,\````"``$`20$``)@J``#(!````@`!
+M`%4!``#L.P``NRL```(``0!C`0``/&D``$,!```"``$`;P$``&`O```N````
+M`@`!`'D!``"(.0``80(```(``0"+`0``D"\``*X````"``$`G`$``$`P```C
+M`0```@`!`*D!``!D,0``I@````(``0"V`0``.#(``($````"``$`PP$```PR
+M```I`````@`!`,X!``"\,@``?`$```(``0#9`0``.#0``!H"```"``$`Y0$`
+M`+0X``!L`````@`!`/,!``!4-@``U`````(``0```@``*#<``(P!```"``$`
+M#0(``"`Y``!E`````@`!``````````````````,`!@`>`@``H&L``((#```"
+M``$`+@(``"1O``!G`````@`!`$4"``",;P``FP4```(``0!0`@``*'4``/T#
+M```"``$`7`(``"AY```[`````@`!`&P"``#4@```9P(```(``0!Y`@``L(@`
+M`"P````"``$`B@(``%B)``!%`````@`!`)\"``"@B0``)0````(``0"R`@``
+MR(D``)$"```"``$`R`(``%R,``!2`0```@`!`-\"``"PC0``E`````(``0#W
+M`@```*```$D````"``$`!0,``$2.``!<`0```@`!`!@#```LH@``HP````(`
+M`0`G`P``B*,``"8"```"``$`.P,``-B5``!8`0```@`!`$X#``"@CP``D0``
+M``(``0!=`P``-)```,`"```"``$`<0,```";``!_`````@`!`(,#``#$F0``
+M.0$```(``0"1`P``-*,``%$````"``$`G@,``/22``!U`````@`!`+0#```D
+MH0``4P````(``0#+`P``;),``*@!```"``$`W@,``!25``#!`````@`!`.X#
+M``!XH0``L0````(``0#[`P``T*(``&(````"``$`#@0``#"7``#1`0```@`!
+M`!\$```$F0``OP````(``0`M!```@)L```X"```"``$`/@0``)"=```=````
+M`@`!`%<$``"PG0``,0$```(``0!N!```Y)X``$(````"``$`@P0``"B?``!J
+M`````@`!`)<$``"4GP``:P````(``0"B!```3*```"P````"``$`M`0``'B@
+M``"L`````@`!`,H$``#(N```<@,```(``0#@!```/+P``,H````"``$`[00`
+M`*`````@`````0`#`/<$``#`````(`````$``P`!!0``X`````0````!``,`
+M"04```0````$`````0`%`!@%``!$O0``HP````(``0`A!0``Z+T``$0````"
+M``$`*04``"R^``!6`````@`!`#0%``#$P```N@````(``0`\!0``G,(``.`!
+M```"``$``````````````````P`(`%0%````````,`<``!(``0!@!0``(```
 M`"0````1``,`;P4`````````````$````'P%``!@````)````!$``P",!0``
-M-'@``"H````2``$`E04`````````````$````*,%`````````````!````"M
-M!0`````````````0````MP4``%@%``#+````$@`!`,$%`````````````!``
-M``#.!0`````````````0````X`4``"0&``"<`0``$@`!`/`%````````````
-M`!````#Y!0``8'@``&,````2``$`!P8``,`'``!6````$@`!`!H&``#@>P``
-M$@```!(``0`H!@``&`@``$`!```2``$`-08``+A[```H````$@`!`$<&``"L
-M=0``8P```!(``0!2!@``6`D``!4!```2``$`908``.@+``!@````$@`!`'8&
-M``"X?@``$P```!(``0""!@``.'T``#D````2``$`D@8``$@,`````0``$@`!
-M`*0&``"$?@``,0```!(``0"T!@``2`T``$(````2``$`QP8``(P-``!O````
-M$@`!`-D&``!@#@``9`$``!(``0#O!@``Q`\``(`````2``$`#`<``$00``"4
-M````$@`!`"`'``#T?@``3@```!(``0`N!P``V!```+<````2``$`10<``)`1
-M```_`0``$@`!`%T'``#0$@``RP```!(``0!S!P`````````````0````?0<`
-M`$@5``"P`0``$@`!`(\'``#,?@``)P```!(``0"@!P``^!8``"L!```2``$`
-MLP<``"08``!V`0``$@`!`,0'``"(&@``8````!(``0#4!P``Z!H````!```2
-M``$`Y0<``.@;```R````$@`!`/<'````````!````!$``P`'"```````````
-M```0````%0@``.QB```P`0``$@`!`"@(`````````````!`````["```````
-M```````0````0@@`````````````$````$D(```<9```80```!(``0!5"```
-M```````````0````=P@```Q^``!X````$@`!`(4(``#$90``4````!(``0"7
-M"```:',``$0"```2``$`J0@```QW```H`0``$@`!`+P(```0=@``_````!(`
-M`0#."``````````````0````X`@`````````````$````/,(```P>P``/@``
-M`!(``0`""0``<'L``$<````2``$`$0D``/1[``!#`0``$@`!`"`)``!T?0``
-M6````!(``0`Q"0``S'T``#\````2``$`/@D``$1_```>````$@`!`$D)``!D
-M?P``-````!(``0!9"0``F'\``/8````2``$`90D``*"D``!#````$@`!`'T)
-M``#DI```2````!(``0"1"0``N+```#T````2``$`K`D``/BP``".````$@`!
-M`-$)``!4I```3````!(``0#H"0``O(```'H````2``$`_PD`````````````
-M$````!,*`````````````!`````D"@`````````````0````-PH``(0````$
-M````$0`#`$H*`````````````!````!<"@`````````````0````;@H`````
-M````````$````(`*`````````````!````"1"@``;+4``#L````2``$`H`H`
-M`*"M``!>````$@`!`+H*````G0``Y`(``!(``0#,"@``G*```)@````2``$`
-MZPH``.2?```?````$@`!``$+```$H```#P```!(``0`3"P``%*````\````2
-M``$`)@L``"2@``!.````$@`!`#T+``!TH```)@```!(``0!1"P``-*$``&0"
-M```2``$`:`L``)BC``"Z````$@`!`'T+```LI0``(````!(``0"4"P``3*4`
-M`'T````2``$`M0L``,RE``#,````$@`!`,\+``"8I@``C````!(``0#H"P``
-M)*<``!$!```2``$```P``#BH``!O`0``$@`!`!<,``"HJ0``4@```!(``0`N
-M#```_*D``+D!```2``$`00P``+BK``"G`0``$@`!`%\,``!@K0``'P```!(`
-M`0!Z#```@*T``"`````2``$`EPP```"N```\````$@`!`+$,```\K@``/```
-M`!(``0#,#```7*\``)`````2``$`Z0P``.RO``"5````$@`!``<-``"$L```
-M,P```!(``0`F#0``B+$``!8#```2``$`/`T``.BV```7`0``$@`!`$,-````
-MN```%P$``!(``0!+#0``U+D``,D````2``$`60T``*"Z```Z````$@`!`&8-
-M``"\O```5P```!(``0``<F%I9"YC`%-T<FEP94=E=$-O;6UA;F0`4W1R:7!E
-M4W!L:71#;VUM86YD`$ES5D1E=D]N;&EN90!$;T-O;G1R;VQ#;60`2D)/1$=E
-M=$-O;6UA;F0`2D)/1%-P;&ET0V]M;6%N9`!?7VAP=%]A;&QO8P!?7VAP=%]F
-M<F5E`&=A<F)A9V5?861D<@!I;FET7VAE87!?8FQO8VL`8V)?86QL;V,`8V)?
-M9G)E90!C8E]Z97)O`&-B7VES7WIE<F\`8V9?:6YI=`!R96UO=F5?<F%N9V5?
-M;&]C:P!A9&1?<F%N9V5?;&]C:P!L;V-K7W)A;F=E`%]?86QL;V-?=&%S:P!F
-M<F5E7W-T<FEP90!A;&QO8U]S=')I<&4`7U]G971?<W1R:7!E`&1A=&%?>&9E
-M<@!R96%D7WAF97)?9&]N90!R86ED-5]D;VYE`'-T<FEP95]D;VYE`&AA;F1L
-M95]S=')I<&4`<F%I9#5?9&ER='D`=&%S:U]D;VYE`'-T<FEP95]C:&5C:U]T
-M87-K`')W7W1A<VM?8G5I;&1S9VP`<G=?9V5T7W)A;F=E`')W7W1A<VM?9&]N
-M90!?7V1O7W)W7W1A<VL`9&]?<G=?=&%S:P!G971?<VQI8V5S`&1O7WAO<E]T
-M87-K`'AO<E]T87-K7V1O;F4`9&]?>&]R,E]T87-K`&1O7WAO<FY?=&%S:P!C
-M:&5C:U]D97!E;F1E;F-Y`&1I<F5C=%]R96%D7W-G;`!D:7)E8W1?<F5A9%]C
-M;VUP;&5T:6]N`&AA;F1L95]C;60`9V5T7W-T<FEP97,`=W)I=&5?>&9E<E]D
-M;VYE`%-Y;F-$:7-K26YF;P!G971%9&UA4F5G3V9F<V5T`'5N;6%S:T5D;6%)
-M;G1E<G)U<'1S`&UA<VM%9&UA26YT97)R=7!T<P!I<T5X=$-O;6UA;F0`=W)I
-M=&5%9&UA4F5Q=65S=$5N=')Y`&1U;7!!=&%$979I8V5296=I<W1E<G,`:&%N
-M9&QE161M849A:6QE9$-O;6UA;F0`<F5M;W9E0V]M;6%N9`!H86YD;&5%9&UA
-M4F5S<&]N<V4`9&5A8W1I=F%T945D;6$`<V5N9$YO;F559&UA0V]M;6%N9`!C
-M;VUP;&5T95!)3T-O;6UA;F0`4V5T1U!)3T-T<FQ296<`:&%N9&QE161M84EN
-M=&5R<G5P=`!?8VAA;FYE;$AA<F1297-E=`!?9FEX4&AY4&%R86US`%]D;T%U
-M=&]&;'5S:`!H86YD;&5$979I8V5);G1E<G)U<'0`9&ES86)L95-A1&5V26YT
-M97)R=71P<P!H86YD;&5024]);G1E<G)U<'0`=')A;G-F97)024]$871A`&%C
-M=&EV871E161M80!%9&UA4F5Q475E=65);G-E<G0`<F5S971%9&UA0VAA;FYE
-M;`!F;'5S:$1M85%U975E`')E=F5R=%-A=&%(0U)E9W,`<F5V97)T1FQA<VA)
-M;G1E<F9A8V5296=S`')E=F5R=%!#24EN=&5R9F%C95)E9W,`8V]M;6%N9'-1
-M=65U94%D9%1A:6P`8V]M;6%N9'-1=65U95)E;6]V90!A9&1#;VUM86YD`%-A
-M1&5V26YT97)R=71P0FET`&5N86)L95-A1&5V26YT97)R=71P<P!I<U-T;W)A
-M9V5$979296%D>49O<E!)3P!?9&]3;V9T4F5S970`7V%D9')-87`Q`%]A9&1R
-M36%P,@!%;G1E<C8P,0!A9&1R36%P`$5X:70V,#$`9V5T7S8P,5]I9`!S971?
-M8W1L`%]?8VAE8VM?<')O=&5C=%]C:7)C=6ET`&9#:&5C:T%R<F%Y`'!F;E-E
-M;F1#;VUM86YD`&9$95)E8617<FET90!P9FY$979I8V5&86EL960`0VAE8VM3
-M=6T`9D1E4V5L96-T36]D90!O<U]M96US970`;W-?;65M8W!Y`&9!9&13<&%R
-M90!'9713<&%R941I<VL`:6]C=&Q?4F5P;W)T179E;G0`1FEX=7!!<G)A>5-T
-M871E`$=E=%-T86UP`%-Y;F-!<G)A>4EN9F\`0VAE8VM!<G)A>4-R:71I8V%L
-M`&9/<T1I<VM&86EL960`9D1E;&5T94%R<F%Y`%5N<F5G:7-T97)61&5V:6-E
-M`')A:60U7V9R964`4W1R:7!E0G5I;&139U1A8FQE`&9204E$,$UE;6)E<D1O
-M;F4`1G)E94-O;6UA;F0`0V%L;$%F=&5R4F5T=7)N`&9204E$,%-E;F1#;VUM
-M86YD`$%L;&]C871E0V]M;6%N9`!F4D%)1#!-96UB97)&86EL960`4D%)1#%"
-M=6EL9%-G5&%B;&4`4D%)1#%-96UB97)#;VUP;&5T:6]N`%)!240Q26YI=%)E
-M8G5I;&1"=6EL9%-G5&%B;&4`4D%)1#%);FET0V]M<&QE=&EO;@!$;U=A:71I
-M;F=,:7-T`%)!240Q4F5B=6EL9$-O;7!L971I;VX`4D%)1#%697)I9GE"=6EL
-M9%-G5&%B;&4`4D%)1#%697)I9GE#;VUP;&5T:6]N`&]S7VUE;6-M<`!F4D%)
-M1#%396YD0V]M;6%N9`!!9&14;U=A:71I;F=,:7-T`&9204E$,4UE;6)E<D9A
-M:6QE9`!*0D]$0G5I;&139U1A8FQE`&9*0D]$365M8F5R1&]N90!F2D)/1%-E
-M;F1#;VUM86YD`&9*0D]$365M8F5R1F%I;&5D`&YU;5]R86ED-5]P86=E<P!O
-M<U]A;&QO8U]P86=E`&9L=7-H7W-T<FEP95]C86-H90!F3W-0:'ES:6-A;$%D
-M9')E<W,`1&]8;W(R`$1O6&]R,0!F;'5S:%]R86ED-0!F0V]M<&QE=&5!;&Q#
-M;VUM86YD<U-Y;F-H<F]N;W5S;'D`0VAE8VM)9&QE0V%L;`!F;'5S:%]R86ED
-M-5]A<WEN8P!F4D%)1#5396YD0V]M;6%N9`!F4D%)1#5-96UB97)&86EL960`
-M:6YI=%]R86ED-5]M96UO<GD`;W-?86QL;V-?9&UA7W!A9V4`9D1E=FEC95-E
-M;F1#;VUM86YD`&9'971&:7)S=$-H:6QD`&9297-E=$)O;W1-87)K`&9#:&5C
-M:T)O;W1A8FQE`$-H96-K4&5N9&EN9T-A;&P`0V%L;%=H96Y)9&QE`&9&;'5S
-M:%9$978`9D9L=7-H5D1E=D%S>6YC`&UV4F5A9%=R:71E`&UV4V%T841I<V%B
-M;&5#:&%N;F5L1&UA`&UV4V%T849L=7-H1&UA475E=64`;793=&]R86=E1&5V
-M051!4V5T1F5A='5R97,`;793=&]R86=E1&5V051!17AE8W5T94YO;E5$34%#
-M;VUM86YD`&UV4V%T845N86)L94-H86YN96Q$;6$`=V%I=$9O<D)U<WE!9G1E
-M<DA297-E=`!M=DUI8W)O4V5C;VYD<T1E;&%Y`$U67U)%1U]214%$7T)95$4`
-M359?4D5'7U=2251%7T173U)$`&UV075T;T9L=7-H3VY%<G)O<@!-5E]214=?
-M5U))5$5?0EE410!-5E]214=?4D5!1%]$5T]21`!-5E]214=?5U))5$5?5T]2
-M1`!-5E]214=?4D5!1%]73U)$`%]D;TUV4V]F=%)E<V5T`'=A:717:&EL95-T
-M;W)A9V5$979)<T)U<WD`;793871A26YI=$%D87!T97(`;793871A27-3=&]R
-M86=E1&5V:6-E0V]N;F5C=&5D`&UV4V%T85-H=71D;W=N061A<'1E<@!M=D5N
-M86)L94%U=&]&;'5S:`!M=D1I<V%B;&5!=71O1FQU<V@`;793871A0V]N9FEG
-M=7)E0VAA;FYE;`!M=E-A=&%296UO=F5#:&%N;F5L`&UV4V%T84-H86YN96Q(
-M87)D4F5S970`;793871A0V]N9FEG161M84UO9&4`;793871A3G5M3V9$;6%#
-M;VUM86YD<P!M=E-A=&%#:&%N;F5L4V5T161M84QO;W!"86-K36]D90!M=E-A
-M=&%3971#:&%N;F5L4&AY4&%R86US`&UV4V%T84-H86YN96Q0:'E3:'5T9&]W
-M;@!M=E-A=&%#:&%N;F5L4&AY4&]W97)/;@!M=E-A=&%'971#:&%N;F5L4W1A
-M='5S`&UV4V%T85%U975E541M84-O;6UA;F0`;793871A475E=65#;VUM86YD
-M`&UV4V%T84EN=&5R<G5P=%-E<G9I8V52;W5T:6YE`&UV4V%T84UA<VM!9&%P
-M=&5R26YT97)R=7!T`&UV4V%T855N;6%S:T%D87!T97));G1E<G)U<'0`96YA
-M8FQE4W1O<F%G941E=DEN=&5R<G5P=`!D:7-A8FQE4W1O<F%G941E=DEN=&5R
-M<G5P=`!M=E-T;W)A9V5$979!5$%)9&QE26UM961I871E`&UV4W1O<F%G941E
-M=D%404ED96YT:69Y1&5V:6-E`&UV4W1O<F%G941E=D%405-O9G1297-E=$1E
-M=FEC90!E>&5C=71E3F]N541-04-O;6UA;F0`0F5E<$]N`$)E97!/9F8`<V5T
-M7V9A:6Q?;&5D<P!S971?9F%I;%]L960`8VAE8VM?<')O=&5C=%]C:7)C=6ET
-M`$P````!8@``<`````)C``"*`````60``*H````"90``V0````)F``#P````
-M`F4``!(!```"9P``E0$```%D``#"`0```6(``%0"```"9P``UP(```%B``#R
-M`@```F@```T#```":```*`,```)H``!#`P```F@``+X#```"9P``0P0```%B
-M``!0!````60``'X$```":```F00```)H``"T!````F@``,\$```":```"@4`
-M``%D``!(!0```60``)H%```":@``Q04```%B``#2!0```60```X&```":P``
-MD08```)L``#S!@```FD``'L'```":0``I0<```)M``"Q!P```FX``/,'```"
-M;````P@```%P```R"````G(``$8(```"<P``AP@```)Q``"Q"````7```-X(
-M```"9P``\@@```)C```1"0```FX``#`)```"9P``&0P```)V```Z#````G<`
-M`&X,```"=P``F@P```)W``#+#````GD``!(-```!=0``&0T```%T``#+#@``
-M`F<``!T/```"9P``7@\```)V``"3#P```G<``'D0```"=@``K1````)W``##
-M$````7\``,D0```"=P``%1$```)G``!:$0```G8``&<1```"=P``?1$```%_
-M``"#$0```G<```\3```"9P``-Q,```*#``!F$P```G8``',3```"=P``B1,`
-M``%_``"/$P```G<```@4```"9P``(10```)Y``!#%````7X``$H4```!?0``
-M@Q0```)Y``"E%````8```*P4```!?0``UQ0```)Y``#Y%````8(````5```!
-M@0``)14```)W```T%0```7\``#H5```"=P``:Q4```)W``!]%0```0(``,\5
-M```!A```W!4```*%``">%@```GD``*T6```":```R!8```%\``#/%@```7L`
-M`)47```":@``K1<```%B``"U%P```60``.P7```":P``#!@```)M```8&```
-M`FX``(D9```":```N1H```)V``#:&@```G<```X;```"=P``.AL```)W``!K
-M&P```GD``+(;```!B```N1L```&'```7'0```HP``*\>```"9P``$A\```)G
-M```()0```F@``"DE```":```%RD```$"```#*@```0(```LJ```"=P``1BH`
-M``%_``!.*@```G<``&\J```!?P``=2H```)W``"]*@```G<``.@J```"C0``
-M$BL```$"```:*P```G<``)LK```"C@``&2T```)V``#M+0```GD``"8N```!
-M`@``,"X```$"``#4,````H\````Q```"D```MC$```*/``",,@```H\``-LS
-M```"CP``"30```*0``#X-0```2X``,@V```!+@``MF,```)W``!%9````HT`
-M`$UD```"D@``560```*3```(9@```HT``%EF```"=P``<&8```%_``!V9@``
-M`G<``*)F```!?P``J&8```)W``#39@```HT``.MG```!!```]F<```$$``#[
-M9P```HX``!!J```"=@``H6P```)G``#!;@```7\``,=N```"=P``^&X```)Y
-M```>;P```0(``"5O```!`@``1W,```$"``!-<P```G<``(MS```"=P``LG,`
-M``)G``#:<P```G<``/US```"=P``D70```)W``#'=````94``-1T```"A0``
-M)G4```&6``!`=0```94``%)U```"A0``774```*-``!N=0```0(``(]U```"
-M=P``P74```*1``!==@```8L``)-V```!BP``OG8```*8``##=@```00``,UV
-M```!BP``V78```*8``#]=@```8L``*YW```":@``S'<```%B``#4=P```60`
-M``!X```":P``&G@```)M```F>````FX``)QX```";@``#WD```)G```M>@``
-M`F@``$)Z```":```6GH```)H``!O>@```F@``/-Z```"90```WL```)E```C
-M>P```F,``&=[```"F@``EWL```*;``#L>P```G(``#%\```"9P``@'P```)C
-M``"_?````F,``%M^```"D@``JGX```)G``!7?P```I$``(-_```"E```N'\`
-M``*B``#$?P```J,``-A_```"I```<H````*E``!^@````J8``-.````"J```
-M^8````*I```5@0```J@``%N!```"J@``;X$```*J``"?@0```JH``*:$```"
-MJ0``NX0```*I``#6A````JD``/&$```"J0``#(4```*I```TA0```JD``$J%
-M```"K```5H4```*I``!IA0```JD``'R%```"J0``CX4```*I``"GA0```JP`
-M`+R%```"J0``SX4```*I```,A@```JT``.2'```"K0``^H<```*J```*B```
-M`JT``!J(```"K0``-8@```*J``!%B````JT``%:(```"J@``R(@```*J```W
-MB0```JT``$^)```"J@``\HD```*J``#^B0```JT```J*```"J```&(H```*J
-M```DB@```JT``#"*```"J```78H```*I```+BP```:L``-&+```"J0``X8L`
-M``*I```)C````JH``)"-```"K@``PXT```*O``!HC@```:L``.V.```"K0``
-M^8X```*H``!LCP```JT``'^/```"J@``B8\```*M``"5CP```J@``+N/```"
-MJ@``7I````*J``!SD````JH``(:0```"J@``GY````*J``"QD````JH``,20
-M```"J@``UI````*J``#LD````JH``!J2```"K0``4I(```*J``"&D@```JT`
-M`.:2```"J@``)9,```*J```QDP```JT``#V3```"J```2Y,```*J``!7DP``
-M`JT``*V3```"J@``O),```*M``#9DP```JH``.V3```"J@```90```*J```5
-ME````JH``"F4```"J@``/90```*J``!1E````JH``&64```"J@``>90```*J
-M``"-E````JH``*&4```"J@``M90```*J``#,E````JH```*5```"J@``%I4`
-M``*J```JE0```JH``#Z5```"J@``4)4```*M``!JE0```JH``)^5```"J@``
-MRI4```*M``#>E0```JH``/&5```"K0``!Y8```*J```9E@```JH``"N6```"
-MJ@``0)8```*J``!2E@```JH``&26```"J@``=I8```*J``"(E@```JH``)J6
-M```"J@``K)8```*J``"^E@```JH``-"6```"J@``L9<```)H``#*EP```F@`
-M`)J8```"J0``RI@```*J``#=F````JH``.V8```"K0``'ID```*J``!]F0``
-M`JH``)&9```"J@``LID```*I``#-F0```JH```B:```"J@``$IH```*M``"#
-MF@```JH``+*:```"L```")L```*I```YFP```JP``$Z;```"K```8YL```*L
-M``!XFP```JP``(V;```"K```VYL```*L``#PFP```JP```6<```"K```&IP`
-M``*L```OG````JP``$2<```"K```69P```*L``!UG````JD``("<```"L0``
-MG)P```*I``#3G````JX``!Z=```"J@``WYT```*M```TG@```JH``$2>```"
-MK0``6IX```*J``!JG@```JT``-F>```"J@``[)X```*J``#_G@```JH``":?
-M```"J@``2)\```*J``!_GP```J@``*B?```"LP``^)\```*J```)H````:L`
-M`!F@```!JP``TJ````*M``#XH````JT```2A```"J```$*$```*M``"JH0``
-M`JH``+:A```"K0``PJ$```*H``#0H0```JH``-RA```"K0``,:(```*M``!A
-MH@```JH``)*B```"K0``GJ(```*H``#2H@```JH```*C```"K0``#J,```*H
-M```<HP```J<``%6C```"J0``W*,```*M```\I````JH``'2E```"K0``E:4`
-M``*J``"TI0```JH``#ZF```"K0``;Z8```*J``#(I@```JT``/6F```"J@``
-M$*<```*M``!IIP```JT``)6G```"J@``L*<```*M``#>IP```JT``!*H```"
-MJ@``@J@```*M``"PJ````K,```"I```"K0``%:D```*M```JJ0```JT``#^I
-M```"K0``5*D```*M``!IJ0```JT``'ZI```"K0``DZD```*M``#TJ0```L(`
-M`-&K```"K0``!*P```*M```9K````JH``%RL```"J@``:*P```*M``!\K```
-M`JH``)BL```"K0``=*T```*J``"5K0```JH``,BM```"J0``Y:T```*H```<
-MK@```JP``#.N```"J0``6*X```*L``!OK@```JD``+6N```"J0``QJX```*H
-M``#>K@```JD``/VN```"LP``$:\```*H```JKP```K$``#RO```"J0``E:\`
-M``+'``"JKP```JP``+BO```"L0``R*\```+&``#9KP```L8``"ZP```"I0``
-ML;````*P``#OL````J4``':Q```"RP``[+$```+'```,L@```JP``".R```"
-MK```.K(```*L``!1L@```JP``&BR```"K```O[(```*L``#4L@```JP``.FR
-M```"K```_K(```*L```3LP```JP``"BS```"K```/;,```*L``!4LP```K$`
-M`&2S```"Q@``=[,```+&``"/LP```K$``)^S```"Q@``N;,```*I``#)LP``
-M`L8``/^S```"L0``&+0```*O```^M````JX``%JT```"L0``:K0```+&``!Z
-MM````L8``(JT```"Q@``P+0```*L``#2M````JD``-ZT```"J```YK0```+&
-M``#RM````J@``/ZT```"J```"K4```*H```6M0```J@``#"U```"J0``3;4`
-M``*H``"2M0```J@``+FU```!!```R+4```*L``#8M0```00``.>U```"J0``
-M`+8```$$```/M@```JD``"BV```!!```-[8```*I``!<M@```00``&NV```"
-MK```>+8```$$``"'M@```JP``*>V```!!```MK8```*L``"^M@```00``,RV
-M```"J0``^[8```*M```/MP```JH``!VW```"K0``0;<```*N``!2MP```JT`
-M`&2W```!!```<[<```*L``!]MP```00``(NW```"K```E;<```$$``"DMP``
-M`JP``*ZW```!!```O+<```*L``#3MP```JH``.&W```"K0``\+<```*J```3
-MN````JT``">X```"J@``-;@```*M``!9N````JX``&JX```"K0``?+@```$$
-M``"+N````JP``)6X```!!```H[@```*L``"MN````00``+RX```"K```QK@`
-M``$$``#4N````JP``.NX```"J@``^;@```*M```(N0```JH``"ZY```!!```
-M/;D```*L``!'N0```00``%6Y```"K```:[D```*I``"*N0```00``)BY```"
-MK```K+D```*H``"ZN0```JD``/"Y```"K0``!;H```*J```3N@```JT``#>Z
-M```"K@``2+H```*M``!@N@```JP``&^Z```"J@``?;H```*M``".N@```JH`
-M`-2Z```"S@``\[H```*M```(NP```JH``!:[```"K0``/+L```*N``!-NP``
-M`JT``%N[```"KP``B;L```$$``"8NP```JP``+*[```"J```QKL```*O``#5
-MNP```00``.2[```"K```#[P```$$```>O````JP``"R\```"KP``:KP```$$
-M``!YO````JP``(J\```"J@``F+P```*M``"IO````JH``,Z\```!!```TKP`
-M``$#``#HO````00``.R\```!`P``(`````&9```D`````9D``"@````!F0``
-M+`````&9```P`````8D``#0````!>```.`````&$``!``````94``&`````!
-M<```9`````%P``!H`````7```&P````!<```<`````&*``!T`````7H``'@`
-M```!A@``@`````&6`````````0(```0````!`@``"`````$"```,`````0(`
-M`!`````!`@``%`````$"```8`````0(``!P````!`@``(`````$"```D````
-D`0(``"@````!`@``+`````$"```P`````0(``#0````!`@``
+M1(```"H````2``$`E04`````````````$````*,%`````````````!````"M
+M!0`````````````0````MP4`````````````$````,$%`````````````!``
+M``#2!0`````````````0````X@4`````````````$````.P%```P!P``VP``
+M`!(``0#V!0`````````````0`````P8`````````````$````!4&```,"```
+MO`$``!(``0`E!@`````````````0````+@8``'"```!C````$@`!`#P&``#(
+M"0``6@```!(``0!/!@``[(,``!(````2``$`708``"0*``!``0``$@`!`&H&
+M``#$@P``*````!(``0!\!@``L'T``&,````2``$`AP8``&0+```5`0``$@`!
+M`)H&`````````````!````"D!@`````````````0````K@8``$P.``!@````
+M$@`!`+\&``#$A@``$P```!(``0#+!@``1(4``#D````2``$`VP8``*P.```$
+M`0``$@`!`.T&``"0A@``,0```!(``0#]!@``L`\``$(````2``$`$`<``/0/
+M``!O````$@`!`"('``#($```9`$``!(``0`X!P``+!(``(`````2``$`50<`
+M`*P2``"\````$@`!`&D'````AP``3@```!(``0!W!P``:!,``-D````2``$`
+MC@<``$04```_`0``$@`!`*8'``"$%0``\P```!(``0"\!P`````````````0
+M````Q@<``$@8``#,`0``$@`!`-@'``#8A@``)P```!(``0#I!P``%!H``%$!
+M```2``$`_`<``&@;``!V`0``$@`!``T(````'@``8````!(``0`="```8!X`
+M```!```2``$`+@@``&`?```R````$@`!`$`(````````!````!$``P!0"```
+M```````````0````7@@``*AG```P`0``$@`!`'$(`````````````!````"$
+M"``````````````0````BP@`````````````$````)((``#8:```80```!(`
+M`0">"``````````````0````P`@``!B&``!X````$@`!`,X(``"`:@``4```
+M`!(``0#@"```9'D``$P$```2``$`\@@``!!_```T`0``$@`!``4)```4?@``
+M_````!(``0`7"0`````````````0````*0D`````````````$````#P)```\
+M@P``/@```!(``0!+"0``?(,``$<````2``$`6@D```"$``!#`0``$@`!`&D)
+M``"`A0``6````!(``0!Z"0``V(4``#\````2``$`APD``%"'```>````$@`!
+M`)()``!PAP``-````!(``0"B"0``I(<```L!```2``$`K@D``$"M``!#````
+M$@`!`,8)``"$K0``2````!(``0#:"0``^+<``#T````2``$`]0D``#BX``".
+M````$@`!`!H*``#TK```3````!(``0`Q"@``W(@``'H````2``$`2`H`````
+M````````$````%P*`````````````!````!M"@`````````````0````@`H`
+M`(0````$````$0`#`),*`````````````!````"E"@`````````````0````
+MMPH``(2I``"8````$@`!`-8*`````````````!````#H"@`````````````0
+M````^0H```B]```[````$@`!``@+``!4M0``C0```!(``0`="P``L*4``!P#
+M```2``$`+PL``'S$``!7````$@`!`$4+``#,J```'P```!(``0!;"P``[*@`
+M``\````2``$`;0L``/RH```/````$@`!`(`+```,J0``3@```!(``0"7"P``
+M7*D``"8````2``$`JPL``!RJ```T`@``$@`!`,(+``!0K```H@```!(``0#7
+M"P``S*T``"`````2``$`[@L``.RM``#0````$@`!``@,``"\K@``C````!(`
+M`0`A#```2*\``!$!```2``$`.0P``%RP``!O`0``$@`!`%`,``#,L0``>0$`
+M`!(``0!C#```2+,``,L!```2``$`@0P``!2U```?````$@`!`)P,```TM0``
+M(````!(``0"Y#```Y+4``#P````2``$`TPP``""V```\````$@`!`.X,``!<
+MM@``E````!(``0`+#0``\+8``-0````2``$`*0T``,2W```S````$@`!`$@-
+M``"$O@``*`$``!(``0!/#0``K+\``!<!```2``$`5PT``(#!``#5````$@`!
+M`&4-``!8P@``1````!(``0!R#0``U,0``"L````2``$`?PT```#%```@`0``
+M$@`!``!R86ED+F,`4W1R:7!E1V5T0V]M;6%N9`!3=')I<&53<&QI=$-O;6UA
+M;F0`27-61&5V3VYL:6YE`$1O0V]N=')O;$-M9`!*0D]$1V5T0V]M;6%N9`!*
+M0D]$4W!L:71#;VUM86YD`%]?:'!T7V%L;&]C`%]?:'!T7V9R964`9V%R8F%G
+M95]A9&1R`&EN:71?:&5A<%]B;&]C:P!C8E]A;&QO8P!C8E]F<F5E`&-B7WIE
+M<F\`8V)?:7-?>F5R;P!C9E]I;FET`')E;6]V95]R86YG95]L;V-K`&%D9%]R
+M86YG95]L;V-K`&QO8VM?<F%N9V4`7U]A;&QO8U]T87-K`&9R965?<W1R:7!E
+M`&%L;&]C7W-T<FEP90!?7V=E=%]S=')I<&4`9&%T85]X9F5R`')E861?>&9E
+M<E]D;VYE`')A:60U7V1O;F4`<W1R:7!E7V1O;F4`:&%N9&QE7W-T<FEP90!R
+M86ED-5]D:7)T>0!T87-K7V1O;F4`<W1R:7!E7V-H96-K7W1A<VL`<G=?=&%S
+M:U]B=6EL9'-G;`!R=U]G971?<F%N9V4`<G=?=&%S:U]D;VYE`%]?9&]?<G=?
+M=&%S:P!D;U]R=U]T87-K`&=E=%]S;&EC97,`9&]?>&]R7W1A<VL`>&]R7W1A
+M<VM?9&]N90!D;U]X;W(R7W1A<VL`9&]?>&]R;E]T87-K`&-H96-K7V1E<&5N
+M9&5N8WD`9&ER96-T7W)E861?<V=L`&1I<F5C=%]R96%D7V-O;7!L971I;VX`
+M:&%N9&QE7V-M9`!G971?<W1R:7!E<P!W<FET95]X9F5R7V1O;F4`4WEN8T1I
+M<VM);F9O`&=E=$5D;6%296=/9F9S970`=6YM87-K161M84EN=&5R<G5P=',`
+M;6%S:T5D;6%);G1E<G)U<'1S`'=R:71E161M85)E<75E<W1%;G1R>0!D=6UP
+M071A1&5V:6-E4F5G:7-T97)S`&AA;F1L945D;6%&86EL961#;VUM86YD`')E
+M;6]V94-O;6UA;F0`:&%N9&QE161M85)E<W!O;G-E`&1E86-T:79A=&5%9&UA
+M`'-E;F1.;VYE561M84-O;6UA;F0`8V]M<&QE=&5024]#;VUM86YD`%-E=$=0
+M24]#=')L4F5G`&AA;F1L945D;6%);G1E<G)U<'0`7V-H86YN96Q(87)D4F5S
+M970`7V9I>%!H>5!A<F%M<P!?9&]!=71O1FQU<V@`:&%N9&QE1&5V:6-E26YT
+M97)R=7!T`&1I<V%B;&53841E=DEN=&5R<G5T<',`:&%N9&QE4$E/26YT97)R
+M=7!T`'1R86YS9F5R4$E/1&%T80!A8W1I=F%T945D;6$`161M85)E<5%U975E
+M26YS97)T`')E<V5T161M84-H86YN96P`9FQU<VA$;6%1=65U90!R979E<G13
+M871A2$-296=S`')E=F5R=$9L87-H26YT97)F86-E4F5G<P!R979E<G100TE)
+M;G1E<F9A8V5296=S`&-O;6UA;F1S475E=65!9&1486EL`&-O;6UA;F1S475E
+M=65296UO=F4`861D0V]M;6%N9`!3841E=DEN=&5R<G5T<$)I=`!E;F%B;&53
+M841E=DEN=&5R<G5T<',`97AE8W5T94YO;E5$34%#;VUM86YD`%]D;U-O9G12
+M97-E=`!?861D<DUA<#$`7V%D9')-87`R`&%D9')-87``9&ES86)L95]B965P
+M97(`16YT97(V,#$`17AI=#8P,0!G971?-C`Q7VED`'-E=%]C=&P`7U]C:&5C
+M:U]P<F]T96-T7V-I<F-U:70`9D-H96-K07)R87D`<&9N4V5N9$-O;6UA;F0`
+M9D1E4F5A9%=R:71E`'!F;D1E=FEC949A:6QE9`!#:&5C:U-U;0!F1&5396QE
+M8W1-;V1E`&]S7VUE;7-E=`!F1&5397140U$`9D1E4V5T3D-1`&9$95-E=%=R
+M:71E0V%C:&4`9D1E4V5T4F5A9$%H96%D`&]S7VUE;6-P>0!F061D4W!A<F4`
+M1V5T4W!A<F5$:7-K`&EO8W1L7U)E<&]R=$5V96YT`$9I>'5P07)R87E3=&%T
+M90!'9713=&%M<`!3>6YC07)R87E);F9O`$-H96-K07)R87E#<FET:6-A;`!F
+M3W-$:7-K1F%I;&5D`&9$96QE=&5!<G)A>0!5;G)E9VES=&5R5D1E=FEC90!R
+M86ED-5]F<F5E`%-T<FEP94)U:6QD4V=486)L90!?7W5D:79D:3,`7U]U;6]D
+M9&DS`&9204E$,$UE;6)E<D1O;F4`1G)E94-O;6UA;F0`0V%L;$%F=&5R4F5T
+M=7)N`&9204E$,%-E;F1#;VUM86YD`$%L;&]C871E0V]M;6%N9`!F4D%)1#!-
+M96UB97)&86EL960`4D%)1#%"=6EL9%-G5&%B;&4`4D%)1#%-96UB97)#;VUP
+M;&5T:6]N`%)!240Q26YI=%)E8G5I;&1"=6EL9%-G5&%B;&4`4D%)1#%);FET
+M0V]M<&QE=&EO;@!$;U=A:71I;F=,:7-T`%)!240Q4F5B=6EL9$-O;7!L971I
+M;VX`4D%)1#%697)I9GE"=6EL9%-G5&%B;&4`4D%)1#%697)I9GE#;VUP;&5T
+M:6]N`&]S7VUE;6-M<`!F4D%)1#%396YD0V]M;6%N9`!!9&14;U=A:71I;F=,
+M:7-T`&9204E$,4UE;6)E<D9A:6QE9`!*0D]$0G5I;&139U1A8FQE`&9*0D]$
+M365M8F5R1&]N90!F2D)/1%-E;F1#;VUM86YD`&9*0D]$365M8F5R1F%I;&5D
+M`&YU;5]R86ED-5]P86=E<P!O<U]A;&QO8U]P86=E`&9L=7-H7W-T<FEP95]C
+M86-H90!F3W-0:'ES:6-A;$%D9')E<W,`1&]8;W(R`$1O6&]R,0!F;'5S:%]R
+M86ED-0!F0V]M<&QE=&5!;&Q#;VUM86YD<U-Y;F-H<F]N;W5S;'D`0VAE8VM)
+M9&QE0V%L;`!F;'5S:%]R86ED-5]A<WEN8P!F4D%)1#5396YD0V]M;6%N9`!F
+M4D%)1#5-96UB97)&86EL960`:6YI=%]R86ED-5]M96UO<GD`;W-?86QL;V-?
+M9&UA7W!A9V4`9D1E=FEC95-E;F1#;VUM86YD`&9'971&:7)S=$-H:6QD`&92
+M97-E=$)O;W1-87)K`&9#:&5C:T)O;W1A8FQE`$-H96-K4&5N9&EN9T-A;&P`
+M0V%L;%=H96Y)9&QE`&9&;'5S:%9$978`9D9L=7-H5D1E=D%S>6YC`&UV4F5A
+M9%=R:71E`&UV4V%T841I<V%B;&5#:&%N;F5L1&UA`&UV4V%T849L=7-H1&UA
+M475E=64`;793=&]R86=E1&5V051!4V5T1F5A='5R97,`;793=&]R86=E1&5V
+M051!17AE8W5T94YO;E5$34%#;VUM86YD`&UV4V%T845N86)L94-H86YN96Q$
+M;6$`=V%I=$9O<D)U<WE!9G1E<DA297-E=`!M=DUI8W)O4V5C;VYD<T1E;&%Y
+M`$U67U)%1U]214%$7T)95$4`359?4D5'7U=2251%7T173U)$`&UV075T;T9L
+M=7-H3VY%<G)O<@!-5E]214=?5U))5$5?0EE410!-5E]214=?4D5!1%]$5T]2
+M1`!M=E-A=&%)<U-T;W)A9V5$979I8V5#;VYN96-T960`359?4D5'7U=2251%
+M7U=/4D0`359?4D5'7U)%041?5T]21`!?9&]-=E-O9G1297-E=`!M=E-T;W)A
+M9V5$979786ET4W1A=`!M=E-A=&%);FET061A<'1E<@!C:&5C:U]P<F]T96-T
+M7V-I<F-U:70`;793871A4VAU=&1O=VY!9&%P=&5R`&UV16YA8FQE075T;T9L
+M=7-H`&UV1&ES86)L94%U=&]&;'5S:`!M=E-A=&%#;VYF:6=U<F5#:&%N;F5L
+M`&UV4V%T85)E;6]V94-H86YN96P`;793871A0VAA;FYE;$AA<F1297-E=`!M
+M=E-A=&%#;VYF:6=%9&UA36]D90!M=E-A=&%.=6U/9D1M84-O;6UA;F1S`&UV
+M4V%T85-E=$-H86YN96Q0:'E087)A;7,`;793871A0VAA;FYE;%!H>5-H=71D
+M;W=N`&UV4V%T84-H86YN96Q0:'E0;W=E<D]N`&UV4V%T84=E=$-H86YN96Q3
+M=&%T=7,`;793871A475E=65#;VUM86YD`&UV4V%T84EN=&5R<G5P=%-E<G9I
+M8V52;W5T:6YE`&UV4V%T84UA<VM!9&%P=&5R26YT97)R=7!T`&UV4V%T855N
+M;6%S:T%D87!T97));G1E<G)U<'0`96YA8FQE4W1O<F%G941E=DEN=&5R<G5P
+M=`!D:7-A8FQE4W1O<F%G941E=DEN=&5R<G5P=`!M=E-T;W)A9V5$979!5$%)
+M9&QE26UM961I871E`&UV4W1O<F%G941E=D%404ED96YT:69Y1&5V:6-E`&UV
+M4W1O<F%G941E=D%405-O9G1297-E=$1E=FEC90!"965P3VX`0F5E<$]F9@!S
+M971?9F%I;%]L961S`'-E=%]F86EL7VQE9`!S>#4P.'A?:6]C=&P`<W@U,#AX
+M7V9L87-H7V%C8V5S<P``5`````%B``""`````F,``)X````!9```O@````)E
+M``#M`````F8```0!```"90``)@$```)G``",`0```F@``*@!```":0``P@$`
+M``)J``#<`0```FL``!$"```!9```/@(```%B```-`P```F<``+\#```!8@``
+MW0,```)L``#X`P```FP``!,$```";```+@0```)L```0!0```F<``-<%```!
+M8@``Y`4```%D```6!@```FP``#8&```";```5@8```)L``!V!@```FP``,$&
+M```!9```(0<```%D``!R!P```FX``*$'```!8@``K@<```%D``#X!P```F\`
+M`'D(```"<```VP@```)M``!K"0```FT``*T)```"<0``N0D```)R``#^"0``
+M`G````X*```!=```/@H```)V``!2"@```G<``),*```"=0``O0H```%T``#J
+M"@```F<``/X*```"8P``'0L```)R```\"P```F<``((-```">0``M@T```)Z
+M``!]#@```GP``)X.```"?0``T@X```)]```!#P```GT``#,/```"?P``>@\`
+M``%[``"!#P```7@``#,1```"9P``A1$```)G``#&$0```GP``/L1```"?0``
+MY!(```)\```[$P```GT``%$3```!A0``61,```)]``"E$P```F<```@4```"
+M?```%Q0```)]```M%````84``#44```"?0``QQ4```)G``#S%0```HD``$(6
+M```"?```3Q8```)]``!E%@```84``&L6```"?0``ZA8```)G```#%P```G\`
+M`"L7```!A```,A<```&#``!Q%P```G\``)D7```!A@``H!<```&#``#1%P``
+M`G\``/D7```!B````!@```&'```E&````GT``#08```!A0``.A@```)]``!K
+M&````GT``'T8```!`@``ZQ@```&*``#X&````HL``+H9```"?P``R1D```)L
+M``#D&0```8(``.L9```!@0``N1H```)N``#1&@```6(``-D:```!9```%QL`
+M``)O``!*&P```G$``%H;```"<@``S1P```)L```Q'@```GP``%(>```"?0``
+MAAX```)]``"R'@```GT``.,>```"?P``*A\```&.```Q'P```8T``(\@```"
+MD@``)R(```)G``"*(@```F<``.@F```">@``Y"@```)L```%*0```FP``(<M
+M```!`@``:RX```$"``!S+@```GT``*XN```!A0``MBX```)]``#7+@```84`
+M`-TN```"?0``)2\```)]``!0+P```I,``'HO```!`@``@B\```)]```#,```
+M`I0``($Q```"?```53(```)_``"9,@```0(``*,R```!`@``2#4```*5``!T
+M-0```I8``"HV```"E0```#<```*5``!/.````I4``'TX```"E@``;#H```$N
+M```\.P```2X``')H```"?0```6D```*3```):0```I@``!%I```"F0``Q&H`
+M``*3```5:P```GT``"QK```!A0``,FL```)]``!>:P```84``&1K```"?0``
+MCVL```*3``#>;````00``.EL```!!```[FP```*4``!$;P```GP``.%Q```"
+M9P``2W0```&%``!3=````GT``(=T```"?P``MW0```$"``"^=````0(``,)U
+M```">@``X74```)Z``!#>0```0(``$EY```"?0``D'D```)]``#">0```F<`
+M`.UY```"?0``%WH```)]``"?>@```GD``'%[```">0``!WP```)]``"X?```
+M`9L``,A\```"BP``'GT```&<```[?0```9L``$U]```"BP``6'T```*3``!M
+M?0```0(``)%]```"?0``Q7T```*7``!A?@```9$``)=^```!D0``PGX```*>
+M``#'?@```00``-%^```!D0``W7X```*>```!?P```9$``+Y_```";@``W'\`
+M``%B``#D?P```60``!"````";P``*H````)Q```V@````G(``*R````"<@``
+M'X$```)G```[@@```FP``%""```";```:((```)L``!]@@```FP``/^"```"
+M90``#X,```)E```O@P```F,``'.#```"H```HX,```*A``#X@P```G8``#V$
+M```"9P``C(0```)C``#+A````F,``&>&```"F```MH8```)G``!CAP```I<`
+M`(^'```"F@``Q(<```*H``#0AP```JD``.2'```"J@``DX@```*K``"?B```
+M`JP``/.(```"K@``&8D```*O```UB0```JX``'N)```"L```CXD```*P``"_
+MB0```K```(>,```"L@``EXP```*O``"LC````J\``,>,```"KP``XHP```*O
+M``#]C````J\``!^-```"L@``*XT```*O```^C0```J\``%&-```"KP``9(T`
+M``*O``!YC0```K(``(R-```"KP``GXT```*O``#<C0```K,``+2/```"LP``
+MRH\```*P``#:CP```K,``.J/```"LP``!9````*P```5D````K,``":0```"
+ML```F)````*P```'D0```K,``!^1```"L```PI$```*P``#.D0```K,``-J1
+M```"K@``Z)$```*P``#TD0```K,```"2```"K@``()(```*T``#9D@```;$`
+M`)V3```"KP``K9,```*O``#5DP```K```'25```"M0``IY4```*V``!,E@``
+M`;$``-&6```"LP``W98```*N``!0EP```K,``&.7```"L```;9<```*S``!Y
+MEP```JX``)^7```"L```0I@```*P``!7F````K```&J8```"L```@Y@```*P
+M``"EF````K```+B8```"L```VI@```*P``#PF````K```!Z:```"LP``5IH`
+M``*P``"*F@```K,``.Z:```"L```+9L```*P```YFP```K,``$6;```"K@``
+M4YL```*P``!?FP```K,``+6;```"L```Q)L```*S``#AFP```K```/6;```"
+ML```"9P```*P```=G````K```#&<```"L```19P```*P``!9G````K```&V<
+M```"L```@9P```*P``"5G````K```*F<```"L```O9P```*P``#4G````K``
+M``J=```"L```'IT```*P```RG0```K```$:=```"L```6)T```*S``!RG0``
+M`K```*>=```"L```TIT```*S``#FG0```K```/F=```"LP``#YX```*P```A
+MG@```K```#.>```"L```2)X```*P``!:G@```K```&R>```"L```?IX```*P
+M``"0G@```K```**>```"L```M)X```*P``#&G@```K```-B>```"L```N9\`
+M``)L``#2GP```FP``*N@```"KP``VZ````*P``#KH````K,```6A```"L```
+M%:$```*S``!,H0```K,``&:A```"L```S:$```*P``#AH0```K````*B```"
+MKP``':(```*P``!BH@```K```'6B```"LP``B:(```*N```GHP```K```%:C
+M```"MP``K*,```*O``#=HP```K(``/*C```"L@``!Z0```*R```<I````K(`
+M`#&D```"L@``?Z0```*R``"4I````K(``*FD```"L@``OJ0```*R``#3I```
+M`K(``.BD```"L@``_:0```*R```9I0```J\``#&E```"N```3:4```*O``"#
+MI0```K4``,ZE```"L```V:4```*Z``"OI@```K,```>G```"L```%Z<```*S
+M```QIP```K```$VG```"LP``P:<```*P``#4IP```K```.>G```"L```#J@`
+M``*P```PJ````K```&>H```"K@``C:@```*T``#@J````K```/&H```!L0``
+M`:D```&Q``"ZJ0```K,``."I```"LP``[*D```*N``#XJ0```K,``.FJ```"
+MLP``&:L```*P``!*JP```K,``%:K```"K@``BJL```*P``"ZJP```K,``,:K
+M```"K@``U*L```*M```.K````J\``(VL```"LP``WJP```*P``!>K@```K,`
+M`(^N```"L```[*X```*S```9KP```K```#2O```"LP``C:\```*S``"YKP``
+M`K```-2O```"LP```K````*S```VL````K```*:P```"LP``U+````*T```D
+ML0```K,``#FQ```"LP``3K$```*S``!CL0```K,``'BQ```"LP``C;$```*S
+M``"BL0```K,``+>Q```"LP``8;,```*S``"FLP```K,``,VS```"L```#K0`
+M``*P```:M````K,``"ZT```"L```2K0```*S```HM0```K```$FU```"L```
+MB[4```*O``"SM0```JX``,*U```"M````+8```*R```7M@```J\``#RV```"
+ML@``4[8```*O``"2M@```LP``*>V```"L@``O;8```*X``#-M@```LL``-VV
+M```"RP``1+<```*K``!PMP```JH``/&W```"MP``+[@```*K```_N0```K@`
+M`%BY```"S```?;D```*R``"6N0```K(``*^Y```"L@``R+D```*R``#AN0``
+M`K(``!NZ```"RP``/;H```*R``!4N@```K(``&NZ```"L@``@KH```*R``"9
+MN@```K(``+"Z```"L@``Q[H```*R``#GN@```K@``/>Z```"RP``"KL```++
+M```KNP```K@``#N[```"RP``5[L```*O``!GNP```LL``*"[```"N```N;L`
+M``*V``#9NP```K4``/:[```"N```!KP```++```6O````LL``":\```"RP``
+M7+P```*R``!NO````J\``'J\```"K@``@KP```++``".O````JX``)J\```"
+MK@``IKP```*N``"RO````JX``,R\```"KP``Z;P```*N```NO0```JX``%6]
+M```!`P``9+T```*R``!TO0```0,``(.]```"KP``G+T```$#``"KO0```J\`
+M`,2]```!`P``T[T```*O``#XO0```0,```>^```"L@``%+X```$#```CO@``
+M`K(``$.^```!`P``4KX```*R``!:O@```0,``&B^```"KP``DKX```$$``"G
+MO@```K,``+N^```"L```R;X```*S``#MO@```K4``/Z^```"LP``$+\```$#
+M```?OP```K(``"F_```!`P``-[\```*R``!!OP```0,``%"_```"L@``6K\`
+M``$#``!HOP```K(``'^_```"L```C;\```*S``"<OP```K```+^_```"LP``
+MT[\```*P``#AOP```K,```7````"M0``%L````*S```HP````0,``#?````"
+ML@``0<````$#``!/P````K(``%G````!`P``:,````*R``!RP````0,``(#`
+M```"L@``E\````*P``"EP````K,``+3````"L```VL````$#``#IP````K(`
+M`//````!`P```<$```*R```7P0```J\``#;!```!`P``1,$```*R``!8P0``
+M`JX``&;!```"KP``G,$```*S``"QP0```K```+_!```"LP``X\$```*U``#T
+MP0```K,```G"```!`P``&,(```*R```GP@```K```#7"```"LP``1L(```*P
+M``"6P@```M(``+/"```"LP``R,(```*P``#6P@```K,``/S"```"M0``#<,`
+M``*S```;PP```K8``$G#```!`P``6,,```*R``!RPP```JX``(;#```"M@``
+ME<,```$#``"DPP```K(``,_#```!`P``WL,```*R``#LPP```K8``"K$```!
+M`P``.<0```*R``!*Q````K```%C$```"LP``:<0```*P``".Q````0,``)+$
+M```!`P``J,0```$#``"LQ````0,``.O$```!!```&<4```*S```NQ0```K``
+M`#S%```"LP``8L4```*U``!SQ0```K,``)S%```"KP``T,4```*R``#MQ0``
+M`K```/O%```"LP``#,8```*P```@`````9\``"0````!GP``*`````&?```L
+M`````9\``#`````!CP``-`````%^```X`````8H``$`````!FP``8`````%T
+M``!D`````70``&@````!=```;`````%T``!P`````9```'0````!@```>```
+M``&,``"``````9P``.`````!`P````````$"```$`````0(```@````!`@``
+M#`````$"```0`````0(``!0````!`@``&`````$"```<`````0(``"`````!
+K`@``)`````$"```H`````0(``"P````!`@``,`````$"```T`````0(`````
 `
 end
diff --git a/sys/dev/hptmv/ioctl.c b/sys/dev/hptmv/ioctl.c
index bf80700..5887822 100644
--- a/sys/dev/hptmv/ioctl.c
+++ b/sys/dev/hptmv/ioctl.c
@@ -209,7 +209,11 @@ lock_driver_idle(IAL_ADAPTER_T *pAdapter)
 #if (__FreeBSD_version < 500000)
 		YIELD_THREAD;
 #else 
+#if (__FreeBSD_version > 700033)
 		pause("switch", 1);
+#else
+		tsleep(lock_driver_idle, PPAUSE, "switch", 1);
+#endif
 #endif
 		oldspl = lock_driver();
 	}
@@ -337,6 +341,7 @@ int Kernel_DeviceIoControl(_VBUS_ARG
 			case HPT_IOCTL_GET_CHANNEL_INFO:
 			case HPT_IOCTL_GET_LOGICAL_DEVICES:
 			case HPT_IOCTL_GET_DEVICE_INFO:
+			case HPT_IOCTL_GET_DEVICE_INFO_V2:				
 			case HPT_IOCTL_GET_EVENT:
 			case HPT_IOCTL_GET_DRIVER_CAPABILITIES:
 				if(hpt_default_ioctl(_VBUS_P dwIoControlCode, lpInBuffer, nInBufferSize, 
@@ -351,10 +356,14 @@ int Kernel_DeviceIoControl(_VBUS_ARG
 				switch(dwIoControlCode) {
 				case HPT_IOCTL_CREATE_ARRAY:
 					pVDev = ID_TO_VDEV(((PCREATE_ARRAY_PARAMS)lpInBuffer)->Members[0]); break;
+				case HPT_IOCTL_CREATE_ARRAY_V2:
+					pVDev = ID_TO_VDEV(((PCREATE_ARRAY_PARAMS_V2)lpInBuffer)->Members[0]); break;
 				case HPT_IOCTL_SET_ARRAY_INFO:
 					pVDev = ID_TO_VDEV(((PHPT_SET_ARRAY_INFO)lpInBuffer)->idArray); break;
 				case HPT_IOCTL_SET_DEVICE_INFO:
 					pVDev = ID_TO_VDEV(((PHPT_SET_DEVICE_INFO)lpInBuffer)->idDisk); break;
+				case HPT_IOCTL_SET_DEVICE_INFO_V2:
+					pVDev = ID_TO_VDEV(((PHPT_SET_DEVICE_INFO_V2)lpInBuffer)->idDisk); break;
 				case HPT_IOCTL_SET_BOOT_MARK:
 				case HPT_IOCTL_ADD_SPARE_DISK:
 				case HPT_IOCTL_REMOVE_SPARE_DISK:
@@ -410,6 +419,22 @@ int Kernel_DeviceIoControl(_VBUS_ARG
                     break;
 				}
 
+
+				case HPT_IOCTL_CREATE_ARRAY_V2:
+				{
+					pAdapter=(IAL_ADAPTER_T *)(ID_TO_VDEV(*(DEVICEID *)lpOutBuffer))->pVBus->OsExt;
+					oldspl = lock_driver();
+				             if(((PCREATE_ARRAY_PARAMS_V2)lpInBuffer)->CreateFlags & CAF_CREATE_AND_DUPLICATE) {
+						  (ID_TO_VDEV(*(DEVICEID *)lpOutBuffer))->u.array.rf_auto_rebuild = 0;
+				                          hpt_queue_dpc((HPT_DPC)hpt_rebuild_data_block, pAdapter, ID_TO_VDEV(*(DEVICEID *)lpOutBuffer), DUPLICATE);
+					} else if(((PCREATE_ARRAY_PARAMS_V2)lpInBuffer)->CreateFlags & CAF_CREATE_R5_ZERO_INIT) {
+				                          hpt_queue_dpc((HPT_DPC)hpt_rebuild_data_block, pAdapter, ID_TO_VDEV(*(DEVICEID *)lpOutBuffer), INITIALIZE);
+					} else if(((PCREATE_ARRAY_PARAMS_V2)lpInBuffer)->CreateFlags & CAF_CREATE_R5_BUILD_PARITY) {
+				                          hpt_queue_dpc((HPT_DPC)hpt_rebuild_data_block, pAdapter, ID_TO_VDEV(*(DEVICEID *)lpOutBuffer), REBUILD_PARITY);
+					}
+					unlock_driver(oldspl);
+					break;
+				}
 				case HPT_IOCTL_ADD_DISK_TO_ARRAY:
 				{
 					PVDevice pArray = ID_TO_VDEV(((PHPT_ADD_DISK_TO_ARRAY)lpInBuffer)->idArray);
@@ -424,7 +449,11 @@ int Kernel_DeviceIoControl(_VBUS_ARG
 						unlock_driver(oldspl);
 						while (!pArray->u.array.rf_rebuilding)
 						{
+#if (__FreeBSD_version > 700033)
 							pause("pause", 1);
+#else
+							tsleep((caddr_t)Kernel_DeviceIoControl, PPAUSE, "pause", 1);
+#endif
 							if ( timeout >= hz*3)
 								break;
 							timeout ++;
@@ -489,7 +518,11 @@ hpt_set_array_state(DEVICEID idArray, DWORD state)
 
 			while (!pVDevice->u.array.rf_rebuilding)
 			{
+#if (__FreeBSD_version > 700033)
 				pause("pause", 1);
+#else
+				tsleep((caddr_t)hpt_set_array_state, PPAUSE, "pause", 1);
+#endif
 				if ( timeout >= hz*20)
 					break;
 				timeout ++;
@@ -514,7 +547,11 @@ hpt_set_array_state(DEVICEID idArray, DWORD state)
 			
 			while (pVDevice->u.array.rf_abort_rebuild)
 			{
+#if (__FreeBSD_version > 700033)
 				pause("pause", 1);
+#else
+				tsleep((caddr_t)hpt_set_array_state, PPAUSE, "pause", 1);
+#endif
 				if ( timeout >= hz*20)
 					break;
 				timeout ++;
@@ -538,7 +575,11 @@ hpt_set_array_state(DEVICEID idArray, DWORD state)
 			
 			while (!pVDevice->u.array.rf_verifying)
 			{
+#if (__FreeBSD_version > 700033)
 				pause("pause", 1);
+#else
+				tsleep((caddr_t)hpt_set_array_state, PPAUSE, "pause", 1);
+#endif
 				if ( timeout >= hz*20)
 					break;
 				timeout ++;
@@ -557,7 +598,11 @@ hpt_set_array_state(DEVICEID idArray, DWORD state)
 			
 			while (pVDevice->u.array.rf_abort_rebuild)
 			{
+#if (__FreeBSD_version > 700033)
 				pause("pause", 1);
+#else
+				tsleep((caddr_t)hpt_set_array_state, PPAUSE, "pause", 1);
+#endif
 				if ( timeout >= hz*80)
 					break;
 				timeout ++;
@@ -578,7 +623,11 @@ hpt_set_array_state(DEVICEID idArray, DWORD state)
 			
 			while (!pVDevice->u.array.rf_initializing)
 			{
+#if (__FreeBSD_version > 700033)
 				pause("pause", 1);
+#else
+				tsleep((caddr_t)hpt_set_array_state, PPAUSE, "pause", 1);
+#endif
 				if ( timeout >= hz*80)
 					break;
 				timeout ++;
@@ -597,7 +646,11 @@ hpt_set_array_state(DEVICEID idArray, DWORD state)
 			
 			while (pVDevice->u.array.rf_abort_rebuild)
 			{
+#if (__FreeBSD_version > 700033)
 				pause("pause", 1);
+#else
+				tsleep((caddr_t)hpt_set_array_state, PPAUSE, "pause", 1);
+#endif
 				if ( timeout >= hz*80)
 					break;
 				timeout ++;
@@ -952,7 +1005,11 @@ fail:
 #if (__FreeBSD_version < 500000)		
 		YIELD_THREAD; 
 #else 
+#if (__FreeBSD_version > 700033)
 		pause("switch", 1);
+#else
+		tsleep(hpt_rebuild_data_block, PPAUSE, "switch", 1);
+#endif
 #endif
 		oldspl = lock_driver();
 	}
diff --git a/sys/dev/hptmv/mvOs.h b/sys/dev/hptmv/mvOs.h
index 510f790..a1ababc 100644
--- a/sys/dev/hptmv/mvOs.h
+++ b/sys/dev/hptmv/mvOs.h
@@ -28,7 +28,7 @@
 #ifndef __INCmvOsBsdh
 #define __INCmvOsBsdh
 
-#ifdef DBG
+#ifdef DEBUG
 #define MV_DEBUG_LOG
 #endif
 
@@ -58,10 +58,10 @@ typedef void *PVOID, *LPVOID;
 typedef void *ADDRESS;
 
 typedef int  LONG;
-typedef unsigned int ULONG, *PULONG, LBA_T;
+typedef unsigned int ULONG, *PULONG;
 typedef unsigned int DWORD, *LPDWORD, *PDWORD;
 typedef unsigned long ULONG_PTR, UINT_PTR, BUS_ADDR;
-typedef unsigned long long HPT_U64;
+typedef unsigned long long HPT_U64, LBA_T;
 
 typedef enum mvBoolean{MV_FALSE, MV_TRUE} MV_BOOLEAN;
 
diff --git a/sys/dev/hptmv/mvSata.h b/sys/dev/hptmv/mvSata.h
index bf77fbf..982743c 100644
--- a/sys/dev/hptmv/mvSata.h
+++ b/sys/dev/hptmv/mvSata.h
@@ -36,15 +36,34 @@
 #define SUPPORT_MV_SATA_GEN_2 0
 #endif
 
-#if SUPPORT_MV_SATA_GEN_1==1 && SUPPORT_MV_SATA_GEN_2==1
+#ifndef SUPPORT_MV_SATA_GEN_2E
+#define SUPPORT_MV_SATA_GEN_2E 0
+#endif
+
+#if (SUPPORT_MV_SATA_GEN_1 + SUPPORT_MV_SATA_GEN_2 + SUPPORT_MV_SATA_GEN_2E) > 1
+
 #define MV_SATA_GEN_1(x) ((x)->sataAdapterGeneration==1)
-#define MV_SATA_GEN_2(x) ((x)->sataAdapterGeneration==2)
+#define MV_SATA_GEN_2(x) ((x)->sataAdapterGeneration>=2)
+#define MV_SATA_GEN_2E(x) ((x)->sataAdapterGeneration==3)
+
 #elif SUPPORT_MV_SATA_GEN_1==1
+
 #define MV_SATA_GEN_1(x) 1
 #define MV_SATA_GEN_2(x) 0
+#define MV_SATA_GEN_2E(x) 0
+
 #elif SUPPORT_MV_SATA_GEN_2==1
+
 #define MV_SATA_GEN_1(x) 0
 #define MV_SATA_GEN_2(x) 1
+#define MV_SATA_GEN_2E(x) 0
+
+#elif SUPPORT_MV_SATA_GEN_2E==1
+
+#define MV_SATA_GEN_1(x)  0
+#define MV_SATA_GEN_2(x)  1 /* gen2E impiles gen2 */
+#define MV_SATA_GEN_2E(x) 1
+
 #else 
 #error "Which IC do you support?"
 #endif
@@ -56,8 +75,15 @@
 #define MV_SATA_DEVICE_ID_5081			   		0x5081
 #define MV_SATA_DEVICE_ID_6080			   		0x6080
 #define MV_SATA_DEVICE_ID_6081			   		0x6081
+
+#if defined(RR2310) || defined(RR1740) || defined(RR2210) || defined (RR2522)
+#define MV_SATA_CHANNELS_NUM					4
+#define MV_SATA_UNITS_NUM						1
+#else 
 #define MV_SATA_CHANNELS_NUM					8
 #define MV_SATA_UNITS_NUM						2
+#endif
+
 #define MV_SATA_PCI_BAR0_SPACE_SIZE				(1<<18) /* 256 Kb*/
 
 #define CHANNEL_QUEUE_LENGTH					32
@@ -156,7 +182,7 @@ typedef MV_BOOLEAN (* HPTLIBAPI mvSataCommandCompletionCallBack_t)(struct mvSata
                                                          MV_COMPLETION_TYPE,
 														 MV_VOID_PTR, MV_U16,
 														 MV_U32,
-											    struct mvStorageDevRegisters FAR*);
+											    struct mvStorageDevRegisters SS_SEG*);
 
 typedef enum mvQueuedCommandType 
 {
@@ -222,8 +248,8 @@ typedef struct mvSataChannel
 	MV_BOOLEAN                  waitingForInterrupt;
 	MV_BOOLEAN                  lba48Address; 
 	MV_BOOLEAN                  maxReadTransfer;
-	struct mvDmaRequestQueueEntry FAR *requestQueue;
-	struct mvDmaResponseQueueEntry FAR *responseQueue;
+	struct mvDmaRequestQueueEntry SS_SEG *requestQueue;
+	struct mvDmaResponseQueueEntry SS_SEG *responseQueue;
 	MV_U32                      requestQueuePciHiAddress;
 	MV_U32                      requestQueuePciLowAddress;
 	MV_U32                      responseQueuePciHiAddress;
@@ -243,7 +269,7 @@ typedef struct mvSataChannel
 	MV_BOOLEAN					queueCommandsEnabled;
 	MV_U8                       noneUdmaOutstandingCommands;
 	MV_U8                       EdmaQueuedCommands;
-    MV_U32                      freeIDsStack[MV_EDMA_QUEUE_LENGTH];
+    MV_U32                      freeIDsStack[CHANNEL_QUEUE_LENGTH];
 	MV_U32                      freeIDsNum;
 	MV_U32                      reqInPtr;
 	MV_U32                      rspOutPtr;
@@ -279,7 +305,10 @@ typedef struct mvSataAdapter
 	MV_BOOLEAN        implement60X1A0Workarounds;
 	MV_BOOLEAN        implement60X1A1Workarounds;
 	MV_BOOLEAN        implement60X1B0Workarounds;
+	MV_BOOLEAN		  implement7042A0Workarounds;
+	MV_BOOLEAN		  implement7042A1Workarounds;
 	MV_U8			  sataAdapterGeneration;
+	MV_BOOLEAN		isPEX;
 	MV_U8             failLEDMask;
     MV_U8			  signalAmps[MV_SATA_CHANNELS_NUM];
 	MV_U8			  pre[MV_SATA_CHANNELS_NUM];
@@ -394,21 +423,9 @@ MV_BOOLEAN HPTLIBAPI mvSataChannelSetEdmaLoopBackMode(MV_SATA_ADAPTER *pAdapter,
 MV_BOOLEAN HPTLIBAPI mvSataGetChannelStatus(MV_SATA_ADAPTER *pAdapter, MV_U8 channelIndex,
 								  MV_SATA_CHANNEL_STATUS *pChannelStatus);
 
-/* Execute UDMA ATA commands */
-MV_EDMA_QUEUE_RESULT HPTLIBAPI mvSataQueueUDmaCommand(MV_SATA_ADAPTER *pAdapter,
-											MV_U8 channelIndex,
-											MV_UDMA_TYPE readWrite,
-											MV_U32 lowLBAAddr,
-											MV_U16 highLBAAddr,
-											MV_U16 sectorCount,
-											MV_U32 prdLowAddr,
-											MV_U32 prdHighAddr,
-									mvSataCommandCompletionCallBack_t callBack,
-											MV_VOID_PTR commandId);
-
 MV_QUEUE_COMMAND_RESULT HPTLIBAPI mvSataQueueCommand(MV_SATA_ADAPTER *pAdapter,
 										   MV_U8 channelIndex,
-										   MV_QUEUE_COMMAND_INFO FAR *pCommandParams);
+										   MV_QUEUE_COMMAND_INFO SS_SEG *pCommandParams);
 
 /* Interrupt Service Routine */
 MV_BOOLEAN HPTLIBAPI mvSataInterruptServiceRoutine(MV_SATA_ADAPTER *pAdapter);
diff --git a/sys/dev/hptmv/mvStorageDev.h b/sys/dev/hptmv/mvStorageDev.h
index 48aad93..d9cdce6 100644
--- a/sys/dev/hptmv/mvStorageDev.h
+++ b/sys/dev/hptmv/mvStorageDev.h
@@ -207,6 +207,9 @@ MV_BOOLEAN HPTLIBAPI mvStorageDevATAFlushWriteCache(MV_SATA_ADAPTER *pAdapter,
 MV_BOOLEAN HPTLIBAPI mvStorageDevATASoftResetDevice(MV_SATA_ADAPTER *pAdapter,
 										  MV_U8 channelIndex);
 
+MV_BOOLEAN HPTLIBAPI mvStorageDevWaitStat(MV_SATA_CHANNEL *pSataChannel,
+								MV_U8 good, MV_U8 bad, MV_U32 loops, MV_U32 delay);
+
 MV_BOOLEAN HPTLIBAPI mvReadWrite(MV_SATA_CHANNEL *pSataChannel, LBA_T Lba, UCHAR Cmd, void *tmpBuffer);
 
 #endif
diff --git a/sys/dev/hptmv/osbsd.h b/sys/dev/hptmv/osbsd.h
index 904bb39..e71e3ed 100644
--- a/sys/dev/hptmv/osbsd.h
+++ b/sys/dev/hptmv/osbsd.h
@@ -42,7 +42,7 @@
 #include <vm/pmap.h>
 #include <vm/vm_extern.h>
 
-#if (__FreeBSD_version < 600030)
+#if (__FreeBSD_version < 600000)
 #include <machine/bus_memio.h>
 #endif
 #include <machine/bus.h>
@@ -102,11 +102,6 @@ typedef struct _INQUIRYDATA {
 	UCHAR Reserved3[40];
 } INQUIRYDATA, *PINQUIRYDATA;
 
-typedef struct _READ_CAPACITY_DATA {
-	ULONG LogicalBlockAddress;
-	ULONG BytesPerBlock;
-} READ_CAPACITY_DATA, *PREAD_CAPACITY_DATA;
-
 #define MV_IAL_HT_SACOALT_DEFAULT	1
 #define MV_IAL_HT_SAITMTH_DEFAULT	1
 
@@ -145,7 +140,13 @@ typedef struct _MV_CHANNEL
 	unsigned int		maxUltraDmaModeSupported;
 	unsigned int		maxDmaModeSupported;
 	unsigned int		maxPioModeSupported;
-	MV_BOOLEAN			online;
+	MV_BOOLEAN		online;
+	MV_BOOLEAN		writeCacheSupported;
+	MV_BOOLEAN		writeCacheEnabled;
+	MV_BOOLEAN		readAheadSupported;
+	MV_BOOLEAN		readAheadEnabled;
+	MV_U8			queueDepth;
+	
 } MV_CHANNEL;
 
 typedef struct _BUS_DMAMAP
@@ -302,12 +303,27 @@ typedef struct _HPT_SET_DEVICE_INFO
 	ALTERABLE_DEVICE_INFO Info;
 } HPT_SET_DEVICE_INFO, *PHPT_SET_DEVICE_INFO;
 
+typedef struct _HPT_SET_DEVICE_INFO_V2
+{
+	DEVICEID idDisk;
+	ALTERABLE_DEVICE_INFO_V2 Info;
+} HPT_SET_DEVICE_INFO_V2, *PHPT_SET_DEVICE_INFO_V2;
+
 typedef struct _HPT_ADD_DISK_TO_ARRAY
 {
 	DEVICEID idArray;
 	DEVICEID idDisk;
 } HPT_ADD_DISK_TO_ARRAY, *PHPT_ADD_DISK_TO_ARRAY;
 
+typedef struct _HPT_DEVICE_IO
+{
+	DEVICEID	id;
+	int			cmd;
+	ULONG		lba;
+	DWORD		nSector;
+	UCHAR		buffer[0];
+} HPT_DEVICE_IO, *PHPT_DEVICE_IO;
+
 int check_VDevice_valid(PVDevice);
 int hpt_default_ioctl(_VBUS_ARG DWORD, PVOID, DWORD, PVOID, DWORD, PDWORD);
 
diff --git a/sys/dev/hptmv/raid5n.h b/sys/dev/hptmv/raid5n.h
index 135177a..52dd95b 100644
--- a/sys/dev/hptmv/raid5n.h
+++ b/sys/dev/hptmv/raid5n.h
@@ -61,10 +61,10 @@ void HPTLIBAPI DoXor2(ULONG *p0, ULONG *p2, UINT nBytes);
 #define xor_init(arg) 0
 #define xor_add_item(handle, dest, src, nsrc, bytes) \
 	do {\
-		if (((void**)src)[0]==dest)\
-			DoXor2((PULONG)(dest), ((PULONG *)src)[1], bytes);\
+		if (((void**)(src))[0]==dest)\
+			DoXor2((PULONG)(dest), ((PULONG *)(src))[1], bytes);\
 		else\
-			DoXor1((PULONG)(dest), ((PULONG *)src)[0], ((PULONG *)src)[1], bytes);\
+			DoXor1((PULONG)(dest), ((PULONG *)(src))[0], ((PULONG *)(src))[1], bytes);\
 	} while(0)
 #define xor_exec(handle, done, tag) done(_VBUS_P tag, 0)
 #define xor_poll()
diff --git a/sys/dev/hptmv/readme.txt b/sys/dev/hptmv/readme.txt
index 6d21e98..6143076 100644
--- a/sys/dev/hptmv/readme.txt
+++ b/sys/dev/hptmv/readme.txt
@@ -1,9 +1,22 @@
-RocketRAID 182x Driver for FreeBSD
-Copyright (C) 2004-2005 HighPoint Technologies, Inc. All rights reserved.
+RocketRAID 18xx Driver for FreeBSD
+Copyright (C) 2007-2008 HighPoint Technologies, Inc. All rights reserved.
 $FreeBSD$
 
 #############################################################################
 Revision History:
+   v1.16 2008-2-29
+         Fix 7.0 compile error.
+         
+   v1.15 2007-8-6
+         Override kernel driver(built-in) to support over 2T RAID array. 
+
+   v1.14 2006-3-21
+         Fix 48-bit LBA compatibility for Seagate drives.
+         Fix 16 bytes CDB support.
+
+   v1.13 2006-2-13
+         Fix fail LED/beeper control.
+         Add 16 bytes CDB support.
 
    v1.12 2005-6-10
          Fix over 4G memory support on amd64.
@@ -21,7 +34,7 @@ Revision History:
 1. Overview
 ---------------------
   This package contains FreeBSD driver source code for HighPoint RocketRAID 
-  182x SATA controller.
+  18xx SATA controller.
 
   NO WARRANTY
 
@@ -41,7 +54,7 @@ Revision History:
   SUCH HARDWARE, OR DATA.
 
 
-2. Rebuild the kernel with RR182x support
+2. Rebuild the kernel with RR18xx support
 --------------------------------------------
 
   1) Install kernel source package and building tools. 
@@ -49,7 +62,7 @@ Revision History:
   2) Extract the driver files under the kernel source tree:
 
      # cd /usr/src/sys/
-     # tar xvzf /your/path/to/rr182x-opensource-v1.12-bsd.tgz
+     # tar xvzf /your/path/to/rr18xx-opensource-v1.12-bsd.tgz
 
   3) Update the kernel configuration file to include the HighPoint source.
      Assume the configure file is GENERIC, and new kernel configure file is 
@@ -61,7 +74,7 @@ Revision History:
   4) Edit MYKERNEL, and add the following line under "RAID controllers 
      interfaced to the SCSI subsystem":
 
-          device  hptmv   #HighPoint RocketRAID 182x
+          device  hptmv   #HighPoint RocketRAID 18xx
 
   5) For i386 system, edit /usr/src/sys/conf/files.i386 and append the lines
      shown below:
@@ -91,13 +104,13 @@ Revision History:
           dev/hptmv/entry.c       optional        hptmv
           dev/hptmv/mv.c          optional        hptmv
 
-     Note FreeBSD 5.3/5.4 i386 already have a built-in RR182x driver, you should
-     replace the old configuration lines with the lines listed above.
+     Note FreeBSD 5.3/5.4/6.x/7.x i386 already have a built-in RR18xx driver,
+     you should replace the old configuration lines with the lines listed above.
 
 
   6) Rebuild and install the kernel:
 
-     a) for FreeBSD 5.x-i386:
+     a) for FreeBSD 5.x/6.x/7.x i386:
      
         # cd /usr/src/sys/i386/conf/
         # /usr/sbin/config MYKERNEL
@@ -106,7 +119,7 @@ Revision History:
         # make 
         # make install
 
-     b) for FreeBSD 5.x-amd64:
+     b) for FreeBSD 5.x/6.x/7.x amd64:
 
         # cd /usr/src/sys/amd64/conf/
         # /usr/sbin/config MYKERNEL
@@ -139,7 +152,7 @@ Revision History:
   2) Extract the driver files under the kernel source tree:
     
      # cd /usr/src/sys/
-     # tar xvzf /your/path/to/rr182x-opensource-v1.12-bsd.tgz
+     # tar xvzf /your/path/to/rr18xx-opensource-v1.12-bsd.tgz
 
 
   4) Build the driver module:
@@ -153,7 +166,7 @@ Revision History:
      
      # cp hptmv.ko /modules/
 
-     For FreeBSD 5.x:
+     For FreeBSD 5.x/6.x/7.x:
     
      # cp hptmv.ko /boot/kernel/
 
@@ -179,7 +192,7 @@ Revision History:
         /modules/hptmv.ko text=0xf571 data=0x2c8+0x254
         ok boot
         
-     For FreeBSD 5.x, you can select 6 on the boot menu to get a loader prompt.
+     For FreeBSD 5.x/6.x/7.x, you can select 6 on the boot menu to get a loader prompt.
   
   7) You can add a below line into /boot/defaults/loader.conf to load the
      driver automatically:
diff --git a/sys/dev/hptmv/vdevice.h b/sys/dev/hptmv/vdevice.h
index 8d06f10..e0bddc7 100644
--- a/sys/dev/hptmv/vdevice.h
+++ b/sys/dev/hptmv/vdevice.h
@@ -35,20 +35,20 @@
 
 typedef  struct _VDevice
 {
-    UCHAR        VDeviceType;
-    UCHAR        vf_bootmark: 1; 	/* is boot device? */
-    UCHAR		 vf_bootable: 1;    /* has active partition */
-    UCHAR        vf_online: 1; 		/* is usable? */
-    UCHAR        vf_cache_disk: 1;  /* Cache enabled */
+	UCHAR        VDeviceType;
+	UCHAR        vf_bootmark: 1; 	/* is boot device? */
+	UCHAR		 vf_bootable: 1;    /* has active partition */
+	UCHAR        vf_online: 1; 		/* is usable? */
+	UCHAR        vf_cache_disk: 1;  /* Cache enabled */
 	UCHAR        vf_format_v2: 1;   /* old array block */
 	UCHAR        vf_freed: 1;       /* memory free */
-    UCHAR        reserve1;
-    UCHAR        bSerialNumber; 	/* valid if pParent!=0 */
+	UCHAR        reserve1;
+	UCHAR        bSerialNumber; 	/* valid if pParent!=0 */
 
-    PVDevice	 pParent;			/* parent array */
+	PVDevice	 pParent;			/* parent array */
 	PVBus        pVBus;				/* vbus this device located. Must not be NULL. */
 
-    LBA_T        VDeviceCapacity;   /* number of blocks */
+	LBA_T        VDeviceCapacity;   /* number of blocks */
 
 	LBA_T        LockedLba;
 	USHORT       LockedSectors;
@@ -58,16 +58,16 @@ typedef  struct _VDevice
 	void  *QuiesceArg;
 	void (* HPTLIBAPI flush_callback)(_VBUS_ARG void *arg);
 	void *flush_callback_arg;
-    
+
 
 #if defined(_RAID5N_)
 	struct stripe **CacheEntry;
 	struct range_lock *range_lock;
 #endif
 
-    void (* HPTLIBAPI pfnSendCommand)(_VBUS_ARG PCommand pCmd);   /* call this to send a command to a VDevice */
-    void (* HPTLIBAPI pfnDeviceFailed)(_VBUS_ARG PVDevice pVDev); /* call this when a VDevice failed */
-	
+	void (* HPTLIBAPI pfnSendCommand)(_VBUS_ARG PCommand pCmd);   /* call this to send a command to a VDevice */
+	void (* HPTLIBAPI pfnDeviceFailed)(_VBUS_ARG PVDevice pVDev); /* call this when a VDevice failed */
+
 	union {
 #ifdef SUPPORT_ARRAY
 		RaidArray array;
@@ -85,10 +85,10 @@ typedef  struct _VDevice
 /*
  * bUserDeviceMode
  */
-#define MEMBER_NOT_SET_MODE  0x5F 
+#define MEMBER_NOT_SET_MODE  0x5F
 
-/* 
- * arrayType 
+/*
+ * arrayType
  */
 #define VD_SPARE             0
 #define VD_REMOVABLE         1
@@ -120,10 +120,10 @@ void HPTLIBAPI fSingleDiskFailed(_VBUS_ARG PVDevice pVDev);
 
 typedef struct _VBus  {
 	/* pVDevice[] may be non-continuous */
-    PVDevice      pVDevice[MAX_VDEVICE_PER_VBUS];
+	PVDevice      pVDevice[MAX_VDEVICE_PER_VBUS];
 
-    UINT          nInstances;
-    PChipInstance pChipInstance[MAX_CHIP_IN_VBUS];
+	UINT          nInstances;
+	PChipInstance pChipInstance[MAX_CHIP_IN_VBUS];
 
 	void *        OsExt; /* for OS private use */
 
@@ -176,16 +176,20 @@ typedef struct _VBus  {
 		for(i = 0; i < MAX_VDEVICE_PER_VBUS; i++) \
 			if ((pVDev=pVBus->pVDevice[i])==0) continue; else
 
-#define FOR_EACH_DEV_ON_ALL_VBUS(pVBus, pVDev, i) \
+
+#define FOR_EACH_VBUS(pVBus) \
 	for(pVBus = gVBus; pVBus < &gVBus[MAX_VBUS]; pVBus++) \
-		for(i = 0; i < MAX_VDEVICE_PER_VBUS; i++) \
-			if ((pVDev=pVBus->pVDevice[i])==0) continue; else
 
 #define FOR_EACH_ARRAY_ON_ALL_VBUS(pVBus, pArray, i) \
 	for(pVBus = gVBus; pVBus < &gVBus[MAX_VBUS]; pVBus++) \
 		for(i = 0; i < MAX_ARRAY_PER_VBUS; i++) \
 			if ((pArray=((PVDevice)&pVBus->_ArrayTables[i*ARRAY_VDEV_SIZE]))->u.array.dArStamp==0) continue; else
 
+#define FOR_EACH_DEV_ON_ALL_VBUS(pVBus, pVDev, i) \
+	FOR_EACH_VBUS(pVBus) \
+		for(i = 0; i < MAX_VDEVICE_PER_VBUS; i++) \
+			if ((pVDev=pVBus->pVDevice[i])==0) continue; else
+
 /***************************************************************************
  * Description:  the functions called by IDE layer
  ***************************************************************************/
@@ -196,10 +200,10 @@ void HPTLIBAPI IdeRegisterDevice(PDevice pDev);
 #endif
 
 /***************************************************************************
- * Description:  the functions OS must provided 
+ * Description:  the functions OS must provided
  ***************************************************************************/
 
-void OsSetDeviceTable(PDevice pDevice, PIDENTIFY_DATA pIdentify);
+void HPTLIBAPI OsSetDeviceTable(PDevice pDevice, PIDENTIFY_DATA pIdentify);
 
 /*
  * allocate and free data structure
@@ -240,12 +244,12 @@ void VBus_Config(PVBus pVBus, char *str);
 #endif
 
 #pragma pack(1)
-struct fdisk_partition_table 
+struct fdisk_partition_table
 {
 	UCHAR 		bootid;   			/* bootable?  0=no, 128=yes  */
 	UCHAR 		beghead;  			/* beginning head number */
 	UCHAR 		begsect;  			/* beginning sector number */
-	UCHAR		begcyl;   			/* 10 bit nmbr, with high 2 bits put in begsect */	
+	UCHAR		begcyl;   			/* 10 bit nmbr, with high 2 bits put in begsect */
 	UCHAR		systid;   			/* Operating System type indicator code */
 	UCHAR 		endhead;  			/* ending head number */
 	UCHAR 		endsect;  			/* ending sector number */
@@ -254,7 +258,7 @@ struct fdisk_partition_table
 	ULONG 		numsect;            /* number of sectors in partition */
 };
 
-typedef struct _Master_Boot_Record 
+typedef struct _Master_Boot_Record
 {
 	UCHAR   bootinst[446];   		/* space to hold actual boot code */
 	struct 	fdisk_partition_table parts[4];
@@ -278,5 +282,5 @@ typedef struct _TIME_RECORD {
 #endif
 #endif
 
-#pragma pack()	
+#pragma pack()
 #endif
diff --git a/sys/dev/ichwd/ichwd.c b/sys/dev/ichwd/ichwd.c
index 917a31a..71662a5 100644
--- a/sys/dev/ichwd/ichwd.c
+++ b/sys/dev/ichwd/ichwd.c
@@ -141,26 +141,55 @@ static devclass_t ichwd_devclass;
 			device_printf(dev, __VA_ARGS__);\
 	} while (0)
 
+/*
+ * Disable the watchdog timeout SMI handler.
+ *
+ * Apparently, some BIOSes install handlers that reset or disable the
+ * watchdog timer instead of resetting the system, so we disable the SMI
+ * (by clearing the SMI_TCO_EN bit of the SMI_EN register) to prevent this
+ * from happening.
+ */
 static __inline void
-ichwd_intr_enable(struct ichwd_softc *sc)
+ichwd_smi_disable(struct ichwd_softc *sc)
 {
 	ichwd_write_smi_4(sc, SMI_EN, ichwd_read_smi_4(sc, SMI_EN) & ~SMI_TCO_EN);
 }
 
+/*
+ * Enable the watchdog timeout SMI handler.  See above for details.
+ */
 static __inline void
-ichwd_intr_disable(struct ichwd_softc *sc)
+ichwd_smi_enable(struct ichwd_softc *sc)
 {
 	ichwd_write_smi_4(sc, SMI_EN, ichwd_read_smi_4(sc, SMI_EN) | SMI_TCO_EN);
 }
 
+/*
+ * Reset the watchdog status bits.
+ */
 static __inline void
 ichwd_sts_reset(struct ichwd_softc *sc)
 {
+	/*
+	 * The watchdog status bits are set to 1 by the hardware to
+	 * indicate various conditions.  They can be cleared by software
+	 * by writing a 1, not a 0.
+	 */
 	ichwd_write_tco_2(sc, TCO1_STS, TCO_TIMEOUT);
+	/*
+	 * XXX The datasheet says that TCO_SECOND_TO_STS must be cleared
+	 * before TCO_BOOT_STS, not the other way around.
+	 */
 	ichwd_write_tco_2(sc, TCO2_STS, TCO_BOOT_STS);
 	ichwd_write_tco_2(sc, TCO2_STS, TCO_SECOND_TO_STS);
 }
 
+/*
+ * Enable the watchdog timer by clearing the TCO_TMR_HALT bit in the
+ * TCO1_CNT register.  This is complicated by the need to preserve bit 9
+ * of that same register, and the requirement that all other bits must be
+ * written back as zero.
+ */
 static __inline void
 ichwd_tmr_enable(struct ichwd_softc *sc)
 {
@@ -172,6 +201,9 @@ ichwd_tmr_enable(struct ichwd_softc *sc)
 	ichwd_verbose_printf(sc->device, "timer enabled\n");
 }
 
+/*
+ * Disable the watchdog timer.  See above for details.
+ */
 static __inline void
 ichwd_tmr_disable(struct ichwd_softc *sc)
 {
@@ -183,6 +215,11 @@ ichwd_tmr_disable(struct ichwd_softc *sc)
 	ichwd_verbose_printf(sc->device, "timer disabled\n");
 }
 
+/*
+ * Reload the watchdog timer: writing anything to any of the lower five
+ * bits of the TCO_RLD register reloads the timer from the last value
+ * written to TCO_TMR.
+ */
 static __inline void
 ichwd_tmr_reload(struct ichwd_softc *sc)
 {
@@ -194,6 +231,10 @@ ichwd_tmr_reload(struct ichwd_softc *sc)
 	ichwd_verbose_printf(sc->device, "timer reloaded\n");
 }
 
+/*
+ * Set the initial timeout value.  Note that this must always be followed
+ * by a reload.
+ */
 static __inline void
 ichwd_tmr_set(struct ichwd_softc *sc, unsigned int timeout)
 {
@@ -262,7 +303,8 @@ ichwd_clear_noreboot(struct ichwd_softc *sc)
 }
 
 /*
- * Watchdog event handler.
+ * Watchdog event handler - called by the framework to enable or disable
+ * the watchdog or change the initial timeout value.
  */
 static void
 ichwd_event(void *arg, unsigned int cmd, int *error)
@@ -426,6 +468,13 @@ ichwd_attach(device_t dev)
 	device_printf(dev, "%s (ICH%d or equivalent)\n",
 	    device_get_desc(dev), sc->ich_version);
 
+	/*
+	 * XXX we should check the status registers (specifically, the
+	 * TCO_SECOND_TO_STS bit in the TCO2_STS register) to see if we
+	 * just came back from a watchdog-induced reset, and let the user
+	 * know.
+	 */
+
 	/* reset the watchdog status registers */
 	ichwd_sts_reset(sc);
 
@@ -435,8 +484,8 @@ ichwd_attach(device_t dev)
 	/* register the watchdog event handler */
 	sc->ev_tag = EVENTHANDLER_REGISTER(watchdog_list, ichwd_event, sc, 0);
 
-	/* enable watchdog timeout interrupts */
-	ichwd_intr_enable(sc);
+	/* disable the SMI handler */
+	ichwd_smi_disable(sc);
 
 	return (0);
  fail:
@@ -466,8 +515,8 @@ ichwd_detach(device_t dev)
 	if (sc->active)
 		ichwd_tmr_disable(sc);
 
-	/* disable watchdog timeout interrupts */
-	ichwd_intr_disable(sc);
+	/* enable the SMI handler */
+	ichwd_smi_enable(sc);
 
 	/* deregister event handler */
 	if (sc->ev_tag != NULL)
diff --git a/sys/dev/if_ndis/if_ndis.c b/sys/dev/if_ndis/if_ndis.c
index 976253c..a3bda2c 100644
--- a/sys/dev/if_ndis/if_ndis.c
+++ b/sys/dev/if_ndis/if_ndis.c
@@ -753,7 +753,8 @@ ndis_attach(dev)
 		ic->ic_ifp = ifp;
 		ic->ic_opmode = IEEE80211_M_STA;
 	        ic->ic_phytype = IEEE80211_T_DS;
-		ic->ic_caps = IEEE80211_C_STA | IEEE80211_C_IBSS;
+		ic->ic_caps = IEEE80211_C_8023ENCAP |
+			IEEE80211_C_STA | IEEE80211_C_IBSS;
 		setbit(ic->ic_modecaps, IEEE80211_MODE_AUTO);
 		len = 0;
 		r = ndis_get_info(sc, OID_802_11_NETWORK_TYPES_SUPPORTED,
@@ -937,8 +938,7 @@ got_crypto:
 		if (r == 0)
 			ic->ic_caps |= IEEE80211_C_TXPMGT;
 
-		bcopy(eaddr, &ic->ic_myaddr, sizeof(eaddr));
-		ieee80211_ifattach(ic);
+		ieee80211_ifattach(ic, eaddr);
 		ic->ic_raw_xmit = ndis_raw_xmit;
 		ic->ic_scan_start = ndis_scan_start;
 		ic->ic_scan_end = ndis_scan_end;
@@ -2408,7 +2408,7 @@ ndis_setstate_80211(sc)
 
 	/* Set the BSSID to our value so the driver doesn't associate */
 	len = IEEE80211_ADDR_LEN;
-	bcopy(ic->ic_myaddr, bssid, len);
+	bcopy(IF_LLADDR(ifp), bssid, len);
 	DPRINTF(("Setting BSSID to %6D\n", (uint8_t *)&bssid, ":"));
 	rval = ndis_set_info(sc, OID_802_11_BSSID, &bssid, &len);
 	if (rval)
@@ -3250,8 +3250,10 @@ ndis_stop(sc)
 
 	NDIS_LOCK(sc);
 	for (i = 0; i < NDIS_EVENTS; i++) {
-		if (sc->ndis_evt[i].ne_sts && sc->ndis_evt[i].ne_buf != NULL)
+		if (sc->ndis_evt[i].ne_sts && sc->ndis_evt[i].ne_buf != NULL) {
 			free(sc->ndis_evt[i].ne_buf, M_TEMP);
+			sc->ndis_evt[i].ne_buf = NULL;
+		}
 		sc->ndis_evt[i].ne_sts = 0;
 		sc->ndis_evt[i].ne_len = 0;
 	}
diff --git a/sys/dev/if_ndis/if_ndis_usb.c b/sys/dev/if_ndis/if_ndis_usb.c
index 7aeed9a..ec9741b 100644
--- a/sys/dev/if_ndis/if_ndis_usb.c
+++ b/sys/dev/if_ndis/if_ndis_usb.c
@@ -210,6 +210,10 @@ ndisusb_detach(device_t self)
 
 	ndis_pnpevent_nic(self, NDIS_PNP_EVENT_SURPRISE_REMOVED);
 
+	if (sc->ndisusb_status & NDISUSB_STATUS_SETUP_EP) {
+		usb2_transfer_unsetup(sc->ndisusb_dread_ep.ne_xfer, 1);
+		usb2_transfer_unsetup(sc->ndisusb_dwrite_ep.ne_xfer, 1);
+	}
 	for (i = 0; i < NDISUSB_ENDPT_MAX; i++) {
 		ne = &sc->ndisusb_ep[i];
 		usb2_transfer_unsetup(ne->ne_xfer, 1);
diff --git a/sys/dev/if_ndis/if_ndisvar.h b/sys/dev/if_ndis/if_ndisvar.h
index c86b267..6db48a6 100644
--- a/sys/dev/if_ndis/if_ndisvar.h
+++ b/sys/dev/if_ndis/if_ndisvar.h
@@ -146,6 +146,7 @@ struct ndisusb_task {
 	unsigned		nt_type;
 #define	NDISUSB_TASK_TSTART	0
 #define	NDISUSB_TASK_IRPCANCEL	1
+#define	NDISUSB_TASK_VENDOR	2
 	void			*nt_ctx;
 	list_entry		nt_tasklist;
 };
@@ -229,6 +230,8 @@ struct ndis_softc {
 
 	struct usb2_device	*ndisusb_dev;
 	struct mtx		ndisusb_mtx;
+	struct ndisusb_ep	ndisusb_dread_ep;
+	struct ndisusb_ep	ndisusb_dwrite_ep;
 #define	NDISUSB_GET_ENDPT(addr) \
 	((UE_GET_DIR(addr) >> 7) | (UE_GET_ADDR(addr) << 1))
 #define	NDISUSB_ENDPT_MAX	((UE_ADDR + 1) * 2)
@@ -241,6 +244,7 @@ struct ndis_softc {
 	kspin_lock		ndisusb_tasklock;
 	int			ndisusb_status;
 #define NDISUSB_STATUS_DETACH	0x1
+#define	NDISUSB_STATUS_SETUP_EP	0x2
 };
 
 #define	NDIS_LOCK(_sc)		mtx_lock(&(_sc)->ndis_mtx)
diff --git a/sys/dev/ipmi/ipmi_linux.c b/sys/dev/ipmi/ipmi_linux.c
new file mode 100644
index 0000000..fcf2bd5
--- /dev/null
+++ b/sys/dev/ipmi/ipmi_linux.c
@@ -0,0 +1,113 @@
+/*-
+ * Copyright (c) 2009 IronPort Systems Inc. <ambrisko@ironport.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * Linux ioctl handler for the ipmi device driver
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/conf.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/file.h>
+#include <sys/proc.h>
+#ifdef __amd64__
+#include <machine/../linux32/linux.h>
+#include <machine/../linux32/linux32_proto.h>
+#else
+#include <machine/../linux/linux.h>
+#include <machine/../linux/linux_proto.h>
+#endif
+#include <compat/linux/linux_ioctl.h>
+#include <sys/ioccom.h>
+#include <sys/ipmi.h>
+
+/* There are multiple ioctl number ranges that need to be handled */
+#define IPMI_LINUX_IOCTL_MIN  0x690b
+#define IPMI_LINUX_IOCTL_MAX  0x6915
+
+/* Linux versions of ioctl's */
+#define L_IPMICTL_RECEIVE_MSG_TRUNC       _IOWR(IPMI_IOC_MAGIC, 11, struct ipmi_recv)
+#define L_IPMICTL_RECEIVE_MSG             _IOWR(IPMI_IOC_MAGIC, 12, struct ipmi_recv)
+#define L_IPMICTL_SEND_COMMAND            _IOW(IPMI_IOC_MAGIC, 13, struct ipmi_req)
+#define L_IPMICTL_REGISTER_FOR_CMD        _IOW(IPMI_IOC_MAGIC, 14, struct ipmi_cmdspec)
+#define L_IPMICTL_UNREGISTER_FOR_CMD      _IOW(IPMI_IOC_MAGIC, 15, struct ipmi_cmdspec)
+#define L_IPMICTL_SET_GETS_EVENTS_CMD     _IOW(IPMI_IOC_MAGIC, 16, int)
+#define L_IPMICTL_SET_MY_ADDRESS_CMD      _IOW(IPMI_IOC_MAGIC, 17, unsigned int)
+#define L_IPMICTL_GET_MY_ADDRESS_CMD      _IOW(IPMI_IOC_MAGIC, 18, unsigned int)
+#define L_IPMICTL_SET_MY_LUN_CMD          _IOW(IPMI_IOC_MAGIC, 19, unsigned int)
+#define L_IPMICTL_GET_MY_LUN_CMD          _IOW(IPMI_IOC_MAGIC, 20, unsigned int)
+
+static linux_ioctl_function_t ipmi_linux_ioctl;
+static struct linux_ioctl_handler ipmi_linux_handler = {ipmi_linux_ioctl,
+						       IPMI_LINUX_IOCTL_MIN,
+						       IPMI_LINUX_IOCTL_MAX};
+
+SYSINIT  (ipmi_linux_register,   SI_SUB_KLD, SI_ORDER_MIDDLE,
+	  linux_ioctl_register_handler, &ipmi_linux_handler);
+SYSUNINIT(ipmi_linux_unregister, SI_SUB_KLD, SI_ORDER_MIDDLE,
+	  linux_ioctl_unregister_handler, &ipmi_linux_handler);
+
+static int
+ipmi_linux_modevent(module_t mod, int type, void *data)
+{
+	/* Do we care about any specific load/unload actions? */
+	return (0);
+}
+
+DEV_MODULE(ipmi_linux, ipmi_linux_modevent, NULL);
+MODULE_DEPEND(ipmi_linux, linux, 1, 1, 1);
+
+static int
+ipmi_linux_ioctl(struct thread *td, struct linux_ioctl_args *args)
+{
+	struct file *fp;
+	u_long cmd;
+	int error;
+
+	if ((error = fget(td, args->fd, &fp)) != 0)
+		return (error);
+	cmd = args->cmd;
+
+	switch(cmd) {
+	case L_IPMICTL_GET_MY_ADDRESS_CMD:
+		cmd = IPMICTL_GET_MY_ADDRESS_CMD;
+		break;
+	case L_IPMICTL_GET_MY_LUN_CMD:
+		cmd = IPMICTL_GET_MY_LUN_CMD;
+		break;
+	}
+	/*
+	 * Pass the ioctl off to our standard handler.
+	 */
+	error = (fo_ioctl(fp, cmd, (caddr_t)args->arg, td->td_ucred, td));
+	fdrop(fp, td);
+	return (error);
+}
diff --git a/sys/dev/ipw/if_ipw.c b/sys/dev/ipw/if_ipw.c
index b20d3f6..b9c5c75 100644
--- a/sys/dev/ipw/if_ipw.c
+++ b/sys/dev/ipw/if_ipw.c
@@ -231,6 +231,7 @@ ipw_attach(device_t dev)
 	struct ieee80211_channel *c;
 	uint16_t val;
 	int error, i;
+	uint8_t macaddr[IEEE80211_ADDR_LEN];
 
 	sc->sc_dev = dev;
 
@@ -315,14 +316,14 @@ ipw_attach(device_t dev)
 
 	/* read MAC address from EEPROM */
 	val = ipw_read_prom_word(sc, IPW_EEPROM_MAC + 0);
-	ic->ic_myaddr[0] = val >> 8;
-	ic->ic_myaddr[1] = val & 0xff;
+	macaddr[0] = val >> 8;
+	macaddr[1] = val & 0xff;
 	val = ipw_read_prom_word(sc, IPW_EEPROM_MAC + 1);
-	ic->ic_myaddr[2] = val >> 8;
-	ic->ic_myaddr[3] = val & 0xff;
+	macaddr[2] = val >> 8;
+	macaddr[3] = val & 0xff;
 	val = ipw_read_prom_word(sc, IPW_EEPROM_MAC + 2);
-	ic->ic_myaddr[4] = val >> 8;
-	ic->ic_myaddr[5] = val & 0xff;
+	macaddr[4] = val >> 8;
+	macaddr[5] = val & 0xff;
 
 	/* set supported .11b channels (read from EEPROM) */
 	if ((val = ipw_read_prom_word(sc, IPW_EEPROM_CHANNEL_LIST)) == 0)
@@ -341,7 +342,7 @@ ipw_attach(device_t dev)
 	if (!(ipw_read_prom_word(sc, IPW_EEPROM_RADIO) & 8))
 		sc->flags |= IPW_FLAG_HAS_RADIO_SWITCH;
 
-	ieee80211_ifattach(ic);
+	ieee80211_ifattach(ic, macaddr);
 	ic->ic_scan_start = ipw_scan_start;
 	ic->ic_scan_end = ipw_scan_end;
 	ic->ic_set_channel = ipw_set_channel;
@@ -1794,11 +1795,6 @@ ipw_start_locked(struct ifnet *ifp)
 			break;
 		}
 		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
-		m = ieee80211_encap(ni, m);
-		if (m == NULL) {
-			ieee80211_free_node(ni);
-			continue;
-		}
 		if (ipw_tx_start(ifp, m, ni) != 0) {
 			ieee80211_free_node(ni);
 			ifp->if_oerrors++;
diff --git a/sys/dev/iwi/if_iwi.c b/sys/dev/iwi/if_iwi.c
index 81e0220..83ab3e5 100644
--- a/sys/dev/iwi/if_iwi.c
+++ b/sys/dev/iwi/if_iwi.c
@@ -280,6 +280,7 @@ iwi_attach(device_t dev)
 	uint16_t val;
 	int i, error;
 	uint8_t bands;
+	uint8_t macaddr[IEEE80211_ADDR_LEN];
 
 	sc->sc_dev = dev;
 
@@ -403,14 +404,14 @@ iwi_attach(device_t dev)
 
 	/* read MAC address from EEPROM */
 	val = iwi_read_prom_word(sc, IWI_EEPROM_MAC + 0);
-	ic->ic_myaddr[0] = val & 0xff;
-	ic->ic_myaddr[1] = val >> 8;
+	macaddr[0] = val & 0xff;
+	macaddr[1] = val >> 8;
 	val = iwi_read_prom_word(sc, IWI_EEPROM_MAC + 1);
-	ic->ic_myaddr[2] = val & 0xff;
-	ic->ic_myaddr[3] = val >> 8;
+	macaddr[2] = val & 0xff;
+	macaddr[3] = val >> 8;
 	val = iwi_read_prom_word(sc, IWI_EEPROM_MAC + 2);
-	ic->ic_myaddr[4] = val & 0xff;
-	ic->ic_myaddr[5] = val >> 8;
+	macaddr[4] = val & 0xff;
+	macaddr[5] = val >> 8;
 	
 	bands = 0;
 	setbit(&bands, IEEE80211_MODE_11B);
@@ -419,7 +420,7 @@ iwi_attach(device_t dev)
 		setbit(&bands, IEEE80211_MODE_11A);
 	ieee80211_init_channels(ic, NULL, &bands);
 
-	ieee80211_ifattach(ic);
+	ieee80211_ifattach(ic, macaddr);
 	/* override default methods */
 	ic->ic_node_alloc = iwi_node_alloc;
 	sc->sc_node_free = ic->ic_node_free;
@@ -1981,13 +1982,6 @@ iwi_start_locked(struct ifnet *ifp)
 		BPF_MTAP(ifp, m);
 
 		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
-		m = ieee80211_encap(ni, m);
-		if (m == NULL) {
-			ieee80211_free_node(ni);
-			ifp->if_oerrors++;
-			continue;
-		}
-
 		if (iwi_tx_start(ifp, m, ni, ac) != 0) {
 			ieee80211_free_node(ni);
 			ifp->if_oerrors++;
@@ -2568,9 +2562,8 @@ iwi_config(struct iwi_softc *sc)
 
 	IWI_LOCK_ASSERT(sc);
 
-	IEEE80211_ADDR_COPY(ic->ic_myaddr, IF_LLADDR(ifp));
-	DPRINTF(("Setting MAC address to %6D\n", ic->ic_myaddr, ":"));
-	error = iwi_cmd(sc, IWI_CMD_SET_MAC_ADDRESS, ic->ic_myaddr,
+	DPRINTF(("Setting MAC address to %6D\n", IF_LLADDR(ifp), ":"));
+	error = iwi_cmd(sc, IWI_CMD_SET_MAC_ADDRESS, IF_LLADDR(ifp),
 	    IEEE80211_ADDR_LEN);
 	if (error != 0)
 		return error;
diff --git a/sys/dev/iwn/if_iwn.c b/sys/dev/iwn/if_iwn.c
index 21d17f1..2928c0f 100644
--- a/sys/dev/iwn/if_iwn.c
+++ b/sys/dev/iwn/if_iwn.c
@@ -132,7 +132,8 @@ void		iwn_cmd_intr(struct iwn_softc *, struct iwn_rx_desc *);
 static void	iwn_bmiss(void *, int);
 void		iwn_notif_intr(struct iwn_softc *);
 void		iwn_intr(void *);
-void		iwn_read_eeprom(struct iwn_softc *);
+void		iwn_read_eeprom(struct iwn_softc *,
+		    uint8_t macaddr[IEEE80211_ADDR_LEN]);
 static void	iwn_read_eeprom_channels(struct iwn_softc *);
 void		iwn_print_power_group(struct iwn_softc *, int);
 uint8_t		iwn_plcp_signal(int);
@@ -254,6 +255,7 @@ iwn_attach(device_t dev)
 	struct ieee80211com *ic;
 	struct ifnet *ifp;
 	int i, error, result;
+	uint8_t macaddr[IEEE80211_ADDR_LEN];
 
 	sc->sc_dev = dev;
 
@@ -410,7 +412,7 @@ iwn_attach(device_t dev)
 		;
 #endif
 	/* read supported channels and MAC address from EEPROM */
-	iwn_read_eeprom(sc);
+	iwn_read_eeprom(sc, macaddr);
 
 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 	ifp->if_softc = sc;
@@ -422,7 +424,7 @@ iwn_attach(device_t dev)
 	ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN;
 	IFQ_SET_READY(&ifp->if_snd);
 
-	ieee80211_ifattach(ic);
+	ieee80211_ifattach(ic, macaddr);
 	ic->ic_vap_create = iwn_vap_create;
 	ic->ic_vap_delete = iwn_vap_delete;
 	ic->ic_raw_xmit = iwn_raw_xmit;
@@ -2120,12 +2122,6 @@ iwn_start_locked(struct ifnet *ifp)
 		ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
 		pri = M_WME_GETAC(m);
 		txq = &sc->txq[pri];
-		m = ieee80211_encap(ni, m);
-		if (m == NULL) {
-			ifp->if_oerrors++;
-			ieee80211_free_node(ni);
-			continue;
-		}
 		if (txq->queued >= IWN_TX_RING_COUNT - 8) {
 			/* XXX not right */
 			/* ring is nearly full, stop flow */
@@ -2134,7 +2130,6 @@ iwn_start_locked(struct ifnet *ifp)
 		if (iwn_tx_data(sc, m, ni, txq) != 0) {
 			ifp->if_oerrors++;
 			ieee80211_free_node(ni);
-			IWN_UNLOCK(sc);
 			break;
 		}
 	}
@@ -2410,10 +2405,8 @@ iwn_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 }
 
 void
-iwn_read_eeprom(struct iwn_softc *sc)
+iwn_read_eeprom(struct iwn_softc *sc, uint8_t macaddr[IEEE80211_ADDR_LEN])
 {
-	struct ifnet *ifp = sc->sc_ifp;
-	struct ieee80211com *ic = ifp->if_l2com;
 	char domain[4];
 	uint16_t val;
 	int i, error;
@@ -2428,8 +2421,8 @@ iwn_read_eeprom(struct iwn_softc *sc)
 	device_printf(sc->sc_dev,"Reg Domain: %.4s", domain);
 
 	/* read and print MAC address */
-	iwn_read_prom_data(sc, IWN_EEPROM_MAC, ic->ic_myaddr, 6);
-	printf(", address %s\n", ether_sprintf(ic->ic_myaddr));
+	iwn_read_prom_data(sc, IWN_EEPROM_MAC, macaddr, 6);
+	printf(", address %6D\n", macaddr, ":");
 
 	/* read the list of authorized channels */
 	iwn_read_eeprom_channels(sc);
@@ -3755,7 +3748,7 @@ iwn_scan(struct iwn_softc *sc)
 	    IEEE80211_FC0_SUBTYPE_PROBE_REQ;
 	wh->i_fc[1] = IEEE80211_FC1_DIR_NODS;
 	IEEE80211_ADDR_COPY(wh->i_addr1, ifp->if_broadcastaddr);
-	IEEE80211_ADDR_COPY(wh->i_addr2, ic->ic_myaddr);
+	IEEE80211_ADDR_COPY(wh->i_addr2, IF_LLADDR(ifp));
 	IEEE80211_ADDR_COPY(wh->i_addr3, ifp->if_broadcastaddr);
 	*(u_int16_t *)&wh->i_dur[0] = 0;	/* filled by h/w */
 	*(u_int16_t *)&wh->i_seq[0] = 0;	/* filled by h/w */
@@ -3893,8 +3886,8 @@ iwn_config(struct iwn_softc *sc)
 
 	/* configure adapter */
 	memset(&sc->config, 0, sizeof (struct iwn_config));
-	IEEE80211_ADDR_COPY(sc->config.myaddr, ic->ic_myaddr);
-	IEEE80211_ADDR_COPY(sc->config.wlap, ic->ic_myaddr);
+	IEEE80211_ADDR_COPY(sc->config.myaddr, IF_LLADDR(ifp));
+	IEEE80211_ADDR_COPY(sc->config.wlap, IF_LLADDR(ifp));
 	/* set default channel */
 	sc->config.chan = htole16(ieee80211_chan2ieee(ic, ic->ic_curchan));
 	sc->config.flags = htole32(IWN_CONFIG_TSF);
diff --git a/sys/dev/ixgbe/LICENSE b/sys/dev/ixgbe/LICENSE
index b6fd3b7..39264e0 100644
--- a/sys/dev/ixgbe/LICENSE
+++ b/sys/dev/ixgbe/LICENSE
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
diff --git a/sys/dev/ixgbe/README b/sys/dev/ixgbe/README
index 40bbe5f..0b27deb 100644
--- a/sys/dev/ixgbe/README
+++ b/sys/dev/ixgbe/README
@@ -1,6 +1,8 @@
 FreeBSD Driver for 10 Gigabit PCI Express Server Adapters
 =============================================
-$FreeBSD$
+/*$FreeBSD$*/
+
+May 14, 2008
 
 
 Contents
@@ -258,7 +260,7 @@ For general information and support, go to the Intel support website at:
 
 If an issue is identified with the released source code on the supported
 kernel with a supported adapter, email the specific information related to 
-the issue to freebsdnic@mailbox.intel.com.
+the issue to freebsd@intel.com.
 
 
 
diff --git a/sys/dev/ixgbe/ixgbe.c b/sys/dev/ixgbe/ixgbe.c
index 3b71cb6..e2827f4 100644
--- a/sys/dev/ixgbe/ixgbe.c
+++ b/sys/dev/ixgbe/ixgbe.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -46,7 +46,7 @@ int             ixgbe_display_debug_stats = 0;
 /*********************************************************************
  *  Driver version
  *********************************************************************/
-char ixgbe_driver_version[] = "1.6.2";
+char ixgbe_driver_version[] = "1.7.4";
 
 /*********************************************************************
  *  PCI Device ID Table
@@ -71,6 +71,8 @@ static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
+	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
+	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
 	/* required last entry */
 	{0, 0, 0, 0, 0}
 };
@@ -104,7 +106,9 @@ static int      ixgbe_allocate_pci_resources(struct adapter *);
 static int      ixgbe_allocate_msix(struct adapter *);
 static int      ixgbe_allocate_legacy(struct adapter *);
 static int	ixgbe_allocate_queues(struct adapter *);
+#if __FreeBSD_version >= 602105
 static int	ixgbe_setup_msix(struct adapter *);
+#endif
 static void	ixgbe_free_pci_resources(struct adapter *);
 static void     ixgbe_local_timer(void *);
 static int      ixgbe_hardware_init(struct adapter *);
@@ -124,6 +128,7 @@ static void     ixgbe_initialize_receive_units(struct adapter *);
 static void     ixgbe_free_receive_structures(struct adapter *);
 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 
+static void	ixgbe_init_moderation(struct adapter *);
 static void     ixgbe_enable_intr(struct adapter *);
 static void     ixgbe_disable_intr(struct adapter *);
 static void     ixgbe_update_stats_counters(struct adapter *);
@@ -146,7 +151,7 @@ static int	ixgbe_dma_malloc(struct adapter *, bus_size_t,
 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 static void	ixgbe_add_rx_process_limit(struct adapter *, const char *,
 		    const char *, int *, int);
-static boolean_t ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
+static int	ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 static boolean_t ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
 static void	ixgbe_set_ivar(struct adapter *, u16, u8, s8);
 static void	ixgbe_configure_ivars(struct adapter *);
@@ -157,6 +162,12 @@ static void	ixgbe_register_vlan(void *, struct ifnet *, u16);
 static void	ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 #endif
 
+#ifdef IXGBE_TIMESYNC
+/* Precision Time sync support */
+static int ixgbe_tsync_init(struct adapter *);
+static void ixgbe_tsync_disable(struct adapter *);
+#endif
+
 static void	ixgbe_update_aim(struct rx_ring *);
 
 /* Support for pluggable optic modules */
@@ -165,14 +176,19 @@ static bool	ixgbe_sfp_probe(struct adapter *);
 /* Legacy (single vector interrupt handler */
 static void	ixgbe_legacy_irq(void *);
 
+#if __FreeBSD_version >= 602105
 /* The MSI/X Interrupt handlers */
 static void	ixgbe_msix_tx(void *);
 static void	ixgbe_msix_rx(void *);
 static void	ixgbe_msix_link(void *);
+#endif
 
-/* Legacy interrupts use deferred handlers */
-static void	ixgbe_handle_tx(void *context, int pending);
-static void	ixgbe_handle_rx(void *context, int pending);
+/* Deferred interrupt tasklets */
+static void	ixgbe_handle_tx(void *, int);
+static void	ixgbe_handle_rx(void *, int);
+static void	ixgbe_handle_link(void *, int);
+static void	ixgbe_handle_msf(void *, int);
+static void	ixgbe_handle_mod(void *, int);
 
 
 /*********************************************************************
@@ -213,8 +229,8 @@ static int ixgbe_enable_aim = TRUE;
 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 static int ixgbe_low_latency = IXGBE_LOW_LATENCY;
 TUNABLE_INT("hw.ixgbe.low_latency", &ixgbe_low_latency);
-static int ixgbe_ave_latency = IXGBE_LOW_LATENCY;
-TUNABLE_INT("hw.ixgbe.ave_latency", &ixgbe_low_latency);
+static int ixgbe_ave_latency = IXGBE_AVE_LATENCY;
+TUNABLE_INT("hw.ixgbe.ave_latency", &ixgbe_ave_latency);
 static int ixgbe_bulk_latency = IXGBE_BULK_LATENCY;
 TUNABLE_INT("hw.ixgbe.bulk_latency", &ixgbe_bulk_latency);
 
@@ -222,8 +238,8 @@ TUNABLE_INT("hw.ixgbe.bulk_latency", &ixgbe_bulk_latency);
 static int ixgbe_rx_process_limit = 100;
 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 
-/* Flow control setting, default to full */
-static int ixgbe_flow_control = ixgbe_fc_none;
+/* Flow control setting, default to off */
+static int ixgbe_flow_control = ixgbe_fc_full;
 TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control);
 
 /*
@@ -239,11 +255,16 @@ TUNABLE_INT("hw.ixgbe.enable_lro", &ixgbe_enable_lro);
  * MSIX should be the default for best performance,
  * but this allows it to be forced off for testing.
  */
+#if __FreeBSD_version >= 602105
 static int ixgbe_enable_msix = 1;
+#else
+static int ixgbe_enable_msix = 0;
+#endif
 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 
 /*
  * Enable RX Header Split
+ *   WARNING: disable this if bridging or forwarding!!
  */
 static int ixgbe_rx_hdr_split = 1;
 TUNABLE_INT("hw.ixgbe.rx_hdr_split", &ixgbe_rx_hdr_split);
@@ -268,6 +289,13 @@ TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 /* Total number of Interfaces - need for config sanity check */
 static int ixgbe_total_ports;
 
+/*
+** The number of scatter-gather segments
+** differs for 82598 and 82599, default to
+** the former.
+*/
+static int ixgbe_num_segs = IXGBE_82598_SCATTER;
+
 /*********************************************************************
  *  Device identification routine
  *
@@ -312,6 +340,7 @@ ixgbe_probe(device_t dev)
 				ixgbe_strings[ent->index],
 				ixgbe_driver_version);
 			device_set_desc_copy(dev, adapter_name);
+			++ixgbe_total_ports;
 			return (0);
 		}
 		ent++;
@@ -333,6 +362,7 @@ static int
 ixgbe_attach(device_t dev)
 {
 	struct adapter *adapter;
+	struct ixgbe_hw *hw;
 	int             error = 0;
 	u16		pci_device_id;
 	u32		ctrl_ext;
@@ -342,37 +372,34 @@ ixgbe_attach(device_t dev)
 	/* Allocate, clear, and link in our adapter structure */
 	adapter = device_get_softc(dev);
 	adapter->dev = adapter->osdep.dev = dev;
+	hw = &adapter->hw;
 
 	/* Core Lock Init*/
 	IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 
-	/* Keep track of number of ports and optics */
+	/* Keep track of optics */
 	pci_device_id = pci_get_device(dev);
 	switch (pci_device_id) {
 		case IXGBE_DEV_ID_82598_CX4_DUAL_PORT :
+		case IXGBE_DEV_ID_82598EB_CX4 :
 			adapter->optics = IFM_10G_CX4;
-			ixgbe_total_ports += 2;
 			break;
 		case IXGBE_DEV_ID_82598AF_DUAL_PORT :
-			adapter->optics = IFM_10G_SR;
-			ixgbe_total_ports += 2;
-			break;
+		case IXGBE_DEV_ID_82598_DA_DUAL_PORT :
 		case IXGBE_DEV_ID_82598AF_SINGLE_PORT :
+		case IXGBE_DEV_ID_82598AT :
 			adapter->optics = IFM_10G_SR;
-			ixgbe_total_ports += 1;
 			break;
 		case IXGBE_DEV_ID_82598EB_XF_LR :
 			adapter->optics = IFM_10G_LR;
-			ixgbe_total_ports += 1;
 			break;
-		case IXGBE_DEV_ID_82598EB_CX4 :
-			adapter->optics = IFM_10G_CX4;
-			ixgbe_total_ports += 1;
+		case IXGBE_DEV_ID_82599_SFP :
+			adapter->optics = IFM_10G_SR;
+			ixgbe_num_segs = IXGBE_82599_SCATTER;
 			break;
-		case IXGBE_DEV_ID_82598AT :
-			ixgbe_total_ports += 1;
-		case IXGBE_DEV_ID_82598_DA_DUAL_PORT :
-			ixgbe_total_ports += 2;
+		case IXGBE_DEV_ID_82599_KX4 :
+			adapter->optics = IFM_10G_CX4;
+			ixgbe_num_segs = IXGBE_82599_SCATTER;
 		default:
 			break;
 	}
@@ -474,7 +501,7 @@ ixgbe_attach(device_t dev)
 	}
 
 	/* Initialize the shared code */
-	error = ixgbe_init_shared_code(&adapter->hw);
+	error = ixgbe_init_shared_code(hw);
 	if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 		/*
 		** No optics in this port, set up
@@ -525,11 +552,11 @@ ixgbe_attach(device_t dev)
 	adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 	    ixgbe_unregister_vlan, 0, EVENTHANDLER_PRI_FIRST);
 #endif
-                
+
 	/* let hardware know driver is loaded */
-	ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
+	ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 	ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
-	IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
+	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 
 	INIT_DEBUGOUT("ixgbe_attach: end");
 	return (0);
@@ -590,6 +617,14 @@ ixgbe_detach(device_t dev)
 		}
 	}
 
+	/* Drain the Link queue */
+	if (adapter->tq) {
+		taskqueue_drain(adapter->tq, &adapter->link_task);
+		taskqueue_drain(adapter->tq, &adapter->mod_task);
+		taskqueue_drain(adapter->tq, &adapter->msf_task);
+		taskqueue_free(adapter->tq);
+	}
+
 	/* let hardware know driver is unloading */
 	ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 	ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
@@ -662,6 +697,19 @@ ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 		IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 		if (m_head == NULL)
 			break;
+		/*
+		 * Force a cleanup if number of TX descriptors
+		 * available is below the threshold. If it fails
+		 * to get above, then abort transmit.
+		 */
+		if (txr->tx_avail <= IXGBE_TX_CLEANUP_THRESHOLD) {
+			ixgbe_txeof(txr);
+			/* Make sure things have improved */
+			if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) {
+				txr->no_tx_desc_avail++;
+				break;
+			}
+		}
 
 		if (ixgbe_xmit(txr, &m_head)) {
 			if (m_head == NULL)
@@ -701,7 +749,8 @@ ixgbe_start(struct ifnet *ifp)
 	txr = &adapter->tx_rings[queue];
 
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
-		IXGBE_TX_LOCK(txr);
+		if (IXGBE_TX_TRYLOCK(txr) == 0)
+			return;
 		ixgbe_start_locked(txr, ifp);
 		IXGBE_TX_UNLOCK(txr);
 	}
@@ -803,6 +852,67 @@ ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
 #endif
 		break;
 	}
+#ifdef IXGBE_TIMESYNC
+	/*
+	** IOCTL support for Precision Time (IEEE 1588) Support
+	*/
+	case IXGBE_TIMESYNC_READTS:
+	    {
+		u32 rx_ctl, tx_ctl;
+		struct ixgbe_tsync_read *tdata;
+
+		tdata = (struct ixgbe_tsync_read *) ifr->ifr_data;
+
+		if (tdata->read_current_time) {
+                        getnanotime(&tdata->system_time);
+                        tdata->network_time = IXGBE_READ_REG(&adapter->hw,
+                            IXGBE_SYSTIML);
+                        tdata->network_time |=
+                            (u64)IXGBE_READ_REG(&adapter->hw,
+                            IXGBE_SYSTIMH ) << 32;
+		}
+  
+		rx_ctl = IXGBE_READ_REG(&adapter->hw, IXGBE_TSYNCRXCTL);
+		tx_ctl = IXGBE_READ_REG(&adapter->hw, IXGBE_TSYNCTXCTL);
+ 
+		if (rx_ctl & 0x1) {
+			u32 tmp;
+			unsigned char *tmp_cp;
+
+			tdata->rx_valid = 1;
+			tdata->rx_stamp = IXGBE_READ_REG(&adapter->hw,
+			    IXGBE_RXSTMPL);
+			tdata->rx_stamp |= (u64)IXGBE_READ_REG(&adapter->hw,
+                            IXGBE_RXSTMPH) << 32;
+
+			tmp = IXGBE_READ_REG(&adapter->hw, IXGBE_RXSATRL);
+			tmp_cp = (unsigned char *) &tmp;
+			tdata->srcid[0] = tmp_cp[0];
+			tdata->srcid[1] = tmp_cp[1];
+			tdata->srcid[2] = tmp_cp[2];
+			tdata->srcid[3] = tmp_cp[3];
+			tmp = IXGBE_READ_REG(&adapter->hw, IXGBE_RXSATRH);
+			tmp_cp = (unsigned char *) &tmp;
+			tdata->srcid[4] = tmp_cp[0];
+			tdata->srcid[5] = tmp_cp[1];
+			tdata->seqid = tmp >> 16;
+			tdata->seqid = htons(tdata->seqid);
+		} else
+                        tdata->rx_valid = 0;
+
+		if (tx_ctl & 0x1) {
+			tdata->tx_valid = 1;
+			tdata->tx_stamp = IXGBE_READ_REG(&adapter->hw,
+			    IXGBE_TXSTMPL);
+			tdata->tx_stamp |= (u64) IXGBE_READ_REG(&adapter->hw,
+			    IXGBE_TXSTMPH) << 32;
+		} else
+			tdata->tx_valid = 0;
+
+		return (0);
+	    }
+#endif  /* IXGBE_TIMESYNC */
+
 	default:
 		IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
 		error = ether_ioctl(ifp, command, data);
@@ -905,13 +1015,12 @@ ixgbe_watchdog(struct adapter *adapter)
 static void
 ixgbe_init_locked(struct adapter *adapter)
 {
-	struct rx_ring *rxr = adapter->rx_rings;
-	struct tx_ring *txr = adapter->tx_rings;
 	struct ifnet   *ifp = adapter->ifp;
 	device_t 	dev = adapter->dev;
 	struct ixgbe_hw *hw;
 	u32		k, txdctl, mhadd, gpie;
 	u32		rxdctl, rxctrl;
+	int		err;
 
 	INIT_DEBUGOUT("ixgbe_init: begin");
 
@@ -951,13 +1060,6 @@ ixgbe_init_locked(struct adapter *adapter)
 
 	ixgbe_initialize_transmit_units(adapter);
 
-	/* TX irq moderation rate is fixed */
-	for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
-		IXGBE_WRITE_REG(&adapter->hw,
-		    IXGBE_EITR(txr->msix), ixgbe_ave_latency);
-		txr->watchdog_timer = FALSE;
-	}
-
 	/* Setup Multicast table */
 	ixgbe_set_multi(adapter);
 
@@ -980,23 +1082,21 @@ ixgbe_init_locked(struct adapter *adapter)
 	/* Configure RX settings */
 	ixgbe_initialize_receive_units(adapter);
 
-	/* RX moderation will be adapted over time, set default */
-	for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) {
-		IXGBE_WRITE_REG(&adapter->hw,
-		    IXGBE_EITR(rxr->msix), ixgbe_low_latency);
-	}
-
-	/* Set Link moderation */
-	IXGBE_WRITE_REG(&adapter->hw,
-	    IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
+	/* Configure Interrupt Moderation */
+	ixgbe_init_moderation(adapter);
 
 	gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
 
+	if (adapter->hw.mac.type == ixgbe_mac_82599EB) {
+		gpie |= IXGBE_SDP1_GPIEN;
+		gpie |= IXGBE_SDP2_GPIEN;
+	}
+
 	/* Enable Fan Failure Interrupt */
-	if (adapter->hw.phy.media_type == ixgbe_media_type_copper)
+	if (hw->device_id == IXGBE_DEV_ID_82598AT)
 		gpie |= IXGBE_SDP1_GPIEN;
 
-	if (adapter->msix) {
+	if (adapter->msix > 2) {
 		/* Enable Enhanced MSIX mode */
 		gpie |= IXGBE_GPIE_MSIX_MODE;
 		gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
@@ -1058,9 +1158,40 @@ ixgbe_init_locked(struct adapter *adapter)
 	/* Set up MSI/X routing */
 	if (ixgbe_enable_msix)
 		ixgbe_configure_ivars(adapter);
+	else {	/* Simple settings for Legacy/MSI */
+                ixgbe_set_ivar(adapter, 0, 0, 0);
+                ixgbe_set_ivar(adapter, 0, 0, 1);
+	}
 
 	ixgbe_enable_intr(adapter);
 
+	/*
+	** Check on any SFP devices that
+	** need to be kick-started
+	*/
+	err = hw->phy.ops.identify(hw);
+	if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
+                device_printf(dev,
+		    "Unsupported SFP+ module type was detected.\n");
+		ixgbe_detach(dev);
+		return;
+        }
+	if (ixgbe_is_sfp(hw)) { 
+		if (hw->phy.multispeed_fiber) {
+			hw->mac.ops.setup_sfp(hw);
+			taskqueue_enqueue(adapter->tq, &adapter->msf_task);
+		} else
+			taskqueue_enqueue(adapter->tq, &adapter->mod_task);
+	} else
+		taskqueue_enqueue(adapter->tq, &adapter->link_task);
+
+
+#ifdef IXGBE_TIMESYNC
+	/* Initialize IEEE 1588 support */
+	if (adapter->hw.mac.type == ixgbe_mac_82599EB)
+		ixgbe_tsync_init(adapter);
+#endif
+
 	/* Now inform the stack we're ready */
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
@@ -1081,7 +1212,7 @@ ixgbe_init(void *arg)
 
 
 /*
-** MSIX Interrupt Handlers
+** MSIX Interrupt Tasklets
 */
 
 static void
@@ -1136,7 +1267,8 @@ ixgbe_legacy_irq(void *arg)
 	struct ixgbe_hw	*hw = &adapter->hw;
 	struct 		tx_ring *txr = adapter->tx_rings;
 	struct		rx_ring *rxr = adapter->rx_rings;
-	u32       	reg_eicr;
+	bool		more;
+	u32       	reg_eicr, loop = MAX_LOOP;
 
 
 	reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
@@ -1148,7 +1280,15 @@ ixgbe_legacy_irq(void *arg)
 
 	if (ixgbe_rxeof(rxr, adapter->rx_process_limit))
 		taskqueue_enqueue(rxr->tq, &rxr->rx_task);
-	if (ixgbe_txeof(txr))
+
+	IXGBE_TX_LOCK(txr);
+	++txr->tx_irq;
+	do {
+		more = ixgbe_txeof(txr);
+	} while (loop-- && more);
+	IXGBE_TX_UNLOCK(txr);
+
+	if (more)
 		taskqueue_enqueue(txr->tq, &txr->tx_task);
 
 	/* Check for fan failure */
@@ -1160,20 +1300,27 @@ ixgbe_legacy_irq(void *arg)
 	}
 
 	/* Link status change */
-	if (reg_eicr & IXGBE_EICR_LSC)
+	if (reg_eicr & IXGBE_EICR_LSC) {
+		ixgbe_check_link(&adapter->hw,
+		    &adapter->link_speed, &adapter->link_up, 0);
         	ixgbe_update_link_status(adapter);
+	}
+
+	/* Update interrupt rate */
+	if (ixgbe_enable_aim == TRUE)
+		ixgbe_update_aim(rxr);
 
 	ixgbe_enable_intr(adapter);
 	return;
 }
 
 
+#if __FreeBSD_version >= 602105
 /*********************************************************************
  *
  *  MSI TX Interrupt Service routine
  *
  **********************************************************************/
-
 void
 ixgbe_msix_tx(void *arg)
 {
@@ -1181,6 +1328,8 @@ ixgbe_msix_tx(void *arg)
 	struct adapter  *adapter = txr->adapter;
 	bool		more;
 
+	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, txr->eims);
+
 	IXGBE_TX_LOCK(txr);
 	++txr->tx_irq;
 	more = ixgbe_txeof(txr);
@@ -1198,7 +1347,6 @@ ixgbe_msix_tx(void *arg)
  *  MSIX RX Interrupt Service routine
  *
  **********************************************************************/
-
 static void
 ixgbe_msix_rx(void *arg)
 {
@@ -1206,18 +1354,72 @@ ixgbe_msix_rx(void *arg)
 	struct adapter  *adapter = rxr->adapter;
 	bool		more;
 
+       	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, rxr->eims);
+
 	++rxr->rx_irq;
-	more = ixgbe_rxeof(rxr, -1);
+	more = ixgbe_rxeof(rxr, adapter->rx_process_limit);
+
+	/* Update interrupt rate */
+	if (ixgbe_enable_aim == TRUE)
+		ixgbe_update_aim(rxr);
+
 	if (more)
 		taskqueue_enqueue(rxr->tq, &rxr->rx_task);
 	else
         	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, rxr->eims);
-	/* Update interrupt rate */
-	if (ixgbe_enable_aim == TRUE)
-		ixgbe_update_aim(rxr);
 	return;
 }
 
+
+static void
+ixgbe_msix_link(void *arg)
+{
+	struct adapter	*adapter = arg;
+	struct ixgbe_hw *hw = &adapter->hw;
+	u32		reg_eicr;
+
+	++adapter->link_irq;
+
+	/* First get the cause */
+	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, IXGBE_EIMS_OTHER);
+	reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
+	/* Clear with write */
+	IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
+
+	/* Link status change */
+	if (reg_eicr & IXGBE_EICR_LSC)
+		taskqueue_enqueue(adapter->tq, &adapter->link_task);
+
+	if (adapter->hw.mac.type == ixgbe_mac_82599EB) {
+		if (reg_eicr & IXGBE_EICR_ECC) {
+                	device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
+			    "Please Reboot!!\n");
+			IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
+		}
+		if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
+                	/* Clear the interrupt */
+                	IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
+			taskqueue_enqueue(adapter->tq, &adapter->msf_task);
+        	} else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
+                	/* Clear the interrupt */
+                	IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
+			taskqueue_enqueue(adapter->tq, &adapter->mod_task);
+		}
+        } 
+
+	/* Check for fan failure */
+	if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
+	    (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
+                device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
+		    "REPLACE IMMEDIATELY!!\n");
+		IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
+	}
+
+	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
+	return;
+}
+#endif /* FreeBSD_version >= 602105 */
+
 /*
 ** Routine to do adjust the RX EITR value based on traffic,
 ** its a simple three state model, but seems to help.
@@ -1267,33 +1469,36 @@ ixgbe_update_aim(struct rx_ring *rxr)
 	return;
 }
 
-
 static void
-ixgbe_msix_link(void *arg)
+ixgbe_init_moderation(struct adapter *adapter)
 {
-	struct adapter	*adapter = arg;
-	struct ixgbe_hw *hw = &adapter->hw;
-	u32		reg_eicr;
-
-	++adapter->link_irq;
+	struct rx_ring *rxr = adapter->rx_rings;
+	struct tx_ring *txr = adapter->tx_rings;
 
-	reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
+	/* Single interrupt - MSI or Legacy? */
+	if (adapter->msix < 2) {
+		IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(0), 100);
+		return;
+	}
 
-	if (reg_eicr & IXGBE_EICR_LSC)
-        	ixgbe_update_link_status(adapter);
+	/* TX irq moderation rate is fixed */
+	for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
+		IXGBE_WRITE_REG(&adapter->hw,
+		    IXGBE_EITR(txr->msix), ixgbe_ave_latency);
+		txr->watchdog_timer = FALSE;
+	}
 
-	/* Check for fan failure */
-	if ((hw->phy.media_type == ixgbe_media_type_copper) &&
-	    (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
-                device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
-		    "REPLACE IMMEDIATELY!!\n");
-		IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
+	/* RX moderation will be adapted over time, set default */
+	for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) {
+		IXGBE_WRITE_REG(&adapter->hw,
+		    IXGBE_EITR(rxr->msix), ixgbe_low_latency);
 	}
 
-	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
-	return;
-}
+	/* Set Link moderation */
+	IXGBE_WRITE_REG(&adapter->hw,
+	    IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
 
+}
 
 /*********************************************************************
  *
@@ -1382,12 +1587,12 @@ static int
 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
 {
 	struct adapter  *adapter = txr->adapter;
-	u32		olinfo_status = 0, cmd_type_len = 0;
+	u32		olinfo_status = 0, cmd_type_len;
 	u32		paylen = 0;
 	int             i, j, error, nsegs;
-	int		first, last = 0;
+	int		first, last = 0, offload = 0;
 	struct mbuf	*m_head;
-	bus_dma_segment_t segs[IXGBE_MAX_SCATTER];
+	bus_dma_segment_t segs[ixgbe_num_segs];
 	bus_dmamap_t	map;
 	struct ixgbe_tx_buf *txbuf, *txbuf_mapped;
 	union ixgbe_adv_tx_desc *txd = NULL;
@@ -1395,26 +1600,12 @@ ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
 	m_head = *m_headp;
 
 	/* Basic descriptor defines */
-        cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA;
-        cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
+        cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
+	    IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
 
 	if (m_head->m_flags & M_VLANTAG)
         	cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
 
-	/*
-	 * Force a cleanup if number of TX descriptors
-	 * available is below the threshold. If it fails
-	 * to get above, then abort transmit.
-	 */
-	if (txr->tx_avail <= IXGBE_TX_CLEANUP_THRESHOLD) {
-		ixgbe_txeof(txr);
-		/* Make sure things have improved */
-		if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) {
-			txr->no_tx_desc_avail++;
-			return (ENOBUFS);
-		}
-	}
-
         /*
          * Important to capture the first descriptor
          * used because it will contain the index of
@@ -1475,19 +1666,27 @@ ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
 	m_head = *m_headp;
 
 	/*
-	** Set the appropriate offload context
+	** Set up the appropriate offload context
 	** this becomes the first descriptor of 
 	** a packet.
 	*/
-	if (ixgbe_tso_setup(txr, m_head, &paylen)) {
-		cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
-		olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
-		olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
-		olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
-		++adapter->tso_tx;
-	} else if (ixgbe_tx_ctx_setup(txr, m_head))
+	if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
+		if (ixgbe_tso_setup(txr, m_head, &paylen)) {
+			cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
+			olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
 			olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
-
+			olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
+			++adapter->tso_tx;
+		} else
+			return (ENXIO);
+	} else /* Offloads other than TSO */
+		offload = ixgbe_tx_ctx_setup(txr, m_head);
+	if (offload == TRUE)
+		olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
+#ifdef IXGBE_TIMESYNC
+	if (offload == IXGBE_TIMESTAMP)
+		cmd_type_len |= IXGBE_ADVTXD_TSTAMP;
+#endif
         /* Record payload length */
 	if (paylen == 0)
         	olinfo_status |= m_head->m_pkthdr.len <<
@@ -1513,6 +1712,7 @@ ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
 			i = 0;
 
 		txbuf->m_head = NULL;
+		txbuf->eop_index = -1;
 	}
 
 	txd->read.cmd_type_len |=
@@ -1526,6 +1726,7 @@ ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
 
         /* Set the index of the descriptor that will be marked done */
         txbuf = &txr->tx_buffers[first];
+	txbuf->eop_index = last;
 
         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
@@ -1685,17 +1886,20 @@ out:
 	callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
 }
 
+/*
+** Note: this routine updates the OS on the link state
+**	the real check of the hardware only happens with
+**	a link interrupt.
+*/
 static void
 ixgbe_update_link_status(struct adapter *adapter)
 {
-	boolean_t link_up = FALSE;
 	struct ifnet	*ifp = adapter->ifp;
 	struct tx_ring *txr = adapter->tx_rings;
 	device_t dev = adapter->dev;
 
-	ixgbe_check_link(&adapter->hw, &adapter->link_speed, &link_up, 0);
 
-	if (link_up){ 
+	if (adapter->link_up){ 
 		if (adapter->link_active == FALSE) {
 			if (bootverbose)
 				device_printf(dev,"Link is up %d Gbps %s \n",
@@ -1720,7 +1924,6 @@ ixgbe_update_link_status(struct adapter *adapter)
 }
 
 
-
 /*********************************************************************
  *
  *  This routine disables all traffic on the adapter by issuing a
@@ -1743,6 +1946,11 @@ ixgbe_stop(void *arg)
 	/* Tell the stack that the interface is no longer active */
 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 
+#ifdef IXGBE_TIMESYNC
+	/* Disable IEEE 1588 support */
+	if (adapter->hw.mac.type == ixgbe_mac_82599EB)
+		ixgbe_tsync_disable(adapter);
+#endif
 	ixgbe_reset_hw(&adapter->hw);
 	adapter->hw.adapter_stopped = FALSE;
 	ixgbe_stop_adapter(&adapter->hw);
@@ -1817,8 +2025,13 @@ ixgbe_allocate_legacy(struct adapter *adapter)
             device_get_nameunit(adapter->dev));
 	taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq",
             device_get_nameunit(adapter->dev));
+
 	if ((error = bus_setup_intr(dev, adapter->res[0],
+#if __FreeBSD_version >= 700000
             INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq,
+#else
+            INTR_TYPE_NET | INTR_MPSAFE, ixgbe_legacy_irq,
+#endif
             adapter, &adapter->tag[0])) != 0) {
 		device_printf(dev, "Failed to register fast interrupt "
 		    "handler: %d\n", error);
@@ -1833,6 +2046,7 @@ ixgbe_allocate_legacy(struct adapter *adapter)
 }
 
 
+#if __FreeBSD_version >= 602105
 /*********************************************************************
  *
  *  Setup MSIX Interrupt resources and handlers 
@@ -1859,7 +2073,10 @@ ixgbe_allocate_msix(struct adapter *adapter)
 		}
 		/* Set the handler function */
 		error = bus_setup_intr(dev, adapter->res[vector],
-		    INTR_TYPE_NET | INTR_MPSAFE, NULL,
+		    INTR_TYPE_NET | INTR_MPSAFE,
+#if __FreeBSD_version > 700000
+		    NULL,
+#endif
 		    ixgbe_msix_tx, txr, &adapter->tag[vector]);
 		if (error) {
 			adapter->res[vector] = NULL;
@@ -1867,7 +2084,7 @@ ixgbe_allocate_msix(struct adapter *adapter)
 			return (error);
 		}
 		txr->msix = vector;
-		txr->eims = IXGBE_IVAR_TX_QUEUE(vector);
+		txr->eims = 1 << vector;
 		TASK_INIT(&txr->tx_task, 0, ixgbe_handle_tx, txr);
 		txr->tq = taskqueue_create_fast("ixgbe_txq", M_NOWAIT,
 		    taskqueue_thread_enqueue, &txr->tq);
@@ -1888,15 +2105,18 @@ ixgbe_allocate_msix(struct adapter *adapter)
 		}
 		/* Set the handler function */
 		error = bus_setup_intr(dev, adapter->res[vector],
-		    INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_msix_rx,
-		    rxr, &adapter->tag[vector]);
+		    INTR_TYPE_NET | INTR_MPSAFE,
+#if __FreeBSD_version > 700000
+		    NULL,
+#endif
+		    ixgbe_msix_rx, rxr, &adapter->tag[vector]);
 		if (error) {
 			adapter->res[vector] = NULL;
 			device_printf(dev, "Failed to register RX handler");
 			return (error);
 		}
 		rxr->msix = vector;
-		rxr->eims = IXGBE_IVAR_RX_QUEUE(vector);
+		rxr->eims = 1 << vector;
 		/* used in local timer */
 		adapter->rx_mask |= rxr->eims;
 		TASK_INIT(&rxr->rx_task, 0, ixgbe_handle_rx, rxr);
@@ -1916,19 +2136,37 @@ ixgbe_allocate_msix(struct adapter *adapter)
 	}
 	/* Set the link handler function */
 	error = bus_setup_intr(dev, adapter->res[vector],
-	    INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_msix_link,
-	    adapter, &adapter->tag[vector]);
+	    INTR_TYPE_NET | INTR_MPSAFE,
+#if __FreeBSD_version > 700000
+		    NULL,
+#endif
+	    ixgbe_msix_link, adapter, &adapter->tag[vector]);
 	if (error) {
 		adapter->res[vector] = NULL;
 		device_printf(dev, "Failed to register LINK handler");
 		return (error);
 	}
 	adapter->linkvec = vector;
+	/* Tasklets for Link, SFP and Multispeed Fiber */
+	TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
+	TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
+	TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
+	adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT,
+	    taskqueue_thread_enqueue, &adapter->tq);
+	taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq",
+	    device_get_nameunit(adapter->dev));
 
 	return (0);
 }
+#else  /* Freebsd 6.1/2 */
+static int
+ixgbe_allocate_msix(struct adapter *adapter)
+{
+	return (1);
+}
+#endif
 
-
+#if __FreeBSD_version >= 602105
 /*
  * Setup Either MSI/X or MSI
  */
@@ -1996,6 +2234,7 @@ msi:
                	device_printf(adapter->dev,"Using MSI interrupt\n");
 	return (msgs);
 }
+#endif /* FreeBSD_version >= 602105 */
 
 static int
 ixgbe_allocate_pci_resources(struct adapter *adapter)
@@ -2031,9 +2270,10 @@ ixgbe_allocate_pci_resources(struct adapter *adapter)
 	adapter->num_tx_queues = 1;
 	adapter->num_rx_queues = 1;
 
+#if __FreeBSD_version >= 602105
 	/* Now setup MSI or MSI/X */
 	adapter->msix = ixgbe_setup_msix(adapter);
-
+#endif
 	adapter->hw.back = &adapter->osdep;
 	return (0);
 }
@@ -2051,7 +2291,10 @@ ixgbe_free_pci_resources(struct adapter * adapter)
 	if (adapter->msix == 0)
 		adapter->msix = 1;
 
-	rid = PCIR_BAR(MSIX_82598_BAR);
+	if (adapter->hw.mac.type == ixgbe_mac_82598EB)
+		rid = PCIR_BAR(MSIX_82598_BAR);
+	else
+		rid = PCIR_BAR(MSIX_82599_BAR);
 
 	/*
 	 * First release all the interrupt resources:
@@ -2071,12 +2314,14 @@ ixgbe_free_pci_resources(struct adapter * adapter)
 		}
 	}
 
+#if __FreeBSD_version >= 602105
 	if (adapter->msix)
 		pci_release_msi(dev);
 
 	if (adapter->msix_mem != NULL)
 		bus_release_resource(dev, SYS_RES_MEMORY,
 		    rid, adapter->msix_mem);
+#endif
 
 	if (adapter->pci_mem != NULL)
 		bus_release_resource(dev, SYS_RES_MEMORY,
@@ -2215,7 +2460,7 @@ ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
 	int             r;
 
 	r = bus_dma_tag_create(NULL,	/* parent */
-			       PAGE_SIZE, 0,	/* alignment, bounds */
+			       1, 0,	/* alignment, bounds */
 			       BUS_SPACE_MAXADDR,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,	/* filter, filterarg */
@@ -2415,12 +2660,12 @@ ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
 	 * Setup DMA descriptor areas.
 	 */
 	if ((error = bus_dma_tag_create(NULL,		/* parent */
-			       PAGE_SIZE, 0,		/* alignment, bounds */
+			       1, 0,		/* alignment, bounds */
 			       BUS_SPACE_MAXADDR,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,		/* filter, filterarg */
 			       IXGBE_TSO_SIZE,		/* maxsize */
-			       IXGBE_MAX_SCATTER,	/* nsegments */
+			       ixgbe_num_segs,		/* nsegments */
 			       PAGE_SIZE,		/* maxsegsize */
 			       0,			/* flags */
 			       NULL,			/* lockfunc */
@@ -2484,6 +2729,8 @@ ixgbe_setup_transmit_ring(struct tx_ring *txr)
 			m_freem(txbuf->m_head);
 			txbuf->m_head = NULL;
 		}
+		/* Clear the EOP index */
+		txbuf->eop_index = -1;
         }
 
 	/* Set number of descriptors available */
@@ -2491,7 +2738,6 @@ ixgbe_setup_transmit_ring(struct tx_ring *txr)
 
 	bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-
 }
 
 /*********************************************************************
@@ -2524,8 +2770,7 @@ ixgbe_initialize_transmit_units(struct adapter *adapter)
 	/* Setup the Base and Length of the Tx Descriptor Ring */
 
 	for (int i = 0; i < adapter->num_tx_queues; i++, txr++) {
-		u64	txhwb = 0, tdba = txr->txdma.dma_paddr;
-		u32	txctrl;
+		u64	tdba = txr->txdma.dma_paddr;
 
 		IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
 		       (tdba & 0x00000000ffffffffULL));
@@ -2533,17 +2778,6 @@ ixgbe_initialize_transmit_units(struct adapter *adapter)
 		IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
 		    adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
 
-		/* Setup for Head WriteBack */
-		txhwb = (u64)vtophys(&txr->tx_hwb);
-		txhwb |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
-		IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(i),
-		    (txhwb & 0x00000000ffffffffULL));
-		IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(i),
-		    (txhwb >> 32));
-		txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
-		txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
-		IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
-
 		/* Setup the HW Tx Head and Tail descriptor pointers */
 		IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
 		IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
@@ -2554,6 +2788,13 @@ ixgbe_initialize_transmit_units(struct adapter *adapter)
 		txr->watchdog_timer = 0;
 	}
 
+	if (hw->mac.type == ixgbe_mac_82599EB) {
+		u32 dmatxctl;
+		dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
+		dmatxctl |= IXGBE_DMATXCTL_TE;
+		IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
+	}
+
 	return;
 }
 
@@ -2634,7 +2875,7 @@ ixgbe_free_transmit_buffers(struct tx_ring *txr)
  *
  **********************************************************************/
 
-static boolean_t
+static int
 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 {
 	struct adapter *adapter = txr->adapter;
@@ -2642,12 +2883,12 @@ ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 	struct ixgbe_tx_buf        *tx_buffer;
 	u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
 	struct ether_vlan_header *eh;
-	struct ip *ip;
+	struct ip *ip = NULL;
 	struct ip6_hdr *ip6;
 	int  ehdrlen, ip_hlen = 0;
 	u16	etype;
 	u8	ipproto = 0;
-	bool	offload = TRUE;
+	bool	offload = FALSE;
 	int ctxd = txr->next_avail_tx_desc;
 #if __FreeBSD_version < 700000
 	struct m_tag	*mtag;
@@ -2656,8 +2897,8 @@ ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 #endif
 
 
-	if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
-		offload = FALSE;
+	if (mp->m_pkthdr.csum_flags & CSUM_OFFLOAD)
+		offload = TRUE;
 
 	tx_buffer = &txr->tx_buffers[ctxd];
 	TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
@@ -2671,14 +2912,14 @@ ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 	if (mtag != NULL) {
 		vlan_macip_lens |=
 		    htole16(VLAN_TAG_VALUE(mtag)) << IXGBE_ADVTXD_VLAN_SHIFT;
-	} else if (offload == FALSE)
-		return FALSE;	/* No need for CTX */
+		offload = TRUE;
+	} 
 #else
 	if (mp->m_flags & M_VLANTAG) {
 		vtag = htole16(mp->m_pkthdr.ether_vtag);
 		vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
-	} else if (offload == FALSE)
-		return FALSE;
+		offload = TRUE;
+	} 
 #endif
 	/*
 	 * Determine where frame payload starts.
@@ -2714,9 +2955,12 @@ ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 			ipproto = ip6->ip6_nxt;
 			type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
 			break;
+#ifdef IXGBE_TIMESYNC
+                case ETHERTYPE_IEEE1588:
+                        return (IXGBE_TIMESTAMP);
+#endif
 		default:
-			offload = FALSE;
-			break;
+			return (FALSE);
 	}
 
 	vlan_macip_lens |= ip_hlen;
@@ -2724,18 +2968,36 @@ ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 
 	switch (ipproto) {
 		case IPPROTO_TCP:
-			if (mp->m_pkthdr.csum_flags & CSUM_TCP)
+			if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
 				type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
+				offload = TRUE;
+			}
 			break;
+
 		case IPPROTO_UDP:
-			if (mp->m_pkthdr.csum_flags & CSUM_UDP)
+		{
+#ifdef IXGBE_TIMESYNC
+			void *hdr = (caddr_t) ip + ip_hlen;
+			struct udphdr *uh = (struct udphdr *)hdr;
+
+			if (uh->uh_dport == htons(TSYNC_UDP_PORT))
+				return (IXGBE_TIMESTAMP);
+#endif
+			if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
 				type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
+				offload = TRUE;
+			}
 			break;
+
+		}
+
 		default:
-			offload = FALSE;
-			break;
+			return (FALSE);
 	}
 
+	if (offload != TRUE )
+		return (FALSE);
+
 	/* Now copy bits into descriptor */
 	TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
 	TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
@@ -2743,6 +3005,7 @@ ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 	TXD->mss_l4len_idx = htole32(0);
 
 	tx_buffer->m_head = NULL;
+	tx_buffer->eop_index = -1;
 
 	/* We've consumed the first desc, adjust counters */
 	if (++ctxd == adapter->num_tx_desc)
@@ -2750,7 +3013,7 @@ ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
 	txr->next_avail_tx_desc = ctxd;
 	--txr->tx_avail;
 
-        return (offload);
+        return (TRUE);
 }
 
 #if __FreeBSD_version >= 700000
@@ -2774,9 +3037,6 @@ ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen)
 	struct ip *ip;
 	struct tcphdr *th;
 
-	if (((mp->m_pkthdr.csum_flags & CSUM_TSO) == 0) ||
-	    (mp->m_pkthdr.len <= IXGBE_TX_BUFFER_SIZE))
-	        return FALSE;
 
 	/*
 	 * Determine where frame payload starts.
@@ -2799,7 +3059,6 @@ ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen)
 	ip = (struct ip *)(mp->m_data + ehdrlen);
 	if (ip->ip_p != IPPROTO_TCP)
 		return FALSE;   /* 0 */
-	ip->ip_len = 0;
 	ip->ip_sum = 0;
 	ip_hlen = ip->ip_hl << 2;
 	th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
@@ -2807,6 +3066,7 @@ ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen)
 	    ip->ip_dst.s_addr, htons(IPPROTO_TCP));
 	tcp_hlen = th->th_off << 2;
 	hdrlen = ehdrlen + ip_hlen + tcp_hlen;
+
 	/* This is used in the transmit desc in encap */
 	*paylen = mp->m_pkthdr.len - hdrlen;
 
@@ -2834,6 +3094,7 @@ ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen)
 
 	TXD->seqnum_seed = htole32(0);
 	tx_buffer->m_head = NULL;
+	tx_buffer->eop_index = -1;
 
 	if (++ctxd == adapter->num_tx_desc)
 		ctxd = 0;
@@ -2867,9 +3128,9 @@ ixgbe_txeof(struct tx_ring *txr)
 	u32	first, last, done, num_avail;
 	u32	cleaned = 0;
 	struct ixgbe_tx_buf *tx_buffer;
-	struct ixgbe_legacy_tx_desc *tx_desc;
+	struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
 
-	mtx_assert(&txr->mtx, MA_OWNED);
+	mtx_assert(&txr->tx_mtx, MA_OWNED);
 
 	if (txr->tx_avail == adapter->num_tx_desc)
 		return FALSE;
@@ -2880,16 +3141,28 @@ ixgbe_txeof(struct tx_ring *txr)
 	tx_buffer = &txr->tx_buffers[first];
 	/* For cleanup we just use legacy struct */
 	tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
+	last = tx_buffer->eop_index;
+	if (last == -1)
+		return FALSE;
 
-	/* Get the HWB */
-	rmb();
-        done = txr->tx_hwb;
+	eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
+	/*
+	** Get the index of the first descriptor
+	** BEYOND the EOP and call that 'done'.
+	** I do this so the comparison in the
+	** inner while loop below can be simple
+	*/
+	if (++last == adapter->num_tx_desc) last = 0;
+	done = last;
 
         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
             BUS_DMASYNC_POSTREAD);
-
-	while (TRUE) {
-		/* We clean the range til last head write back */
+	/*
+	** Only the EOP descriptor of a packet now has the DD
+	** bit set, this is what we look for...
+	*/
+	while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
+		/* We clean the range of the packet */
 		while (first != done) {
 			tx_desc->upper.data = 0;
 			tx_desc->lower.data = 0;
@@ -2907,6 +3180,7 @@ ixgbe_txeof(struct tx_ring *txr)
 				tx_buffer->m_head = NULL;
 				tx_buffer->map = NULL;
 			}
+			tx_buffer->eop_index = -1;
 
 			if (++first == adapter->num_tx_desc)
 				first = 0;
@@ -2916,10 +3190,14 @@ ixgbe_txeof(struct tx_ring *txr)
 			    (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
 		}
 		/* See if there is more work now */
-		last = done;
-		rmb();
-        	done = txr->tx_hwb;
-		if (last == done)
+		last = tx_buffer->eop_index;
+		if (last != -1) {
+			eop_desc =
+			    (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
+			/* Get next done point */
+			if (++last == adapter->num_tx_desc) last = 0;
+			done = last;
+		} else
 			break;
 	}
 	bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
@@ -3091,7 +3369,7 @@ ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
 	** it may not always use this.
 	*/
 	if ((error = bus_dma_tag_create(NULL,		/* parent */
-				   PAGE_SIZE, 0,	/* alignment, bounds */
+				   1, 0,	/* alignment, bounds */
 				   BUS_SPACE_MAXADDR,	/* lowaddr */
 				   BUS_SPACE_MAXADDR,	/* highaddr */
 				   NULL, NULL,		/* filter, filterarg */
@@ -3244,7 +3522,7 @@ fail:
 	 * the rings that completed, the failing case will have
 	 * cleaned up for itself. 'j' failed, so its the terminus.
 	 */
-	for (int i = 0; i < j; i++) {
+	for (int i = 0; i < j; ++i) {
 		rxr = &adapter->rx_rings[i];
 		for (int n = 0; n < adapter->num_rx_desc; n++) {
 			struct ixgbe_rx_buf *rxbuf;
@@ -3314,9 +3592,21 @@ ixgbe_initialize_receive_units(struct adapter *adapter)
 		srrctl |= ((IXGBE_RX_HDR << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
 		    & IXGBE_SRRCTL_BSIZEHDR_MASK);
 		srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
+		if (adapter->hw.mac.type == ixgbe_mac_82599EB) {
+			/* PSRTYPE must be initialized in 82599 */
+			u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
+				      IXGBE_PSRTYPE_UDPHDR |
+				      IXGBE_PSRTYPE_IPV4HDR |
+				      IXGBE_PSRTYPE_IPV6HDR;
+			psrtype |= (7 << 29);
+			IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
+		}
 	} else
 		srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
 
+	if (adapter->hw.mac.type == ixgbe_mac_82599EB)
+		srrctl |= IXGBE_SRRCTL_DROP_EN;
+
 	IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(0), srrctl);
 
 	for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) {
@@ -3356,6 +3646,9 @@ ixgbe_initialize_receive_units(struct adapter *adapter)
 			IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
 
 		/* Perform hash on these packet types */
+		if (adapter->hw.mac.type == ixgbe_mac_82599EB)
+			mrqc = IXGBE_MRQC_VMDQRSS32EN;
+
 		mrqc |= IXGBE_MRQC_RSSEN
 		     | IXGBE_MRQC_RSS_FIELD_IPV4
 		     | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
@@ -3380,6 +3673,14 @@ ixgbe_initialize_receive_units(struct adapter *adapter)
 
 	IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
 
+	if (adapter->hw.mac.type == ixgbe_mac_82599EB) {
+		u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
+		rdrxctl |= IXGBE_RDRXCTL_AGGDIS;
+		rdrxctl |= IXGBE_RDRXCTL_RSCLLIDIS;
+		rdrxctl |= IXGBE_RDRXCTL_MVMEN;
+		IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
+        }
+
 	return;
 }
 
@@ -3594,9 +3895,15 @@ ixgbe_rxeof(struct rx_ring *rxr, int count)
 				else
 					rxr->fmp->m_pkthdr.csum_flags = 0;
 				if (staterr & IXGBE_RXD_STAT_VP) {
+#if __FreeBSD_version >= 700000
 					rxr->fmp->m_pkthdr.ether_vtag =
 					    le16toh(cur->wb.upper.vlan);
 					rxr->fmp->m_flags |= M_VLANTAG;
+#else
+					VLAN_INPUT_TAG_NEW(ifp, rxr->fmp,
+					    (le16toh(cur->wb.upper.vlan) &
+					    IXGBE_RX_DESC_SPECIAL_VLAN_MASK));
+#endif
 				}
 				sendmp = rxr->fmp;
 				rxr->fmp = NULL;
@@ -3765,14 +4072,23 @@ ixgbe_enable_intr(struct adapter *adapter)
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32 mask = IXGBE_EIMS_ENABLE_MASK;
 
+
 	/* Enable Fan Failure detection */
-	if (hw->phy.media_type == ixgbe_media_type_copper)
+	if (hw->device_id == IXGBE_DEV_ID_82598AT)
 		    mask |= IXGBE_EIMS_GPI_SDP1;
 
+	/* 82599 specific interrupts */
+	if (adapter->hw.mac.type == ixgbe_mac_82599EB) {
+		    mask |= IXGBE_EIMS_ECC;
+		    mask |= IXGBE_EIMS_GPI_SDP1;
+		    mask |= IXGBE_EIMS_GPI_SDP2;
+	}
+
 	IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
 
 	/* With RSS we use auto clear */
 	if (adapter->msix_mem) {
+		mask = IXGBE_EIMS_ENABLE_MASK;
 		/* Dont autoclear Link */
 		mask &= ~IXGBE_EIMS_OTHER;
 		mask &= ~IXGBE_EIMS_LSC;
@@ -3805,6 +4121,15 @@ ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
 	return (value);
 }
 
+void
+ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
+{
+	pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
+	    reg, value, 2);
+
+	return;
+}
+
 /*
 ** Setup the correct IVAR register for a particular MSIX interrupt
 **   (yes this is all very magic and confusing :)
@@ -3834,6 +4159,21 @@ ixgbe_set_ivar(struct adapter *adapter, u16 entry, u8 vector, s8 type)
 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
 		break;
 
+	case ixgbe_mac_82599EB:
+		if (type == -1) { /* MISC IVAR */
+			index = (entry & 1) * 8;
+			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
+			ivar &= ~(0xFF << index);
+			ivar |= (vector << index);
+			IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
+		} else {	/* RX/TX IVARS */
+			index = (16 * (entry & 1)) + (8 * type);
+			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
+			ivar &= ~(0xFF << index);
+			ivar |= (vector << index);
+			IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
+		}
+
 	default:
 		break;
 	}
@@ -3886,6 +4226,63 @@ out:
 	return (result);
 }
 
+/*
+** Tasklet handler for MSIX Link interrupts
+**  - do outside interrupt since it might sleep
+*/
+static void
+ixgbe_handle_link(void *context, int pending)
+{
+	struct adapter  *adapter = context;
+
+	ixgbe_check_link(&adapter->hw,
+	    &adapter->link_speed, &adapter->link_up, 0);
+       	ixgbe_update_link_status(adapter);
+}
+
+/*
+** Tasklet for handling SFP module interrupts
+*/
+static void
+ixgbe_handle_mod(void *context, int pending)
+{
+	struct adapter  *adapter = context;
+	struct ixgbe_hw *hw = &adapter->hw;
+	device_t	dev = adapter->dev;
+	u32 err;
+
+	err = hw->phy.ops.identify_sfp(hw);
+	if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
+		device_printf(dev,
+		    "Unsupported SFP+ module type was detected.\n");
+		return;
+	}
+	hw->mac.ops.setup_sfp(hw);
+	taskqueue_enqueue(adapter->tq, &adapter->msf_task);
+	return;
+}
+
+
+/*
+** Tasklet for handling MSF (multispeed fiber) interrupts
+*/
+static void
+ixgbe_handle_msf(void *context, int pending)
+{
+	struct adapter  *adapter = context;
+	struct ixgbe_hw *hw = &adapter->hw;
+	u32 autoneg;
+
+	if (hw->mac.ops.get_link_capabilities)
+		hw->mac.ops.get_link_capabilities(hw, &autoneg,
+                                                  &hw->mac.autoneg);
+	if (hw->mac.ops.setup_link_speed)
+		hw->mac.ops.setup_link_speed(hw, autoneg, TRUE, TRUE);
+	ixgbe_check_link(&adapter->hw,
+	    &adapter->link_speed, &adapter->link_up, 0);
+       	ixgbe_update_link_status(adapter);
+	return;
+}
 
 /**********************************************************************
  *
@@ -3935,6 +4332,9 @@ ixgbe_update_stats_counters(struct adapter *adapter)
 	adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
 	adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
 
+	adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
+	adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
+
 	lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
 	adapter->stats.lxontxc += lxon;
 	lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
@@ -4147,7 +4547,7 @@ ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
 			adapter->hw.fc.requested_mode = ixgbe_fc_none;
 	}
 
-	ixgbe_setup_fc(&adapter->hw, 0);
+	ixgbe_fc_enable(&adapter->hw, 0);
 	return error;
 }
 
@@ -4160,3 +4560,97 @@ ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
 }
+
+#ifdef IXGBE_TIMESYNC
+/*
+ * Initialize the Time Sync Feature
+ */
+static int
+ixgbe_tsync_init(struct adapter *adapter)
+{
+	device_t	dev = adapter->dev;
+	u32		tx_ctl, rx_ctl;
+
+
+	IXGBE_WRITE_REG(&adapter->hw, IXGBE_TIMINCA, (1<<24) |
+	    20833/PICOSECS_PER_TICK);
+
+	adapter->last_stamp =  IXGBE_READ_REG(&adapter->hw, IXGBE_SYSTIML);
+	adapter->last_stamp |= (u64)IXGBE_READ_REG(&adapter->hw,
+	    IXGBE_SYSTIMH) << 32ULL;
+
+	/* Enable the TX side */
+	tx_ctl =  IXGBE_READ_REG(&adapter->hw, IXGBE_TSYNCTXCTL);
+	tx_ctl |= 0x10;
+	IXGBE_WRITE_REG(&adapter->hw, IXGBE_TSYNCTXCTL, tx_ctl);
+	IXGBE_WRITE_FLUSH(&adapter->hw);
+
+	tx_ctl = IXGBE_READ_REG(&adapter->hw, IXGBE_TSYNCTXCTL);
+	if ((tx_ctl & 0x10) == 0) {
+     		device_printf(dev, "Failed to enable TX timestamping\n");
+		return (ENXIO);
+	} 
+
+	/* Enable RX */
+	rx_ctl = IXGBE_READ_REG(&adapter->hw, IXGBE_TSYNCRXCTL);
+	rx_ctl |= 0x10; /* Enable the feature */
+	rx_ctl |= 0x04; /* This value turns on Ver 1 and 2 */
+	IXGBE_WRITE_REG(&adapter->hw, IXGBE_TSYNCRXCTL, rx_ctl);
+
+	/*
+	 * Ethertype Filter Queue Filter[0][15:0] = 0x88F7 (Ethertype)
+	 * Ethertype Filter Queue Filter[0][30] = 0x1 (Enable filter)
+	 * Ethertype Filter Queue Filter[0][31] = 0x1 (Enable Timestamping)
+	 */
+	IXGBE_WRITE_REG(&adapter->hw, IXGBE_ETQF(0), 0xC00088f7);
+
+	IXGBE_WRITE_FLUSH(&adapter->hw);
+
+	rx_ctl = IXGBE_READ_REG(&adapter->hw, IXGBE_TSYNCRXCTL);
+	if ((rx_ctl & 0x10) == 0) {
+     		device_printf(dev, "Failed to enable RX timestamping\n");
+		return (ENXIO);
+	} 
+
+	device_printf(dev, "IEEE 1588 Precision Time Protocol enabled\n");
+
+	return (0);
+}
+
+/*
+ * Disable the Time Sync Feature
+ */
+static void
+ixgbe_tsync_disable(struct adapter *adapter)
+{
+	u32		tx_ctl, rx_ctl;
+ 
+	tx_ctl =  IXGBE_READ_REG(&adapter->hw, IXGBE_TSYNCTXCTL);
+	tx_ctl &= ~0x10;
+	IXGBE_WRITE_REG(&adapter->hw, IXGBE_TSYNCTXCTL, tx_ctl);
+	IXGBE_WRITE_FLUSH(&adapter->hw);
+   
+	/* Invalidate TX Timestamp */
+	IXGBE_READ_REG(&adapter->hw, IXGBE_TXSTMPH);
+ 
+	tx_ctl = IXGBE_READ_REG(&adapter->hw, IXGBE_TSYNCTXCTL);
+	if (tx_ctl & 0x10)
+     		HW_DEBUGOUT("Failed to disable TX timestamping\n");
+   
+	rx_ctl = IXGBE_READ_REG(&adapter->hw, IXGBE_TSYNCRXCTL);
+	rx_ctl &= ~0x10;
+   
+	IXGBE_WRITE_REG(&adapter->hw, IXGBE_TSYNCRXCTL, rx_ctl);
+	IXGBE_WRITE_FLUSH(&adapter->hw);
+   
+	/* Invalidate RX Timestamp */
+	IXGBE_READ_REG(&adapter->hw, IXGBE_RXSATRH);
+ 
+	rx_ctl = IXGBE_READ_REG(&adapter->hw, IXGBE_TSYNCRXCTL);
+	if (rx_ctl & 0x10)
+		HW_DEBUGOUT("Failed to disable RX timestamping\n");
+ 
+	return;
+}
+
+#endif /* IXGBE_TIMESYNC */
diff --git a/sys/dev/ixgbe/ixgbe.h b/sys/dev/ixgbe/ixgbe.h
index 5a5e2ef..e6bebd1 100644
--- a/sys/dev/ixgbe/ixgbe.h
+++ b/sys/dev/ixgbe/ixgbe.h
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -64,6 +64,7 @@
 #include <netinet/ip.h>
 #include <netinet/ip6.h>
 #include <netinet/tcp.h>
+#include <netinet/tcp_lro.h>
 #include <netinet/udp.h>
 
 #include <machine/in_cksum.h>
@@ -83,8 +84,12 @@
 #include <sys/taskqueue.h>
 #include <sys/pcpu.h>
 
+#ifdef IXGBE_TIMESYNC
+#include <sys/ioccom.h>
+#include <sys/time.h>
+#endif
+
 #include "ixgbe_api.h"
-#include "tcp_lro.h"
 
 /* Tunables */
 
@@ -95,7 +100,7 @@
  * bytes. Performance tests have show the 2K value to be optimal for top
  * performance.
  */
-#define DEFAULT_TXD	256
+#define DEFAULT_TXD	1024
 #define PERFORM_TXD	2048
 #define MAX_TXD		4096
 #define MIN_TXD		64
@@ -110,7 +115,7 @@
  *	against the system mbuf pool limit, you can tune nmbclusters
  *	to adjust for this.
  */
-#define DEFAULT_RXD	256
+#define DEFAULT_RXD	1024
 #define PERFORM_RXD	2048
 #define MAX_RXD		4096
 #define MIN_RXD		64
@@ -159,7 +164,8 @@
 #define HW_DEBUGOUT2(S, A, B)       if (DEBUG_HW) printf(S "\n", A, B)
 
 #define MAX_NUM_MULTICAST_ADDRESSES     128
-#define IXGBE_MAX_SCATTER		100
+#define IXGBE_82598_SCATTER		100
+#define IXGBE_82599_SCATTER		32
 #define MSIX_82598_BAR			3
 #define MSIX_82599_BAR			4
 #define IXGBE_TSO_SIZE			65535
@@ -171,12 +177,13 @@
 #define IXGBE_MSGS			18
 
 /* For 6.X code compatibility */
-#if __FreeBSD_version < 700000
+#if !defined(ETHER_BPF_MTAP)
 #define ETHER_BPF_MTAP		BPF_MTAP
+#endif
+
+#if __FreeBSD_version < 700000
 #define CSUM_TSO		0
 #define IFCAP_TSO4		0
-#define FILTER_STRAY
-#define FILTER_HANDLED
 #endif
 
 /*
@@ -214,6 +221,7 @@ typedef struct _ixgbe_vendor_info_t {
 
 
 struct ixgbe_tx_buf {
+	u32		eop_index;
 	struct mbuf	*m_head;
 	bus_dmamap_t	map;
 };
@@ -314,8 +322,6 @@ struct adapter {
 
 	/*
 	 * Interrupt resources:
-	 *  Oplin has 20 MSIX messages
-	 *  so allocate that for now.
 	 */
 	void		*tag[IXGBE_MSGS];
 	struct resource *res[IXGBE_MSGS];
@@ -334,6 +340,7 @@ struct adapter {
 	bool		link_active;
 	u16		max_frame_size;
 	u32		link_speed;
+	bool		link_up;
 	u32 		linkvec;
 	u32		tx_int_delay;
 	u32		tx_abs_int_delay;
@@ -343,8 +350,12 @@ struct adapter {
 	/* Mbuf cluster size */
 	u32		rx_mbuf_sz;
 
-	/* Check for missing optics */
+	/* Support for pluggable optics */
 	bool		sfp_probe;
+	struct task     link_task; 	/* Link tasklet */
+	struct task     mod_task; 	/* SFP tasklet */
+	struct task     msf_task; 	/* Multispeed Fiber tasklet */
+	struct taskqueue	*tq;
 
 	/*
 	 * Transmit rings:
@@ -364,6 +375,12 @@ struct adapter {
 	u32		rx_mask;
 	u32		rx_process_limit;
 
+#ifdef IXGBE_TIMESYNC
+	u64		last_stamp;
+	u64		last_sec;
+	u32		last_ns;
+#endif
+
 	/* Misc stats maintained by the driver */
 	unsigned long   dropped_pkts;
 	unsigned long   mbuf_defrag_failed;
@@ -378,6 +395,32 @@ struct adapter {
 	struct ixgbe_hw_stats stats;
 };
 
+#ifdef IXGBE_TIMESYNC
+/* Precision Time Sync (IEEE 1588) defines */
+#define ETHERTYPE_IEEE1588      0x88F7
+#define PICOSECS_PER_TICK       20833
+#define TSYNC_UDP_PORT          319 /* UDP port for the protocol */
+#define IXGBE_ADVTXD_TSTAMP	0x00080000
+
+/* TIMESYNC IOCTL defines */
+#define IXGBE_TIMESYNC_READTS     _IOWR('i', 127, struct ixgbe_tsync_read)
+#define IXGBE_TIMESTAMP           5       /* A unique return value */
+ 
+/* Used in the READTS IOCTL */
+struct ixgbe_tsync_read {
+        int read_current_time;
+        struct timespec system_time;
+        u64 network_time;
+        u64 rx_stamp;
+        u64 tx_stamp;
+        u16 seqid;
+        unsigned char srcid[6];
+        int rx_valid;
+        int tx_valid;
+};
+
+#endif /* IXGBE_TIMESYNC */
+
 #define IXGBE_CORE_LOCK_INIT(_sc, _name) \
         mtx_init(&(_sc)->core_mtx, _name, "IXGBE Core Lock", MTX_DEF)
 #define IXGBE_CORE_LOCK_DESTROY(_sc)      mtx_destroy(&(_sc)->core_mtx)
@@ -385,6 +428,7 @@ struct adapter {
 #define IXGBE_RX_LOCK_DESTROY(_sc)                mtx_destroy(&(_sc)->rx_mtx)
 #define IXGBE_CORE_LOCK(_sc)              mtx_lock(&(_sc)->core_mtx)
 #define IXGBE_TX_LOCK(_sc)                        mtx_lock(&(_sc)->tx_mtx)
+#define IXGBE_TX_TRYLOCK(_sc)                     mtx_trylock(&(_sc)->tx_mtx)
 #define IXGBE_RX_LOCK(_sc)                        mtx_lock(&(_sc)->rx_mtx)
 #define IXGBE_CORE_UNLOCK(_sc)            mtx_unlock(&(_sc)->core_mtx)
 #define IXGBE_TX_UNLOCK(_sc)              mtx_unlock(&(_sc)->tx_mtx)
@@ -393,4 +437,20 @@ struct adapter {
 #define IXGBE_TX_LOCK_ASSERT(_sc)         mtx_assert(&(_sc)->tx_mtx, MA_OWNED)
 
 
+static inline bool
+ixgbe_is_sfp(struct ixgbe_hw *hw)
+{
+	switch (hw->phy.type) {
+	case ixgbe_phy_sfp_avago:
+	case ixgbe_phy_sfp_ftl:
+	case ixgbe_phy_sfp_intel:
+	case ixgbe_phy_sfp_unknown:
+	case ixgbe_phy_tw_tyco:
+	case ixgbe_phy_tw_unknown:
+		return TRUE;
+	default:
+		return FALSE;
+	}
+}
+
 #endif /* _IXGBE_H_ */
diff --git a/sys/dev/ixgbe/ixgbe_82598.c b/sys/dev/ixgbe/ixgbe_82598.c
index c94b2f9..fd90d16 100644
--- a/sys/dev/ixgbe/ixgbe_82598.c
+++ b/sys/dev/ixgbe/ixgbe_82598.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -37,15 +37,12 @@
 #include "ixgbe_common.h"
 #include "ixgbe_phy.h"
 
+u32 ixgbe_get_pcie_msix_count_82598(struct ixgbe_hw *hw);
 s32 ixgbe_init_ops_82598(struct ixgbe_hw *hw);
 static s32 ixgbe_get_link_capabilities_82598(struct ixgbe_hw *hw,
                                              ixgbe_link_speed *speed,
                                              bool *autoneg);
-s32 ixgbe_get_copper_link_capabilities_82598(struct ixgbe_hw *hw,
-                                             ixgbe_link_speed *speed,
-                                             bool *autoneg);
 static enum ixgbe_media_type ixgbe_get_media_type_82598(struct ixgbe_hw *hw);
-s32 ixgbe_setup_fc_82598(struct ixgbe_hw *hw, s32 packetbuf_num);
 s32 ixgbe_fc_enable_82598(struct ixgbe_hw *hw, s32 packetbuf_num);
 static s32 ixgbe_setup_mac_link_82598(struct ixgbe_hw *hw);
 static s32 ixgbe_check_mac_link_82598(struct ixgbe_hw *hw,
@@ -60,22 +57,41 @@ static s32 ixgbe_setup_copper_link_speed_82598(struct ixgbe_hw *hw,
                                                ixgbe_link_speed speed,
                                                bool autoneg,
                                                bool autoneg_wait_to_complete);
-#ifndef NO_82598_A0_SUPPORT
-s32 ixgbe_reset_hw_rev_0_82598(struct ixgbe_hw *hw);
-#endif
 static s32 ixgbe_reset_hw_82598(struct ixgbe_hw *hw);
 s32 ixgbe_set_vmdq_82598(struct ixgbe_hw *hw, u32 rar, u32 vmdq);
 static s32 ixgbe_clear_vmdq_82598(struct ixgbe_hw *hw, u32 rar, u32 vmdq);
 s32 ixgbe_set_vfta_82598(struct ixgbe_hw *hw, u32 vlan,
                          u32 vind, bool vlan_on);
 static s32 ixgbe_clear_vfta_82598(struct ixgbe_hw *hw);
-static s32 ixgbe_blink_led_stop_82598(struct ixgbe_hw *hw, u32 index);
-static s32 ixgbe_blink_led_start_82598(struct ixgbe_hw *hw, u32 index);
 s32 ixgbe_read_analog_reg8_82598(struct ixgbe_hw *hw, u32 reg, u8 *val);
 s32 ixgbe_write_analog_reg8_82598(struct ixgbe_hw *hw, u32 reg, u8 val);
 s32 ixgbe_read_i2c_eeprom_82598(struct ixgbe_hw *hw, u8 byte_offset,
                                 u8 *eeprom_data);
 u32 ixgbe_get_supported_physical_layer_82598(struct ixgbe_hw *hw);
+s32 ixgbe_init_phy_ops_82598(struct ixgbe_hw *hw);
+
+/**
+ *  ixgbe_get_pcie_msix_count_82598 - Gets MSI-X vector count
+ *  @hw: pointer to hardware structure
+ *
+ *  Read PCIe configuration space, and get the MSI-X vector count from
+ *  the capabilities table.
+ **/
+u32 ixgbe_get_pcie_msix_count_82598(struct ixgbe_hw *hw)
+{
+	u32 msix_count = 18;
+
+	if (hw->mac.msix_vectors_from_pcie) {
+		msix_count = IXGBE_READ_PCIE_WORD(hw,
+		                                  IXGBE_PCIE_MSIX_82598_CAPS);
+		msix_count &= IXGBE_PCIE_MSIX_TBL_SZ_MASK;
+
+		/* MSI-X count is zero-based in HW, so increment to give
+		 * proper value */
+		msix_count++;
+	}
+	return msix_count;
+}
 
 /**
  *  ixgbe_init_ops_82598 - Inits func ptrs and MAC type
@@ -89,30 +105,21 @@ s32 ixgbe_init_ops_82598(struct ixgbe_hw *hw)
 	struct ixgbe_mac_info *mac = &hw->mac;
 	struct ixgbe_phy_info *phy = &hw->phy;
 	s32 ret_val;
-	u16 list_offset, data_offset;
 
 	ret_val = ixgbe_init_phy_ops_generic(hw);
 	ret_val = ixgbe_init_ops_generic(hw);
 
+	/* PHY */
+	phy->ops.init = &ixgbe_init_phy_ops_82598;
+
 	/* MAC */
-#ifndef NO_82598_A0_SUPPORT
-	if (hw->revision_id == 0)
-		mac->ops.reset_hw = &ixgbe_reset_hw_rev_0_82598;
-	else
-		mac->ops.reset_hw = &ixgbe_reset_hw_82598;
-#else
 	mac->ops.reset_hw = &ixgbe_reset_hw_82598;
-#endif
 	mac->ops.get_media_type = &ixgbe_get_media_type_82598;
 	mac->ops.get_supported_physical_layer =
 	                            &ixgbe_get_supported_physical_layer_82598;
 	mac->ops.read_analog_reg8 = &ixgbe_read_analog_reg8_82598;
 	mac->ops.write_analog_reg8 = &ixgbe_write_analog_reg8_82598;
 
-	/* LEDs */
-	mac->ops.blink_led_start = &ixgbe_blink_led_start_82598;
-	mac->ops.blink_led_stop = &ixgbe_blink_led_stop_82598;
-
 	/* RAR, Multicast, VLAN */
 	mac->ops.set_vmdq = &ixgbe_set_vmdq_82598;
 	mac->ops.clear_vmdq = &ixgbe_clear_vmdq_82598;
@@ -120,42 +127,67 @@ s32 ixgbe_init_ops_82598(struct ixgbe_hw *hw)
 	mac->ops.clear_vfta = &ixgbe_clear_vfta_82598;
 
 	/* Flow Control */
-	mac->ops.setup_fc = &ixgbe_setup_fc_82598;
-
-	/* Link */
-	mac->ops.check_link = &ixgbe_check_mac_link_82598;
-	if (mac->ops.get_media_type(hw) == ixgbe_media_type_copper) {
-		mac->ops.setup_link = &ixgbe_setup_copper_link_82598;
-		mac->ops.setup_link_speed =
-		                     &ixgbe_setup_copper_link_speed_82598;
-		mac->ops.get_link_capabilities =
-		                     &ixgbe_get_copper_link_capabilities_82598;
-	} else {
-		mac->ops.setup_link = &ixgbe_setup_mac_link_82598;
-		mac->ops.setup_link_speed = &ixgbe_setup_mac_link_speed_82598;
-		mac->ops.get_link_capabilities =
-		                       &ixgbe_get_link_capabilities_82598;
-	}
+	mac->ops.fc_enable = &ixgbe_fc_enable_82598;
 
 	mac->mcft_size       = 128;
 	mac->vft_size        = 128;
 	mac->num_rar_entries = 16;
 	mac->max_tx_queues   = 32;
 	mac->max_rx_queues   = 64;
+	mac->max_msix_vectors = ixgbe_get_pcie_msix_count_82598(hw);
 
 	/* SFP+ Module */
 	phy->ops.read_i2c_eeprom = &ixgbe_read_i2c_eeprom_82598;
 
-	/* Call PHY identify routine to get the phy type */
+	/* Link */
+	mac->ops.check_link = &ixgbe_check_mac_link_82598;
+	mac->ops.setup_link = &ixgbe_setup_mac_link_82598;
+	mac->ops.setup_link_speed = &ixgbe_setup_mac_link_speed_82598;
+	mac->ops.get_link_capabilities =
+	                       &ixgbe_get_link_capabilities_82598;
+
+	return ret_val;
+}
+
+/**
+ *  ixgbe_init_phy_ops_82598 - PHY/SFP specific init
+ *  @hw: pointer to hardware structure
+ *
+ *  Initialize any function pointers that were not able to be
+ *  set during init_shared_code because the PHY/SFP type was
+ *  not known.  Perform the SFP init if necessary.
+ *
+ **/
+s32 ixgbe_init_phy_ops_82598(struct ixgbe_hw *hw)
+{
+	struct ixgbe_mac_info *mac = &hw->mac;
+	struct ixgbe_phy_info *phy = &hw->phy;
+	s32 ret_val = IXGBE_SUCCESS;
+	u16 list_offset, data_offset;
+
+
+	/* Identify the PHY */
 	phy->ops.identify(hw);
 
-	/* PHY Init */
+	/* Overwrite the link function pointers if copper PHY */
+	if (mac->ops.get_media_type(hw) == ixgbe_media_type_copper) {
+		mac->ops.setup_link = &ixgbe_setup_copper_link_82598;
+		mac->ops.setup_link_speed =
+		                     &ixgbe_setup_copper_link_speed_82598;
+		mac->ops.get_link_capabilities =
+		                  &ixgbe_get_copper_link_capabilities_generic;
+	}
+
 	switch (hw->phy.type) {
 	case ixgbe_phy_tn:
 		phy->ops.check_link = &ixgbe_check_phy_link_tnx;
 		phy->ops.get_firmware_version =
 		             &ixgbe_get_phy_firmware_version_tnx;
 		break;
+	case ixgbe_phy_aq:
+		phy->ops.get_firmware_version =
+		             &ixgbe_get_phy_firmware_version_aq;
+		break;
 	case ixgbe_phy_nl:
 		phy->ops.reset = &ixgbe_reset_phy_nl;
 
@@ -198,18 +230,19 @@ static s32 ixgbe_get_link_capabilities_82598(struct ixgbe_hw *hw,
                                              bool *autoneg)
 {
 	s32 status = IXGBE_SUCCESS;
-	s32 autoc_reg;
+	u32 autoc = 0;
 
-	autoc_reg = IXGBE_READ_REG(hw, IXGBE_AUTOC);
-
-	if (hw->mac.link_settings_loaded) {
-		autoc_reg &= ~IXGBE_AUTOC_LMS_ATTACH_TYPE;
-		autoc_reg &= ~IXGBE_AUTOC_LMS_MASK;
-		autoc_reg |= hw->mac.link_attach_type;
-		autoc_reg |= hw->mac.link_mode_select;
-	}
+	/*
+	 * Determine link capabilities based on the stored value of AUTOC,
+	 * which represents EEPROM defaults.  If AUTOC value has not been
+	 * stored, use the current register value.
+	 */
+	if (hw->mac.orig_link_settings_stored)
+		autoc = hw->mac.orig_autoc;
+	else
+		autoc = IXGBE_READ_REG(hw, IXGBE_AUTOC);
 
-	switch (autoc_reg & IXGBE_AUTOC_LMS_MASK) {
+	switch (autoc & IXGBE_AUTOC_LMS_MASK) {
 	case IXGBE_AUTOC_LMS_1G_LINK_NO_AN:
 		*speed = IXGBE_LINK_SPEED_1GB_FULL;
 		*autoneg = FALSE;
@@ -228,9 +261,9 @@ static s32 ixgbe_get_link_capabilities_82598(struct ixgbe_hw *hw,
 	case IXGBE_AUTOC_LMS_KX4_AN:
 	case IXGBE_AUTOC_LMS_KX4_AN_1G_AN:
 		*speed = IXGBE_LINK_SPEED_UNKNOWN;
-		if (autoc_reg & IXGBE_AUTOC_KX4_SUPP)
+		if (autoc & IXGBE_AUTOC_KX4_SUPP)
 			*speed |= IXGBE_LINK_SPEED_10GB_FULL;
-		if (autoc_reg & IXGBE_AUTOC_KX_SUPP)
+		if (autoc & IXGBE_AUTOC_KX_SUPP)
 			*speed |= IXGBE_LINK_SPEED_1GB_FULL;
 		*autoneg = TRUE;
 		break;
@@ -244,38 +277,6 @@ static s32 ixgbe_get_link_capabilities_82598(struct ixgbe_hw *hw,
 }
 
 /**
- *  ixgbe_get_copper_link_capabilities_82598 - Determines link capabilities
- *  @hw: pointer to hardware structure
- *  @speed: pointer to link speed
- *  @autoneg: boolean auto-negotiation value
- *
- *  Determines the link capabilities by reading the AUTOC register.
- **/
-s32 ixgbe_get_copper_link_capabilities_82598(struct ixgbe_hw *hw,
-                                             ixgbe_link_speed *speed,
-                                             bool *autoneg)
-{
-	s32 status = IXGBE_ERR_LINK_SETUP;
-	u16 speed_ability;
-
-	*speed = 0;
-	*autoneg = TRUE;
-
-	status = hw->phy.ops.read_reg(hw, IXGBE_MDIO_PHY_SPEED_ABILITY,
-	                              IXGBE_MDIO_PMA_PMD_DEV_TYPE,
-	                              &speed_ability);
-
-	if (status == IXGBE_SUCCESS) {
-		if (speed_ability & IXGBE_MDIO_PHY_SPEED_10G)
-			*speed |= IXGBE_LINK_SPEED_10GB_FULL;
-		if (speed_ability & IXGBE_MDIO_PHY_SPEED_1G)
-			*speed |= IXGBE_LINK_SPEED_1GB_FULL;
-	}
-
-	return status;
-}
-
-/**
  *  ixgbe_get_media_type_82598 - Determines media type
  *  @hw: pointer to hardware structure
  *
@@ -285,9 +286,18 @@ static enum ixgbe_media_type ixgbe_get_media_type_82598(struct ixgbe_hw *hw)
 {
 	enum ixgbe_media_type media_type;
 
+	/* Detect if there is a copper PHY attached. */
+	if (hw->phy.type == ixgbe_phy_cu_unknown ||
+	    hw->phy.type == ixgbe_phy_tn ||
+	    hw->phy.type == ixgbe_phy_aq) {
+		media_type = ixgbe_media_type_copper;
+		goto out;
+	}
+
 	/* Media type for I82598 is based on device ID */
 	switch (hw->device_id) {
 	case IXGBE_DEV_ID_82598:
+	case IXGBE_DEV_ID_82598_BX:
 		/* Default device ID is mezzanine card KX/KX4 */
 		media_type = ixgbe_media_type_backplane;
 		break;
@@ -308,7 +318,7 @@ static enum ixgbe_media_type ixgbe_get_media_type_82598(struct ixgbe_hw *hw)
 		media_type = ixgbe_media_type_unknown;
 		break;
 	}
-
+out:
 	return media_type;
 }
 
@@ -328,6 +338,12 @@ s32 ixgbe_fc_enable_82598(struct ixgbe_hw *hw, s32 packetbuf_num)
 
 	DEBUGFUNC("ixgbe_fc_enable_82598");
 
+	/* Negotiate the fc mode to use */
+	ret_val = ixgbe_fc_autoneg(hw);
+	if (ret_val)
+		goto out;
+
+	/* Disable any previous flow control settings */
 	fctrl_reg = IXGBE_READ_REG(hw, IXGBE_FCTRL);
 	fctrl_reg &= ~(IXGBE_FCTRL_RFCE | IXGBE_FCTRL_RPFCE);
 
@@ -339,14 +355,16 @@ s32 ixgbe_fc_enable_82598(struct ixgbe_hw *hw, s32 packetbuf_num)
 	 * 0: Flow control is completely disabled
 	 * 1: Rx flow control is enabled (we can receive pause frames,
 	 *    but not send pause frames).
-	 * 2:  Tx flow control is enabled (we can send pause frames but
+	 * 2: Tx flow control is enabled (we can send pause frames but
 	 *     we do not support receiving pause frames).
 	 * 3: Both Rx and Tx flow control (symmetric) are enabled.
 	 * other: Invalid.
 	 */
 	switch (hw->fc.current_mode) {
 	case ixgbe_fc_none:
-		/* Flow control completely disabled by software override. */
+		/* Flow control is disabled by software override or autoneg.
+		 * The code below will actually disable it in the HW.
+		 */
 		break;
 	case ixgbe_fc_rx_pause:
 		/*
@@ -378,7 +396,8 @@ s32 ixgbe_fc_enable_82598(struct ixgbe_hw *hw, s32 packetbuf_num)
 		break;
 	}
 
-	/* Enable 802.3x based flow control settings. */
+	/* Set 802.3x based flow control settings. */
+	fctrl_reg |= IXGBE_FCTRL_DPF;
 	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl_reg);
 	IXGBE_WRITE_REG(hw, IXGBE_RMCS, rmcs_reg);
 
@@ -411,73 +430,6 @@ out:
 }
 
 /**
- *  ixgbe_setup_fc_82598 - Set up flow control
- *  @hw: pointer to hardware structure
- *
- *  Sets up flow control.
- **/
-s32 ixgbe_setup_fc_82598(struct ixgbe_hw *hw, s32 packetbuf_num)
-{
-	s32 ret_val = IXGBE_SUCCESS;
-	ixgbe_link_speed speed;
-	bool link_up;
-
-	/* Validate the packetbuf configuration */
-	if (packetbuf_num < 0 || packetbuf_num > 7) {
-		DEBUGOUT1("Invalid packet buffer number [%d], expected range is"
-		          " 0-7\n", packetbuf_num);
-		ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
-		goto out;
-	}
-
-	/*
-	 * Validate the water mark configuration.  Zero water marks are invalid
-	 * because it causes the controller to just blast out fc packets.
-	 */
-	if (!hw->fc.low_water || !hw->fc.high_water || !hw->fc.pause_time) {
-		DEBUGOUT("Invalid water mark configuration\n");
-		ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
-		goto out;
-	}
-
-	/*
-	 * Validate the requested mode.  Strict IEEE mode does not allow
-	 * ixgbe_fc_rx_pause because it will cause testing anomalies.
-	 */
-	if (hw->fc.strict_ieee && hw->fc.requested_mode == ixgbe_fc_rx_pause) {
-		DEBUGOUT("ixgbe_fc_rx_pause not valid in strict IEEE mode\n");
-		ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
-		goto out;
-	}
-
-	/*
-	 * 10gig parts do not have a word in the EEPROM to determine the
-	 * default flow control setting, so we explicitly set it to full.
-	 */
-	if (hw->fc.requested_mode == ixgbe_fc_default)
-		hw->fc.requested_mode = ixgbe_fc_full;
-
-	/*
-	 * Save off the requested flow control mode for use later.  Depending
-	 * on the link partner's capabilities, we may or may not use this mode.
-	 */
-	hw->fc.current_mode = hw->fc.requested_mode;
-
-	/* Decide whether to use autoneg or not. */
-	hw->mac.ops.check_link(hw, &speed, &link_up, FALSE);
-	if (hw->phy.multispeed_fiber && (speed == IXGBE_LINK_SPEED_1GB_FULL))
-		ret_val = ixgbe_fc_autoneg(hw);
-
-	if (ret_val)
-		goto out;
-
-	ret_val = ixgbe_fc_enable_82598(hw, packetbuf_num);
-
-out:
-	return ret_val;
-}
-
-/**
  *  ixgbe_setup_mac_link_82598 - Configures MAC link settings
  *  @hw: pointer to hardware structure
  *
@@ -491,27 +443,17 @@ static s32 ixgbe_setup_mac_link_82598(struct ixgbe_hw *hw)
 	u32 i;
 	s32 status = IXGBE_SUCCESS;
 
-	autoc_reg = IXGBE_READ_REG(hw, IXGBE_AUTOC);
-
-	if (hw->mac.link_settings_loaded) {
-		autoc_reg &= ~IXGBE_AUTOC_LMS_ATTACH_TYPE;
-		autoc_reg &= ~IXGBE_AUTOC_LMS_MASK;
-		autoc_reg |= hw->mac.link_attach_type;
-		autoc_reg |= hw->mac.link_mode_select;
-
-		IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc_reg);
-		IXGBE_WRITE_FLUSH(hw);
-		msec_delay(50);
-	}
-
 	/* Restart link */
+	autoc_reg = IXGBE_READ_REG(hw, IXGBE_AUTOC);
 	autoc_reg |= IXGBE_AUTOC_AN_RESTART;
 	IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc_reg);
 
 	/* Only poll for autoneg to complete if specified to do so */
 	if (hw->phy.autoneg_wait_to_complete) {
-		if (hw->mac.link_mode_select == IXGBE_AUTOC_LMS_KX4_AN ||
-		    hw->mac.link_mode_select == IXGBE_AUTOC_LMS_KX4_AN_1G_AN) {
+		if ((autoc_reg & IXGBE_AUTOC_LMS_MASK) ==
+		     IXGBE_AUTOC_LMS_KX4_AN ||
+		    (autoc_reg & IXGBE_AUTOC_LMS_MASK) ==
+		     IXGBE_AUTOC_LMS_KX4_AN_1G_AN) {
 			links_reg = 0; /* Just in case Autoneg time = 0 */
 			for (i = 0; i < IXGBE_AUTO_NEG_TIME; i++) {
 				links_reg = IXGBE_READ_REG(hw, IXGBE_LINKS);
@@ -526,9 +468,6 @@ static s32 ixgbe_setup_mac_link_82598(struct ixgbe_hw *hw)
 		}
 	}
 
-	/* Set up flow control */
-	status = ixgbe_setup_fc_82598(hw, 0);
-
 	/* Add delay to filter out noises during initial link setup */
 	msec_delay(50);
 
@@ -616,6 +555,11 @@ static s32 ixgbe_check_mac_link_82598(struct ixgbe_hw *hw,
 	else
 		*speed = IXGBE_LINK_SPEED_1GB_FULL;
 
+	/* if link is down, zero out the current_mode */
+	if (*link_up == FALSE) {
+		hw->fc.current_mode = ixgbe_fc_none;
+		hw->fc.fc_was_autonegged = FALSE;
+	}
 out:
 	return IXGBE_SUCCESS;
 }
@@ -633,28 +577,34 @@ static s32 ixgbe_setup_mac_link_speed_82598(struct ixgbe_hw *hw,
                                            ixgbe_link_speed speed, bool autoneg,
                                            bool autoneg_wait_to_complete)
 {
-	s32 status = IXGBE_SUCCESS;
+	s32              status            = IXGBE_SUCCESS;
+	ixgbe_link_speed link_capabilities = IXGBE_LINK_SPEED_UNKNOWN;
+	u32              curr_autoc        = IXGBE_READ_REG(hw, IXGBE_AUTOC);
+	u32              autoc             = curr_autoc;
+	u32              link_mode         = autoc & IXGBE_AUTOC_LMS_MASK;
 
-	/* If speed is 10G, then check for CX4 or XAUI. */
-	if ((speed == IXGBE_LINK_SPEED_10GB_FULL) &&
-	    (!(hw->mac.link_attach_type & IXGBE_AUTOC_10G_KX4))) {
-		hw->mac.link_mode_select = IXGBE_AUTOC_LMS_10G_LINK_NO_AN;
-	} else if ((speed == IXGBE_LINK_SPEED_1GB_FULL) && (!autoneg)) {
-		hw->mac.link_mode_select = IXGBE_AUTOC_LMS_1G_LINK_NO_AN;
-	} else if (autoneg) {
-		/* BX mode - Autonegotiate 1G */
-		if (!(hw->mac.link_attach_type & IXGBE_AUTOC_1G_PMA_PMD))
-			hw->mac.link_mode_select = IXGBE_AUTOC_LMS_1G_AN;
-		else /* KX/KX4 mode */
-			hw->mac.link_mode_select = IXGBE_AUTOC_LMS_KX4_AN_1G_AN;
-	} else {
+	/* Check to see if speed passed in is supported. */
+	ixgbe_get_link_capabilities(hw, &link_capabilities, &autoneg);
+	speed &= link_capabilities;
+
+	if (speed == IXGBE_LINK_SPEED_UNKNOWN)
 		status = IXGBE_ERR_LINK_SETUP;
+
+	/* Set KX4/KX support according to speed requested */
+	else if (link_mode == IXGBE_AUTOC_LMS_KX4_AN ||
+	         link_mode == IXGBE_AUTOC_LMS_KX4_AN_1G_AN) {
+		autoc &= ~IXGBE_AUTOC_KX4_KX_SUPP_MASK;
+		if (speed & IXGBE_LINK_SPEED_10GB_FULL)
+			autoc |= IXGBE_AUTOC_KX4_SUPP;
+		if (speed & IXGBE_LINK_SPEED_1GB_FULL)
+			autoc |= IXGBE_AUTOC_KX_SUPP;
+		if (autoc != curr_autoc)
+			IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc);
 	}
 
 	if (status == IXGBE_SUCCESS) {
 		hw->phy.autoneg_wait_to_complete = autoneg_wait_to_complete;
 
-		hw->mac.link_settings_loaded = TRUE;
 		/*
 		 * Setup and restart the link based on the new values in
 		 * ixgbe_hw This will write the AUTOC register based on the new
@@ -683,10 +633,6 @@ static s32 ixgbe_setup_copper_link_82598(struct ixgbe_hw *hw)
 	/* Restart autonegotiation on PHY */
 	status = hw->phy.ops.setup_link(hw);
 
-	/* Set MAC to KX/KX4 autoneg, which defaults to Parallel detection */
-	hw->mac.link_attach_type = (IXGBE_AUTOC_10G_KX4 | IXGBE_AUTOC_1G_KX);
-	hw->mac.link_mode_select = IXGBE_AUTOC_LMS_KX4_AN;
-
 	/* Set up MAC */
 	ixgbe_setup_mac_link_82598(hw);
 
@@ -712,110 +658,12 @@ static s32 ixgbe_setup_copper_link_speed_82598(struct ixgbe_hw *hw,
 	/* Setup the PHY according to input speed */
 	status = hw->phy.ops.setup_link_speed(hw, speed, autoneg,
 	                                      autoneg_wait_to_complete);
-
-	/* Set MAC to KX/KX4 autoneg, which defaults to Parallel detection */
-	hw->mac.link_attach_type = (IXGBE_AUTOC_10G_KX4 | IXGBE_AUTOC_1G_KX);
-	hw->mac.link_mode_select = IXGBE_AUTOC_LMS_KX4_AN;
-
 	/* Set up MAC */
 	ixgbe_setup_mac_link_82598(hw);
 
 	return status;
 }
 
-#ifndef NO_82598_A0_SUPPORT
-/**
- *  ixgbe_reset_hw_rev_0_82598 - Performs hardware reset
- *  @hw: pointer to hardware structure
- *
- *  Resets the hardware by resetting the transmit and receive units, masks and
- *  clears all interrupts, performing a PHY reset, and performing a link (MAC)
- *  reset.
- **/
-s32 ixgbe_reset_hw_rev_0_82598(struct ixgbe_hw *hw)
-{
-	s32 status = IXGBE_SUCCESS;
-	u32 ctrl;
-	u32 gheccr;
-	u32 autoc;
-	u32 i;
-	u32 resets;
-
-	/* Call adapter stop to disable tx/rx and clear interrupts */
-	hw->mac.ops.stop_adapter(hw);
-
-	/* Reset PHY */
-	hw->phy.ops.reset(hw);
-
-	for (resets = 0; resets < 10; resets++) {
-		/*
-		 * Prevent the PCI-E bus from from hanging by disabling PCI-E
-		 * master access and verify no pending requests before reset
-		 */
-		if (ixgbe_disable_pcie_master(hw) != IXGBE_SUCCESS) {
-			status = IXGBE_ERR_MASTER_REQUESTS_PENDING;
-			DEBUGOUT("PCI-E Master disable polling has failed.\n");
-		}
-
-		/*
-		 * Issue global reset to the MAC.  This needs to be a SW reset.
-		 * If link reset is used, it might reset the MAC when mng is
-		 * using it.
-		 */
-		ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
-		IXGBE_WRITE_REG(hw, IXGBE_CTRL, (ctrl | IXGBE_CTRL_RST));
-		IXGBE_WRITE_FLUSH(hw);
-
-		/*
-		 * Poll for reset bit to self-clear indicating reset is
-		 * complete
-		 */
-		for (i = 0; i < 10; i++) {
-			usec_delay(1);
-			ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
-			if (!(ctrl & IXGBE_CTRL_RST))
-				break;
-		}
-		if (ctrl & IXGBE_CTRL_RST) {
-			status = IXGBE_ERR_RESET_FAILED;
-			DEBUGOUT("Reset polling failed to complete.\n");
-		}
-	}
-
-	msec_delay(50);
-
-	gheccr = IXGBE_READ_REG(hw, IXGBE_GHECCR);
-	gheccr &= ~((1 << 21) | (1 << 18) | (1 << 9) | (1 << 6));
-	IXGBE_WRITE_REG(hw, IXGBE_GHECCR, gheccr);
-
-	/*
-	 * AUTOC register which stores link settings gets cleared
-	 * and reloaded from EEPROM after reset. We need to restore
-	 * our stored value from init in case SW changed the attach
-	 * type or speed.  If this is the first time and link settings
-	 * have not been stored, store default settings from AUTOC.
-	 */
-	autoc = IXGBE_READ_REG(hw, IXGBE_AUTOC);
-	if (hw->mac.link_settings_loaded) {
-		autoc &= ~(IXGBE_AUTOC_LMS_ATTACH_TYPE);
-		autoc &= ~(IXGBE_AUTOC_LMS_MASK);
-		autoc |= hw->mac.link_attach_type;
-		autoc |= hw->mac.link_mode_select;
-		IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc);
-	} else {
-		hw->mac.link_attach_type =
-		                         (autoc & IXGBE_AUTOC_LMS_ATTACH_TYPE);
-		hw->mac.link_mode_select = (autoc & IXGBE_AUTOC_LMS_MASK);
-		hw->mac.link_settings_loaded = TRUE;
-	}
-
-	/* Store the permanent mac address */
-	hw->mac.ops.get_mac_addr(hw, hw->mac.perm_addr);
-
-	return status;
-}
-
-#endif /* NO_A0_SUPPORT */
 /**
  *  ixgbe_reset_hw_82598 - Performs hardware reset
  *  @hw: pointer to hardware structure
@@ -870,14 +718,23 @@ static s32 ixgbe_reset_hw_82598(struct ixgbe_hw *hw)
 	}
 
 	/* Reset PHY */
-	if (hw->phy.reset_disable == FALSE)
+	if (hw->phy.reset_disable == FALSE) {
+		/* PHY ops must be identified and initialized prior to reset */
+
+		/* Init PHY and function pointers, perform SFP setup */
+		status = hw->phy.ops.init(hw);
+		if (status == IXGBE_ERR_SFP_NOT_SUPPORTED)
+			goto reset_hw_out;
+
 		hw->phy.ops.reset(hw);
+	}
 
 	/*
 	 * Prevent the PCI-E bus from from hanging by disabling PCI-E master
 	 * access and verify no pending requests before reset
 	 */
-	if (ixgbe_disable_pcie_master(hw) != IXGBE_SUCCESS) {
+	status = ixgbe_disable_pcie_master(hw);
+	if (status != IXGBE_SUCCESS) {
 		status = IXGBE_ERR_MASTER_REQUESTS_PENDING;
 		DEBUGOUT("PCI-E Master disable polling has failed.\n");
 	}
@@ -909,29 +766,27 @@ static s32 ixgbe_reset_hw_82598(struct ixgbe_hw *hw)
 	IXGBE_WRITE_REG(hw, IXGBE_GHECCR, gheccr);
 
 	/*
-	 * AUTOC register which stores link settings gets cleared
-	 * and reloaded from EEPROM after reset. We need to restore
-	 * our stored value from init in case SW changed the attach
-	 * type or speed.  If this is the first time and link settings
-	 * have not been stored, store default settings from AUTOC.
+	 * Store the original AUTOC value if it has not been
+	 * stored off yet.  Otherwise restore the stored original
+	 * AUTOC value since the reset operation sets back to deaults.
 	 */
 	autoc = IXGBE_READ_REG(hw, IXGBE_AUTOC);
-	if (hw->mac.link_settings_loaded) {
-		autoc &= ~(IXGBE_AUTOC_LMS_ATTACH_TYPE);
-		autoc &= ~(IXGBE_AUTOC_LMS_MASK);
-		autoc |= hw->mac.link_attach_type;
-		autoc |= hw->mac.link_mode_select;
-		IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc);
-	} else {
-		hw->mac.link_attach_type =
-		                         (autoc & IXGBE_AUTOC_LMS_ATTACH_TYPE);
-		hw->mac.link_mode_select = (autoc & IXGBE_AUTOC_LMS_MASK);
-		hw->mac.link_settings_loaded = TRUE;
-	}
+	if (hw->mac.orig_link_settings_stored == FALSE) {
+		hw->mac.orig_autoc = autoc;
+		hw->mac.orig_link_settings_stored = TRUE;
+	} else if (autoc != hw->mac.orig_autoc)
+		IXGBE_WRITE_REG(hw, IXGBE_AUTOC, hw->mac.orig_autoc);
+
+	/*
+	 * Store MAC address from RAR0, clear receive address registers, and
+	 * clear the multicast table
+	 */
+	hw->mac.ops.init_rx_addrs(hw);
 
 	/* Store the permanent mac address */
 	hw->mac.ops.get_mac_addr(hw, hw->mac.perm_addr);
 
+reset_hw_out:
 	return status;
 }
 
@@ -1049,61 +904,6 @@ static s32 ixgbe_clear_vfta_82598(struct ixgbe_hw *hw)
 }
 
 /**
- *  ixgbe_blink_led_start_82598 - Blink LED based on index.
- *  @hw: pointer to hardware structure
- *  @index: led number to blink
- **/
-static s32 ixgbe_blink_led_start_82598(struct ixgbe_hw *hw, u32 index)
-{
-	ixgbe_link_speed speed = 0;
-	bool link_up = 0;
-	u32 autoc_reg = IXGBE_READ_REG(hw, IXGBE_AUTOC);
-	u32 led_reg = IXGBE_READ_REG(hw, IXGBE_LEDCTL);
-
-	/*
-	 * Link must be up to auto-blink the LEDs on the 82598EB MAC;
-	 * force it if link is down.
-	 */
-	hw->mac.ops.check_link(hw, &speed, &link_up, FALSE);
-
-	if (!link_up) {
-		autoc_reg |= IXGBE_AUTOC_FLU;
-		IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc_reg);
-		msec_delay(10);
-	}
-
-	led_reg &= ~IXGBE_LED_MODE_MASK(index);
-	led_reg |= IXGBE_LED_BLINK(index);
-	IXGBE_WRITE_REG(hw, IXGBE_LEDCTL, led_reg);
-	IXGBE_WRITE_FLUSH(hw);
-
-	return IXGBE_SUCCESS;
-}
-
-/**
- *  ixgbe_blink_led_stop_82598 - Stop blinking LED based on index.
- *  @hw: pointer to hardware structure
- *  @index: led number to stop blinking
- **/
-static s32 ixgbe_blink_led_stop_82598(struct ixgbe_hw *hw, u32 index)
-{
-	u32 autoc_reg = IXGBE_READ_REG(hw, IXGBE_AUTOC);
-	u32 led_reg = IXGBE_READ_REG(hw, IXGBE_LEDCTL);
-
-	autoc_reg &= ~IXGBE_AUTOC_FLU;
-	autoc_reg |= IXGBE_AUTOC_AN_RESTART;
-	IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc_reg);
-
-	led_reg &= ~IXGBE_LED_MODE_MASK(index);
-	led_reg &= ~IXGBE_LED_BLINK(index);
-	led_reg |= IXGBE_LED_LINK_ACTIVE << IXGBE_LED_MODE_SHIFT(index);
-	IXGBE_WRITE_REG(hw, IXGBE_LEDCTL, led_reg);
-	IXGBE_WRITE_FLUSH(hw);
-
-	return IXGBE_SUCCESS;
-}
-
-/**
  *  ixgbe_read_analog_reg8_82598 - Reads 8 bit Atlas analog register
  *  @hw: pointer to hardware structure
  *  @reg: analog register to read
@@ -1216,33 +1016,56 @@ out:
 u32 ixgbe_get_supported_physical_layer_82598(struct ixgbe_hw *hw)
 {
 	u32 physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN;
+	u32 autoc = IXGBE_READ_REG(hw, IXGBE_AUTOC);
+	u32 pma_pmd_10g = autoc & IXGBE_AUTOC_10G_PMA_PMD_MASK;
+	u32 pma_pmd_1g = autoc & IXGBE_AUTOC_1G_PMA_PMD_MASK;
+	u16 ext_ability = 0;
+
+	hw->phy.ops.identify(hw);
+
+	/* Copper PHY must be checked before AUTOC LMS to determine correct
+	 * physical layer because 10GBase-T PHYs use LMS = KX4/KX */
+	if (hw->phy.type == ixgbe_phy_tn ||
+	    hw->phy.type == ixgbe_phy_cu_unknown) {
+		hw->phy.ops.read_reg(hw, IXGBE_MDIO_PHY_EXT_ABILITY,
+		IXGBE_MDIO_PMA_PMD_DEV_TYPE, &ext_ability);
+		if (ext_ability & IXGBE_MDIO_PHY_10GBASET_ABILITY)
+			physical_layer |= IXGBE_PHYSICAL_LAYER_10GBASE_T;
+		if (ext_ability & IXGBE_MDIO_PHY_1000BASET_ABILITY)
+			physical_layer |= IXGBE_PHYSICAL_LAYER_1000BASE_T;
+		if (ext_ability & IXGBE_MDIO_PHY_100BASETX_ABILITY)
+			physical_layer |= IXGBE_PHYSICAL_LAYER_100BASE_TX;
+		goto out;
+	}
 
-	switch (hw->device_id) {
-	case IXGBE_DEV_ID_82598:
-		/* Default device ID is mezzanine card KX/KX4 */
-		physical_layer = (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
-		                  IXGBE_PHYSICAL_LAYER_1000BASE_KX);
-		break;
-	case IXGBE_DEV_ID_82598EB_CX4:
-	case IXGBE_DEV_ID_82598_CX4_DUAL_PORT:
-		physical_layer = IXGBE_PHYSICAL_LAYER_10GBASE_CX4;
-		break;
-	case IXGBE_DEV_ID_82598_DA_DUAL_PORT:
-		physical_layer = IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU;
+	switch (autoc & IXGBE_AUTOC_LMS_MASK) {
+	case IXGBE_AUTOC_LMS_1G_AN:
+	case IXGBE_AUTOC_LMS_1G_LINK_NO_AN:
+		if (pma_pmd_1g == IXGBE_AUTOC_1G_KX)
+			physical_layer = IXGBE_PHYSICAL_LAYER_1000BASE_KX;
+		else
+			physical_layer = IXGBE_PHYSICAL_LAYER_1000BASE_BX;
 		break;
-	case IXGBE_DEV_ID_82598AF_DUAL_PORT:
-	case IXGBE_DEV_ID_82598AF_SINGLE_PORT:
-	case IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM:
-		physical_layer = IXGBE_PHYSICAL_LAYER_10GBASE_SR;
+	case IXGBE_AUTOC_LMS_10G_LINK_NO_AN:
+		if (pma_pmd_10g == IXGBE_AUTOC_10G_CX4)
+			physical_layer = IXGBE_PHYSICAL_LAYER_10GBASE_CX4;
+		else if (pma_pmd_10g == IXGBE_AUTOC_10G_KX4)
+			physical_layer = IXGBE_PHYSICAL_LAYER_10GBASE_KX4;
+		else /* XAUI */
+			physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN;
 		break;
-	case IXGBE_DEV_ID_82598EB_XF_LR:
-		physical_layer = IXGBE_PHYSICAL_LAYER_10GBASE_LR;
+	case IXGBE_AUTOC_LMS_KX4_AN:
+	case IXGBE_AUTOC_LMS_KX4_AN_1G_AN:
+		if (autoc & IXGBE_AUTOC_KX_SUPP)
+			physical_layer |= IXGBE_PHYSICAL_LAYER_1000BASE_KX;
+		if (autoc & IXGBE_AUTOC_KX4_SUPP)
+			physical_layer |= IXGBE_PHYSICAL_LAYER_10GBASE_KX4;
 		break;
-	case IXGBE_DEV_ID_82598AT:
-		physical_layer = (IXGBE_PHYSICAL_LAYER_10GBASE_T |
-		                  IXGBE_PHYSICAL_LAYER_1000BASE_T);
+	default:
 		break;
-	case IXGBE_DEV_ID_82598EB_SFP_LOM:
+	}
+
+	if (hw->phy.type == ixgbe_phy_nl) {
 		hw->phy.ops.identify_sfp(hw);
 
 		switch (hw->phy.sfp_type) {
@@ -1259,12 +1082,24 @@ u32 ixgbe_get_supported_physical_layer_82598(struct ixgbe_hw *hw)
 			physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN;
 			break;
 		}
-		break;
+	}
 
+	switch (hw->device_id) {
+	case IXGBE_DEV_ID_82598_DA_DUAL_PORT:
+		physical_layer = IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU;
+		break;
+	case IXGBE_DEV_ID_82598AF_DUAL_PORT:
+	case IXGBE_DEV_ID_82598AF_SINGLE_PORT:
+	case IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM:
+		physical_layer = IXGBE_PHYSICAL_LAYER_10GBASE_SR;
+		break;
+	case IXGBE_DEV_ID_82598EB_XF_LR:
+		physical_layer = IXGBE_PHYSICAL_LAYER_10GBASE_LR;
+		break;
 	default:
-		physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN;
 		break;
 	}
 
+out:
 	return physical_layer;
 }
diff --git a/sys/dev/ixgbe/ixgbe_82599.c b/sys/dev/ixgbe/ixgbe_82599.c
new file mode 100644
index 0000000..4f438a2
--- /dev/null
+++ b/sys/dev/ixgbe/ixgbe_82599.c
@@ -0,0 +1,2444 @@
+/******************************************************************************
+
+  Copyright (c) 2001-2009, Intel Corporation 
+  All rights reserved.
+  
+  Redistribution and use in source and binary forms, with or without 
+  modification, are permitted provided that the following conditions are met:
+  
+   1. Redistributions of source code must retain the above copyright notice, 
+      this list of conditions and the following disclaimer.
+  
+   2. Redistributions in binary form must reproduce the above copyright 
+      notice, this list of conditions and the following disclaimer in the 
+      documentation and/or other materials provided with the distribution.
+  
+   3. Neither the name of the Intel Corporation nor the names of its 
+      contributors may be used to endorse or promote products derived from 
+      this software without specific prior written permission.
+  
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
+  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
+  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
+  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
+  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
+  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
+  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
+  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
+  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+  POSSIBILITY OF SUCH DAMAGE.
+
+******************************************************************************/
+/*$FreeBSD$*/
+
+#include "ixgbe_type.h"
+#include "ixgbe_api.h"
+#include "ixgbe_common.h"
+#include "ixgbe_phy.h"
+
+u32 ixgbe_get_pcie_msix_count_82599(struct ixgbe_hw *hw);
+s32 ixgbe_init_ops_82599(struct ixgbe_hw *hw);
+s32 ixgbe_get_link_capabilities_82599(struct ixgbe_hw *hw,
+                                      ixgbe_link_speed *speed,
+                                      bool *autoneg);
+enum ixgbe_media_type ixgbe_get_media_type_82599(struct ixgbe_hw *hw);
+s32 ixgbe_setup_mac_link_multispeed_fiber(struct ixgbe_hw *hw);
+s32 ixgbe_setup_mac_link_speed_multispeed_fiber(struct ixgbe_hw *hw,
+                                     ixgbe_link_speed speed, bool autoneg,
+                                     bool autoneg_wait_to_complete);
+s32 ixgbe_setup_mac_link_82599(struct ixgbe_hw *hw);
+s32 ixgbe_check_mac_link_82599(struct ixgbe_hw *hw,
+                               ixgbe_link_speed *speed,
+                               bool *link_up, bool link_up_wait_to_complete);
+s32 ixgbe_setup_mac_link_speed_82599(struct ixgbe_hw *hw,
+                                     ixgbe_link_speed speed,
+                                     bool autoneg,
+                                     bool autoneg_wait_to_complete);
+static s32 ixgbe_setup_copper_link_82599(struct ixgbe_hw *hw);
+static s32 ixgbe_setup_copper_link_speed_82599(struct ixgbe_hw *hw,
+                                               ixgbe_link_speed speed,
+                                               bool autoneg,
+                                               bool autoneg_wait_to_complete);
+s32 ixgbe_setup_sfp_modules_82599(struct ixgbe_hw *hw);
+void ixgbe_init_mac_link_ops_82599(struct ixgbe_hw *hw);
+s32 ixgbe_reset_hw_82599(struct ixgbe_hw *hw);
+s32 ixgbe_set_vmdq_82599(struct ixgbe_hw *hw, u32 rar, u32 vmdq);
+s32 ixgbe_clear_vmdq_82599(struct ixgbe_hw *hw, u32 rar, u32 vmdq);
+s32 ixgbe_insert_mac_addr_82599(struct ixgbe_hw *hw, u8 *addr, u32 vmdq);
+s32 ixgbe_set_vfta_82599(struct ixgbe_hw *hw, u32 vlan,
+                         u32 vind, bool vlan_on);
+s32 ixgbe_clear_vfta_82599(struct ixgbe_hw *hw);
+s32 ixgbe_init_uta_tables_82599(struct ixgbe_hw *hw);
+s32 ixgbe_read_analog_reg8_82599(struct ixgbe_hw *hw, u32 reg, u8 *val);
+s32 ixgbe_write_analog_reg8_82599(struct ixgbe_hw *hw, u32 reg, u8 val);
+s32 ixgbe_start_hw_rev_1_82599(struct ixgbe_hw *hw);
+s32 ixgbe_identify_phy_82599(struct ixgbe_hw *hw);
+s32 ixgbe_init_phy_ops_82599(struct ixgbe_hw *hw);
+u32 ixgbe_get_supported_physical_layer_82599(struct ixgbe_hw *hw);
+s32 ixgbe_enable_rx_dma_82599(struct ixgbe_hw *hw, u32 regval);
+s32 ixgbe_get_san_mac_addr_offset_82599(struct ixgbe_hw *hw,
+                                        u16 *san_mac_offset);
+s32 ixgbe_get_san_mac_addr_82599(struct ixgbe_hw *hw, u8 *san_mac_addr);
+s32 ixgbe_set_san_mac_addr_82599(struct ixgbe_hw *hw, u8 *san_mac_addr);
+s32 ixgbe_get_device_caps_82599(struct ixgbe_hw *hw, u16 *device_caps);
+
+void ixgbe_init_mac_link_ops_82599(struct ixgbe_hw *hw)
+{
+	struct ixgbe_mac_info *mac = &hw->mac;
+
+	DEBUGFUNC("ixgbe_init_mac_link_ops_82599");
+
+	if (hw->phy.multispeed_fiber) {
+		/* Set up dual speed SFP+ support */
+		mac->ops.setup_link =
+		          &ixgbe_setup_mac_link_multispeed_fiber;
+		mac->ops.setup_link_speed =
+		          &ixgbe_setup_mac_link_speed_multispeed_fiber;
+	} else {
+		mac->ops.setup_link =
+			&ixgbe_setup_mac_link_82599;
+		mac->ops.setup_link_speed =
+		          &ixgbe_setup_mac_link_speed_82599;
+	}
+}
+
+/**
+ *  ixgbe_init_phy_ops_82599 - PHY/SFP specific init
+ *  @hw: pointer to hardware structure
+ *
+ *  Initialize any function pointers that were not able to be
+ *  set during init_shared_code because the PHY/SFP type was
+ *  not known.  Perform the SFP init if necessary.
+ *
+ **/
+s32 ixgbe_init_phy_ops_82599(struct ixgbe_hw *hw)
+{
+	struct ixgbe_mac_info *mac = &hw->mac;
+	struct ixgbe_phy_info *phy = &hw->phy;
+	s32 ret_val = IXGBE_SUCCESS;
+
+	DEBUGFUNC("ixgbe_init_phy_ops_82599");
+
+	/* Identify the PHY or SFP module */
+	ret_val = phy->ops.identify(hw);
+	if (ret_val == IXGBE_ERR_SFP_NOT_SUPPORTED)
+		goto init_phy_ops_out;
+
+	/* Setup function pointers based on detected SFP module and speeds */
+	ixgbe_init_mac_link_ops_82599(hw);
+	if (hw->phy.sfp_type != ixgbe_sfp_type_unknown)
+		hw->phy.ops.reset = NULL;
+
+	/* If copper media, overwrite with copper function pointers */
+	if (mac->ops.get_media_type(hw) == ixgbe_media_type_copper) {
+		mac->ops.setup_link = &ixgbe_setup_copper_link_82599;
+		mac->ops.setup_link_speed =
+		                     &ixgbe_setup_copper_link_speed_82599;
+		mac->ops.get_link_capabilities =
+		                  &ixgbe_get_copper_link_capabilities_generic;
+	}
+
+	/* Set necessary function pointers based on phy type */
+	switch (hw->phy.type) {
+	case ixgbe_phy_tn:
+		phy->ops.check_link = &ixgbe_check_phy_link_tnx;
+		phy->ops.get_firmware_version =
+		             &ixgbe_get_phy_firmware_version_tnx;
+		break;
+	case ixgbe_phy_aq:
+		phy->ops.get_firmware_version =
+		             &ixgbe_get_phy_firmware_version_aq;
+		break;
+	default:
+		break;
+	}
+init_phy_ops_out:
+	return ret_val;
+}
+
+s32 ixgbe_setup_sfp_modules_82599(struct ixgbe_hw *hw)
+{
+	s32 ret_val = IXGBE_SUCCESS;
+	u16 list_offset, data_offset, data_value;
+
+	DEBUGFUNC("ixgbe_setup_sfp_modules_82599");
+
+	if (hw->phy.sfp_type != ixgbe_sfp_type_unknown) {
+		ixgbe_init_mac_link_ops_82599(hw);
+
+		hw->phy.ops.reset = NULL;
+
+		ret_val = ixgbe_get_sfp_init_sequence_offsets(hw, &list_offset,
+		                                              &data_offset);
+		if (ret_val != IXGBE_SUCCESS)
+			goto setup_sfp_out;
+
+		hw->eeprom.ops.read(hw, ++data_offset, &data_value);
+		while (data_value != 0xffff) {
+			IXGBE_WRITE_REG(hw, IXGBE_CORECTL, data_value);
+			IXGBE_WRITE_FLUSH(hw);
+			hw->eeprom.ops.read(hw, ++data_offset, &data_value);
+		}
+		/* Now restart DSP */
+		IXGBE_WRITE_REG(hw, IXGBE_CORECTL, 0x00000102);
+		IXGBE_WRITE_REG(hw, IXGBE_CORECTL, 0x00000b1d);
+		IXGBE_WRITE_FLUSH(hw);
+	}
+
+setup_sfp_out:
+	return ret_val;
+}
+
+/**
+ *  ixgbe_get_pcie_msix_count_82599 - Gets MSI-X vector count
+ *  @hw: pointer to hardware structure
+ *
+ *  Read PCIe configuration space, and get the MSI-X vector count from
+ *  the capabilities table.
+ **/
+u32 ixgbe_get_pcie_msix_count_82599(struct ixgbe_hw *hw)
+{
+	u32 msix_count = 64;
+
+	if (hw->mac.msix_vectors_from_pcie) {
+		msix_count = IXGBE_READ_PCIE_WORD(hw,
+		                                  IXGBE_PCIE_MSIX_82599_CAPS);
+		msix_count &= IXGBE_PCIE_MSIX_TBL_SZ_MASK;
+
+		/* MSI-X count is zero-based in HW, so increment to give
+		 * proper value */
+		msix_count++;
+	}
+
+	return msix_count;
+}
+
+/**
+ *  ixgbe_init_ops_82599 - Inits func ptrs and MAC type
+ *  @hw: pointer to hardware structure
+ *
+ *  Initialize the function pointers and assign the MAC type for 82599.
+ *  Does not touch the hardware.
+ **/
+
+s32 ixgbe_init_ops_82599(struct ixgbe_hw *hw)
+{
+	struct ixgbe_mac_info *mac = &hw->mac;
+	struct ixgbe_phy_info *phy = &hw->phy;
+	s32 ret_val;
+
+	ret_val = ixgbe_init_phy_ops_generic(hw);
+	ret_val = ixgbe_init_ops_generic(hw);
+
+	/* PHY */
+	phy->ops.identify = &ixgbe_identify_phy_82599;
+	phy->ops.init = &ixgbe_init_phy_ops_82599;
+
+	/* MAC */
+	mac->ops.reset_hw = &ixgbe_reset_hw_82599;
+	mac->ops.get_media_type = &ixgbe_get_media_type_82599;
+	mac->ops.get_supported_physical_layer =
+	                            &ixgbe_get_supported_physical_layer_82599;
+	mac->ops.enable_rx_dma = &ixgbe_enable_rx_dma_82599;
+	mac->ops.read_analog_reg8 = &ixgbe_read_analog_reg8_82599;
+	mac->ops.write_analog_reg8 = &ixgbe_write_analog_reg8_82599;
+	mac->ops.start_hw = &ixgbe_start_hw_rev_1_82599;
+	mac->ops.get_san_mac_addr = &ixgbe_get_san_mac_addr_82599;
+	mac->ops.set_san_mac_addr = &ixgbe_set_san_mac_addr_82599;
+	mac->ops.get_device_caps = &ixgbe_get_device_caps_82599;
+
+	/* RAR, Multicast, VLAN */
+	mac->ops.set_vmdq = &ixgbe_set_vmdq_82599;
+	mac->ops.clear_vmdq = &ixgbe_clear_vmdq_82599;
+	mac->ops.insert_mac_addr = &ixgbe_insert_mac_addr_82599;
+	mac->rar_highwater = 1;
+	mac->ops.set_vfta = &ixgbe_set_vfta_82599;
+	mac->ops.clear_vfta = &ixgbe_clear_vfta_82599;
+	mac->ops.init_uta_tables = &ixgbe_init_uta_tables_82599;
+	mac->ops.setup_sfp = &ixgbe_setup_sfp_modules_82599;
+
+	/* Link */
+	mac->ops.get_link_capabilities = &ixgbe_get_link_capabilities_82599;
+	mac->ops.check_link            = &ixgbe_check_mac_link_82599;
+	ixgbe_init_mac_link_ops_82599(hw);
+
+	mac->mcft_size        = 128;
+	mac->vft_size         = 128;
+	mac->num_rar_entries  = 128;
+	mac->max_tx_queues    = 128;
+	mac->max_rx_queues    = 128;
+	mac->max_msix_vectors = ixgbe_get_pcie_msix_count_82599(hw);
+
+	return ret_val;
+}
+
+/**
+ *  ixgbe_get_link_capabilities_82599 - Determines link capabilities
+ *  @hw: pointer to hardware structure
+ *  @speed: pointer to link speed
+ *  @negotiation: TRUE when autoneg or autotry is enabled
+ *
+ *  Determines the link capabilities by reading the AUTOC register.
+ **/
+s32 ixgbe_get_link_capabilities_82599(struct ixgbe_hw *hw,
+                                      ixgbe_link_speed *speed,
+                                      bool *negotiation)
+{
+	s32 status = IXGBE_SUCCESS;
+	u32 autoc = 0;
+
+	/*
+	 * Determine link capabilities based on the stored value of AUTOC,
+	 * which represents EEPROM defaults.  If AUTOC value has not
+	 * been stored, use the current register values.
+	 */
+	if (hw->mac.orig_link_settings_stored)
+		autoc = hw->mac.orig_autoc;
+	else
+		autoc = IXGBE_READ_REG(hw, IXGBE_AUTOC);
+
+	switch (autoc & IXGBE_AUTOC_LMS_MASK) {
+	case IXGBE_AUTOC_LMS_1G_LINK_NO_AN:
+		*speed = IXGBE_LINK_SPEED_1GB_FULL;
+		*negotiation = FALSE;
+		break;
+
+	case IXGBE_AUTOC_LMS_10G_LINK_NO_AN:
+		*speed = IXGBE_LINK_SPEED_10GB_FULL;
+		*negotiation = FALSE;
+		break;
+
+	case IXGBE_AUTOC_LMS_1G_AN:
+		*speed = IXGBE_LINK_SPEED_1GB_FULL;
+		*negotiation = TRUE;
+		break;
+
+	case IXGBE_AUTOC_LMS_10G_SERIAL:
+		*speed = IXGBE_LINK_SPEED_10GB_FULL;
+		*negotiation = FALSE;
+		break;
+
+	case IXGBE_AUTOC_LMS_KX4_KX_KR:
+	case IXGBE_AUTOC_LMS_KX4_KX_KR_1G_AN:
+		*speed = IXGBE_LINK_SPEED_UNKNOWN;
+		if (autoc & IXGBE_AUTOC_KR_SUPP)
+			*speed |= IXGBE_LINK_SPEED_10GB_FULL;
+		if (autoc & IXGBE_AUTOC_KX4_SUPP)
+			*speed |= IXGBE_LINK_SPEED_10GB_FULL;
+		if (autoc & IXGBE_AUTOC_KX_SUPP)
+			*speed |= IXGBE_LINK_SPEED_1GB_FULL;
+		*negotiation = TRUE;
+		break;
+
+	case IXGBE_AUTOC_LMS_KX4_KX_KR_SGMII:
+		*speed = IXGBE_LINK_SPEED_100_FULL;
+		if (autoc & IXGBE_AUTOC_KR_SUPP)
+			*speed |= IXGBE_LINK_SPEED_10GB_FULL;
+		if (autoc & IXGBE_AUTOC_KX4_SUPP)
+			*speed |= IXGBE_LINK_SPEED_10GB_FULL;
+		if (autoc & IXGBE_AUTOC_KX_SUPP)
+			*speed |= IXGBE_LINK_SPEED_1GB_FULL;
+		*negotiation = TRUE;
+		break;
+
+	case IXGBE_AUTOC_LMS_SGMII_1G_100M:
+		*speed = IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_100_FULL;
+		*negotiation = FALSE;
+		break;
+
+	default:
+		status = IXGBE_ERR_LINK_SETUP;
+		goto out;
+		break;
+	}
+
+	if (hw->phy.multispeed_fiber) {
+		*speed |= IXGBE_LINK_SPEED_10GB_FULL |
+		          IXGBE_LINK_SPEED_1GB_FULL;
+		*negotiation = TRUE;
+	}
+
+out:
+	return status;
+}
+
+/**
+ *  ixgbe_get_media_type_82599 - Get media type
+ *  @hw: pointer to hardware structure
+ *
+ *  Returns the media type (fiber, copper, backplane)
+ **/
+enum ixgbe_media_type ixgbe_get_media_type_82599(struct ixgbe_hw *hw)
+{
+	enum ixgbe_media_type media_type;
+
+	/* Detect if there is a copper PHY attached. */
+	if (hw->phy.type == ixgbe_phy_cu_unknown ||
+	    hw->phy.type == ixgbe_phy_tn ||
+	    hw->phy.type == ixgbe_phy_aq) {
+		media_type = ixgbe_media_type_copper;
+		goto out;
+	}
+
+	switch (hw->device_id) {
+	case IXGBE_DEV_ID_82599_KX4:
+		/* Default device ID is mezzanine card KX/KX4 */
+		media_type = ixgbe_media_type_backplane;
+		break;
+	case IXGBE_DEV_ID_82599_SFP:
+		media_type = ixgbe_media_type_fiber;
+		break;
+	case IXGBE_DEV_ID_82599_CX4:
+		media_type = ixgbe_media_type_fiber;
+		break;
+	default:
+		media_type = ixgbe_media_type_unknown;
+		break;
+	}
+out:
+	return media_type;
+}
+
+/**
+ *  ixgbe_setup_mac_link_82599 - Setup MAC link settings
+ *  @hw: pointer to hardware structure
+ *
+ *  Configures link settings based on values in the ixgbe_hw struct.
+ *  Restarts the link.  Performs autonegotiation if needed.
+ **/
+s32 ixgbe_setup_mac_link_82599(struct ixgbe_hw *hw)
+{
+	u32 autoc_reg;
+	u32 links_reg;
+	u32 i;
+	s32 status = IXGBE_SUCCESS;
+
+	/* Restart link */
+	autoc_reg = IXGBE_READ_REG(hw, IXGBE_AUTOC);
+	autoc_reg |= IXGBE_AUTOC_AN_RESTART;
+	IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc_reg);
+
+	/* Only poll for autoneg to complete if specified to do so */
+	if (hw->phy.autoneg_wait_to_complete) {
+		if ((autoc_reg & IXGBE_AUTOC_LMS_MASK) ==
+		     IXGBE_AUTOC_LMS_KX4_KX_KR ||
+		    (autoc_reg & IXGBE_AUTOC_LMS_MASK) ==
+		     IXGBE_AUTOC_LMS_KX4_KX_KR_1G_AN
+		    || (autoc_reg & IXGBE_AUTOC_LMS_MASK) ==
+		     IXGBE_AUTOC_LMS_KX4_KX_KR_SGMII) {
+			links_reg = 0; /* Just in case Autoneg time = 0 */
+			for (i = 0; i < IXGBE_AUTO_NEG_TIME; i++) {
+				links_reg = IXGBE_READ_REG(hw, IXGBE_LINKS);
+				if (links_reg & IXGBE_LINKS_KX_AN_COMP)
+					break;
+				msec_delay(100);
+			}
+			if (!(links_reg & IXGBE_LINKS_KX_AN_COMP)) {
+				status = IXGBE_ERR_AUTONEG_NOT_COMPLETE;
+				DEBUGOUT("Autoneg did not complete.\n");
+			}
+		}
+	}
+
+	/* Add delay to filter out noises during initial link setup */
+	msec_delay(50);
+
+	return status;
+}
+
+/**
+ *  ixgbe_setup_mac_link_multispeed_fiber - Setup MAC link settings
+ *  @hw: pointer to hardware structure
+ *
+ *  Configures link settings based on values in the ixgbe_hw struct.
+ *  Restarts the link for multi-speed fiber at 1G speed, if link
+ *  fails at 10G.
+ *  Performs autonegotiation if needed.
+ **/
+s32 ixgbe_setup_mac_link_multispeed_fiber(struct ixgbe_hw *hw)
+{
+	s32 status = IXGBE_SUCCESS;
+	ixgbe_link_speed link_speed = IXGBE_LINK_SPEED_82599_AUTONEG;
+	DEBUGFUNC("ixgbe_setup_mac_link_multispeed_fiber");
+
+	status = ixgbe_setup_mac_link_speed_multispeed_fiber(hw,
+	                                               link_speed, TRUE, true);
+	return status;
+}
+
+/**
+ *  ixgbe_setup_mac_link_speed_multispeed_fiber - Set MAC link speed
+ *  @hw: pointer to hardware structure
+ *  @speed: new link speed
+ *  @autoneg: TRUE if autonegotiation enabled
+ *  @autoneg_wait_to_complete: TRUE when waiting for completion is needed
+ *
+ *  Set the link speed in the AUTOC register and restarts link.
+ **/
+s32 ixgbe_setup_mac_link_speed_multispeed_fiber(struct ixgbe_hw *hw,
+                                     ixgbe_link_speed speed, bool autoneg,
+                                     bool autoneg_wait_to_complete)
+{
+	s32 status = IXGBE_SUCCESS;
+	ixgbe_link_speed link_speed;
+	ixgbe_link_speed highest_link_speed = IXGBE_LINK_SPEED_UNKNOWN;
+	u32 speedcnt = 0;
+	u32 esdp_reg = IXGBE_READ_REG(hw, IXGBE_ESDP);
+	bool link_up = FALSE;
+	bool negotiation;
+
+	/* Mask off requested but non-supported speeds */
+	status = ixgbe_get_link_capabilities(hw, &link_speed, &negotiation);
+	if (status != IXGBE_SUCCESS)
+		goto out;
+
+	speed &= link_speed;
+
+	/*
+	 * Try each speed one by one, highest priority first.  We do this in
+	 * software because 10gb fiber doesn't support speed autonegotiation.
+	 */
+	if (speed & IXGBE_LINK_SPEED_10GB_FULL) {
+		speedcnt++;
+		highest_link_speed = IXGBE_LINK_SPEED_10GB_FULL;
+
+		/* If we already have link at this speed, just jump out */
+		status = ixgbe_check_link(hw, &link_speed, &link_up, FALSE);
+		if (status != IXGBE_SUCCESS)
+			goto out;
+
+		if ((link_speed == IXGBE_LINK_SPEED_10GB_FULL) && link_up)
+			goto out;
+
+		/* Set hardware SDP's */
+		esdp_reg |= (IXGBE_ESDP_SDP5_DIR | IXGBE_ESDP_SDP5);
+		IXGBE_WRITE_REG(hw, IXGBE_ESDP, esdp_reg);
+
+		/* Allow module to change analog characteristics (1G->10G) */
+		msec_delay(40);
+
+		status = ixgbe_setup_mac_link_speed_82599(
+			hw, IXGBE_LINK_SPEED_10GB_FULL, autoneg,
+			autoneg_wait_to_complete);
+		if (status != IXGBE_SUCCESS)
+			goto out;
+
+		msec_delay(100);
+
+		/* If we have link, just jump out */
+		status = ixgbe_check_link(hw, &link_speed, &link_up, FALSE);
+		if (status != IXGBE_SUCCESS)
+			goto out;
+
+		if (link_up)
+			goto out;
+	}
+
+	if (speed & IXGBE_LINK_SPEED_1GB_FULL) {
+		speedcnt++;
+		if (highest_link_speed == IXGBE_LINK_SPEED_UNKNOWN)
+			highest_link_speed = IXGBE_LINK_SPEED_1GB_FULL;
+
+		/* If we already have link at this speed, just jump out */
+		status = ixgbe_check_link(hw, &link_speed, &link_up, FALSE);
+		if (status != IXGBE_SUCCESS)
+			goto out;
+
+		if ((link_speed == IXGBE_LINK_SPEED_1GB_FULL) && link_up)
+			goto out;
+
+		/* Set hardware SDP's */
+		esdp_reg &= ~IXGBE_ESDP_SDP5;
+		esdp_reg |= IXGBE_ESDP_SDP5_DIR;
+		IXGBE_WRITE_REG(hw, IXGBE_ESDP, esdp_reg);
+
+		/* Allow module to change analog characteristics (10G->1G) */
+		msec_delay(40);
+
+		status = ixgbe_setup_mac_link_speed_82599(
+			hw, IXGBE_LINK_SPEED_1GB_FULL, autoneg,
+			autoneg_wait_to_complete);
+		if (status != IXGBE_SUCCESS)
+			goto out;
+
+		msec_delay(100);
+
+		/* If we have link, just jump out */
+		status = ixgbe_check_link(hw, &link_speed, &link_up, FALSE);
+		if (status != IXGBE_SUCCESS)
+			goto out;
+
+		if (link_up)
+			goto out;
+	}
+
+	/*
+	 * We didn't get link.  Configure back to the highest speed we tried,
+	 * (if there was more than one).  We call ourselves back with just the
+	 * single highest speed that the user requested.
+	 */
+	if (speedcnt > 1)
+		status = ixgbe_setup_mac_link_speed_multispeed_fiber(hw,
+		        highest_link_speed, autoneg, autoneg_wait_to_complete);
+
+out:
+	return status;
+}
+
+/**
+ *  ixgbe_check_mac_link_82599 - Determine link and speed status
+ *  @hw: pointer to hardware structure
+ *  @speed: pointer to link speed
+ *  @link_up: TRUE when link is up
+ *  @link_up_wait_to_complete: bool used to wait for link up or not
+ *
+ *  Reads the links register to determine if link is up and the current speed
+ **/
+s32 ixgbe_check_mac_link_82599(struct ixgbe_hw *hw, ixgbe_link_speed *speed,
+                               bool *link_up, bool link_up_wait_to_complete)
+{
+	u32 links_reg;
+	u32 i;
+
+	links_reg = IXGBE_READ_REG(hw, IXGBE_LINKS);
+	if (link_up_wait_to_complete) {
+		for (i = 0; i < IXGBE_LINK_UP_TIME; i++) {
+			if (links_reg & IXGBE_LINKS_UP) {
+				*link_up = TRUE;
+				break;
+			} else {
+				*link_up = FALSE;
+			}
+			msec_delay(100);
+			links_reg = IXGBE_READ_REG(hw, IXGBE_LINKS);
+		}
+	} else {
+		if (links_reg & IXGBE_LINKS_UP)
+			*link_up = TRUE;
+		else
+			*link_up = FALSE;
+	}
+
+	if ((links_reg & IXGBE_LINKS_SPEED_82599) ==
+	    IXGBE_LINKS_SPEED_10G_82599)
+		*speed = IXGBE_LINK_SPEED_10GB_FULL;
+	else if ((links_reg & IXGBE_LINKS_SPEED_82599) ==
+	         IXGBE_LINKS_SPEED_1G_82599)
+		*speed = IXGBE_LINK_SPEED_1GB_FULL;
+	else
+		*speed = IXGBE_LINK_SPEED_100_FULL;
+
+	/* if link is down, zero out the current_mode */
+	if (*link_up == FALSE) {
+		hw->fc.current_mode = ixgbe_fc_none;
+		hw->fc.fc_was_autonegged = FALSE;
+	}
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_setup_mac_link_speed_82599 - Set MAC link speed
+ *  @hw: pointer to hardware structure
+ *  @speed: new link speed
+ *  @autoneg: TRUE if autonegotiation enabled
+ *  @autoneg_wait_to_complete: TRUE when waiting for completion is needed
+ *
+ *  Set the link speed in the AUTOC register and restarts link.
+ **/
+s32 ixgbe_setup_mac_link_speed_82599(struct ixgbe_hw *hw,
+                                     ixgbe_link_speed speed, bool autoneg,
+                                     bool autoneg_wait_to_complete)
+{
+	s32 status = IXGBE_SUCCESS;
+	u32 autoc = IXGBE_READ_REG(hw, IXGBE_AUTOC);
+	u32 autoc2 = IXGBE_READ_REG(hw, IXGBE_AUTOC2);
+	u32 orig_autoc = 0;
+	u32 link_mode = autoc & IXGBE_AUTOC_LMS_MASK;
+	u32 pma_pmd_1g = autoc & IXGBE_AUTOC_1G_PMA_PMD_MASK;
+	u32 pma_pmd_10g_serial = autoc2 & IXGBE_AUTOC2_10G_SERIAL_PMA_PMD_MASK;
+	u32 links_reg;
+	u32 i;
+	ixgbe_link_speed link_capabilities = IXGBE_LINK_SPEED_UNKNOWN;
+
+	/* Check to see if speed passed in is supported. */
+	status = ixgbe_get_link_capabilities(hw, &link_capabilities, &autoneg);
+	if (status != IXGBE_SUCCESS)
+		goto out;
+
+	speed &= link_capabilities;
+
+	/* Use stored value (EEPROM defaults) of AUTOC to find KR/KX4 support*/
+	if (hw->mac.orig_link_settings_stored)
+		orig_autoc = hw->mac.orig_autoc;
+	else
+		orig_autoc = autoc;
+
+
+	if (speed == IXGBE_LINK_SPEED_UNKNOWN) {
+		status = IXGBE_ERR_LINK_SETUP;
+	} else if (link_mode == IXGBE_AUTOC_LMS_KX4_KX_KR ||
+	         link_mode == IXGBE_AUTOC_LMS_KX4_KX_KR_1G_AN ||
+	         link_mode == IXGBE_AUTOC_LMS_KX4_KX_KR_SGMII) {
+		/* Set KX4/KX/KR support according to speed requested */
+		autoc &= ~(IXGBE_AUTOC_KX4_KX_SUPP_MASK | IXGBE_AUTOC_KR_SUPP);
+		if (speed & IXGBE_LINK_SPEED_10GB_FULL)
+			if (orig_autoc & IXGBE_AUTOC_KX4_SUPP)
+				autoc |= IXGBE_AUTOC_KX4_SUPP;
+			if (orig_autoc & IXGBE_AUTOC_KR_SUPP)
+				autoc |= IXGBE_AUTOC_KR_SUPP;
+		if (speed & IXGBE_LINK_SPEED_1GB_FULL)
+			autoc |= IXGBE_AUTOC_KX_SUPP;
+	} else if ((pma_pmd_1g == IXGBE_AUTOC_1G_SFI) &&
+	         (link_mode == IXGBE_AUTOC_LMS_1G_LINK_NO_AN ||
+	          link_mode == IXGBE_AUTOC_LMS_1G_AN)) {
+		/* Switch from 1G SFI to 10G SFI if requested */
+		if ((speed == IXGBE_LINK_SPEED_10GB_FULL) &&
+		    (pma_pmd_10g_serial == IXGBE_AUTOC2_10G_SFI)) {
+			autoc &= ~IXGBE_AUTOC_LMS_MASK;
+			autoc |= IXGBE_AUTOC_LMS_10G_SERIAL;
+		}
+	} else if ((pma_pmd_10g_serial == IXGBE_AUTOC2_10G_SFI) &&
+	         (link_mode == IXGBE_AUTOC_LMS_10G_SERIAL)) {
+		/* Switch from 10G SFI to 1G SFI if requested */
+		if ((speed == IXGBE_LINK_SPEED_1GB_FULL) &&
+		    (pma_pmd_1g == IXGBE_AUTOC_1G_SFI)) {
+			autoc &= ~IXGBE_AUTOC_LMS_MASK;
+			if (autoneg)
+				autoc |= IXGBE_AUTOC_LMS_1G_AN;
+			else
+				autoc |= IXGBE_AUTOC_LMS_1G_LINK_NO_AN;
+		}
+	}
+
+	if (status == IXGBE_SUCCESS) {
+		/* Restart link */
+		autoc |= IXGBE_AUTOC_AN_RESTART;
+		IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc);
+
+		/* Only poll for autoneg to complete if specified to do so */
+		if (autoneg_wait_to_complete) {
+			if (link_mode == IXGBE_AUTOC_LMS_KX4_KX_KR ||
+			    link_mode == IXGBE_AUTOC_LMS_KX4_KX_KR_1G_AN ||
+			    link_mode == IXGBE_AUTOC_LMS_KX4_KX_KR_SGMII) {
+				links_reg = 0; /*Just in case Autoneg time=0*/
+				for (i = 0; i < IXGBE_AUTO_NEG_TIME; i++) {
+					links_reg =
+					       IXGBE_READ_REG(hw, IXGBE_LINKS);
+					if (links_reg & IXGBE_LINKS_KX_AN_COMP)
+						break;
+					msec_delay(100);
+				}
+				if (!(links_reg & IXGBE_LINKS_KX_AN_COMP)) {
+					status =
+						IXGBE_ERR_AUTONEG_NOT_COMPLETE;
+					DEBUGOUT("Autoneg did not complete.\n");
+				}
+			}
+		}
+
+		/* Add delay to filter out noises during initial link setup */
+		msec_delay(50);
+	}
+
+out:
+	return status;
+}
+
+/**
+ *  ixgbe_setup_copper_link_82599 - Setup copper link settings
+ *  @hw: pointer to hardware structure
+ *
+ *  Restarts the link on PHY and then MAC. Performs autonegotiation if needed.
+ **/
+static s32 ixgbe_setup_copper_link_82599(struct ixgbe_hw *hw)
+{
+	s32 status;
+
+	/* Restart autonegotiation on PHY */
+	status = hw->phy.ops.setup_link(hw);
+
+	/* Set up MAC */
+	ixgbe_setup_mac_link_82599(hw);
+
+	return status;
+}
+
+/**
+ *  ixgbe_setup_copper_link_speed_82599 - Set the PHY autoneg advertised field
+ *  @hw: pointer to hardware structure
+ *  @speed: new link speed
+ *  @autoneg: TRUE if autonegotiation enabled
+ *  @autoneg_wait_to_complete: TRUE if waiting is needed to complete
+ *
+ *  Restarts link on PHY and MAC based on settings passed in.
+ **/
+static s32 ixgbe_setup_copper_link_speed_82599(struct ixgbe_hw *hw,
+                                               ixgbe_link_speed speed,
+                                               bool autoneg,
+                                               bool autoneg_wait_to_complete)
+{
+	s32 status;
+
+	/* Setup the PHY according to input speed */
+	status = hw->phy.ops.setup_link_speed(hw, speed, autoneg,
+	                                      autoneg_wait_to_complete);
+	/* Set up MAC */
+	ixgbe_setup_mac_link_82599(hw);
+
+	return status;
+}
+/**
+ *  ixgbe_reset_hw_82599 - Perform hardware reset
+ *  @hw: pointer to hardware structure
+ *
+ *  Resets the hardware by resetting the transmit and receive units, masks
+ *  and clears all interrupts, perform a PHY reset, and perform a link (MAC)
+ *  reset.
+ **/
+s32 ixgbe_reset_hw_82599(struct ixgbe_hw *hw)
+{
+	s32 status = IXGBE_SUCCESS;
+	u32 ctrl, ctrl_ext;
+	u32 i;
+	u32 autoc;
+	u32 autoc2;
+
+	/* Call adapter stop to disable tx/rx and clear interrupts */
+	hw->mac.ops.stop_adapter(hw);
+
+	/* PHY ops must be identified and initialized prior to reset */
+
+	/* Identify PHY and related function pointers */
+	status = hw->phy.ops.init(hw);
+
+	if (status == IXGBE_ERR_SFP_NOT_SUPPORTED)
+		goto reset_hw_out;
+
+
+	/* Setup SFP module if there is one present. */
+	if (hw->phy.sfp_setup_needed) {
+		status = hw->mac.ops.setup_sfp(hw);
+		hw->phy.sfp_setup_needed = FALSE;
+	}
+
+	if (status == IXGBE_ERR_SFP_NOT_SUPPORTED)
+		goto reset_hw_out;
+
+	/* Reset PHY */
+	if (hw->phy.reset_disable == FALSE && hw->phy.ops.reset != NULL)
+		hw->phy.ops.reset(hw);
+
+	/*
+	 * Prevent the PCI-E bus from from hanging by disabling PCI-E master
+	 * access and verify no pending requests before reset
+	 */
+	status = ixgbe_disable_pcie_master(hw);
+	if (status != IXGBE_SUCCESS) {
+		status = IXGBE_ERR_MASTER_REQUESTS_PENDING;
+		DEBUGOUT("PCI-E Master disable polling has failed.\n");
+	}
+
+	/*
+	 * Issue global reset to the MAC.  This needs to be a SW reset.
+	 * If link reset is used, it might reset the MAC when mng is using it
+	 */
+	ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
+	IXGBE_WRITE_REG(hw, IXGBE_CTRL, (ctrl | IXGBE_CTRL_RST));
+	IXGBE_WRITE_FLUSH(hw);
+
+	/* Poll for reset bit to self-clear indicating reset is complete */
+	for (i = 0; i < 10; i++) {
+		usec_delay(1);
+		ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
+		if (!(ctrl & IXGBE_CTRL_RST))
+			break;
+	}
+	if (ctrl & IXGBE_CTRL_RST) {
+		status = IXGBE_ERR_RESET_FAILED;
+		DEBUGOUT("Reset polling failed to complete.\n");
+	}
+	/* Clear PF Reset Done bit so PF/VF Mail Ops can work */
+	ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
+	ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
+	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
+
+	msec_delay(50);
+
+
+
+	/*
+	 * Store the original AUTOC/AUTOC2 values if they have not been
+	 * stored off yet.  Otherwise restore the stored original
+	 * values since the reset operation sets back to defaults.
+	 */
+	autoc = IXGBE_READ_REG(hw, IXGBE_AUTOC);
+	autoc2 = IXGBE_READ_REG(hw, IXGBE_AUTOC2);
+	if (hw->mac.orig_link_settings_stored == FALSE) {
+		hw->mac.orig_autoc = autoc;
+		hw->mac.orig_autoc2 = autoc2;
+		hw->mac.orig_link_settings_stored = TRUE;
+	} else {
+		if (autoc != hw->mac.orig_autoc)
+			IXGBE_WRITE_REG(hw, IXGBE_AUTOC, (hw->mac.orig_autoc |
+					IXGBE_AUTOC_AN_RESTART));
+
+		if ((autoc2 & IXGBE_AUTOC2_UPPER_MASK) !=
+		    (hw->mac.orig_autoc2 & IXGBE_AUTOC2_UPPER_MASK)) {
+			autoc2 &= ~IXGBE_AUTOC2_UPPER_MASK;
+			autoc2 |= (hw->mac.orig_autoc2 &
+			           IXGBE_AUTOC2_UPPER_MASK);
+			IXGBE_WRITE_REG(hw, IXGBE_AUTOC2, autoc2);
+		}
+	}
+
+	/*
+	 * Store MAC address from RAR0, clear receive address registers, and
+	 * clear the multicast table.  Also reset num_rar_entries to 128,
+	 * since we modify this value when programming the SAN MAC address.
+	 */
+	hw->mac.num_rar_entries = 128;
+	hw->mac.ops.init_rx_addrs(hw);
+
+	/* Store the permanent mac address */
+	hw->mac.ops.get_mac_addr(hw, hw->mac.perm_addr);
+
+	/* Store the permanent SAN mac address */
+	hw->mac.ops.get_san_mac_addr(hw, hw->mac.san_addr);
+
+	/* Add the SAN MAC address to the RAR only if it's a valid address */
+	if (ixgbe_validate_mac_addr(hw->mac.san_addr) == 0) {
+		hw->mac.ops.set_rar(hw, hw->mac.num_rar_entries - 1,
+		                    hw->mac.san_addr, 0, IXGBE_RAH_AV);
+
+		/* Reserve the last RAR for the SAN MAC address */
+		hw->mac.num_rar_entries--;
+       }
+
+reset_hw_out:
+	return status;
+}
+
+/**
+ *  ixgbe_insert_mac_addr_82599 - Find a RAR for this mac address
+ *  @hw: pointer to hardware structure
+ *  @addr: Address to put into receive address register
+ *  @vmdq: VMDq pool to assign
+ *
+ *  Puts an ethernet address into a receive address register, or
+ *  finds the rar that it is aleady in; adds to the pool list
+ **/
+s32 ixgbe_insert_mac_addr_82599(struct ixgbe_hw *hw, u8 *addr, u32 vmdq)
+{
+	static const u32 NO_EMPTY_RAR_FOUND = 0xFFFFFFFF;
+	u32 first_empty_rar = NO_EMPTY_RAR_FOUND;
+	u32 rar;
+	u32 rar_low, rar_high;
+	u32 addr_low, addr_high;
+
+	/* swap bytes for HW little endian */
+	addr_low  = addr[0] | (addr[1] << 8)
+			    | (addr[2] << 16)
+			    | (addr[3] << 24);
+	addr_high = addr[4] | (addr[5] << 8);
+
+	/*
+	 * Either find the mac_id in rar or find the first empty space.
+	 * rar_highwater points to just after the highest currently used
+	 * rar in order to shorten the search.  It grows when we add a new
+	 * rar to the top.
+	 */
+	for (rar = 0; rar < hw->mac.rar_highwater; rar++) {
+		rar_high = IXGBE_READ_REG(hw, IXGBE_RAH(rar));
+
+		if (((IXGBE_RAH_AV & rar_high) == 0)
+		    && first_empty_rar == NO_EMPTY_RAR_FOUND) {
+			first_empty_rar = rar;
+		} else if ((rar_high & 0xFFFF) == addr_high) {
+			rar_low = IXGBE_READ_REG(hw, IXGBE_RAL(rar));
+			if (rar_low == addr_low)
+				break;    /* found it already in the rars */
+		}
+	}
+
+	if (rar < hw->mac.rar_highwater) {
+		/* already there so just add to the pool bits */
+		ixgbe_set_vmdq(hw, rar, vmdq);
+	} else if (first_empty_rar != NO_EMPTY_RAR_FOUND) {
+		/* stick it into first empty RAR slot we found */
+		rar = first_empty_rar;
+		ixgbe_set_rar(hw, rar, addr, vmdq, IXGBE_RAH_AV);
+	} else if (rar == hw->mac.rar_highwater) {
+		/* add it to the top of the list and inc the highwater mark */
+		ixgbe_set_rar(hw, rar, addr, vmdq, IXGBE_RAH_AV);
+		hw->mac.rar_highwater++;
+	} else if (rar >= hw->mac.num_rar_entries) {
+		return IXGBE_ERR_INVALID_MAC_ADDR;
+	}
+
+	/*
+	 * If we found rar[0], make sure the default pool bit (we use pool 0)
+	 * remains cleared to be sure default pool packets will get delivered
+	 */
+	if (rar == 0)
+		ixgbe_clear_vmdq(hw, rar, 0);
+
+	return rar;
+}
+
+/**
+ *  ixgbe_clear_vmdq_82599 - Disassociate a VMDq pool index from a rx address
+ *  @hw: pointer to hardware struct
+ *  @rar: receive address register index to disassociate
+ *  @vmdq: VMDq pool index to remove from the rar
+ **/
+s32 ixgbe_clear_vmdq_82599(struct ixgbe_hw *hw, u32 rar, u32 vmdq)
+{
+	u32 mpsar_lo, mpsar_hi;
+	u32 rar_entries = hw->mac.num_rar_entries;
+
+	if (rar < rar_entries) {
+		mpsar_lo = IXGBE_READ_REG(hw, IXGBE_MPSAR_LO(rar));
+		mpsar_hi = IXGBE_READ_REG(hw, IXGBE_MPSAR_HI(rar));
+
+		if (!mpsar_lo && !mpsar_hi)
+			goto done;
+
+		if (vmdq == IXGBE_CLEAR_VMDQ_ALL) {
+			if (mpsar_lo) {
+				IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(rar), 0);
+				mpsar_lo = 0;
+			}
+			if (mpsar_hi) {
+				IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(rar), 0);
+				mpsar_hi = 0;
+			}
+		} else if (vmdq < 32) {
+			mpsar_lo &= ~(1 << vmdq);
+			IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(rar), mpsar_lo);
+		} else {
+			mpsar_hi &= ~(1 << (vmdq - 32));
+			IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(rar), mpsar_hi);
+		}
+
+		/* was that the last pool using this rar? */
+		if (mpsar_lo == 0 && mpsar_hi == 0 && rar != 0)
+			hw->mac.ops.clear_rar(hw, rar);
+	} else {
+		DEBUGOUT1("RAR index %d is out of range.\n", rar);
+	}
+
+done:
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_set_vmdq_82599 - Associate a VMDq pool index with a rx address
+ *  @hw: pointer to hardware struct
+ *  @rar: receive address register index to associate with a VMDq index
+ *  @vmdq: VMDq pool index
+ **/
+s32 ixgbe_set_vmdq_82599(struct ixgbe_hw *hw, u32 rar, u32 vmdq)
+{
+	u32 mpsar;
+	u32 rar_entries = hw->mac.num_rar_entries;
+
+	if (rar < rar_entries) {
+		if (vmdq < 32) {
+			mpsar = IXGBE_READ_REG(hw, IXGBE_MPSAR_LO(rar));
+			mpsar |= 1 << vmdq;
+			IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(rar), mpsar);
+		} else {
+			mpsar = IXGBE_READ_REG(hw, IXGBE_MPSAR_HI(rar));
+			mpsar |= 1 << (vmdq - 32);
+			IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(rar), mpsar);
+		}
+	} else {
+		DEBUGOUT1("RAR index %d is out of range.\n", rar);
+	}
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_set_vfta_82599 - Set VLAN filter table
+ *  @hw: pointer to hardware structure
+ *  @vlan: VLAN id to write to VLAN filter
+ *  @vind: VMDq output index that maps queue to VLAN id in VFVFB
+ *  @vlan_on: boolean flag to turn on/off VLAN in VFVF
+ *
+ *  Turn on/off specified VLAN in the VLAN filter table.
+ **/
+s32 ixgbe_set_vfta_82599(struct ixgbe_hw *hw, u32 vlan, u32 vind,
+                           bool vlan_on)
+{
+	u32 regindex;
+	u32 bitindex;
+	u32 bits;
+	u32 first_empty_slot;
+
+	if (vlan > 4095)
+		return IXGBE_ERR_PARAM;
+
+	/*
+	 * this is a 2 part operation - first the VFTA, then the
+	 * VLVF and VLVFB if vind is set
+	 */
+
+	/* Part 1
+	 * The VFTA is a bitstring made up of 128 32-bit registers
+	 * that enable the particular VLAN id, much like the MTA:
+	 *    bits[11-5]: which register
+	 *    bits[4-0]:  which bit in the register
+	 */
+	regindex = (vlan >> 5) & 0x7F;
+	bitindex = vlan & 0x1F;
+	bits = IXGBE_READ_REG(hw, IXGBE_VFTA(regindex));
+	if (vlan_on)
+		bits |= (1 << bitindex);
+	else
+		bits &= ~(1 << bitindex);
+	IXGBE_WRITE_REG(hw, IXGBE_VFTA(regindex), bits);
+
+
+	/* Part 2
+	 * If the vind is set
+	 *   Either vlan_on
+	 *     make sure the vlan is in VLVF
+	 *     set the vind bit in the matching VLVFB
+	 *   Or !vlan_on
+	 *     clear the pool bit and possibly the vind
+	 */
+	if (vind) {
+		/* find the vlanid or the first empty slot */
+		first_empty_slot = 0;
+
+		for (regindex = 1; regindex < IXGBE_VLVF_ENTRIES; regindex++) {
+			bits = IXGBE_READ_REG(hw, IXGBE_VLVF(regindex));
+			if (!bits && !first_empty_slot)
+				first_empty_slot = regindex;
+			else if ((bits & 0x0FFF) == vlan)
+				break;
+		}
+
+		if (regindex >= IXGBE_VLVF_ENTRIES) {
+			if (first_empty_slot)
+				regindex = first_empty_slot;
+			else {
+				DEBUGOUT("No space in VLVF.\n");
+				goto out;
+			}
+		}
+
+
+		if (vlan_on) {
+			/* set the pool bit */
+			if (vind < 32) {
+				bits =
+				   IXGBE_READ_REG(hw, IXGBE_VLVFB(regindex*2));
+				bits |= (1 << vind);
+				IXGBE_WRITE_REG(hw,
+				                IXGBE_VLVFB(regindex*2), bits);
+			} else {
+				bits = IXGBE_READ_REG(hw,
+				                  IXGBE_VLVFB((regindex*2)+1));
+				bits |= (1 << vind);
+				IXGBE_WRITE_REG(hw,
+				            IXGBE_VLVFB((regindex*2)+1), bits);
+			}
+		} else {
+			/* clear the pool bit */
+			if (vind < 32) {
+				bits = IXGBE_READ_REG(hw,
+				     IXGBE_VLVFB(regindex*2));
+			bits &= ~(1 << vind);
+				IXGBE_WRITE_REG(hw,
+				                IXGBE_VLVFB(regindex*2), bits);
+				bits |= IXGBE_READ_REG(hw,
+				                  IXGBE_VLVFB((regindex*2)+1));
+			} else {
+				bits = IXGBE_READ_REG(hw,
+				                  IXGBE_VLVFB((regindex*2)+1));
+				bits &= ~(1 << vind);
+				IXGBE_WRITE_REG(hw,
+	                                    IXGBE_VLVFB((regindex*2)+1), bits);
+				bits |= IXGBE_READ_REG(hw,
+				                      IXGBE_VLVFB(regindex*2));
+			}
+		}
+
+		if (bits)
+			IXGBE_WRITE_REG(hw, IXGBE_VLVF(regindex),
+			                             (IXGBE_VLVF_VIEN | vlan));
+		else
+			IXGBE_WRITE_REG(hw, IXGBE_VLVF(regindex), 0);
+	}
+
+out:
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_clear_vfta_82599 - Clear VLAN filter table
+ *  @hw: pointer to hardware structure
+ *
+ *  Clears the VLAN filer table, and the VMDq index associated with the filter
+ **/
+s32 ixgbe_clear_vfta_82599(struct ixgbe_hw *hw)
+{
+	u32 offset;
+
+	for (offset = 0; offset < hw->mac.vft_size; offset++)
+		IXGBE_WRITE_REG(hw, IXGBE_VFTA(offset), 0);
+
+	for (offset = 0; offset < IXGBE_VLVF_ENTRIES; offset++) {
+		IXGBE_WRITE_REG(hw, IXGBE_VLVF(offset), 0);
+		IXGBE_WRITE_REG(hw, IXGBE_VLVFB(offset*2), 0);
+		IXGBE_WRITE_REG(hw, IXGBE_VLVFB((offset*2)+1), 0);
+	}
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_init_uta_tables_82599 - Initialize the Unicast Table Array
+ *  @hw: pointer to hardware structure
+ **/
+s32 ixgbe_init_uta_tables_82599(struct ixgbe_hw *hw)
+{
+	int i;
+	DEBUGOUT(" Clearing UTA\n");
+
+	for (i = 0; i < 128; i++)
+		IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0);
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_reinit_fdir_tables_82599 - Reinitialize Flow Director tables.
+ *  @hw: pointer to hardware structure
+ **/
+s32 ixgbe_reinit_fdir_tables_82599(struct ixgbe_hw *hw)
+{
+	u32 fdirctrl = IXGBE_READ_REG(hw, IXGBE_FDIRCTRL);
+	fdirctrl &= ~IXGBE_FDIRCTRL_INIT_DONE;
+	IXGBE_WRITE_REG(hw, IXGBE_FDIRFREE, 0);
+	IXGBE_WRITE_FLUSH(hw);
+	IXGBE_WRITE_REG(hw, IXGBE_FDIRCTRL, fdirctrl);
+
+	return IXGBE_SUCCESS;
+}
+
+#define IXGBE_FDIR_INIT_DONE_POLL 10
+/**
+ *  ixgbe_init_fdir_signature_82599 - Initialize Flow Director signature filters
+ *  @hw: pointer to hardware structure
+ *  @pballoc: which mode to allocate filters with
+ **/
+s32 ixgbe_init_fdir_signature_82599(struct ixgbe_hw *hw, u32 pballoc)
+{
+	u32 fdirctrl = 0;
+	u32 pbsize;
+	int i;
+
+	/*
+	 * Before enabling Flow Director, the Rx Packet Buffer size
+	 * must be reduced.  The new value is the current size minus
+	 * flow director memory usage size.
+	 */
+	pbsize = (1 << (IXGBE_FDIR_PBALLOC_SIZE_SHIFT + pballoc));
+	IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(0),
+	    (IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) - pbsize));
+
+	/*
+	 * The defaults in the HW for RX PB 1-7 are not zero and so should be
+	 * intialized to zero for non DCB mode otherwise actual total RX PB
+	 * would be bigger than programmed and filter space would run into
+	 * the PB 0 region.
+	 */
+	for (i = 1; i < 8; i++)
+		IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), 0);
+
+	/* Send interrupt when 64 filters are left */
+	fdirctrl |= 4 << IXGBE_FDIRCTRL_FULL_THRESH_SHIFT;
+
+	/* Set the maximum length per hash bucket to 0xA filters */
+	fdirctrl |= 0xA << IXGBE_FDIRCTRL_MAX_LENGTH_SHIFT;
+
+	switch (pballoc) {
+	case IXGBE_FDIR_PBALLOC_64K:
+		/* 8k - 1 signature filters */
+		fdirctrl |= IXGBE_FDIRCTRL_PBALLOC_64K;
+		break;
+	case IXGBE_FDIR_PBALLOC_128K:
+		/* 16k - 1 signature filters */
+		fdirctrl |= IXGBE_FDIRCTRL_PBALLOC_128K;
+		break;
+	case IXGBE_FDIR_PBALLOC_256K:
+		/* 32k - 1 signature filters */
+		fdirctrl |= IXGBE_FDIRCTRL_PBALLOC_256K;
+		break;
+	default:
+		/* bad value */
+		return IXGBE_ERR_CONFIG;
+	};
+
+	/* Move the flexible bytes to use the ethertype - shift 6 words */
+	fdirctrl |= (0x6 << IXGBE_FDIRCTRL_FLEX_SHIFT);
+
+
+	/* Prime the keys for hashing */
+	IXGBE_WRITE_REG(hw, IXGBE_FDIRHKEY,
+	                IXGBE_HTONL(IXGBE_ATR_BUCKET_HASH_KEY));
+	IXGBE_WRITE_REG(hw, IXGBE_FDIRSKEY,
+	                IXGBE_HTONL(IXGBE_ATR_SIGNATURE_HASH_KEY));
+
+	/*
+	 * Poll init-done after we write the register.  Estimated times:
+	 *      10G: PBALLOC = 11b, timing is 60us
+	 *       1G: PBALLOC = 11b, timing is 600us
+	 *     100M: PBALLOC = 11b, timing is 6ms
+	 *
+	 *     Multiple these timings by 4 if under full Rx load
+	 *
+	 * So we'll poll for IXGBE_FDIR_INIT_DONE_POLL times, sleeping for
+	 * 1 msec per poll time.  If we're at line rate and drop to 100M, then
+	 * this might not finish in our poll time, but we can live with that
+	 * for now.
+	 */
+	IXGBE_WRITE_REG(hw, IXGBE_FDIRCTRL, fdirctrl);
+	IXGBE_WRITE_FLUSH(hw);
+	for (i = 0; i < IXGBE_FDIR_INIT_DONE_POLL; i++) {
+		if (IXGBE_READ_REG(hw, IXGBE_FDIRCTRL) &
+		                   IXGBE_FDIRCTRL_INIT_DONE)
+			break;
+		msec_delay(1);
+	}
+	if (i >= IXGBE_FDIR_INIT_DONE_POLL)
+		DEBUGOUT("Flow Director Signature poll time exceeded!\n");
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_init_fdir_perfect_82599 - Initialize Flow Director perfect filters
+ *  @hw: pointer to hardware structure
+ *  @pballoc: which mode to allocate filters with
+ **/
+s32 ixgbe_init_fdir_perfect_82599(struct ixgbe_hw *hw, u32 pballoc)
+{
+	u32 fdirctrl = 0;
+	u32 pbsize;
+	int i;
+
+	/*
+	 * Before enabling Flow Director, the Rx Packet Buffer size
+	 * must be reduced.  The new value is the current size minus
+	 * flow director memory usage size.
+	 */
+
+	pbsize = (1 << (IXGBE_FDIR_PBALLOC_SIZE_SHIFT + pballoc));
+	IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(0),
+	    (IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) - pbsize));
+
+	/*
+	 * The defaults in the HW for RX PB 1-7 are not zero and so should be
+	 * intialized to zero for non DCB mode otherwise actual total RX PB
+	 * would be bigger than programmed and filter space would run into
+	 * the PB 0 region.
+	 */
+	for (i = 1; i < 8; i++)
+		IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), 0);
+
+	/* Send interrupt when 64 filters are left */
+	fdirctrl |= 4 << IXGBE_FDIRCTRL_FULL_THRESH_SHIFT;
+
+	switch (pballoc) {
+	case IXGBE_FDIR_PBALLOC_64K:
+		/* 2k - 1 perfect filters */
+		fdirctrl |= IXGBE_FDIRCTRL_PBALLOC_64K;
+		break;
+	case IXGBE_FDIR_PBALLOC_128K:
+		/* 4k - 1 perfect filters */
+		fdirctrl |= IXGBE_FDIRCTRL_PBALLOC_128K;
+		break;
+	case IXGBE_FDIR_PBALLOC_256K:
+		/* 8k - 1 perfect filters */
+		fdirctrl |= IXGBE_FDIRCTRL_PBALLOC_256K;
+		break;
+	default:
+		/* bad value */
+		return IXGBE_ERR_CONFIG;
+	};
+
+	/* Turn perfect match filtering on */
+	fdirctrl |= IXGBE_FDIRCTRL_PERFECT_MATCH;
+	fdirctrl |= IXGBE_FDIRCTRL_REPORT_STATUS;
+
+	/* Move the flexible bytes to use the ethertype - shift 6 words */
+	fdirctrl |= (0x6 << IXGBE_FDIRCTRL_FLEX_SHIFT);
+
+	/* Prime the keys for hashing */
+	IXGBE_WRITE_REG(hw, IXGBE_FDIRHKEY,
+	                IXGBE_HTONL(IXGBE_ATR_BUCKET_HASH_KEY));
+	IXGBE_WRITE_REG(hw, IXGBE_FDIRSKEY,
+	                IXGBE_HTONL(IXGBE_ATR_SIGNATURE_HASH_KEY));
+
+	/*
+	 * Poll init-done after we write the register.  Estimated times:
+	 *      10G: PBALLOC = 11b, timing is 60us
+	 *       1G: PBALLOC = 11b, timing is 600us
+	 *     100M: PBALLOC = 11b, timing is 6ms
+	 *
+	 *     Multiple these timings by 4 if under full Rx load
+	 *
+	 * So we'll poll for IXGBE_FDIR_INIT_DONE_POLL times, sleeping for
+	 * 1 msec per poll time.  If we're at line rate and drop to 100M, then
+	 * this might not finish in our poll time, but we can live with that
+	 * for now.
+	 */
+
+	/* Set the maximum length per hash bucket to 0xA filters */
+	fdirctrl |= (0xA << IXGBE_FDIRCTRL_MAX_LENGTH_SHIFT);
+
+	IXGBE_WRITE_REG(hw, IXGBE_FDIRCTRL, fdirctrl);
+	IXGBE_WRITE_FLUSH(hw);
+	for (i = 0; i < IXGBE_FDIR_INIT_DONE_POLL; i++) {
+		if (IXGBE_READ_REG(hw, IXGBE_FDIRCTRL) &
+		                   IXGBE_FDIRCTRL_INIT_DONE)
+			break;
+		msec_delay(1);
+	}
+	if (i >= IXGBE_FDIR_INIT_DONE_POLL)
+		DEBUGOUT("Flow Director Perfect poll time exceeded!\n");
+
+	return IXGBE_SUCCESS;
+}
+
+
+/**
+ *  ixgbe_atr_compute_hash_82599 - Compute the hashes for SW ATR
+ *  @stream: input bitstream to compute the hash on
+ *  @key: 32-bit hash key
+ **/
+u16 ixgbe_atr_compute_hash_82599(struct ixgbe_atr_input *atr_input, u32 key)
+{
+	/*
+	 * The algorithm is as follows:
+	 *    Hash[15:0] = Sum { S[n] x K[n+16] }, n = 0...350
+	 *    where Sum {A[n]}, n = 0...n is bitwise XOR of A[0], A[1]...A[n]
+	 *    and A[n] x B[n] is bitwise AND between same length strings
+	 *
+	 *    K[n] is 16 bits, defined as:
+	 *       for n modulo 32 >= 15, K[n] = K[n % 32 : (n % 32) - 15]
+	 *       for n modulo 32 < 15, K[n] =
+	 *             K[(n % 32:0) | (31:31 - (14 - (n % 32)))]
+	 *
+	 *    S[n] is 16 bits, defined as:
+	 *       for n >= 15, S[n] = S[n:n - 15]
+	 *       for n < 15, S[n] = S[(n:0) | (350:350 - (14 - n))]
+	 *
+	 *    To simplify for programming, the algorithm is implemented
+	 *    in software this way:
+	 *
+	 *    Key[31:0], Stream[335:0]
+	 *
+	 *    tmp_key[11 * 32 - 1:0] = 11{Key[31:0] = key concatenated 11 times
+	 *    int_key[350:0] = tmp_key[351:1]
+	 *    int_stream[365:0] = Stream[14:0] | Stream[335:0] | Stream[335:321]
+	 *
+	 *    hash[15:0] = 0;
+	 *    for (i = 0; i < 351; i++) {
+	 *        if (int_key[i])
+	 *            hash ^= int_stream[(i + 15):i];
+	 *    }
+	 */
+
+	union {
+		u32    key[11];
+		u8     key_stream[44];
+	} tmp_key;
+
+	u8   *stream = (u8 *)atr_input;
+	u8   int_key[44];      /* upper-most bit unused */
+	u8   hash_str[46];     /* upper-most 2 bits unused */
+	u16  hash_result = 0;
+	u16  tmp = 0;
+	int  i, j, k, h;
+
+	memset(&tmp_key, 0, sizeof(tmp_key));
+	/* First load the temporary key stream */
+	for (i = 0; i < 11; i++)
+		tmp_key.key[i] = key;
+
+	/*
+	 * Set the interim key for the hashing.  Bit 352 is unused, so we must
+	 * shift and compensate when building the key.
+	 */
+	int_key[0] = tmp_key.key_stream[0] >> 1;
+	for (i = 1, j = 0; i < 44; i++) {
+		int_key[i] = (tmp_key.key_stream[j] & 0x1) << 7;
+		j++;
+		int_key[i] |= tmp_key.key_stream[j] >> 1;
+	}
+
+	/*
+	 * Set the interim bit string for the hashing.  Bits 368 and 367 are
+	 * unused, so shift and compensate when building the string.
+	 */
+	hash_str[0] = (stream[40] & 0x7f) >> 1;
+	for (i = 1, j = 40; i < 46; i++) {
+		hash_str[i] = (stream[j] & 0x1) << 7;
+		j++;
+		if (j > 41)
+			j = 0;
+		hash_str[i] |= stream[j] >> 1;
+	}
+
+	/*
+	 * Now compute the hash.  i is the index into hash_str, j is into our
+	 * key stream, k is counting the number of bits, and h interates within
+	 * each byte.
+	 */
+	for (i = 45, j = 43, k = 0; k < 351 && i >= 2 && j >= 0; i--, j--) {
+		for (h = 0; h < 8 && k < 351; h++, k++) {
+			if ((int_key[j] >> h) & 0x1) {
+				/*
+				 * Key bit is set, XOR in the current 16-bit
+				 * string.  Example of processing:
+				 *    h = 0,
+				 *      tmp = (hash_str[i - 2] & 0 << 16) |
+				 *            (hash_str[i - 1] & 0xff << 8) |
+				 *            (hash_str[i] & 0xff >> 0)
+				 *      So tmp = hash_str[15 + k:k], since the
+				 *      i + 2 clause rolls off the 16-bit value
+				 *    h = 7,
+				 *      tmp = (hash_str[i - 2] & 0x7f << 9) |
+				 *            (hash_str[i - 1] & 0xff << 1) |
+				 *            (hash_str[i] & 0x80 >> 7)
+				 */
+				tmp = ((hash_str[i] & (0xff << h)) >> h);
+				tmp |= ((hash_str[i - 1] & 0xff) << (8 - h));
+				tmp |= (hash_str[i - 2] & (0xff >> (8 - h)))
+				        << (16 - h);
+				hash_result ^= tmp;
+			}
+		}
+	}
+
+	return hash_result;
+}
+
+/**
+ *  ixgbe_atr_set_vlan_id_82599 - Sets the VLAN id in the ATR input stream
+ *  @input: input stream to modify
+ *  @vlan: the VLAN id to load
+ **/
+s32 ixgbe_atr_set_vlan_id_82599(struct ixgbe_atr_input *input, u16 vlan)
+{
+	input->byte_stream[IXGBE_ATR_VLAN_OFFSET + 1] = vlan >> 8;
+	input->byte_stream[IXGBE_ATR_VLAN_OFFSET] = vlan & 0xff;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_set_src_ipv4_82599 - Sets the source IPv4 address
+ *  @input: input stream to modify
+ *  @src_addr: the IP address to load
+ **/
+s32 ixgbe_atr_set_src_ipv4_82599(struct ixgbe_atr_input *input, u32 src_addr)
+{
+	input->byte_stream[IXGBE_ATR_SRC_IPV4_OFFSET + 3] = src_addr >> 24;
+	input->byte_stream[IXGBE_ATR_SRC_IPV4_OFFSET + 2] =
+	                                               (src_addr >> 16) & 0xff;
+	input->byte_stream[IXGBE_ATR_SRC_IPV4_OFFSET + 1] =
+	                                                (src_addr >> 8) & 0xff;
+	input->byte_stream[IXGBE_ATR_SRC_IPV4_OFFSET] = src_addr & 0xff;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_set_dst_ipv4_82599 - Sets the destination IPv4 address
+ *  @input: input stream to modify
+ *  @dst_addr: the IP address to load
+ **/
+s32 ixgbe_atr_set_dst_ipv4_82599(struct ixgbe_atr_input *input, u32 dst_addr)
+{
+	input->byte_stream[IXGBE_ATR_DST_IPV4_OFFSET + 3] = dst_addr >> 24;
+	input->byte_stream[IXGBE_ATR_DST_IPV4_OFFSET + 2] =
+	                                               (dst_addr >> 16) & 0xff;
+	input->byte_stream[IXGBE_ATR_DST_IPV4_OFFSET + 1] =
+	                                                (dst_addr >> 8) & 0xff;
+	input->byte_stream[IXGBE_ATR_DST_IPV4_OFFSET] = dst_addr & 0xff;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_set_src_ipv6_82599 - Sets the source IPv6 address
+ *  @input: input stream to modify
+ *  @src_addr_1: the first 4 bytes of the IP address to load
+ *  @src_addr_2: the second 4 bytes of the IP address to load
+ *  @src_addr_3: the third 4 bytes of the IP address to load
+ *  @src_addr_4: the fourth 4 bytes of the IP address to load
+ **/
+s32 ixgbe_atr_set_src_ipv6_82599(struct ixgbe_atr_input *input,
+                                 u32 src_addr_1, u32 src_addr_2,
+                                 u32 src_addr_3, u32 src_addr_4)
+{
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET] = src_addr_4 & 0xff;
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 1] =
+	                                               (src_addr_4 >> 8) & 0xff;
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 2] =
+	                                              (src_addr_4 >> 16) & 0xff;
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 3] = src_addr_4 >> 24;
+
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 4] = src_addr_3 & 0xff;
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 5] =
+	                                               (src_addr_3 >> 8) & 0xff;
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 6] =
+	                                              (src_addr_3 >> 16) & 0xff;
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 7] = src_addr_3 >> 24;
+
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 8] = src_addr_2 & 0xff;
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 9] =
+	                                               (src_addr_2 >> 8) & 0xff;
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 10] =
+	                                              (src_addr_2 >> 16) & 0xff;
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 11] = src_addr_2 >> 24;
+
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 12] = src_addr_1 & 0xff;
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 13] =
+	                                               (src_addr_1 >> 8) & 0xff;
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 14] =
+	                                              (src_addr_1 >> 16) & 0xff;
+	input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 15] = src_addr_1 >> 24;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_set_dst_ipv6_82599 - Sets the destination IPv6 address
+ *  @input: input stream to modify
+ *  @dst_addr_1: the first 4 bytes of the IP address to load
+ *  @dst_addr_2: the second 4 bytes of the IP address to load
+ *  @dst_addr_3: the third 4 bytes of the IP address to load
+ *  @dst_addr_4: the fourth 4 bytes of the IP address to load
+ **/
+s32 ixgbe_atr_set_dst_ipv6_82599(struct ixgbe_atr_input *input,
+                                 u32 dst_addr_1, u32 dst_addr_2,
+                                 u32 dst_addr_3, u32 dst_addr_4)
+{
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET] = dst_addr_4 & 0xff;
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 1] =
+	                                               (dst_addr_4 >> 8) & 0xff;
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 2] =
+	                                              (dst_addr_4 >> 16) & 0xff;
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 3] = dst_addr_4 >> 24;
+
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 4] = dst_addr_3 & 0xff;
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 5] =
+	                                               (dst_addr_3 >> 8) & 0xff;
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 6] =
+	                                              (dst_addr_3 >> 16) & 0xff;
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 7] = dst_addr_3 >> 24;
+
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 8] = dst_addr_2 & 0xff;
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 9] =
+	                                               (dst_addr_2 >> 8) & 0xff;
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 10] =
+	                                              (dst_addr_2 >> 16) & 0xff;
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 11] = dst_addr_2 >> 24;
+
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 12] = dst_addr_1 & 0xff;
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 13] =
+	                                               (dst_addr_1 >> 8) & 0xff;
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 14] =
+	                                              (dst_addr_1 >> 16) & 0xff;
+	input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 15] = dst_addr_1 >> 24;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_set_src_port_82599 - Sets the source port
+ *  @input: input stream to modify
+ *  @src_port: the source port to load
+ **/
+s32 ixgbe_atr_set_src_port_82599(struct ixgbe_atr_input *input, u16 src_port)
+{
+	input->byte_stream[IXGBE_ATR_SRC_PORT_OFFSET + 1] = src_port >> 8;
+	input->byte_stream[IXGBE_ATR_SRC_PORT_OFFSET] = src_port & 0xff;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_set_dst_port_82599 - Sets the destination port
+ *  @input: input stream to modify
+ *  @dst_port: the destination port to load
+ **/
+s32 ixgbe_atr_set_dst_port_82599(struct ixgbe_atr_input *input, u16 dst_port)
+{
+	input->byte_stream[IXGBE_ATR_DST_PORT_OFFSET + 1] = dst_port >> 8;
+	input->byte_stream[IXGBE_ATR_DST_PORT_OFFSET] = dst_port & 0xff;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_set_flex_byte_82599 - Sets the flexible bytes
+ *  @input: input stream to modify
+ *  @flex_bytes: the flexible bytes to load
+ **/
+s32 ixgbe_atr_set_flex_byte_82599(struct ixgbe_atr_input *input, u16 flex_byte)
+{
+	input->byte_stream[IXGBE_ATR_FLEX_BYTE_OFFSET + 1] = flex_byte >> 8;
+	input->byte_stream[IXGBE_ATR_FLEX_BYTE_OFFSET] = flex_byte & 0xff;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_set_vm_pool_82599 - Sets the Virtual Machine pool
+ *  @input: input stream to modify
+ *  @vm_pool: the Virtual Machine pool to load
+ **/
+s32 ixgbe_atr_set_vm_pool_82599(struct ixgbe_atr_input *input, u8 vm_pool)
+{
+	input->byte_stream[IXGBE_ATR_VM_POOL_OFFSET] = vm_pool;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_set_l4type_82599 - Sets the layer 4 packet type
+ *  @input: input stream to modify
+ *  @l4type: the layer 4 type value to load
+ **/
+s32 ixgbe_atr_set_l4type_82599(struct ixgbe_atr_input *input, u8 l4type)
+{
+	input->byte_stream[IXGBE_ATR_L4TYPE_OFFSET] = l4type;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_get_vlan_id_82599 - Gets the VLAN id from the ATR input stream
+ *  @input: input stream to search
+ *  @vlan: the VLAN id to load
+ **/
+s32 ixgbe_atr_get_vlan_id_82599(struct ixgbe_atr_input *input, u16 *vlan)
+{
+	*vlan = input->byte_stream[IXGBE_ATR_VLAN_OFFSET];
+	*vlan |= input->byte_stream[IXGBE_ATR_VLAN_OFFSET + 1] << 8;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_get_src_ipv4_82599 - Gets the source IPv4 address
+ *  @input: input stream to search
+ *  @src_addr: the IP address to load
+ **/
+s32 ixgbe_atr_get_src_ipv4_82599(struct ixgbe_atr_input *input, u32 *src_addr)
+{
+	*src_addr = input->byte_stream[IXGBE_ATR_SRC_IPV4_OFFSET];
+	*src_addr |= input->byte_stream[IXGBE_ATR_SRC_IPV4_OFFSET + 1] << 8;
+	*src_addr |= input->byte_stream[IXGBE_ATR_SRC_IPV4_OFFSET + 2] << 16;
+	*src_addr |= input->byte_stream[IXGBE_ATR_SRC_IPV4_OFFSET + 3] << 24;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_get_dst_ipv4_82599 - Gets the destination IPv4 address
+ *  @input: input stream to search
+ *  @dst_addr: the IP address to load
+ **/
+s32 ixgbe_atr_get_dst_ipv4_82599(struct ixgbe_atr_input *input, u32 *dst_addr)
+{
+	*dst_addr = input->byte_stream[IXGBE_ATR_DST_IPV4_OFFSET];
+	*dst_addr |= input->byte_stream[IXGBE_ATR_DST_IPV4_OFFSET + 1] << 8;
+	*dst_addr |= input->byte_stream[IXGBE_ATR_DST_IPV4_OFFSET + 2] << 16;
+	*dst_addr |= input->byte_stream[IXGBE_ATR_DST_IPV4_OFFSET + 3] << 24;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_get_src_ipv6_82599 - Gets the source IPv6 address
+ *  @input: input stream to search
+ *  @src_addr_1: the first 4 bytes of the IP address to load
+ *  @src_addr_2: the second 4 bytes of the IP address to load
+ *  @src_addr_3: the third 4 bytes of the IP address to load
+ *  @src_addr_4: the fourth 4 bytes of the IP address to load
+ **/
+s32 ixgbe_atr_get_src_ipv6_82599(struct ixgbe_atr_input *input,
+                                 u32 *src_addr_1, u32 *src_addr_2,
+                                 u32 *src_addr_3, u32 *src_addr_4)
+{
+	*src_addr_1 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 12];
+	*src_addr_1 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 13] << 8;
+	*src_addr_1 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 14] << 16;
+	*src_addr_1 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 15] << 24;
+
+	*src_addr_2 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 8];
+	*src_addr_2 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 9] << 8;
+	*src_addr_2 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 10] << 16;
+	*src_addr_2 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 11] << 24;
+
+	*src_addr_3 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 4];
+	*src_addr_3 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 5] << 8;
+	*src_addr_3 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 6] << 16;
+	*src_addr_3 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 7] << 24;
+
+	*src_addr_4 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET];
+	*src_addr_4 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 1] << 8;
+	*src_addr_4 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 2] << 16;
+	*src_addr_4 = input->byte_stream[IXGBE_ATR_SRC_IPV6_OFFSET + 3] << 24;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_get_dst_ipv6_82599 - Gets the destination IPv6 address
+ *  @input: input stream to search
+ *  @dst_addr_1: the first 4 bytes of the IP address to load
+ *  @dst_addr_2: the second 4 bytes of the IP address to load
+ *  @dst_addr_3: the third 4 bytes of the IP address to load
+ *  @dst_addr_4: the fourth 4 bytes of the IP address to load
+ **/
+s32 ixgbe_atr_get_dst_ipv6_82599(struct ixgbe_atr_input *input,
+                                 u32 *dst_addr_1, u32 *dst_addr_2,
+                                 u32 *dst_addr_3, u32 *dst_addr_4)
+{
+	*dst_addr_1 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 12];
+	*dst_addr_1 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 13] << 8;
+	*dst_addr_1 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 14] << 16;
+	*dst_addr_1 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 15] << 24;
+
+	*dst_addr_2 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 8];
+	*dst_addr_2 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 9] << 8;
+	*dst_addr_2 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 10] << 16;
+	*dst_addr_2 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 11] << 24;
+
+	*dst_addr_3 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 4];
+	*dst_addr_3 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 5] << 8;
+	*dst_addr_3 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 6] << 16;
+	*dst_addr_3 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 7] << 24;
+
+	*dst_addr_4 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET];
+	*dst_addr_4 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 1] << 8;
+	*dst_addr_4 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 2] << 16;
+	*dst_addr_4 = input->byte_stream[IXGBE_ATR_DST_IPV6_OFFSET + 3] << 24;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_get_src_port_82599 - Gets the source port
+ *  @input: input stream to modify
+ *  @src_port: the source port to load
+ *
+ *  Even though the input is given in big-endian, the FDIRPORT registers
+ *  expect the ports to be programmed in little-endian.  Hence the need to swap
+ *  endianness when retrieving the data.  This can be confusing since the
+ *  internal hash engine expects it to be big-endian.
+ **/
+s32 ixgbe_atr_get_src_port_82599(struct ixgbe_atr_input *input, u16 *src_port)
+{
+	*src_port = input->byte_stream[IXGBE_ATR_SRC_PORT_OFFSET] << 8;
+	*src_port |= input->byte_stream[IXGBE_ATR_SRC_PORT_OFFSET + 1];
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_get_dst_port_82599 - Gets the destination port
+ *  @input: input stream to modify
+ *  @dst_port: the destination port to load
+ *
+ *  Even though the input is given in big-endian, the FDIRPORT registers
+ *  expect the ports to be programmed in little-endian.  Hence the need to swap
+ *  endianness when retrieving the data.  This can be confusing since the
+ *  internal hash engine expects it to be big-endian.
+ **/
+s32 ixgbe_atr_get_dst_port_82599(struct ixgbe_atr_input *input, u16 *dst_port)
+{
+	*dst_port = input->byte_stream[IXGBE_ATR_DST_PORT_OFFSET] << 8;
+	*dst_port |= input->byte_stream[IXGBE_ATR_DST_PORT_OFFSET + 1];
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_get_flex_byte_82599 - Gets the flexible bytes
+ *  @input: input stream to modify
+ *  @flex_bytes: the flexible bytes to load
+ **/
+s32 ixgbe_atr_get_flex_byte_82599(struct ixgbe_atr_input *input, u16 *flex_byte)
+{
+	*flex_byte = input->byte_stream[IXGBE_ATR_FLEX_BYTE_OFFSET];
+	*flex_byte |= input->byte_stream[IXGBE_ATR_FLEX_BYTE_OFFSET + 1] << 8;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_get_vm_pool_82599 - Gets the Virtual Machine pool
+ *  @input: input stream to modify
+ *  @vm_pool: the Virtual Machine pool to load
+ **/
+s32 ixgbe_atr_get_vm_pool_82599(struct ixgbe_atr_input *input, u8 *vm_pool)
+{
+	*vm_pool = input->byte_stream[IXGBE_ATR_VM_POOL_OFFSET];
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_get_l4type_82599 - Gets the layer 4 packet type
+ *  @input: input stream to modify
+ *  @l4type: the layer 4 type value to load
+ **/
+s32 ixgbe_atr_get_l4type_82599(struct ixgbe_atr_input *input, u8 *l4type)
+{
+	*l4type = input->byte_stream[IXGBE_ATR_L4TYPE_OFFSET];
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_atr_add_signature_filter_82599 - Adds a signature hash filter
+ *  @hw: pointer to hardware structure
+ *  @stream: input bitstream
+ *  @queue: queue index to direct traffic to
+ **/
+s32 ixgbe_fdir_add_signature_filter_82599(struct ixgbe_hw *hw,
+                                          struct ixgbe_atr_input *input,
+                                          u8 queue)
+{
+	u64  fdirhashcmd;
+	u64  fdircmd;
+	u32  fdirhash;
+	u16  bucket_hash, sig_hash;
+	u8   l4type;
+
+	bucket_hash = ixgbe_atr_compute_hash_82599(input,
+	                                           IXGBE_ATR_BUCKET_HASH_KEY);
+
+	/* bucket_hash is only 15 bits */
+	bucket_hash &= IXGBE_ATR_HASH_MASK;
+
+	sig_hash = ixgbe_atr_compute_hash_82599(input,
+	                                        IXGBE_ATR_SIGNATURE_HASH_KEY);
+
+	/* Get the l4type in order to program FDIRCMD properly */
+	/* lowest 2 bits are FDIRCMD.L4TYPE, third lowest bit is FDIRCMD.IPV6 */
+	ixgbe_atr_get_l4type_82599(input, &l4type);
+
+	/*
+	 * The lower 32-bits of fdirhashcmd is for FDIRHASH, the upper 32-bits
+	 * is for FDIRCMD.  Then do a 64-bit register write from FDIRHASH.
+	 */
+	fdirhash = sig_hash << IXGBE_FDIRHASH_SIG_SW_INDEX_SHIFT | bucket_hash;
+
+	fdircmd = (IXGBE_FDIRCMD_CMD_ADD_FLOW | IXGBE_FDIRCMD_FILTER_UPDATE |
+	           IXGBE_FDIRCMD_LAST | IXGBE_FDIRCMD_QUEUE_EN);
+
+	switch (l4type & IXGBE_ATR_L4TYPE_MASK) {
+	case IXGBE_ATR_L4TYPE_TCP:
+		fdircmd |= IXGBE_FDIRCMD_L4TYPE_TCP;
+		break;
+	case IXGBE_ATR_L4TYPE_UDP:
+		fdircmd |= IXGBE_FDIRCMD_L4TYPE_UDP;
+		break;
+	case IXGBE_ATR_L4TYPE_SCTP:
+		fdircmd |= IXGBE_FDIRCMD_L4TYPE_SCTP;
+		break;
+	default:
+		DEBUGOUT(" Error on l4type input\n");
+		return IXGBE_ERR_CONFIG;
+	}
+
+	if (l4type & IXGBE_ATR_L4TYPE_IPV6_MASK)
+		fdircmd |= IXGBE_FDIRCMD_IPV6;
+
+	fdircmd |= ((u64)queue << IXGBE_FDIRCMD_RX_QUEUE_SHIFT);
+	fdirhashcmd = ((fdircmd << 32) | fdirhash);
+
+	DEBUGOUT2("Tx Queue=%x hash=%x\n", queue, fdirhash & 0x7FFF7FFF);
+	IXGBE_WRITE_REG64(hw, IXGBE_FDIRHASH, fdirhashcmd);
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_fdir_add_perfect_filter_82599 - Adds a perfect filter
+ *  @hw: pointer to hardware structure
+ *  @input: input bitstream
+ *  @queue: queue index to direct traffic to
+ *
+ *  Note that the caller to this function must lock before calling, since the
+ *  hardware writes must be protected from one another.
+ **/
+s32 ixgbe_fdir_add_perfect_filter_82599(struct ixgbe_hw *hw,
+                                        struct ixgbe_atr_input *input,
+                                        u16 soft_id,
+                                        u8 queue)
+{
+	u32 fdircmd = 0;
+	u32 fdirhash;
+	u32 src_ipv4, dst_ipv4;
+	u32 src_ipv6_1, src_ipv6_2, src_ipv6_3, src_ipv6_4;
+	u16 src_port, dst_port, vlan_id, flex_bytes;
+	u16 bucket_hash;
+	u8  l4type;
+
+	/* Get our input values */
+	ixgbe_atr_get_l4type_82599(input, &l4type);
+
+	/*
+	 * Check l4type formatting, and bail out before we touch the hardware
+	 * if there's a configuration issue
+	 */
+	switch (l4type & IXGBE_ATR_L4TYPE_MASK) {
+	case IXGBE_ATR_L4TYPE_TCP:
+		fdircmd |= IXGBE_FDIRCMD_L4TYPE_TCP;
+		break;
+	case IXGBE_ATR_L4TYPE_UDP:
+		fdircmd |= IXGBE_FDIRCMD_L4TYPE_UDP;
+		break;
+	case IXGBE_ATR_L4TYPE_SCTP:
+		fdircmd |= IXGBE_FDIRCMD_L4TYPE_SCTP;
+		break;
+	default:
+		DEBUGOUT(" Error on l4type input\n");
+		return IXGBE_ERR_CONFIG;
+	}
+
+	bucket_hash = ixgbe_atr_compute_hash_82599(input,
+	                                           IXGBE_ATR_BUCKET_HASH_KEY);
+
+	/* bucket_hash is only 15 bits */
+	bucket_hash &= IXGBE_ATR_HASH_MASK;
+
+	ixgbe_atr_get_vlan_id_82599(input, &vlan_id);
+	ixgbe_atr_get_src_port_82599(input, &src_port);
+	ixgbe_atr_get_dst_port_82599(input, &dst_port);
+	ixgbe_atr_get_flex_byte_82599(input, &flex_bytes);
+
+	fdirhash = soft_id << IXGBE_FDIRHASH_SIG_SW_INDEX_SHIFT | bucket_hash;
+
+	/* Now figure out if we're IPv4 or IPv6 */
+	if (l4type & IXGBE_ATR_L4TYPE_IPV6_MASK) {
+		/* IPv6 */
+		ixgbe_atr_get_src_ipv6_82599(input, &src_ipv6_1, &src_ipv6_2,
+	                                     &src_ipv6_3, &src_ipv6_4);
+
+		IXGBE_WRITE_REG(hw, IXGBE_FDIRSIPv6(0), src_ipv6_1);
+		IXGBE_WRITE_REG(hw, IXGBE_FDIRSIPv6(1), src_ipv6_2);
+		IXGBE_WRITE_REG(hw, IXGBE_FDIRSIPv6(2), src_ipv6_3);
+		/* The last 4 bytes is the same register as IPv4 */
+		IXGBE_WRITE_REG(hw, IXGBE_FDIRIPSA, src_ipv6_4);
+
+		fdircmd |= IXGBE_FDIRCMD_IPV6;
+		fdircmd |= IXGBE_FDIRCMD_IPv6DMATCH;
+	} else {
+		/* IPv4 */
+		ixgbe_atr_get_src_ipv4_82599(input, &src_ipv4);
+		IXGBE_WRITE_REG(hw, IXGBE_FDIRIPSA, src_ipv4);
+
+	}
+
+	ixgbe_atr_get_dst_ipv4_82599(input, &dst_ipv4);
+	IXGBE_WRITE_REG(hw, IXGBE_FDIRIPDA, dst_ipv4);
+
+	IXGBE_WRITE_REG(hw, IXGBE_FDIRVLAN, (vlan_id |
+	                            (flex_bytes << IXGBE_FDIRVLAN_FLEX_SHIFT)));
+	IXGBE_WRITE_REG(hw, IXGBE_FDIRPORT, (src_port |
+	                       (dst_port << IXGBE_FDIRPORT_DESTINATION_SHIFT)));
+
+	fdircmd |= IXGBE_FDIRCMD_CMD_ADD_FLOW;
+	fdircmd |= IXGBE_FDIRCMD_FILTER_UPDATE;
+	fdircmd |= IXGBE_FDIRCMD_LAST;
+	fdircmd |= IXGBE_FDIRCMD_QUEUE_EN;
+	fdircmd |= queue << IXGBE_FDIRCMD_RX_QUEUE_SHIFT;
+
+	IXGBE_WRITE_REG(hw, IXGBE_FDIRHASH, fdirhash);
+	IXGBE_WRITE_REG(hw, IXGBE_FDIRCMD, fdircmd);
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_read_analog_reg8_82599 - Reads 8 bit Omer analog register
+ *  @hw: pointer to hardware structure
+ *  @reg: analog register to read
+ *  @val: read value
+ *
+ *  Performs read operation to Omer analog register specified.
+ **/
+s32 ixgbe_read_analog_reg8_82599(struct ixgbe_hw *hw, u32 reg, u8 *val)
+{
+	u32  core_ctl;
+
+	IXGBE_WRITE_REG(hw, IXGBE_CORECTL, IXGBE_CORECTL_WRITE_CMD |
+	                (reg << 8));
+	IXGBE_WRITE_FLUSH(hw);
+	usec_delay(10);
+	core_ctl = IXGBE_READ_REG(hw, IXGBE_CORECTL);
+	*val = (u8)core_ctl;
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_write_analog_reg8_82599 - Writes 8 bit Omer analog register
+ *  @hw: pointer to hardware structure
+ *  @reg: atlas register to write
+ *  @val: value to write
+ *
+ *  Performs write operation to Omer analog register specified.
+ **/
+s32 ixgbe_write_analog_reg8_82599(struct ixgbe_hw *hw, u32 reg, u8 val)
+{
+	u32  core_ctl;
+
+	core_ctl = (reg << 8) | val;
+	IXGBE_WRITE_REG(hw, IXGBE_CORECTL, core_ctl);
+	IXGBE_WRITE_FLUSH(hw);
+	usec_delay(10);
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_start_hw_rev_1_82599 - Prepare hardware for Tx/Rx
+ *  @hw: pointer to hardware structure
+ *
+ *  Starts the hardware using the generic start_hw function.
+ *  Then performs revision-specific operations:
+ *  Clears the rate limiter registers.
+ **/
+s32 ixgbe_start_hw_rev_1_82599(struct ixgbe_hw *hw)
+{
+	u32 q_num;
+	s32 ret_val = IXGBE_SUCCESS;
+
+	ret_val = ixgbe_start_hw_generic(hw);
+
+	/* Clear the rate limiters */
+	for (q_num = 0; q_num < hw->mac.max_tx_queues; q_num++) {
+		IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, q_num);
+		IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, 0);
+	}
+	IXGBE_WRITE_FLUSH(hw);
+
+	return ret_val;
+}
+
+/**
+ *  ixgbe_identify_phy_82599 - Get physical layer module
+ *  @hw: pointer to hardware structure
+ *
+ *  Determines the physical layer module found on the current adapter.
+ *  If PHY already detected, maintains current PHY type in hw struct,
+ *  otherwise executes the PHY detection routine.
+ **/
+s32 ixgbe_identify_phy_82599(struct ixgbe_hw *hw)
+{
+	s32 status = IXGBE_ERR_PHY_ADDR_INVALID;
+
+	/* Detect PHY if not unknown - returns success if already detected. */
+	status = ixgbe_identify_phy_generic(hw);
+	if (status != IXGBE_SUCCESS)
+		status = ixgbe_identify_sfp_module_generic(hw);
+	/* Set PHY type none if no PHY detected */
+	if (hw->phy.type == ixgbe_phy_unknown) {
+		hw->phy.type = ixgbe_phy_none;
+		status = IXGBE_SUCCESS;
+	}
+
+	/* Return error if SFP module has been detected but is not supported */
+	if (hw->phy.type == ixgbe_phy_sfp_unsupported)
+		status = IXGBE_ERR_SFP_NOT_SUPPORTED;
+
+	return status;
+}
+
+/**
+ *  ixgbe_get_supported_physical_layer_82599 - Returns physical layer type
+ *  @hw: pointer to hardware structure
+ *
+ *  Determines physical layer capabilities of the current configuration.
+ **/
+u32 ixgbe_get_supported_physical_layer_82599(struct ixgbe_hw *hw)
+{
+	u32 physical_layer = IXGBE_PHYSICAL_LAYER_UNKNOWN;
+	u32 autoc = IXGBE_READ_REG(hw, IXGBE_AUTOC);
+	u32 autoc2 = IXGBE_READ_REG(hw, IXGBE_AUTOC2);
+	u32 pma_pmd_10g_serial = autoc2 & IXGBE_AUTOC2_10G_SERIAL_PMA_PMD_MASK;
+	u32 pma_pmd_10g_parallel = autoc & IXGBE_AUTOC_10G_PMA_PMD_MASK;
+	u32 pma_pmd_1g = autoc & IXGBE_AUTOC_1G_PMA_PMD_MASK;
+	u16 ext_ability = 0;
+	u8 comp_codes_10g = 0;
+
+	hw->phy.ops.identify(hw);
+
+	if (hw->phy.type == ixgbe_phy_tn ||
+	    hw->phy.type == ixgbe_phy_aq ||
+	    hw->phy.type == ixgbe_phy_cu_unknown) {
+		hw->phy.ops.read_reg(hw, IXGBE_MDIO_PHY_EXT_ABILITY,
+		IXGBE_MDIO_PMA_PMD_DEV_TYPE, &ext_ability);
+		if (ext_ability & IXGBE_MDIO_PHY_10GBASET_ABILITY)
+			physical_layer |= IXGBE_PHYSICAL_LAYER_10GBASE_T;
+		if (ext_ability & IXGBE_MDIO_PHY_1000BASET_ABILITY)
+			physical_layer |= IXGBE_PHYSICAL_LAYER_1000BASE_T;
+		if (ext_ability & IXGBE_MDIO_PHY_100BASETX_ABILITY)
+			physical_layer |= IXGBE_PHYSICAL_LAYER_100BASE_TX;
+		goto out;
+	}
+
+	switch (autoc & IXGBE_AUTOC_LMS_MASK) {
+	case IXGBE_AUTOC_LMS_1G_AN:
+	case IXGBE_AUTOC_LMS_1G_LINK_NO_AN:
+		if (pma_pmd_1g == IXGBE_AUTOC_1G_KX_BX) {
+			physical_layer = IXGBE_PHYSICAL_LAYER_1000BASE_KX |
+			    IXGBE_PHYSICAL_LAYER_1000BASE_BX;
+			goto out;
+		} else
+			/* SFI mode so read SFP module */
+			goto sfp_check;
+		break;
+	case IXGBE_AUTOC_LMS_10G_LINK_NO_AN:
+		if (pma_pmd_10g_parallel == IXGBE_AUTOC_10G_CX4)
+			physical_layer = IXGBE_PHYSICAL_LAYER_10GBASE_CX4;
+		else if (pma_pmd_10g_parallel == IXGBE_AUTOC_10G_KX4)
+			physical_layer = IXGBE_PHYSICAL_LAYER_10GBASE_KX4;
+		goto out;
+		break;
+	case IXGBE_AUTOC_LMS_10G_SERIAL:
+		if (pma_pmd_10g_serial == IXGBE_AUTOC2_10G_KR) {
+			physical_layer = IXGBE_PHYSICAL_LAYER_10GBASE_KR;
+			goto out;
+		} else if (pma_pmd_10g_serial == IXGBE_AUTOC2_10G_SFI)
+			goto sfp_check;
+		break;
+	case IXGBE_AUTOC_LMS_KX4_KX_KR:
+	case IXGBE_AUTOC_LMS_KX4_KX_KR_1G_AN:
+		if (autoc & IXGBE_AUTOC_KX_SUPP)
+			physical_layer |= IXGBE_PHYSICAL_LAYER_1000BASE_KX;
+		if (autoc & IXGBE_AUTOC_KX4_SUPP)
+			physical_layer |= IXGBE_PHYSICAL_LAYER_10GBASE_KX4;
+		if (autoc & IXGBE_AUTOC_KR_SUPP)
+			physical_layer |= IXGBE_PHYSICAL_LAYER_10GBASE_KR;
+		goto out;
+		break;
+	default:
+		goto out;
+		break;
+	}
+
+sfp_check:
+	/* SFP check must be done last since DA modules are sometimes used to
+	 * test KR mode -  we need to id KR mode correctly before SFP module.
+	 * Call identify_sfp because the pluggable module may have changed */
+	hw->phy.ops.identify_sfp(hw);
+	if (hw->phy.sfp_type == ixgbe_sfp_type_not_present)
+		goto out;
+
+	switch (hw->phy.type) {
+	case ixgbe_phy_tw_tyco:
+	case ixgbe_phy_tw_unknown:
+		physical_layer = IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU;
+		break;
+	case ixgbe_phy_sfp_avago:
+	case ixgbe_phy_sfp_ftl:
+	case ixgbe_phy_sfp_intel:
+	case ixgbe_phy_sfp_unknown:
+		hw->phy.ops.read_i2c_eeprom(hw,
+		      IXGBE_SFF_10GBE_COMP_CODES, &comp_codes_10g);
+		if (comp_codes_10g & IXGBE_SFF_10GBASESR_CAPABLE)
+			physical_layer = IXGBE_PHYSICAL_LAYER_10GBASE_SR;
+		else if (comp_codes_10g & IXGBE_SFF_10GBASELR_CAPABLE)
+			physical_layer = IXGBE_PHYSICAL_LAYER_10GBASE_LR;
+		break;
+	default:
+		break;
+	}
+
+out:
+	return physical_layer;
+}
+
+/**
+ *  ixgbe_enable_rx_dma_82599 - Enable the Rx DMA unit on 82599
+ *  @hw: pointer to hardware structure
+ *  @regval: register value to write to RXCTRL
+ *
+ *  Enables the Rx DMA unit for 82599
+ **/
+s32 ixgbe_enable_rx_dma_82599(struct ixgbe_hw *hw, u32 regval)
+{
+#define IXGBE_MAX_SECRX_POLL 30
+	int i;
+	int secrxreg;
+
+	/*
+	 * Workaround for 82599 silicon errata when enabling the Rx datapath.
+	 * If traffic is incoming before we enable the Rx unit, it could hang
+	 * the Rx DMA unit.  Therefore, make sure the security engine is
+	 * completely disabled prior to enabling the Rx unit.
+	 */
+	secrxreg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
+	secrxreg |= IXGBE_SECRXCTRL_RX_DIS;
+	IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, secrxreg);
+	for (i = 0; i < IXGBE_MAX_SECRX_POLL; i++) {
+		secrxreg = IXGBE_READ_REG(hw, IXGBE_SECRXSTAT);
+		if (secrxreg & IXGBE_SECRXSTAT_SECRX_RDY)
+			break;
+		else
+			/* Use interrupt-safe sleep just in case */
+			usec_delay(10);
+	}
+
+	/* For informational purposes only */
+	if (i >= IXGBE_MAX_SECRX_POLL)
+		DEBUGOUT("Rx unit being enabled before security "
+		         "path fully disabled.  Continuing with init.\n");
+
+	IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, regval);
+	secrxreg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
+	secrxreg &= ~IXGBE_SECRXCTRL_RX_DIS;
+	IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, secrxreg);
+	IXGBE_WRITE_FLUSH(hw);
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_get_device_caps_82599 - Get additional device capabilities
+ *  @hw: pointer to hardware structure
+ *  @device_caps: the EEPROM word with the extra device capabilities
+ *
+ *  This function will read the EEPROM location for the device capabilities,
+ *  and return the word through device_caps.
+ **/
+s32 ixgbe_get_device_caps_82599(struct ixgbe_hw *hw, u16 *device_caps)
+{
+	hw->eeprom.ops.read(hw, IXGBE_DEVICE_CAPS, device_caps);
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_get_san_mac_addr_offset_82599 - SAN MAC address offset for 82599
+ *  @hw: pointer to hardware structure
+ *  @san_mac_offset: SAN MAC address offset
+ *
+ *  This function will read the EEPROM location for the SAN MAC address
+ *  pointer, and returns the value at that location.  This is used in both
+ *  get and set mac_addr routines.
+ **/
+s32 ixgbe_get_san_mac_addr_offset_82599(struct ixgbe_hw *hw,
+                                        u16 *san_mac_offset)
+{
+	/*
+	 * First read the EEPROM pointer to see if the MAC addresses are
+	 * available.
+	 */
+	hw->eeprom.ops.read(hw, IXGBE_SAN_MAC_ADDR_PTR, san_mac_offset);
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_get_san_mac_addr_82599 - SAN MAC address retrieval for 82599
+ *  @hw: pointer to hardware structure
+ *  @san_mac_addr: SAN MAC address
+ *
+ *  Reads the SAN MAC address from the EEPROM, if it's available.  This is
+ *  per-port, so set_lan_id() must be called before reading the addresses.
+ *  set_lan_id() is called by identify_sfp(), but this cannot be relied
+ *  upon for non-SFP connections, so we must call it here.
+ **/
+s32 ixgbe_get_san_mac_addr_82599(struct ixgbe_hw *hw, u8 *san_mac_addr)
+{
+	u16 san_mac_data, san_mac_offset;
+	u8 i;
+
+	/*
+	 * First read the EEPROM pointer to see if the MAC addresses are
+	 * available.  If they're not, no point in calling set_lan_id() here.
+	 */
+	ixgbe_get_san_mac_addr_offset_82599(hw, &san_mac_offset);
+
+	if ((san_mac_offset == 0) || (san_mac_offset == 0xFFFF)) {
+		/*
+		 * No addresses available in this EEPROM.  It's not an
+		 * error though, so just wipe the local address and return.
+		 */
+		for (i = 0; i < 6; i++)
+			san_mac_addr[i] = 0xFF;
+
+		goto san_mac_addr_out;
+	}
+
+	/* make sure we know which port we need to program */
+	hw->mac.ops.set_lan_id(hw);
+	/* apply the port offset to the address offset */
+	(hw->bus.func) ? (san_mac_offset += IXGBE_SAN_MAC_ADDR_PORT1_OFFSET) :
+	                 (san_mac_offset += IXGBE_SAN_MAC_ADDR_PORT0_OFFSET);
+	for (i = 0; i < 3; i++) {
+		hw->eeprom.ops.read(hw, san_mac_offset, &san_mac_data);
+		san_mac_addr[i * 2] = (u8)(san_mac_data);
+		san_mac_addr[i * 2 + 1] = (u8)(san_mac_data >> 8);
+		san_mac_offset++;
+	}
+
+san_mac_addr_out:
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_set_san_mac_addr_82599 - Write the SAN MAC address to the EEPROM
+ *  @hw: pointer to hardware structure
+ *  @san_mac_addr: SAN MAC address
+ *
+ *  Write a SAN MAC address to the EEPROM.
+ **/
+s32 ixgbe_set_san_mac_addr_82599(struct ixgbe_hw *hw, u8 *san_mac_addr)
+{
+	s32 status = IXGBE_SUCCESS;
+	u16 san_mac_data, san_mac_offset;
+	u8 i;
+
+	/* Look for SAN mac address pointer.  If not defined, return */
+	ixgbe_get_san_mac_addr_offset_82599(hw, &san_mac_offset);
+
+	if ((san_mac_offset == 0) || (san_mac_offset == 0xFFFF)) {
+		status = IXGBE_ERR_NO_SAN_ADDR_PTR;
+		goto san_mac_addr_out;
+	}
+
+	/* Make sure we know which port we need to write */
+	hw->mac.ops.set_lan_id(hw);
+	/* Apply the port offset to the address offset */
+	(hw->bus.func) ? (san_mac_offset += IXGBE_SAN_MAC_ADDR_PORT1_OFFSET) :
+	                 (san_mac_offset += IXGBE_SAN_MAC_ADDR_PORT0_OFFSET);
+
+	for (i = 0; i < 3; i++) {
+		san_mac_data = (u16)((u16)(san_mac_addr[i * 2 + 1]) << 8);
+		san_mac_data |= (u16)(san_mac_addr[i * 2]);
+		hw->eeprom.ops.write(hw, san_mac_offset, san_mac_data);
+		san_mac_offset++;
+	}
+
+san_mac_addr_out:
+	return status;
+}
diff --git a/sys/dev/ixgbe/ixgbe_api.c b/sys/dev/ixgbe/ixgbe_api.c
index 49dac7d..a3e7b35 100644
--- a/sys/dev/ixgbe/ixgbe_api.c
+++ b/sys/dev/ixgbe/ixgbe_api.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -36,6 +36,7 @@
 #include "ixgbe_common.h"
 
 extern s32 ixgbe_init_ops_82598(struct ixgbe_hw *hw);
+extern s32 ixgbe_init_ops_82599(struct ixgbe_hw *hw);
 
 /**
  *  ixgbe_init_shared_code - Initialize the shared code
@@ -62,6 +63,9 @@ s32 ixgbe_init_shared_code(struct ixgbe_hw *hw)
 	case ixgbe_mac_82598EB:
 		status = ixgbe_init_ops_82598(hw);
 		break;
+	case ixgbe_mac_82599EB:
+		status = ixgbe_init_ops_82599(hw);
+		break;
 	default:
 		status = IXGBE_ERR_DEVICE_NOT_SUPPORTED;
 		break;
@@ -86,6 +90,7 @@ s32 ixgbe_set_mac_type(struct ixgbe_hw *hw)
 	if (hw->vendor_id == IXGBE_INTEL_VENDOR_ID) {
 		switch (hw->device_id) {
 		case IXGBE_DEV_ID_82598:
+		case IXGBE_DEV_ID_82598_BX:
 		case IXGBE_DEV_ID_82598AF_SINGLE_PORT:
 		case IXGBE_DEV_ID_82598AF_DUAL_PORT:
 		case IXGBE_DEV_ID_82598AT:
@@ -97,6 +102,11 @@ s32 ixgbe_set_mac_type(struct ixgbe_hw *hw)
 		case IXGBE_DEV_ID_82598EB_SFP_LOM:
 			hw->mac.type = ixgbe_mac_82598EB;
 			break;
+		case IXGBE_DEV_ID_82599_KX4:
+		case IXGBE_DEV_ID_82599_SFP:
+		case IXGBE_DEV_ID_82599_CX4:
+			hw->mac.type = ixgbe_mac_82599EB;
+			break;
 		default:
 			ret_val = IXGBE_ERR_DEVICE_NOT_SUPPORTED;
 			break;
@@ -193,6 +203,46 @@ s32 ixgbe_get_mac_addr(struct ixgbe_hw *hw, u8 *mac_addr)
 }
 
 /**
+ *  ixgbe_get_san_mac_addr - Get SAN MAC address
+ *  @hw: pointer to hardware structure
+ *  @san_mac_addr: SAN MAC address
+ *
+ *  Reads the SAN MAC address from the EEPROM, if it's available.  This is
+ *  per-port, so set_lan_id() must be called before reading the addresses.
+ **/
+s32 ixgbe_get_san_mac_addr(struct ixgbe_hw *hw, u8 *san_mac_addr)
+{
+	return ixgbe_call_func(hw, hw->mac.ops.get_san_mac_addr,
+	                       (hw, san_mac_addr), IXGBE_NOT_IMPLEMENTED);
+}
+
+/**
+ *  ixgbe_set_san_mac_addr - Write a SAN MAC address
+ *  @hw: pointer to hardware structure
+ *  @san_mac_addr: SAN MAC address
+ *
+ *  Writes A SAN MAC address to the EEPROM.
+ **/
+s32 ixgbe_set_san_mac_addr(struct ixgbe_hw *hw, u8 *san_mac_addr)
+{
+	return ixgbe_call_func(hw, hw->mac.ops.set_san_mac_addr,
+	                       (hw, san_mac_addr), IXGBE_NOT_IMPLEMENTED);
+}
+
+/**
+ *  ixgbe_get_device_caps - Get additional device capabilities
+ *  @hw: pointer to hardware structure
+ *  @device_caps: the EEPROM word for device capabilities
+ *
+ *  Reads the extra device capabilities from the EEPROM
+ **/
+s32 ixgbe_get_device_caps(struct ixgbe_hw *hw, u16 *device_caps)
+{
+	return ixgbe_call_func(hw, hw->mac.ops.get_device_caps,
+	                       (hw, device_caps), IXGBE_NOT_IMPLEMENTED);
+}
+
+/**
  *  ixgbe_get_bus_info - Set PCI bus info
  *  @hw: pointer to hardware structure
  *
@@ -319,6 +369,9 @@ s32 ixgbe_get_phy_firmware_version(struct ixgbe_hw *hw, u16 *firmware_version)
 s32 ixgbe_read_phy_reg(struct ixgbe_hw *hw, u32 reg_addr, u32 device_type,
                        u16 *phy_data)
 {
+	if (hw->phy.id == 0)
+		ixgbe_identify_phy(hw);
+
 	return ixgbe_call_func(hw, hw->phy.ops.read_reg, (hw, reg_addr,
 	                       device_type, phy_data), IXGBE_NOT_IMPLEMENTED);
 }
@@ -334,6 +387,9 @@ s32 ixgbe_read_phy_reg(struct ixgbe_hw *hw, u32 reg_addr, u32 device_type,
 s32 ixgbe_write_phy_reg(struct ixgbe_hw *hw, u32 reg_addr, u32 device_type,
                         u16 phy_data)
 {
+	if (hw->phy.id == 0)
+		ixgbe_identify_phy(hw);
+
 	return ixgbe_call_func(hw, hw->phy.ops.write_reg, (hw, reg_addr,
 	                       device_type, phy_data), IXGBE_NOT_IMPLEMENTED);
 }
@@ -557,6 +613,22 @@ s32 ixgbe_update_eeprom_checksum(struct ixgbe_hw *hw)
 }
 
 /**
+ *  ixgbe_insert_mac_addr - Find a RAR for this mac address
+ *  @hw: pointer to hardware structure
+ *  @addr: Address to put into receive address register
+ *  @vmdq: VMDq pool to assign
+ *
+ *  Puts an ethernet address into a receive address register, or
+ *  finds the rar that it is aleady in; adds to the pool list
+ **/
+s32 ixgbe_insert_mac_addr(struct ixgbe_hw *hw, u8 *addr, u32 vmdq)
+{
+	return ixgbe_call_func(hw, hw->mac.ops.insert_mac_addr,
+	                       (hw, addr, vmdq),
+			       IXGBE_NOT_IMPLEMENTED);
+}
+
+/**
  *  ixgbe_set_rar - Set Rx address register
  *  @hw: pointer to hardware structure
  *  @index: Receive address register to write
@@ -724,15 +796,15 @@ s32 ixgbe_set_vfta(struct ixgbe_hw *hw, u32 vlan, u32 vind, bool vlan_on)
 }
 
 /**
- *  ixgbe_setup_fc - Set flow control
+ *  ixgbe_fc_enable - Enable flow control
  *  @hw: pointer to hardware structure
  *  @packetbuf_num: packet buffer number (0-7)
  *
  *  Configures the flow control settings based on SW configuration.
  **/
-s32 ixgbe_setup_fc(struct ixgbe_hw *hw, s32 packetbuf_num)
+s32 ixgbe_fc_enable(struct ixgbe_hw *hw, s32 packetbuf_num)
 {
-	return ixgbe_call_func(hw, hw->mac.ops.setup_fc, (hw, packetbuf_num),
+	return ixgbe_call_func(hw, hw->mac.ops.fc_enable, (hw, packetbuf_num),
 	                       IXGBE_NOT_IMPLEMENTED);
 }
 
@@ -778,6 +850,53 @@ s32 ixgbe_init_uta_tables(struct ixgbe_hw *hw)
 }
 
 /**
+ *  ixgbe_read_i2c_byte - Reads 8 bit word over I2C at specified device address
+ *  @hw: pointer to hardware structure
+ *  @byte_offset: byte offset to read
+ *  @data: value read
+ *
+ *  Performs byte read operation to SFP module's EEPROM over I2C interface.
+ **/
+s32 ixgbe_read_i2c_byte(struct ixgbe_hw *hw, u8 byte_offset, u8 dev_addr,
+                        u8 *data)
+{
+	return ixgbe_call_func(hw, hw->phy.ops.read_i2c_byte, (hw, byte_offset,
+	                       dev_addr, data), IXGBE_NOT_IMPLEMENTED);
+}
+
+/**
+ *  ixgbe_write_i2c_byte - Writes 8 bit word over I2C
+ *  @hw: pointer to hardware structure
+ *  @byte_offset: byte offset to write
+ *  @data: value to write
+ *
+ *  Performs byte write operation to SFP module's EEPROM over I2C interface
+ *  at a specified device address.
+ **/
+s32 ixgbe_write_i2c_byte(struct ixgbe_hw *hw, u8 byte_offset, u8 dev_addr,
+                         u8 data)
+{
+	return ixgbe_call_func(hw, hw->phy.ops.write_i2c_byte, (hw, byte_offset,
+	                       dev_addr, data), IXGBE_NOT_IMPLEMENTED);
+}
+
+/**
+ *  ixgbe_write_i2c_eeprom - Writes 8 bit EEPROM word over I2C interface
+ *  @hw: pointer to hardware structure
+ *  @byte_offset: EEPROM byte offset to write
+ *  @eeprom_data: value to write
+ *
+ *  Performs byte write operation to SFP module's EEPROM over I2C interface.
+ **/
+s32 ixgbe_write_i2c_eeprom(struct ixgbe_hw *hw,
+                           u8 byte_offset, u8 eeprom_data)
+{
+	return ixgbe_call_func(hw, hw->phy.ops.write_i2c_eeprom,
+	                       (hw, byte_offset, eeprom_data),
+	                       IXGBE_NOT_IMPLEMENTED);
+}
+
+/**
  *  ixgbe_read_i2c_eeprom - Reads 8 bit EEPROM word over I2C interface
  *  @hw: pointer to hardware structure
  *  @byte_offset: EEPROM byte offset to read
@@ -803,3 +922,16 @@ u32 ixgbe_get_supported_physical_layer(struct ixgbe_hw *hw)
 	return ixgbe_call_func(hw, hw->mac.ops.get_supported_physical_layer,
 	                       (hw), IXGBE_PHYSICAL_LAYER_UNKNOWN);
 }
+
+/**
+ *  ixgbe_enable_rx_dma - Enables Rx DMA unit, dependant on device specifics
+ *  @hw: pointer to hardware structure
+ *  @regval: bitfield to write to the Rx DMA register
+ *
+ *  Enables the Rx DMA unit of the device.
+ **/
+s32 ixgbe_enable_rx_dma(struct ixgbe_hw *hw, u32 regval)
+{
+	return ixgbe_call_func(hw, hw->mac.ops.enable_rx_dma,
+	                       (hw, regval), IXGBE_NOT_IMPLEMENTED);
+}
diff --git a/sys/dev/ixgbe/ixgbe_api.h b/sys/dev/ixgbe/ixgbe_api.h
index 8022f90..ba987ed 100644
--- a/sys/dev/ixgbe/ixgbe_api.h
+++ b/sys/dev/ixgbe/ixgbe_api.h
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -85,6 +85,7 @@ s32 ixgbe_read_eeprom(struct ixgbe_hw *hw, u16 offset, u16 *data);
 s32 ixgbe_validate_eeprom_checksum(struct ixgbe_hw *hw, u16 *checksum_val);
 s32 ixgbe_update_eeprom_checksum(struct ixgbe_hw *hw);
 
+s32 ixgbe_insert_mac_addr(struct ixgbe_hw *hw, u8 *addr, u32 vmdq);
 s32 ixgbe_set_rar(struct ixgbe_hw *hw, u32 index, u8 *addr, u32 vmdq,
                   u32 enable_addr);
 s32 ixgbe_clear_rar(struct ixgbe_hw *hw, u32 index);
@@ -102,7 +103,7 @@ s32 ixgbe_clear_vfta(struct ixgbe_hw *hw);
 s32 ixgbe_set_vfta(struct ixgbe_hw *hw, u32 vlan,
                    u32 vind, bool vlan_on);
 
-s32 ixgbe_setup_fc(struct ixgbe_hw *hw, s32 packetbuf_num);
+s32 ixgbe_fc_enable(struct ixgbe_hw *hw, s32 packetbuf_num);
 
 void ixgbe_set_mta(struct ixgbe_hw *hw, u8 *mc_addr);
 s32 ixgbe_get_phy_firmware_version(struct ixgbe_hw *hw,
@@ -112,5 +113,54 @@ s32 ixgbe_write_analog_reg8(struct ixgbe_hw *hw, u32 reg, u8 val);
 s32 ixgbe_init_uta_tables(struct ixgbe_hw *hw);
 s32 ixgbe_read_i2c_eeprom(struct ixgbe_hw *hw, u8 byte_offset, u8 *eeprom_data);
 u32 ixgbe_get_supported_physical_layer(struct ixgbe_hw *hw);
+s32 ixgbe_enable_rx_dma(struct ixgbe_hw *hw, u32 regval);
+s32 ixgbe_reinit_fdir_tables_82599(struct ixgbe_hw *hw);
+s32 ixgbe_init_fdir_signature_82599(struct ixgbe_hw *hw, u32 pballoc);
+s32 ixgbe_init_fdir_perfect_82599(struct ixgbe_hw *hw, u32 pballoc);
+s32 ixgbe_fdir_add_signature_filter_82599(struct ixgbe_hw *hw,
+                                          struct ixgbe_atr_input *input,
+                                          u8 queue);
+s32 ixgbe_fdir_add_perfect_filter_82599(struct ixgbe_hw *hw,
+                                        struct ixgbe_atr_input *input,
+                                        u16 soft_id,
+                                        u8 queue);
+u16 ixgbe_atr_compute_hash_82599(struct ixgbe_atr_input *input, u32 key);
+s32 ixgbe_atr_set_vlan_id_82599(struct ixgbe_atr_input *input, u16 vlan_id);
+s32 ixgbe_atr_set_src_ipv4_82599(struct ixgbe_atr_input *input, u32 src_addr);
+s32 ixgbe_atr_set_dst_ipv4_82599(struct ixgbe_atr_input *input, u32 dst_addr);
+s32 ixgbe_atr_set_src_ipv6_82599(struct ixgbe_atr_input *input, u32 src_addr_1,
+                                 u32 src_addr_2, u32 src_addr_3,
+                                 u32 src_addr_4);
+s32 ixgbe_atr_set_dst_ipv6_82599(struct ixgbe_atr_input *input, u32 dst_addr_1,
+                                 u32 dst_addr_2, u32 dst_addr_3,
+                                 u32 dst_addr_4);
+s32 ixgbe_atr_set_src_port_82599(struct ixgbe_atr_input *input, u16 src_port);
+s32 ixgbe_atr_set_dst_port_82599(struct ixgbe_atr_input *input, u16 dst_port);
+s32 ixgbe_atr_set_flex_byte_82599(struct ixgbe_atr_input *input, u16 flex_byte);
+s32 ixgbe_atr_set_vm_pool_82599(struct ixgbe_atr_input *input, u8 vm_pool);
+s32 ixgbe_atr_set_l4type_82599(struct ixgbe_atr_input *input, u8 l4type);
+s32 ixgbe_atr_get_vlan_id_82599(struct ixgbe_atr_input *input, u16 *vlan_id);
+s32 ixgbe_atr_get_src_ipv4_82599(struct ixgbe_atr_input *input, u32 *src_addr);
+s32 ixgbe_atr_get_dst_ipv4_82599(struct ixgbe_atr_input *input, u32 *dst_addr);
+s32 ixgbe_atr_get_src_ipv6_82599(struct ixgbe_atr_input *input, u32 *src_addr_1,
+                                 u32 *src_addr_2, u32 *src_addr_3,
+                                 u32 *src_addr_4);
+s32 ixgbe_atr_get_dst_ipv6_82599(struct ixgbe_atr_input *input, u32 *dst_addr_1,
+                                 u32 *dst_addr_2, u32 *dst_addr_3,
+                                 u32 *dst_addr_4);
+s32 ixgbe_atr_get_src_port_82599(struct ixgbe_atr_input *input, u16 *src_port);
+s32 ixgbe_atr_get_dst_port_82599(struct ixgbe_atr_input *input, u16 *dst_port);
+s32 ixgbe_atr_get_flex_byte_82599(struct ixgbe_atr_input *input,
+                                  u16 *flex_byte);
+s32 ixgbe_atr_get_vm_pool_82599(struct ixgbe_atr_input *input, u8 *vm_pool);
+s32 ixgbe_atr_get_l4type_82599(struct ixgbe_atr_input *input, u8 *l4type);
+s32 ixgbe_read_i2c_byte(struct ixgbe_hw *hw, u8 byte_offset, u8 dev_addr,
+                        u8 *data);
+s32 ixgbe_write_i2c_byte(struct ixgbe_hw *hw, u8 byte_offset, u8 dev_addr,
+                         u8 data);
+s32 ixgbe_write_i2c_eeprom(struct ixgbe_hw *hw, u8 byte_offset, u8 eeprom_data);
+s32 ixgbe_get_san_mac_addr(struct ixgbe_hw *hw, u8 *san_mac_addr);
+s32 ixgbe_set_san_mac_addr(struct ixgbe_hw *hw, u8 *san_mac_addr);
+s32 ixgbe_get_device_caps(struct ixgbe_hw *hw, u16 *device_caps);
 
 #endif /* _IXGBE_API_H_ */
diff --git a/sys/dev/ixgbe/ixgbe_common.c b/sys/dev/ixgbe/ixgbe_common.c
index 53e4aff..030605c 100644
--- a/sys/dev/ixgbe/ixgbe_common.c
+++ b/sys/dev/ixgbe/ixgbe_common.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -49,11 +49,7 @@ static void ixgbe_lower_eeprom_clk(struct ixgbe_hw *hw, u32 *eec);
 static void ixgbe_release_eeprom(struct ixgbe_hw *hw);
 static u16 ixgbe_calc_eeprom_checksum(struct ixgbe_hw *hw);
 
-static void ixgbe_enable_rar(struct ixgbe_hw *hw, u32 index);
-static void ixgbe_disable_rar(struct ixgbe_hw *hw, u32 index);
 static s32 ixgbe_mta_vector(struct ixgbe_hw *hw, u8 *mc_addr);
-void ixgbe_add_mc_addr(struct ixgbe_hw *hw, u8 *mc_addr);
-void ixgbe_add_uc_addr(struct ixgbe_hw *hw, u8 *addr, u32 vmdq);
 
 /**
  *  ixgbe_init_ops_generic - Inits function ptrs
@@ -86,6 +82,7 @@ s32 ixgbe_init_ops_generic(struct ixgbe_hw *hw)
 	mac->ops.clear_hw_cntrs = &ixgbe_clear_hw_cntrs_generic;
 	mac->ops.get_media_type = NULL;
 	mac->ops.get_supported_physical_layer = NULL;
+	mac->ops.enable_rx_dma = &ixgbe_enable_rx_dma_generic;
 	mac->ops.get_mac_addr = &ixgbe_get_mac_addr_generic;
 	mac->ops.stop_adapter = &ixgbe_stop_adapter_generic;
 	mac->ops.get_bus_info = &ixgbe_get_bus_info_generic;
@@ -94,12 +91,13 @@ s32 ixgbe_init_ops_generic(struct ixgbe_hw *hw)
 	/* LEDs */
 	mac->ops.led_on = &ixgbe_led_on_generic;
 	mac->ops.led_off = &ixgbe_led_off_generic;
-	mac->ops.blink_led_start = NULL;
-	mac->ops.blink_led_stop = NULL;
+	mac->ops.blink_led_start = &ixgbe_blink_led_start_generic;
+	mac->ops.blink_led_stop = &ixgbe_blink_led_stop_generic;
 
 	/* RAR, Multicast, VLAN */
 	mac->ops.set_rar = &ixgbe_set_rar_generic;
 	mac->ops.clear_rar = &ixgbe_clear_rar_generic;
+	mac->ops.insert_mac_addr = NULL;
 	mac->ops.set_vmdq = NULL;
 	mac->ops.clear_vmdq = NULL;
 	mac->ops.init_rx_addrs = &ixgbe_init_rx_addrs_generic;
@@ -111,6 +109,8 @@ s32 ixgbe_init_ops_generic(struct ixgbe_hw *hw)
 	mac->ops.set_vfta = NULL;
 	mac->ops.init_uta_tables = NULL;
 
+	/* Flow Control */
+	mac->ops.fc_enable = &ixgbe_fc_enable_generic;
 
 	/* Link */
 	mac->ops.get_link_capabilities = NULL;
@@ -133,28 +133,16 @@ s32 ixgbe_init_ops_generic(struct ixgbe_hw *hw)
 s32 ixgbe_start_hw_generic(struct ixgbe_hw *hw)
 {
 	u32 ctrl_ext;
+	s32 ret_val = IXGBE_SUCCESS;
 
 	/* Set the media type */
 	hw->phy.media_type = hw->mac.ops.get_media_type(hw);
 
-	/* Set bus info */
-	hw->mac.ops.get_bus_info(hw);
-
-	/* Identify the PHY */
-	hw->phy.ops.identify(hw);
-
-	/*
-	 * Store MAC address from RAR0, clear receive address registers, and
-	 * clear the multicast table
-	 */
-	hw->mac.ops.init_rx_addrs(hw);
+	/* PHY ops initialization must be done in reset_hw() */
 
 	/* Clear the VLAN filter table */
 	hw->mac.ops.clear_vfta(hw);
 
-	/* Set up link */
-	hw->mac.ops.setup_link(hw);
-
 	/* Clear statistics registers */
 	hw->mac.ops.clear_hw_cntrs(hw);
 
@@ -164,10 +152,13 @@ s32 ixgbe_start_hw_generic(struct ixgbe_hw *hw)
 	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 	IXGBE_WRITE_FLUSH(hw);
 
+	/* Setup flow control */
+	ixgbe_setup_fc(hw, 0);
+
 	/* Clear adapter stopped flag */
 	hw->adapter_stopped = FALSE;
 
-	return IXGBE_SUCCESS;
+	return ret_val;
 }
 
 /**
@@ -182,13 +173,17 @@ s32 ixgbe_start_hw_generic(struct ixgbe_hw *hw)
  **/
 s32 ixgbe_init_hw_generic(struct ixgbe_hw *hw)
 {
+	s32 status = IXGBE_SUCCESS;
+
 	/* Reset the hardware */
-	hw->mac.ops.reset_hw(hw);
+	status = hw->mac.ops.reset_hw(hw);
 
-	/* Start the HW */
-	hw->mac.ops.start_hw(hw);
+	if (status == IXGBE_SUCCESS) {
+		/* Start the HW */
+		status = hw->mac.ops.start_hw(hw);
+	}
 
-	return IXGBE_SUCCESS;
+	return status;
 }
 
 /**
@@ -214,15 +209,28 @@ s32 ixgbe_clear_hw_cntrs_generic(struct ixgbe_hw *hw)
 	IXGBE_READ_REG(hw, IXGBE_RLEC);
 	IXGBE_READ_REG(hw, IXGBE_LXONTXC);
 	IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
-	IXGBE_READ_REG(hw, IXGBE_LXONRXC);
-	IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
+	if (hw->mac.type >= ixgbe_mac_82599EB) {
+		IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
+		IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
+	} else {
+		IXGBE_READ_REG(hw, IXGBE_LXONRXC);
+		IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
+	}
 
 	for (i = 0; i < 8; i++) {
 		IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
 		IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
-		IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
-		IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
+		if (hw->mac.type >= ixgbe_mac_82599EB) {
+			IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
+			IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
+		} else {
+			IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
+			IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
+		}
 	}
+	if (hw->mac.type >= ixgbe_mac_82599EB)
+		for (i = 0; i < 8; i++)
+			IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
 	IXGBE_READ_REG(hw, IXGBE_PRC64);
 	IXGBE_READ_REG(hw, IXGBE_PRC127);
 	IXGBE_READ_REG(hw, IXGBE_PRC255);
@@ -391,6 +399,7 @@ void ixgbe_set_lan_id_multi_port_pcie(struct ixgbe_hw *hw)
 
 	reg = IXGBE_READ_REG(hw, IXGBE_STATUS);
 	bus->func = (reg & IXGBE_STATUS_LAN_ID) >> IXGBE_STATUS_LAN_ID_SHIFT;
+	bus->lan_id = bus->func;
 
 	/* check for a port swap */
 	reg = IXGBE_READ_REG(hw, IXGBE_FACTPS);
@@ -595,7 +604,6 @@ s32 ixgbe_write_eeprom_generic(struct ixgbe_hw *hw, u16 offset, u16 data)
 		ixgbe_shift_out_eeprom_bits(hw, data, 16);
 		ixgbe_standby_eeprom(hw);
 
-		msec_delay(hw->eeprom.semaphore_delay);
 		/* Done with writing - release the EEPROM */
 		ixgbe_release_eeprom(hw);
 	}
@@ -802,7 +810,7 @@ static s32 ixgbe_get_eeprom_semaphore(struct ixgbe_hw *hw)
 			status = IXGBE_SUCCESS;
 			break;
 		}
-		msec_delay(1);
+		usec_delay(50);
 	}
 
 	/* Now get the semaphore between SW/FW through the SWESMBI bit */
@@ -830,11 +838,14 @@ static s32 ixgbe_get_eeprom_semaphore(struct ixgbe_hw *hw)
 		 * was not granted because we don't have access to the EEPROM
 		 */
 		if (i >= timeout) {
-			DEBUGOUT("Driver can't access the Eeprom - Semaphore "
+			DEBUGOUT("SWESMBI Software EEPROM semaphore "
 			         "not granted.\n");
 			ixgbe_release_eeprom_semaphore(hw);
 			status = IXGBE_ERR_EEPROM;
 		}
+	} else {
+		DEBUGOUT("Software semaphore SMBI between device drivers "
+		         "not granted.\n");
 	}
 
 	return status;
@@ -1296,38 +1307,6 @@ s32 ixgbe_clear_rar_generic(struct ixgbe_hw *hw, u32 index)
 }
 
 /**
- *  ixgbe_enable_rar - Enable Rx address register
- *  @hw: pointer to hardware structure
- *  @index: index into the RAR table
- *
- *  Enables the select receive address register.
- **/
-static void ixgbe_enable_rar(struct ixgbe_hw *hw, u32 index)
-{
-	u32 rar_high;
-
-	rar_high = IXGBE_READ_REG(hw, IXGBE_RAH(index));
-	rar_high |= IXGBE_RAH_AV;
-	IXGBE_WRITE_REG(hw, IXGBE_RAH(index), rar_high);
-}
-
-/**
- *  ixgbe_disable_rar - Disable Rx address register
- *  @hw: pointer to hardware structure
- *  @index: index into the RAR table
- *
- *  Disables the select receive address register.
- **/
-static void ixgbe_disable_rar(struct ixgbe_hw *hw, u32 index)
-{
-	u32 rar_high;
-
-	rar_high = IXGBE_READ_REG(hw, IXGBE_RAH(index));
-	rar_high &= (~IXGBE_RAH_AV);
-	IXGBE_WRITE_REG(hw, IXGBE_RAH(index), rar_high);
-}
-
-/**
  *  ixgbe_init_rx_addrs_generic - Initializes receive address filters.
  *  @hw: pointer to hardware structure
  *
@@ -1378,7 +1357,6 @@ s32 ixgbe_init_rx_addrs_generic(struct ixgbe_hw *hw)
 	}
 
 	/* Clear the MTA */
-	hw->addr_ctrl.mc_addr_in_rar_count = 0;
 	hw->addr_ctrl.mta_in_use = 0;
 	IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL, hw->mac.mc_filter_type);
 
@@ -1411,8 +1389,7 @@ void ixgbe_add_uc_addr(struct ixgbe_hw *hw, u8 *addr, u32 vmdq)
 	 * else put the controller into promiscuous mode
 	 */
 	if (hw->addr_ctrl.rar_used_count < rar_entries) {
-		rar = hw->addr_ctrl.rar_used_count -
-		      hw->addr_ctrl.mc_addr_in_rar_count;
+		rar = hw->addr_ctrl.rar_used_count;
 		hw->mac.ops.set_rar(hw, rar, addr, vmdq, IXGBE_RAH_AV);
 		DEBUGOUT1("Added a secondary address to RAR[%d]\n", rar);
 		hw->addr_ctrl.rar_used_count++;
@@ -1451,14 +1428,13 @@ s32 ixgbe_update_uc_addr_list_generic(struct ixgbe_hw *hw, u8 *addr_list,
 	 * Clear accounting of old secondary address list,
 	 * don't count RAR[0]
 	 */
-	uc_addr_in_use = hw->addr_ctrl.rar_used_count -
-	                 hw->addr_ctrl.mc_addr_in_rar_count - 1;
+	uc_addr_in_use = hw->addr_ctrl.rar_used_count - 1;
 	hw->addr_ctrl.rar_used_count -= uc_addr_in_use;
 	hw->addr_ctrl.overflow_promisc = 0;
 
 	/* Zero out the other receive addresses */
-	DEBUGOUT1("Clearing RAR[1-%d]\n", uc_addr_in_use);
-	for (i = 1; i <= uc_addr_in_use; i++) {
+	DEBUGOUT1("Clearing RAR[1-%d]\n", hw->addr_ctrl.rar_used_count);
+	for (i = 1; i <= hw->addr_ctrl.rar_used_count; i++) {
 		IXGBE_WRITE_REG(hw, IXGBE_RAL(i), 0);
 		IXGBE_WRITE_REG(hw, IXGBE_RAH(i), 0);
 	}
@@ -1568,40 +1544,6 @@ void ixgbe_set_mta(struct ixgbe_hw *hw, u8 *mc_addr)
 }
 
 /**
- *  ixgbe_add_mc_addr - Adds a multicast address.
- *  @hw: pointer to hardware structure
- *  @mc_addr: new multicast address
- *
- *  Adds it to unused receive address register or to the multicast table.
- **/
-void ixgbe_add_mc_addr(struct ixgbe_hw *hw, u8 *mc_addr)
-{
-	u32 rar_entries = hw->mac.num_rar_entries;
-	u32 rar;
-
-	DEBUGOUT6(" MC Addr =%.2X %.2X %.2X %.2X %.2X %.2X\n",
-	          mc_addr[0], mc_addr[1], mc_addr[2],
-	          mc_addr[3], mc_addr[4], mc_addr[5]);
-
-	/*
-	 * Place this multicast address in the RAR if there is room,
-	 * else put it in the MTA
-	 */
-	if (hw->addr_ctrl.rar_used_count < rar_entries) {
-		/* use RAR from the end up for multicast */
-		rar = rar_entries - hw->addr_ctrl.mc_addr_in_rar_count - 1;
-		hw->mac.ops.set_rar(hw, rar, mc_addr, 0, IXGBE_RAH_AV);
-		DEBUGOUT1("Added a multicast address to RAR[%d]\n", rar);
-		hw->addr_ctrl.rar_used_count++;
-		hw->addr_ctrl.mc_addr_in_rar_count++;
-	} else {
-		ixgbe_set_mta(hw, mc_addr);
-	}
-
-	DEBUGOUT("ixgbe_add_mc_addr Complete\n");
-}
-
-/**
  *  ixgbe_update_mc_addr_list_generic - Updates MAC list of multicast addresses
  *  @hw: pointer to hardware structure
  *  @mc_addr_list: the list of new multicast addresses
@@ -1617,7 +1559,6 @@ s32 ixgbe_update_mc_addr_list_generic(struct ixgbe_hw *hw, u8 *mc_addr_list,
                                       u32 mc_addr_count, ixgbe_mc_addr_itr next)
 {
 	u32 i;
-	u32 rar_entries = hw->mac.num_rar_entries;
 	u32 vmdq;
 
 	/*
@@ -1625,18 +1566,8 @@ s32 ixgbe_update_mc_addr_list_generic(struct ixgbe_hw *hw, u8 *mc_addr_list,
 	 * use.
 	 */
 	hw->addr_ctrl.num_mc_addrs = mc_addr_count;
-	hw->addr_ctrl.rar_used_count -= hw->addr_ctrl.mc_addr_in_rar_count;
-	hw->addr_ctrl.mc_addr_in_rar_count = 0;
 	hw->addr_ctrl.mta_in_use = 0;
 
-	/* Zero out the other receive addresses. */
-	DEBUGOUT2("Clearing RAR[%d-%d]\n", hw->addr_ctrl.rar_used_count,
-	          rar_entries - 1);
-	for (i = hw->addr_ctrl.rar_used_count; i < rar_entries; i++) {
-		IXGBE_WRITE_REG(hw, IXGBE_RAL(i), 0);
-		IXGBE_WRITE_REG(hw, IXGBE_RAH(i), 0);
-	}
-
 	/* Clear the MTA */
 	DEBUGOUT(" Clearing MTA\n");
 	for (i = 0; i < hw->mac.mcft_size; i++)
@@ -1645,7 +1576,7 @@ s32 ixgbe_update_mc_addr_list_generic(struct ixgbe_hw *hw, u8 *mc_addr_list,
 	/* Add the new addresses */
 	for (i = 0; i < mc_addr_count; i++) {
 		DEBUGOUT(" Adding the multicast addresses:\n");
-		ixgbe_add_mc_addr(hw, next(hw, &mc_addr_list, &vmdq));
+		ixgbe_set_mta(hw, next(hw, &mc_addr_list, &vmdq));
 	}
 
 	/* Enable mta */
@@ -1665,15 +1596,8 @@ s32 ixgbe_update_mc_addr_list_generic(struct ixgbe_hw *hw, u8 *mc_addr_list,
  **/
 s32 ixgbe_enable_mc_generic(struct ixgbe_hw *hw)
 {
-	u32 i;
-	u32 rar_entries = hw->mac.num_rar_entries;
 	struct ixgbe_addr_filter_info *a = &hw->addr_ctrl;
 
-	if (a->mc_addr_in_rar_count > 0)
-		for (i = (rar_entries - a->mc_addr_in_rar_count);
-		     i < rar_entries; i++)
-			ixgbe_enable_rar(hw, i);
-
 	if (a->mta_in_use > 0)
 		IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL, IXGBE_MCSTCTRL_MFE |
 		                hw->mac.mc_filter_type);
@@ -1689,52 +1613,56 @@ s32 ixgbe_enable_mc_generic(struct ixgbe_hw *hw)
  **/
 s32 ixgbe_disable_mc_generic(struct ixgbe_hw *hw)
 {
-	u32 i;
-	u32 rar_entries = hw->mac.num_rar_entries;
 	struct ixgbe_addr_filter_info *a = &hw->addr_ctrl;
 
-	if (a->mc_addr_in_rar_count > 0)
-		for (i = (rar_entries - a->mc_addr_in_rar_count);
-		     i < rar_entries; i++)
-			ixgbe_disable_rar(hw, i);
-
 	if (a->mta_in_use > 0)
 		IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL, hw->mac.mc_filter_type);
 
 	return IXGBE_SUCCESS;
 }
 
-
 /**
- *  ixgbe_fc_autoneg - Configure flow control
+ *  ixgbe_fc_enable_generic - Enable flow control
  *  @hw: pointer to hardware structure
+ *  @packetbuf_num: packet buffer number (0-7)
  *
- *  Negotiates flow control capabilities with link partner using autoneg and
- *  applies the results.
+ *  Enable flow control according to the current settings.
  **/
-s32 ixgbe_fc_autoneg(struct ixgbe_hw *hw)
+s32 ixgbe_fc_enable_generic(struct ixgbe_hw *hw, s32 packetbuf_num)
 {
 	s32 ret_val = IXGBE_SUCCESS;
-	u32 i, reg, pcs_anadv_reg, pcs_lpab_reg;
+	u32 mflcn_reg, fccfg_reg;
+	u32 reg;
 
-	DEBUGFUNC("ixgbe_fc_autoneg");
+	DEBUGFUNC("ixgbe_fc_enable_generic");
 
-	reg = IXGBE_READ_REG(hw, IXGBE_PCS1GANA);
+	/* Negotiate the fc mode to use */
+	ret_val = ixgbe_fc_autoneg(hw);
+	if (ret_val)
+		goto out;
+
+	/* Disable any previous flow control settings */
+	mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
+	mflcn_reg &= ~(IXGBE_MFLCN_RFCE | IXGBE_MFLCN_RPFCE);
+
+	fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
+	fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
 
 	/*
 	 * The possible values of fc.current_mode are:
-	 * 0:  Flow control is completely disabled
-	 * 1:  Rx flow control is enabled (we can receive pause frames,
-	 *     but not send pause frames).
-	 * 2:  Tx flow control is enabled (we can send pause frames but
-	 *     we do not support receiving pause frames).
-	 * 3:  Both Rx and Tx flow control (symmetric) are enabled.
+	 * 0: Flow control is completely disabled
+	 * 1: Rx flow control is enabled (we can receive pause frames,
+	 *    but not send pause frames).
+	 * 2: Tx flow control is enabled (we can send pause frames but
+	 *    we do not support receiving pause frames).
+	 * 3: Both Rx and Tx flow control (symmetric) are enabled.
 	 * other: Invalid.
 	 */
 	switch (hw->fc.current_mode) {
 	case ixgbe_fc_none:
-		/* Flow control completely disabled by software override. */
-		reg &= ~(IXGBE_PCS1GANA_SYM_PAUSE | IXGBE_PCS1GANA_ASM_PAUSE);
+		/* Flow control is disabled by software override or autoneg.
+		 * The code below will actually disable it in the HW.
+		 */
 		break;
 	case ixgbe_fc_rx_pause:
 		/*
@@ -1745,19 +1673,19 @@ s32 ixgbe_fc_autoneg(struct ixgbe_hw *hw)
 		 * symmetric and asymmetric Rx PAUSE.  Later, we will
 		 * disable the adapter's ability to send PAUSE frames.
 		 */
-		reg |= (IXGBE_PCS1GANA_SYM_PAUSE | IXGBE_PCS1GANA_ASM_PAUSE);
+		mflcn_reg |= IXGBE_MFLCN_RFCE;
 		break;
 	case ixgbe_fc_tx_pause:
 		/*
 		 * Tx Flow control is enabled, and Rx Flow control is
 		 * disabled by software override.
 		 */
-		reg |= (IXGBE_PCS1GANA_ASM_PAUSE);
-		reg &= ~(IXGBE_PCS1GANA_SYM_PAUSE);
+		fccfg_reg |= IXGBE_FCCFG_TFCE_802_3X;
 		break;
 	case ixgbe_fc_full:
 		/* Flow control (both Rx and Tx) is enabled by SW override. */
-		reg |= (IXGBE_PCS1GANA_SYM_PAUSE | IXGBE_PCS1GANA_ASM_PAUSE);
+		mflcn_reg |= IXGBE_MFLCN_RFCE;
+		fccfg_reg |= IXGBE_FCCFG_TFCE_802_3X;
 		break;
 	default:
 		DEBUGOUT("Flow control param set incorrectly\n");
@@ -1766,39 +1694,80 @@ s32 ixgbe_fc_autoneg(struct ixgbe_hw *hw)
 		break;
 	}
 
-	IXGBE_WRITE_REG(hw, IXGBE_PCS1GANA, reg);
-	reg = IXGBE_READ_REG(hw, IXGBE_PCS1GLCTL);
+	/* Set 802.3x based flow control settings. */
+	mflcn_reg |= IXGBE_MFLCN_DPF;
+	IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
+	IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
 
-	/* Set PCS register for autoneg */
-	/* Enable and restart autoneg */
-	reg |= IXGBE_PCS1GLCTL_AN_ENABLE | IXGBE_PCS1GLCTL_AN_RESTART;
+	/* Set up and enable Rx high/low water mark thresholds, enable XON. */
+	if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
+		if (hw->fc.send_xon) {
+			IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(packetbuf_num),
+			                (hw->fc.low_water | IXGBE_FCRTL_XONE));
+		} else {
+			IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(packetbuf_num),
+			                hw->fc.low_water);
+		}
 
-	/* Disable AN timeout */
-	if (hw->fc.strict_ieee)
-		reg &= ~IXGBE_PCS1GLCTL_AN_1G_TIMEOUT_EN;
+		IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(packetbuf_num),
+		                (hw->fc.high_water | IXGBE_FCRTH_FCEN));
+	}
 
-	DEBUGOUT1("Configuring Autoneg; PCS_LCTL = 0x%08X\n", reg);
-	IXGBE_WRITE_REG(hw, IXGBE_PCS1GLCTL, reg);
+	/* Configure pause time (2 TCs per register) */
+	reg = IXGBE_READ_REG(hw, IXGBE_FCTTV(packetbuf_num));
+	if ((packetbuf_num & 1) == 0)
+		reg = (reg & 0xFFFF0000) | hw->fc.pause_time;
+	else
+		reg = (reg & 0x0000FFFF) | (hw->fc.pause_time << 16);
+	IXGBE_WRITE_REG(hw, IXGBE_FCTTV(packetbuf_num / 2), reg);
 
-	/* See if autonegotiation has succeeded */
-	hw->mac.autoneg_succeeded = 0;
-	for (i = 0; i < FIBER_LINK_UP_LIMIT; i++) {
-		msec_delay(10);
-		reg = IXGBE_READ_REG(hw, IXGBE_PCS1GLSTA);
-		if ((reg & (IXGBE_PCS1GLSTA_LINK_OK |
-		     IXGBE_PCS1GLSTA_AN_COMPLETE)) ==
-		    (IXGBE_PCS1GLSTA_LINK_OK |
-		     IXGBE_PCS1GLSTA_AN_COMPLETE)) {
-			if (!(reg & IXGBE_PCS1GLSTA_AN_TIMED_OUT))
-				hw->mac.autoneg_succeeded = 1;
-			break;
-		}
-	}
+	IXGBE_WRITE_REG(hw, IXGBE_FCRTV, (hw->fc.pause_time >> 1));
 
-	if (!hw->mac.autoneg_succeeded) {
-		/* Autoneg failed to achieve a link, so we turn fc off */
-		hw->fc.current_mode = ixgbe_fc_none;
-		DEBUGOUT("Flow Control = NONE.\n");
+out:
+	return ret_val;
+}
+
+/**
+ *  ixgbe_fc_autoneg - Configure flow control
+ *  @hw: pointer to hardware structure
+ *
+ *  Compares our advertised flow control capabilities to those advertised by
+ *  our link partner, and determines the proper flow control mode to use.
+ **/
+s32 ixgbe_fc_autoneg(struct ixgbe_hw *hw)
+{
+	s32 ret_val = IXGBE_SUCCESS;
+	ixgbe_link_speed speed;
+	u32 pcs_anadv_reg, pcs_lpab_reg, linkstat;
+	bool link_up;
+
+	DEBUGFUNC("ixgbe_fc_autoneg");
+
+	/*
+	 * AN should have completed when the cable was plugged in.
+	 * Look for reasons to bail out.  Bail out if:
+	 * - FC autoneg is disabled, or if
+	 * - we don't have multispeed fiber, or if
+	 * - we're not running at 1G, or if
+	 * - link is not up, or if
+	 * - link is up but AN did not complete, or if
+	 * - link is up and AN completed but timed out
+	 *
+	 * Since we're being called from an LSC, link is already know to be up.
+	 * So use link_up_wait_to_complete=FALSE.
+	 */
+	hw->mac.ops.check_link(hw, &speed, &link_up, FALSE);
+	linkstat = IXGBE_READ_REG(hw, IXGBE_PCS1GLSTA);
+
+	if (hw->fc.disable_fc_autoneg ||
+	    !hw->phy.multispeed_fiber ||
+	    (speed != IXGBE_LINK_SPEED_1GB_FULL) ||
+	    !link_up ||
+	    ((linkstat & IXGBE_PCS1GLSTA_AN_COMPLETE) == 0) ||
+	    ((linkstat & IXGBE_PCS1GLSTA_AN_TIMED_OUT) == 1)) {
+		hw->fc.fc_was_autonegged = FALSE;
+		hw->fc.current_mode = hw->fc.requested_mode;
+		DEBUGOUT("Autoneg FC was skipped.\n");
 		goto out;
 	}
 
@@ -1841,10 +1810,128 @@ s32 ixgbe_fc_autoneg(struct ixgbe_hw *hw)
 		DEBUGOUT("Flow Control = NONE.\n");
 	}
 
+	/* Record that current_mode is the result of a successful autoneg */
+	hw->fc.fc_was_autonegged = TRUE;
+
 out:
 	return ret_val;
 }
 
+/**
+ *  ixgbe_setup_fc - Set up flow control
+ *  @hw: pointer to hardware structure
+ *
+ *  Called at init time to set up flow control.
+ **/
+s32 ixgbe_setup_fc(struct ixgbe_hw *hw, s32 packetbuf_num)
+{
+	s32 ret_val = IXGBE_SUCCESS;
+	u32 reg;
+
+
+	/* Validate the packetbuf configuration */
+	if (packetbuf_num < 0 || packetbuf_num > 7) {
+		DEBUGOUT1("Invalid packet buffer number [%d], expected range is"
+		          " 0-7\n", packetbuf_num);
+		ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
+		goto out;
+	}
+
+	/*
+	 * Validate the water mark configuration.  Zero water marks are invalid
+	 * because it causes the controller to just blast out fc packets.
+	 */
+	if (!hw->fc.low_water || !hw->fc.high_water || !hw->fc.pause_time) {
+		DEBUGOUT("Invalid water mark configuration\n");
+		ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
+		goto out;
+	}
+
+	/*
+	 * Validate the requested mode.  Strict IEEE mode does not allow
+	 * ixgbe_fc_rx_pause because it will cause us to fail at UNH.
+	 */
+	if (hw->fc.strict_ieee && hw->fc.requested_mode == ixgbe_fc_rx_pause) {
+		DEBUGOUT("ixgbe_fc_rx_pause not valid in strict IEEE mode\n");
+		ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
+		goto out;
+	}
+
+	/*
+	 * 10gig parts do not have a word in the EEPROM to determine the
+	 * default flow control setting, so we explicitly set it to full.
+	 */
+	if (hw->fc.requested_mode == ixgbe_fc_default)
+		hw->fc.requested_mode = ixgbe_fc_full;
+
+	/*
+	 * Set up the 1G flow control advertisement registers so the HW will be
+	 * able to do fc autoneg once the cable is plugged in.  If we end up
+	 * using 10g instead, this is harmless.
+	 */
+	reg = IXGBE_READ_REG(hw, IXGBE_PCS1GANA);
+
+	/*
+	 * The possible values of fc.requested_mode are:
+	 * 0: Flow control is completely disabled
+	 * 1: Rx flow control is enabled (we can receive pause frames,
+	 *    but not send pause frames).
+	 * 2: Tx flow control is enabled (we can send pause frames but
+	 *    we do not support receiving pause frames).
+	 * 3: Both Rx and Tx flow control (symmetric) are enabled.
+	 * other: Invalid.
+	 */
+	switch (hw->fc.requested_mode) {
+	case ixgbe_fc_none:
+		/* Flow control completely disabled by software override. */
+		reg &= ~(IXGBE_PCS1GANA_SYM_PAUSE | IXGBE_PCS1GANA_ASM_PAUSE);
+		break;
+	case ixgbe_fc_rx_pause:
+		/*
+		 * Rx Flow control is enabled and Tx Flow control is
+		 * disabled by software override. Since there really
+		 * isn't a way to advertise that we are capable of RX
+		 * Pause ONLY, we will advertise that we support both
+		 * symmetric and asymmetric Rx PAUSE.  Later, we will
+		 * disable the adapter's ability to send PAUSE frames.
+		 */
+		reg |= (IXGBE_PCS1GANA_SYM_PAUSE | IXGBE_PCS1GANA_ASM_PAUSE);
+		break;
+	case ixgbe_fc_tx_pause:
+		/*
+		 * Tx Flow control is enabled, and Rx Flow control is
+		 * disabled by software override.
+		 */
+		reg |= (IXGBE_PCS1GANA_ASM_PAUSE);
+		reg &= ~(IXGBE_PCS1GANA_SYM_PAUSE);
+		break;
+	case ixgbe_fc_full:
+		/* Flow control (both Rx and Tx) is enabled by SW override. */
+		reg |= (IXGBE_PCS1GANA_SYM_PAUSE | IXGBE_PCS1GANA_ASM_PAUSE);
+		break;
+	default:
+		DEBUGOUT("Flow control param set incorrectly\n");
+		ret_val = -IXGBE_ERR_CONFIG;
+		goto out;
+		break;
+	}
+
+	IXGBE_WRITE_REG(hw, IXGBE_PCS1GANA, reg);
+	reg = IXGBE_READ_REG(hw, IXGBE_PCS1GLCTL);
+
+	/* Enable and restart autoneg to inform the link partner */
+	reg |= IXGBE_PCS1GLCTL_AN_ENABLE | IXGBE_PCS1GLCTL_AN_RESTART;
+
+	/* Disable AN timeout */
+	if (hw->fc.strict_ieee)
+		reg &= ~IXGBE_PCS1GLCTL_AN_1G_TIMEOUT_EN;
+
+	IXGBE_WRITE_REG(hw, IXGBE_PCS1GLCTL, reg);
+	DEBUGOUT1("Set up FC; PCS1GLCTL = 0x%08X\n", reg);
+
+out:
+	return ret_val;
+}
 
 /**
  *  ixgbe_disable_pcie_master - Disable PCI-express master access
@@ -1904,6 +1991,10 @@ s32 ixgbe_acquire_swfw_sync(struct ixgbe_hw *hw, u16 mask)
 	s32 timeout = 200;
 
 	while (timeout) {
+		/*
+		 * SW EEPROM semaphore bit is used for access to all
+		 * SW_FW_SYNC/GSSR bits (not just EEPROM)
+		 */
 		if (ixgbe_get_eeprom_semaphore(hw))
 			return -IXGBE_ERR_SWFW_SYNC;
 
@@ -1921,7 +2012,7 @@ s32 ixgbe_acquire_swfw_sync(struct ixgbe_hw *hw, u16 mask)
 	}
 
 	if (!timeout) {
-		DEBUGOUT("Driver can't access resource, GSSR timeout.\n");
+		DEBUGOUT("Driver can't access resource, SW_FW_SYNC timeout.\n");
 		return -IXGBE_ERR_SWFW_SYNC;
 	}
 
@@ -1952,5 +2043,77 @@ void ixgbe_release_swfw_sync(struct ixgbe_hw *hw, u16 mask)
 	IXGBE_WRITE_REG(hw, IXGBE_GSSR, gssr);
 
 	ixgbe_release_eeprom_semaphore(hw);
+
+	/* Delay before attempt to obtain semaphore again to allow FW access */
+	msec_delay(hw->eeprom.semaphore_delay);
+}
+
+/**
+ *  ixgbe_enable_rx_dma_generic - Enable the Rx DMA unit
+ *  @hw: pointer to hardware structure
+ *  @regval: register value to write to RXCTRL
+ *
+ *  Enables the Rx DMA unit
+ **/
+s32 ixgbe_enable_rx_dma_generic(struct ixgbe_hw *hw, u32 regval)
+{
+	IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, regval);
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_blink_led_start_generic - Blink LED based on index.
+ *  @hw: pointer to hardware structure
+ *  @index: led number to blink
+ **/
+s32 ixgbe_blink_led_start_generic(struct ixgbe_hw *hw, u32 index)
+{
+	ixgbe_link_speed speed = 0;
+	bool link_up = 0;
+	u32 autoc_reg = IXGBE_READ_REG(hw, IXGBE_AUTOC);
+	u32 led_reg = IXGBE_READ_REG(hw, IXGBE_LEDCTL);
+
+	/*
+	 * Link must be up to auto-blink the LEDs;
+	 * Force it if link is down.
+	 */
+	hw->mac.ops.check_link(hw, &speed, &link_up, FALSE);
+
+	if (!link_up) {
+		autoc_reg |= IXGBE_AUTOC_FLU;
+		IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc_reg);
+		msec_delay(10);
+	}
+
+	led_reg &= ~IXGBE_LED_MODE_MASK(index);
+	led_reg |= IXGBE_LED_BLINK(index);
+	IXGBE_WRITE_REG(hw, IXGBE_LEDCTL, led_reg);
+	IXGBE_WRITE_FLUSH(hw);
+
+	return IXGBE_SUCCESS;
+}
+
+/**
+ *  ixgbe_blink_led_stop_generic - Stop blinking LED based on index.
+ *  @hw: pointer to hardware structure
+ *  @index: led number to stop blinking
+ **/
+s32 ixgbe_blink_led_stop_generic(struct ixgbe_hw *hw, u32 index)
+{
+	u32 autoc_reg = IXGBE_READ_REG(hw, IXGBE_AUTOC);
+	u32 led_reg = IXGBE_READ_REG(hw, IXGBE_LEDCTL);
+
+	autoc_reg &= ~IXGBE_AUTOC_FLU;
+	autoc_reg |= IXGBE_AUTOC_AN_RESTART;
+	IXGBE_WRITE_REG(hw, IXGBE_AUTOC, autoc_reg);
+
+	led_reg &= ~IXGBE_LED_MODE_MASK(index);
+	led_reg &= ~IXGBE_LED_BLINK(index);
+	led_reg |= IXGBE_LED_LINK_ACTIVE << IXGBE_LED_MODE_SHIFT(index);
+	IXGBE_WRITE_REG(hw, IXGBE_LEDCTL, led_reg);
+	IXGBE_WRITE_FLUSH(hw);
+
+	return IXGBE_SUCCESS;
 }
 
diff --git a/sys/dev/ixgbe/ixgbe_common.h b/sys/dev/ixgbe/ixgbe_common.h
index d94f674..0c53575 100644
--- a/sys/dev/ixgbe/ixgbe_common.h
+++ b/sys/dev/ixgbe/ixgbe_common.h
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -36,6 +36,11 @@
 #define _IXGBE_COMMON_H_
 
 #include "ixgbe_type.h"
+#define IXGBE_WRITE_REG64(hw, reg, value) \
+	do { \
+		IXGBE_WRITE_REG(hw, reg, (u32) value); \
+		IXGBE_WRITE_REG(hw, reg + 4, (u32) (value >> 32)); \
+	} while (0)
 
 s32 ixgbe_init_ops_generic(struct ixgbe_hw *hw);
 s32 ixgbe_init_hw_generic(struct ixgbe_hw *hw);
@@ -68,11 +73,13 @@ s32 ixgbe_update_mc_addr_list_generic(struct ixgbe_hw *hw, u8 *mc_addr_list,
                                       ixgbe_mc_addr_itr func);
 s32 ixgbe_update_uc_addr_list_generic(struct ixgbe_hw *hw, u8 *addr_list,
                                       u32 addr_count, ixgbe_mc_addr_itr func);
+void ixgbe_add_uc_addr(struct ixgbe_hw *hw, u8 *addr, u32 vmdq);
 s32 ixgbe_enable_mc_generic(struct ixgbe_hw *hw);
 s32 ixgbe_disable_mc_generic(struct ixgbe_hw *hw);
+s32 ixgbe_enable_rx_dma_generic(struct ixgbe_hw *hw, u32 regval);
 
-s32 ixgbe_setup_fc_generic(struct ixgbe_hw *hw, s32 packetbuf_num);
-s32 ixgbe_fc_enable(struct ixgbe_hw *hw, s32 packtetbuf_num);
+s32 ixgbe_setup_fc(struct ixgbe_hw *hw, s32 packetbuf_num);
+s32 ixgbe_fc_enable_generic(struct ixgbe_hw *hw, s32 packtetbuf_num);
 s32 ixgbe_fc_autoneg(struct ixgbe_hw *hw);
 
 s32 ixgbe_validate_mac_addr(u8 *mac_addr);
@@ -82,4 +89,7 @@ s32 ixgbe_disable_pcie_master(struct ixgbe_hw *hw);
 
 s32 ixgbe_read_analog_reg8_generic(struct ixgbe_hw *hw, u32 reg, u8 *val);
 s32 ixgbe_write_analog_reg8_generic(struct ixgbe_hw *hw, u32 reg, u8 val);
+s32 ixgbe_blink_led_start_generic(struct ixgbe_hw *hw, u32 index);
+s32 ixgbe_blink_led_stop_generic(struct ixgbe_hw *hw, u32 index);
+
 #endif /* IXGBE_COMMON */
diff --git a/sys/dev/ixgbe/ixgbe_osdep.h b/sys/dev/ixgbe/ixgbe_osdep.h
index be6f577..53b64f5 100644
--- a/sys/dev/ixgbe/ixgbe_osdep.h
+++ b/sys/dev/ixgbe/ixgbe_osdep.h
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -96,6 +96,7 @@ typedef boolean_t	bool;
 
 #define le16_to_cpu 
 
+#if __FreeBSD_version < 800000
 #if defined(__i386__) || defined(__amd64__)
 #define mb()	__asm volatile("mfence" ::: "memory")
 #define wmb()	__asm volatile("sfence" ::: "memory")
@@ -105,6 +106,7 @@ typedef boolean_t	bool;
 #define rmb()
 #define wmb()
 #endif
+#endif
 
 struct ixgbe_osdep
 {
@@ -113,11 +115,14 @@ struct ixgbe_osdep
 	struct device     *dev;
 };
 
-/* This is needed by the shared code */
+/* These routines are needed by the shared code */
 struct ixgbe_hw; 
 extern u16 ixgbe_read_pci_cfg(struct ixgbe_hw *, u32);
 #define IXGBE_READ_PCIE_WORD ixgbe_read_pci_cfg
 
+extern void ixgbe_write_pci_cfg(struct ixgbe_hw *, u32, u16);
+#define IXGBE_WRITE_PCIE_WORD ixgbe_write_pci_cfg
+
 #define IXGBE_WRITE_FLUSH(a) IXGBE_READ_REG(a, IXGBE_STATUS)
 
 #define IXGBE_READ_REG(a, reg) (\
diff --git a/sys/dev/ixgbe/ixgbe_phy.c b/sys/dev/ixgbe/ixgbe_phy.c
index 291d454..afd30ee 100644
--- a/sys/dev/ixgbe/ixgbe_phy.c
+++ b/sys/dev/ixgbe/ixgbe_phy.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -36,6 +36,19 @@
 #include "ixgbe_common.h"
 #include "ixgbe_phy.h"
 
+static void ixgbe_i2c_start(struct ixgbe_hw *hw);
+static void ixgbe_i2c_stop(struct ixgbe_hw *hw);
+static s32 ixgbe_clock_in_i2c_byte(struct ixgbe_hw *hw, u8 *data);
+static s32 ixgbe_clock_out_i2c_byte(struct ixgbe_hw *hw, u8 data);
+static s32 ixgbe_get_i2c_ack(struct ixgbe_hw *hw);
+static s32 ixgbe_clock_in_i2c_bit(struct ixgbe_hw *hw, bool *data);
+static s32 ixgbe_clock_out_i2c_bit(struct ixgbe_hw *hw, bool data);
+static s32 ixgbe_raise_i2c_clk(struct ixgbe_hw *hw, u32 *i2cctl);
+static void ixgbe_lower_i2c_clk(struct ixgbe_hw *hw, u32 *i2cctl);
+static s32 ixgbe_set_i2c_data(struct ixgbe_hw *hw, u32 *i2cctl, bool data);
+static bool ixgbe_get_i2c_data(u32 *i2cctl);
+void ixgbe_i2c_bus_clear(struct ixgbe_hw *hw);
+
 /**
  *  ixgbe_init_phy_ops_generic - Inits PHY function ptrs
  *  @hw: pointer to the hardware structure
@@ -55,6 +68,11 @@ s32 ixgbe_init_phy_ops_generic(struct ixgbe_hw *hw)
 	phy->ops.setup_link_speed = &ixgbe_setup_phy_link_speed_generic;
 	phy->ops.check_link = NULL;
 	phy->ops.get_firmware_version = NULL;
+	phy->ops.read_i2c_byte = &ixgbe_read_i2c_byte_generic;
+	phy->ops.write_i2c_byte = &ixgbe_write_i2c_byte_generic;
+	phy->ops.read_i2c_eeprom = &ixgbe_read_i2c_eeprom_generic;
+	phy->ops.write_i2c_eeprom = &ixgbe_write_i2c_eeprom_generic;
+	phy->ops.i2c_bus_clear = &ixgbe_i2c_bus_clear;
 	phy->ops.identify_sfp = &ixgbe_identify_sfp_module_generic;
 	phy->sfp_type = ixgbe_sfp_type_unknown;
 
@@ -71,6 +89,7 @@ s32 ixgbe_identify_phy_generic(struct ixgbe_hw *hw)
 {
 	s32 status = IXGBE_ERR_PHY_ADDR_INVALID;
 	u32 phy_addr;
+	u16 ext_ability = 0;
 
 	if (hw->phy.type == ixgbe_phy_unknown) {
 		for (phy_addr = 0; phy_addr < IXGBE_MAX_PHY_ADDR; phy_addr++) {
@@ -79,10 +98,29 @@ s32 ixgbe_identify_phy_generic(struct ixgbe_hw *hw)
 				ixgbe_get_phy_id(hw);
 				hw->phy.type =
 				        ixgbe_get_phy_type_from_id(hw->phy.id);
+
+				if (hw->phy.type == ixgbe_phy_unknown) {
+					hw->phy.ops.read_reg(hw,
+						  IXGBE_MDIO_PHY_EXT_ABILITY,
+					          IXGBE_MDIO_PMA_PMD_DEV_TYPE,
+					          &ext_ability);
+					if (ext_ability &
+					    IXGBE_MDIO_PHY_10GBASET_ABILITY ||
+					    ext_ability &
+					    IXGBE_MDIO_PHY_1000BASET_ABILITY)
+						hw->phy.type =
+						         ixgbe_phy_cu_unknown;
+					else
+						hw->phy.type =
+						         ixgbe_phy_generic;
+				}
+
 				status = IXGBE_SUCCESS;
 				break;
 			}
 		}
+		if (status != IXGBE_SUCCESS)
+			hw->phy.addr = 0;
 	} else {
 		status = IXGBE_SUCCESS;
 	}
@@ -149,6 +187,9 @@ enum ixgbe_phy_type ixgbe_get_phy_type_from_id(u32 phy_id)
 	case TN1010_PHY_ID:
 		phy_type = ixgbe_phy_tn;
 		break;
+	case AQ1002_PHY_ID:
+		phy_type = ixgbe_phy_aq;
+		break;
 	case QT2022_PHY_ID:
 		phy_type = ixgbe_phy_qt;
 		break;
@@ -170,13 +211,40 @@ enum ixgbe_phy_type ixgbe_get_phy_type_from_id(u32 phy_id)
  **/
 s32 ixgbe_reset_phy_generic(struct ixgbe_hw *hw)
 {
+	u32 i;
+	u16 ctrl = 0;
+	s32 status = IXGBE_SUCCESS;
+
+	if (hw->phy.type == ixgbe_phy_unknown)
+		status = ixgbe_identify_phy_generic(hw);
+
+	if (status != IXGBE_SUCCESS || hw->phy.type == ixgbe_phy_none)
+		goto out;
+
 	/*
 	 * Perform soft PHY reset to the PHY_XS.
 	 * This will cause a soft reset to the PHY
 	 */
-	return hw->phy.ops.write_reg(hw, IXGBE_MDIO_PHY_XS_CONTROL,
-	                             IXGBE_MDIO_PHY_XS_DEV_TYPE,
-	                             IXGBE_MDIO_PHY_XS_RESET);
+	hw->phy.ops.write_reg(hw, IXGBE_MDIO_PHY_XS_CONTROL,
+	                      IXGBE_MDIO_PHY_XS_DEV_TYPE,
+	                      IXGBE_MDIO_PHY_XS_RESET);
+
+	/* Poll for reset bit to self-clear indicating reset is complete */
+	for (i = 0; i < 500; i++) {
+		msec_delay(1);
+		hw->phy.ops.read_reg(hw, IXGBE_MDIO_PHY_XS_CONTROL,
+		                     IXGBE_MDIO_PHY_XS_DEV_TYPE, &ctrl);
+		if (!(ctrl & IXGBE_MDIO_PHY_XS_RESET))
+			break;
+	}
+
+	if (ctrl & IXGBE_MDIO_PHY_XS_RESET) {
+		status = IXGBE_ERR_RESET_FAILED;
+		DEBUGOUT("PHY reset polling failed to complete.\n");
+	}
+
+out:
+	return status;
 }
 
 /**
@@ -454,6 +522,9 @@ s32 ixgbe_setup_phy_link_speed_generic(struct ixgbe_hw *hw,
 	if (speed & IXGBE_LINK_SPEED_1GB_FULL)
 		hw->phy.autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL;
 
+	if (speed & IXGBE_LINK_SPEED_100_FULL)
+		hw->phy.autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL;
+
 	/* Setup link based on the new speed settings */
 	hw->phy.ops.setup_link(hw);
 
@@ -461,6 +532,40 @@ s32 ixgbe_setup_phy_link_speed_generic(struct ixgbe_hw *hw,
 }
 
 /**
+ *  ixgbe_get_copper_link_capabilities_generic - Determines link capabilities
+ *  @hw: pointer to hardware structure
+ *  @speed: pointer to link speed
+ *  @autoneg: boolean auto-negotiation value
+ *
+ *  Determines the link capabilities by reading the AUTOC register.
+ **/
+s32 ixgbe_get_copper_link_capabilities_generic(struct ixgbe_hw *hw,
+                                             ixgbe_link_speed *speed,
+                                             bool *autoneg)
+{
+	s32 status = IXGBE_ERR_LINK_SETUP;
+	u16 speed_ability;
+
+	*speed = 0;
+	*autoneg = TRUE;
+
+	status = hw->phy.ops.read_reg(hw, IXGBE_MDIO_PHY_SPEED_ABILITY,
+	                              IXGBE_MDIO_PMA_PMD_DEV_TYPE,
+	                              &speed_ability);
+
+	if (status == IXGBE_SUCCESS) {
+		if (speed_ability & IXGBE_MDIO_PHY_SPEED_10G)
+			*speed |= IXGBE_LINK_SPEED_10GB_FULL;
+		if (speed_ability & IXGBE_MDIO_PHY_SPEED_1G)
+			*speed |= IXGBE_LINK_SPEED_1GB_FULL;
+		if (speed_ability & IXGBE_MDIO_PHY_SPEED_100M)
+			*speed |= IXGBE_LINK_SPEED_100_FULL;
+	}
+
+	return status;
+}
+
+/**
  *  ixgbe_check_phy_link_tnx - Determine link and speed status
  *  @hw: pointer to hardware structure
  *
@@ -525,6 +630,24 @@ s32 ixgbe_get_phy_firmware_version_tnx(struct ixgbe_hw *hw,
 	return status;
 }
 
+
+/**
+ *  ixgbe_get_phy_firmware_version_aq - Gets the PHY Firmware Version
+ *  @hw: pointer to hardware structure
+ *  @firmware_version: pointer to the PHY Firmware Version
+ **/
+s32 ixgbe_get_phy_firmware_version_aq(struct ixgbe_hw *hw,
+                                       u16 *firmware_version)
+{
+	s32 status = IXGBE_SUCCESS;
+
+	status = hw->phy.ops.read_reg(hw, AQ_FW_REV,
+	                              IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE,
+	                              firmware_version);
+
+	return status;
+}
+
 /**
  *  ixgbe_reset_phy_nl - Performs a PHY reset
  *  @hw: pointer to hardware structure
@@ -632,20 +755,30 @@ s32 ixgbe_identify_sfp_module_generic(struct ixgbe_hw *hw)
 {
 	s32 status = IXGBE_ERR_PHY_ADDR_INVALID;
 	u32 vendor_oui = 0;
+	enum ixgbe_sfp_type stored_sfp_type = hw->phy.sfp_type;
 	u8 identifier = 0;
 	u8 comp_codes_1g = 0;
 	u8 comp_codes_10g = 0;
-	u8 oui_bytes[4] = {0, 0, 0, 0};
+	u8 oui_bytes[3] = {0, 0, 0};
 	u8 transmission_media = 0;
+	u16 enforce_sfp = 0;
 
 	status = hw->phy.ops.read_i2c_eeprom(hw, IXGBE_SFF_IDENTIFIER,
 	                                     &identifier);
 
-	if (status == IXGBE_ERR_SFP_NOT_PRESENT) {
+	if (status == IXGBE_ERR_SFP_NOT_PRESENT || status == IXGBE_ERR_I2C) {
+		status = IXGBE_ERR_SFP_NOT_PRESENT;
 		hw->phy.sfp_type = ixgbe_sfp_type_not_present;
+		if (hw->phy.type != ixgbe_phy_nl) {
+			hw->phy.id = 0;
+			hw->phy.type = ixgbe_phy_unknown;
+		}
 		goto out;
 	}
 
+	/* LAN ID is needed for sfp_type determination */
+	hw->mac.ops.set_lan_id(hw);
+
 	if (identifier == IXGBE_SFF_IDENTIFIER_SFP) {
 		hw->phy.ops.read_i2c_eeprom(hw, IXGBE_SFF_1GBE_COMP_CODES,
 		                           &comp_codes_1g);
@@ -659,22 +792,57 @@ s32 ixgbe_identify_sfp_module_generic(struct ixgbe_hw *hw)
 		  * 0   SFP_DA_CU
 		  * 1   SFP_SR
 		  * 2   SFP_LR
+		  * 3   SFP_DA_CORE0 - 82599-specific
+		  * 4   SFP_DA_CORE1 - 82599-specific
+		  * 5   SFP_SR/LR_CORE0 - 82599-specific
+		  * 6   SFP_SR/LR_CORE1 - 82599-specific
 		  */
-		if (transmission_media & IXGBE_SFF_TWIN_AX_CAPABLE)
-			hw->phy.sfp_type = ixgbe_sfp_type_da_cu;
-		else if (comp_codes_10g & IXGBE_SFF_10GBASESR_CAPABLE)
-			hw->phy.sfp_type = ixgbe_sfp_type_sr;
-		else if (comp_codes_10g & IXGBE_SFF_10GBASELR_CAPABLE)
-			hw->phy.sfp_type = ixgbe_sfp_type_lr;
-		else
-			hw->phy.sfp_type = ixgbe_sfp_type_unknown;
+		if (hw->mac.type == ixgbe_mac_82598EB) {
+			if (transmission_media & IXGBE_SFF_TWIN_AX_CAPABLE)
+				hw->phy.sfp_type = ixgbe_sfp_type_da_cu;
+			else if (comp_codes_10g & IXGBE_SFF_10GBASESR_CAPABLE)
+				hw->phy.sfp_type = ixgbe_sfp_type_sr;
+			else if (comp_codes_10g & IXGBE_SFF_10GBASELR_CAPABLE)
+				hw->phy.sfp_type = ixgbe_sfp_type_lr;
+			else
+				hw->phy.sfp_type = ixgbe_sfp_type_unknown;
+		} else if (hw->mac.type == ixgbe_mac_82599EB) {
+			if (transmission_media & IXGBE_SFF_TWIN_AX_CAPABLE)
+				if (hw->bus.lan_id == 0)
+					hw->phy.sfp_type =
+					             ixgbe_sfp_type_da_cu_core0;
+				else
+					hw->phy.sfp_type =
+					             ixgbe_sfp_type_da_cu_core1;
+			else if (comp_codes_10g & IXGBE_SFF_10GBASESR_CAPABLE)
+				if (hw->bus.lan_id == 0)
+					hw->phy.sfp_type =
+					              ixgbe_sfp_type_srlr_core0;
+				else
+					hw->phy.sfp_type =
+					              ixgbe_sfp_type_srlr_core1;
+			else if (comp_codes_10g & IXGBE_SFF_10GBASELR_CAPABLE)
+				if (hw->bus.lan_id == 0)
+					hw->phy.sfp_type =
+					              ixgbe_sfp_type_srlr_core0;
+				else
+					hw->phy.sfp_type =
+					              ixgbe_sfp_type_srlr_core1;
+			else
+				hw->phy.sfp_type = ixgbe_sfp_type_unknown;
+		}
+
+		if (hw->phy.sfp_type != stored_sfp_type)
+			hw->phy.sfp_setup_needed = TRUE;
 
 		/* Determine if the SFP+ PHY is dual speed or not. */
-		if ((comp_codes_1g & IXGBE_SFF_1GBASESX_CAPABLE) &&
-			(comp_codes_10g & IXGBE_SFF_10GBASESR_CAPABLE))
+		if (((comp_codes_1g & IXGBE_SFF_1GBASESX_CAPABLE) &&
+		   (comp_codes_10g & IXGBE_SFF_10GBASESR_CAPABLE)) ||
+		   ((comp_codes_1g & IXGBE_SFF_1GBASELX_CAPABLE) &&
+		   (comp_codes_10g & IXGBE_SFF_10GBASELR_CAPABLE)))
 			hw->phy.multispeed_fiber = TRUE;
 		/* Determine PHY vendor */
-		if (hw->phy.type == ixgbe_phy_unknown) {
+		if (hw->phy.type != ixgbe_phy_nl) {
 			hw->phy.id = identifier;
 			hw->phy.ops.read_i2c_eeprom(hw,
 			                            IXGBE_SFF_VENDOR_OUI_BYTE0,
@@ -703,6 +871,9 @@ s32 ixgbe_identify_sfp_module_generic(struct ixgbe_hw *hw)
 			case IXGBE_SFF_VENDOR_OUI_AVAGO:
 				hw->phy.type = ixgbe_phy_sfp_avago;
 				break;
+			case IXGBE_SFF_VENDOR_OUI_INTEL:
+				hw->phy.type = ixgbe_phy_sfp_intel;
+				break;
 			default:
 				if (transmission_media &
 				    IXGBE_SFF_TWIN_AX_CAPABLE)
@@ -712,7 +883,34 @@ s32 ixgbe_identify_sfp_module_generic(struct ixgbe_hw *hw)
 				break;
 			}
 		}
-		status = IXGBE_SUCCESS;
+
+		if (comp_codes_10g == 0) {
+			hw->phy.type = ixgbe_phy_sfp_unsupported;
+			status = IXGBE_ERR_SFP_NOT_SUPPORTED;
+			goto out;
+		}
+		if (hw->mac.type == ixgbe_mac_82598EB ||
+		    (hw->phy.sfp_type != ixgbe_sfp_type_sr &&
+		     hw->phy.sfp_type != ixgbe_sfp_type_lr &&
+		     hw->phy.sfp_type != ixgbe_sfp_type_srlr_core0 &&
+		     hw->phy.sfp_type != ixgbe_sfp_type_srlr_core1)) {
+			status = IXGBE_SUCCESS;
+			goto out;
+		}
+
+		ixgbe_get_device_caps(hw, &enforce_sfp);
+		if (!(enforce_sfp & IXGBE_DEVICE_CAPS_ALLOW_ANY_SFP)) {
+			/* Make sure we're a supported PHY type */
+			if (hw->phy.type == ixgbe_phy_sfp_intel) {
+				status = IXGBE_SUCCESS;
+			} else {
+				DEBUGOUT("SFP+ module not supported\n");
+				hw->phy.type = ixgbe_phy_sfp_unsupported;
+				status = IXGBE_ERR_SFP_NOT_SUPPORTED;
+			}
+		} else {
+			status = IXGBE_SUCCESS;
+		}
 	}
 
 out:
@@ -748,7 +946,7 @@ s32 ixgbe_get_sfp_init_sequence_offsets(struct ixgbe_hw *hw,
 	hw->eeprom.ops.read(hw, IXGBE_PHY_INIT_OFFSET_NL, list_offset);
 
 	if ((!*list_offset) || (*list_offset == 0xFFFF))
-		return IXGBE_ERR_PHY;
+		return IXGBE_ERR_SFP_NO_INIT_SEQ_PRESENT;
 
 	/* Shift offset to first ID word */
 	(*list_offset)++;
@@ -784,3 +982,549 @@ s32 ixgbe_get_sfp_init_sequence_offsets(struct ixgbe_hw *hw,
 	return IXGBE_SUCCESS;
 }
 
+/**
+ *  ixgbe_read_i2c_eeprom_generic - Reads 8 bit EEPROM word over I2C interface
+ *  @hw: pointer to hardware structure
+ *  @byte_offset: EEPROM byte offset to read
+ *  @eeprom_data: value read
+ *
+ *  Performs byte read operation to SFP module's EEPROM over I2C interface.
+ **/
+s32 ixgbe_read_i2c_eeprom_generic(struct ixgbe_hw *hw, u8 byte_offset,
+                                  u8 *eeprom_data)
+{
+	DEBUGFUNC("ixgbe_read_i2c_eeprom_generic");
+
+	return hw->phy.ops.read_i2c_byte(hw, byte_offset,
+	                                 IXGBE_I2C_EEPROM_DEV_ADDR,
+	                                 eeprom_data);
+}
+
+/**
+ *  ixgbe_write_i2c_eeprom_generic - Writes 8 bit EEPROM word over I2C interface
+ *  @hw: pointer to hardware structure
+ *  @byte_offset: EEPROM byte offset to write
+ *  @eeprom_data: value to write
+ *
+ *  Performs byte write operation to SFP module's EEPROM over I2C interface.
+ **/
+s32 ixgbe_write_i2c_eeprom_generic(struct ixgbe_hw *hw, u8 byte_offset,
+                                   u8 eeprom_data)
+{
+	DEBUGFUNC("ixgbe_write_i2c_eeprom_generic");
+
+	return hw->phy.ops.write_i2c_byte(hw, byte_offset,
+	                                  IXGBE_I2C_EEPROM_DEV_ADDR,
+	                                  eeprom_data);
+}
+
+/**
+ *  ixgbe_read_i2c_byte_generic - Reads 8 bit word over I2C
+ *  @hw: pointer to hardware structure
+ *  @byte_offset: byte offset to read
+ *  @data: value read
+ *
+ *  Performs byte read operation to SFP module's EEPROM over I2C interface at
+ *  a specified deivce address.
+ **/
+s32 ixgbe_read_i2c_byte_generic(struct ixgbe_hw *hw, u8 byte_offset,
+                                u8 dev_addr, u8 *data)
+{
+	s32 status = IXGBE_SUCCESS;
+	u32 max_retry = 1;
+	u32 retry = 0;
+	u16 swfw_mask = 0;
+	bool nack = 1;
+
+	DEBUGFUNC("ixgbe_read_i2c_byte_generic");
+
+	if (IXGBE_READ_REG(hw, IXGBE_STATUS) & IXGBE_STATUS_LAN_ID_1)
+		swfw_mask = IXGBE_GSSR_PHY1_SM;
+	else
+		swfw_mask = IXGBE_GSSR_PHY0_SM;
+
+	if (ixgbe_acquire_swfw_sync(hw, swfw_mask) != IXGBE_SUCCESS) {
+		status = IXGBE_ERR_SWFW_SYNC;
+		goto read_byte_out;
+	}
+
+	do {
+		ixgbe_i2c_start(hw);
+
+		/* Device Address and write indication */
+		status = ixgbe_clock_out_i2c_byte(hw, dev_addr);
+		if (status != IXGBE_SUCCESS)
+			goto fail;
+
+		status = ixgbe_get_i2c_ack(hw);
+		if (status != IXGBE_SUCCESS)
+			goto fail;
+
+		status = ixgbe_clock_out_i2c_byte(hw, byte_offset);
+		if (status != IXGBE_SUCCESS)
+			goto fail;
+
+		status = ixgbe_get_i2c_ack(hw);
+		if (status != IXGBE_SUCCESS)
+			goto fail;
+
+		ixgbe_i2c_start(hw);
+
+		/* Device Address and read indication */
+		status = ixgbe_clock_out_i2c_byte(hw, (dev_addr | 0x1));
+		if (status != IXGBE_SUCCESS)
+			goto fail;
+
+		status = ixgbe_get_i2c_ack(hw);
+		if (status != IXGBE_SUCCESS)
+			goto fail;
+
+		status = ixgbe_clock_in_i2c_byte(hw, data);
+		if (status != IXGBE_SUCCESS)
+			goto fail;
+
+		status = ixgbe_clock_out_i2c_bit(hw, nack);
+		if (status != IXGBE_SUCCESS)
+			goto fail;
+
+		ixgbe_i2c_stop(hw);
+		break;
+
+fail:
+		ixgbe_i2c_bus_clear(hw);
+		retry++;
+		if (retry < max_retry)
+			DEBUGOUT("I2C byte read error - Retrying.\n");
+		else
+			DEBUGOUT("I2C byte read error.\n");
+
+	} while (retry < max_retry);
+
+	ixgbe_release_swfw_sync(hw, swfw_mask);
+
+read_byte_out:
+	return status;
+}
+
+/**
+ *  ixgbe_write_i2c_byte_generic - Writes 8 bit word over I2C
+ *  @hw: pointer to hardware structure
+ *  @byte_offset: byte offset to write
+ *  @data: value to write
+ *
+ *  Performs byte write operation to SFP module's EEPROM over I2C interface at
+ *  a specified device address.
+ **/
+s32 ixgbe_write_i2c_byte_generic(struct ixgbe_hw *hw, u8 byte_offset,
+                                 u8 dev_addr, u8 data)
+{
+	s32 status = IXGBE_SUCCESS;
+	u32 max_retry = 1;
+	u32 retry = 0;
+	u16 swfw_mask = 0;
+
+	DEBUGFUNC("ixgbe_write_i2c_byte_generic");
+
+	if (IXGBE_READ_REG(hw, IXGBE_STATUS) & IXGBE_STATUS_LAN_ID_1)
+		swfw_mask = IXGBE_GSSR_PHY1_SM;
+	else
+		swfw_mask = IXGBE_GSSR_PHY0_SM;
+
+	if (ixgbe_acquire_swfw_sync(hw, swfw_mask) != IXGBE_SUCCESS) {
+		status = IXGBE_ERR_SWFW_SYNC;
+		goto write_byte_out;
+	}
+
+	do {
+		ixgbe_i2c_start(hw);
+
+		status = ixgbe_clock_out_i2c_byte(hw, dev_addr);
+		if (status != IXGBE_SUCCESS)
+			goto fail;
+
+		status = ixgbe_get_i2c_ack(hw);
+		if (status != IXGBE_SUCCESS)
+			goto fail;
+
+		status = ixgbe_clock_out_i2c_byte(hw, byte_offset);
+		if (status != IXGBE_SUCCESS)
+			goto fail;
+
+		status = ixgbe_get_i2c_ack(hw);
+		if (status != IXGBE_SUCCESS)
+			goto fail;
+
+		status = ixgbe_clock_out_i2c_byte(hw, data);
+		if (status != IXGBE_SUCCESS)
+			goto fail;
+
+		status = ixgbe_get_i2c_ack(hw);
+		if (status != IXGBE_SUCCESS)
+			goto fail;
+
+		ixgbe_i2c_stop(hw);
+		break;
+
+fail:
+		ixgbe_i2c_bus_clear(hw);
+		retry++;
+		if (retry < max_retry)
+			DEBUGOUT("I2C byte write error - Retrying.\n");
+		else
+			DEBUGOUT("I2C byte write error.\n");
+	} while (retry < max_retry);
+
+	ixgbe_release_swfw_sync(hw, swfw_mask);
+
+write_byte_out:
+	return status;
+}
+
+/**
+ *  ixgbe_i2c_start - Sets I2C start condition
+ *  @hw: pointer to hardware structure
+ *
+ *  Sets I2C start condition (High -> Low on SDA while SCL is High)
+ **/
+static void ixgbe_i2c_start(struct ixgbe_hw *hw)
+{
+	u32 i2cctl = IXGBE_READ_REG(hw, IXGBE_I2CCTL);
+
+	DEBUGFUNC("ixgbe_i2c_start");
+
+	/* Start condition must begin with data and clock high */
+	ixgbe_set_i2c_data(hw, &i2cctl, 1);
+	ixgbe_raise_i2c_clk(hw, &i2cctl);
+
+	/* Setup time for start condition (4.7us) */
+	usec_delay(IXGBE_I2C_T_SU_STA);
+
+	ixgbe_set_i2c_data(hw, &i2cctl, 0);
+
+	/* Hold time for start condition (4us) */
+	usec_delay(IXGBE_I2C_T_HD_STA);
+
+	ixgbe_lower_i2c_clk(hw, &i2cctl);
+
+	/* Minimum low period of clock is 4.7 us */
+	usec_delay(IXGBE_I2C_T_LOW);
+
+}
+
+/**
+ *  ixgbe_i2c_stop - Sets I2C stop condition
+ *  @hw: pointer to hardware structure
+ *
+ *  Sets I2C stop condition (Low -> High on SDA while SCL is High)
+ **/
+static void ixgbe_i2c_stop(struct ixgbe_hw *hw)
+{
+	u32 i2cctl = IXGBE_READ_REG(hw, IXGBE_I2CCTL);
+
+	DEBUGFUNC("ixgbe_i2c_stop");
+
+	/* Stop condition must begin with data low and clock high */
+	ixgbe_set_i2c_data(hw, &i2cctl, 0);
+	ixgbe_raise_i2c_clk(hw, &i2cctl);
+
+	/* Setup time for stop condition (4us) */
+	usec_delay(IXGBE_I2C_T_SU_STO);
+
+	ixgbe_set_i2c_data(hw, &i2cctl, 1);
+
+	/* bus free time between stop and start (4.7us)*/
+	usec_delay(IXGBE_I2C_T_BUF);
+}
+
+/**
+ *  ixgbe_clock_in_i2c_byte - Clocks in one byte via I2C
+ *  @hw: pointer to hardware structure
+ *  @data: data byte to clock in
+ *
+ *  Clocks in one byte data via I2C data/clock
+ **/
+static s32 ixgbe_clock_in_i2c_byte(struct ixgbe_hw *hw, u8 *data)
+{
+	s32 status = IXGBE_SUCCESS;
+	s32 i;
+	bool bit = 0;
+
+	DEBUGFUNC("ixgbe_clock_in_i2c_byte");
+
+	for (i = 7; i >= 0; i--) {
+		status = ixgbe_clock_in_i2c_bit(hw, &bit);
+		*data |= bit<<i;
+
+		if (status != IXGBE_SUCCESS)
+			break;
+	}
+
+	return status;
+}
+
+/**
+ *  ixgbe_clock_out_i2c_byte - Clocks out one byte via I2C
+ *  @hw: pointer to hardware structure
+ *  @data: data byte clocked out
+ *
+ *  Clocks out one byte data via I2C data/clock
+ **/
+static s32 ixgbe_clock_out_i2c_byte(struct ixgbe_hw *hw, u8 data)
+{
+	s32 status = IXGBE_SUCCESS;
+	s32 i;
+	u32 i2cctl;
+	bool bit = 0;
+
+	DEBUGFUNC("ixgbe_clock_out_i2c_byte");
+
+	for (i = 7; i >= 0; i--) {
+		bit = (data >> i) & 0x1;
+		status = ixgbe_clock_out_i2c_bit(hw, bit);
+
+		if (status != IXGBE_SUCCESS)
+			break;
+	}
+
+	/* Release SDA line (set high) */
+	i2cctl = IXGBE_READ_REG(hw, IXGBE_I2CCTL);
+	i2cctl |= IXGBE_I2C_DATA_OUT;
+	IXGBE_WRITE_REG(hw, IXGBE_I2CCTL, i2cctl);
+
+	return status;
+}
+
+/**
+ *  ixgbe_get_i2c_ack - Polls for I2C ACK
+ *  @hw: pointer to hardware structure
+ *
+ *  Clocks in/out one bit via I2C data/clock
+ **/
+static s32 ixgbe_get_i2c_ack(struct ixgbe_hw *hw)
+{
+	s32 status;
+	u32 i = 0;
+	u32 i2cctl = IXGBE_READ_REG(hw, IXGBE_I2CCTL);
+	u32 timeout = 10;
+	bool ack = 1;
+
+	DEBUGFUNC("ixgbe_get_i2c_ack");
+
+	status = ixgbe_raise_i2c_clk(hw, &i2cctl);
+
+	if (status != IXGBE_SUCCESS)
+		goto out;
+
+	/* Minimum high period of clock is 4us */
+	usec_delay(IXGBE_I2C_T_HIGH);
+
+	/* Poll for ACK.  Note that ACK in I2C spec is
+	 * transition from 1 to 0 */
+	for (i = 0; i < timeout; i++) {
+		i2cctl = IXGBE_READ_REG(hw, IXGBE_I2CCTL);
+		ack = ixgbe_get_i2c_data(&i2cctl);
+
+		usec_delay(1);
+		if (ack == 0)
+			break;
+	}
+
+	if (ack == 1) {
+		DEBUGOUT("I2C ack was not received.\n");
+		status = IXGBE_ERR_I2C;
+	}
+
+	ixgbe_lower_i2c_clk(hw, &i2cctl);
+
+	/* Minimum low period of clock is 4.7 us */
+	usec_delay(IXGBE_I2C_T_LOW);
+
+out:
+	return status;
+}
+
+/**
+ *  ixgbe_clock_in_i2c_bit - Clocks in one bit via I2C data/clock
+ *  @hw: pointer to hardware structure
+ *  @data: read data value
+ *
+ *  Clocks in one bit via I2C data/clock
+ **/
+static s32 ixgbe_clock_in_i2c_bit(struct ixgbe_hw *hw, bool *data)
+{
+	s32 status;
+	u32 i2cctl = IXGBE_READ_REG(hw, IXGBE_I2CCTL);
+
+	status = ixgbe_raise_i2c_clk(hw, &i2cctl);
+
+	/* Minimum high period of clock is 4us */
+	usec_delay(IXGBE_I2C_T_HIGH);
+
+	i2cctl = IXGBE_READ_REG(hw, IXGBE_I2CCTL);
+	*data = ixgbe_get_i2c_data(&i2cctl);
+
+	ixgbe_lower_i2c_clk(hw, &i2cctl);
+
+	/* Minimum low period of clock is 4.7 us */
+	usec_delay(IXGBE_I2C_T_LOW);
+
+	return status;
+}
+
+/**
+ *  ixgbe_clock_out_i2c_bit - Clocks in/out one bit via I2C data/clock
+ *  @hw: pointer to hardware structure
+ *  @data: data value to write
+ *
+ *  Clocks out one bit via I2C data/clock
+ **/
+static s32 ixgbe_clock_out_i2c_bit(struct ixgbe_hw *hw, bool data)
+{
+	s32 status;
+	u32 i2cctl = IXGBE_READ_REG(hw, IXGBE_I2CCTL);
+
+	status = ixgbe_set_i2c_data(hw, &i2cctl, data);
+	if (status == IXGBE_SUCCESS) {
+		status = ixgbe_raise_i2c_clk(hw, &i2cctl);
+
+		/* Minimum high period of clock is 4us */
+		usec_delay(IXGBE_I2C_T_HIGH);
+
+		ixgbe_lower_i2c_clk(hw, &i2cctl);
+
+		/* Minimum low period of clock is 4.7 us.
+		 * This also takes care of the data hold time.
+		 */
+		usec_delay(IXGBE_I2C_T_LOW);
+	} else {
+		status = IXGBE_ERR_I2C;
+		DEBUGOUT1("I2C data was not set to %X\n", data);
+	}
+
+	return status;
+}
+/**
+ *  ixgbe_raise_i2c_clk - Raises the I2C SCL clock
+ *  @hw: pointer to hardware structure
+ *  @i2cctl: Current value of I2CCTL register
+ *
+ *  Raises the I2C clock line '0'->'1'
+ **/
+static s32 ixgbe_raise_i2c_clk(struct ixgbe_hw *hw, u32 *i2cctl)
+{
+	s32 status = IXGBE_SUCCESS;
+
+	*i2cctl |= IXGBE_I2C_CLK_OUT;
+
+	IXGBE_WRITE_REG(hw, IXGBE_I2CCTL, *i2cctl);
+
+	/* SCL rise time (1000ns) */
+	usec_delay(IXGBE_I2C_T_RISE);
+
+	return status;
+}
+
+/**
+ *  ixgbe_lower_i2c_clk - Lowers the I2C SCL clock
+ *  @hw: pointer to hardware structure
+ *  @i2cctl: Current value of I2CCTL register
+ *
+ *  Lowers the I2C clock line '1'->'0'
+ **/
+static void ixgbe_lower_i2c_clk(struct ixgbe_hw *hw, u32 *i2cctl)
+{
+
+	*i2cctl &= ~IXGBE_I2C_CLK_OUT;
+
+	IXGBE_WRITE_REG(hw, IXGBE_I2CCTL, *i2cctl);
+
+	/* SCL fall time (300ns) */
+	usec_delay(IXGBE_I2C_T_FALL);
+}
+
+/**
+ *  ixgbe_set_i2c_data - Sets the I2C data bit
+ *  @hw: pointer to hardware structure
+ *  @i2cctl: Current value of I2CCTL register
+ *  @data: I2C data value (0 or 1) to set
+ *
+ *  Sets the I2C data bit
+ **/
+static s32 ixgbe_set_i2c_data(struct ixgbe_hw *hw, u32 *i2cctl, bool data)
+{
+	s32 status = IXGBE_SUCCESS;
+
+	if (data)
+		*i2cctl |= IXGBE_I2C_DATA_OUT;
+	else
+		*i2cctl &= ~IXGBE_I2C_DATA_OUT;
+
+	IXGBE_WRITE_REG(hw, IXGBE_I2CCTL, *i2cctl);
+
+	/* Data rise/fall (1000ns/300ns) and set-up time (250ns) */
+	usec_delay(IXGBE_I2C_T_RISE + IXGBE_I2C_T_FALL + IXGBE_I2C_T_SU_DATA);
+
+	/* Verify data was set correctly */
+	*i2cctl = IXGBE_READ_REG(hw, IXGBE_I2CCTL);
+	if (data != ixgbe_get_i2c_data(i2cctl)) {
+		status = IXGBE_ERR_I2C;
+		DEBUGOUT1("Error - I2C data was not set to %X.\n", data);
+	}
+
+	return status;
+}
+
+/**
+ *  ixgbe_get_i2c_data - Reads the I2C SDA data bit
+ *  @hw: pointer to hardware structure
+ *  @i2cctl: Current value of I2CCTL register
+ *
+ *  Returns the I2C data bit value
+ **/
+static bool ixgbe_get_i2c_data(u32 *i2cctl)
+{
+	bool data;
+
+	if (*i2cctl & IXGBE_I2C_DATA_IN)
+		data = 1;
+	else
+		data = 0;
+
+	return data;
+}
+
+/**
+ *  ixgbe_i2c_bus_clear - Clears the I2C bus
+ *  @hw: pointer to hardware structure
+ *
+ *  Clears the I2C bus by sending nine clock pulses.
+ *  Used when data line is stuck low.
+ **/
+void ixgbe_i2c_bus_clear(struct ixgbe_hw *hw)
+{
+	u32 i2cctl = IXGBE_READ_REG(hw, IXGBE_I2CCTL);
+	u32 i;
+
+	DEBUGFUNC("ixgbe_i2c_bus_clear");
+
+	ixgbe_i2c_start(hw);
+
+	ixgbe_set_i2c_data(hw, &i2cctl, 1);
+
+	for (i = 0; i < 9; i++) {
+		ixgbe_raise_i2c_clk(hw, &i2cctl);
+
+		/* Min high period of clock is 4us */
+		usec_delay(IXGBE_I2C_T_HIGH);
+
+		ixgbe_lower_i2c_clk(hw, &i2cctl);
+
+		/* Min low period of clock is 4.7us*/
+		usec_delay(IXGBE_I2C_T_LOW);
+	}
+
+	ixgbe_i2c_start(hw);
+
+	/* Put the i2c bus back to default state */
+	ixgbe_i2c_stop(hw);
+}
diff --git a/sys/dev/ixgbe/ixgbe_phy.h b/sys/dev/ixgbe/ixgbe_phy.h
index 7a1e4c7..047d31d 100644
--- a/sys/dev/ixgbe/ixgbe_phy.h
+++ b/sys/dev/ixgbe/ixgbe_phy.h
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -51,6 +51,7 @@
 /* Bitmasks */
 #define IXGBE_SFF_TWIN_AX_CAPABLE            0x80
 #define IXGBE_SFF_1GBASESX_CAPABLE           0x1
+#define IXGBE_SFF_1GBASELX_CAPABLE           0x2
 #define IXGBE_SFF_10GBASESR_CAPABLE          0x10
 #define IXGBE_SFF_10GBASELR_CAPABLE          0x20
 #define IXGBE_I2C_EEPROM_READ_MASK           0x100
@@ -61,14 +62,15 @@
 #define IXGBE_I2C_EEPROM_STATUS_IN_PROGRESS  0x3
 
 /* Bit-shift macros */
-#define IXGBE_SFF_VENDOR_OUI_BYTE0_SHIFT    12
-#define IXGBE_SFF_VENDOR_OUI_BYTE1_SHIFT    8
-#define IXGBE_SFF_VENDOR_OUI_BYTE2_SHIFT    4
+#define IXGBE_SFF_VENDOR_OUI_BYTE0_SHIFT    24
+#define IXGBE_SFF_VENDOR_OUI_BYTE1_SHIFT    16
+#define IXGBE_SFF_VENDOR_OUI_BYTE2_SHIFT    8
 
 /* Vendor OUIs: format of OUI is 0x[byte0][byte1][byte2][00] */
 #define IXGBE_SFF_VENDOR_OUI_TYCO     0x00407600
 #define IXGBE_SFF_VENDOR_OUI_FTL      0x00906500
 #define IXGBE_SFF_VENDOR_OUI_AVAGO    0x00176A00
+#define IXGBE_SFF_VENDOR_OUI_INTEL    0x001B2100
 
 /* I2C SDA and SCL timing parameters for standard mode */
 #define IXGBE_I2C_T_HD_STA  4
@@ -98,6 +100,9 @@ s32 ixgbe_setup_phy_link_speed_generic(struct ixgbe_hw *hw,
                                        ixgbe_link_speed speed,
                                        bool autoneg,
                                        bool autoneg_wait_to_complete);
+s32 ixgbe_get_copper_link_capabilities_generic(struct ixgbe_hw *hw,
+                                             ixgbe_link_speed *speed,
+                                             bool *autoneg);
 
 /* PHY specific */
 s32 ixgbe_check_phy_link_tnx(struct ixgbe_hw *hw,
@@ -105,10 +110,20 @@ s32 ixgbe_check_phy_link_tnx(struct ixgbe_hw *hw,
                              bool *link_up);
 s32 ixgbe_get_phy_firmware_version_tnx(struct ixgbe_hw *hw,
                                        u16 *firmware_version);
+s32 ixgbe_get_phy_firmware_version_aq(struct ixgbe_hw *hw,
+                                       u16 *firmware_version);
 
 s32 ixgbe_reset_phy_nl(struct ixgbe_hw *hw);
 s32 ixgbe_identify_sfp_module_generic(struct ixgbe_hw *hw);
 s32 ixgbe_get_sfp_init_sequence_offsets(struct ixgbe_hw *hw,
                                         u16 *list_offset,
                                         u16 *data_offset);
+s32 ixgbe_read_i2c_byte_generic(struct ixgbe_hw *hw, u8 byte_offset,
+                                u8 dev_addr, u8 *data);
+s32 ixgbe_write_i2c_byte_generic(struct ixgbe_hw *hw, u8 byte_offset,
+                                 u8 dev_addr, u8 data);
+s32 ixgbe_read_i2c_eeprom_generic(struct ixgbe_hw *hw, u8 byte_offset,
+                                  u8 *eeprom_data);
+s32 ixgbe_write_i2c_eeprom_generic(struct ixgbe_hw *hw, u8 byte_offset,
+                                   u8 eeprom_data);
 #endif /* _IXGBE_PHY_H_ */
diff --git a/sys/dev/ixgbe/ixgbe_type.h b/sys/dev/ixgbe/ixgbe_type.h
index f265768..595a863 100644
--- a/sys/dev/ixgbe/ixgbe_type.h
+++ b/sys/dev/ixgbe/ixgbe_type.h
@@ -1,6 +1,6 @@
 /******************************************************************************
 
-  Copyright (c) 2001-2008, Intel Corporation 
+  Copyright (c) 2001-2009, Intel Corporation 
   All rights reserved.
   
   Redistribution and use in source and binary forms, with or without 
@@ -42,6 +42,7 @@
 
 /* Device IDs */
 #define IXGBE_DEV_ID_82598               0x10B6
+#define IXGBE_DEV_ID_82598_BX            0x1508
 #define IXGBE_DEV_ID_82598AF_DUAL_PORT   0x10C6
 #define IXGBE_DEV_ID_82598AF_SINGLE_PORT 0x10C7
 #define IXGBE_DEV_ID_82598AT             0x10C8
@@ -51,6 +52,9 @@
 #define IXGBE_DEV_ID_82598_DA_DUAL_PORT  0x10F1
 #define IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM      0x10E1
 #define IXGBE_DEV_ID_82598EB_XF_LR       0x10F4
+#define IXGBE_DEV_ID_82599_KX4     0x10F7
+#define IXGBE_DEV_ID_82599_CX4 0x10F9
+#define IXGBE_DEV_ID_82599_SFP 0x10FB
 
 /* General Registers */
 #define IXGBE_CTRL      0x00000
@@ -58,9 +62,12 @@
 #define IXGBE_CTRL_EXT  0x00018
 #define IXGBE_ESDP      0x00020
 #define IXGBE_EODSDP    0x00028
+#define IXGBE_I2CCTL    0x00028
 #define IXGBE_LEDCTL    0x00200
 #define IXGBE_FRTIMER   0x00048
 #define IXGBE_TCPTIMER  0x0004C
+#define IXGBE_CORESPARE 0x00600
+#define IXGBE_EXVET     0x05078
 
 /* NVM Registers */
 #define IXGBE_EEC       0x10010
@@ -74,6 +81,19 @@
 #define IXGBE_FLOP      0x1013C
 #define IXGBE_GRC       0x10200
 
+/* General Receive Control */
+#define IXGBE_GRC_MNG  0x00000001 /* Manageability Enable */
+#define IXGBE_GRC_APME 0x00000002 /* Advanced Power Management Enable */
+
+#define IXGBE_VPDDIAG0  0x10204
+#define IXGBE_VPDDIAG1  0x10208
+
+/* I2CCTL Bit Masks */
+#define IXGBE_I2C_CLK_IN    0x00000001
+#define IXGBE_I2C_CLK_OUT   0x00000002
+#define IXGBE_I2C_DATA_IN   0x00000004
+#define IXGBE_I2C_DATA_OUT  0x00000008
+
 /* Interrupt Registers */
 #define IXGBE_EICR      0x00800
 #define IXGBE_EICS      0x00808
@@ -81,21 +101,45 @@
 #define IXGBE_EIMC      0x00888
 #define IXGBE_EIAC      0x00810
 #define IXGBE_EIAM      0x00890
+#define IXGBE_EICS_EX(_i)   (0x00A90 + (_i) * 4)
+#define IXGBE_EIMS_EX(_i)   (0x00AA0 + (_i) * 4)
+#define IXGBE_EIMC_EX(_i)   (0x00AB0 + (_i) * 4)
+#define IXGBE_EIAM_EX(_i)   (0x00AD0 + (_i) * 4)
+/* 82599 EITR is only 12 bits, with the lower 3 always zero */
+/*
+ * 82598 EITR is 16 bits but set the limits based on the max
+ * supported by all ixgbe hardware
+ */
+#define IXGBE_MAX_INT_RATE 488281
+#define IXGBE_MIN_INT_RATE 956
+#define IXGBE_MAX_EITR     0x00000FF8
+#define IXGBE_MIN_EITR     8
 #define IXGBE_EITR(_i)  (((_i) <= 23) ? (0x00820 + ((_i) * 4)) : \
-                         (0x012300 + ((_i) * 4)))
-#define IXGBE_EITR_ITR_INT_MASK 0x00000FFF
+                         (0x012300 + (((_i) - 24) * 4)))
+#define IXGBE_EITR_ITR_INT_MASK 0x00000FF8
+#define IXGBE_EITR_LLI_MOD      0x00008000
+#define IXGBE_EITR_CNT_WDIS     0x80000000
 #define IXGBE_IVAR(_i)  (0x00900 + ((_i) * 4)) /* 24 at 0x900-0x960 */
+#define IXGBE_IVAR_MISC 0x00A00 /* misc MSI-X interrupt causes */
+#define IXGBE_EITRSEL   0x00894
 #define IXGBE_MSIXT     0x00000 /* MSI-X Table. 0x0000 - 0x01C */
 #define IXGBE_MSIXPBA   0x02000 /* MSI-X Pending bit array */
 #define IXGBE_PBACL(_i) (((_i) == 0) ? (0x11068) : (0x110C0 + ((_i) * 4)))
 #define IXGBE_GPIE      0x00898
 
 /* Flow Control Registers */
+#define IXGBE_FCADBUL   0x03210
+#define IXGBE_FCADBUH   0x03214
+#define IXGBE_FCAMACL   0x04328
+#define IXGBE_FCAMACH   0x0432C
+#define IXGBE_FCRTH_82599(_i) (0x03260 + ((_i) * 4)) /* 8 of these (0-7) */
+#define IXGBE_FCRTL_82599(_i) (0x03220 + ((_i) * 4)) /* 8 of these (0-7) */
 #define IXGBE_PFCTOP    0x03008
 #define IXGBE_FCTTV(_i) (0x03200 + ((_i) * 4)) /* 4 of these (0-3) */
 #define IXGBE_FCRTL(_i) (0x03220 + ((_i) * 8)) /* 8 of these (0-7) */
 #define IXGBE_FCRTH(_i) (0x03260 + ((_i) * 8)) /* 8 of these (0-7) */
 #define IXGBE_FCRTV     0x032A0
+#define IXGBE_FCCFG     0x03D00
 #define IXGBE_TFCS      0x0CE00
 
 /* Receive DMA Registers */
@@ -111,6 +155,12 @@
                          (0x0D018 + ((_i - 64) * 0x40)))
 #define IXGBE_RXDCTL(_i) (((_i) < 64) ? (0x01028 + ((_i) * 0x40)) : \
                           (0x0D028 + ((_i - 64) * 0x40)))
+#define IXGBE_RSCCTL(_i) (((_i) < 64) ? (0x0102C + ((_i) * 0x40)) : \
+                          (0x0D02C + ((_i - 64) * 0x40)))
+#define IXGBE_RSCDBU     0x03028
+#define IXGBE_RDDCC      0x02F20
+#define IXGBE_RXMEMWRAP  0x03190
+#define IXGBE_STARCTRL   0x03024
 /*
  * Split and Replication Receive Control Registers
  * 00-15 : 0x02100 + n*4
@@ -130,6 +180,7 @@
                                  (((_i) < 64) ? (0x0100C + ((_i) * 0x40)) : \
                                  (0x0D00C + ((_i - 64) * 0x40))))
 #define IXGBE_RDRXCTL           0x02F00
+#define IXGBE_RDRXCTL_RSC_PUSH  0x80
 #define IXGBE_RXPBSIZE(_i)      (0x03C00 + ((_i) * 4))
                                              /* 8 of these 0x03C00 - 0x03C1C */
 #define IXGBE_RXCTRL    0x03000
@@ -147,6 +198,8 @@
                          (0x0A200 + ((_i) * 8)))
 #define IXGBE_RAH(_i)   (((_i) <= 15) ? (0x05404 + ((_i) * 8)) : \
                          (0x0A204 + ((_i) * 8)))
+#define IXGBE_MPSAR_LO(_i) (0x0A600 + ((_i) * 8))
+#define IXGBE_MPSAR_HI(_i) (0x0A604 + ((_i) * 8))
 /* Packet split receive type */
 #define IXGBE_PSRTYPE(_i)    (((_i) <= 15) ? (0x05480 + ((_i) * 4)) : \
                               (0x0EA00 + ((_i) * 4)))
@@ -158,6 +211,28 @@
 #define IXGBE_VLNCTRL   0x05088
 #define IXGBE_MCSTCTRL  0x05090
 #define IXGBE_MRQC      0x05818
+#define IXGBE_SAQF(_i)  (0x0E000 + ((_i) * 4)) /* Source Address Queue Filter */
+#define IXGBE_DAQF(_i)  (0x0E200 + ((_i) * 4)) /* Dest. Address Queue Filter */
+#define IXGBE_SDPQF(_i) (0x0E400 + ((_i) * 4)) /* Src Dest. Addr Queue Filter */
+#define IXGBE_FTQF(_i)  (0x0E600 + ((_i) * 4)) /* Five Tuple Queue Filter */
+#define IXGBE_ETQF(_i)  (0x05128 + ((_i) * 4)) /* EType Queue Filter */
+#define IXGBE_ETQS(_i)  (0x0EC00 + ((_i) * 4)) /* EType Queue Select */
+#define IXGBE_SYNQF     0x0EC30 /* SYN Packet Queue Filter */
+#define IXGBE_RQTC      0x0EC70
+#define IXGBE_MTQC      0x08120
+#define IXGBE_VLVF(_i)  (0x0F100 + ((_i) * 4))  /* 64 of these (0-63) */
+#define IXGBE_VLVFB(_i) (0x0F200 + ((_i) * 4))  /* 128 of these (0-127) */
+#define IXGBE_VT_CTL    0x051B0
+#define IXGBE_VFRE(_i)  (0x051E0 + ((_i) * 4))
+#define IXGBE_VFTE(_i)  (0x08110 + ((_i) * 4))
+#define IXGBE_QDE       0x2F04
+#define IXGBE_VMOLR(_i) (0x0F000 + ((_i) * 4)) /* 64 total */
+#define IXGBE_UTA(_i)   (0x0F400 + ((_i) * 4))
+#define IXGBE_VMRCTL(_i)        (0x0F600 + ((_i) * 4))
+#define IXGBE_VMRVLAN(_i)       (0x0F610 + ((_i) * 4))
+#define IXGBE_VMRVM(_i)         (0x0F630 + ((_i) * 4))
+#define IXGBE_L34T_IMIR(_i)      (0x0E800 + ((_i) * 4)) /*128 of these (0-127)*/
+#define IXGBE_LLITHRESH 0x0EC90
 #define IXGBE_IMIR(_i)  (0x05A80 + ((_i) * 4))  /* 8 of these (0-7) */
 #define IXGBE_IMIREXT(_i)       (0x05AA0 + ((_i) * 4))  /* 8 of these (0-7) */
 #define IXGBE_IMIRVP    0x05AC0
@@ -165,6 +240,33 @@
 #define IXGBE_RETA(_i)  (0x05C00 + ((_i) * 4))  /* 32 of these (0-31) */
 #define IXGBE_RSSRK(_i) (0x05C80 + ((_i) * 4))  /* 10 of these (0-9) */
 
+/* Flow Director registers */
+#define IXGBE_FDIRCTRL  0x0EE00
+#define IXGBE_FDIRHKEY  0x0EE68
+#define IXGBE_FDIRSKEY  0x0EE6C
+#define IXGBE_FDIRDIP4M 0x0EE3C
+#define IXGBE_FDIRSIP4M 0x0EE40
+#define IXGBE_FDIRTCPM  0x0EE44
+#define IXGBE_FDIRUDPM  0x0EE48
+#define IXGBE_FDIRIP6M  0x0EE74
+#define IXGBE_FDIRM     0x0EE70
+
+/* Flow Director Stats registers */
+#define IXGBE_FDIRFREE  0x0EE38
+#define IXGBE_FDIRLEN   0x0EE4C
+#define IXGBE_FDIRUSTAT 0x0EE50
+#define IXGBE_FDIRFSTAT 0x0EE54
+#define IXGBE_FDIRMATCH 0x0EE58
+#define IXGBE_FDIRMISS  0x0EE5C
+
+/* Flow Director Programming registers */
+#define IXGBE_FDIRSIPv6(_i) (0x0EE0C + ((_i) * 4)) /* 3 of these (0-2) */
+#define IXGBE_FDIRIPSA      0x0EE18
+#define IXGBE_FDIRIPDA      0x0EE1C
+#define IXGBE_FDIRPORT      0x0EE20
+#define IXGBE_FDIRVLAN      0x0EE24
+#define IXGBE_FDIRHASH      0x0EE28
+#define IXGBE_FDIRCMD       0x0EE2C
 
 /* Transmit DMA registers */
 #define IXGBE_TDBAL(_i) (0x06000 + ((_i) * 0x40)) /* 32 of these (0-31)*/
@@ -177,7 +279,20 @@
 #define IXGBE_TDWBAH(_i) (0x0603C + ((_i) * 0x40))
 #define IXGBE_DTXCTL    0x07E00
 
+#define IXGBE_DMATXCTL  0x04A80
+#define IXGBE_DTXMXSZRQ     0x08100
+#define IXGBE_DTXTCPFLGL    0x04A88
+#define IXGBE_DTXTCPFLGH    0x04A8C
+#define IXGBE_LBDRPEN       0x0CA00
+#define IXGBE_TXPBTHRESH(_i) (0x04950 + ((_i) * 4)) /* 8 of these 0 - 7 */
+
+#define IXGBE_DMATXCTL_TE       0x1 /* Transmit Enable */
+#define IXGBE_DMATXCTL_NS       0x2 /* No Snoop LSO hdr buffer */
+#define IXGBE_DMATXCTL_GDV      0x8 /* Global Double VLAN */
+#define IXGBE_DMATXCTL_VT_SHIFT 16  /* VLAN EtherType */
 #define IXGBE_DCA_TXCTRL(_i)    (0x07200 + ((_i) * 4)) /* 16 of these (0-15) */
+/* Tx DCA Control register : 128 of these (0-127) */
+#define IXGBE_DCA_TXCTRL_82599(_i)  (0x0600C + ((_i) * 0x40))
 #define IXGBE_TIPG      0x0CB00
 #define IXGBE_TXPBSIZE(_i)      (0x0CC00 + ((_i) * 4)) /* 8 of these */
 #define IXGBE_MNGTXMAP  0x0CD10
@@ -269,6 +384,193 @@
 #define IXGBE_TDPT2TCSR(_i)     (0x0CD40 + ((_i) * 4)) /* 8 of these (0-7) */
 
 
+/* Security Control Registers */
+#define IXGBE_SECTXCTRL         0x08800
+#define IXGBE_SECTXSTAT         0x08804
+#define IXGBE_SECTXBUFFAF       0x08808
+#define IXGBE_SECTXMINIFG       0x08810
+#define IXGBE_SECTXSTAT         0x08804
+#define IXGBE_SECRXCTRL         0x08D00
+#define IXGBE_SECRXSTAT         0x08D04
+
+/* Security Bit Fields and Masks */
+#define IXGBE_SECTXCTRL_SECTX_DIS       0x00000001
+#define IXGBE_SECTXCTRL_TX_DIS          0x00000002
+#define IXGBE_SECTXCTRL_STORE_FORWARD   0x00000004
+
+#define IXGBE_SECTXSTAT_SECTX_RDY       0x00000001
+#define IXGBE_SECTXSTAT_ECC_TXERR       0x00000002
+
+#define IXGBE_SECRXCTRL_SECRX_DIS       0x00000001
+#define IXGBE_SECRXCTRL_RX_DIS          0x00000002
+
+#define IXGBE_SECRXSTAT_SECRX_RDY       0x00000001
+#define IXGBE_SECRXSTAT_ECC_RXERR       0x00000002
+
+/* LinkSec (MacSec) Registers */
+#define IXGBE_LSECTXCAP         0x08A00
+#define IXGBE_LSECRXCAP         0x08F00
+#define IXGBE_LSECTXCTRL        0x08A04
+#define IXGBE_LSECTXSCL         0x08A08 /* SCI Low */
+#define IXGBE_LSECTXSCH         0x08A0C /* SCI High */
+#define IXGBE_LSECTXSA          0x08A10
+#define IXGBE_LSECTXPN0         0x08A14
+#define IXGBE_LSECTXPN1         0x08A18
+#define IXGBE_LSECTXKEY0(_n)    (0x08A1C + (4 * (_n))) /* 4 of these (0-3) */
+#define IXGBE_LSECTXKEY1(_n)    (0x08A2C + (4 * (_n))) /* 4 of these (0-3) */
+#define IXGBE_LSECRXCTRL        0x08F04
+#define IXGBE_LSECRXSCL         0x08F08
+#define IXGBE_LSECRXSCH         0x08F0C
+#define IXGBE_LSECRXSA(_i)      (0x08F10 + (4 * (_i))) /* 2 of these (0-1) */
+#define IXGBE_LSECRXPN(_i)      (0x08F18 + (4 * (_i))) /* 2 of these (0-1) */
+#define IXGBE_LSECRXKEY(_n, _m) (0x08F20 + ((0x10 * (_n)) + (4 * (_m))))
+#define IXGBE_LSECTXUT          0x08A3C /* OutPktsUntagged */
+#define IXGBE_LSECTXPKTE        0x08A40 /* OutPktsEncrypted */
+#define IXGBE_LSECTXPKTP        0x08A44 /* OutPktsProtected */
+#define IXGBE_LSECTXOCTE        0x08A48 /* OutOctetsEncrypted */
+#define IXGBE_LSECTXOCTP        0x08A4C /* OutOctetsProtected */
+#define IXGBE_LSECRXUT          0x08F40 /* InPktsUntagged/InPktsNoTag */
+#define IXGBE_LSECRXOCTD        0x08F44 /* InOctetsDecrypted */
+#define IXGBE_LSECRXOCTV        0x08F48 /* InOctetsValidated */
+#define IXGBE_LSECRXBAD         0x08F4C /* InPktsBadTag */
+#define IXGBE_LSECRXNOSCI       0x08F50 /* InPktsNoSci */
+#define IXGBE_LSECRXUNSCI       0x08F54 /* InPktsUnknownSci */
+#define IXGBE_LSECRXUNCH        0x08F58 /* InPktsUnchecked */
+#define IXGBE_LSECRXDELAY       0x08F5C /* InPktsDelayed */
+#define IXGBE_LSECRXLATE        0x08F60 /* InPktsLate */
+#define IXGBE_LSECRXOK(_n)      (0x08F64 + (0x04 * (_n))) /* InPktsOk */
+#define IXGBE_LSECRXINV(_n)     (0x08F6C + (0x04 * (_n))) /* InPktsInvalid */
+#define IXGBE_LSECRXNV(_n)      (0x08F74 + (0x04 * (_n))) /* InPktsNotValid */
+#define IXGBE_LSECRXUNSA        0x08F7C /* InPktsUnusedSa */
+#define IXGBE_LSECRXNUSA        0x08F80 /* InPktsNotUsingSa */
+
+/* LinkSec (MacSec) Bit Fields and Masks */
+#define IXGBE_LSECTXCAP_SUM_MASK        0x00FF0000
+#define IXGBE_LSECTXCAP_SUM_SHIFT       16
+#define IXGBE_LSECRXCAP_SUM_MASK        0x00FF0000
+#define IXGBE_LSECRXCAP_SUM_SHIFT       16
+
+#define IXGBE_LSECTXCTRL_EN_MASK        0x00000003
+#define IXGBE_LSECTXCTRL_DISABLE        0x0
+#define IXGBE_LSECTXCTRL_AUTH           0x1
+#define IXGBE_LSECTXCTRL_AUTH_ENCRYPT   0x2
+#define IXGBE_LSECTXCTRL_AISCI          0x00000020
+#define IXGBE_LSECTXCTRL_PNTHRSH_MASK   0xFFFFFF00
+#define IXGBE_LSECTXCTRL_RSV_MASK       0x000000D8
+
+#define IXGBE_LSECRXCTRL_EN_MASK        0x0000000C
+#define IXGBE_LSECRXCTRL_EN_SHIFT       2
+#define IXGBE_LSECRXCTRL_DISABLE        0x0
+#define IXGBE_LSECRXCTRL_CHECK          0x1
+#define IXGBE_LSECRXCTRL_STRICT         0x2
+#define IXGBE_LSECRXCTRL_DROP           0x3
+#define IXGBE_LSECRXCTRL_PLSH           0x00000040
+#define IXGBE_LSECRXCTRL_RP             0x00000080
+#define IXGBE_LSECRXCTRL_RSV_MASK       0xFFFFFF33
+
+/* IpSec Registers */
+#define IXGBE_IPSTXIDX          0x08900
+#define IXGBE_IPSTXSALT         0x08904
+#define IXGBE_IPSTXKEY(_i)      (0x08908 + (4 * (_i))) /* 4 of these (0-3) */
+#define IXGBE_IPSRXIDX          0x08E00
+#define IXGBE_IPSRXIPADDR(_i)   (0x08E04 + (4 * (_i))) /* 4 of these (0-3) */
+#define IXGBE_IPSRXSPI          0x08E14
+#define IXGBE_IPSRXIPIDX        0x08E18
+#define IXGBE_IPSRXKEY(_i)      (0x08E1C + (4 * (_i))) /* 4 of these (0-3) */
+#define IXGBE_IPSRXSALT         0x08E2C
+#define IXGBE_IPSRXMOD          0x08E30
+
+#define IXGBE_SECTXCTRL_STORE_FORWARD_ENABLE    0x4
+
+/* DCB registers */
+#define IXGBE_RTRPCS      0x02430
+#define IXGBE_RTTDCS      0x04900
+#define IXGBE_RTTDCS_ARBDIS     0x00000040 /* DCB arbiter disable */
+#define IXGBE_RTTPCS      0x0CD00
+#define IXGBE_RTRUP2TC    0x03020
+#define IXGBE_RTTUP2TC    0x0C800
+#define IXGBE_RTRPT4C(_i) (0x02140 + ((_i) * 4)) /* 8 of these (0-7) */
+#define IXGBE_RTRPT4S(_i) (0x02160 + ((_i) * 4)) /* 8 of these (0-7) */
+#define IXGBE_RTTDT2C(_i) (0x04910 + ((_i) * 4)) /* 8 of these (0-7) */
+#define IXGBE_RTTDT2S(_i) (0x04930 + ((_i) * 4)) /* 8 of these (0-7) */
+#define IXGBE_RTTPT2C(_i) (0x0CD20 + ((_i) * 4)) /* 8 of these (0-7) */
+#define IXGBE_RTTPT2S(_i) (0x0CD40 + ((_i) * 4)) /* 8 of these (0-7) */
+#define IXGBE_RTTDQSEL    0x04904
+#define IXGBE_RTTDT1C     0x04908
+#define IXGBE_RTTDT1S     0x0490C
+#define IXGBE_RTTDTECC    0x04990
+#define IXGBE_RTTDTECC_NO_BCN   0x00000100
+
+#define IXGBE_RTTBCNRC    0x04984
+
+/* BCN (for DCB) Registers */
+#define IXGBE_RTTBCNRM    0x04980
+#define IXGBE_RTTBCNRS    0x04988
+#define IXGBE_RTTBCNCR    0x08B00
+#define IXGBE_RTTBCNACH   0x08B04
+#define IXGBE_RTTBCNACL   0x08B08
+#define IXGBE_RTTBCNTG    0x04A90
+#define IXGBE_RTTBCNIDX   0x08B0C
+#define IXGBE_RTTBCNCP    0x08B10
+#define IXGBE_RTFRTIMER   0x08B14
+#define IXGBE_RTTBCNRTT   0x05150
+#define IXGBE_RTTBCNRD    0x0498C
+
+/* FCoE DMA Context Registers */
+#define IXGBE_FCPTRL    0x02410 /* FC User Desc. PTR Low */
+#define IXGBE_FCPTRH    0x02414 /* FC USer Desc. PTR High */
+#define IXGBE_FCBUFF    0x02418 /* FC Buffer Control */
+#define IXGBE_FCDMARW   0x02420 /* FC Receive DMA RW */
+#define IXGBE_FCINVST0  0x03FC0 /* FC Invalid DMA Context Status Reg 0 */
+#define IXGBE_FCINVST(_i)       (IXGBE_FCINVST0 + ((_i) * 4))
+#define IXGBE_FCBUFF_VALID      (1 << 0)   /* DMA Context Valid */
+#define IXGBE_FCBUFF_BUFFSIZE   (3 << 3)   /* User Buffer Size */
+#define IXGBE_FCBUFF_WRCONTX    (1 << 7)   /* 0: Initiator, 1: Target */
+#define IXGBE_FCBUFF_BUFFCNT    0x0000ff00 /* Number of User Buffers */
+#define IXGBE_FCBUFF_OFFSET     0xffff0000 /* User Buffer Offset */
+#define IXGBE_FCBUFF_BUFFSIZE_SHIFT  3
+#define IXGBE_FCBUFF_BUFFCNT_SHIFT   8
+#define IXGBE_FCBUFF_OFFSET_SHIFT    16
+#define IXGBE_FCDMARW_WE        (1 << 14)   /* Write enable */
+#define IXGBE_FCDMARW_RE        (1 << 15)   /* Read enable */
+#define IXGBE_FCDMARW_FCOESEL   0x000001ff  /* FC X_ID: 11 bits */
+#define IXGBE_FCDMARW_LASTSIZE  0xffff0000  /* Last User Buffer Size */
+#define IXGBE_FCDMARW_LASTSIZE_SHIFT 16
+/* FCoE SOF/EOF */
+#define IXGBE_TEOFF     0x04A94 /* Tx FC EOF */
+#define IXGBE_TSOFF     0x04A98 /* Tx FC SOF */
+#define IXGBE_REOFF     0x05158 /* Rx FC EOF */
+#define IXGBE_RSOFF     0x051F8 /* Rx FC SOF */
+/* FCoE Filter Context Registers */
+#define IXGBE_FCFLT     0x05108 /* FC FLT Context */
+#define IXGBE_FCFLTRW   0x05110 /* FC Filter RW Control */
+#define IXGBE_FCPARAM   0x051d8 /* FC Offset Parameter */
+#define IXGBE_FCFLT_VALID       (1 << 0)   /* Filter Context Valid */
+#define IXGBE_FCFLT_FIRST       (1 << 1)   /* Filter First */
+#define IXGBE_FCFLT_SEQID       0x00ff0000 /* Sequence ID */
+#define IXGBE_FCFLT_SEQCNT      0xff000000 /* Sequence Count */
+#define IXGBE_FCFLTRW_RVALDT    (1 << 13)  /* Fast Re-Validation */
+#define IXGBE_FCFLTRW_WE        (1 << 14)  /* Write Enable */
+#define IXGBE_FCFLTRW_RE        (1 << 15)  /* Read Enable */
+/* FCoE Receive Control */
+#define IXGBE_FCRXCTRL  0x05100 /* FC Receive Control */
+#define IXGBE_FCRXCTRL_FCOELLI  (1 << 0)   /* Low latency interrupt */
+#define IXGBE_FCRXCTRL_SAVBAD   (1 << 1)   /* Save Bad Frames */
+#define IXGBE_FCRXCTRL_FRSTRDH  (1 << 2)   /* EN 1st Read Header */
+#define IXGBE_FCRXCTRL_LASTSEQH (1 << 3)   /* EN Last Header in Seq */
+#define IXGBE_FCRXCTRL_ALLH     (1 << 4)   /* EN All Headers */
+#define IXGBE_FCRXCTRL_FRSTSEQH (1 << 5)   /* EN 1st Seq. Header */
+#define IXGBE_FCRXCTRL_ICRC     (1 << 6)   /* Ignore Bad FC CRC */
+#define IXGBE_FCRXCTRL_FCCRCBO  (1 << 7)   /* FC CRC Byte Ordering */
+#define IXGBE_FCRXCTRL_FCOEVER  0x00000f00 /* FCoE Version: 4 bits */
+#define IXGBE_FCRXCTRL_FCOEVER_SHIFT 8
+/* FCoE Redirection */
+#define IXGBE_FCRECTL   0x0ED00 /* FC Redirection Control */
+#define IXGBE_FCRETA0   0x0ED10 /* FC Redirection Table 0 */
+#define IXGBE_FCRETA(_i)        (IXGBE_FCRETA0 + ((_i) * 4)) /* FCoE Redir */
+#define IXGBE_FCRECTL_ENA       0x1        /* FCoE Redir Table Enable */
+#define IXGBE_FCRETA_SIZE       8          /* Max entries in FCRETA */
+#define IXGBE_FCRETA_ENTRY_MASK 0x0000007f /* 7 bits for the queue index */
 
 /* Stats registers */
 #define IXGBE_CRCERRS   0x04000
@@ -283,6 +585,11 @@
 #define IXGBE_LXONRXC   0x0CF60
 #define IXGBE_LXOFFTXC  0x03F68
 #define IXGBE_LXOFFRXC  0x0CF68
+#define IXGBE_LXONRXCNT 0x041A4
+#define IXGBE_LXOFFRXCNT 0x041A8
+#define IXGBE_PXONRXCNT(_i)     (0x04140 + ((_i) * 4)) /* 8 of these */
+#define IXGBE_PXOFFRXCNT(_i)    (0x04160 + ((_i) * 4)) /* 8 of these */
+#define IXGBE_PXON2OFFCNT(_i)   (0x03240 + ((_i) * 4)) /* 8 of these */
 #define IXGBE_PXONTXC(_i)       (0x03F00 + ((_i) * 4)) /* 8 of these 3F00-3F1C*/
 #define IXGBE_PXONRXC(_i)       (0x0CF00 + ((_i) * 4)) /* 8 of these CF00-CF1C*/
 #define IXGBE_PXOFFTXC(_i)      (0x03F20 + ((_i) * 4)) /* 8 of these 3F20-3F3C*/
@@ -322,15 +629,29 @@
 #define IXGBE_MPTC      0x040F0
 #define IXGBE_BPTC      0x040F4
 #define IXGBE_XEC       0x04120
+#define IXGBE_SSVPC     0x08780
 
 #define IXGBE_RQSMR(_i) (0x02300 + ((_i) * 4))
 #define IXGBE_TQSMR(_i) (((_i) <= 7) ? (0x07300 + ((_i) * 4)) : \
                          (0x08600 + ((_i) * 4)))
+#define IXGBE_TQSM(_i)  (0x08600 + ((_i) * 4))
 
 #define IXGBE_QPRC(_i) (0x01030 + ((_i) * 0x40)) /* 16 of these */
 #define IXGBE_QPTC(_i) (0x06030 + ((_i) * 0x40)) /* 16 of these */
 #define IXGBE_QBRC(_i) (0x01034 + ((_i) * 0x40)) /* 16 of these */
 #define IXGBE_QBTC(_i) (0x06034 + ((_i) * 0x40)) /* 16 of these */
+#define IXGBE_QPRDC(_i) (0x01430 + ((_i) * 0x40)) /* 16 of these */
+#define IXGBE_QBTC_L(_i) (0x08700 + ((_i) * 0x8)) /* 16 of these */
+#define IXGBE_QBTC_H(_i) (0x08704 + ((_i) * 0x8)) /* 16 of these */
+#define IXGBE_FCCRC     0x05118 /* Count of Good Eth CRC w/ Bad FC CRC */
+#define IXGBE_FCOERPDC  0x0241C /* FCoE Rx Packets Dropped Count */
+#define IXGBE_FCLAST    0x02424 /* FCoE Last Error Count */
+#define IXGBE_FCOEPRC   0x02428 /* Number of FCoE Packets Received */
+#define IXGBE_FCOEDWRC  0x0242C /* Number of FCoE DWords Received */
+#define IXGBE_FCOEPTC   0x08784 /* Number of FCoE Packets Transmitted */
+#define IXGBE_FCOEDWTC  0x08788 /* Number of FCoE DWords Transmitted */
+#define IXGBE_FCCRC_CNT_MASK    0x0000FFFF /* CRC_CNT: bit 0 - 15 */
+#define IXGBE_FCLAST_CNT_MASK   0x0000FFFF /* Last_CNT: bit 0 - 15 */
 
 /* Management */
 #define IXGBE_MAVTV(_i) (0x05010 + ((_i) * 4)) /* 8 of these (0-7) */
@@ -343,6 +664,9 @@
 #define IXGBE_MMAL(_i)  (0x05910 + ((_i) * 8)) /* 4 of these (0-3) */
 #define IXGBE_MMAH(_i)  (0x05914 + ((_i) * 8)) /* 4 of these (0-3) */
 #define IXGBE_FTFT      0x09400 /* 0x9400-0x97FC */
+#define IXGBE_METF(_i)  (0x05190 + ((_i) * 4)) /* 4 of these (0-3) */
+#define IXGBE_MDEF_EXT(_i) (0x05160 + ((_i) * 4)) /* 8 of these (0-7) */
+#define IXGBE_LSWFW     0x15014
 
 /* ARC Subsystem registers */
 #define IXGBE_HICR      0x15F00
@@ -375,16 +699,65 @@
 #define IXGBE_DCA_ID    0x11070
 #define IXGBE_DCA_CTRL  0x11074
 
+/* PCI-E registers 82599-Specific */
+#define IXGBE_GCR_EXT           0x11050
+#define IXGBE_GSCL_5_82599      0x11030
+#define IXGBE_GSCL_6_82599      0x11034
+#define IXGBE_GSCL_7_82599      0x11038
+#define IXGBE_GSCL_8_82599      0x1103C
+#define IXGBE_PHYADR_82599      0x11040
+#define IXGBE_PHYDAT_82599      0x11044
+#define IXGBE_PHYCTL_82599      0x11048
+#define IXGBE_PBACLR_82599      0x11068
+#define IXGBE_CIAA_82599        0x11088
+#define IXGBE_CIAD_82599        0x1108C
+#define IXGBE_PCIE_DIAG_0_82599 0x11090
+#define IXGBE_PCIE_DIAG_1_82599 0x11094
+#define IXGBE_PCIE_DIAG_2_82599 0x11098
+#define IXGBE_PCIE_DIAG_3_82599 0x1109C
+#define IXGBE_PCIE_DIAG_4_82599 0x110A0
+#define IXGBE_PCIE_DIAG_5_82599 0x110A4
+#define IXGBE_PCIE_DIAG_6_82599 0x110A8
+#define IXGBE_PCIE_DIAG_7_82599 0x110C0
+#define IXGBE_INTRPT_CSR_82599  0x110B0
+#define IXGBE_INTRPT_MASK_82599 0x110B8
+#define IXGBE_CDQ_MBR_82599     0x110B4
+#define IXGBE_MISC_REG_82599    0x110F0
+#define IXGBE_ECC_CTRL_0_82599  0x11100
+#define IXGBE_ECC_CTRL_1_82599  0x11104
+#define IXGBE_ECC_STATUS_82599  0x110E0
+#define IXGBE_BAR_CTRL_82599    0x110F4
+
+/* Time Sync Registers */
+#define IXGBE_TSYNCRXCTL 0x05188 /* Rx Time Sync Control register - RW */
+#define IXGBE_TSYNCTXCTL 0x08C00 /* Tx Time Sync Control register - RW */
+#define IXGBE_RXSTMPL    0x051E8 /* Rx timestamp Low - RO */
+#define IXGBE_RXSTMPH    0x051A4 /* Rx timestamp High - RO */
+#define IXGBE_RXSATRL    0x051A0 /* Rx timestamp attribute low - RO */
+#define IXGBE_RXSATRH    0x051A8 /* Rx timestamp attribute high - RO */
+#define IXGBE_RXMTRL     0x05120 /* RX message type register low - RW */
+#define IXGBE_TXSTMPL    0x08C04 /* Tx timestamp value Low - RO */
+#define IXGBE_TXSTMPH    0x08C08 /* Tx timestamp value High - RO */
+#define IXGBE_SYSTIML    0x08C0C /* System time register Low - RO */
+#define IXGBE_SYSTIMH    0x08C10 /* System time register High - RO */
+#define IXGBE_TIMINCA    0x08C14 /* Increment attributes register - RW */
+#define IXGBE_RXUDP      0x08C1C /* Time Sync Rx UDP Port - RW */
+
 /* Diagnostic Registers */
 #define IXGBE_RDSTATCTL   0x02C20
 #define IXGBE_RDSTAT(_i)  (0x02C00 + ((_i) * 4)) /* 0x02C00-0x02C1C */
 #define IXGBE_RDHMPN      0x02F08
 #define IXGBE_RIC_DW(_i)  (0x02F10 + ((_i) * 4))
 #define IXGBE_RDPROBE     0x02F20
+#define IXGBE_RDMAM       0x02F30
+#define IXGBE_RDMAD       0x02F34
 #define IXGBE_TDSTATCTL   0x07C20
 #define IXGBE_TDSTAT(_i)  (0x07C00 + ((_i) * 4)) /* 0x07C00 - 0x07C1C */
 #define IXGBE_TDHMPN      0x07F08
+#define IXGBE_TDHMPN2     0x082FC
+#define IXGBE_TXDESCIC    0x082CC
 #define IXGBE_TIC_DW(_i)  (0x07F10 + ((_i) * 4))
+#define IXGBE_TIC_DW2(_i) (0x082B0 + ((_i) * 4))
 #define IXGBE_TDPROBE     0x07F20
 #define IXGBE_TXBUFCTRL   0x0C600
 #define IXGBE_TXBUFDATA0  0x0C610
@@ -412,6 +785,10 @@
 #define IXGBE_TXDATARDPTR(_i)   (0x0C720 + ((_i) * 4)) /* 8 of these C720-C72C*/
 #define IXGBE_TXDESCRDPTR(_i)   (0x0C730 + ((_i) * 4)) /* 8 of these C730-C73C*/
 #define IXGBE_PCIEECCCTL 0x1106C
+#define IXGBE_PCIEECCCTL0 0x11100
+#define IXGBE_PCIEECCCTL1 0x11104
+#define IXGBE_RXDBUECC  0x03F70
+#define IXGBE_TXDBUECC  0x0CF70
 #define IXGBE_PBTXECC   0x0C300
 #define IXGBE_PBRXECC   0x03300
 #define IXGBE_GHECCR    0x110B0
@@ -437,24 +814,74 @@
 #define IXGBE_MSRWD     0x04260
 #define IXGBE_MLADD     0x04264
 #define IXGBE_MHADD     0x04268
+#define IXGBE_MAXFRS    0x04268
 #define IXGBE_TREG      0x0426C
 #define IXGBE_PCSS1     0x04288
 #define IXGBE_PCSS2     0x0428C
 #define IXGBE_XPCSS     0x04290
+#define IXGBE_MFLCN     0x04294
 #define IXGBE_SERDESC   0x04298
 #define IXGBE_MACS      0x0429C
 #define IXGBE_AUTOC     0x042A0
 #define IXGBE_LINKS     0x042A4
+#define IXGBE_LINKS2    0x04324
 #define IXGBE_AUTOC2    0x042A8
 #define IXGBE_AUTOC3    0x042AC
 #define IXGBE_ANLP1     0x042B0
 #define IXGBE_ANLP2     0x042B4
 #define IXGBE_ATLASCTL  0x04800
+#define IXGBE_MMNGC     0x042D0
+#define IXGBE_ANLPNP1   0x042D4
+#define IXGBE_ANLPNP2   0x042D8
+#define IXGBE_KRPCSFC   0x042E0
+#define IXGBE_KRPCSS    0x042E4
+#define IXGBE_FECS1     0x042E8
+#define IXGBE_FECS2     0x042EC
+#define IXGBE_SMADARCTL 0x14F10
+#define IXGBE_MPVC      0x04318
+#define IXGBE_SGMIIC    0x04314
+
+/* Omer CORECTL */
+#define IXGBE_CORECTL           0x014F00
+/* BARCTRL */
+#define IXGBE_BARCTRL           0x110F4
+#define IXGBE_BARCTRL_FLSIZE    0x0700
+#define IXGBE_BARCTRL_CSRSIZE   0x2000
+
+/* RSCCTL Bit Masks */
+#define IXGBE_RSCCTL_RSCEN          0x01
+#define IXGBE_RSCCTL_MAXDESC_1      0x00
+#define IXGBE_RSCCTL_MAXDESC_4      0x04
+#define IXGBE_RSCCTL_MAXDESC_8      0x08
+#define IXGBE_RSCCTL_MAXDESC_16     0x0C
+
+/* RSCDBU Bit Masks */
+#define IXGBE_RSCDBU_RSCSMALDIS_MASK    0x0000007F
+#define IXGBE_RSCDBU_RSCACKDIS          0x00000080
 
 /* RDRXCTL Bit Masks */
 #define IXGBE_RDRXCTL_RDMTS_1_2     0x00000000 /* Rx Desc Min Threshold Size */
+#define IXGBE_RDRXCTL_CRCSTRIP      0x00000002 /* CRC Strip */
 #define IXGBE_RDRXCTL_MVMEN         0x00000020
 #define IXGBE_RDRXCTL_DMAIDONE      0x00000008 /* DMA init cycle done */
+#define IXGBE_RDRXCTL_AGGDIS        0x00010000 /* Aggregation disable */
+#define IXGBE_RDRXCTL_RSCFRSTSIZE   0x003E0000 /* RSC First packet size */
+#define IXGBE_RDRXCTL_RSCLLIDIS     0x00800000 /* Disable RSC compl on LLI */
+
+/* RQTC Bit Masks and Shifts */
+#define IXGBE_RQTC_SHIFT_TC(_i)     ((_i) * 4)
+#define IXGBE_RQTC_TC0_MASK         (0x7 << 0)
+#define IXGBE_RQTC_TC1_MASK         (0x7 << 4)
+#define IXGBE_RQTC_TC2_MASK         (0x7 << 8)
+#define IXGBE_RQTC_TC3_MASK         (0x7 << 12)
+#define IXGBE_RQTC_TC4_MASK         (0x7 << 16)
+#define IXGBE_RQTC_TC5_MASK         (0x7 << 20)
+#define IXGBE_RQTC_TC6_MASK         (0x7 << 24)
+#define IXGBE_RQTC_TC7_MASK         (0x7 << 28)
+
+/* PSRTYPE.RQPL Bit masks and shift */
+#define IXGBE_PSRTYPE_RQPL_MASK     0x7
+#define IXGBE_PSRTYPE_RQPL_SHIFT    29
 
 /* CTRL Bit Masks */
 #define IXGBE_CTRL_GIO_DIS      0x00000004 /* Global IO Master Disable bit */
@@ -482,11 +909,18 @@
 #define IXGBE_DCA_CTRL_DCA_MODE_CB2 0x02 /* DCA Mode CB2 */
 
 #define IXGBE_DCA_RXCTRL_CPUID_MASK 0x0000001F /* Rx CPUID Mask */
+#define IXGBE_DCA_RXCTRL_CPUID_MASK_82599  0xFF000000 /* Rx CPUID Mask */
+#define IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599 24 /* Rx CPUID Shift */
 #define IXGBE_DCA_RXCTRL_DESC_DCA_EN (1 << 5) /* DCA Rx Desc enable */
 #define IXGBE_DCA_RXCTRL_HEAD_DCA_EN (1 << 6) /* DCA Rx Desc header enable */
 #define IXGBE_DCA_RXCTRL_DATA_DCA_EN (1 << 7) /* DCA Rx Desc payload enable */
+#define IXGBE_DCA_RXCTRL_DESC_RRO_EN (1 << 9) /* DCA Rx rd Desc Relax Order */
+#define IXGBE_DCA_RXCTRL_DESC_WRO_EN (1 << 13) /* DCA Rx wr Desc Relax Order */
+#define IXGBE_DCA_RXCTRL_DESC_HSRO_EN (1 << 15) /* DCA Rx Split Header RO */
 
 #define IXGBE_DCA_TXCTRL_CPUID_MASK 0x0000001F /* Tx CPUID Mask */
+#define IXGBE_DCA_TXCTRL_CPUID_MASK_82599  0xFF000000 /* Tx CPUID Mask */
+#define IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599 24 /* Tx CPUID Shift */
 #define IXGBE_DCA_TXCTRL_DESC_DCA_EN (1 << 5) /* DCA Tx Desc enable */
 #define IXGBE_DCA_TXCTRL_TX_WB_RO_EN (1 << 11) /* Tx Desc writeback RO bit */
 #define IXGBE_DCA_MAX_QUEUES_82598   16 /* DCA regs only on 16 queues */
@@ -530,6 +964,8 @@
 #define IXGBE_ATLAS_PDN_TX_1G_QL_ALL    0xF0
 #define IXGBE_ATLAS_PDN_TX_AN_QL_ALL    0xF0
 
+/* Omer bit masks */
+#define IXGBE_CORECTL_WRITE_CMD         0x00010000
 
 /* Device Type definitions for new protocol MDIO commands */
 #define IXGBE_MDIO_PMA_PMD_DEV_TYPE               0x1
@@ -557,6 +993,11 @@
 #define IXGBE_MDIO_PHY_SPEED_ABILITY   0x4 /* Speed Ability Reg */
 #define IXGBE_MDIO_PHY_SPEED_10G       0x0001 /* 10G capable */
 #define IXGBE_MDIO_PHY_SPEED_1G        0x0010 /* 1G capable */
+#define IXGBE_MDIO_PHY_SPEED_100M      0x0020 /* 100M capable */
+#define IXGBE_MDIO_PHY_EXT_ABILITY        0xB /* Ext Ability Reg */
+#define IXGBE_MDIO_PHY_10GBASET_ABILITY   0x0004 /* 10GBaseT capable */
+#define IXGBE_MDIO_PHY_1000BASET_ABILITY  0x0020 /* 1000BaseT capable */
+#define IXGBE_MDIO_PHY_100BASETX_ABILITY  0x0080 /* 100BaseTX capable */
 
 #define IXGBE_MDIO_PMA_PMD_SDA_SCL_ADDR     0xC30A /* PHY_XS SDA/SCL Addr Reg */
 #define IXGBE_MDIO_PMA_PMD_SDA_SCL_DATA     0xC30B /* PHY_XS SDA/SCL Data Reg */
@@ -576,6 +1017,8 @@
 /* PHY IDs*/
 #define TN1010_PHY_ID    0x00A19410
 #define TNX_FW_REV       0xB
+#define AQ1002_PHY_ID    0x03A1B420
+#define AQ_FW_REV        0x20
 #define QT2022_PHY_ID    0x0043A400
 #define ATH_PHY_ID       0x03429050
 
@@ -597,11 +1040,17 @@
 /* General purpose Interrupt Enable */
 #define IXGBE_SDP0_GPIEN         0x00000001 /* SDP0 */
 #define IXGBE_SDP1_GPIEN         0x00000002 /* SDP1 */
+#define IXGBE_SDP2_GPIEN         0x00000004 /* SDP2 */
 #define IXGBE_GPIE_MSIX_MODE     0x00000010 /* MSI-X mode */
 #define IXGBE_GPIE_OCD           0x00000020 /* Other Clear Disable */
 #define IXGBE_GPIE_EIMEN         0x00000040 /* Immediate Interrupt Enable */
 #define IXGBE_GPIE_EIAME         0x40000000
 #define IXGBE_GPIE_PBA_SUPPORT   0x80000000
+#define IXGBE_GPIE_RSC_DELAY_SHIFT 11
+#define IXGBE_GPIE_VTMODE_MASK   0x0000C000 /* VT Mode Mask */
+#define IXGBE_GPIE_VTMODE_16     0x00004000 /* 16 VFs 8 queues per VF */
+#define IXGBE_GPIE_VTMODE_32     0x00008000 /* 32 VFs 4 queues per VF */
+#define IXGBE_GPIE_VTMODE_64     0x0000C000 /* 64 VFs 2 queues per VF */
 
 /* Transmit Flow Control status */
 #define IXGBE_TFCS_TXOFF         0x00000001
@@ -642,6 +1091,23 @@
 #define IXGBE_VMD_CTL_VMDQ_EN     0x00000001
 #define IXGBE_VMD_CTL_VMDQ_FILTER 0x00000002
 
+/* VT_CTL bitmasks */
+#define IXGBE_VT_CTL_DIS_DEFPL  0x20000000 /* disable default pool */
+#define IXGBE_VT_CTL_REPLEN     0x40000000 /* replication enabled */
+#define IXGBE_VT_CTL_VT_ENABLE  0x00000001  /* Enable VT Mode */
+#define IXGBE_VT_CTL_POOL_SHIFT 7
+#define IXGBE_VT_CTL_POOL_MASK  (0x3F << IXGBE_VT_CTL_POOL_SHIFT)
+
+/* VMOLR bitmasks */
+#define IXGBE_VMOLR_AUPE        0x01000000 /* accept untagged packets */
+#define IXGBE_VMOLR_ROMPE       0x02000000 /* accept packets in MTA tbl */
+#define IXGBE_VMOLR_ROPE        0x04000000 /* accept packets in UC tbl */
+#define IXGBE_VMOLR_BAM         0x08000000 /* accept broadcast packets */
+#define IXGBE_VMOLR_MPE         0x10000000 /* multicast promiscuous */
+
+/* VFRE bitmask */
+#define IXGBE_VFRE_ENABLE_ALL   0xFFFFFFFF
+
 /* RDHMPN and TDHMPN bitmasks */
 #define IXGBE_RDHMPN_RDICADDR       0x007FF800
 #define IXGBE_RDHMPN_RDICRDREQ      0x00800000
@@ -650,6 +1116,41 @@
 #define IXGBE_TDHMPN_TDICRDREQ      0x00800000
 #define IXGBE_TDHMPN_TDICADDR_SHIFT 11
 
+#define IXGBE_RDMAM_MEM_SEL_SHIFT   13
+#define IXGBE_RDMAM_DWORD_SHIFT     9
+#define IXGBE_RDMAM_DESC_COMP_FIFO  1
+#define IXGBE_RDMAM_DFC_CMD_FIFO    2
+#define IXGBE_RDMAM_RSC_HEADER_ADDR 3
+#define IXGBE_RDMAM_TCN_STATUS_RAM  4
+#define IXGBE_RDMAM_WB_COLL_FIFO    5
+#define IXGBE_RDMAM_QSC_CNT_RAM     6
+#define IXGBE_RDMAM_QSC_FCOE_RAM    7
+#define IXGBE_RDMAM_QSC_QUEUE_CNT   8
+#define IXGBE_RDMAM_QSC_QUEUE_RAM   0xA
+#define IXGBE_RDMAM_QSC_RSC_RAM     0xB
+#define IXGBE_RDMAM_DESC_COM_FIFO_RANGE     135
+#define IXGBE_RDMAM_DESC_COM_FIFO_COUNT     4
+#define IXGBE_RDMAM_DFC_CMD_FIFO_RANGE      48
+#define IXGBE_RDMAM_DFC_CMD_FIFO_COUNT      7
+#define IXGBE_RDMAM_RSC_HEADER_ADDR_RANGE   32
+#define IXGBE_RDMAM_RSC_HEADER_ADDR_COUNT   4
+#define IXGBE_RDMAM_TCN_STATUS_RAM_RANGE    256
+#define IXGBE_RDMAM_TCN_STATUS_RAM_COUNT    9
+#define IXGBE_RDMAM_WB_COLL_FIFO_RANGE      8
+#define IXGBE_RDMAM_WB_COLL_FIFO_COUNT      4
+#define IXGBE_RDMAM_QSC_CNT_RAM_RANGE       64
+#define IXGBE_RDMAM_QSC_CNT_RAM_COUNT       4
+#define IXGBE_RDMAM_QSC_FCOE_RAM_RANGE      512
+#define IXGBE_RDMAM_QSC_FCOE_RAM_COUNT      5
+#define IXGBE_RDMAM_QSC_QUEUE_CNT_RANGE     32
+#define IXGBE_RDMAM_QSC_QUEUE_CNT_COUNT     4
+#define IXGBE_RDMAM_QSC_QUEUE_RAM_RANGE     128
+#define IXGBE_RDMAM_QSC_QUEUE_RAM_COUNT     8
+#define IXGBE_RDMAM_QSC_RSC_RAM_RANGE       32
+#define IXGBE_RDMAM_QSC_RSC_RAM_COUNT       8
+
+#define IXGBE_TXDESCIC_READY        0x80000000
+
 /* Receive Checksum Control */
 #define IXGBE_RXCSUM_IPPCSE     0x00001000   /* IP payload checksum enable */
 #define IXGBE_RXCSUM_PCSD       0x00002000   /* packet checksum disabled */
@@ -670,15 +1171,25 @@
 #define IXGBE_RMCS_TFCE_PRIORITY       0x00000010 /* Tx Priority FC ena */
 #define IXGBE_RMCS_ARBDIS       0x00000040 /* Arbitration disable bit */
 
+/* FCCFG Bit Masks */
+#define IXGBE_FCCFG_TFCE_802_3X         0x00000008 /* Tx link FC enable */
+#define IXGBE_FCCFG_TFCE_PRIORITY       0x00000010 /* Tx priority FC enable */
 
 /* Interrupt register bitmasks */
 
 /* Extended Interrupt Cause Read */
 #define IXGBE_EICR_RTX_QUEUE    0x0000FFFF /* RTx Queue Interrupt */
+#define IXGBE_EICR_FLOW_DIR     0x00010000 /* FDir Exception */
+#define IXGBE_EICR_RX_MISS      0x00020000 /* Packet Buffer Overrun */
+#define IXGBE_EICR_PCI          0x00040000 /* PCI Exception */
+#define IXGBE_EICR_MAILBOX      0x00080000 /* VF to PF Mailbox Interrupt */
 #define IXGBE_EICR_LSC          0x00100000 /* Link Status Change */
+#define IXGBE_EICR_LINKSEC      0x00200000 /* PN Threshold */
 #define IXGBE_EICR_MNG          0x00400000 /* Manageability Event Interrupt */
 #define IXGBE_EICR_GPI_SDP0     0x01000000 /* Gen Purpose Interrupt on SDP0 */
 #define IXGBE_EICR_GPI_SDP1     0x02000000 /* Gen Purpose Interrupt on SDP1 */
+#define IXGBE_EICR_GPI_SDP2     0x04000000 /* Gen Purpose Interrupt on SDP2 */
+#define IXGBE_EICR_ECC          0x10000000 /* ECC Error */
 #define IXGBE_EICR_PBUR         0x10000000 /* Packet Buffer Handler Error */
 #define IXGBE_EICR_DHER         0x20000000 /* Descriptor Handler Error */
 #define IXGBE_EICR_TCP_TIMER    0x40000000 /* TCP Timer */
@@ -686,10 +1197,16 @@
 
 /* Extended Interrupt Cause Set */
 #define IXGBE_EICS_RTX_QUEUE    IXGBE_EICR_RTX_QUEUE /* RTx Queue Interrupt */
+#define IXGBE_EICS_FLOW_DIR     IXGBE_EICR_FLOW_DIR  /* FDir Exception */
+#define IXGBE_EICS_RX_MISS      IXGBE_EICR_RX_MISS   /* Pkt Buffer Overrun */
+#define IXGBE_EICS_PCI          IXGBE_EICR_PCI       /* PCI Exception */
+#define IXGBE_EICS_MAILBOX      IXGBE_EICR_MAILBOX   /* VF to PF Mailbox Int */
 #define IXGBE_EICS_LSC          IXGBE_EICR_LSC       /* Link Status Change */
 #define IXGBE_EICS_MNG          IXGBE_EICR_MNG       /* MNG Event Interrupt */
 #define IXGBE_EICS_GPI_SDP0     IXGBE_EICR_GPI_SDP0  /* SDP0 Gen Purpose Int */
 #define IXGBE_EICS_GPI_SDP1     IXGBE_EICR_GPI_SDP1  /* SDP1 Gen Purpose Int */
+#define IXGBE_EICS_GPI_SDP2     IXGBE_EICR_GPI_SDP2  /* SDP2 Gen Purpose Int */
+#define IXGBE_EICS_ECC          IXGBE_EICR_ECC       /* ECC Error */
 #define IXGBE_EICS_PBUR         IXGBE_EICR_PBUR      /* Pkt Buf Handler Err */
 #define IXGBE_EICS_DHER         IXGBE_EICR_DHER      /* Desc Handler Error */
 #define IXGBE_EICS_TCP_TIMER    IXGBE_EICR_TCP_TIMER /* TCP Timer */
@@ -697,10 +1214,16 @@
 
 /* Extended Interrupt Mask Set */
 #define IXGBE_EIMS_RTX_QUEUE    IXGBE_EICR_RTX_QUEUE /* RTx Queue Interrupt */
+#define IXGBE_EIMS_FLOW_DIR     IXGBE_EICR_FLOW_DIR  /* FDir Exception */
+#define IXGBE_EIMS_RX_MISS      IXGBE_EICR_RX_MISS   /* Packet Buffer Overrun */
+#define IXGBE_EIMS_PCI          IXGBE_EICR_PCI       /* PCI Exception */
+#define IXGBE_EIMS_MAILBOX      IXGBE_EICR_MAILBOX   /* VF to PF Mailbox Int */
 #define IXGBE_EIMS_LSC          IXGBE_EICR_LSC       /* Link Status Change */
 #define IXGBE_EIMS_MNG          IXGBE_EICR_MNG       /* MNG Event Interrupt */
 #define IXGBE_EIMS_GPI_SDP0     IXGBE_EICR_GPI_SDP0  /* SDP0 Gen Purpose Int */
 #define IXGBE_EIMS_GPI_SDP1     IXGBE_EICR_GPI_SDP1  /* SDP1 Gen Purpose Int */
+#define IXGBE_EIMS_GPI_SDP2     IXGBE_EICR_GPI_SDP2  /* SDP2 Gen Purpose Int */
+#define IXGBE_EIMS_ECC          IXGBE_EICR_ECC       /* ECC Error */
 #define IXGBE_EIMS_PBUR         IXGBE_EICR_PBUR      /* Pkt Buf Handler Err */
 #define IXGBE_EIMS_DHER         IXGBE_EICR_DHER      /* Descr Handler Error */
 #define IXGBE_EIMS_TCP_TIMER    IXGBE_EICR_TCP_TIMER /* TCP Timer */
@@ -708,10 +1231,16 @@
 
 /* Extended Interrupt Mask Clear */
 #define IXGBE_EIMC_RTX_QUEUE    IXGBE_EICR_RTX_QUEUE /* RTx Queue Interrupt */
+#define IXGBE_EIMC_FLOW_DIR     IXGBE_EICR_FLOW_DIR  /* FDir Exception */
+#define IXGBE_EIMC_RX_MISS      IXGBE_EICR_RX_MISS   /* Packet Buffer Overrun */
+#define IXGBE_EIMC_PCI          IXGBE_EICR_PCI       /* PCI Exception */
+#define IXGBE_EIMC_MAILBOX      IXGBE_EICR_MAILBOX   /* VF to PF Mailbox Int */
 #define IXGBE_EIMC_LSC          IXGBE_EICR_LSC       /* Link Status Change */
 #define IXGBE_EIMC_MNG          IXGBE_EICR_MNG       /* MNG Event Interrupt */
 #define IXGBE_EIMC_GPI_SDP0     IXGBE_EICR_GPI_SDP0  /* SDP0 Gen Purpose Int */
 #define IXGBE_EIMC_GPI_SDP1     IXGBE_EICR_GPI_SDP1  /* SDP1 Gen Purpose Int */
+#define IXGBE_EIMC_GPI_SDP2     IXGBE_EICR_GPI_SDP2  /* SDP2 Gen Purpose Int */
+#define IXGBE_EIMC_ECC          IXGBE_EICR_ECC       /* ECC Error */
 #define IXGBE_EIMC_PBUR         IXGBE_EICR_PBUR      /* Pkt Buf Handler Err */
 #define IXGBE_EIMC_DHER         IXGBE_EICR_DHER      /* Desc Handler Err */
 #define IXGBE_EIMC_TCP_TIMER    IXGBE_EICR_TCP_TIMER /* TCP Timer */
@@ -734,12 +1263,45 @@
 #define IXGBE_IMIREXT_CTRL_SYN    0x00020000  /* Check SYN bit in header */
 #define IXGBE_IMIREXT_CTRL_FIN    0x00040000  /* Check FIN bit in header */
 #define IXGBE_IMIREXT_CTRL_BP     0x00080000  /* Bypass check of control bits */
+#define IXGBE_IMIR_SIZE_BP_82599  0x00001000 /* Packet size bypass */
+#define IXGBE_IMIR_CTRL_URG_82599 0x00002000 /* Check URG bit in header */
+#define IXGBE_IMIR_CTRL_ACK_82599 0x00004000 /* Check ACK bit in header */
+#define IXGBE_IMIR_CTRL_PSH_82599 0x00008000 /* Check PSH bit in header */
+#define IXGBE_IMIR_CTRL_RST_82599 0x00010000 /* Check RST bit in header */
+#define IXGBE_IMIR_CTRL_SYN_82599 0x00020000 /* Check SYN bit in header */
+#define IXGBE_IMIR_CTRL_FIN_82599 0x00040000 /* Check FIN bit in header */
+#define IXGBE_IMIR_CTRL_BP_82599  0x00080000 /* Bypass check of control bits */
+#define IXGBE_IMIR_LLI_EN_82599   0x00100000 /* Enables low latency Int */
+#define IXGBE_IMIR_RX_QUEUE_MASK_82599  0x0000007F /* Rx Queue Mask */
+#define IXGBE_IMIR_RX_QUEUE_SHIFT_82599 21 /* Rx Queue Shift */
+#define IXGBE_IMIRVP_PRIORITY_MASK      0x00000007 /* VLAN priority mask */
+#define IXGBE_IMIRVP_PRIORITY_EN        0x00000008 /* VLAN priority enable */
+
+#define IXGBE_MAX_FTQF_FILTERS          128
+#define IXGBE_FTQF_PROTOCOL_MASK        0x00000003
+#define IXGBE_FTQF_PROTOCOL_TCP         0x00000000
+#define IXGBE_FTQF_PROTOCOL_UDP         0x00000001
+#define IXGBE_FTQF_PROTOCOL_SCTP        2
+#define IXGBE_FTQF_PRIORITY_MASK        0x00000007
+#define IXGBE_FTQF_PRIORITY_SHIFT       2
+#define IXGBE_FTQF_POOL_MASK            0x0000003F
+#define IXGBE_FTQF_POOL_SHIFT           8
+#define IXGBE_FTQF_5TUPLE_MASK_MASK     0x0000001F
+#define IXGBE_FTQF_5TUPLE_MASK_SHIFT    25
+#define IXGBE_FTQF_SOURCE_ADDR_MASK     0x1E
+#define IXGBE_FTQF_DEST_ADDR_MASK       0x1D
+#define IXGBE_FTQF_SOURCE_PORT_MASK     0x1B
+#define IXGBE_FTQF_DEST_PORT_MASK       0x17
+#define IXGBE_FTQF_PROTOCOL_COMP_MASK   0x0F
+#define IXGBE_FTQF_POOL_MASK_EN         0x40000000
+#define IXGBE_FTQF_QUEUE_ENABLE         0x80000000
 
 /* Interrupt clear mask */
 #define IXGBE_IRQ_CLEAR_MASK    0xFFFFFFFF
 
 /* Interrupt Vector Allocation Registers */
 #define IXGBE_IVAR_REG_NUM      25
+#define IXGBE_IVAR_REG_NUM_82599           64
 #define IXGBE_IVAR_TXRX_ENTRY   96
 #define IXGBE_IVAR_RX_ENTRY     64
 #define IXGBE_IVAR_RX_QUEUE(_i)    (0 + (_i))
@@ -753,6 +1315,32 @@
 
 #define IXGBE_IVAR_ALLOC_VAL    0x80 /* Interrupt Allocation valid */
 
+/* ETYPE Queue Filter/Select Bit Masks */
+#define IXGBE_MAX_ETQF_FILTERS  8
+#define IXGBE_ETQF_FCOE         0x08000000 /* bit 27 */
+#define IXGBE_ETQF_BCN          0x10000000 /* bit 28 */
+#define IXGBE_ETQF_1588         0x40000000 /* bit 30 */
+#define IXGBE_ETQF_FILTER_EN    0x80000000 /* bit 31 */
+#define IXGBE_ETQF_POOL_ENABLE   (1 << 26) /* bit 26 */
+
+#define IXGBE_ETQS_RX_QUEUE     0x007F0000 /* bits 22:16 */
+#define IXGBE_ETQS_RX_QUEUE_SHIFT       16
+#define IXGBE_ETQS_LLI          0x20000000 /* bit 29 */
+#define IXGBE_ETQS_QUEUE_EN     0x80000000 /* bit 31 */
+
+/*
+ * ETQF filter list: one static filter per filter consumer. This is
+ *                   to avoid filter collisions later. Add new filters
+ *                   here!!
+ *
+ * Current filters:
+ *    EAPOL 802.1x (0x888e): Filter 0
+ *    FCoE (0x8906):         Filter 2
+ *    1588 (0x88f7):         Filter 3
+ */
+#define IXGBE_ETQF_FILTER_EAPOL          0
+#define IXGBE_ETQF_FILTER_FCOE           2
+#define IXGBE_ETQF_FILTER_1588           3
 /* VLAN Control Bit Masks */
 #define IXGBE_VLNCTRL_VET       0x0000FFFF  /* bits 0-15 */
 #define IXGBE_VLNCTRL_CFI       0x10000000  /* bit 28 */
@@ -760,6 +1348,9 @@
 #define IXGBE_VLNCTRL_VFE       0x40000000  /* bit 30 */
 #define IXGBE_VLNCTRL_VME       0x80000000  /* bit 31 */
 
+/* VLAN pool filtering masks */
+#define IXGBE_VLVF_VIEN         0x80000000  /* filter is valid */
+#define IXGBE_VLVF_ENTRIES      64
 
 #define IXGBE_ETHERNET_IEEE_VLAN_TYPE 0x8100  /* 802.1q protocol */
 
@@ -772,7 +1363,8 @@
 #define IXGBE_STATUS_LAN_ID_1   0x00000004 /* LAN ID 1 */
 
 /* ESDP Bit Masks */
-#define IXGBE_ESDP_SDP1 0x00000001
+#define IXGBE_ESDP_SDP0 0x00000001
+#define IXGBE_ESDP_SDP1 0x00000002
 #define IXGBE_ESDP_SDP4 0x00000010 /* SDP4 Data Value */
 #define IXGBE_ESDP_SDP5 0x00000020 /* SDP5 Data Value */
 #define IXGBE_ESDP_SDP6 0x00000040 /* SDP6 Data Value */
@@ -800,7 +1392,7 @@
 #define IXGBE_LED_OFF           0xF
 
 /* AUTOC Bit Masks */
-#define IXGBE_AUTOC_KX4_KX_SUPP 0xC0000000
+#define IXGBE_AUTOC_KX4_KX_SUPP_MASK 0xC0000000
 #define IXGBE_AUTOC_KX4_SUPP    0x80000000
 #define IXGBE_AUTOC_KX_SUPP     0x40000000
 #define IXGBE_AUTOC_PAUSE       0x30000000
@@ -809,9 +1401,17 @@
 #define IXGBE_AUTOC_AN_RX_LOOSE 0x01000000
 #define IXGBE_AUTOC_AN_RX_DRIFT 0x00800000
 #define IXGBE_AUTOC_AN_RX_ALIGN 0x007C0000
+#define IXGBE_AUTOC_FECA        0x00040000
+#define IXGBE_AUTOC_FECR        0x00020000
+#define IXGBE_AUTOC_KR_SUPP     0x00010000
 #define IXGBE_AUTOC_AN_RESTART  0x00001000
 #define IXGBE_AUTOC_FLU         0x00000001
 #define IXGBE_AUTOC_LMS_SHIFT   13
+#define IXGBE_AUTOC_LMS_10G_SERIAL      (0x3 << IXGBE_AUTOC_LMS_SHIFT)
+#define IXGBE_AUTOC_LMS_KX4_KX_KR       (0x4 << IXGBE_AUTOC_LMS_SHIFT)
+#define IXGBE_AUTOC_LMS_SGMII_1G_100M   (0x5 << IXGBE_AUTOC_LMS_SHIFT)
+#define IXGBE_AUTOC_LMS_KX4_KX_KR_1G_AN (0x6 << IXGBE_AUTOC_LMS_SHIFT)
+#define IXGBE_AUTOC_LMS_KX4_KX_KR_SGMII (0x7 << IXGBE_AUTOC_LMS_SHIFT)
 #define IXGBE_AUTOC_LMS_MASK            (0x7 << IXGBE_AUTOC_LMS_SHIFT)
 #define IXGBE_AUTOC_LMS_1G_LINK_NO_AN   (0x0 << IXGBE_AUTOC_LMS_SHIFT)
 #define IXGBE_AUTOC_LMS_10G_LINK_NO_AN  (0x1 << IXGBE_AUTOC_LMS_SHIFT)
@@ -820,15 +1420,24 @@
 #define IXGBE_AUTOC_LMS_KX4_AN_1G_AN    (0x6 << IXGBE_AUTOC_LMS_SHIFT)
 #define IXGBE_AUTOC_LMS_ATTACH_TYPE     (0x7 << IXGBE_AUTOC_10G_PMA_PMD_SHIFT)
 
-#define IXGBE_AUTOC_1G_PMA_PMD         0x00000200
-#define IXGBE_AUTOC_10G_PMA_PMD        0x00000180
-#define IXGBE_AUTOC_10G_PMA_PMD_SHIFT 7
-#define IXGBE_AUTOC_1G_PMA_PMD_SHIFT 9
+#define IXGBE_AUTOC_1G_PMA_PMD_MASK    0x00000200
+#define IXGBE_AUTOC_1G_PMA_PMD_SHIFT   9
+#define IXGBE_AUTOC_10G_PMA_PMD_MASK   0x00000180
+#define IXGBE_AUTOC_10G_PMA_PMD_SHIFT  7
 #define IXGBE_AUTOC_10G_XAUI   (0x0 << IXGBE_AUTOC_10G_PMA_PMD_SHIFT)
 #define IXGBE_AUTOC_10G_KX4    (0x1 << IXGBE_AUTOC_10G_PMA_PMD_SHIFT)
 #define IXGBE_AUTOC_10G_CX4    (0x2 << IXGBE_AUTOC_10G_PMA_PMD_SHIFT)
 #define IXGBE_AUTOC_1G_BX      (0x0 << IXGBE_AUTOC_1G_PMA_PMD_SHIFT)
 #define IXGBE_AUTOC_1G_KX      (0x1 << IXGBE_AUTOC_1G_PMA_PMD_SHIFT)
+#define IXGBE_AUTOC_1G_SFI     (0x0 << IXGBE_AUTOC_1G_PMA_PMD_SHIFT)
+#define IXGBE_AUTOC_1G_KX_BX   (0x1 << IXGBE_AUTOC_1G_PMA_PMD_SHIFT)
+
+#define IXGBE_AUTOC2_UPPER_MASK  0xFFFF0000
+#define IXGBE_AUTOC2_10G_SERIAL_PMA_PMD_MASK  0x00030000
+#define IXGBE_AUTOC2_10G_SERIAL_PMA_PMD_SHIFT 16
+#define IXGBE_AUTOC2_10G_KR  (0x0 << IXGBE_AUTOC2_10G_SERIAL_PMA_PMD_SHIFT)
+#define IXGBE_AUTOC2_10G_XFI (0x1 << IXGBE_AUTOC2_10G_SERIAL_PMA_PMD_SHIFT)
+#define IXGBE_AUTOC2_10G_SFI (0x2 << IXGBE_AUTOC2_10G_SERIAL_PMA_PMD_SHIFT)
 
 /* LINKS Bit Masks */
 #define IXGBE_LINKS_KX_AN_COMP  0x80000000
@@ -838,6 +1447,7 @@
 #define IXGBE_LINKS_RX_MODE     0x06000000
 #define IXGBE_LINKS_TX_MODE     0x01800000
 #define IXGBE_LINKS_XGXS_EN     0x00400000
+#define IXGBE_LINKS_SGMII_EN    0x02000000
 #define IXGBE_LINKS_PCS_1G_EN   0x00200000
 #define IXGBE_LINKS_1G_AN_EN    0x00100000
 #define IXGBE_LINKS_KX_AN_IDLE  0x00080000
@@ -847,11 +1457,13 @@
 #define IXGBE_LINKS_TL_FAULT    0x00001000
 #define IXGBE_LINKS_SIGNAL      0x00000F00
 
+#define IXGBE_LINKS_SPEED_82599     0x30000000
+#define IXGBE_LINKS_SPEED_10G_82599 0x30000000
+#define IXGBE_LINKS_SPEED_1G_82599  0x20000000
+#define IXGBE_LINKS_SPEED_100_82599 0x10000000
 #define IXGBE_LINK_UP_TIME      90 /* 9.0 Seconds */
 #define IXGBE_AUTO_NEG_TIME     45 /* 4.5 Seconds */
 
-#define FIBER_LINK_UP_LIMIT     50
-
 /* PCS1GLSTA Bit Masks */
 #define IXGBE_PCS1GLSTA_LINK_OK         1
 #define IXGBE_PCS1GLSTA_SYNK_OK         0x10
@@ -923,6 +1535,13 @@
 #define IXGBE_FW_PTR            0x0F
 #define IXGBE_PBANUM0_PTR       0x15
 #define IXGBE_PBANUM1_PTR       0x16
+#define IXGBE_SAN_MAC_ADDR_PTR  0x28
+#define IXGBE_DEVICE_CAPS       0x2C
+#define IXGBE_PCIE_MSIX_82599_CAPS  0x72
+#define IXGBE_PCIE_MSIX_82598_CAPS  0x62
+
+/* MSI-X capability fields masks */
+#define IXGBE_PCIE_MSIX_TBL_SZ_MASK     0x7FF
 
 /* Legacy EEPROM word offsets */
 #define IXGBE_ISCSI_BOOT_CAPS           0x0033
@@ -961,6 +1580,11 @@
 #define IXGBE_EERD_ATTEMPTS 100000
 #endif
 
+#define IXGBE_SAN_MAC_ADDR_PORT0_OFFSET  0x0
+#define IXGBE_SAN_MAC_ADDR_PORT1_OFFSET  0x3
+#define IXGBE_DEVICE_CAPS_ALLOW_ANY_SFP  0x1
+#define IXGBE_DEVICE_CAPS_FCOE_OFFLOADS  0x2
+
 /* PCI Bus Info */
 #define IXGBE_PCI_LINK_STATUS     0xB2
 #define IXGBE_PCI_LINK_WIDTH      0x3F0
@@ -1025,6 +1649,7 @@
 #define IXGBE_RXCTRL_RXEN       0x00000001  /* Enable Receiver */
 #define IXGBE_RXCTRL_DMBYPS     0x00000002  /* Descriptor Monitor Bypass */
 #define IXGBE_RXDCTL_ENABLE     0x02000000  /* Enable specific Rx Queue */
+#define IXGBE_RXDCTL_VME        0x40000000  /* VLAN mode enable */
 
 #define IXGBE_FCTRL_SBP 0x00000002 /* Store Bad Packet */
 #define IXGBE_FCTRL_MPE 0x00000100 /* Multicast Promiscuous Ena*/
@@ -1035,9 +1660,23 @@
 /* Receive Priority Flow Control Enable */
 #define IXGBE_FCTRL_RPFCE 0x00004000
 #define IXGBE_FCTRL_RFCE 0x00008000 /* Receive Flow Control Ena */
+#define IXGBE_MFLCN_PMCF        0x00000001 /* Pass MAC Control Frames */
+#define IXGBE_MFLCN_DPF         0x00000002 /* Discard Pause Frame */
+#define IXGBE_MFLCN_RPFCE       0x00000004 /* Receive Priority FC Enable */
+#define IXGBE_MFLCN_RFCE        0x00000008 /* Receive FC Enable */
 
 /* Multiple Receive Queue Control */
 #define IXGBE_MRQC_RSSEN                 0x00000001  /* RSS Enable */
+#define IXGBE_MRQC_MRQE_MASK                    0xF /* Bits 3:0 */
+#define IXGBE_MRQC_RT8TCEN               0x00000002 /* 8 TC no RSS */
+#define IXGBE_MRQC_RT4TCEN               0x00000003 /* 4 TC no RSS */
+#define IXGBE_MRQC_RTRSS8TCEN            0x00000004 /* 8 TC w/ RSS */
+#define IXGBE_MRQC_RTRSS4TCEN            0x00000005 /* 4 TC w/ RSS */
+#define IXGBE_MRQC_VMDQEN                0x00000008 /* VMDq2 64 pools no RSS */
+#define IXGBE_MRQC_VMDQRSS32EN           0x0000000A /* VMDq2 32 pools w/ RSS */
+#define IXGBE_MRQC_VMDQRSS64EN           0x0000000B /* VMDq2 64 pools w/ RSS */
+#define IXGBE_MRQC_VMDQRT8TCEN           0x0000000C /* VMDq2/RT 16 pool 8 TC */
+#define IXGBE_MRQC_VMDQRT4TCEN           0x0000000D /* VMDq2/RT 32 pool 4 TC */
 #define IXGBE_MRQC_RSS_FIELD_MASK        0xFFFF0000
 #define IXGBE_MRQC_RSS_FIELD_IPV4_TCP    0x00010000
 #define IXGBE_MRQC_RSS_FIELD_IPV4        0x00020000
@@ -1048,6 +1687,12 @@
 #define IXGBE_MRQC_RSS_FIELD_IPV4_UDP    0x00400000
 #define IXGBE_MRQC_RSS_FIELD_IPV6_UDP    0x00800000
 #define IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP 0x01000000
+#define IXGBE_MRQC_L3L4TXSWEN            0x00008000
+
+/* Queue Drop Enable */
+#define IXGBE_QDE_ENABLE     0x00000001
+#define IXGBE_QDE_IDX_MASK   0x00007F00
+#define IXGBE_QDE_IDX_SHIFT           8
 
 #define IXGBE_TXD_POPTS_IXSM 0x01       /* Insert IP checksum */
 #define IXGBE_TXD_POPTS_TXSM 0x02       /* Insert TCP/UDP checksum */
@@ -1059,10 +1704,26 @@
 #define IXGBE_TXD_CMD_VLE    0x40000000 /* Add VLAN tag */
 #define IXGBE_TXD_STAT_DD    0x00000001 /* Descriptor Done */
 
+#define IXGBE_RXDADV_IPSEC_STATUS_SECP                  0x00020000
+#define IXGBE_RXDADV_IPSEC_ERROR_INVALID_PROTOCOL       0x08000000
+#define IXGBE_RXDADV_IPSEC_ERROR_INVALID_LENGTH         0x10000000
+#define IXGBE_RXDADV_IPSEC_ERROR_AUTH_FAILED            0x18000000
+#define IXGBE_RXDADV_IPSEC_ERROR_BIT_MASK               0x18000000
+/* Multiple Transmit Queue Command Register */
+#define IXGBE_MTQC_RT_ENA       0x1 /* DCB Enable */
+#define IXGBE_MTQC_VT_ENA       0x2 /* VMDQ2 Enable */
+#define IXGBE_MTQC_64Q_1PB      0x0 /* 64 queues 1 pack buffer */
+#define IXGBE_MTQC_32VF         0x8 /* 4 TX Queues per pool w/32VF's */
+#define IXGBE_MTQC_64VF         0x4 /* 2 TX Queues per pool w/64VF's */
+#define IXGBE_MTQC_8TC_8TQ      0xC /* 8 TC if RT_ENA or 8 TQ if VT_ENA */
+
 /* Receive Descriptor bit definitions */
 #define IXGBE_RXD_STAT_DD       0x01    /* Descriptor Done */
 #define IXGBE_RXD_STAT_EOP      0x02    /* End of Packet */
+#define IXGBE_RXD_STAT_FLM      0x04    /* FDir Match */
 #define IXGBE_RXD_STAT_VP       0x08    /* IEEE VLAN Packet */
+#define IXGBE_RXDADV_NEXTP_MASK   0x000FFFF0 /* Next Descriptor Index */
+#define IXGBE_RXDADV_NEXTP_SHIFT  0x00000004
 #define IXGBE_RXD_STAT_UDPCS    0x10    /* UDP xsum calculated */
 #define IXGBE_RXD_STAT_L4CS     0x20    /* L4 xsum calculated */
 #define IXGBE_RXD_STAT_IPCS     0x40    /* IP xsum calculated */
@@ -1071,6 +1732,10 @@
 #define IXGBE_RXD_STAT_VEXT     0x200   /* 1st VLAN found */
 #define IXGBE_RXD_STAT_UDPV     0x400   /* Valid UDP checksum */
 #define IXGBE_RXD_STAT_DYNINT   0x800   /* Pkt caused INT via DYNINT */
+#define IXGBE_RXD_STAT_LLINT    0x800   /* Pkt caused Low Latency Interrupt */
+#define IXGBE_RXD_STAT_TS       0x10000 /* Time Stamp */
+#define IXGBE_RXD_STAT_SECP     0x20000 /* Security Processing */
+#define IXGBE_RXD_STAT_LB       0x40000 /* Loopback Status */
 #define IXGBE_RXD_STAT_ACK      0x8000  /* ACK Packet indication */
 #define IXGBE_RXD_ERR_CE        0x01    /* CRC Error */
 #define IXGBE_RXD_ERR_LE        0x02    /* Length Error */
@@ -1079,6 +1744,13 @@
 #define IXGBE_RXD_ERR_USE       0x20    /* Undersize Error */
 #define IXGBE_RXD_ERR_TCPE      0x40    /* TCP/UDP Checksum Error */
 #define IXGBE_RXD_ERR_IPE       0x80    /* IP Checksum Error */
+#define IXGBE_RXDADV_ERR_MASK           0xfff00000 /* RDESC.ERRORS mask */
+#define IXGBE_RXDADV_ERR_SHIFT          20         /* RDESC.ERRORS shift */
+#define IXGBE_RXDADV_ERR_FCEOFE         0x80000000 /* FCoEFe/IPE */
+#define IXGBE_RXDADV_ERR_FCERR          0x00700000 /* FCERR/FDIRERR */
+#define IXGBE_RXDADV_ERR_FDIR_LEN       0x00100000 /* FDIR Length error */
+#define IXGBE_RXDADV_ERR_FDIR_DROP      0x00200000 /* FDIR Drop error */
+#define IXGBE_RXDADV_ERR_FDIR_COLL      0x00400000 /* FDIR Collision error */
 #define IXGBE_RXDADV_ERR_HBO    0x00800000 /*Header Buffer Overflow */
 #define IXGBE_RXDADV_ERR_CE     0x01000000 /* CRC Error */
 #define IXGBE_RXDADV_ERR_LE     0x02000000 /* Length Error */
@@ -1093,9 +1765,29 @@
 #define IXGBE_RXD_CFI_MASK      0x1000  /* CFI is bit 12 */
 #define IXGBE_RXD_CFI_SHIFT     12
 
+#define IXGBE_RXDADV_STAT_DD            IXGBE_RXD_STAT_DD  /* Done */
+#define IXGBE_RXDADV_STAT_EOP           IXGBE_RXD_STAT_EOP /* End of Packet */
+#define IXGBE_RXDADV_STAT_FLM           IXGBE_RXD_STAT_FLM /* FDir Match */
+#define IXGBE_RXDADV_STAT_VP            IXGBE_RXD_STAT_VP  /* IEEE VLAN Pkt */
+#define IXGBE_RXDADV_STAT_MASK          0x000fffff /* Stat/NEXTP: bit 0-19 */
+#define IXGBE_RXDADV_STAT_FCEOFS        0x00000040 /* FCoE EOF/SOF Stat */
+#define IXGBE_RXDADV_STAT_FCSTAT        0x00000030 /* FCoE Pkt Stat */
+#define IXGBE_RXDADV_STAT_FCSTAT_NOMTCH 0x00000000 /* 00: No Ctxt Match */
+#define IXGBE_RXDADV_STAT_FCSTAT_NODDP  0x00000010 /* 01: Ctxt w/o DDP */
+#define IXGBE_RXDADV_STAT_FCSTAT_FCPRSP 0x00000020 /* 10: Recv. FCP_RSP */
+#define IXGBE_RXDADV_STAT_FCSTAT_DDP    0x00000030 /* 11: Ctxt w/ DDP */
+
+/* PSRTYPE bit definitions */
+#define IXGBE_PSRTYPE_TCPHDR    0x00000010
+#define IXGBE_PSRTYPE_UDPHDR    0x00000020
+#define IXGBE_PSRTYPE_IPV4HDR   0x00000100
+#define IXGBE_PSRTYPE_IPV6HDR   0x00000200
 
 /* SRRCTL bit definitions */
 #define IXGBE_SRRCTL_BSIZEPKT_SHIFT     10     /* so many KBs */
+#define IXGBE_SRRCTL_RDMTS_SHIFT        22
+#define IXGBE_SRRCTL_RDMTS_MASK         0x01C00000
+#define IXGBE_SRRCTL_DROP_EN            0x10000000
 #define IXGBE_SRRCTL_BSIZEPKT_MASK      0x0000007F
 #define IXGBE_SRRCTL_BSIZEHDR_MASK      0x00003F00
 #define IXGBE_SRRCTL_DESCTYPE_LEGACY    0x00000000
@@ -1110,7 +1802,10 @@
 
 #define IXGBE_RXDADV_RSSTYPE_MASK       0x0000000F
 #define IXGBE_RXDADV_PKTTYPE_MASK       0x0000FFF0
+#define IXGBE_RXDADV_PKTTYPE_MASK_EX    0x0001FFF0
 #define IXGBE_RXDADV_HDRBUFLEN_MASK     0x00007FE0
+#define IXGBE_RXDADV_RSCCNT_MASK        0x001E0000
+#define IXGBE_RXDADV_RSCCNT_SHIFT       17
 #define IXGBE_RXDADV_HDRBUFLEN_SHIFT    5
 #define IXGBE_RXDADV_SPLITHEADER_EN     0x00001000
 #define IXGBE_RXDADV_SPH                0x8000
@@ -1137,6 +1832,20 @@
 #define IXGBE_RXDADV_PKTTYPE_UDP        0x00000200 /* UDP hdr present */
 #define IXGBE_RXDADV_PKTTYPE_SCTP       0x00000400 /* SCTP hdr present */
 #define IXGBE_RXDADV_PKTTYPE_NFS        0x00000800 /* NFS hdr present */
+#define IXGBE_RXDADV_PKTTYPE_IPSEC_ESP  0x00001000 /* IPSec ESP */
+#define IXGBE_RXDADV_PKTTYPE_IPSEC_AH   0x00002000 /* IPSec AH */
+#define IXGBE_RXDADV_PKTTYPE_LINKSEC    0x00004000 /* LinkSec Encap */
+#define IXGBE_RXDADV_PKTTYPE_ETQF       0x00008000 /* PKTTYPE is ETQF index */
+#define IXGBE_RXDADV_PKTTYPE_ETQF_MASK  0x00000070 /* ETQF has 8 indices */
+#define IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT 4          /* Right-shift 4 bits */
+
+/* Security Processing bit Indication */
+#define IXGBE_RXDADV_LNKSEC_STATUS_SECP         0x00020000
+#define IXGBE_RXDADV_LNKSEC_ERROR_NO_SA_MATCH   0x08000000
+#define IXGBE_RXDADV_LNKSEC_ERROR_REPLAY_ERROR  0x10000000
+#define IXGBE_RXDADV_LNKSEC_ERROR_BIT_MASK      0x18000000
+#define IXGBE_RXDADV_LNKSEC_ERROR_BAD_SIG       0x18000000
+
 /* Masks to determine if packets should be dropped due to frame errors */
 #define IXGBE_RXD_ERR_FRAME_ERR_MASK ( \
                                       IXGBE_RXD_ERR_CE | \
@@ -1166,10 +1875,14 @@
 #define IXGBE_RX_DESC_SPECIAL_PRI_SHIFT  0x000D /* Priority in upper 3 of 16 */
 #define IXGBE_TX_DESC_SPECIAL_PRI_SHIFT  IXGBE_RX_DESC_SPECIAL_PRI_SHIFT
 
-#ifndef __le16
 /* Little Endian defines */
+#ifndef __le16
 #define __le16  u16
+#endif
+#ifndef __le32
 #define __le32  u32
+#endif
+#ifndef __le64
 #define __le64  u64
 
 #endif
@@ -1180,6 +1893,77 @@
 #define __be64  u64
 
 #endif
+enum ixgbe_fdir_pballoc_type {
+	IXGBE_FDIR_PBALLOC_64K = 0,
+	IXGBE_FDIR_PBALLOC_128K,
+	IXGBE_FDIR_PBALLOC_256K,
+};
+#define IXGBE_FDIR_PBALLOC_SIZE_SHIFT           16
+
+/* Flow Director register values */
+#define IXGBE_FDIRCTRL_PBALLOC_64K              0x00000001
+#define IXGBE_FDIRCTRL_PBALLOC_128K             0x00000002
+#define IXGBE_FDIRCTRL_PBALLOC_256K             0x00000003
+#define IXGBE_FDIRCTRL_INIT_DONE                0x00000008
+#define IXGBE_FDIRCTRL_PERFECT_MATCH            0x00000010
+#define IXGBE_FDIRCTRL_REPORT_STATUS            0x00000020
+#define IXGBE_FDIRCTRL_REPORT_STATUS_ALWAYS     0x00000080
+#define IXGBE_FDIRCTRL_DROP_Q_SHIFT             8
+#define IXGBE_FDIRCTRL_FLEX_SHIFT               16
+#define IXGBE_FDIRCTRL_SEARCHLIM                0x00800000
+#define IXGBE_FDIRCTRL_MAX_LENGTH_SHIFT         24
+#define IXGBE_FDIRCTRL_FULL_THRESH_MASK         0xF0000000
+#define IXGBE_FDIRCTRL_FULL_THRESH_SHIFT        28
+
+#define IXGBE_FDIRTCPM_DPORTM_SHIFT             16
+#define IXGBE_FDIRUDPM_DPORTM_SHIFT             16
+#define IXGBE_FDIRIP6M_DIPM_SHIFT               16
+#define IXGBE_FDIRM_VLANID                      0x00000001
+#define IXGBE_FDIRM_VLANP                       0x00000002
+#define IXGBE_FDIRM_POOL                        0x00000004
+#define IXGBE_FDIRM_L3P                         0x00000008
+#define IXGBE_FDIRM_L4P                         0x00000010
+#define IXGBE_FDIRM_FLEX                        0x00000020
+#define IXGBE_FDIRM_DIPv6                       0x00000040
+
+#define IXGBE_FDIRFREE_FREE_MASK                0xFFFF
+#define IXGBE_FDIRFREE_FREE_SHIFT               0
+#define IXGBE_FDIRFREE_COLL_MASK                0x7FFF0000
+#define IXGBE_FDIRFREE_COLL_SHIFT               16
+#define IXGBE_FDIRLEN_MAXLEN_MASK               0x3F
+#define IXGBE_FDIRLEN_MAXLEN_SHIFT              0
+#define IXGBE_FDIRLEN_MAXHASH_MASK              0x7FFF0000
+#define IXGBE_FDIRLEN_MAXHASH_SHIFT             16
+#define IXGBE_FDIRUSTAT_ADD_MASK                0xFFFF
+#define IXGBE_FDIRUSTAT_ADD_SHIFT               0
+#define IXGBE_FDIRUSTAT_REMOVE_MASK             0xFFFF0000
+#define IXGBE_FDIRUSTAT_REMOVE_SHIFT            16
+#define IXGBE_FDIRFSTAT_FADD_MASK               0x00FF
+#define IXGBE_FDIRFSTAT_FADD_SHIFT              0
+#define IXGBE_FDIRFSTAT_FREMOVE_MASK            0xFF00
+#define IXGBE_FDIRFSTAT_FREMOVE_SHIFT           8
+#define IXGBE_FDIRPORT_DESTINATION_SHIFT        16
+#define IXGBE_FDIRVLAN_FLEX_SHIFT               16
+#define IXGBE_FDIRHASH_BUCKET_VALID_SHIFT       15
+#define IXGBE_FDIRHASH_SIG_SW_INDEX_SHIFT       16
+
+#define IXGBE_FDIRCMD_CMD_ADD_FLOW              0x00000001
+#define IXGBE_FDIRCMD_CMD_REMOVE_FLOW           0x00000002
+#define IXGBE_FDIRCMD_CMD_QUERY_REM_FILT        0x00000003
+#define IXGBE_FDIRCMD_CMD_QUERY_REM_HASH        0x00000007
+#define IXGBE_FDIRCMD_FILTER_UPDATE             0x00000008
+#define IXGBE_FDIRCMD_IPv6DMATCH                0x00000010
+#define IXGBE_FDIRCMD_L4TYPE_UDP                0x00000020
+#define IXGBE_FDIRCMD_L4TYPE_TCP                0x00000040
+#define IXGBE_FDIRCMD_L4TYPE_SCTP               0x00000060
+#define IXGBE_FDIRCMD_IPV6                      0x00000080
+#define IXGBE_FDIRCMD_DROP                      0x00000200
+#define IXGBE_FDIRCMD_INT                       0x00000400
+#define IXGBE_FDIRCMD_LAST                      0x00000800
+#define IXGBE_FDIRCMD_COLLISION                 0x00001000
+#define IXGBE_FDIRCMD_QUEUE_EN                  0x00008000
+#define IXGBE_FDIRCMD_RX_QUEUE_SHIFT            16
+#define IXGBE_FDIRCMD_VT_POOL_SHIFT             24
 
 /* Transmit Descriptor - Legacy */
 struct ixgbe_legacy_tx_desc {
@@ -1267,6 +2051,9 @@ struct ixgbe_adv_tx_context_desc {
 
 /* Adv Transmit Descriptor Config Masks */
 #define IXGBE_ADVTXD_DTALEN_MASK      0x0000FFFF /* Data buf length(bytes) */
+#define IXGBE_ADVTXD_MAC_LINKSEC      0x00040000 /* Insert LinkSec */
+#define IXGBE_ADVTXD_IPSEC_SA_INDEX_MASK   0x000003FF /* IPSec SA index */
+#define IXGBE_ADVTXD_IPSEC_ESP_LEN_MASK    0x000001FF /* IPSec ESP length */
 #define IXGBE_ADVTXD_DTYP_MASK  0x00F00000 /* DTYP mask */
 #define IXGBE_ADVTXD_DTYP_CTXT  0x00200000 /* Advanced Context Desc */
 #define IXGBE_ADVTXD_DTYP_DATA  0x00300000 /* Advanced Data Descriptor */
@@ -1301,6 +2088,19 @@ struct ixgbe_adv_tx_context_desc {
 #define IXGBE_ADVTXD_TUCMD_L4T_TCP   0x00000800  /* L4 Packet TYPE of TCP */
 #define IXGBE_ADVTXD_TUCMD_L4T_SCTP  0x00001000  /* L4 Packet TYPE of SCTP */
 #define IXGBE_ADVTXD_TUCMD_MKRREQ    0x00002000 /*Req requires Markers and CRC*/
+#define IXGBE_ADVTXD_POPTS_IPSEC      0x00000400 /* IPSec offload request */
+#define IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP 0x00002000 /* IPSec Type ESP */
+#define IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN 0x00004000/* ESP Encrypt Enable */
+#define IXGBE_ADVTXT_TUCMD_FCOE      0x00008000       /* FCoE Frame Type */
+#define IXGBE_ADVTXD_FCOEF_EOF_MASK  (0x3 << 10)      /* FC EOF index */
+#define IXGBE_ADVTXD_FCOEF_SOF       ((1 << 2) << 10) /* FC SOF index */
+#define IXGBE_ADVTXD_FCOEF_PARINC    ((1 << 3) << 10) /* Rel_Off in F_CTL */
+#define IXGBE_ADVTXD_FCOEF_ORIE      ((1 << 4) << 10) /* Orientation: End */
+#define IXGBE_ADVTXD_FCOEF_ORIS      ((1 << 5) << 10) /* Orientation: Start */
+#define IXGBE_ADVTXD_FCOEF_EOF_N     (0x0 << 10)      /* 00: EOFn */
+#define IXGBE_ADVTXD_FCOEF_EOF_T     (0x1 << 10)      /* 01: EOFt */
+#define IXGBE_ADVTXD_FCOEF_EOF_NI    (0x2 << 10)      /* 10: EOFni */
+#define IXGBE_ADVTXD_FCOEF_EOF_A     (0x3 << 10)      /* 11: EOFa */
 #define IXGBE_ADVTXD_L4LEN_SHIFT     8  /* Adv ctxt L4LEN shift */
 #define IXGBE_ADVTXD_MSS_SHIFT       16  /* Adv ctxt MSS shift */
 
@@ -1314,13 +2114,17 @@ typedef u32 ixgbe_link_speed;
 #define IXGBE_LINK_SPEED_10GB_FULL 0x0080
 #define IXGBE_LINK_SPEED_82598_AUTONEG (IXGBE_LINK_SPEED_1GB_FULL | \
                                         IXGBE_LINK_SPEED_10GB_FULL)
+#define IXGBE_LINK_SPEED_82599_AUTONEG (IXGBE_LINK_SPEED_100_FULL | \
+                                        IXGBE_LINK_SPEED_1GB_FULL | \
+                                        IXGBE_LINK_SPEED_10GB_FULL)
+
 
 /* Physical layer type */
 typedef u32 ixgbe_physical_layer;
 #define IXGBE_PHYSICAL_LAYER_UNKNOWN      0
 #define IXGBE_PHYSICAL_LAYER_10GBASE_T    0x0001
 #define IXGBE_PHYSICAL_LAYER_1000BASE_T   0x0002
-#define IXGBE_PHYSICAL_LAYER_100BASE_T    0x0004
+#define IXGBE_PHYSICAL_LAYER_100BASE_TX   0x0004
 #define IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU  0x0008
 #define IXGBE_PHYSICAL_LAYER_10GBASE_LR   0x0010
 #define IXGBE_PHYSICAL_LAYER_10GBASE_LRM  0x0020
@@ -1329,7 +2133,46 @@ typedef u32 ixgbe_physical_layer;
 #define IXGBE_PHYSICAL_LAYER_10GBASE_CX4  0x0100
 #define IXGBE_PHYSICAL_LAYER_1000BASE_KX  0x0200
 #define IXGBE_PHYSICAL_LAYER_1000BASE_BX  0x0400
-
+#define IXGBE_PHYSICAL_LAYER_10GBASE_KR   0x0800
+
+/* Software ATR hash keys */
+#define IXGBE_ATR_BUCKET_HASH_KEY    0xE214AD3D
+#define IXGBE_ATR_SIGNATURE_HASH_KEY 0x14364D17
+
+/* Software ATR input stream offsets and masks */
+#define IXGBE_ATR_VLAN_OFFSET       0
+#define IXGBE_ATR_SRC_IPV6_OFFSET   2
+#define IXGBE_ATR_SRC_IPV4_OFFSET  14
+#define IXGBE_ATR_DST_IPV6_OFFSET  18
+#define IXGBE_ATR_DST_IPV4_OFFSET  30
+#define IXGBE_ATR_SRC_PORT_OFFSET  34
+#define IXGBE_ATR_DST_PORT_OFFSET  36
+#define IXGBE_ATR_FLEX_BYTE_OFFSET 38
+#define IXGBE_ATR_VM_POOL_OFFSET   40
+#define IXGBE_ATR_L4TYPE_OFFSET    41
+
+#define IXGBE_ATR_L4TYPE_MASK      0x3
+#define IXGBE_ATR_L4TYPE_IPV6_MASK 0x4
+#define IXGBE_ATR_L4TYPE_UDP       0x1
+#define IXGBE_ATR_L4TYPE_TCP       0x2
+#define IXGBE_ATR_L4TYPE_SCTP      0x3
+#define IXGBE_ATR_HASH_MASK     0x7fff
+
+/* Flow Director ATR input struct. */
+struct ixgbe_atr_input {
+	/* Byte layout in order, all values with MSB first:
+	 *
+	 * vlan_id    - 2 bytes
+	 * src_ip     - 16 bytes
+	 * dst_ip     - 16 bytes
+	 * src_port   - 2 bytes
+	 * dst_port   - 2 bytes
+	 * flex_bytes - 2 bytes
+	 * vm_pool    - 1 byte
+	 * l4type     - 1 byte
+	 */
+	u8 byte_stream[42];
+};
 
 enum ixgbe_eeprom_type {
 	ixgbe_eeprom_uninitialized = 0,
@@ -1340,12 +2183,16 @@ enum ixgbe_eeprom_type {
 enum ixgbe_mac_type {
 	ixgbe_mac_unknown = 0,
 	ixgbe_mac_82598EB,
+	ixgbe_mac_82599EB,
 	ixgbe_num_macs
 };
 
 enum ixgbe_phy_type {
 	ixgbe_phy_unknown = 0,
+	ixgbe_phy_none,
 	ixgbe_phy_tn,
+	ixgbe_phy_aq,
+	ixgbe_phy_cu_unknown,
 	ixgbe_phy_qt,
 	ixgbe_phy_xaui,
 	ixgbe_phy_nl,
@@ -1354,6 +2201,8 @@ enum ixgbe_phy_type {
 	ixgbe_phy_sfp_avago,
 	ixgbe_phy_sfp_ftl,
 	ixgbe_phy_sfp_unknown,
+	ixgbe_phy_sfp_intel,
+	ixgbe_phy_sfp_unsupported, /*Enforce bit set with unsupported module*/
 	ixgbe_phy_generic
 };
 
@@ -1365,11 +2214,19 @@ enum ixgbe_phy_type {
  * 0	SFP_DA_CU
  * 1	SFP_SR
  * 2	SFP_LR
+ * 3    SFP_DA_CU_CORE0 - 82599-specific
+ * 4    SFP_DA_CU_CORE1 - 82599-specific
+ * 5    SFP_SR/LR_CORE0 - 82599-specific
+ * 6    SFP_SR/LR_CORE1 - 82599-specific
  */
 enum ixgbe_sfp_type {
 	ixgbe_sfp_type_da_cu = 0,
 	ixgbe_sfp_type_sr = 1,
 	ixgbe_sfp_type_lr = 2,
+	ixgbe_sfp_type_da_cu_core0 = 3,
+	ixgbe_sfp_type_da_cu_core1 = 4,
+	ixgbe_sfp_type_srlr_core0 = 5,
+	ixgbe_sfp_type_srlr_core1 = 6,
 	ixgbe_sfp_type_not_present = 0xFFFE,
 	ixgbe_sfp_type_unknown = 0xFFFF
 };
@@ -1428,7 +2285,6 @@ enum ixgbe_bus_width {
 struct ixgbe_addr_filter_info {
 	u32 num_mc_addrs;
 	u32 rar_used_count;
-	u32 mc_addr_in_rar_count;
 	u32 mta_in_use;
 	u32 overflow_promisc;
 	bool user_set_promisc;
@@ -1441,6 +2297,7 @@ struct ixgbe_bus_info {
 	enum ixgbe_bus_type type;
 
 	u16 func;
+	u16 lan_id;
 };
 
 /* Flow control parameters */
@@ -1450,6 +2307,8 @@ struct ixgbe_fc_info {
 	u16 pause_time; /* Flow Control Pause timer */
 	bool send_xon; /* Flow control send XON */
 	bool strict_ieee; /* Strict IEEE mode */
+	bool disable_fc_autoneg; /* Do not autonegotiate FC */
+	bool fc_was_autonegged; /* Is current_mode the result of autonegging? */
 	enum ixgbe_fc_mode current_mode; /* FC mode in effect */
 	enum ixgbe_fc_mode requested_mode; /* FC mode requested by caller */
 };
@@ -1511,6 +2370,21 @@ struct ixgbe_hw_stats {
 	u64 qptc[16];
 	u64 qbrc[16];
 	u64 qbtc[16];
+	u64 qprdc[16];
+	u64 pxon2offc[8];
+	u64 fdirustat_add;
+	u64 fdirustat_remove;
+	u64 fdirfstat_fadd;
+	u64 fdirfstat_fremove;
+	u64 fdirmatch;
+	u64 fdirmiss;
+	u64 fccrc;
+	u64 fclast;
+	u64 fcoerpdc;
+	u64 fcoeprc;
+	u64 fcoeptc;
+	u64 fcoedwrc;
+	u64 fcoedwtc;
 };
 
 /* forward declaration */
@@ -1537,11 +2411,16 @@ struct ixgbe_mac_operations {
 	enum ixgbe_media_type (*get_media_type)(struct ixgbe_hw *);
 	u32 (*get_supported_physical_layer)(struct ixgbe_hw *);
 	s32 (*get_mac_addr)(struct ixgbe_hw *, u8 *);
+	s32 (*get_san_mac_addr)(struct ixgbe_hw *, u8 *);
+	s32 (*set_san_mac_addr)(struct ixgbe_hw *, u8 *);
+	s32 (*get_device_caps)(struct ixgbe_hw *, u16 *);
 	s32 (*stop_adapter)(struct ixgbe_hw *);
 	s32 (*get_bus_info)(struct ixgbe_hw *);
 	void (*set_lan_id)(struct ixgbe_hw *);
 	s32 (*read_analog_reg8)(struct ixgbe_hw*, u32, u8*);
 	s32 (*write_analog_reg8)(struct ixgbe_hw*, u32, u8);
+	s32 (*setup_sfp)(struct ixgbe_hw *);
+	s32 (*enable_rx_dma)(struct ixgbe_hw *, u32);
 
 	/* Link */
 	s32 (*setup_link)(struct ixgbe_hw *);
@@ -1560,6 +2439,7 @@ struct ixgbe_mac_operations {
 	/* RAR, Multicast, VLAN */
 	s32 (*set_rar)(struct ixgbe_hw *, u32, u8 *, u32, u32);
 	s32 (*clear_rar)(struct ixgbe_hw *, u32);
+	s32 (*insert_mac_addr)(struct ixgbe_hw *, u8 *, u32);
 	s32 (*set_vmdq)(struct ixgbe_hw *, u32, u32);
 	s32 (*clear_vmdq)(struct ixgbe_hw *, u32, u32);
 	s32 (*init_rx_addrs)(struct ixgbe_hw *);
@@ -1574,12 +2454,13 @@ struct ixgbe_mac_operations {
 	s32 (*init_uta_tables)(struct ixgbe_hw *);
 
 	/* Flow Control */
-	s32 (*setup_fc)(struct ixgbe_hw *, s32);
+	s32 (*fc_enable)(struct ixgbe_hw *, s32);
 };
 
 struct ixgbe_phy_operations {
 	s32 (*identify)(struct ixgbe_hw *);
 	s32 (*identify_sfp)(struct ixgbe_hw *);
+	s32 (*init)(struct ixgbe_hw *);
 	s32 (*reset)(struct ixgbe_hw *);
 	s32 (*read_reg)(struct ixgbe_hw *, u32, u32, u16 *);
 	s32 (*write_reg)(struct ixgbe_hw *, u32, u32, u16);
@@ -1592,12 +2473,13 @@ struct ixgbe_phy_operations {
 	s32 (*write_i2c_byte)(struct ixgbe_hw *, u8, u8, u8);
 	s32 (*read_i2c_eeprom)(struct ixgbe_hw *, u8 , u8 *);
 	s32 (*write_i2c_eeprom)(struct ixgbe_hw *, u8, u8);
+	void (*i2c_bus_clear)(struct ixgbe_hw *);
 };
 
 struct ixgbe_eeprom_info {
 	struct ixgbe_eeprom_operations  ops;
 	enum ixgbe_eeprom_type          type;
-	u32				semaphore_delay;
+	u32                             semaphore_delay;
 	u16                             word_size;
 	u16                             address_bits;
 };
@@ -1607,16 +2489,19 @@ struct ixgbe_mac_info {
 	enum ixgbe_mac_type             type;
 	u8                              addr[IXGBE_ETH_LENGTH_OF_ADDRESS];
 	u8                              perm_addr[IXGBE_ETH_LENGTH_OF_ADDRESS];
+	u8                              san_addr[IXGBE_ETH_LENGTH_OF_ADDRESS];
 	s32                             mc_filter_type;
 	u32                             mcft_size;
 	u32                             vft_size;
 	u32                             num_rar_entries;
+	u32                             rar_highwater;
 	u32                             max_tx_queues;
 	u32                             max_rx_queues;
-	u32                             link_attach_type;
-	u32                             link_mode_select;
-	u32                             link_kx4_kx_supp;
-	bool                            link_settings_loaded;
+	u32                             max_msix_vectors;
+	bool                            msix_vectors_from_pcie;
+	u32                             orig_autoc;
+	u32                             orig_autoc2;
+	bool                            orig_link_settings_stored;
 	bool                            autoneg;
 	bool                            autoneg_succeeded;
 };
@@ -1627,6 +2512,7 @@ struct ixgbe_phy_info {
 	u32                             addr;
 	u32                             id;
 	enum ixgbe_sfp_type             sfp_type;
+	bool                            sfp_setup_needed;
 	u32                             revision;
 	enum ixgbe_media_type           media_type;
 	bool                            reset_disable;
@@ -1677,6 +2563,8 @@ struct ixgbe_hw {
 #define IXGBE_ERR_I2C                           -18
 #define IXGBE_ERR_SFP_NOT_SUPPORTED             -19
 #define IXGBE_ERR_SFP_NOT_PRESENT               -20
+#define IXGBE_ERR_SFP_NO_INIT_SEQ_PRESENT       -21
+#define IXGBE_ERR_NO_SAN_ADDR_PTR               -22
 #define IXGBE_NOT_IMPLEMENTED                   0x7FFFFFFF
 
 #define UNREFERENCED_PARAMETER(_p)
diff --git a/sys/dev/kbd/kbdreg.h b/sys/dev/kbd/kbdreg.h
index 5b81123..ff4a694 100644
--- a/sys/dev/kbd/kbdreg.h
+++ b/sys/dev/kbd/kbdreg.h
@@ -60,6 +60,7 @@ struct keyboard {
 #define KB_INITIALIZED	(1 << 19)	/* device initialized */
 #define KB_REGISTERED	(1 << 20)	/* device registered to kbdio */
 #define KB_BUSY		(1 << 21)	/* device used by a client */
+#define KB_POLLED	(1 << 22)	/* device is polled */
 	int		kb_active;	/* 0: inactive */
 	void		*kb_token;	/* id of the current client */
 	keyboard_callback_t kb_callback;/* callback function */
@@ -107,6 +108,9 @@ struct keyboard {
 #define KBD_IS_BUSY(k)		((k)->kb_flags & KB_BUSY)
 #define KBD_BUSY(k)		((k)->kb_flags |= KB_BUSY)
 #define KBD_UNBUSY(k)		((k)->kb_flags &= ~KB_BUSY)
+#define KBD_IS_POLLED(k)	((k)->kb_flags & KB_POLLED)
+#define KBD_POLL(k)		((k)->kb_flags |= KB_POLLED)
+#define KBD_UNPOLL(k)		((k)->kb_flags &= ~KB_POLLED)
 #define KBD_IS_ACTIVE(k)	((k)->kb_active)
 #define KBD_ACTIVATE(k)		(++(k)->kb_active)
 #define KBD_DEACTIVATE(k)	(--(k)->kb_active)
@@ -170,7 +174,7 @@ typedef struct keyboard_switch {
 	(*kbdsw[(kbd)->kb_index]->intr)((kbd), (arg))
 #define kbdd_test_if(kbd)						\
 	(*kbdsw[(kbd)->kb_index]->test_if)((kbd))
-#define kbdd_enable(kbd)							\
+#define kbdd_enable(kbd)						\
 	(*kbdsw[(kbd)->kb_index]->enable)((kbd))
 #define kbdd_disable(kbd)						\
 	(*kbdsw[(kbd)->kb_index]->disable)((kbd))
@@ -194,7 +198,7 @@ typedef struct keyboard_switch {
 	(*kbdsw[(kbd)->kb_index]->get_state)((kbd), (buf), (len))
 #define kbdd_set_state(kbd, buf, len)					\
 	(*kbdsw[(kbd)->kb_index]->set_state)((kbd), (buf), (len))
-#define kbdd_get_fkeystr(kbd, fkey, len)					\
+#define kbdd_get_fkeystr(kbd, fkey, len)				\
 	(*kbdsw[(kbd)->kb_index]->get_fkeystr)((kbd), (fkey), (len))
 #define kbdd_poll(kbd, on)						\
 	(*kbdsw[(kbd)->kb_index]->poll)((kbd), (on))
diff --git a/sys/dev/kbdmux/kbdmux.c b/sys/dev/kbdmux/kbdmux.c
index 07b592f..ac6095c 100644
--- a/sys/dev/kbdmux/kbdmux.c
+++ b/sys/dev/kbdmux/kbdmux.c
@@ -104,10 +104,10 @@ MALLOC_DEFINE(M_KBDMUX, KEYBOARD_NAME, "Keyboard multiplexor");
 
 #define KBDMUX_LOCK_DESTROY(s)
 
-#define KBDMUX_LOCK(s) \
-	mtx_lock(&Giant)
-#define KBDMUX_UNLOCK(s) \
-	mtx_unlock(&Giant)
+#define KBDMUX_LOCK(s)
+
+#define KBDMUX_UNLOCK(s)
+
 #define KBDMUX_LOCK_ASSERT(s, w)
 
 #define KBDMUX_SLEEP(s, f, d, t) \
diff --git a/sys/dev/malo/if_malo.c b/sys/dev/malo/if_malo.c
index 681c824..c17e98c 100644
--- a/sys/dev/malo/if_malo.c
+++ b/sys/dev/malo/if_malo.c
@@ -175,16 +175,10 @@ malo_bar0_write4(struct malo_softc *sc, bus_size_t off, uint32_t val)
 	bus_space_write_4(sc->malo_io0t, sc->malo_io0h, off, val);
 }
 
-static uint8_t
-malo_bar1_read1(struct malo_softc *sc, bus_size_t off)
-{
-	return bus_space_read_1(sc->malo_io1t, sc->malo_io1h, off);
-}
-
 int
 malo_attach(uint16_t devid, struct malo_softc *sc)
 {
-	int error, i;
+	int error;
 	struct ieee80211com *ic;
 	struct ifnet *ifp;
 	struct malo_hal *mh;
@@ -203,16 +197,6 @@ malo_attach(uint16_t devid, struct malo_softc *sc)
 	if_initname(ifp, device_get_name(sc->malo_dev),
 	    device_get_unit(sc->malo_dev));
 
-	/*
-	 * NB: get mac address from hardware directly here before we set DMAs
-	 * for HAL because we don't want to disturb operations of HAL at BAR 1.
-	 */
-	for (i = 0; i < IEEE80211_ADDR_LEN; i++) {
-		/* XXX remove a magic number but we don't have documents.  */
-		ic->ic_myaddr[i] = malo_bar1_read1(sc, 0xa528 + i);
-		DELAY(1000);
-	}
-
 	mh = malo_hal_attach(sc->malo_dev, devid,
 	    sc->malo_io1h, sc->malo_io1t, sc->malo_dmat);
 	if (mh == NULL) {
@@ -276,7 +260,7 @@ malo_attach(uint16_t devid, struct malo_softc *sc)
 	}
 	error = malo_setup_hwdma(sc);	/* push to firmware */
 	if (error != 0)			/* NB: malo_setupdma prints msg */
-		goto bad1;
+		goto bad2;
 
 	sc->malo_tq = taskqueue_create_fast("malo_taskq", M_NOWAIT,
 		taskqueue_thread_enqueue, &sc->malo_tq);
@@ -319,11 +303,8 @@ malo_attach(uint16_t devid, struct malo_softc *sc)
 	ic->ic_headroom = sizeof(struct malo_txrec) -
 		sizeof(struct ieee80211_frame);
 
-	/* get mac address from hardware */
-	IEEE80211_ADDR_COPY(ic->ic_myaddr, sc->malo_hwspecs.macaddr);
-
 	/* call MI attach routine. */
-	ieee80211_ifattach(ic);
+	ieee80211_ifattach(ic, sc->malo_hwspecs.macaddr);
 	/* override default methods */
 	ic->ic_vap_create = malo_vap_create;
 	ic->ic_vap_delete = malo_vap_delete;
@@ -348,6 +329,8 @@ malo_attach(uint16_t devid, struct malo_softc *sc)
 	malo_announce(sc);
 
 	return 0;
+bad2:
+	malo_dma_cleanup(sc);
 bad1:
 	malo_hal_detach(mh);
 bad:
@@ -1307,20 +1290,9 @@ malo_start(struct ifnet *ifp)
 			break;
 		}
 		/*
-		 * Encapsulate the packet in prep for transmission.
-		 */
-		m = ieee80211_encap(ni, m);
-		if (m == NULL) {
-			DPRINTF(sc, MALO_DEBUG_XMIT,
-			    "%s: encapsulation failure\n", __func__);
-			sc->malo_stats.mst_tx_encap++;
-			goto bad;
-		}
-		/*
 		 * Pass the frame to the h/w for transmission.
 		 */
 		if (malo_tx_start(sc, ni, bf, m)) {
-	bad:
 			ifp->if_oerrors++;
 			if (bf != NULL) {
 				bf->bf_m = NULL;
@@ -1636,14 +1608,6 @@ malo_mode_init(struct malo_softc *sc)
 	struct malo_hal *mh = sc->malo_mh;
 
 	/*
-	 * Handle any link-level address change.  Note that we only
-	 * need to force ic_myaddr; any other addresses are handled
-	 * as a byproduct of the ifnet code marking the interface
-	 * down then up.
-	 */
-	IEEE80211_ADDR_COPY(ic->ic_myaddr, IF_LLADDR(ifp));
-
-	/*
 	 * NB: Ignore promisc in hostap mode; it's set by the
 	 * bridge.  This is wrong but we have no way to
 	 * identify internal requests (from the bridge)
diff --git a/sys/dev/malo/if_malo_pci.c b/sys/dev/malo/if_malo_pci.c
index 8a3b346..2df6dd1 100644
--- a/sys/dev/malo/if_malo_pci.c
+++ b/sys/dev/malo/if_malo_pci.c
@@ -245,7 +245,7 @@ malo_pci_attach(device_t dev)
 			       BUS_SPACE_MAXADDR,	/* maxsize */
 			       0,			/* nsegments */
 			       BUS_SPACE_MAXADDR,	/* maxsegsize */
-			       BUS_DMA_ALLOCNOW,	/* flags */
+			       0,			/* flags */
 			       NULL,			/* lockfunc */
 			       NULL,			/* lockarg */
 			       &sc->malo_dmat)) {
@@ -260,12 +260,13 @@ malo_pci_attach(device_t dev)
 
 	error = malo_attach(pci_get_device(dev), sc);
 
-	if (error != 0) {
-		malo_pci_detach(dev);
-		return (error);
-	}
+	if (error != 0)
+		goto bad2;
 
 	return (error);
+
+bad2:
+	bus_dma_tag_destroy(sc->malo_dmat);
 bad1:
 	if (psc->malo_msi == 0)
 		bus_teardown_intr(dev, psc->malo_res_irq[0],
@@ -275,10 +276,11 @@ bad1:
 			bus_teardown_intr(dev, psc->malo_res_irq[i],
 			    psc->malo_intrhand[i]);
 	}
-
+	bus_release_resources(dev, psc->malo_irq_spec, psc->malo_res_irq);
 bad:
 	if (psc->malo_msi != 0)
 		pci_release_msi(dev);
+	bus_release_resources(dev, psc->malo_mem_spec, psc->malo_res_mem);
 
 	return (error);
 }
diff --git a/sys/dev/malo/if_malohal.c b/sys/dev/malo/if_malohal.c
index b14bff2..d244b98 100644
--- a/sys/dev/malo/if_malohal.c
+++ b/sys/dev/malo/if_malohal.c
@@ -128,7 +128,7 @@ malo_hal_attach(device_t dev, uint16_t devid,
 		       NULL,			/* lockarg */
 		       &mh->mh_dmat);
 	if (error != 0) {
-		device_printf(dev, "unable to allocate memory for cmd buffer, "
+		device_printf(dev, "unable to allocate memory for cmd tag, "
 			"error %u\n", error);
 		goto fail;
 	}
@@ -163,8 +163,6 @@ malo_hal_attach(device_t dev, uint16_t devid,
 	return (mh);
 
 fail:
-	free(mh, M_DEVBUF);
-
 	if (mh->mh_dmamap != NULL) {
 		bus_dmamap_unload(mh->mh_dmat, mh->mh_dmamap);
 		if (mh->mh_cmdbuf != NULL)
@@ -174,6 +172,7 @@ fail:
 	}
 	if (mh->mh_dmat)
 		bus_dma_tag_destroy(mh->mh_dmat);
+	free(mh, M_DEVBUF);
 
 	return (NULL);
 }
diff --git a/sys/dev/mii/axphy.c b/sys/dev/mii/axphy.c
new file mode 100644
index 0000000..21ac6f0
--- /dev/null
+++ b/sys/dev/mii/axphy.c
@@ -0,0 +1,207 @@
+/*-
+ * Copyright (c) 2009, M. Warner Losh
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * driver for internal phy in the AX88x9x chips.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/socket.h>
+#include <sys/bus.h>
+
+#include <net/if.h>
+#include <net/if_media.h>
+
+#include <dev/mii/mii.h>
+#include <dev/mii/miivar.h>
+#include "miidevs.h"
+
+#include <dev/mii/axphyreg.h>
+
+#include "miibus_if.h"
+
+static int 	axphy_probe(device_t dev);
+static int 	axphy_attach(device_t dev);
+
+static device_method_t axphy_methods[] = {
+	/* device interface */
+	DEVMETHOD(device_probe,		axphy_probe),
+	DEVMETHOD(device_attach,	axphy_attach),
+	DEVMETHOD(device_detach,	mii_phy_detach),
+	DEVMETHOD(device_shutdown,	bus_generic_shutdown),
+	{ 0, 0 }
+};
+
+static devclass_t axphy_devclass;
+
+static driver_t axphy_driver = {
+	"axphy",
+	axphy_methods,
+	sizeof(struct mii_softc)
+};
+
+DRIVER_MODULE(axphy, miibus, axphy_driver, axphy_devclass, 0, 0);
+
+static int	axphy_service(struct mii_softc *, struct mii_data *, int);
+static void	axphy_status(struct mii_softc *);
+
+static const struct mii_phydesc axphys[] = {
+	MII_PHY_DESC(ASIX, AX88X9X),
+	MII_PHY_END
+};
+
+static int
+axphy_probe(device_t dev)
+{
+
+	return (mii_phy_dev_probe(dev, axphys, BUS_PROBE_DEFAULT));
+}
+
+static int
+axphy_attach(device_t dev)
+{
+	struct mii_softc *sc;
+	struct mii_attach_args *ma;
+	struct mii_data *mii;
+
+	sc = device_get_softc(dev);
+	ma = device_get_ivars(dev);
+	sc->mii_dev = device_get_parent(dev);
+	mii = device_get_softc(sc->mii_dev);
+	LIST_INSERT_HEAD(&mii->mii_phys, sc, mii_list);
+
+	sc->mii_inst = mii->mii_instance;
+	sc->mii_phy = ma->mii_phyno;
+	sc->mii_service = axphy_service;
+	sc->mii_pdata = mii;
+	sc->mii_flags |= MIIF_NOISOLATE;
+	mii->mii_instance++;
+
+	mii_phy_reset(sc);
+
+	sc->mii_capabilities = PHY_READ(sc, MII_BMSR) & ma->mii_capmask;
+	device_printf(dev, " ");
+	mii_phy_add_media(sc);
+	printf("\n");
+
+	MIIBUS_MEDIAINIT(sc->mii_dev);
+	mii_phy_setmedia(sc);
+
+	return (0);
+}
+
+static int
+axphy_service(struct mii_softc *sc, struct mii_data *mii, int cmd)
+{
+	struct ifmedia_entry *ife = mii->mii_media.ifm_cur;
+	int reg;
+
+	switch (cmd) {
+	case MII_POLLSTAT:
+		if (IFM_INST(ife->ifm_media) != sc->mii_inst)
+			return (0);
+		break;
+
+	case MII_MEDIACHG:
+		if (IFM_INST(ife->ifm_media) != sc->mii_inst) {
+			reg = PHY_READ(sc, MII_BMCR);
+			PHY_WRITE(sc, MII_BMCR, reg | BMCR_ISO);
+			return (0);
+		}
+
+		/*
+		 * If the interface is not up, don't do anything.
+		 */
+		if ((mii->mii_ifp->if_flags & IFF_UP) == 0)
+			break;
+
+		mii_phy_setmedia(sc);
+		break;
+
+	case MII_TICK:
+		if (IFM_INST(ife->ifm_media) != sc->mii_inst)
+			return (0);
+		if (mii_phy_tick(sc) == EJUSTRETURN)
+			return (0);
+		break;
+	}
+
+	/* Update the media status. */
+	axphy_status(sc);
+
+	/* Callback if something changed. */
+	mii_phy_update(sc, cmd);
+	return (0);
+}
+
+static void
+axphy_status(struct mii_softc *sc)
+{
+	struct mii_data *mii = sc->mii_pdata;
+	struct ifmedia_entry *ife = mii->mii_media.ifm_cur;
+	int bmsr, bmcr;
+
+	mii->mii_media_status = IFM_AVALID;
+	mii->mii_media_active = IFM_ETHER;
+
+	bmsr = PHY_READ(sc, MII_BMSR) | PHY_READ(sc, MII_BMSR);
+	if (bmsr & BMSR_LINK)
+		mii->mii_media_status |= IFM_ACTIVE;
+
+	bmcr = PHY_READ(sc, MII_BMCR);
+	if (bmcr & BMCR_ISO) {
+		mii->mii_media_active |= IFM_NONE;
+		mii->mii_media_status = 0;
+		return;
+	}
+
+	if (bmcr & BMCR_LOOP)
+		mii->mii_media_active |= IFM_LOOP;
+
+	if (bmcr & BMCR_AUTOEN) {
+		if ((bmsr & BMSR_ACOMP) == 0) {
+			mii->mii_media_active |= IFM_NONE;
+			return;
+		}
+
+#if 0
+		scr = PHY_READ(sc, MII_AXPHY_SCR);
+		if (scr & SCR_S100)
+			mii->mii_media_active |= IFM_100_TX;
+		else
+			mii->mii_media_active |= IFM_10_T;
+		if (scr & SCR_FDX)
+			mii->mii_media_active |= IFM_FDX;
+#endif
+	} else
+		mii->mii_media_active = ife->ifm_media;
+}
diff --git a/sys/mips/atheros/apbvar.h b/sys/dev/mii/axphyreg.h
index e1d29f4..2d9e933 100644
--- a/sys/mips/atheros/apbvar.h
+++ b/sys/dev/mii/axphyreg.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2009, Oleksandr Tymoshenko <gonzo@FreeBSD.org>
+ * Copyright (c) 2009, M. Warner Losh
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -23,26 +23,6 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
+ *	$FreeBSD$
  */
-
-#ifndef _APBVAR_H_
-#define _APBVAR_H_
-
-#define	APB_IRQ_BASE		0
-#define	APB_IRQ_END		7
-#define	APB_NIRQS		8
-
-struct apb_softc {
-	struct rman		apb_irq_rman;
-	/* IRQ events structs for child devices */
-	struct intr_event	*sc_eventstab[APB_NIRQS];	
-	/* Resources and cookies for MIPS CPU INTs */
-	struct resource		*sc_misc_irq;
-	void			*sc_misc_ih;
-};
-
-struct apb_ivar {
-	struct resource_list	resources;
-};
-
-#endif /* _APBVAR_H_ */
diff --git a/sys/dev/mii/miidevs b/sys/dev/mii/miidevs
index 7cc8868..6bc07fa 100644
--- a/sys/dev/mii/miidevs
+++ b/sys/dev/mii/miidevs
@@ -52,6 +52,7 @@ $FreeBSD$
 oui AGERE			0x00a0bc	Agere Systems
 oui ALTIMA			0x0010a9	Altima Communications
 oui AMD				0x00001a	Advanced Micro Devices
+oui ASIX			0x00602e	Asix Semiconductor
 oui ATHEROS			0x001374	Atheros Communications
 oui BROADCOM			0x001018	Broadcom Corporation
 oui BROADCOM2			0x000af7	Broadcom Corporation
@@ -120,6 +121,9 @@ model AMD 79c973phy		0x0036 Am79c973 internal PHY
 model AMD 79c978		0x0039 Am79c978 HomePNA PHY
 model xxAMD 79C873		0x0000 Am79C873/DM9101 10/100 media interface
 
+/* Asix semiconductor PHYs. */
+model ASIX AX88X9X		0x0031 Ax88x9x internal PHY
+
 /* Atheros Communications/Attansic PHYs. */
 model ATHEROS F1		0x0001 Atheros F1 10/100/1000 PHY
 model ATHEROS F2		0x0002 Atheros F2 10/100 PHY
diff --git a/sys/dev/msk/if_msk.c b/sys/dev/msk/if_msk.c
index 2620f9f..17bced0 100644
--- a/sys/dev/msk/if_msk.c
+++ b/sys/dev/msk/if_msk.c
@@ -287,9 +287,8 @@ static int msk_miibus_writereg(device_t, int, int, int);
 static void msk_miibus_statchg(device_t);
 static void msk_link_task(void *, int);
 
-static void msk_setmulti(struct msk_if_softc *);
+static void msk_rxfilter(struct msk_if_softc *);
 static void msk_setvlan(struct msk_if_softc *, struct ifnet *);
-static void msk_setpromisc(struct msk_if_softc *);
 
 static void msk_stats_clear(struct msk_if_softc *);
 static void msk_stats_update(struct msk_if_softc *);
@@ -560,7 +559,7 @@ msk_link_task(void *arg, int pending)
 }
 
 static void
-msk_setmulti(struct msk_if_softc *sc_if)
+msk_rxfilter(struct msk_if_softc *sc_if)
 {
 	struct msk_softc *sc;
 	struct ifnet *ifp;
@@ -577,15 +576,14 @@ msk_setmulti(struct msk_if_softc *sc_if)
 
 	bzero(mchash, sizeof(mchash));
 	mode = GMAC_READ_2(sc, sc_if->msk_port, GM_RX_CTRL);
-	mode |= GM_RXCR_UCF_ENA;
-	if ((ifp->if_flags & (IFF_PROMISC | IFF_ALLMULTI)) != 0) {
-		if ((ifp->if_flags & IFF_PROMISC) != 0)
-			mode &= ~(GM_RXCR_UCF_ENA | GM_RXCR_MCF_ENA);
-		else if ((ifp->if_flags & IFF_ALLMULTI) != 0) {
-			mchash[0] = 0xffff;
-			mchash[1] = 0xffff;
-		}
+	if ((ifp->if_flags & IFF_PROMISC) != 0)
+		mode &= ~(GM_RXCR_UCF_ENA | GM_RXCR_MCF_ENA);
+	else if ((ifp->if_flags & IFF_ALLMULTI) != 0) {
+		mode |= GM_RXCR_UCF_ENA | GM_RXCR_MCF_ENA;
+		mchash[0] = 0xffff;
+		mchash[1] = 0xffff;
 	} else {
+		mode |= GM_RXCR_UCF_ENA;
 		IF_ADDR_LOCK(ifp);
 		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 			if (ifma->ifma_addr->sa_family != AF_LINK)
@@ -598,7 +596,8 @@ msk_setmulti(struct msk_if_softc *sc_if)
 			mchash[crc >> 5] |= 1 << (crc & 0x1f);
 		}
 		IF_ADDR_UNLOCK(ifp);
-		mode |= GM_RXCR_MCF_ENA;
+		if (mchash[0] != 0 || mchash[1] != 0)
+			mode |= GM_RXCR_MCF_ENA;
 	}
 
 	GMAC_WRITE_2(sc, sc_if->msk_port, GM_MC_ADDR_H1,
@@ -631,26 +630,6 @@ msk_setvlan(struct msk_if_softc *sc_if, struct ifnet *ifp)
 	}
 }
 
-static void
-msk_setpromisc(struct msk_if_softc *sc_if)
-{
-	struct msk_softc *sc;
-	struct ifnet *ifp;
-	uint16_t mode;
-
-	MSK_IF_LOCK_ASSERT(sc_if);
-
-	sc = sc_if->msk_softc;
-	ifp = sc_if->msk_ifp;
-
-	mode = GMAC_READ_2(sc, sc_if->msk_port, GM_RX_CTRL);
-	if (ifp->if_flags & IFF_PROMISC)
-		mode &= ~(GM_RXCR_UCF_ENA | GM_RXCR_MCF_ENA);
-	else
-		mode |= (GM_RXCR_UCF_ENA | GM_RXCR_MCF_ENA);
-	GMAC_WRITE_2(sc, sc_if->msk_port, GM_RX_CTRL, mode);
-}
-
 static int
 msk_init_rx_ring(struct msk_if_softc *sc_if)
 {
@@ -954,10 +933,8 @@ msk_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 		if ((ifp->if_flags & IFF_UP) != 0) {
 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
 				if (((ifp->if_flags ^ sc_if->msk_if_flags)
-				    & IFF_PROMISC) != 0) {
-					msk_setpromisc(sc_if);
-					msk_setmulti(sc_if);
-				}
+				    & (IFF_PROMISC | IFF_ALLMULTI)) != 0)
+					msk_rxfilter(sc_if);
 			} else {
 				if (sc_if->msk_detach == 0)
 					msk_init_locked(sc_if);
@@ -973,7 +950,7 @@ msk_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 	case SIOCDELMULTI:
 		MSK_IF_LOCK(sc_if);
 		if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
-			msk_setmulti(sc_if);
+			msk_rxfilter(sc_if);
 		MSK_IF_UNLOCK(sc_if);
 		break;
 	case SIOCGIFMEDIA:
@@ -3594,11 +3571,8 @@ msk_init_locked(struct msk_if_softc *sc_if)
 	CSR_WRITE_4(sc, MR_ADDR(sc_if->msk_port, RX_GMF_CTRL_T),
 	    GMF_OPER_ON | GMF_RX_F_FL_ON);
 
-	/* Set promiscuous mode. */
-	msk_setpromisc(sc_if);
-
-	/* Set multicast filter. */
-	msk_setmulti(sc_if);
+	/* Set receive filter. */
+	msk_rxfilter(sc_if);
 
 	/* Flush Rx MAC FIFO on any flow control or error. */
 	CSR_WRITE_4(sc, MR_ADDR(sc_if->msk_port, RX_GMF_FL_MSK),
@@ -4196,7 +4170,7 @@ msk_sysctl_node(struct msk_if_softc *sc_if)
 	    child, rx_pkts_too_long, "frames too long");
 	MSK_SYSCTL_STAT32(sc_if, ctx, "jabbers",
 	    child, rx_pkts_jabbers, "Jabber errors");
-	MSK_SYSCTL_STAT32(sc_if, ctx, "jabbers",
+	MSK_SYSCTL_STAT32(sc_if, ctx, "overflows",
 	    child, rx_fifo_oflows, "FIFO overflows");
 
 	tree = SYSCTL_ADD_NODE(ctx, schild, OID_AUTO, "tx", CTLFLAG_RD,
diff --git a/sys/dev/my/if_my.c b/sys/dev/my/if_my.c
index 65c0dcc..b25ae22 100644
--- a/sys/dev/my/if_my.c
+++ b/sys/dev/my/if_my.c
@@ -1700,7 +1700,7 @@ my_watchdog(struct ifnet * ifp)
 	my_init_locked(sc);
 	if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 		my_start_locked(ifp);
-	MY_LOCK(sc);
+	MY_UNLOCK(sc);
 	return;
 }
 
diff --git a/sys/dev/ofw/ofw_standard.c b/sys/dev/ofw/ofw_standard.c
index e097145..a05f072 100644
--- a/sys/dev/ofw/ofw_standard.c
+++ b/sys/dev/ofw/ofw_standard.c
@@ -204,7 +204,7 @@ ofw_std_interpret(ofw_t ofw, const char *cmd, int nreturns,
 	}
 	status = args.slot[i++];
 	while (i < 1 + nreturns)
-		returns[j] = args.slot[i++];
+		returns[j++] = args.slot[i++];
 	return (status);
 }
 
diff --git a/sys/dev/ofw/openfirm.c b/sys/dev/ofw/openfirm.c
index 4c33c9f..06b58d0 100644
--- a/sys/dev/ofw/openfirm.c
+++ b/sys/dev/ofw/openfirm.c
@@ -164,9 +164,11 @@ OF_interpret(const char *cmd, int nreturns, ...)
 	int status;
 
 	status = OFW_INTERPRET(ofw_obj, cmd, nreturns, slots);
+	if (status == -1)
+		return (status);
 
 	va_start(ap, nreturns);
-	while (i < 1 + nreturns)
+	while (i < nreturns)
 		*va_arg(ap, cell_t *) = slots[i++];
 	va_end(ap);
 
diff --git a/sys/dev/pccard/pccarddevs b/sys/dev/pccard/pccarddevs
index 83872d1..2c42ebf 100644
--- a/sys/dev/pccard/pccarddevs
+++ b/sys/dev/pccard/pccarddevs
@@ -146,8 +146,8 @@ vendor ARTEM			0x0268	ARtem
 vendor SYMBOL			0x026c	Symbol
 vendor BUFFALO			0x026f	BUFFALO (Melco Corporation)
 vendor BROMAX			0x0274	Bromax communications, Inc
-vendor IODATA2			0x028a	I-O DATA
 vendor NECINFORTIA		0x0288	NEC Inforntia
+vendor IODATA2			0x028a	I-O DATA
 vendor ASUS			0x02aa	ASUS
 vendor SIEMENS			0x02ac	Siemens
 vendor UNGERMANN		0x02c0	Ungermann Bass
@@ -181,6 +181,7 @@ vendor RALINK			0x1814	Ralink Technology
  * with '0xc' look coherent enough that maybe somebody other than PCMCIA is
  * assigning numbers in that range.  Maybe JEITA?
  */
+vendor RIOS			0x492f	RIOS Systems Co
 vendor AIRVAST			0x50c2	AirVast Technology
 vendor ARCHOS			0x5241	Archos
 vendor DUAL			0x890f	Dual
@@ -265,6 +266,7 @@ product BREEZECOM BREEZENET	0x0102 BreezeCOM BreezeNET
 
 /* Bromax Communications, Inc (Linksys OEM) */
 product BROMAX IPORT		0x1103 iPort 10/100 Ethernet
+product BROMAX AXNET		0x1106 AXNET based 10/100 Ethernet
 product BROMAX IPORT2		0x1121 iPort 10Mbps Ethernet
 product BROMAX IWN		0x1612 Instant Wireless Network PC Card
 product BROMAX IWN3		0x1613 Instant Wireless Network PC Card, V3
@@ -342,6 +344,7 @@ product FUJITSU WL110		0x2003 PEGA-WL110 Wireless LAN
 product GEMPLUS GPR400		0x3004 GPR400 Smartcard Reader
 
 /* GlobalVillage */
+product GLOBALVILLAGE POWERPORT	0x0103 GlobalVillage PowerPort PlatinumPro
 product GLOBALVILLAGE LANMODEM	0x0105 GlobalVillage Ethernet + Modem
 
 /* Grey Cell Systems, Ltd */
@@ -545,6 +548,9 @@ product RAYTHEON WLAN		0x0000 WLAN Adapter
 /* RELIA Technologies Corporation */
 product RELIA COMBO		0x2452 Reliable Combo-L/M-56K
 
+/* RIOS Systems Co */
+product RIOS PCCARD3		0x0000 PC Card Ethernet
+
 /* Roland */
 product ROLAND SCP55		0x0001 Roland SCP-55
 
@@ -671,6 +677,7 @@ vendor IODATA3			-1	I-O DATA
 vendor LANTECH			-1	Lantech Computer Company
 vendor MELCO2			-1	Melco Corporation
 vendor MICRORESEARCH		-1	Micro Research
+vendor MITSUBISHI		-1	Mitsubishi Electronics Corp
 vendor NDC			-1	NDC
 vendor NEC			-1	NEC
 vendor OEM2			-1	Generic OEM
@@ -745,6 +752,7 @@ product MACNICA MPS100		{ "MACNICA", "MIRACLE&spSCSI", "mPS100", "D.0" } Macnica
 product MEGAHERTZ XJ2288 	{ "MEGAHERTZ", "MODEM&spXJ2288", NULL, NULL } Megahertz XJ2288 Modem
 product MELCO2 LPC2_TX		{ "MELCO", "LPC2-TX", NULL, NULL } Melco LPC2-TX
 product MICRORESEARCH MR10TPC	{ "MICRO-RESEARCH-MR10TPC", "Ethernet", NULL, NULL } Micro Research MR10TPC
+product MITSUBISHI B8895	{ "MITSUBISHI&spELECTRIC&spCORPORATION", "B8895", NULL, NULL } Mitsubishi Electronics Corporation B8895
 product NANOSPEED PRISM2	{ "NANOSPEED", "HFA384x/IEEE", "Version&sp01.02", NULL } NANOSPEED ROOT-RZ2000 WLAN Card
 product NDC ND5100_E		{ "NDC", "Ethernet", "A", NULL } Sohoware ND5100E NE2000 Compatible Card
 product NEC CMZ_RT_WP		{ "NEC", "Wireless&spCard&spCMZ-RT-WP", "Version&sp01.01", NULL } NEC Wireless Card CMZ-RT-WP
diff --git a/sys/dev/pci/pci.c b/sys/dev/pci/pci.c
index d90cd6a..ad6220f 100644
--- a/sys/dev/pci/pci.c
+++ b/sys/dev/pci/pci.c
@@ -1030,8 +1030,9 @@ pci_get_vpd_readonly_method(device_t dev, device_t child, const char *kw,
 }
 
 /*
- * Return the offset in configuration space of the requested extended
- * capability entry or 0 if the specified capability was not found.
+ * Find the requested extended capability and return the offset in
+ * configuration space via the pointer provided. The function returns
+ * 0 on success and error code otherwise.
  */
 int
 pci_find_extcap_method(device_t dev, device_t child, int capability,
diff --git a/sys/dev/pci/pci_pci.c b/sys/dev/pci/pci_pci.c
index 4e1c8ef..15cad92 100644
--- a/sys/dev/pci/pci_pci.c
+++ b/sys/dev/pci/pci_pci.c
@@ -413,12 +413,14 @@ pcib_alloc_resource(device_t dev, device_t child, int type, int *rid,
 			}
 		} else {
 			ok = 1;
-#if 1
-			if (start < sc->iobase && end > sc->iolimit) {
-				start = sc->iobase;
-				end = sc->iolimit;
-			}
-#endif			
+#if 0
+			/*
+			 * If we overlap with the subtractive range, then
+			 * pick the upper range to use.
+			 */
+			if (start < sc->iolimit && end > sc->iobase)
+				start = sc->iolimit + 1;
+#endif
 		}
 		if (end < start) {
 			device_printf(dev, "ioport: end (%lx) < start (%lx)\n",
@@ -478,18 +480,16 @@ pcib_alloc_resource(device_t dev, device_t child, int type, int *rid,
 			}
 		} else if (!ok) {
 			ok = 1;	/* subtractive bridge: always ok */
+#if 0
 			if (pcib_is_nonprefetch_open(sc)) {
-				if (start < sc->membase && end > sc->memlimit) {
-					start = sc->membase;
-					end = sc->memlimit;
-				}
+				if (start < sc->memlimit && end > sc->membase)
+					start = sc->memlimit + 1;
 			}
 			if (pcib_is_prefetch_open(sc)) {
-				if (start < sc->pmembase && end > sc->pmemlimit) {
-					start = sc->pmembase;
-					end = sc->pmemlimit;
-				}
+				if (start < sc->pmemlimit && end > sc->pmembase)
+					start = sc->pmemlimit + 1;
 			}
+#endif
 		}
 		if (end < start) {
 			device_printf(dev, "memory: end (%lx) < start (%lx)\n",
@@ -536,13 +536,13 @@ pcib_maxslots(device_t dev)
  * Since we are a child of a PCI bus, its parent must support the pcib interface.
  */
 uint32_t
-pcib_read_config(device_t dev, int b, int s, int f, int reg, int width)
+pcib_read_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, int width)
 {
     return(PCIB_READ_CONFIG(device_get_parent(device_get_parent(dev)), b, s, f, reg, width));
 }
 
 void
-pcib_write_config(device_t dev, int b, int s, int f, int reg, uint32_t val, int width)
+pcib_write_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, uint32_t val, int width)
 {
     PCIB_WRITE_CONFIG(device_get_parent(device_get_parent(dev)), b, s, f, reg, val, width);
 }
diff --git a/sys/dev/pci/pcib_private.h b/sys/dev/pci/pcib_private.h
index 6159839..c7d6818 100644
--- a/sys/dev/pci/pcib_private.h
+++ b/sys/dev/pci/pcib_private.h
@@ -74,8 +74,8 @@ int		pcib_write_ivar(device_t dev, device_t child, int which, uintptr_t value);
 struct resource *pcib_alloc_resource(device_t dev, device_t child, int type, int *rid, 
 					    u_long start, u_long end, u_long count, u_int flags);
 int		pcib_maxslots(device_t dev);
-uint32_t	pcib_read_config(device_t dev, int b, int s, int f, int reg, int width);
-void		pcib_write_config(device_t dev, int b, int s, int f, int reg, uint32_t val, int width);
+uint32_t	pcib_read_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, int width);
+void		pcib_write_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, uint32_t val, int width);
 int		pcib_route_interrupt(device_t pcib, device_t dev, int pin);
 int		pcib_alloc_msi(device_t pcib, device_t dev, int count, int maxcount, int *irqs);
 int		pcib_release_msi(device_t pcib, device_t dev, int count, int *irqs);
diff --git a/sys/dev/powermac_nvram/powermac_nvram.c b/sys/dev/powermac_nvram/powermac_nvram.c
index a55fcae..f6e930b 100644
--- a/sys/dev/powermac_nvram/powermac_nvram.c
+++ b/sys/dev/powermac_nvram/powermac_nvram.c
@@ -131,19 +131,25 @@ powermac_nvram_attach(device_t dev)
 {
 	struct powermac_nvram_softc *sc;
 	phandle_t node;
-	u_int32_t reg[2];
-	int gen0, gen1;
+	u_int32_t reg[3];
+	int gen0, gen1, i;
 
 	node = ofw_bus_get_node(dev);
 	sc = device_get_softc(dev);
 
-	if (OF_getprop(node, "reg", reg, sizeof(reg)) < 8)
+	if ((i = OF_getprop(node, "reg", reg, sizeof(reg))) < 8)
 		return ENXIO;
 
 	sc->sc_dev = dev;
 	sc->sc_node = node;
 
-	sc->sc_bank0 = (vm_offset_t)pmap_mapdev(reg[0], NVRAM_SIZE * 2);
+	/*
+	 * Find which byte of reg corresponds to the 32-bit physical address.
+	 * We should probably read #address-cells from /chosen instead.
+	 */
+	i = (i/4) - 2;
+
+	sc->sc_bank0 = (vm_offset_t)pmap_mapdev(reg[i], NVRAM_SIZE * 2);
 	sc->sc_bank1 = sc->sc_bank0 + NVRAM_SIZE;
 
 	gen0 = powermac_nvram_check((void *)sc->sc_bank0);
diff --git a/sys/dev/ral/rt2560.c b/sys/dev/ral/rt2560.c
index a0b06a3..936fe8e 100644
--- a/sys/dev/ral/rt2560.c
+++ b/sys/dev/ral/rt2560.c
@@ -52,7 +52,6 @@ __FBSDID("$FreeBSD$");
 #include <net/if_types.h>
 
 #include <net80211/ieee80211_var.h>
-#include <net80211/ieee80211_phy.h>
 #include <net80211/ieee80211_radiotap.h>
 #include <net80211/ieee80211_regdomain.h>
 #include <net80211/ieee80211_amrr.h>
@@ -202,6 +201,7 @@ rt2560_attach(device_t dev, int id)
 	struct ifnet *ifp;
 	int error;
 	uint8_t bands;
+	uint8_t macaddr[IEEE80211_ADDR_LEN];
 
 	sc->sc_dev = dev;
 
@@ -260,7 +260,7 @@ rt2560_attach(device_t dev, int id)
 	ic = ifp->if_l2com;
 
 	/* retrieve MAC address */
-	rt2560_get_macaddr(sc, ic->ic_myaddr);
+	rt2560_get_macaddr(sc, macaddr);
 
 	ifp->if_softc = sc;
 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
@@ -300,7 +300,7 @@ rt2560_attach(device_t dev, int id)
 		setbit(&bands, IEEE80211_MODE_11A);
 	ieee80211_init_channels(ic, NULL, &bands);
 
-	ieee80211_ifattach(ic);
+	ieee80211_ifattach(ic, macaddr);
 	ic->ic_newassoc = rt2560_newassoc;
 	ic->ic_raw_xmit = rt2560_raw_xmit;
 	ic->ic_updateslot = rt2560_update_slot;
@@ -1476,7 +1476,7 @@ rt2560_setup_tx_desc(struct rt2560_softc *sc, struct rt2560_tx_desc *desc,
 	desc->plcp_service = 4;
 
 	len += IEEE80211_CRC_LEN;
-	if (ieee80211_rate2phytype(sc->sc_rates, rate) == IEEE80211_T_OFDM) {
+	if (ieee80211_rate2phytype(ic->ic_rt, rate) == IEEE80211_T_OFDM) {
 		desc->flags |= htole32(RT2560_TX_OFDM);
 
 		plcp_length = len & 0xfff;
@@ -1621,7 +1621,7 @@ rt2560_tx_mgt(struct rt2560_softc *sc, struct mbuf *m0,
 	if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) {
 		flags |= RT2560_TX_ACK;
 
-		dur = ieee80211_ack_duration(sc->sc_rates,
+		dur = ieee80211_ack_duration(ic->ic_rt,
 		    rate, ic->ic_flags & IEEE80211_F_SHPREAMBLE);
 		*(uint16_t *)wh->i_dur = htole16(dur);
 
@@ -1671,16 +1671,16 @@ rt2560_sendprot(struct rt2560_softc *sc,
 	wh = mtod(m, const struct ieee80211_frame *);
 	pktlen = m->m_pkthdr.len + IEEE80211_CRC_LEN;
 
-	protrate = ieee80211_ctl_rate(sc->sc_rates, rate);
-	ackrate = ieee80211_ack_rate(sc->sc_rates, rate);
+	protrate = ieee80211_ctl_rate(ic->ic_rt, rate);
+	ackrate = ieee80211_ack_rate(ic->ic_rt, rate);
 
 	isshort = (ic->ic_flags & IEEE80211_F_SHPREAMBLE) != 0;
-	dur = ieee80211_compute_duration(sc->sc_rates, pktlen, rate, isshort)
-	    + ieee80211_ack_duration(sc->sc_rates, rate, isshort);
+	dur = ieee80211_compute_duration(ic->ic_rt, pktlen, rate, isshort)
+	    + ieee80211_ack_duration(ic->ic_rt, rate, isshort);
 	flags = RT2560_TX_MORE_FRAG;
 	if (prot == IEEE80211_PROT_RTSCTS) {
 		/* NB: CTS is the same size as an ACK */
-		dur += ieee80211_ack_duration(sc->sc_rates, rate, isshort);
+		dur += ieee80211_ack_duration(ic->ic_rt, rate, isshort);
 		flags |= RT2560_TX_ACK;
 		mprot = ieee80211_alloc_rts(ic, wh->i_addr1, wh->i_addr2, dur);
 	} else {
@@ -1851,7 +1851,7 @@ rt2560_tx_data(struct rt2560_softc *sc, struct mbuf *m0,
 		if (m0->m_pkthdr.len + IEEE80211_CRC_LEN > vap->iv_rtsthreshold)
 			prot = IEEE80211_PROT_RTSCTS;
 		else if ((ic->ic_flags & IEEE80211_F_USEPROT) &&
-		    ieee80211_rate2phytype(sc->sc_rates, rate) == IEEE80211_T_OFDM)
+		    ieee80211_rate2phytype(ic->ic_rt, rate) == IEEE80211_T_OFDM)
 			prot = ic->ic_protmode;
 		if (prot != IEEE80211_PROT_NONE) {
 			error = rt2560_sendprot(sc, m0, ni, prot, rate);
@@ -1921,7 +1921,7 @@ rt2560_tx_data(struct rt2560_softc *sc, struct mbuf *m0,
 	if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) {
 		flags |= RT2560_TX_ACK;
 
-		dur = ieee80211_ack_duration(sc->sc_rates,
+		dur = ieee80211_ack_duration(ic->ic_rt,
 		    rate, ic->ic_flags & IEEE80211_F_SHPREAMBLE);
 		*(uint16_t *)wh->i_dur = htole16(dur);
 	}
@@ -1963,15 +1963,7 @@ rt2560_start_locked(struct ifnet *ifp)
 			sc->sc_flags |= RT2560_F_DATA_OACTIVE;
 			break;
 		}
-
 		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
-		m = ieee80211_encap(ni, m);
-		if (m == NULL) {
-			ieee80211_free_node(ni);
-			ifp->if_oerrors++;
-			continue;
-		}
-
 		if (rt2560_tx_data(sc, m, ni) != 0) {
 			ieee80211_free_node(ni);
 			ifp->if_oerrors++;
@@ -2145,8 +2137,6 @@ rt2560_set_chan(struct rt2560_softc *sc, struct ieee80211_channel *c)
 	chan = ieee80211_chan2ieee(ic, c);
 	KASSERT(chan != 0 && chan != IEEE80211_CHAN_ANY, ("chan 0x%x", chan));
 
-	sc->sc_rates = ieee80211_get_ratetable(c);
-
 	if (IEEE80211_IS_CHAN_2GHZ(c))
 		power = min(sc->txpow[chan - 1], 31);
 	else
@@ -2683,8 +2673,7 @@ rt2560_init_locked(struct rt2560_softc *sc)
 	for (i = 0; i < N(rt2560_def_mac); i++)
 		RAL_WRITE(sc, rt2560_def_mac[i].reg, rt2560_def_mac[i].val);
 
-	IEEE80211_ADDR_COPY(ic->ic_myaddr, IF_LLADDR(ifp));
-	rt2560_set_macaddr(sc, ic->ic_myaddr);
+	rt2560_set_macaddr(sc, IF_LLADDR(ifp));
 
 	/* set basic rate set (will be updated later) */
 	RAL_WRITE(sc, RT2560_ARSP_PLCP_1, 0x153);
diff --git a/sys/dev/ral/rt2560var.h b/sys/dev/ral/rt2560var.h
index 0841444..d53c0d1 100644
--- a/sys/dev/ral/rt2560var.h
+++ b/sys/dev/ral/rt2560var.h
@@ -122,8 +122,6 @@ struct rt2560_softc {
 	int			sc_tx_timer;
 	int                     sc_invalid;
 	int			sc_debug;
-
-	const struct ieee80211_rate_table *sc_rates;
 /*
  * The same in both up to here
  * ------------------------------------------------
diff --git a/sys/dev/ral/rt2661.c b/sys/dev/ral/rt2661.c
index 215a1b7..a3b1986 100644
--- a/sys/dev/ral/rt2661.c
+++ b/sys/dev/ral/rt2661.c
@@ -53,7 +53,6 @@ __FBSDID("$FreeBSD$");
 #include <net/if_types.h>
 
 #include <net80211/ieee80211_var.h>
-#include <net80211/ieee80211_phy.h>
 #include <net80211/ieee80211_radiotap.h>
 #include <net80211/ieee80211_regdomain.h>
 #include <net80211/ieee80211_amrr.h>
@@ -155,7 +154,7 @@ static int		rt2661_wme_update(struct ieee80211com *) __unused;
 static void		rt2661_update_slot(struct ifnet *);
 static const char	*rt2661_get_rf(int);
 static void		rt2661_read_eeprom(struct rt2661_softc *,
-			    struct ieee80211com *);
+			    uint8_t macaddr[IEEE80211_ADDR_LEN]);
 static int		rt2661_bbp_init(struct rt2661_softc *);
 static void		rt2661_init_locked(struct rt2661_softc *);
 static void		rt2661_init(void *);
@@ -204,6 +203,7 @@ rt2661_attach(device_t dev, int id)
 	uint32_t val;
 	int error, ac, ntries;
 	uint8_t bands;
+	uint8_t macaddr[IEEE80211_ADDR_LEN];
 
 	sc->sc_id = id;
 	sc->sc_dev = dev;
@@ -234,7 +234,7 @@ rt2661_attach(device_t dev, int id)
 	}
 
 	/* retrieve RF rev. no and various other things from EEPROM */
-	rt2661_read_eeprom(sc, ic);
+	rt2661_read_eeprom(sc, macaddr);
 
 	device_printf(dev, "MAC/BBP RT%X, RF %s\n", val,
 	    rt2661_get_rf(sc->rf_rev));
@@ -303,7 +303,7 @@ rt2661_attach(device_t dev, int id)
 		setbit(&bands, IEEE80211_MODE_11A);
 	ieee80211_init_channels(ic, NULL, &bands);
 
-	ieee80211_ifattach(ic);
+	ieee80211_ifattach(ic, macaddr);
 	ic->ic_newassoc = rt2661_newassoc;
 	ic->ic_node_alloc = rt2661_node_alloc;
 #if 0
@@ -319,8 +319,6 @@ rt2661_attach(device_t dev, int id)
 	ic->ic_vap_create = rt2661_vap_create;
 	ic->ic_vap_delete = rt2661_vap_delete;
 
-	sc->sc_rates = ieee80211_get_ratetable(ic->ic_curchan);
-
 	bpfattach(ifp, DLT_IEEE802_11_RADIO,
 	    sizeof (struct ieee80211_frame) + sizeof (sc->sc_txtap));
 
@@ -1301,7 +1299,7 @@ rt2661_setup_tx_desc(struct rt2661_softc *sc, struct rt2661_tx_desc *desc,
 	desc->plcp_service = 4;
 
 	len += IEEE80211_CRC_LEN;
-	if (ieee80211_rate2phytype(sc->sc_rates, rate) == IEEE80211_T_OFDM) {
+	if (ieee80211_rate2phytype(ic->ic_rt, rate) == IEEE80211_T_OFDM) {
 		desc->flags |= htole32(RT2661_TX_OFDM);
 
 		plcp_length = len & 0xfff;
@@ -1387,7 +1385,7 @@ rt2661_tx_mgt(struct rt2661_softc *sc, struct mbuf *m0,
 	if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) {
 		flags |= RT2661_TX_NEED_ACK;
 
-		dur = ieee80211_ack_duration(sc->sc_rates,
+		dur = ieee80211_ack_duration(ic->ic_rt,
 		    rate, ic->ic_flags & IEEE80211_F_SHPREAMBLE);
 		*(uint16_t *)wh->i_dur = htole16(dur);
 
@@ -1437,16 +1435,16 @@ rt2661_sendprot(struct rt2661_softc *sc, int ac,
 	wh = mtod(m, const struct ieee80211_frame *);
 	pktlen = m->m_pkthdr.len + IEEE80211_CRC_LEN;
 
-	protrate = ieee80211_ctl_rate(sc->sc_rates, rate);
-	ackrate = ieee80211_ack_rate(sc->sc_rates, rate);
+	protrate = ieee80211_ctl_rate(ic->ic_rt, rate);
+	ackrate = ieee80211_ack_rate(ic->ic_rt, rate);
 
 	isshort = (ic->ic_flags & IEEE80211_F_SHPREAMBLE) != 0;
-	dur = ieee80211_compute_duration(sc->sc_rates, pktlen, rate, isshort)
-	    + ieee80211_ack_duration(sc->sc_rates, rate, isshort);
+	dur = ieee80211_compute_duration(ic->ic_rt, pktlen, rate, isshort)
+	    + ieee80211_ack_duration(ic->ic_rt, rate, isshort);
 	flags = RT2661_TX_MORE_FRAG;
 	if (prot == IEEE80211_PROT_RTSCTS) {
 		/* NB: CTS is the same size as an ACK */
-		dur += ieee80211_ack_duration(sc->sc_rates, rate, isshort);
+		dur += ieee80211_ack_duration(ic->ic_rt, rate, isshort);
 		flags |= RT2661_TX_NEED_ACK;
 		mprot = ieee80211_alloc_rts(ic, wh->i_addr1, wh->i_addr2, dur);
 	} else {
@@ -1543,7 +1541,7 @@ rt2661_tx_data(struct rt2661_softc *sc, struct mbuf *m0,
 		if (m0->m_pkthdr.len + IEEE80211_CRC_LEN > vap->iv_rtsthreshold)
 			prot = IEEE80211_PROT_RTSCTS;
 		else if ((ic->ic_flags & IEEE80211_F_USEPROT) &&
-		    ieee80211_rate2phytype(sc->sc_rates, rate) == IEEE80211_T_OFDM)
+		    ieee80211_rate2phytype(ic->ic_rt, rate) == IEEE80211_T_OFDM)
 			prot = ic->ic_protmode;
 		if (prot != IEEE80211_PROT_NONE) {
 			error = rt2661_sendprot(sc, ac, m0, ni, prot, rate);
@@ -1614,7 +1612,7 @@ rt2661_tx_data(struct rt2661_softc *sc, struct mbuf *m0,
 	if (!noack && !IEEE80211_IS_MULTICAST(wh->i_addr1)) {
 		flags |= RT2661_TX_NEED_ACK;
 
-		dur = ieee80211_ack_duration(sc->sc_rates,
+		dur = ieee80211_ack_duration(ic->ic_rt,
 		    rate, ic->ic_flags & IEEE80211_F_SHPREAMBLE);
 		*(uint16_t *)wh->i_dur = htole16(dur);
 	}
@@ -1662,15 +1660,7 @@ rt2661_start_locked(struct ifnet *ifp)
 			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 			break;
 		}
-
 		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
-		m = ieee80211_encap(ni, m);
-		if (m == NULL) {
-			ieee80211_free_node(ni);
-			ifp->if_oerrors++;
-			continue;
-		}
-
 		if (rt2661_tx_data(sc, m, ni, ac) != 0) {
 			ieee80211_free_node(ni);
 			ifp->if_oerrors++;
@@ -2040,8 +2030,6 @@ rt2661_set_chan(struct rt2661_softc *sc, struct ieee80211_channel *c)
 	chan = ieee80211_chan2ieee(ic, c);
 	KASSERT(chan != 0 && chan != IEEE80211_CHAN_ANY, ("chan 0x%x", chan));
 
-	sc->sc_rates = ieee80211_get_ratetable(c);
-
 	/* select the appropriate RF settings based on what EEPROM says */
 	rfprog = (sc->rfprog == 0) ? rt2661_rf5225_1 : rt2661_rf5225_2;
 
@@ -2219,23 +2207,23 @@ rt2661_get_rf(int rev)
 }
 
 static void
-rt2661_read_eeprom(struct rt2661_softc *sc, struct ieee80211com *ic)
+rt2661_read_eeprom(struct rt2661_softc *sc, uint8_t macaddr[IEEE80211_ADDR_LEN])
 {
 	uint16_t val;
 	int i;
 
 	/* read MAC address */
 	val = rt2661_eeprom_read(sc, RT2661_EEPROM_MAC01);
-	ic->ic_myaddr[0] = val & 0xff;
-	ic->ic_myaddr[1] = val >> 8;
+	macaddr[0] = val & 0xff;
+	macaddr[1] = val >> 8;
 
 	val = rt2661_eeprom_read(sc, RT2661_EEPROM_MAC23);
-	ic->ic_myaddr[2] = val & 0xff;
-	ic->ic_myaddr[3] = val >> 8;
+	macaddr[2] = val & 0xff;
+	macaddr[3] = val >> 8;
 
 	val = rt2661_eeprom_read(sc, RT2661_EEPROM_MAC45);
-	ic->ic_myaddr[4] = val & 0xff;
-	ic->ic_myaddr[5] = val >> 8;
+	macaddr[4] = val & 0xff;
+	macaddr[5] = val >> 8;
 
 	val = rt2661_eeprom_read(sc, RT2661_EEPROM_ANTENNA);
 	/* XXX: test if different from 0xffff? */
@@ -2413,8 +2401,7 @@ rt2661_init_locked(struct rt2661_softc *sc)
 	for (i = 0; i < N(rt2661_def_mac); i++)
 		RAL_WRITE(sc, rt2661_def_mac[i].reg, rt2661_def_mac[i].val);
 
-	IEEE80211_ADDR_COPY(ic->ic_myaddr, IF_LLADDR(ifp));
-	rt2661_set_macaddr(sc, ic->ic_myaddr);
+	rt2661_set_macaddr(sc, IF_LLADDR(ifp));
 
 	/* set host ready */
 	RAL_WRITE(sc, RT2661_MAC_CSR1, 3);
diff --git a/sys/dev/ral/rt2661var.h b/sys/dev/ral/rt2661var.h
index 0c15840..6fc958a 100644
--- a/sys/dev/ral/rt2661var.h
+++ b/sys/dev/ral/rt2661var.h
@@ -114,8 +114,6 @@ struct rt2661_softc {
 	int				sc_tx_timer;
 	int                             sc_invalid;
 	int				sc_debug;
-
-	const struct ieee80211_rate_table *sc_rates;
 /*
  * The same in both up to here
  * ------------------------------------------------
diff --git a/sys/dev/re/if_re.c b/sys/dev/re/if_re.c
index 7a017c6..fe7dad9 100644
--- a/sys/dev/re/if_re.c
+++ b/sys/dev/re/if_re.c
@@ -419,6 +419,7 @@ re_gmii_readreg(device_t dev, int phy, int reg)
 	}
 
 	CSR_WRITE_4(sc, RL_PHYAR, reg << 16);
+	DELAY(1000);
 
 	for (i = 0; i < RL_PHY_TIMEOUT; i++) {
 		rval = CSR_READ_4(sc, RL_PHYAR);
@@ -446,6 +447,7 @@ re_gmii_writereg(device_t dev, int phy, int reg, int data)
 
 	CSR_WRITE_4(sc, RL_PHYAR, (reg << 16) |
 	    (data & RL_PHYAR_PHYDATA) | RL_PHYAR_BUSY);
+	DELAY(1000);
 
 	for (i = 0; i < RL_PHY_TIMEOUT; i++) {
 		rval = CSR_READ_4(sc, RL_PHYAR);
diff --git a/sys/dev/sound/pci/hda/hdac.c b/sys/dev/sound/pci/hda/hdac.c
index 1f997f7..e580944 100644
--- a/sys/dev/sound/pci/hda/hdac.c
+++ b/sys/dev/sound/pci/hda/hdac.c
@@ -83,7 +83,7 @@
 
 #include "mixer_if.h"
 
-#define HDA_DRV_TEST_REV	"20090226_0129"
+#define HDA_DRV_TEST_REV	"20090401_0132"
 
 SND_DECLARE_FILE("$FreeBSD$");
 
@@ -247,6 +247,7 @@ SND_DECLARE_FILE("$FreeBSD$");
 #define ACER_A4715_SUBVENDOR	HDA_MODEL_CONSTRUCT(ACER, 0x0133)
 #define ACER_3681WXM_SUBVENDOR	HDA_MODEL_CONSTRUCT(ACER, 0x0110)
 #define ACER_T6292_SUBVENDOR	HDA_MODEL_CONSTRUCT(ACER, 0x011b)
+#define ACER_T5320_SUBVENDOR	HDA_MODEL_CONSTRUCT(ACER, 0x011f)
 #define ACER_ALL_SUBVENDOR	HDA_MODEL_CONSTRUCT(ACER, 0xffff)
 
 /* Asus */
@@ -260,6 +261,7 @@ SND_DECLARE_FILE("$FreeBSD$");
 #define ASUS_A7T_SUBVENDOR	HDA_MODEL_CONSTRUCT(ASUS, 0x13c2)
 #define ASUS_W2J_SUBVENDOR	HDA_MODEL_CONSTRUCT(ASUS, 0x1971)
 #define ASUS_M5200_SUBVENDOR	HDA_MODEL_CONSTRUCT(ASUS, 0x1993)
+#define ASUS_P5PL2_SUBVENDOR	HDA_MODEL_CONSTRUCT(ASUS, 0x817f)
 #define ASUS_P1AH2_SUBVENDOR	HDA_MODEL_CONSTRUCT(ASUS, 0x81cb)
 #define ASUS_M2NPVMX_SUBVENDOR	HDA_MODEL_CONSTRUCT(ASUS, 0x81cb)
 #define ASUS_M2V_SUBVENDOR	HDA_MODEL_CONSTRUCT(ASUS, 0x81e7)
@@ -2424,13 +2426,20 @@ hdac_widget_pin_getconfig(struct hdac_widget *w)
 	/* New patches */
 	if (id == HDA_CODEC_AD1986A &&
 	    (sc->pci_subvendor == ASUS_M2NPVMX_SUBVENDOR ||
-	    sc->pci_subvendor == ASUS_A8NVMCSM_SUBVENDOR)) {
+	    sc->pci_subvendor == ASUS_A8NVMCSM_SUBVENDOR ||
+	    sc->pci_subvendor == ASUS_P5PL2_SUBVENDOR)) {
 		switch (nid) {
-		case 28: /* 5.1 out => 2.0 out + 2 inputs */
+		case 26: /* Headphones with redirection */
+			patch = "as=1 seq=15";
+			break;
+		case 28: /* 5.1 out => 2.0 out + 1 input */
 			patch = "device=Line-in as=8 seq=1";
 			break;
-		case 29:
-			patch = "device=Mic as=8 seq=2";
+		case 29: /* Can't use this as input, as the only available mic
+			  * preamplifier is busy by front panel mic (nid 31).
+			  * If you want to use this rear connector as mic input,
+			  * you have to disable the front panel one. */
+			patch = "as=0";
 			break;
 		case 31: /* Lot of inputs configured with as=15 and unusable */
 			patch = "as=8 seq=3";
@@ -2452,13 +2461,14 @@ hdac_widget_pin_getconfig(struct hdac_widget *w)
 			patch = "seq=15 device=Headphones";
 			break;
 		}
-	} else if (id == HDA_CODEC_ALC268 &&
-	    HDA_DEV_MATCH(ACER_ALL_SUBVENDOR, sc->pci_subvendor)) {
+	} else if (id == HDA_CODEC_ALC268) {
+	    if (sc->pci_subvendor == ACER_T5320_SUBVENDOR) {
 		switch (nid) {
-		case 28:
-			patch = "device=CD conn=fixed";
+		case 20: /* Headphones Jack */
+			patch = "as=1 seq=15";
 			break;
 		}
+	    }
 	}
 
 	if (patch != NULL)
@@ -4729,6 +4739,35 @@ hdac_vendor_patch_parse(struct hdac_devinfo *devinfo)
 		w = hdac_widget_get(devinfo, 15);
 		if (w != NULL)
 			w->connsenable[3] = 0;
+		/* There is only one mic preamplifier, use it effectively. */
+		w = hdac_widget_get(devinfo, 31);
+		if (w != NULL) {
+			if ((w->wclass.pin.config &
+			    HDA_CONFIG_DEFAULTCONF_DEVICE_MASK) ==
+			    HDA_CONFIG_DEFAULTCONF_DEVICE_MIC_IN) {
+				w = hdac_widget_get(devinfo, 16);
+				if (w != NULL)
+				    w->connsenable[2] = 0;
+			} else {
+				w = hdac_widget_get(devinfo, 15);
+				if (w != NULL)
+				    w->connsenable[0] = 0;
+			}
+		}
+		w = hdac_widget_get(devinfo, 32);
+		if (w != NULL) {
+			if ((w->wclass.pin.config &
+			    HDA_CONFIG_DEFAULTCONF_DEVICE_MASK) ==
+			    HDA_CONFIG_DEFAULTCONF_DEVICE_MIC_IN) {
+				w = hdac_widget_get(devinfo, 16);
+				if (w != NULL)
+				    w->connsenable[0] = 0;
+			} else {
+				w = hdac_widget_get(devinfo, 15);
+				if (w != NULL)
+				    w->connsenable[1] = 0;
+			}
+		}
 
 		if (subvendor == ASUS_A8X_SUBVENDOR) {
 			/*
@@ -5278,7 +5317,7 @@ hdac_audio_bind_as(struct hdac_devinfo *devinfo)
 		    sizeof(struct hdac_chan) * cnt,
 		    M_HDAC, M_ZERO | M_NOWAIT);
 		if (sc->chans == NULL) {
-			device_printf(devinfo->codec->sc->dev,
+			device_printf(sc->dev,
 			    "Channels memory allocation failed!\n");
 			return;
 		}
@@ -5288,17 +5327,20 @@ hdac_audio_bind_as(struct hdac_devinfo *devinfo)
 		    M_HDAC, M_ZERO | M_NOWAIT);
 		if (sc->chans == NULL) {
 			sc->num_chans = 0;
-			device_printf(devinfo->codec->sc->dev,
+			device_printf(sc->dev,
 			    "Channels memory allocation failed!\n");
 			return;
 		}
+		/* Fixup relative pointers after realloc */
+		for (j = 0; j < sc->num_chans; j++)
+			sc->chans[j].caps.fmtlist = sc->chans[j].fmtlist;
 	}
 	free = sc->num_chans;
 	sc->num_chans += cnt;
 
 	for (j = free; j < free + cnt; j++) {
-		devinfo->codec->sc->chans[j].devinfo = devinfo;
-		devinfo->codec->sc->chans[j].as = -1;
+		sc->chans[j].devinfo = devinfo;
+		sc->chans[j].as = -1;
 	}
 
 	/* Assign associations in order of their numbers, */
@@ -5307,10 +5349,10 @@ hdac_audio_bind_as(struct hdac_devinfo *devinfo)
 			continue;
 		
 		as[j].chan = free;
-		devinfo->codec->sc->chans[free].as = j;
-		devinfo->codec->sc->chans[free].dir =
+		sc->chans[free].as = j;
+		sc->chans[free].dir =
 		    (as[j].dir == HDA_CTL_IN) ? PCMDIR_REC : PCMDIR_PLAY;
-		hdac_pcmchannel_setup(&devinfo->codec->sc->chans[free]);
+		hdac_pcmchannel_setup(&sc->chans[free]);
 		free++;
 	}
 }
@@ -7923,9 +7965,9 @@ hdac_pcm_attach(device_t dev)
 		device_printf(dev, "+--------------------------------------+\n");
 		hdac_dump_pcmchannels(pdevinfo);
 		device_printf(dev, "\n");
-		device_printf(dev, "+--------------------------------+\n");
-		device_printf(dev, "| DUMPING Playback/Record Pathes |\n");
-		device_printf(dev, "+--------------------------------+\n");
+		device_printf(dev, "+-------------------------------+\n");
+		device_printf(dev, "| DUMPING Playback/Record Paths |\n");
+		device_printf(dev, "+-------------------------------+\n");
 		hdac_dump_dac(pdevinfo);
 		hdac_dump_adc(pdevinfo);
 		hdac_dump_mix(pdevinfo);
diff --git a/sys/dev/sound/usb/uaudio.c b/sys/dev/sound/usb/uaudio.c
index 50728ee..50866da 100644
--- a/sys/dev/sound/usb/uaudio.c
+++ b/sys/dev/sound/usb/uaudio.c
@@ -400,20 +400,20 @@ static const struct usb2_config
 		.type = UE_ISOCHRONOUS,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = 0,	/* use "wMaxPacketSize * frames" */
-		.mh.frames = UAUDIO_MINFRAMES,
-		.mh.flags = {.short_xfer_ok = 1,},
-		.mh.callback = &uaudio_chan_record_callback,
+		.bufsize = 0,	/* use "wMaxPacketSize * frames" */
+		.frames = UAUDIO_MINFRAMES,
+		.flags = {.short_xfer_ok = 1,},
+		.callback = &uaudio_chan_record_callback,
 	},
 
 	[1] = {
 		.type = UE_ISOCHRONOUS,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = 0,	/* use "wMaxPacketSize * frames" */
-		.mh.frames = UAUDIO_MINFRAMES,
-		.mh.flags = {.short_xfer_ok = 1,},
-		.mh.callback = &uaudio_chan_record_callback,
+		.bufsize = 0,	/* use "wMaxPacketSize * frames" */
+		.frames = UAUDIO_MINFRAMES,
+		.flags = {.short_xfer_ok = 1,},
+		.callback = &uaudio_chan_record_callback,
 	},
 };
 
@@ -423,20 +423,20 @@ static const struct usb2_config
 		.type = UE_ISOCHRONOUS,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = 0,	/* use "wMaxPacketSize * frames" */
-		.mh.frames = UAUDIO_MINFRAMES,
-		.mh.flags = {.short_xfer_ok = 1,},
-		.mh.callback = &uaudio_chan_play_callback,
+		.bufsize = 0,	/* use "wMaxPacketSize * frames" */
+		.frames = UAUDIO_MINFRAMES,
+		.flags = {.short_xfer_ok = 1,},
+		.callback = &uaudio_chan_play_callback,
 	},
 
 	[1] = {
 		.type = UE_ISOCHRONOUS,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = 0,	/* use "wMaxPacketSize * frames" */
-		.mh.frames = UAUDIO_MINFRAMES,
-		.mh.flags = {.short_xfer_ok = 1,},
-		.mh.callback = &uaudio_chan_play_callback,
+		.bufsize = 0,	/* use "wMaxPacketSize * frames" */
+		.frames = UAUDIO_MINFRAMES,
+		.flags = {.short_xfer_ok = 1,},
+		.callback = &uaudio_chan_play_callback,
 	},
 };
 
@@ -446,9 +446,9 @@ static const struct usb2_config
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = (sizeof(struct usb2_device_request) + 4),
-		.mh.callback = &uaudio_mixer_write_cfg_callback,
-		.mh.timeout = 1000,	/* 1 second */
+		.bufsize = (sizeof(struct usb2_device_request) + 4),
+		.callback = &uaudio_mixer_write_cfg_callback,
+		.timeout = 1000,	/* 1 second */
 	},
 };
 
@@ -478,40 +478,40 @@ static const struct usb2_config
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = UMIDI_BULK_SIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &umidi_bulk_write_callback,
+		.bufsize = UMIDI_BULK_SIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &umidi_bulk_write_callback,
 	},
 
 	[1] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = UMIDI_BULK_SIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &umidi_bulk_read_callback,
+		.bufsize = UMIDI_BULK_SIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &umidi_bulk_read_callback,
 	},
 
 	[2] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &umidi_write_clear_stall_callback,
-		.mh.timeout = 1000,	/* 1 second */
-		.mh.interval = 50,	/* 50ms */
+		.bufsize = sizeof(struct usb2_device_request),
+		.flags = {},
+		.callback = &umidi_write_clear_stall_callback,
+		.timeout = 1000,	/* 1 second */
+		.interval = 50,	/* 50ms */
 	},
 
 	[3] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &umidi_read_clear_stall_callback,
-		.mh.timeout = 1000,	/* 1 second */
-		.mh.interval = 50,	/* 50ms */
+		.bufsize = sizeof(struct usb2_device_request),
+		.flags = {},
+		.callback = &umidi_read_clear_stall_callback,
+		.timeout = 1000,	/* 1 second */
+		.interval = 50,	/* 50ms */
 	},
 };
 
diff --git a/sys/dev/syscons/syscons.c b/sys/dev/syscons/syscons.c
index 6791d64..77f4e61 100644
--- a/sys/dev/syscons/syscons.c
+++ b/sys/dev/syscons/syscons.c
@@ -173,7 +173,6 @@ static void scshutdown(void *arg, int howto);
 static u_int scgetc(sc_softc_t *sc, u_int flags);
 #define SCGETC_CN	1
 #define SCGETC_NONBLOCK	2
-static int sccngetch(int flags);
 static void sccnupdate(scr_stat *scp);
 static scr_stat *alloc_scp(sc_softc_t *sc, int vty);
 static void init_scp(sc_softc_t *sc, int vty, scr_stat *scp);
@@ -1533,12 +1532,6 @@ sc_cnputc(struct consdev *cd, int c)
 static int
 sc_cngetc(struct consdev *cd)
 {
-    return sccngetch(SCGETC_NONBLOCK);
-}
-
-static int
-sccngetch(int flags)
-{
     static struct fkeytab fkey;
     static int fkeycp;
     scr_stat *scp;
@@ -1579,7 +1572,7 @@ sccngetch(int flags)
     kbdd_ioctl(scp->sc->kbd, KDSKBMODE, (caddr_t)&scp->kbd_mode);
 
     kbdd_poll(scp->sc->kbd, TRUE);
-    c = scgetc(scp->sc, SCGETC_CN | flags);
+    c = scgetc(scp->sc, SCGETC_CN | SCGETC_NONBLOCK);
     kbdd_poll(scp->sc->kbd, FALSE);
 
     scp->kbd_mode = cur_mode;
diff --git a/sys/dev/syscons/teken/teken.c b/sys/dev/syscons/teken/teken.c
index f5555b0..30845c4 100644
--- a/sys/dev/syscons/teken/teken.c
+++ b/sys/dev/syscons/teken/teken.c
@@ -361,8 +361,6 @@ void
 teken_set_winsize(teken_t *t, const teken_pos_t *p)
 {
 
-	teken_assert(p->tp_col <= T_NUMCOL);
-
 	t->t_winsize = *p;
 	/* XXX: bounds checking with cursor/etc! */
 	t->t_scrollreg.ts_begin = 0;
diff --git a/sys/dev/syscons/teken/teken_subr.h b/sys/dev/syscons/teken/teken_subr.h
index 2ca2c30..21982e2 100644
--- a/sys/dev/syscons/teken/teken_subr.h
+++ b/sys/dev/syscons/teken/teken_subr.h
@@ -37,7 +37,8 @@ teken_tab_isset(teken_t *t, unsigned int col)
 {
 	unsigned int b, o;
 
-	teken_assert(col <= T_NUMCOL);
+	if (col >= T_NUMCOL)
+		return ((col % 8) == 0);
 
 	b = col / (sizeof(unsigned int) * 8);
 	o = col % (sizeof(unsigned int) * 8);
@@ -50,7 +51,8 @@ teken_tab_clear(teken_t *t, unsigned int col)
 {
 	unsigned int b, o;
 
-	teken_assert(col <= T_NUMCOL);
+	if (col >= T_NUMCOL)
+		return;
 
 	b = col / (sizeof(unsigned int) * 8);
 	o = col % (sizeof(unsigned int) * 8);
@@ -63,7 +65,8 @@ teken_tab_set(teken_t *t, unsigned int col)
 {
 	unsigned int b, o;
 
-	teken_assert(col <= T_NUMCOL);
+	if (col >= T_NUMCOL)
+		return;
 
 	b = col / (sizeof(unsigned int) * 8);
 	o = col % (sizeof(unsigned int) * 8);
diff --git a/sys/dev/twa/tw_cl_init.c b/sys/dev/twa/tw_cl_init.c
index 52864b2..ca282fa 100644
--- a/sys/dev/twa/tw_cl_init.c
+++ b/sys/dev/twa/tw_cl_init.c
@@ -692,7 +692,7 @@ tw_cli_init_connection(struct tw_cli_ctlr_context *ctlr,
 	init_connect->message_credits = TW_CL_SWAP16(message_credits);
 	init_connect->features = TW_CL_SWAP32(set_features);
 	if (ctlr->flags & TW_CL_64BIT_ADDRESSES)
-		init_connect->features |= TWA_64BIT_SG_ADDRESSES;
+		init_connect->features |= TW_CL_SWAP32(TWA_64BIT_SG_ADDRESSES);
 	if (set_features & TWA_EXTENDED_INIT_CONNECT) {
 		/*
 		 * Fill in the extra fields needed for an extended
diff --git a/sys/dev/twa/tw_osl.h b/sys/dev/twa/tw_osl.h
index d68cc5e..6082253 100644
--- a/sys/dev/twa/tw_osl.h
+++ b/sys/dev/twa/tw_osl.h
@@ -57,6 +57,12 @@
 #define TW_OSLI_DEFERRED_INTR_USED
 */
 
+#ifdef PAE
+#define	TW_OSLI_DMA_BOUNDARY		(1u << 31)
+#else
+#define	TW_OSLI_DMA_BOUNDARY		((bus_size_t)((uint64_t)1 << 32))
+#endif
+
 /* Possible values of req->state. */
 #define TW_OSLI_REQ_STATE_INIT		0x0	/* being initialized */
 #define TW_OSLI_REQ_STATE_BUSY		0x1	/* submitted to CL */
diff --git a/sys/dev/twa/tw_osl_freebsd.c b/sys/dev/twa/tw_osl_freebsd.c
index d18fd0a..85b3af1 100644
--- a/sys/dev/twa/tw_osl_freebsd.c
+++ b/sys/dev/twa/tw_osl_freebsd.c
@@ -491,8 +491,8 @@ tw_osli_alloc_mem(struct twa_softc *sc)
 	/* Create the parent dma tag. */
 	if (bus_dma_tag_create(NULL,			/* parent */
 				sc->alignment,		/* alignment */
-				0,			/* boundary */
-				BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
+				TW_OSLI_DMA_BOUNDARY,	/* boundary */
+				BUS_SPACE_MAXADDR,	/* lowaddr */
 				BUS_SPACE_MAXADDR, 	/* highaddr */
 				NULL, NULL, 		/* filter, filterarg */
 				TW_CL_MAX_IO_SIZE,	/* maxsize */
@@ -515,7 +515,7 @@ tw_osli_alloc_mem(struct twa_softc *sc)
 	if (bus_dma_tag_create(sc->parent_tag,		/* parent */
 				sc->alignment,		/* alignment */
 				0,			/* boundary */
-				BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
+				BUS_SPACE_MAXADDR,	/* lowaddr */
 				BUS_SPACE_MAXADDR, 	/* highaddr */
 				NULL, NULL, 		/* filter, filterarg */
 				dma_mem_size,		/* maxsize */
@@ -562,7 +562,7 @@ tw_osli_alloc_mem(struct twa_softc *sc)
 	if (bus_dma_tag_create(sc->parent_tag,		/* parent */
 				sc->alignment,		/* alignment */
 				0,			/* boundary */
-				BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
+				BUS_SPACE_MAXADDR,	/* lowaddr */
 				BUS_SPACE_MAXADDR, 	/* highaddr */
 				NULL, NULL, 		/* filter, filterarg */
 				TW_CL_MAX_IO_SIZE,	/* maxsize */
@@ -588,7 +588,7 @@ tw_osli_alloc_mem(struct twa_softc *sc)
 	if (bus_dma_tag_create(sc->parent_tag,		/* parent */
 				sc->alignment,		/* alignment */
 				0,			/* boundary */
-				BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
+				BUS_SPACE_MAXADDR,	/* lowaddr */
 				BUS_SPACE_MAXADDR, 	/* highaddr */
 				NULL, NULL, 		/* filter, filterarg */
 				TW_CL_MAX_IO_SIZE,	/* maxsize */
@@ -1347,7 +1347,7 @@ static TW_VOID
 twa_map_load_callback(TW_VOID *arg, bus_dma_segment_t *segs,
 	TW_INT32 nsegments, TW_INT32 error)
 {
-	*((bus_addr_t *)arg) = segs[0].ds_addr;
+	*((TW_UINT64 *)arg) = segs[0].ds_addr;
 }
 
 
diff --git a/sys/dev/uart/uart_cpu_powerpc.c b/sys/dev/uart/uart_cpu_powerpc.c
index 0945970..f97eda0 100644
--- a/sys/dev/uart/uart_cpu_powerpc.c
+++ b/sys/dev/uart/uart_cpu_powerpc.c
@@ -31,6 +31,8 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
 
 #include <machine/bus.h>
 
@@ -53,8 +55,11 @@ bus_space_tag_t uart_bus_space_mem = &bs_le_tag;
 int
 uart_cpu_eqres(struct uart_bas *b1, struct uart_bas *b2)
 {
-
+#ifdef MPC85XX
 	return ((b1->bsh == b2->bsh) ? 1 : 0);
+#else
+	return ((pmap_kextract(b1->bsh) == pmap_kextract(b2->bsh)) ? 1 : 0);
+#endif
 }
 
 #ifdef MPC85XX
@@ -116,7 +121,16 @@ uart_cpu_getdev(int devtype, struct uart_devinfo *di)
 		return (ENXIO);
 	if (OF_getprop(input, "name", buf, sizeof(buf)) == -1)
 		return (ENXIO);
-	if (strcmp(buf, "ch-a"))
+
+	if (strcmp(buf, "ch-a") == 0) {
+		class = &uart_z8530_class;
+		di->bas.regshft = 4;
+		di->bas.chan = 1;
+	} else if (strcmp(buf,"serial") == 0) {
+		class = &uart_ns8250_class;
+		di->bas.regshft = 0;
+		di->bas.chan = 0;
+	} else
 		return (ENXIO);
 
 	error = OF_decode_addr(input, 0, &di->bas.bst, &di->bas.bsh);
@@ -125,11 +139,13 @@ uart_cpu_getdev(int devtype, struct uart_devinfo *di)
 
 	di->ops = uart_getops(class);
 
-	di->bas.rclk = 230400;
-	di->bas.chan = 1;
-	di->bas.regshft = 4;
+	if (OF_getprop(input, "clock-frequency", &di->bas.rclk, 
+	    sizeof(di->bas.rclk)) == -1)
+		di->bas.rclk = 230400;
+	if (OF_getprop(input, "current-speed", &di->baudrate, 
+	    sizeof(di->baudrate)) == -1)
+		di->baudrate = 0;
 
-	di->baudrate = 0;
 	di->databits = 8;
 	di->stopbits = 1;
 	di->parity = UART_PARITY_NONE;
diff --git a/sys/dev/uart/uart_dev_ns8250.c b/sys/dev/uart/uart_dev_ns8250.c
index 81642b2..c01fd61 100644
--- a/sys/dev/uart/uart_dev_ns8250.c
+++ b/sys/dev/uart/uart_dev_ns8250.c
@@ -50,14 +50,16 @@ __FBSDID("$FreeBSD$");
 static void
 ns8250_clrint(struct uart_bas *bas)
 {
-	uint8_t iir;
+	uint8_t iir, lsr;
 
 	iir = uart_getreg(bas, REG_IIR);
 	while ((iir & IIR_NOPEND) == 0) {
 		iir &= IIR_IMASK;
-		if (iir == IIR_RLS)
-			(void)uart_getreg(bas, REG_LSR);
-		else if (iir == IIR_RXRDY || iir == IIR_RXTOUT)
+		if (iir == IIR_RLS) {
+			lsr = uart_getreg(bas, REG_LSR);
+			if (lsr & (LSR_BI|LSR_FE|LSR_PE))
+				(void)uart_getreg(bas, REG_DATA);
+		} else if (iir == IIR_RXRDY || iir == IIR_RXTOUT)
 			(void)uart_getreg(bas, REG_DATA);
 		else if (iir == IIR_MLSC)
 			(void)uart_getreg(bas, REG_MSR);
@@ -587,7 +589,6 @@ ns8250_bus_ipend(struct uart_softc *sc)
 	ipend = 0;
 	if (iir & IIR_RXRDY) {
 		lsr = uart_getreg(bas, REG_LSR);
-		uart_unlock(sc->sc_hwmtx);
 		if (lsr & LSR_OE)
 			ipend |= SER_INT_OVERRUN;
 		if (lsr & LSR_BI)
@@ -595,12 +596,14 @@ ns8250_bus_ipend(struct uart_softc *sc)
 		if (lsr & LSR_RXRDY)
 			ipend |= SER_INT_RXREADY;
 	} else {
-		uart_unlock(sc->sc_hwmtx);
 		if (iir & IIR_TXRDY)
 			ipend |= SER_INT_TXIDLE;
 		else
 			ipend |= SER_INT_SIGCHG;
 	}
+	if (ipend == 0)
+		ns8250_clrint(bas);
+	uart_unlock(sc->sc_hwmtx);
 	return ((sc->sc_leaving) ? 0 : ipend);
 }
 
diff --git a/sys/dev/usb/bluetooth/ng_ubt.c b/sys/dev/usb/bluetooth/ng_ubt.c
index 5e94578..a5a19ded 100644
--- a/sys/dev/usb/bluetooth/ng_ubt.c
+++ b/sys/dev/usb/bluetooth/ng_ubt.c
@@ -276,9 +276,9 @@ static const struct usb2_config		ubt_config[UBT_N_TRANSFER] =
 		.endpoint =	UE_ADDR_ANY,
 		.direction =	UE_DIR_OUT,
 		.if_index = 	0,
-		.mh.bufsize =	UBT_BULK_WRITE_BUFFER_SIZE,
-		.mh.flags =	{ .pipe_bof = 1, .force_short_xfer = 1, },
-		.mh.callback =	&ubt_bulk_write_callback,
+		.bufsize =	UBT_BULK_WRITE_BUFFER_SIZE,
+		.flags =	{ .pipe_bof = 1, .force_short_xfer = 1, },
+		.callback =	&ubt_bulk_write_callback,
 	},
 	/* Incoming bulk transfer - ACL packets */
 	[UBT_IF_0_BULK_DT_RD] = {
@@ -286,9 +286,9 @@ static const struct usb2_config		ubt_config[UBT_N_TRANSFER] =
 		.endpoint =	UE_ADDR_ANY,
 		.direction =	UE_DIR_IN,
 		.if_index = 	0,
-		.mh.bufsize =	UBT_BULK_READ_BUFFER_SIZE,
-		.mh.flags =	{ .pipe_bof = 1, .short_xfer_ok = 1, },
-		.mh.callback =	&ubt_bulk_read_callback,
+		.bufsize =	UBT_BULK_READ_BUFFER_SIZE,
+		.flags =	{ .pipe_bof = 1, .short_xfer_ok = 1, },
+		.callback =	&ubt_bulk_read_callback,
 	},
 	/* Incoming interrupt transfer - HCI events */
 	[UBT_IF_0_INTR_DT_RD] = {
@@ -296,9 +296,9 @@ static const struct usb2_config		ubt_config[UBT_N_TRANSFER] =
 		.endpoint =	UE_ADDR_ANY,
 		.direction =	UE_DIR_IN,
 		.if_index = 	0,
-		.mh.flags =	{ .pipe_bof = 1, .short_xfer_ok = 1, },
-		.mh.bufsize =	UBT_INTR_BUFFER_SIZE,
-		.mh.callback =	&ubt_intr_read_callback,
+		.flags =	{ .pipe_bof = 1, .short_xfer_ok = 1, },
+		.bufsize =	UBT_INTR_BUFFER_SIZE,
+		.callback =	&ubt_intr_read_callback,
 	},
 	/* Outgoing control transfer - HCI commands */
 	[UBT_IF_0_CTRL_DT_WR] = {
@@ -306,9 +306,9 @@ static const struct usb2_config		ubt_config[UBT_N_TRANSFER] =
 		.endpoint =	0x00,	/* control pipe */
 		.direction =	UE_DIR_ANY,
 		.if_index = 	0,
-		.mh.bufsize =	UBT_CTRL_BUFFER_SIZE,
-		.mh.callback =	&ubt_ctrl_write_callback,
-		.mh.timeout =	5000,	/* 5 seconds */
+		.bufsize =	UBT_CTRL_BUFFER_SIZE,
+		.callback =	&ubt_ctrl_write_callback,
+		.timeout =	5000,	/* 5 seconds */
 	},
 
 	/*
@@ -321,10 +321,10 @@ static const struct usb2_config		ubt_config[UBT_N_TRANSFER] =
 		.endpoint =	UE_ADDR_ANY,
 		.direction =	UE_DIR_IN,
 		.if_index = 	1,
-		.mh.bufsize =	0,	/* use "wMaxPacketSize * frames" */
-		.mh.frames =	UBT_ISOC_NFRAMES,
-		.mh.flags =	{ .short_xfer_ok = 1, },
-		.mh.callback =	&ubt_isoc_read_callback,
+		.bufsize =	0,	/* use "wMaxPacketSize * frames" */
+		.frames =	UBT_ISOC_NFRAMES,
+		.flags =	{ .short_xfer_ok = 1, },
+		.callback =	&ubt_isoc_read_callback,
 	},
 	/* Incoming isochronous transfer #2 - SCO packets */
 	[UBT_IF_1_ISOC_DT_RD2] = {
@@ -332,10 +332,10 @@ static const struct usb2_config		ubt_config[UBT_N_TRANSFER] =
 		.endpoint =	UE_ADDR_ANY,
 		.direction =	UE_DIR_IN,
 		.if_index = 	1,
-		.mh.bufsize =	0,	/* use "wMaxPacketSize * frames" */
-		.mh.frames =	UBT_ISOC_NFRAMES,
-		.mh.flags =	{ .short_xfer_ok = 1, },
-		.mh.callback =	&ubt_isoc_read_callback,
+		.bufsize =	0,	/* use "wMaxPacketSize * frames" */
+		.frames =	UBT_ISOC_NFRAMES,
+		.flags =	{ .short_xfer_ok = 1, },
+		.callback =	&ubt_isoc_read_callback,
 	},
 	/* Outgoing isochronous transfer #1 - SCO packets */
 	[UBT_IF_1_ISOC_DT_WR1] = {
@@ -343,10 +343,10 @@ static const struct usb2_config		ubt_config[UBT_N_TRANSFER] =
 		.endpoint =	UE_ADDR_ANY,
 		.direction =	UE_DIR_OUT,
 		.if_index = 	1,
-		.mh.bufsize =	0,	/* use "wMaxPacketSize * frames" */
-		.mh.frames =	UBT_ISOC_NFRAMES,
-		.mh.flags =	{ .short_xfer_ok = 1, },
-		.mh.callback =	&ubt_isoc_write_callback,
+		.bufsize =	0,	/* use "wMaxPacketSize * frames" */
+		.frames =	UBT_ISOC_NFRAMES,
+		.flags =	{ .short_xfer_ok = 1, },
+		.callback =	&ubt_isoc_write_callback,
 	},
 	/* Outgoing isochronous transfer #2 - SCO packets */
 	[UBT_IF_1_ISOC_DT_WR2] = {
@@ -354,10 +354,10 @@ static const struct usb2_config		ubt_config[UBT_N_TRANSFER] =
 		.endpoint =	UE_ADDR_ANY,
 		.direction =	UE_DIR_OUT,
 		.if_index = 	1,
-		.mh.bufsize =	0,	/* use "wMaxPacketSize * frames" */
-		.mh.frames =	UBT_ISOC_NFRAMES,
-		.mh.flags =	{ .short_xfer_ok = 1, },
-		.mh.callback =	&ubt_isoc_write_callback,
+		.bufsize =	0,	/* use "wMaxPacketSize * frames" */
+		.frames =	UBT_ISOC_NFRAMES,
+		.flags =	{ .short_xfer_ok = 1, },
+		.callback =	&ubt_isoc_write_callback,
 	},
 };
 
@@ -426,6 +426,7 @@ ubt_attach(device_t dev)
 	struct usb2_attach_arg		*uaa = device_get_ivars(dev);
 	struct ubt_softc		*sc = device_get_softc(dev);
 	struct usb2_endpoint_descriptor	*ed;
+	struct usb2_interface_descriptor *id;
 	uint16_t			wMaxPacketSize;
 	uint8_t				alt_index, i, j;
 	uint8_t				iface_index[2] = { 0, 1 };
@@ -496,31 +497,34 @@ ubt_attach(device_t dev)
 	alt_index = 0;
 	i = 0;
 	j = 0;
+	ed = NULL;
 
-	/* Search through all the descriptors looking for bidir mode */
-	while (1) {
-		uint16_t temp;
-
-		ed = usb2_find_edesc(usb2_get_config_descriptor(uaa->device),
-				1, i, j);
-		if (ed == NULL) {
-			if (j != 0) {
-				/* next interface */
-				j = 0;
-				i ++;
-				continue;
-			}
-
-			break;	/* end of interfaces */
+	/* 
+	 * Search through all the descriptors looking for the largest
+	 * packet size:
+	 */
+	while ((ed = (struct usb2_endpoint_descriptor *)usb2_desc_foreach(
+	    usb2_get_config_descriptor(uaa->device), 
+	    (struct usb2_descriptor *)ed))) {
+
+		if ((ed->bDescriptorType == UDESC_INTERFACE) &&
+		    (ed->bLength >= sizeof(*id))) {
+			id = (struct usb2_interface_descriptor *)ed;
+			i = id->bInterfaceNumber;
+			j = id->bAlternateSetting;
 		}
 
-		temp = UGETW(ed->wMaxPacketSize);
-		if (temp > wMaxPacketSize) {
-			wMaxPacketSize = temp;
-			alt_index = i;
-		}
+		if ((ed->bDescriptorType == UDESC_ENDPOINT) &&
+		    (ed->bLength >= sizeof(*ed)) &&
+		    (i == 1)) {
+			uint16_t temp;
 
-		j ++;
+			temp = UGETW(ed->wMaxPacketSize);
+			if (temp > wMaxPacketSize) {
+				wMaxPacketSize = temp;
+				alt_index = j;
+			}
+		}
 	}
 
 	/* Set alt configuration on interface #1 only if we found it */
diff --git a/sys/dev/usb/bluetooth/ubtbcmfw.c b/sys/dev/usb/bluetooth/ubtbcmfw.c
index bfa390f..1932a39 100644
--- a/sys/dev/usb/bluetooth/ubtbcmfw.c
+++ b/sys/dev/usb/bluetooth/ubtbcmfw.c
@@ -118,10 +118,10 @@ static const struct usb2_config	ubtbcmfw_config[UBTBCMFW_N_TRANSFER] =
 		.endpoint =	0x02,	/* fixed */
 		.direction =	UE_DIR_OUT,
 		.if_index =	UBTBCMFW_IFACE_IDX,
-		.mh.bufsize =	UBTBCMFW_BSIZE,
-		.mh.flags =	{ .pipe_bof = 1, .force_short_xfer = 1,
+		.bufsize =	UBTBCMFW_BSIZE,
+		.flags =	{ .pipe_bof = 1, .force_short_xfer = 1,
 				  .proxy_buffer = 1, },
-		.mh.callback =	&ubtbcmfw_write_callback,
+		.callback =	&ubtbcmfw_write_callback,
 	},
 
 	[UBTBCMFW_INTR_DT_RD] = {
@@ -129,10 +129,10 @@ static const struct usb2_config	ubtbcmfw_config[UBTBCMFW_N_TRANSFER] =
 		.endpoint =	0x01,	/* fixed */
 		.direction =	UE_DIR_IN,
 		.if_index =	UBTBCMFW_IFACE_IDX,
-		.mh.bufsize =	UBTBCMFW_BSIZE,
-		.mh.flags =	{ .pipe_bof = 1, .short_xfer_ok = 1,
+		.bufsize =	UBTBCMFW_BSIZE,
+		.flags =	{ .pipe_bof = 1, .short_xfer_ok = 1,
 				  .proxy_buffer = 1, },
-		.mh.callback =	&ubtbcmfw_read_callback,
+		.callback =	&ubtbcmfw_read_callback,
 	},
 };
 
diff --git a/sys/dev/usb/controller/at91dci.c b/sys/dev/usb/controller/at91dci.c
index fec3b27..ade0c2a 100644
--- a/sys/dev/usb/controller/at91dci.c
+++ b/sys/dev/usb/controller/at91dci.c
@@ -38,7 +38,7 @@ __FBSDID("$FreeBSD$");
 
 /*
  * NOTE: The "fifo_bank" is not reset in hardware when the endpoint is
- * reset !
+ * reset.
  *
  * NOTE: When the chip detects BUS-reset it will also reset the
  * endpoints, Function-address and more.
@@ -47,7 +47,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR at91dcidebug
 
@@ -55,7 +54,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb_debug.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_transfer.h>
 #include <dev/usb/usb_device.h>
 #include <dev/usb/usb_hub.h>
@@ -67,10 +65,10 @@ __FBSDID("$FreeBSD$");
 
 #define	AT9100_DCI_BUS2SC(bus) \
    ((struct at91dci_softc *)(((uint8_t *)(bus)) - \
-   USB_P2U(&(((struct at91dci_softc *)0)->sc_bus))))
+    ((uint8_t *)&(((struct at91dci_softc *)0)->sc_bus))))
 
 #define	AT9100_DCI_PC2SC(pc) \
-   AT9100_DCI_BUS2SC((pc)->tag_parent->info->bus)
+   AT9100_DCI_BUS2SC(USB_DMATAG_TO_XROOT((pc)->tag_parent)->bus)
 
 #if USB_DEBUG
 static int at91dcidebug = 0;
@@ -89,8 +87,6 @@ struct usb2_pipe_methods at91dci_device_bulk_methods;
 struct usb2_pipe_methods at91dci_device_ctrl_methods;
 struct usb2_pipe_methods at91dci_device_intr_methods;
 struct usb2_pipe_methods at91dci_device_isoc_fs_methods;
-struct usb2_pipe_methods at91dci_root_ctrl_methods;
-struct usb2_pipe_methods at91dci_root_intr_methods;
 
 static at91dci_cmd_t at91dci_setup_rx;
 static at91dci_cmd_t at91dci_data_rx;
@@ -98,11 +94,8 @@ static at91dci_cmd_t at91dci_data_tx;
 static at91dci_cmd_t at91dci_data_tx_sync;
 static void	at91dci_device_done(struct usb2_xfer *, usb2_error_t);
 static void	at91dci_do_poll(struct usb2_bus *);
-static void	at91dci_root_ctrl_poll(struct at91dci_softc *);
 static void	at91dci_standard_done(struct usb2_xfer *);
-
-static usb2_sw_transfer_func_t at91dci_root_intr_done;
-static usb2_sw_transfer_func_t at91dci_root_ctrl_done;
+static void	at91dci_root_intr(struct at91dci_softc *sc);
 
 /*
  * NOTE: Some of the bits in the CSR register have inverse meaning so
@@ -257,10 +250,8 @@ at91dci_pull_down(struct at91dci_softc *sc)
 }
 
 static void
-at91dci_wakeup_peer(struct usb2_xfer *xfer)
+at91dci_wakeup_peer(struct at91dci_softc *sc)
 {
-	struct at91dci_softc *sc = AT9100_DCI_BUS2SC(xfer->xroot->bus);
-
 	if (!(sc->sc_flags.status_suspend)) {
 		return;
 	}
@@ -306,14 +297,11 @@ at91dci_setup_rx(struct at91dci_td *td)
 	    AT91_UDP_CSR_TXCOMP);
 
 	if (!(csr & AT91_UDP_CSR_RXSETUP)) {
-		/* abort any ongoing transfer */
-		if (!td->did_stall) {
-			DPRINTFN(5, "stalling\n");
-			temp |= AT91_UDP_CSR_FORCESTALL;
-			td->did_stall = 1;
-		}
 		goto not_complete;
 	}
+	/* clear did stall */
+	td->did_stall = 0;
+
 	/* get the packet byte count */
 	count = (csr & AT91_UDP_CSR_RXBYTECNT) >> 16;
 
@@ -363,6 +351,13 @@ at91dci_setup_rx(struct at91dci_td *td)
 	return (0);			/* complete */
 
 not_complete:
+	/* abort any ongoing transfer */
+	if (!td->did_stall) {
+		DPRINTFN(5, "stalling\n");
+		temp |= AT91_UDP_CSR_FORCESTALL;
+		td->did_stall = 1;
+	}
+
 	/* clear interrupts, if any */
 	if (temp) {
 		DPRINTFN(5, "clearing 0x%08x\n", temp);
@@ -733,9 +728,7 @@ at91dci_vbus_interrupt(struct at91dci_softc *sc, uint8_t is_on)
 			sc->sc_flags.status_vbus = 1;
 
 			/* complete root HUB interrupt endpoint */
-
-			usb2_sw_transfer(&sc->sc_root_intr,
-			    &at91dci_root_intr_done);
+			at91dci_root_intr(sc);
 		}
 	} else {
 		if (sc->sc_flags.status_vbus) {
@@ -746,9 +739,7 @@ at91dci_vbus_interrupt(struct at91dci_softc *sc, uint8_t is_on)
 			sc->sc_flags.change_connect = 1;
 
 			/* complete root HUB interrupt endpoint */
-
-			usb2_sw_transfer(&sc->sc_root_intr,
-			    &at91dci_root_intr_done);
+			at91dci_root_intr(sc);
 		}
 	}
 	USB_BUS_UNLOCK(&sc->sc_bus);
@@ -825,9 +816,7 @@ at91dci_interrupt(struct at91dci_softc *sc)
 			}
 		}
 		/* complete root HUB interrupt endpoint */
-
-		usb2_sw_transfer(&sc->sc_root_intr,
-		    &at91dci_root_intr_done);
+		at91dci_root_intr(sc);
 	}
 	/* check for any endpoint interrupts */
 
@@ -888,8 +877,8 @@ at91dci_setup_standard_chain(struct usb2_xfer *xfer)
 
 	temp.td = NULL;
 	temp.td_next = xfer->td_start[0];
-	temp.setup_alt_next = xfer->flags_int.short_frames_ok;
 	temp.offset = 0;
+	temp.setup_alt_next = xfer->flags_int.short_frames_ok;
 
 	sc = AT9100_DCI_BUS2SC(xfer->xroot->bus);
 	ep_no = (xfer->endpoint & UE_ADDR);
@@ -903,6 +892,12 @@ at91dci_setup_standard_chain(struct usb2_xfer *xfer)
 			temp.len = xfer->frlengths[0];
 			temp.pc = xfer->frbuffers + 0;
 			temp.short_pkt = temp.len ? 1 : 0;
+			/* check for last frame */
+			if (xfer->nframes == 1) {
+				/* no STATUS stage yet, SETUP is last */
+				if (xfer->flags_int.control_act)
+					temp.setup_alt_next = 0;
+			}
 
 			at91dci_setup_standard_chain_sub(&temp);
 		}
@@ -934,7 +929,13 @@ at91dci_setup_standard_chain(struct usb2_xfer *xfer)
 		x++;
 
 		if (x == xfer->nframes) {
-			temp.setup_alt_next = 0;
+			if (xfer->flags_int.control_xfr) {
+				if (xfer->flags_int.control_act) {
+					temp.setup_alt_next = 0;
+				}
+			} else {
+				temp.setup_alt_next = 0;
+			}
 		}
 		if (temp.len == 0) {
 
@@ -959,47 +960,46 @@ at91dci_setup_standard_chain(struct usb2_xfer *xfer)
 		}
 	}
 
-	/* always setup a valid "pc" pointer for status and sync */
-	temp.pc = xfer->frbuffers + 0;
-
-	/* check if we need to sync */
-	if (need_sync && xfer->flags_int.control_xfr) {
-
-		/* we need a SYNC point after TX */
-		temp.func = &at91dci_data_tx_sync;
-		temp.len = 0;
-		temp.short_pkt = 0;
-
-		at91dci_setup_standard_chain_sub(&temp);
-	}
-	/* check if we should append a status stage */
-	if (xfer->flags_int.control_xfr &&
-	    !xfer->flags_int.control_act) {
+	/* check for control transfer */
+	if (xfer->flags_int.control_xfr) {
 
-		/*
-		 * Send a DATA1 message and invert the current
-		 * endpoint direction.
-		 */
-		if (xfer->endpoint & UE_DIR_IN) {
-			temp.func = &at91dci_data_rx;
-			need_sync = 0;
-		} else {
-			temp.func = &at91dci_data_tx;
-			need_sync = 1;
-		}
+		/* always setup a valid "pc" pointer for status and sync */
+		temp.pc = xfer->frbuffers + 0;
 		temp.len = 0;
 		temp.short_pkt = 0;
+		temp.setup_alt_next = 0;
 
-		at91dci_setup_standard_chain_sub(&temp);
+		/* check if we need to sync */
 		if (need_sync) {
 			/* we need a SYNC point after TX */
 			temp.func = &at91dci_data_tx_sync;
-			temp.len = 0;
-			temp.short_pkt = 0;
+			at91dci_setup_standard_chain_sub(&temp);
+		}
+
+		/* check if we should append a status stage */
+		if (!xfer->flags_int.control_act) {
+
+			/*
+			 * Send a DATA1 message and invert the current
+			 * endpoint direction.
+			 */
+			if (xfer->endpoint & UE_DIR_IN) {
+				temp.func = &at91dci_data_rx;
+				need_sync = 0;
+			} else {
+				temp.func = &at91dci_data_tx;
+				need_sync = 1;
+			}
 
 			at91dci_setup_standard_chain_sub(&temp);
+			if (need_sync) {
+				/* we need a SYNC point after TX */
+				temp.func = &at91dci_data_tx_sync;
+				at91dci_setup_standard_chain_sub(&temp);
+			}
 		}
 	}
+
 	/* must have at least one frame! */
 	td = temp.td;
 	xfer->td_transfer_last = td;
@@ -1056,31 +1056,17 @@ at91dci_start_standard_chain(struct usb2_xfer *xfer)
 }
 
 static void
-at91dci_root_intr_done(struct usb2_xfer *xfer,
-    struct usb2_sw_transfer *std)
+at91dci_root_intr(struct at91dci_softc *sc)
 {
-	struct at91dci_softc *sc = AT9100_DCI_BUS2SC(xfer->xroot->bus);
-
 	DPRINTFN(9, "\n");
 
 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 
-	if (std->state != USB_SW_TR_PRE_DATA) {
-		if (std->state == USB_SW_TR_PRE_CALLBACK) {
-			/* transfer transferred */
-			at91dci_device_done(xfer, std->err);
-		}
-		goto done;
-	}
-	/* setup buffer */
-	std->ptr = sc->sc_hub_idata;
-	std->len = sizeof(sc->sc_hub_idata);
-
 	/* set port bit */
 	sc->sc_hub_idata[0] = 0x02;	/* we only have one port */
 
-done:
-	return;
+	uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
+	    sizeof(sc->sc_hub_idata));
 }
 
 static usb2_error_t
@@ -1474,7 +1460,6 @@ at91dci_do_poll(struct usb2_bus *bus)
 
 	USB_BUS_LOCK(&sc->sc_bus);
 	at91dci_interrupt_poll(sc);
-	at91dci_root_ctrl_poll(sc);
 	USB_BUS_UNLOCK(&sc->sc_bus);
 }
 
@@ -1513,8 +1498,6 @@ struct usb2_pipe_methods at91dci_device_bulk_methods =
 	.close = at91dci_device_bulk_close,
 	.enter = at91dci_device_bulk_enter,
 	.start = at91dci_device_bulk_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -1552,8 +1535,6 @@ struct usb2_pipe_methods at91dci_device_ctrl_methods =
 	.close = at91dci_device_ctrl_close,
 	.enter = at91dci_device_ctrl_enter,
 	.start = at91dci_device_ctrl_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -1591,8 +1572,6 @@ struct usb2_pipe_methods at91dci_device_intr_methods =
 	.close = at91dci_device_intr_close,
 	.enter = at91dci_device_intr_enter,
 	.start = at91dci_device_intr_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -1675,38 +1654,14 @@ struct usb2_pipe_methods at91dci_device_isoc_fs_methods =
 	.close = at91dci_device_isoc_fs_close,
 	.enter = at91dci_device_isoc_fs_enter,
 	.start = at91dci_device_isoc_fs_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
  * at91dci root control support
  *------------------------------------------------------------------------*
- * simulate a hardware HUB by handling
- * all the necessary requests
+ * Simulate a hardware HUB by handling all the necessary requests.
  *------------------------------------------------------------------------*/
 
-static void
-at91dci_root_ctrl_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-at91dci_root_ctrl_close(struct usb2_xfer *xfer)
-{
-	struct at91dci_softc *sc = AT9100_DCI_BUS2SC(xfer->xroot->bus);
-
-	if (sc->sc_root_ctrl.xfer == xfer) {
-		sc->sc_root_ctrl.xfer = NULL;
-	}
-	at91dci_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-/*
- * USB descriptors for the virtual Root HUB:
- */
-
 static const struct usb2_device_descriptor at91dci_devd = {
 	.bLength = sizeof(struct usb2_device_descriptor),
 	.bDescriptorType = UDESC_DEVICE,
@@ -1751,7 +1706,6 @@ static const struct at91dci_config_desc at91dci_confd = {
 		.bInterfaceSubClass = UISUBCLASS_HUB,
 		.bInterfaceProtocol = UIPROTO_HSHUBSTT,
 	},
-
 	.endpd = {
 		.bLength = sizeof(struct usb2_endpoint_descriptor),
 		.bDescriptorType = UDESC_ENDPOINT,
@@ -1791,44 +1745,15 @@ USB_MAKE_STRING_DESC(STRING_VENDOR, at91dci_vendor);
 USB_MAKE_STRING_DESC(STRING_PRODUCT, at91dci_product);
 
 static void
-at91dci_root_ctrl_enter(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-at91dci_root_ctrl_start(struct usb2_xfer *xfer)
-{
-	struct at91dci_softc *sc = AT9100_DCI_BUS2SC(xfer->xroot->bus);
-
-	sc->sc_root_ctrl.xfer = xfer;
-
-	usb2_bus_roothub_exec(xfer->xroot->bus);
-}
-
-static void
-at91dci_root_ctrl_task(struct usb2_bus *bus)
-{
-	at91dci_root_ctrl_poll(AT9100_DCI_BUS2SC(bus));
-}
-
-static void
-at91dci_root_ctrl_done(struct usb2_xfer *xfer,
-    struct usb2_sw_transfer *std)
+at91dci_roothub_exec(struct usb2_bus *bus)
 {
-	struct at91dci_softc *sc = AT9100_DCI_BUS2SC(xfer->xroot->bus);
+	struct at91dci_softc *sc = AT9100_DCI_BUS2SC(bus);
+	struct usb2_sw_transfer *std = &sc->sc_bus.roothub_req;
 	uint16_t value;
 	uint16_t index;
 
 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 
-	if (std->state != USB_SW_TR_SETUP) {
-		if (std->state == USB_SW_TR_PRE_CALLBACK) {
-			/* transfer transferred */
-			at91dci_device_done(xfer, std->err);
-		}
-		goto done;
-	}
 	/* buffer reset */
 	std->ptr = USB_ADD_BYTES(&sc->sc_hub_temp, 0);
 	std->len = 0;
@@ -2087,7 +2012,7 @@ tr_handle_clear_port_feature:
 
 	switch (value) {
 	case UHF_PORT_SUSPEND:
-		at91dci_wakeup_peer(xfer);
+		at91dci_wakeup_peer(sc);
 		break;
 
 	case UHF_PORT_ENABLE:
@@ -2211,67 +2136,6 @@ done:
 }
 
 static void
-at91dci_root_ctrl_poll(struct at91dci_softc *sc)
-{
-	usb2_sw_transfer(&sc->sc_root_ctrl,
-	    &at91dci_root_ctrl_done);
-}
-
-struct usb2_pipe_methods at91dci_root_ctrl_methods =
-{
-	.open = at91dci_root_ctrl_open,
-	.close = at91dci_root_ctrl_close,
-	.enter = at91dci_root_ctrl_enter,
-	.start = at91dci_root_ctrl_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 0,
-};
-
-/*------------------------------------------------------------------------*
- * at91dci root interrupt support
- *------------------------------------------------------------------------*/
-static void
-at91dci_root_intr_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-at91dci_root_intr_close(struct usb2_xfer *xfer)
-{
-	struct at91dci_softc *sc = AT9100_DCI_BUS2SC(xfer->xroot->bus);
-
-	if (sc->sc_root_intr.xfer == xfer) {
-		sc->sc_root_intr.xfer = NULL;
-	}
-	at91dci_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-static void
-at91dci_root_intr_enter(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-at91dci_root_intr_start(struct usb2_xfer *xfer)
-{
-	struct at91dci_softc *sc = AT9100_DCI_BUS2SC(xfer->xroot->bus);
-
-	sc->sc_root_intr.xfer = xfer;
-}
-
-struct usb2_pipe_methods at91dci_root_intr_methods =
-{
-	.open = at91dci_root_intr_open,
-	.close = at91dci_root_intr_close,
-	.enter = at91dci_root_intr_enter,
-	.start = at91dci_root_intr_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
-};
-
-static void
 at91dci_xfer_setup(struct usb2_setup_params *parm)
 {
 	const struct usb2_hw_ep_profile *pf;
@@ -2397,24 +2261,7 @@ at91dci_pipe_init(struct usb2_device *udev, struct usb2_endpoint_descriptor *ede
 	    edesc->bEndpointAddress, udev->flags.usb2_mode,
 	    sc->sc_rt_addr);
 
-	if (udev->device_index == sc->sc_rt_addr) {
-
-		if (udev->flags.usb2_mode != USB_MODE_HOST) {
-			/* not supported */
-			return;
-		}
-		switch (edesc->bEndpointAddress) {
-		case USB_CONTROL_ENDPOINT:
-			pipe->methods = &at91dci_root_ctrl_methods;
-			break;
-		case UE_DIR_IN | AT9100_DCI_INTR_ENDPT:
-			pipe->methods = &at91dci_root_intr_methods;
-			break;
-		default:
-			/* do nothing */
-			break;
-		}
-	} else {
+	if (udev->device_index != sc->sc_rt_addr) {
 
 		if (udev->flags.usb2_mode != USB_MODE_DEVICE) {
 			/* not supported */
@@ -2452,5 +2299,5 @@ struct usb2_bus_methods at91dci_bus_methods =
 	.get_hw_ep_profile = &at91dci_get_hw_ep_profile,
 	.set_stall = &at91dci_set_stall,
 	.clear_stall = &at91dci_clear_stall,
-	.roothub_exec = &at91dci_root_ctrl_task,
+	.roothub_exec = &at91dci_roothub_exec,
 };
diff --git a/sys/dev/usb/controller/at91dci.h b/sys/dev/usb/controller/at91dci.h
index 6464ad7..8a745c8 100644
--- a/sys/dev/usb/controller/at91dci.h
+++ b/sys/dev/usb/controller/at91dci.h
@@ -27,8 +27,8 @@
  */
 
 /*
- * USB Device Port (UDP) register definition, based on
- * "AT91RM9200.h" provided by ATMEL.
+ * USB Device Port (UDP) register definition, based on "AT91RM9200.h" provided
+ * by ATMEL.
  */
 
 #ifndef _AT9100_DCI_H_
@@ -204,8 +204,6 @@ struct at91dci_softc {
 	struct usb2_bus sc_bus;
 	union at91dci_hub_temp sc_hub_temp;
 	LIST_HEAD(, usb2_xfer) sc_interrupt_list_head;
-	struct usb2_sw_transfer sc_root_ctrl;
-	struct usb2_sw_transfer sc_root_intr;
 
 	struct usb2_device *sc_devices[AT91_MAX_DEVICES];
 	struct resource *sc_io_res;
diff --git a/sys/dev/usb/controller/at91dci_atmelarm.c b/sys/dev/usb/controller/at91dci_atmelarm.c
index 71d2937..aedd30f 100644
--- a/sys/dev/usb/controller/at91dci_atmelarm.c
+++ b/sys/dev/usb/controller/at91dci_atmelarm.c
@@ -28,13 +28,11 @@ __FBSDID("$FreeBSD$");
 
 
 #include <dev/usb/usb_mfunc.h>
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb.h>
 
 #include <dev/usb/usb_core.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_util.h>
 
 #include <dev/usb/usb_controller.h>
@@ -205,10 +203,10 @@ at91_udp_attach(device_t dev)
 
 #if (__FreeBSD_version >= 700031)
 	err = bus_setup_intr(dev, sc->sc_dci.sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    NULL, (void *)at91dci_interrupt, sc, &sc->sc_dci.sc_intr_hdl);
+	    NULL, (driver_intr_t *)at91dci_interrupt, sc, &sc->sc_dci.sc_intr_hdl);
 #else
 	err = bus_setup_intr(dev, sc->sc_dci.sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    (void *)at91dci_interrupt, sc, &sc->sc_dci.sc_intr_hdl);
+	    (driver_intr_t *)at91dci_interrupt, sc, &sc->sc_dci.sc_intr_hdl);
 #endif
 	if (err) {
 		sc->sc_dci.sc_intr_hdl = NULL;
@@ -216,10 +214,10 @@ at91_udp_attach(device_t dev)
 	}
 #if (__FreeBSD_version >= 700031)
 	err = bus_setup_intr(dev, sc->sc_vbus_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    NULL, (void *)at91_vbus_poll, sc, &sc->sc_vbus_intr_hdl);
+	    NULL, (driver_intr_t *)at91_vbus_poll, sc, &sc->sc_vbus_intr_hdl);
 #else
 	err = bus_setup_intr(dev, sc->sc_vbus_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    (void *)at91_vbus_poll, sc, &sc->sc_vbus_intr_hdl);
+	    (driver_intr_t *)at91_vbus_poll, sc, &sc->sc_vbus_intr_hdl);
 #endif
 	if (err) {
 		sc->sc_vbus_intr_hdl = NULL;
diff --git a/sys/dev/usb/controller/atmegadci.c b/sys/dev/usb/controller/atmegadci.c
index edaef19..e182fc1 100644
--- a/sys/dev/usb/controller/atmegadci.c
+++ b/sys/dev/usb/controller/atmegadci.c
@@ -27,9 +27,8 @@ __FBSDID("$FreeBSD$");
  */
 
 /*
- * This file contains the driver for the ATMEGA series USB OTG
- * Controller. This driver currently only supports the DCI mode of the
- * USB hardware.
+ * This file contains the driver for the ATMEGA series USB OTG Controller. This
+ * driver currently only supports the DCI mode of the USB hardware.
  */
 
 /*
@@ -40,7 +39,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR atmegadci_debug
 
@@ -48,7 +46,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb_debug.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_transfer.h>
 #include <dev/usb/usb_device.h>
 #include <dev/usb/usb_hub.h>
@@ -60,10 +57,10 @@ __FBSDID("$FreeBSD$");
 
 #define	ATMEGA_BUS2SC(bus) \
    ((struct atmegadci_softc *)(((uint8_t *)(bus)) - \
-   USB_P2U(&(((struct atmegadci_softc *)0)->sc_bus))))
+    ((uint8_t *)&(((struct atmegadci_softc *)0)->sc_bus))))
 
 #define	ATMEGA_PC2SC(pc) \
-   ATMEGA_BUS2SC((pc)->tag_parent->info->bus)
+   ATMEGA_BUS2SC(USB_DMATAG_TO_XROOT((pc)->tag_parent)->bus)
 
 #if USB_DEBUG
 static int atmegadci_debug = 0;
@@ -78,12 +75,8 @@ SYSCTL_INT(_hw_usb2_atmegadci, OID_AUTO, debug, CTLFLAG_RW,
 /* prototypes */
 
 struct usb2_bus_methods atmegadci_bus_methods;
-struct usb2_pipe_methods atmegadci_device_bulk_methods;
-struct usb2_pipe_methods atmegadci_device_ctrl_methods;
-struct usb2_pipe_methods atmegadci_device_intr_methods;
+struct usb2_pipe_methods atmegadci_device_non_isoc_methods;
 struct usb2_pipe_methods atmegadci_device_isoc_fs_methods;
-struct usb2_pipe_methods atmegadci_root_ctrl_methods;
-struct usb2_pipe_methods atmegadci_root_intr_methods;
 
 static atmegadci_cmd_t atmegadci_setup_rx;
 static atmegadci_cmd_t atmegadci_data_rx;
@@ -91,11 +84,8 @@ static atmegadci_cmd_t atmegadci_data_tx;
 static atmegadci_cmd_t atmegadci_data_tx_sync;
 static void atmegadci_device_done(struct usb2_xfer *, usb2_error_t);
 static void atmegadci_do_poll(struct usb2_bus *);
-static void atmegadci_root_ctrl_poll(struct atmegadci_softc *);
 static void atmegadci_standard_done(struct usb2_xfer *);
-
-static usb2_sw_transfer_func_t atmegadci_root_intr_done;
-static usb2_sw_transfer_func_t atmegadci_root_ctrl_done;
+static void atmegadci_root_intr(struct atmegadci_softc *sc);
 
 /*
  * Here is a list of what the chip supports:
@@ -202,9 +192,8 @@ atmegadci_pull_down(struct atmegadci_softc *sc)
 }
 
 static void
-atmegadci_wakeup_peer(struct usb2_xfer *xfer)
+atmegadci_wakeup_peer(struct atmegadci_softc *sc)
 {
-	struct atmegadci_softc *sc = ATMEGA_BUS2SC(xfer->xroot->bus);
 	uint8_t temp;
 
 	if (!sc->sc_flags.status_suspend) {
@@ -251,16 +240,10 @@ atmegadci_setup_rx(struct atmegadci_td *td)
 	DPRINTFN(5, "UEINTX=0x%02x\n", temp);
 
 	if (!(temp & ATMEGA_UEINTX_RXSTPI)) {
-		/* abort any ongoing transfer */
-		if (!td->did_stall) {
-			DPRINTFN(5, "stalling\n");
-			ATMEGA_WRITE_1(sc, ATMEGA_UECONX,
-			    ATMEGA_UECONX_EPEN |
-			    ATMEGA_UECONX_STALLRQ);
-			td->did_stall = 1;
-		}
 		goto not_complete;
 	}
+	/* clear did stall */
+	td->did_stall = 0;
 	/* get the packet byte count */
 	count =
 	    (ATMEGA_READ_1(sc, ATMEGA_UEBCHX) << 8) |
@@ -305,6 +288,18 @@ atmegadci_setup_rx(struct atmegadci_td *td)
 	return (0);			/* complete */
 
 not_complete:
+	/* abort any ongoing transfer */
+	if (!td->did_stall) {
+		DPRINTFN(5, "stalling\n");
+		ATMEGA_WRITE_1(sc, ATMEGA_UECONX,
+		    ATMEGA_UECONX_EPEN |
+		    ATMEGA_UECONX_STALLRQ);
+		td->did_stall = 1;
+	}
+	if (temp & ATMEGA_UEINTX_RXSTPI) {
+		/* clear SETUP packet interrupt */
+		ATMEGA_WRITE_1(sc, ATMEGA_UEINTX, ~ATMEGA_UEINTX_RXSTPI);
+	}
 	/* we only want to know if there is a SETUP packet */
 	ATMEGA_WRITE_1(sc, ATMEGA_UEIENX, ATMEGA_UEIENX_RXSTPE);
 	return (1);			/* not complete */
@@ -620,8 +615,7 @@ atmegadci_vbus_interrupt(struct atmegadci_softc *sc, uint8_t is_on)
 
 			/* complete root HUB interrupt endpoint */
 
-			usb2_sw_transfer(&sc->sc_root_intr,
-			    &atmegadci_root_intr_done);
+			atmegadci_root_intr(sc);
 		}
 	} else {
 		if (sc->sc_flags.status_vbus) {
@@ -633,8 +627,7 @@ atmegadci_vbus_interrupt(struct atmegadci_softc *sc, uint8_t is_on)
 
 			/* complete root HUB interrupt endpoint */
 
-			usb2_sw_transfer(&sc->sc_root_intr,
-			    &atmegadci_root_intr_done);
+			atmegadci_root_intr(sc);
 		}
 	}
 }
@@ -650,7 +643,7 @@ atmegadci_interrupt(struct atmegadci_softc *sc)
 	status = ATMEGA_READ_1(sc, ATMEGA_UDINT);
 
 	/* clear all set interrupts */
-	ATMEGA_WRITE_1(sc, ATMEGA_UDINT, ~status);
+	ATMEGA_WRITE_1(sc, ATMEGA_UDINT, (~status) & 0x7D);
 
 	DPRINTFN(14, "UDINT=0x%02x\n", status);
 
@@ -671,8 +664,7 @@ atmegadci_interrupt(struct atmegadci_softc *sc)
 		    ATMEGA_UDINT_EORSTE);
 
 		/* complete root HUB interrupt endpoint */
-		usb2_sw_transfer(&sc->sc_root_intr,
-		    &atmegadci_root_intr_done);
+		atmegadci_root_intr(sc);
 	}
 	/*
 	 * If resume and suspend is set at the same time we interpret
@@ -694,8 +686,7 @@ atmegadci_interrupt(struct atmegadci_softc *sc)
 			    ATMEGA_UDINT_EORSTE);
 
 			/* complete root HUB interrupt endpoint */
-			usb2_sw_transfer(&sc->sc_root_intr,
-			    &atmegadci_root_intr_done);
+			atmegadci_root_intr(sc);
 		}
 	} else if (status & ATMEGA_UDINT_SUSPI) {
 
@@ -712,15 +703,14 @@ atmegadci_interrupt(struct atmegadci_softc *sc)
 			    ATMEGA_UDINT_EORSTE);
 
 			/* complete root HUB interrupt endpoint */
-			usb2_sw_transfer(&sc->sc_root_intr,
-			    &atmegadci_root_intr_done);
+			atmegadci_root_intr(sc);
 		}
 	}
 	/* check VBUS */
 	status = ATMEGA_READ_1(sc, ATMEGA_USBINT);
 
 	/* clear all set interrupts */
-	ATMEGA_WRITE_1(sc, ATMEGA_USBINT, ~status);
+	ATMEGA_WRITE_1(sc, ATMEGA_USBINT, (~status) & 0x03);
 
 	if (status & ATMEGA_USBINT_VBUSTI) {
 		uint8_t temp;
@@ -732,10 +722,7 @@ atmegadci_interrupt(struct atmegadci_softc *sc)
 	}
 	/* check for any endpoint interrupts */
 	status = ATMEGA_READ_1(sc, ATMEGA_UEINT);
-
-	/* clear all set interrupts */
-	ATMEGA_WRITE_1(sc, ATMEGA_UEINT, ~status);
-
+	/* the hardware will clear the UEINT bits automatically */
 	if (status) {
 
 		DPRINTFN(5, "real endpoint interrupt UEINT=0x%02x\n", status);
@@ -792,8 +779,8 @@ atmegadci_setup_standard_chain(struct usb2_xfer *xfer)
 
 	temp.td = NULL;
 	temp.td_next = xfer->td_start[0];
-	temp.setup_alt_next = xfer->flags_int.short_frames_ok;
 	temp.offset = 0;
+	temp.setup_alt_next = xfer->flags_int.short_frames_ok;
 
 	sc = ATMEGA_BUS2SC(xfer->xroot->bus);
 	ep_no = (xfer->endpoint & UE_ADDR);
@@ -807,6 +794,12 @@ atmegadci_setup_standard_chain(struct usb2_xfer *xfer)
 			temp.len = xfer->frlengths[0];
 			temp.pc = xfer->frbuffers + 0;
 			temp.short_pkt = temp.len ? 1 : 0;
+			/* check for last frame */
+			if (xfer->nframes == 1) {
+				/* no STATUS stage yet, SETUP is last */
+				if (xfer->flags_int.control_act)
+					temp.setup_alt_next = 0;
+			}
 
 			atmegadci_setup_standard_chain_sub(&temp);
 		}
@@ -838,7 +831,13 @@ atmegadci_setup_standard_chain(struct usb2_xfer *xfer)
 		x++;
 
 		if (x == xfer->nframes) {
-			temp.setup_alt_next = 0;
+			if (xfer->flags_int.control_xfr) {
+				if (xfer->flags_int.control_act) {
+					temp.setup_alt_next = 0;
+				}
+			} else {
+				temp.setup_alt_next = 0;
+			}
 		}
 		if (temp.len == 0) {
 
@@ -863,45 +862,42 @@ atmegadci_setup_standard_chain(struct usb2_xfer *xfer)
 		}
 	}
 
-	/* always setup a valid "pc" pointer for status and sync */
-	temp.pc = xfer->frbuffers + 0;
-
-	/* check if we need to sync */
-	if (need_sync && xfer->flags_int.control_xfr) {
-
-		/* we need a SYNC point after TX */
-		temp.func = &atmegadci_data_tx_sync;
-		temp.len = 0;
-		temp.short_pkt = 0;
-
-		atmegadci_setup_standard_chain_sub(&temp);
-	}
-	/* check if we should append a status stage */
-	if (xfer->flags_int.control_xfr &&
-	    !xfer->flags_int.control_act) {
+	if (xfer->flags_int.control_xfr) {
 
-		/*
-		 * Send a DATA1 message and invert the current
-		 * endpoint direction.
-		 */
-		if (xfer->endpoint & UE_DIR_IN) {
-			temp.func = &atmegadci_data_rx;
-			need_sync = 0;
-		} else {
-			temp.func = &atmegadci_data_tx;
-			need_sync = 1;
-		}
+		/* always setup a valid "pc" pointer for status and sync */
+		temp.pc = xfer->frbuffers + 0;
 		temp.len = 0;
 		temp.short_pkt = 0;
+		temp.setup_alt_next = 0;
 
-		atmegadci_setup_standard_chain_sub(&temp);
+		/* check if we need to sync */
 		if (need_sync) {
 			/* we need a SYNC point after TX */
 			temp.func = &atmegadci_data_tx_sync;
-			temp.len = 0;
-			temp.short_pkt = 0;
+			atmegadci_setup_standard_chain_sub(&temp);
+		}
+
+		/* check if we should append a status stage */
+		if (!xfer->flags_int.control_act) {
+
+			/*
+			 * Send a DATA1 message and invert the current
+			 * endpoint direction.
+			 */
+			if (xfer->endpoint & UE_DIR_IN) {
+				temp.func = &atmegadci_data_rx;
+				need_sync = 0;
+			} else {
+				temp.func = &atmegadci_data_tx;
+				need_sync = 1;
+			}
 
 			atmegadci_setup_standard_chain_sub(&temp);
+			if (need_sync) {
+				/* we need a SYNC point after TX */
+				temp.func = &atmegadci_data_tx_sync;
+				atmegadci_setup_standard_chain_sub(&temp);
+			}
 		}
 	}
 	/* must have at least one frame! */
@@ -942,32 +938,18 @@ atmegadci_start_standard_chain(struct usb2_xfer *xfer)
 }
 
 static void
-atmegadci_root_intr_done(struct usb2_xfer *xfer,
-    struct usb2_sw_transfer *std)
+atmegadci_root_intr(struct atmegadci_softc *sc)
 {
-	struct atmegadci_softc *sc = ATMEGA_BUS2SC(xfer->xroot->bus);
-
 	DPRINTFN(9, "\n");
 
 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 
-	if (std->state != USB_SW_TR_PRE_DATA) {
-		if (std->state == USB_SW_TR_PRE_CALLBACK) {
-			/* transfer transferred */
-			atmegadci_device_done(xfer, std->err);
-		}
-		goto done;
-	}
-	/* setup buffer */
-	std->ptr = sc->sc_hub_idata;
-	std->len = sizeof(sc->sc_hub_idata);
-
 	/* set port bit */
 	sc->sc_hub_idata[0] = 0x02;	/* we only have one port */
 
-done:
-	return;
-}
+	uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
+	    sizeof(sc->sc_hub_idata));
+ }
 
 static usb2_error_t
 atmegadci_standard_done_sub(struct usb2_xfer *xfer)
@@ -1241,6 +1223,12 @@ atmegadci_init(struct atmegadci_softc *sc)
 	ATMEGA_WRITE_1(sc, ATMEGA_UHWCON,
 	    ATMEGA_UHWCON_UVREGE | ATMEGA_UHWCON_UIMOD);
 #endif
+	/* make sure USB is enabled */
+	ATMEGA_WRITE_1(sc, ATMEGA_USBCON,
+	    ATMEGA_USBCON_USBE |
+	    ATMEGA_USBCON_OTGPADE |
+	    ATMEGA_USBCON_VBUSTE);
+
 	/* turn on clocks */
 	(sc->sc_clocks_on) (&sc->sc_bus);
 
@@ -1346,125 +1334,46 @@ atmegadci_do_poll(struct usb2_bus *bus)
 
 	USB_BUS_LOCK(&sc->sc_bus);
 	atmegadci_interrupt_poll(sc);
-	atmegadci_root_ctrl_poll(sc);
 	USB_BUS_UNLOCK(&sc->sc_bus);
 }
 
 /*------------------------------------------------------------------------*
  * at91dci bulk support
- *------------------------------------------------------------------------*/
-static void
-atmegadci_device_bulk_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-atmegadci_device_bulk_close(struct usb2_xfer *xfer)
-{
-	atmegadci_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-static void
-atmegadci_device_bulk_enter(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-atmegadci_device_bulk_start(struct usb2_xfer *xfer)
-{
-	/* setup TDs */
-	atmegadci_setup_standard_chain(xfer);
-	atmegadci_start_standard_chain(xfer);
-}
-
-struct usb2_pipe_methods atmegadci_device_bulk_methods =
-{
-	.open = atmegadci_device_bulk_open,
-	.close = atmegadci_device_bulk_close,
-	.enter = atmegadci_device_bulk_enter,
-	.start = atmegadci_device_bulk_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
-};
-
-/*------------------------------------------------------------------------*
  * at91dci control support
- *------------------------------------------------------------------------*/
-static void
-atmegadci_device_ctrl_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-atmegadci_device_ctrl_close(struct usb2_xfer *xfer)
-{
-	atmegadci_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-static void
-atmegadci_device_ctrl_enter(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-atmegadci_device_ctrl_start(struct usb2_xfer *xfer)
-{
-	/* setup TDs */
-	atmegadci_setup_standard_chain(xfer);
-	atmegadci_start_standard_chain(xfer);
-}
-
-struct usb2_pipe_methods atmegadci_device_ctrl_methods =
-{
-	.open = atmegadci_device_ctrl_open,
-	.close = atmegadci_device_ctrl_close,
-	.enter = atmegadci_device_ctrl_enter,
-	.start = atmegadci_device_ctrl_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
-};
-
-/*------------------------------------------------------------------------*
  * at91dci interrupt support
  *------------------------------------------------------------------------*/
 static void
-atmegadci_device_intr_open(struct usb2_xfer *xfer)
+atmegadci_device_non_isoc_open(struct usb2_xfer *xfer)
 {
 	return;
 }
 
 static void
-atmegadci_device_intr_close(struct usb2_xfer *xfer)
+atmegadci_device_non_isoc_close(struct usb2_xfer *xfer)
 {
 	atmegadci_device_done(xfer, USB_ERR_CANCELLED);
 }
 
 static void
-atmegadci_device_intr_enter(struct usb2_xfer *xfer)
+atmegadci_device_non_isoc_enter(struct usb2_xfer *xfer)
 {
 	return;
 }
 
 static void
-atmegadci_device_intr_start(struct usb2_xfer *xfer)
+atmegadci_device_non_isoc_start(struct usb2_xfer *xfer)
 {
 	/* setup TDs */
 	atmegadci_setup_standard_chain(xfer);
 	atmegadci_start_standard_chain(xfer);
 }
 
-struct usb2_pipe_methods atmegadci_device_intr_methods =
+struct usb2_pipe_methods atmegadci_device_non_isoc_methods =
 {
-	.open = atmegadci_device_intr_open,
-	.close = atmegadci_device_intr_close,
-	.enter = atmegadci_device_intr_enter,
-	.start = atmegadci_device_intr_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
+	.open = atmegadci_device_non_isoc_open,
+	.close = atmegadci_device_non_isoc_close,
+	.enter = atmegadci_device_non_isoc_enter,
+	.start = atmegadci_device_non_isoc_start,
 };
 
 /*------------------------------------------------------------------------*
@@ -1551,38 +1460,14 @@ struct usb2_pipe_methods atmegadci_device_isoc_fs_methods =
 	.close = atmegadci_device_isoc_fs_close,
 	.enter = atmegadci_device_isoc_fs_enter,
 	.start = atmegadci_device_isoc_fs_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
  * at91dci root control support
  *------------------------------------------------------------------------*
- * simulate a hardware HUB by handling
- * all the necessary requests
+ * Simulate a hardware HUB by handling all the necessary requests.
  *------------------------------------------------------------------------*/
 
-static void
-atmegadci_root_ctrl_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-atmegadci_root_ctrl_close(struct usb2_xfer *xfer)
-{
-	struct atmegadci_softc *sc = ATMEGA_BUS2SC(xfer->xroot->bus);
-
-	if (sc->sc_root_ctrl.xfer == xfer) {
-		sc->sc_root_ctrl.xfer = NULL;
-	}
-	atmegadci_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-/*
- * USB descriptors for the virtual Root HUB:
- */
-
 static const struct usb2_device_descriptor atmegadci_devd = {
 	.bLength = sizeof(struct usb2_device_descriptor),
 	.bDescriptorType = UDESC_DEVICE,
@@ -1627,7 +1512,6 @@ static const struct atmegadci_config_desc atmegadci_confd = {
 		.bInterfaceSubClass = UISUBCLASS_HUB,
 		.bInterfaceProtocol = UIPROTO_HSHUBSTT,
 	},
-
 	.endpd = {
 		.bLength = sizeof(struct usb2_endpoint_descriptor),
 		.bDescriptorType = UDESC_ENDPOINT,
@@ -1667,45 +1551,16 @@ USB_MAKE_STRING_DESC(STRING_VENDOR, atmegadci_vendor);
 USB_MAKE_STRING_DESC(STRING_PRODUCT, atmegadci_product);
 
 static void
-atmegadci_root_ctrl_enter(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-atmegadci_root_ctrl_start(struct usb2_xfer *xfer)
-{
-	struct atmegadci_softc *sc = ATMEGA_BUS2SC(xfer->xroot->bus);
-
-	sc->sc_root_ctrl.xfer = xfer;
-
-	usb2_bus_roothub_exec(xfer->xroot->bus);
-}
-
-static void
-atmegadci_root_ctrl_task(struct usb2_bus *bus)
-{
-	atmegadci_root_ctrl_poll(ATMEGA_BUS2SC(bus));
-}
-
-static void
-atmegadci_root_ctrl_done(struct usb2_xfer *xfer,
-    struct usb2_sw_transfer *std)
+atmegadci_roothub_exec(struct usb2_bus *bus)
 {
-	struct atmegadci_softc *sc = ATMEGA_BUS2SC(xfer->xroot->bus);
+	struct atmegadci_softc *sc = ATMEGA_BUS2SC(bus);
+	struct usb2_sw_transfer *std = &sc->sc_bus.roothub_req;
 	uint16_t value;
 	uint16_t index;
 	uint8_t temp;
 
 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 
-	if (std->state != USB_SW_TR_SETUP) {
-		if (std->state == USB_SW_TR_PRE_CALLBACK) {
-			/* transfer transferred */
-			atmegadci_device_done(xfer, std->err);
-		}
-		goto done;
-	}
 	/* buffer reset */
 	std->ptr = USB_ADD_BYTES(&sc->sc_hub_temp, 0);
 	std->len = 0;
@@ -1964,7 +1819,7 @@ tr_handle_clear_port_feature:
 
 	switch (value) {
 	case UHF_PORT_SUSPEND:
-		atmegadci_wakeup_peer(xfer);
+		atmegadci_wakeup_peer(sc);
 		break;
 
 	case UHF_PORT_ENABLE:
@@ -2113,67 +1968,6 @@ done:
 }
 
 static void
-atmegadci_root_ctrl_poll(struct atmegadci_softc *sc)
-{
-	usb2_sw_transfer(&sc->sc_root_ctrl,
-	    &atmegadci_root_ctrl_done);
-}
-
-struct usb2_pipe_methods atmegadci_root_ctrl_methods =
-{
-	.open = atmegadci_root_ctrl_open,
-	.close = atmegadci_root_ctrl_close,
-	.enter = atmegadci_root_ctrl_enter,
-	.start = atmegadci_root_ctrl_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 0,
-};
-
-/*------------------------------------------------------------------------*
- * at91dci root interrupt support
- *------------------------------------------------------------------------*/
-static void
-atmegadci_root_intr_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-atmegadci_root_intr_close(struct usb2_xfer *xfer)
-{
-	struct atmegadci_softc *sc = ATMEGA_BUS2SC(xfer->xroot->bus);
-
-	if (sc->sc_root_intr.xfer == xfer) {
-		sc->sc_root_intr.xfer = NULL;
-	}
-	atmegadci_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-static void
-atmegadci_root_intr_enter(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-atmegadci_root_intr_start(struct usb2_xfer *xfer)
-{
-	struct atmegadci_softc *sc = ATMEGA_BUS2SC(xfer->xroot->bus);
-
-	sc->sc_root_intr.xfer = xfer;
-}
-
-struct usb2_pipe_methods atmegadci_root_intr_methods =
-{
-	.open = atmegadci_root_intr_open,
-	.close = atmegadci_root_intr_close,
-	.enter = atmegadci_root_intr_enter,
-	.start = atmegadci_root_intr_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
-};
-
-static void
 atmegadci_xfer_setup(struct usb2_setup_params *parm)
 {
 	const struct usb2_hw_ep_profile *pf;
@@ -2201,34 +1995,21 @@ atmegadci_xfer_setup(struct usb2_setup_params *parm)
 	/*
 	 * compute maximum number of TDs
 	 */
-	if (parm->methods == &atmegadci_device_ctrl_methods) {
+	if ((xfer->pipe->edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL) {
 
-		ntd = xfer->nframes + 1 /* STATUS */ + 1	/* SYNC 1 */
+		ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC 1 */
 		    + 1 /* SYNC 2 */ ;
-
-	} else if (parm->methods == &atmegadci_device_bulk_methods) {
-
-		ntd = xfer->nframes + 1 /* SYNC */ ;
-
-	} else if (parm->methods == &atmegadci_device_intr_methods) {
-
-		ntd = xfer->nframes + 1 /* SYNC */ ;
-
-	} else if (parm->methods == &atmegadci_device_isoc_fs_methods) {
-
-		ntd = xfer->nframes + 1 /* SYNC */ ;
-
 	} else {
 
-		ntd = 0;
+		ntd = xfer->nframes + 1 /* SYNC */ ;
 	}
 
 	/*
 	 * check if "usb2_transfer_setup_sub" set an error
 	 */
-	if (parm->err) {
+	if (parm->err)
 		return;
-	}
+
 	/*
 	 * allocate transfer descriptors
 	 */
@@ -2237,19 +2018,13 @@ atmegadci_xfer_setup(struct usb2_setup_params *parm)
 	/*
 	 * get profile stuff
 	 */
-	if (ntd) {
-
-		ep_no = xfer->endpoint & UE_ADDR;
-		atmegadci_get_hw_ep_profile(parm->udev, &pf, ep_no);
+	ep_no = xfer->endpoint & UE_ADDR;
+	atmegadci_get_hw_ep_profile(parm->udev, &pf, ep_no);
 
-		if (pf == NULL) {
-			/* should not happen */
-			parm->err = USB_ERR_INVAL;
-			return;
-		}
-	} else {
-		ep_no = 0;
-		pf = NULL;
+	if (pf == NULL) {
+		/* should not happen */
+		parm->err = USB_ERR_INVAL;
+		return;
 	}
 
 	/* align data */
@@ -2296,24 +2071,7 @@ atmegadci_pipe_init(struct usb2_device *udev, struct usb2_endpoint_descriptor *e
 	    edesc->bEndpointAddress, udev->flags.usb2_mode,
 	    sc->sc_rt_addr, udev->device_index);
 
-	if (udev->device_index == sc->sc_rt_addr) {
-
-		if (udev->flags.usb2_mode != USB_MODE_HOST) {
-			/* not supported */
-			return;
-		}
-		switch (edesc->bEndpointAddress) {
-		case USB_CONTROL_ENDPOINT:
-			pipe->methods = &atmegadci_root_ctrl_methods;
-			break;
-		case UE_DIR_IN | ATMEGA_INTR_ENDPT:
-			pipe->methods = &atmegadci_root_intr_methods;
-			break;
-		default:
-			/* do nothing */
-			break;
-		}
-	} else {
+	if (udev->device_index != sc->sc_rt_addr) {
 
 		if (udev->flags.usb2_mode != USB_MODE_DEVICE) {
 			/* not supported */
@@ -2323,23 +2081,10 @@ atmegadci_pipe_init(struct usb2_device *udev, struct usb2_endpoint_descriptor *e
 			/* not supported */
 			return;
 		}
-		switch (edesc->bmAttributes & UE_XFERTYPE) {
-		case UE_CONTROL:
-			pipe->methods = &atmegadci_device_ctrl_methods;
-			break;
-		case UE_INTERRUPT:
-			pipe->methods = &atmegadci_device_intr_methods;
-			break;
-		case UE_ISOCHRONOUS:
+		if ((edesc->bmAttributes & UE_XFERTYPE) == UE_ISOCHRONOUS)
 			pipe->methods = &atmegadci_device_isoc_fs_methods;
-			break;
-		case UE_BULK:
-			pipe->methods = &atmegadci_device_bulk_methods;
-			break;
-		default:
-			/* do nothing */
-			break;
-		}
+		else
+			pipe->methods = &atmegadci_device_non_isoc_methods;
 	}
 }
 
@@ -2351,5 +2096,5 @@ struct usb2_bus_methods atmegadci_bus_methods =
 	.get_hw_ep_profile = &atmegadci_get_hw_ep_profile,
 	.set_stall = &atmegadci_set_stall,
 	.clear_stall = &atmegadci_clear_stall,
-	.roothub_exec = &atmegadci_root_ctrl_task,
+	.roothub_exec = &atmegadci_roothub_exec,
 };
diff --git a/sys/dev/usb/controller/atmegadci.h b/sys/dev/usb/controller/atmegadci.h
index 1e7d964..af61eeb 100644
--- a/sys/dev/usb/controller/atmegadci.h
+++ b/sys/dev/usb/controller/atmegadci.h
@@ -25,8 +25,8 @@
  */
 
 /*
- * USB Device Port register definitions, copied from ATMEGA
- * documentation provided by ATMEL.
+ * USB Device Port register definitions, copied from ATMEGA documentation
+ * provided by ATMEL.
  */
 
 #ifndef _ATMEGADCI_H_
@@ -34,10 +34,6 @@
 
 #define	ATMEGA_MAX_DEVICES (USB_MIN_DEVICES + 1)
 
-#ifndef ATMEGA_HAVE_BUS_SPACE
-#define	ATMEGA_HAVE_BUS_SPACE 1
-#endif
-
 #define	ATMEGA_UEINT 0xF4
 #define	ATMEGA_UEINT_MASK(n) (1 << (n))	/* endpoint interrupt mask */
 
@@ -241,8 +237,6 @@ struct atmegadci_softc {
 	struct usb2_bus sc_bus;
 	union atmegadci_hub_temp sc_hub_temp;
 	LIST_HEAD(, usb2_xfer) sc_interrupt_list_head;
-	struct usb2_sw_transfer sc_root_ctrl;
-	struct usb2_sw_transfer sc_root_intr;
 
 	/* must be set by by the bus interface layer */
 	atmegadci_clocks_t *sc_clocks_on;
@@ -251,11 +245,10 @@ struct atmegadci_softc {
 	struct usb2_device *sc_devices[ATMEGA_MAX_DEVICES];
 	struct resource *sc_irq_res;
 	void   *sc_intr_hdl;
-#if (ATMEGA_HAVE_BUS_SPACE != 0)
 	struct resource *sc_io_res;
 	bus_space_tag_t sc_io_tag;
 	bus_space_handle_t sc_io_hdl;
-#endif
+
 	uint8_t	sc_rt_addr;		/* root hub address */
 	uint8_t	sc_dv_addr;		/* device address */
 	uint8_t	sc_conf;		/* root hub config */
diff --git a/sys/dev/usb/controller/atmegadci_atmelarm.c b/sys/dev/usb/controller/atmegadci_atmelarm.c
index 7cf2eb4..df305ac 100644
--- a/sys/dev/usb/controller/atmegadci_atmelarm.c
+++ b/sys/dev/usb/controller/atmegadci_atmelarm.c
@@ -26,13 +26,11 @@ __FBSDID("$FreeBSD$");
  * SUCH DAMAGE.
  */
 
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb.h>
 
 #include <dev/usb/usb_core.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_util.h>
 
 #include <dev/usb/usb_controller.h>
@@ -114,7 +112,7 @@ atmegadci_attach(device_t dev)
 	device_set_ivars(sc->sc_otg.sc_bus.bdev, &sc->sc_otg.sc_bus);
 
 	err = bus_setup_intr(dev, sc->sc_otg.sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    NULL, (void *)atmegadci_interrupt, sc, &sc->sc_otg.sc_intr_hdl);
+	    NULL, (driver_intr_t *)atmegadci_interrupt, sc, &sc->sc_otg.sc_intr_hdl);
 	if (err) {
 		sc->sc_otg.sc_intr_hdl = NULL;
 		goto error;
diff --git a/sys/dev/usb/controller/ehci.c b/sys/dev/usb/controller/ehci.c
index 7c873d9..8be6bec 100644
--- a/sys/dev/usb/controller/ehci.c
+++ b/sys/dev/usb/controller/ehci.c
@@ -39,7 +39,7 @@
  */
 
 /*
- * TODO:
+ * TODO: 
  * 1) command failures are not recovered correctly
  */
 
@@ -49,7 +49,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR ehcidebug
 
@@ -57,7 +56,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb_debug.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_transfer.h>
 #include <dev/usb/usb_device.h>
 #include <dev/usb/usb_hub.h>
@@ -67,8 +65,9 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb_bus.h>
 #include <dev/usb/controller/ehci.h>
 
-#define	EHCI_BUS2SC(bus) ((ehci_softc_t *)(((uint8_t *)(bus)) - \
-   USB_P2U(&(((ehci_softc_t *)0)->sc_bus))))
+#define	EHCI_BUS2SC(bus) \
+   ((ehci_softc_t *)(((uint8_t *)(bus)) - \
+    ((uint8_t *)&(((ehci_softc_t *)0)->sc_bus))))
 
 #if USB_DEBUG
 static int ehcidebug = 0;
@@ -93,17 +92,12 @@ extern struct usb2_pipe_methods ehci_device_ctrl_methods;
 extern struct usb2_pipe_methods ehci_device_intr_methods;
 extern struct usb2_pipe_methods ehci_device_isoc_fs_methods;
 extern struct usb2_pipe_methods ehci_device_isoc_hs_methods;
-extern struct usb2_pipe_methods ehci_root_ctrl_methods;
-extern struct usb2_pipe_methods ehci_root_intr_methods;
 
 static void ehci_do_poll(struct usb2_bus *bus);
-static void ehci_root_ctrl_poll(ehci_softc_t *sc);
 static void ehci_device_done(struct usb2_xfer *xfer, usb2_error_t error);
 static uint8_t ehci_check_transfer(struct usb2_xfer *xfer);
 static void ehci_timeout(void *arg);
-
-static usb2_sw_transfer_func_t ehci_root_intr_done;
-static usb2_sw_transfer_func_t ehci_root_ctrl_done;
+static void ehci_root_intr(ehci_softc_t *sc);
 
 struct ehci_std_temp {
 	ehci_softc_t *sc;
@@ -117,7 +111,7 @@ struct ehci_std_temp {
 	uint8_t	shortpkt;
 	uint8_t	auto_data_toggle;
 	uint8_t	setup_alt_next;
-	uint8_t	short_frames_ok;
+	uint8_t	last_frame;
 };
 
 void
@@ -1415,8 +1409,7 @@ ehci_pcd_enable(ehci_softc_t *sc)
 	/* acknowledge any PCD interrupt */
 	EOWRITE4(sc, EHCI_USBSTS, EHCI_STS_PCD);
 
-	usb2_sw_transfer(&sc->sc_root_intr,
-	    &ehci_root_intr_done);
+	ehci_root_intr(sc);
 }
 
 static void
@@ -1486,8 +1479,7 @@ ehci_interrupt(ehci_softc_t *sc)
 		sc->sc_eintrs &= ~EHCI_STS_PCD;
 		EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 
-		usb2_sw_transfer(&sc->sc_root_intr,
-		    &ehci_root_intr_done);
+		ehci_root_intr(sc);
 
 		/* do not allow RHSC interrupts > 1 per second */
 		usb2_callout_reset(&sc->sc_tmo_pcd, hz,
@@ -1531,7 +1523,6 @@ ehci_do_poll(struct usb2_bus *bus)
 
 	USB_BUS_LOCK(&sc->sc_bus);
 	ehci_interrupt_poll(sc);
-	ehci_root_ctrl_poll(sc);
 	USB_BUS_UNLOCK(&sc->sc_bus);
 }
 
@@ -1546,10 +1537,12 @@ ehci_setup_standard_chain_sub(struct ehci_std_temp *temp)
 	uint32_t buf_offset;
 	uint32_t average;
 	uint32_t len_old;
+	uint32_t terminate;
 	uint8_t shortpkt_old;
 	uint8_t precompute;
 
-	qtd_altnext = htohc32(temp->sc, EHCI_LINK_TERMINATE);
+	terminate = htohc32(temp->sc, EHCI_LINK_TERMINATE);
+	qtd_altnext = terminate;
 	td_alt_next = NULL;
 	buf_offset = 0;
 	shortpkt_old = temp->shortpkt;
@@ -1696,14 +1689,17 @@ restart:
 		precompute = 0;
 
 		/* setup alt next pointer, if any */
-		if (temp->short_frames_ok) {
-			if (temp->setup_alt_next) {
-				td_alt_next = td_next;
-				qtd_altnext = td_next->qtd_self;
-			}
+		if (temp->last_frame) {
+			td_alt_next = NULL;
+			qtd_altnext = terminate;
 		} else {
 			/* we use this field internally */
 			td_alt_next = td_next;
+			if (temp->setup_alt_next) {
+				qtd_altnext = td_next->qtd_self;
+			} else {
+				qtd_altnext = terminate;
+			}
 		}
 
 		/* restore */
@@ -1730,7 +1726,7 @@ ehci_setup_standard_chain(struct usb2_xfer *xfer, ehci_qh_t **qh_last)
 	    xfer->address, UE_GET_ADDR(xfer->endpoint),
 	    xfer->sumlen, usb2_get_speed(xfer->xroot->udev));
 
-	temp.average = xfer->max_usb2_frame_size;
+	temp.average = xfer->max_hc_frame_size;
 	temp.max_frame_size = xfer->max_frame_size;
 	temp.sc = EHCI_BUS2SC(xfer->xroot->bus);
 
@@ -1746,8 +1742,8 @@ ehci_setup_standard_chain(struct usb2_xfer *xfer, ehci_qh_t **qh_last)
 	temp.td = NULL;
 	temp.td_next = td;
 	temp.qtd_status = 0;
+	temp.last_frame = 0;
 	temp.setup_alt_next = xfer->flags_int.short_frames_ok;
-	temp.short_frames_ok = xfer->flags_int.short_frames_ok;
 
 	if (xfer->flags_int.control_xfr) {
 		if (xfer->pipe->toggle_next) {
@@ -1780,7 +1776,14 @@ ehci_setup_standard_chain(struct usb2_xfer *xfer, ehci_qh_t **qh_last)
 			temp.len = xfer->frlengths[0];
 			temp.pc = xfer->frbuffers + 0;
 			temp.shortpkt = temp.len ? 1 : 0;
-
+			/* check for last frame */
+			if (xfer->nframes == 1) {
+				/* no STATUS stage yet, SETUP is last */
+				if (xfer->flags_int.control_act) {
+					temp.last_frame = 1;
+					temp.setup_alt_next = 0;
+				}
+			}
 			ehci_setup_standard_chain_sub(&temp);
 		}
 		x = 1;
@@ -1798,7 +1801,16 @@ ehci_setup_standard_chain(struct usb2_xfer *xfer, ehci_qh_t **qh_last)
 		x++;
 
 		if (x == xfer->nframes) {
-			temp.setup_alt_next = 0;
+			if (xfer->flags_int.control_xfr) {
+				/* no STATUS stage yet, DATA is last */
+				if (xfer->flags_int.control_act) {
+					temp.last_frame = 1;
+					temp.setup_alt_next = 0;
+				}
+			} else {
+				temp.last_frame = 1;
+				temp.setup_alt_next = 0;
+			}
 		}
 		/* keep previous data toggle and error count */
 
@@ -1855,6 +1867,8 @@ ehci_setup_standard_chain(struct usb2_xfer *xfer, ehci_qh_t **qh_last)
 		temp.len = 0;
 		temp.pc = NULL;
 		temp.shortpkt = 0;
+		temp.last_frame = 1;
+		temp.setup_alt_next = 0;
 
 		ehci_setup_standard_chain_sub(&temp);
 	}
@@ -1956,28 +1970,15 @@ ehci_setup_standard_chain(struct usb2_xfer *xfer, ehci_qh_t **qh_last)
 }
 
 static void
-ehci_root_intr_done(struct usb2_xfer *xfer,
-    struct usb2_sw_transfer *std)
+ehci_root_intr(ehci_softc_t *sc)
 {
-	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
 	uint16_t i;
 	uint16_t m;
 
 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 
-	if (std->state != USB_SW_TR_PRE_DATA) {
-		if (std->state == USB_SW_TR_PRE_CALLBACK) {
-			/* transfer transferred */
-			ehci_device_done(xfer, std->err);
-		}
-		goto done;
-	}
-	/* setup buffer */
-	std->ptr = sc->sc_hub_idata;
-	std->len = sizeof(sc->sc_hub_idata);
-
 	/* clear any old interrupt data */
-	bzero(sc->sc_hub_idata, sizeof(sc->sc_hub_idata));
+	memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
 
 	/* set bits */
 	m = (sc->sc_noport + 1);
@@ -1991,8 +1992,8 @@ ehci_root_intr_done(struct usb2_xfer *xfer,
 			DPRINTF("port %d changed\n", i);
 		}
 	}
-done:
-	return;
+	uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
+	    sizeof(sc->sc_hub_idata));
 }
 
 static void
@@ -2208,8 +2209,6 @@ struct usb2_pipe_methods ehci_device_bulk_methods =
 	.close = ehci_device_bulk_close,
 	.enter = ehci_device_bulk_enter,
 	.start = ehci_device_bulk_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -2251,8 +2250,6 @@ struct usb2_pipe_methods ehci_device_ctrl_methods =
 	.close = ehci_device_ctrl_close,
 	.enter = ehci_device_ctrl_enter,
 	.start = ehci_device_ctrl_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -2349,8 +2346,6 @@ struct usb2_pipe_methods ehci_device_intr_methods =
 	.close = ehci_device_intr_close,
 	.enter = ehci_device_intr_enter,
 	.start = ehci_device_intr_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -2634,8 +2629,6 @@ struct usb2_pipe_methods ehci_device_isoc_fs_methods =
 	.close = ehci_device_isoc_fs_close,
 	.enter = ehci_device_isoc_fs_enter,
 	.start = ehci_device_isoc_fs_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -2902,38 +2895,14 @@ struct usb2_pipe_methods ehci_device_isoc_hs_methods =
 	.close = ehci_device_isoc_hs_close,
 	.enter = ehci_device_isoc_hs_enter,
 	.start = ehci_device_isoc_hs_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
  * ehci root control support
  *------------------------------------------------------------------------*
- * simulate a hardware hub by handling
- * all the necessary requests
+ * Simulate a hardware hub by handling all the necessary requests.
  *------------------------------------------------------------------------*/
 
-static void
-ehci_root_ctrl_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-ehci_root_ctrl_close(struct usb2_xfer *xfer)
-{
-	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
-
-	if (sc->sc_root_ctrl.xfer == xfer) {
-		sc->sc_root_ctrl.xfer = NULL;
-	}
-	ehci_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-/* data structures and routines
- * to emulate the root hub:
- */
-
 static const
 struct usb2_device_descriptor ehci_devd =
 {
@@ -2974,7 +2943,6 @@ static const struct ehci_config_desc ehci_confd = {
 		.bmAttributes = UC_SELF_POWERED,
 		.bMaxPower = 0		/* max power */
 	},
-
 	.ifcd = {
 		.bLength = sizeof(struct usb2_interface_descriptor),
 		.bDescriptorType = UDESC_INTERFACE,
@@ -2984,7 +2952,6 @@ static const struct ehci_config_desc ehci_confd = {
 		.bInterfaceProtocol = UIPROTO_HSHUBSTT,
 		0
 	},
-
 	.endpd = {
 		.bLength = sizeof(struct usb2_endpoint_descriptor),
 		.bDescriptorType = UDESC_ENDPOINT,
@@ -3021,34 +2988,10 @@ ehci_disown(ehci_softc_t *sc, uint16_t index, uint8_t lowspeed)
 }
 
 static void
-ehci_root_ctrl_enter(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-ehci_root_ctrl_start(struct usb2_xfer *xfer)
+ehci_roothub_exec(struct usb2_bus *bus)
 {
-	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
-
-	DPRINTF("\n");
-
-	sc->sc_root_ctrl.xfer = xfer;
-
-	usb2_bus_roothub_exec(xfer->xroot->bus);
-}
-
-static void
-ehci_root_ctrl_task(struct usb2_bus *bus)
-{
-	ehci_root_ctrl_poll(EHCI_BUS2SC(bus));
-}
-
-static void
-ehci_root_ctrl_done(struct usb2_xfer *xfer,
-    struct usb2_sw_transfer *std)
-{
-	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
+	ehci_softc_t *sc = EHCI_BUS2SC(bus);
+	struct usb2_sw_transfer *std = &sc->sc_bus.roothub_req;
 	char *ptr;
 	uint32_t port;
 	uint32_t v;
@@ -3059,13 +3002,6 @@ ehci_root_ctrl_done(struct usb2_xfer *xfer,
 
 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 
-	if (std->state != USB_SW_TR_SETUP) {
-		if (std->state == USB_SW_TR_PRE_CALLBACK) {
-			/* transfer transferred */
-			ehci_device_done(xfer, std->err);
-		}
-		goto done;
-	}
 	/* buffer reset */
 	std->ptr = sc->sc_hub_desc.temp;
 	std->len = 0;
@@ -3168,7 +3104,7 @@ ehci_root_ctrl_done(struct usb2_xfer *xfer,
 		USETW(sc->sc_hub_desc.stat.wStatus, 0);
 		break;
 	case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
-		if (value >= USB_MAX_DEVICES) {
+		if (value >= EHCI_MAX_DEVICES) {
 			std->err = USB_ERR_IOERROR;
 			goto done;
 		}
@@ -3439,67 +3375,6 @@ done:
 }
 
 static void
-ehci_root_ctrl_poll(ehci_softc_t *sc)
-{
-	usb2_sw_transfer(&sc->sc_root_ctrl,
-	    &ehci_root_ctrl_done);
-}
-
-struct usb2_pipe_methods ehci_root_ctrl_methods =
-{
-	.open = ehci_root_ctrl_open,
-	.close = ehci_root_ctrl_close,
-	.enter = ehci_root_ctrl_enter,
-	.start = ehci_root_ctrl_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 0,
-};
-
-/*------------------------------------------------------------------------*
- * ehci root interrupt support
- *------------------------------------------------------------------------*/
-static void
-ehci_root_intr_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-ehci_root_intr_close(struct usb2_xfer *xfer)
-{
-	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
-
-	if (sc->sc_root_intr.xfer == xfer) {
-		sc->sc_root_intr.xfer = NULL;
-	}
-	ehci_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-static void
-ehci_root_intr_enter(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-ehci_root_intr_start(struct usb2_xfer *xfer)
-{
-	ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
-
-	sc->sc_root_intr.xfer = xfer;
-}
-
-struct usb2_pipe_methods ehci_root_intr_methods =
-{
-	.open = ehci_root_intr_open,
-	.close = ehci_root_intr_close,
-	.enter = ehci_root_intr_enter,
-	.start = ehci_root_intr_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
-};
-
-static void
 ehci_xfer_setup(struct usb2_setup_params *parm)
 {
 	struct usb2_page_search page_info;
@@ -3564,7 +3439,7 @@ ehci_xfer_setup(struct usb2_setup_params *parm)
 
 		nqh = 1;
 		nqtd = ((2 * xfer->nframes) + 1	/* STATUS */
-		    + (xfer->max_data_length / xfer->max_usb2_frame_size));
+		    + (xfer->max_data_length / xfer->max_hc_frame_size));
 
 	} else if (parm->methods == &ehci_device_bulk_methods) {
 
@@ -3577,7 +3452,7 @@ ehci_xfer_setup(struct usb2_setup_params *parm)
 
 		nqh = 1;
 		nqtd = ((2 * xfer->nframes)
-		    + (xfer->max_data_length / xfer->max_usb2_frame_size));
+		    + (xfer->max_data_length / xfer->max_hc_frame_size));
 
 	} else if (parm->methods == &ehci_device_intr_methods) {
 
@@ -3599,7 +3474,7 @@ ehci_xfer_setup(struct usb2_setup_params *parm)
 
 		nqh = 1;
 		nqtd = ((2 * xfer->nframes)
-		    + (xfer->max_data_length / xfer->max_usb2_frame_size));
+		    + (xfer->max_data_length / xfer->max_hc_frame_size));
 
 	} else if (parm->methods == &ehci_device_isoc_fs_methods) {
 
@@ -3771,19 +3646,8 @@ ehci_pipe_init(struct usb2_device *udev, struct usb2_endpoint_descriptor *edesc,
 		/* not supported */
 		return;
 	}
-	if (udev->device_index == sc->sc_addr) {
-		switch (edesc->bEndpointAddress) {
-		case USB_CONTROL_ENDPOINT:
-			pipe->methods = &ehci_root_ctrl_methods;
-			break;
-		case UE_DIR_IN | EHCI_INTR_ENDPT:
-			pipe->methods = &ehci_root_intr_methods;
-			break;
-		default:
-			/* do nothing */
-			break;
-		}
-	} else {
+	if (udev->device_index != sc->sc_addr) {
+
 		if ((udev->speed != USB_SPEED_HIGH) &&
 		    ((udev->hs_hub_addr == 0) ||
 		    (udev->hs_port_no == 0) ||
@@ -3941,5 +3805,5 @@ struct usb2_bus_methods ehci_bus_methods =
 	.device_resume = ehci_device_resume,
 	.device_suspend = ehci_device_suspend,
 	.set_hw_power = ehci_set_hw_power,
-	.roothub_exec = ehci_root_ctrl_task,
+	.roothub_exec = ehci_roothub_exec,
 };
diff --git a/sys/dev/usb/controller/ehci.h b/sys/dev/usb/controller/ehci.h
index 5b54809..0f3580f 100644
--- a/sys/dev/usb/controller/ehci.h
+++ b/sys/dev/usb/controller/ehci.h
@@ -38,9 +38,9 @@
 #ifndef _EHCI_H_
 #define	_EHCI_H_
 
-#define	EHCI_MAX_DEVICES USB_MAX_DEVICES
+#define	EHCI_MAX_DEVICES MIN(USB_MAX_DEVICES, 128)
 
-/* PCI config registers */
+/* PCI config registers  */
 #define	PCI_CBMEM		0x10	/* configuration base MEM */
 #define	PCI_INTERFACE_EHCI	0x20
 #define	PCI_USBREV		0x60	/* RO USB protocol revision */
@@ -457,8 +457,6 @@ typedef struct ehci_softc {
 	struct usb2_bus sc_bus;		/* base device */
 	struct usb2_callout sc_tmo_pcd;
 	union ehci_hub_desc sc_hub_desc;
-	struct usb2_sw_transfer sc_root_ctrl;
-	struct usb2_sw_transfer sc_root_intr;
 
 	struct usb2_device *sc_devices[EHCI_MAX_DEVICES];
 	struct resource *sc_io_res;
diff --git a/sys/dev/usb/controller/ehci_ixp4xx.c b/sys/dev/usb/controller/ehci_ixp4xx.c
index 1e0e253..672a6ee 100644
--- a/sys/dev/usb/controller/ehci_ixp4xx.c
+++ b/sys/dev/usb/controller/ehci_ixp4xx.c
@@ -32,13 +32,11 @@ __FBSDID("$FreeBSD$");
 #include "opt_bus.h"
 
 #include <dev/usb/usb_mfunc.h>
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb.h>
 
 #include <dev/usb/usb_core.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_util.h>
 
 #include <dev/usb/usb_controller.h>
@@ -190,7 +188,7 @@ ehci_ixp_attach(device_t self)
 
 
 	err = bus_setup_intr(self, sc->sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    NULL, (void *)(void *)ehci_interrupt, sc, &sc->sc_intr_hdl);
+	    NULL, (driver_intr_t *)ehci_interrupt, sc, &sc->sc_intr_hdl);
 	if (err) {
 		device_printf(self, "Could not setup irq, %d\n", err);
 		sc->sc_intr_hdl = NULL;
diff --git a/sys/dev/usb/controller/ehci_mbus.c b/sys/dev/usb/controller/ehci_mbus.c
index 66cf8cc..c508269 100644
--- a/sys/dev/usb/controller/ehci_mbus.c
+++ b/sys/dev/usb/controller/ehci_mbus.c
@@ -39,13 +39,11 @@ __FBSDID("$FreeBSD$");
 #include "opt_bus.h"
 
 #include <dev/usb/usb_mfunc.h>
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb.h>
 
 #include <dev/usb/usb_core.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_util.h>
 
 #include <dev/usb/usb_controller.h>
@@ -211,7 +209,7 @@ ehci_mbus_attach(device_t self)
 	    MV_USB_DEVICE_UNDERFLOW);
 
 	err = bus_setup_intr(self, sc->sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    NULL, (void *)(void *)ehci_interrupt, sc, &sc->sc_intr_hdl);
+	    NULL, (driver_intr_t *)ehci_interrupt, sc, &sc->sc_intr_hdl);
 	if (err) {
 		device_printf(self, "Could not setup irq, %d\n", err);
 		sc->sc_intr_hdl = NULL;
diff --git a/sys/dev/usb/controller/ehci_pci.c b/sys/dev/usb/controller/ehci_pci.c
index 2f097fc..744f5f5 100644
--- a/sys/dev/usb/controller/ehci_pci.c
+++ b/sys/dev/usb/controller/ehci_pci.c
@@ -53,13 +53,11 @@ __FBSDID("$FreeBSD$");
  */
 
 #include <dev/usb/usb_mfunc.h>
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb.h>
 
 #include <dev/usb/usb_core.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_util.h>
 
 #include <dev/usb/usb_controller.h>
@@ -339,10 +337,10 @@ ehci_pci_attach(device_t self)
 
 #if (__FreeBSD_version >= 700031)
 	err = bus_setup_intr(self, sc->sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    NULL, (void *)(void *)ehci_interrupt, sc, &sc->sc_intr_hdl);
+	    NULL, (driver_intr_t *)ehci_interrupt, sc, &sc->sc_intr_hdl);
 #else
 	err = bus_setup_intr(self, sc->sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    (void *)(void *)ehci_interrupt, sc, &sc->sc_intr_hdl);
+	    (driver_intr_t *)ehci_interrupt, sc, &sc->sc_intr_hdl);
 #endif
 	if (err) {
 		device_printf(self, "Could not setup irq, %d\n", err);
diff --git a/sys/dev/usb/controller/musb_otg.c b/sys/dev/usb/controller/musb_otg.c
index cda1a67..249b0b7 100644
--- a/sys/dev/usb/controller/musb_otg.c
+++ b/sys/dev/usb/controller/musb_otg.c
@@ -25,8 +25,8 @@
  */
 
 /*
- * Thanks to Mentor Graphics for providing a reference driver for this
- * USB chip at their homepage.
+ * Thanks to Mentor Graphics for providing a reference driver for this USB chip
+ * at their homepage.
  */
 
 /*
@@ -39,7 +39,6 @@
 #include <dev/usb/usb.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR musbotgdebug
 
@@ -47,7 +46,6 @@
 #include <dev/usb/usb_debug.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_transfer.h>
 #include <dev/usb/usb_device.h>
 #include <dev/usb/usb_hub.h>
@@ -64,7 +62,7 @@
    USB_P2U(&(((struct musbotg_softc *)0)->sc_bus))))
 
 #define	MUSBOTG_PC2SC(pc) \
-   MUSBOTG_BUS2SC((pc)->tag_parent->info->bus)
+   MUSBOTG_BUS2SC(USB_DMATAG_TO_XROOT((pc)->tag_parent)->bus)
 
 #if USB_DEBUG
 static int musbotgdebug = 0;
@@ -81,8 +79,6 @@ struct usb2_pipe_methods musbotg_device_bulk_methods;
 struct usb2_pipe_methods musbotg_device_ctrl_methods;
 struct usb2_pipe_methods musbotg_device_intr_methods;
 struct usb2_pipe_methods musbotg_device_isoc_methods;
-struct usb2_pipe_methods musbotg_root_ctrl_methods;
-struct usb2_pipe_methods musbotg_root_intr_methods;
 
 static musbotg_cmd_t musbotg_setup_rx;
 static musbotg_cmd_t musbotg_setup_data_rx;
@@ -92,12 +88,9 @@ static musbotg_cmd_t musbotg_data_rx;
 static musbotg_cmd_t musbotg_data_tx;
 static void	musbotg_device_done(struct usb2_xfer *, usb2_error_t);
 static void	musbotg_do_poll(struct usb2_bus *);
-static void	musbotg_root_ctrl_poll(struct musbotg_softc *);
 static void	musbotg_standard_done(struct usb2_xfer *);
 static void	musbotg_interrupt_poll(struct musbotg_softc *);
-
-static usb2_sw_transfer_func_t musbotg_root_intr_done;
-static usb2_sw_transfer_func_t musbotg_root_ctrl_done;
+static void	musbotg_root_intr(struct musbotg_softc *);
 
 /*
  * Here is a configuration that the chip supports.
@@ -202,9 +195,8 @@ musbotg_pull_down(struct musbotg_softc *sc)
 }
 
 static void
-musbotg_wakeup_peer(struct usb2_xfer *xfer)
+musbotg_wakeup_peer(struct musbotg_softc *sc)
 {
-	struct musbotg_softc *sc = MUSBOTG_BUS2SC(xfer->xroot->bus);
 	uint8_t temp;
 
 	if (!(sc->sc_flags.status_suspend)) {
@@ -256,6 +248,8 @@ musbotg_setup_rx(struct musbotg_td *td)
 	 * callback, hence the status stage is not complete.
 	 */
 	if (csr & MUSB2_MASK_CSR0L_DATAEND) {
+		/* do not stall at this point */
+		td->did_stall = 1;
 		/* wait for interrupt */
 		goto not_complete;
 	}
@@ -277,18 +271,13 @@ musbotg_setup_rx(struct musbotg_td *td)
 		sc->sc_ep0_busy = 0;
 	}
 	if (sc->sc_ep0_busy) {
-		/* abort any ongoing transfer */
-		if (!td->did_stall) {
-			DPRINTFN(4, "stalling\n");
-			MUSB2_WRITE_1(sc, MUSB2_REG_TXCSRL,
-			    MUSB2_MASK_CSR0L_SENDSTALL);
-			td->did_stall = 1;
-		}
 		goto not_complete;
 	}
 	if (!(csr & MUSB2_MASK_CSR0L_RXPKTRDY)) {
 		goto not_complete;
 	}
+	/* clear did stall flag */
+	td->did_stall = 0;
 	/* get the packet byte count */
 	count = MUSB2_READ_2(sc, MUSB2_REG_RXCOUNT);
 
@@ -296,11 +285,15 @@ musbotg_setup_rx(struct musbotg_td *td)
 	if (count != td->remainder) {
 		DPRINTFN(0, "Invalid SETUP packet "
 		    "length, %d bytes\n", count);
+		MUSB2_WRITE_1(sc, MUSB2_REG_TXCSRL,
+		      MUSB2_MASK_CSR0L_RXPKTRDY_CLR);
 		goto not_complete;
 	}
 	if (count != sizeof(req)) {
 		DPRINTFN(0, "Unsupported SETUP packet "
 		    "length, %d bytes\n", count);
+		MUSB2_WRITE_1(sc, MUSB2_REG_TXCSRL,
+		      MUSB2_MASK_CSR0L_RXPKTRDY_CLR);
 		goto not_complete;
 	}
 	/* receive data */
@@ -329,6 +322,13 @@ musbotg_setup_rx(struct musbotg_td *td)
 	return (0);			/* complete */
 
 not_complete:
+	/* abort any ongoing transfer */
+	if (!td->did_stall) {
+		DPRINTFN(4, "stalling\n");
+		MUSB2_WRITE_1(sc, MUSB2_REG_TXCSRL,
+		    MUSB2_MASK_CSR0L_SENDSTALL);
+		td->did_stall = 1;
+	}
 	return (1);			/* not complete */
 }
 
@@ -958,9 +958,7 @@ musbotg_vbus_interrupt(struct musbotg_softc *sc, uint8_t is_on)
 			sc->sc_flags.status_vbus = 1;
 
 			/* complete root HUB interrupt endpoint */
-
-			usb2_sw_transfer(&sc->sc_root_intr,
-			    &musbotg_root_intr_done);
+			musbotg_root_intr(sc);
 		}
 	} else {
 		if (sc->sc_flags.status_vbus) {
@@ -971,9 +969,7 @@ musbotg_vbus_interrupt(struct musbotg_softc *sc, uint8_t is_on)
 			sc->sc_flags.change_connect = 1;
 
 			/* complete root HUB interrupt endpoint */
-
-			usb2_sw_transfer(&sc->sc_root_intr,
-			    &musbotg_root_intr_done);
+			musbotg_root_intr(sc);
 		}
 	}
 
@@ -1067,9 +1063,7 @@ repeat:
 			}
 		}
 		/* complete root HUB interrupt endpoint */
-
-		usb2_sw_transfer(&sc->sc_root_intr,
-		    &musbotg_root_intr_done);
+		musbotg_root_intr(sc);
 	}
 	/* check for any endpoint interrupts */
 
@@ -1133,8 +1127,8 @@ musbotg_setup_standard_chain(struct usb2_xfer *xfer)
 
 	temp.td = NULL;
 	temp.td_next = xfer->td_start[0];
-	temp.setup_alt_next = xfer->flags_int.short_frames_ok;
 	temp.offset = 0;
+	temp.setup_alt_next = xfer->flags_int.short_frames_ok;
 
 	sc = MUSBOTG_BUS2SC(xfer->xroot->bus);
 	ep_no = (xfer->endpoint & UE_ADDR);
@@ -1181,7 +1175,13 @@ musbotg_setup_standard_chain(struct usb2_xfer *xfer)
 		x++;
 
 		if (x == xfer->nframes) {
-			temp.setup_alt_next = 0;
+			if (xfer->flags_int.control_xfr) {
+				if (xfer->flags_int.control_act) {
+					temp.setup_alt_next = 0;
+				}
+			} else {
+				temp.setup_alt_next = 0;
+			}
 		}
 		if (temp.len == 0) {
 
@@ -1206,23 +1206,24 @@ musbotg_setup_standard_chain(struct usb2_xfer *xfer)
 		}
 	}
 
-	/* always setup a valid "pc" pointer for status and sync */
-	temp.pc = xfer->frbuffers + 0;
-
-	/* check if we should append a status stage */
-
-	if (xfer->flags_int.control_xfr &&
-	    !xfer->flags_int.control_act) {
+	/* check for control transfer */
+	if (xfer->flags_int.control_xfr) {
 
-		/*
-		 * Send a DATA1 message and invert the current
-		 * endpoint direction.
-		 */
-		temp.func = &musbotg_setup_status;
+		/* always setup a valid "pc" pointer for status and sync */
+		temp.pc = xfer->frbuffers + 0;
 		temp.len = 0;
 		temp.short_pkt = 0;
+		temp.setup_alt_next = 0;
 
-		musbotg_setup_standard_chain_sub(&temp);
+		/* check if we should append a status stage */
+		if (!xfer->flags_int.control_act) {
+			/*
+			 * Send a DATA1 message and invert the current
+			 * endpoint direction.
+			 */
+			temp.func = &musbotg_setup_status;
+			musbotg_setup_standard_chain_sub(&temp);
+		}
 	}
 	/* must have at least one frame! */
 	td = temp.td;
@@ -1306,31 +1307,17 @@ musbotg_start_standard_chain(struct usb2_xfer *xfer)
 }
 
 static void
-musbotg_root_intr_done(struct usb2_xfer *xfer,
-    struct usb2_sw_transfer *std)
+musbotg_root_intr(struct musbotg_softc *sc)
 {
-	struct musbotg_softc *sc = MUSBOTG_BUS2SC(xfer->xroot->bus);
-
 	DPRINTFN(8, "\n");
 
 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 
-	if (std->state != USB_SW_TR_PRE_DATA) {
-		if (std->state == USB_SW_TR_PRE_CALLBACK) {
-			/* transfer transferred */
-			musbotg_device_done(xfer, std->err);
-		}
-		goto done;
-	}
-	/* setup buffer */
-	std->ptr = sc->sc_hub_idata;
-	std->len = sizeof(sc->sc_hub_idata);
-
 	/* set port bit */
 	sc->sc_hub_idata[0] = 0x02;	/* we only have one port */
 
-done:
-	return;
+	uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
+	    sizeof(sc->sc_hub_idata));
 }
 
 static usb2_error_t
@@ -1873,7 +1860,6 @@ musbotg_do_poll(struct usb2_bus *bus)
 
 	USB_BUS_LOCK(&sc->sc_bus);
 	musbotg_interrupt_poll(sc);
-	musbotg_root_ctrl_poll(sc);
 	USB_BUS_UNLOCK(&sc->sc_bus);
 }
 
@@ -1912,8 +1898,6 @@ struct usb2_pipe_methods musbotg_device_bulk_methods =
 	.close = musbotg_device_bulk_close,
 	.enter = musbotg_device_bulk_enter,
 	.start = musbotg_device_bulk_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -1951,8 +1935,6 @@ struct usb2_pipe_methods musbotg_device_ctrl_methods =
 	.close = musbotg_device_ctrl_close,
 	.enter = musbotg_device_ctrl_enter,
 	.start = musbotg_device_ctrl_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -1990,8 +1972,6 @@ struct usb2_pipe_methods musbotg_device_intr_methods =
 	.close = musbotg_device_intr_close,
 	.enter = musbotg_device_intr_enter,
 	.start = musbotg_device_intr_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -2081,36 +2061,13 @@ struct usb2_pipe_methods musbotg_device_isoc_methods =
 	.close = musbotg_device_isoc_close,
 	.enter = musbotg_device_isoc_enter,
 	.start = musbotg_device_isoc_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
  * musbotg root control support
  *------------------------------------------------------------------------*
- * simulate a hardware HUB by handling
- * all the necessary requests
+ * Simulate a hardware HUB by handling all the necessary requests.
  *------------------------------------------------------------------------*/
-static void
-musbotg_root_ctrl_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-musbotg_root_ctrl_close(struct usb2_xfer *xfer)
-{
-	struct musbotg_softc *sc = MUSBOTG_BUS2SC(xfer->xroot->bus);
-
-	if (sc->sc_root_ctrl.xfer == xfer) {
-		sc->sc_root_ctrl.xfer = NULL;
-	}
-	musbotg_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-/*
- * USB descriptors for the virtual Root HUB:
- */
 
 static const struct usb2_device_descriptor musbotg_devd = {
 	.bLength = sizeof(struct usb2_device_descriptor),
@@ -2156,7 +2113,6 @@ static const struct musbotg_config_desc musbotg_confd = {
 		.bInterfaceSubClass = UISUBCLASS_HUB,
 		.bInterfaceProtocol = UIPROTO_HSHUBSTT,
 	},
-
 	.endpd = {
 		.bLength = sizeof(struct usb2_endpoint_descriptor),
 		.bDescriptorType = UDESC_ENDPOINT,
@@ -2197,44 +2153,15 @@ USB_MAKE_STRING_DESC(STRING_VENDOR, musbotg_vendor);
 USB_MAKE_STRING_DESC(STRING_PRODUCT, musbotg_product);
 
 static void
-musbotg_root_ctrl_enter(struct usb2_xfer *xfer)
+musbotg_roothub_exec(struct usb2_bus *bus)
 {
-	return;
-}
-
-static void
-musbotg_root_ctrl_start(struct usb2_xfer *xfer)
-{
-	struct musbotg_softc *sc = MUSBOTG_BUS2SC(xfer->xroot->bus);
-
-	sc->sc_root_ctrl.xfer = xfer;
-
-	usb2_bus_roothub_exec(xfer->xroot->bus);
-}
-
-static void
-musbotg_root_ctrl_task(struct usb2_bus *bus)
-{
-	musbotg_root_ctrl_poll(MUSBOTG_BUS2SC(bus));
-}
-
-static void
-musbotg_root_ctrl_done(struct usb2_xfer *xfer,
-    struct usb2_sw_transfer *std)
-{
-	struct musbotg_softc *sc = MUSBOTG_BUS2SC(xfer->xroot->bus);
+	struct musbotg_softc *sc = MUSBOTG_BUS2SC(bus);
+	struct usb2_sw_transfer *std = &sc->sc_bus.roothub_req;
 	uint16_t value;
 	uint16_t index;
 
 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 
-	if (std->state != USB_SW_TR_SETUP) {
-		if (std->state == USB_SW_TR_PRE_CALLBACK) {
-			/* transfer transferred */
-			musbotg_device_done(xfer, std->err);
-		}
-		goto done;
-	}
 	/* buffer reset */
 	std->ptr = USB_ADD_BYTES(&sc->sc_hub_temp, 0);
 	std->len = 0;
@@ -2493,7 +2420,7 @@ tr_handle_clear_port_feature:
 
 	switch (value) {
 	case UHF_PORT_SUSPEND:
-		musbotg_wakeup_peer(xfer);
+		musbotg_wakeup_peer(sc);
 		break;
 
 	case UHF_PORT_ENABLE:
@@ -2620,67 +2547,6 @@ done:
 }
 
 static void
-musbotg_root_ctrl_poll(struct musbotg_softc *sc)
-{
-	usb2_sw_transfer(&sc->sc_root_ctrl,
-	    &musbotg_root_ctrl_done);
-}
-
-struct usb2_pipe_methods musbotg_root_ctrl_methods =
-{
-	.open = musbotg_root_ctrl_open,
-	.close = musbotg_root_ctrl_close,
-	.enter = musbotg_root_ctrl_enter,
-	.start = musbotg_root_ctrl_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 0,
-};
-
-/*------------------------------------------------------------------------*
- * musbotg root interrupt support
- *------------------------------------------------------------------------*/
-static void
-musbotg_root_intr_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-musbotg_root_intr_close(struct usb2_xfer *xfer)
-{
-	struct musbotg_softc *sc = MUSBOTG_BUS2SC(xfer->xroot->bus);
-
-	if (sc->sc_root_intr.xfer == xfer) {
-		sc->sc_root_intr.xfer = NULL;
-	}
-	musbotg_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-static void
-musbotg_root_intr_enter(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-musbotg_root_intr_start(struct usb2_xfer *xfer)
-{
-	struct musbotg_softc *sc = MUSBOTG_BUS2SC(xfer->xroot->bus);
-
-	sc->sc_root_intr.xfer = xfer;
-}
-
-struct usb2_pipe_methods musbotg_root_intr_methods =
-{
-	.open = musbotg_root_intr_open,
-	.close = musbotg_root_intr_close,
-	.enter = musbotg_root_intr_enter,
-	.start = musbotg_root_intr_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
-};
-
-static void
 musbotg_xfer_setup(struct usb2_setup_params *parm)
 {
 	const struct usb2_hw_ep_profile *pf;
@@ -2804,24 +2670,7 @@ musbotg_pipe_init(struct usb2_device *udev, struct usb2_endpoint_descriptor *ede
 	    edesc->bEndpointAddress, udev->flags.usb2_mode,
 	    sc->sc_rt_addr);
 
-	if (udev->device_index == sc->sc_rt_addr) {
-
-		if (udev->flags.usb2_mode != USB_MODE_HOST) {
-			/* not supported */
-			return;
-		}
-		switch (edesc->bEndpointAddress) {
-		case USB_CONTROL_ENDPOINT:
-			pipe->methods = &musbotg_root_ctrl_methods;
-			break;
-		case UE_DIR_IN | MUSBOTG_INTR_ENDPT:
-			pipe->methods = &musbotg_root_intr_methods;
-			break;
-		default:
-			/* do nothing */
-			break;
-		}
-	} else {
+	if (udev->device_index != sc->sc_rt_addr) {
 
 		if (udev->flags.usb2_mode != USB_MODE_DEVICE) {
 			/* not supported */
@@ -2860,5 +2709,5 @@ struct usb2_bus_methods musbotg_bus_methods =
 	.get_hw_ep_profile = &musbotg_get_hw_ep_profile,
 	.set_stall = &musbotg_set_stall,
 	.clear_stall = &musbotg_clear_stall,
-	.roothub_exec = &musbotg_root_ctrl_task,
+	.roothub_exec = &musbotg_roothub_exec,
 };
diff --git a/sys/dev/usb/controller/musb_otg.h b/sys/dev/usb/controller/musb_otg.h
index 0d880e1..2dd2ce3 100644
--- a/sys/dev/usb/controller/musb_otg.h
+++ b/sys/dev/usb/controller/musb_otg.h
@@ -25,8 +25,8 @@
  */
 
 /*
- * This header file defines the registers of the Mentor Graphics
- * USB OnTheGo Inventra chip.
+ * This header file defines the registers of the Mentor Graphics USB OnTheGo
+ * Inventra chip.
  */
 
 #ifndef _MUSB2_OTG_H_
@@ -363,8 +363,6 @@ struct musbotg_flags {
 struct musbotg_softc {
 	struct usb2_bus sc_bus;
 	union musbotg_hub_temp sc_hub_temp;
-	struct usb2_sw_transfer sc_root_ctrl;
-	struct usb2_sw_transfer sc_root_intr;
 	struct usb2_hw_ep_profile sc_hw_ep_profile[16];
 
 	struct usb2_device *sc_devices[MUSB2_MAX_DEVICES];
diff --git a/sys/dev/usb/controller/musb_otg_atmelarm.c b/sys/dev/usb/controller/musb_otg_atmelarm.c
index 3ee5cfb..2589287 100644
--- a/sys/dev/usb/controller/musb_otg_atmelarm.c
+++ b/sys/dev/usb/controller/musb_otg_atmelarm.c
@@ -25,13 +25,11 @@
  */
 
 #include <dev/usb/usb_mfunc.h>
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb.h>
 
 #include <dev/usb/usb_core.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_util.h>
 
 #include <dev/usb/usb_controller.h>
@@ -131,10 +129,10 @@ musbotg_attach(device_t dev)
 
 #if (__FreeBSD_version >= 700031)
 	err = bus_setup_intr(dev, sc->sc_otg.sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    NULL, (void *)musbotg_interrupt, sc, &sc->sc_otg.sc_intr_hdl);
+	    NULL, (driver_intr_t *)musbotg_interrupt, sc, &sc->sc_otg.sc_intr_hdl);
 #else
 	err = bus_setup_intr(dev, sc->sc_otg.sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    (void *)musbotg_interrupt, sc, &sc->sc_otg.sc_intr_hdl);
+	    (driver_intr_t *)musbotg_interrupt, sc, &sc->sc_otg.sc_intr_hdl);
 #endif
 	if (err) {
 		sc->sc_otg.sc_intr_hdl = NULL;
diff --git a/sys/dev/usb/controller/ohci.c b/sys/dev/usb/controller/ohci.c
index ad2b1cc..fc05ece 100644
--- a/sys/dev/usb/controller/ohci.c
+++ b/sys/dev/usb/controller/ohci.c
@@ -32,13 +32,12 @@ __FBSDID("$FreeBSD$");
  * USB Open Host Controller driver.
  *
  * OHCI spec: http://www.compaq.com/productinfo/development/openhci.html
- * USB spec: http://www.usb.org/developers/docs/usbspec.zip
+ * USB spec:  http://www.usb.org/developers/docs/usbspec.zip
  */
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR ohcidebug
 
@@ -46,7 +45,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb_debug.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_transfer.h>
 #include <dev/usb/usb_device.h>
 #include <dev/usb/usb_hub.h>
@@ -56,8 +54,9 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb_bus.h>
 #include <dev/usb/controller/ohci.h>
 
-#define	OHCI_BUS2SC(bus) ((ohci_softc_t *)(((uint8_t *)(bus)) - \
-   USB_P2U(&(((ohci_softc_t *)0)->sc_bus))))
+#define	OHCI_BUS2SC(bus) \
+   ((ohci_softc_t *)(((uint8_t *)(bus)) - \
+    ((uint8_t *)&(((ohci_softc_t *)0)->sc_bus))))
 
 #if USB_DEBUG
 static int ohcidebug = 0;
@@ -93,17 +92,12 @@ extern struct usb2_pipe_methods ohci_device_bulk_methods;
 extern struct usb2_pipe_methods ohci_device_ctrl_methods;
 extern struct usb2_pipe_methods ohci_device_intr_methods;
 extern struct usb2_pipe_methods ohci_device_isoc_methods;
-extern struct usb2_pipe_methods ohci_root_ctrl_methods;
-extern struct usb2_pipe_methods ohci_root_intr_methods;
 
-static void ohci_root_ctrl_poll(struct ohci_softc *sc);
 static void ohci_do_poll(struct usb2_bus *bus);
 static void ohci_device_done(struct usb2_xfer *xfer, usb2_error_t error);
-
-static usb2_sw_transfer_func_t ohci_root_intr_done;
-static usb2_sw_transfer_func_t ohci_root_ctrl_done;
 static void ohci_timeout(void *arg);
 static uint8_t ohci_check_transfer(struct usb2_xfer *xfer);
+static void ohci_root_intr(ohci_softc_t *sc);
 
 struct ohci_std_temp {
 	struct usb2_page_cache *pc;
@@ -115,7 +109,7 @@ struct ohci_std_temp {
 	uint16_t max_frame_size;
 	uint8_t	shortpkt;
 	uint8_t	setup_alt_next;
-	uint8_t	short_frames_ok;
+	uint8_t last_frame;
 };
 
 static struct ohci_hcca *
@@ -1020,6 +1014,23 @@ ohci_check_transfer_sub(struct usb2_xfer *xfer)
 		usb2_pc_cpu_flush(ed->page_cache);
 
 		DPRINTFN(13, "xfer=%p following alt next\n", xfer);
+
+		/*
+		 * Make sure that the OHCI re-scans the schedule by
+		 * writing the BLF and CLF bits:
+		 */
+
+		if (xfer->xroot->udev->pwr_save.suspended) {
+			/* nothing to do */
+		} else if (xfer->pipe->methods == &ohci_device_bulk_methods) {
+			ohci_softc_t *sc = OHCI_BUS2SC(xfer->xroot->bus);
+
+			OWRITE4(sc, OHCI_COMMAND_STATUS, OHCI_BLF);
+		} else if (xfer->pipe->methods == &ohci_device_ctrl_methods) {
+			ohci_softc_t *sc = OHCI_BUS2SC(xfer->xroot->bus);
+
+			OWRITE4(sc, OHCI_COMMAND_STATUS, OHCI_CLF);
+		}
 	}
 }
 
@@ -1034,7 +1045,6 @@ static uint8_t
 ohci_check_transfer(struct usb2_xfer *xfer)
 {
 	ohci_ed_t *ed;
-	uint32_t ed_flags;
 	uint32_t ed_headp;
 	uint32_t ed_tailp;
 
@@ -1043,12 +1053,10 @@ ohci_check_transfer(struct usb2_xfer *xfer)
 	ed = xfer->qh_start[xfer->flags_int.curr_dma_set];
 
 	usb2_pc_cpu_invalidate(ed->page_cache);
-	ed_flags = le32toh(ed->ed_flags);
 	ed_headp = le32toh(ed->ed_headp);
 	ed_tailp = le32toh(ed->ed_tailp);
 
-	if ((ed_flags & OHCI_ED_SKIP) ||
-	    (ed_headp & OHCI_HALTED) ||
+	if ((ed_headp & OHCI_HALTED) ||
 	    (((ed_headp ^ ed_tailp) & (~0xF)) == 0)) {
 		if (xfer->pipe->methods == &ohci_device_isoc_methods) {
 			/* isochronous transfer */
@@ -1090,8 +1098,7 @@ ohci_rhsc_enable(ohci_softc_t *sc)
 	/* acknowledge any RHSC interrupt */
 	OWRITE4(sc, OHCI_INTERRUPT_STATUS, OHCI_RHSC);
 
-	usb2_sw_transfer(&sc->sc_root_intr,
-	    &ohci_root_intr_done);
+	ohci_root_intr(sc);
 }
 
 static void
@@ -1205,8 +1212,7 @@ ohci_interrupt(ohci_softc_t *sc)
 			sc->sc_eintrs &= ~OHCI_RHSC;
 			OWRITE4(sc, OHCI_INTERRUPT_DISABLE, OHCI_RHSC);
 
-			usb2_sw_transfer(&sc->sc_root_intr,
-			    &ohci_root_intr_done);
+			ohci_root_intr(sc);
 
 			/* do not allow RHSC interrupts > 1 per second */
 			usb2_callout_reset(&sc->sc_tmo_rhsc, hz,
@@ -1251,7 +1257,6 @@ ohci_do_poll(struct usb2_bus *bus)
 
 	USB_BUS_LOCK(&sc->sc_bus);
 	ohci_interrupt_poll(sc);
-	ohci_root_ctrl_poll(sc);
 	USB_BUS_UNLOCK(&sc->sc_bus);
 }
 
@@ -1336,9 +1341,12 @@ restart:
 		temp->td_flags &= ~htole32(OHCI_TD_TOGGLE_MASK);
 
 		if (average == 0) {
-
+			/*
+			 * The buffer start and end phys addresses should be
+			 * 0x0 for a zero length packet.
+			 */
 			td->td_cbp = 0;
-			td->td_be = ~0;
+			td->td_be = 0;
 			td->len = 0;
 
 		} else {
@@ -1379,10 +1387,9 @@ restart:
 		precompute = 0;
 
 		/* setup alt next pointer, if any */
-		if (temp->short_frames_ok) {
-			if (temp->setup_alt_next) {
-				td_alt_next = td_next;
-			}
+		if (temp->last_frame) {
+			/* no alternate next */
+			td_alt_next = NULL;
 		} else {
 			/* we use this field internally */
 			td_alt_next = td_next;
@@ -1411,7 +1418,7 @@ ohci_setup_standard_chain(struct usb2_xfer *xfer, ohci_ed_t **ed_last)
 	    xfer->address, UE_GET_ADDR(xfer->endpoint),
 	    xfer->sumlen, usb2_get_speed(xfer->xroot->udev));
 
-	temp.average = xfer->max_usb2_frame_size;
+	temp.average = xfer->max_hc_frame_size;
 	temp.max_frame_size = xfer->max_frame_size;
 
 	/* toggle the DMA set we are using */
@@ -1425,8 +1432,8 @@ ohci_setup_standard_chain(struct usb2_xfer *xfer, ohci_ed_t **ed_last)
 
 	temp.td = NULL;
 	temp.td_next = td;
+	temp.last_frame = 0;
 	temp.setup_alt_next = xfer->flags_int.short_frames_ok;
-	temp.short_frames_ok = xfer->flags_int.short_frames_ok;
 
 	methods = xfer->pipe->methods;
 
@@ -1441,7 +1448,14 @@ ohci_setup_standard_chain(struct usb2_xfer *xfer, ohci_ed_t **ed_last)
 			temp.len = xfer->frlengths[0];
 			temp.pc = xfer->frbuffers + 0;
 			temp.shortpkt = temp.len ? 1 : 0;
-
+			/* check for last frame */
+			if (xfer->nframes == 1) {
+				/* no STATUS stage yet, SETUP is last */
+				if (xfer->flags_int.control_act) {
+					temp.last_frame = 1;
+					temp.setup_alt_next = 0;
+				}
+			}
 			ohci_setup_standard_chain_sub(&temp);
 
 			/*
@@ -1482,7 +1496,16 @@ ohci_setup_standard_chain(struct usb2_xfer *xfer, ohci_ed_t **ed_last)
 		x++;
 
 		if (x == xfer->nframes) {
-			temp.setup_alt_next = 0;
+			if (xfer->flags_int.control_xfr) {
+				/* no STATUS stage yet, DATA is last */
+				if (xfer->flags_int.control_act) {
+					temp.last_frame = 1;
+					temp.setup_alt_next = 0;
+				}
+			} else {
+				temp.last_frame = 1;
+				temp.setup_alt_next = 0;
+			}
 		}
 		if (temp.len == 0) {
 
@@ -1523,11 +1546,14 @@ ohci_setup_standard_chain(struct usb2_xfer *xfer, ohci_ed_t **ed_last)
 		temp.len = 0;
 		temp.pc = NULL;
 		temp.shortpkt = 0;
+		temp.last_frame = 1;
+		temp.setup_alt_next = 0;
 
 		ohci_setup_standard_chain_sub(&temp);
 	}
 	td = temp.td;
 
+	/* Ensure that last TD is terminating: */
 	td->td_next = htole32(OHCI_TD_NEXT_END);
 	td->td_flags &= ~htole32(OHCI_TD_INTR_MASK);
 	td->td_flags |= htole32(OHCI_TD_SET_DI(1));
@@ -1583,33 +1609,20 @@ ohci_setup_standard_chain(struct usb2_xfer *xfer, ohci_ed_t **ed_last)
 }
 
 static void
-ohci_root_intr_done(struct usb2_xfer *xfer,
-    struct usb2_sw_transfer *std)
+ohci_root_intr(ohci_softc_t *sc)
 {
-	ohci_softc_t *sc = OHCI_BUS2SC(xfer->xroot->bus);
 	uint32_t hstatus;
 	uint16_t i;
 	uint16_t m;
 
 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 
-	if (std->state != USB_SW_TR_PRE_DATA) {
-		if (std->state == USB_SW_TR_PRE_CALLBACK) {
-			/* transfer transferred */
-			ohci_device_done(xfer, std->err);
-		}
-		goto done;
-	}
-	/* setup buffer */
-	std->ptr = sc->sc_hub_idata;
-	std->len = sizeof(sc->sc_hub_idata);
-
 	/* clear any old interrupt data */
-	bzero(sc->sc_hub_idata, sizeof(sc->sc_hub_idata));
+	memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
 
 	hstatus = OREAD4(sc, OHCI_RH_STATUS);
-	DPRINTF("sc=%p xfer=%p hstatus=0x%08x\n",
-	    sc, xfer, hstatus);
+	DPRINTF("sc=%p hstatus=0x%08x\n",
+	    sc, hstatus);
 
 	/* set bits */
 	m = (sc->sc_noport + 1);
@@ -1623,8 +1636,9 @@ ohci_root_intr_done(struct usb2_xfer *xfer,
 			DPRINTF("port %d changed\n", i);
 		}
 	}
-done:
-	return;
+
+	uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
+	    sizeof(sc->sc_hub_idata));
 }
 
 /* NOTE: "done" can be run two times in a row,
@@ -1705,8 +1719,6 @@ struct usb2_pipe_methods ohci_device_bulk_methods =
 	.close = ohci_device_bulk_close,
 	.enter = ohci_device_bulk_enter,
 	.start = ohci_device_bulk_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -1748,8 +1760,6 @@ struct usb2_pipe_methods ohci_device_ctrl_methods =
 	.close = ohci_device_ctrl_close,
 	.enter = ohci_device_ctrl_enter,
 	.start = ohci_device_ctrl_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -1822,8 +1832,6 @@ struct usb2_pipe_methods ohci_device_intr_methods =
 	.close = ohci_device_intr_close,
 	.enter = ohci_device_intr_enter,
 	.start = ohci_device_intr_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -2032,37 +2040,14 @@ struct usb2_pipe_methods ohci_device_isoc_methods =
 	.close = ohci_device_isoc_close,
 	.enter = ohci_device_isoc_enter,
 	.start = ohci_device_isoc_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
  * ohci root control support
  *------------------------------------------------------------------------*
- * simulate a hardware hub by handling
- * all the necessary requests
+ * Simulate a hardware hub by handling all the necessary requests.
  *------------------------------------------------------------------------*/
 
-static void
-ohci_root_ctrl_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-ohci_root_ctrl_close(struct usb2_xfer *xfer)
-{
-	ohci_softc_t *sc = OHCI_BUS2SC(xfer->xroot->bus);
-
-	if (sc->sc_root_ctrl.xfer == xfer) {
-		sc->sc_root_ctrl.xfer = NULL;
-	}
-	ohci_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-/* data structures and routines
- * to emulate the root hub:
- */
 static const
 struct usb2_device_descriptor ohci_devd =
 {
@@ -2091,7 +2076,6 @@ struct ohci_config_desc ohci_confd =
 		.bmAttributes = UC_SELF_POWERED,
 		.bMaxPower = 0,		/* max power */
 	},
-
 	.ifcd = {
 		.bLength = sizeof(struct usb2_interface_descriptor),
 		.bDescriptorType = UDESC_INTERFACE,
@@ -2100,7 +2084,6 @@ struct ohci_config_desc ohci_confd =
 		.bInterfaceSubClass = UISUBCLASS_HUB,
 		.bInterfaceProtocol = UIPROTO_FSHUB,
 	},
-
 	.endpd = {
 		.bLength = sizeof(struct usb2_endpoint_descriptor),
 		.bDescriptorType = UDESC_ENDPOINT,
@@ -2124,32 +2107,10 @@ struct usb2_hub_descriptor ohci_hubd =
 };
 
 static void
-ohci_root_ctrl_enter(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-ohci_root_ctrl_start(struct usb2_xfer *xfer)
-{
-	ohci_softc_t *sc = OHCI_BUS2SC(xfer->xroot->bus);
-
-	sc->sc_root_ctrl.xfer = xfer;
-
-	usb2_bus_roothub_exec(xfer->xroot->bus);
-}
-
-static void
-ohci_root_ctrl_task(struct usb2_bus *bus)
-{
-	ohci_root_ctrl_poll(OHCI_BUS2SC(bus));
-}
-
-static void
-ohci_root_ctrl_done(struct usb2_xfer *xfer,
-    struct usb2_sw_transfer *std)
+ohci_roothub_exec(struct usb2_bus *bus)
 {
-	ohci_softc_t *sc = OHCI_BUS2SC(xfer->xroot->bus);
+	ohci_softc_t *sc = OHCI_BUS2SC(bus);
+	struct usb2_sw_transfer *std = &sc->sc_bus.roothub_req;
 	char *ptr;
 	uint32_t port;
 	uint32_t v;
@@ -2159,13 +2120,6 @@ ohci_root_ctrl_done(struct usb2_xfer *xfer,
 
 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 
-	if (std->state != USB_SW_TR_SETUP) {
-		if (std->state == USB_SW_TR_PRE_CALLBACK) {
-			/* transfer transferred */
-			ohci_device_done(xfer, std->err);
-		}
-		goto done;
-	}
 	/* buffer reset */
 	std->ptr = sc->sc_hub_desc.temp;
 	std->len = 0;
@@ -2256,7 +2210,7 @@ ohci_root_ctrl_done(struct usb2_xfer *xfer,
 		USETW(sc->sc_hub_desc.stat.wStatus, 0);
 		break;
 	case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
-		if (value >= USB_MAX_DEVICES) {
+		if (value >= OHCI_MAX_DEVICES) {
 			std->err = USB_ERR_IOERROR;
 			goto done;
 		}
@@ -2439,67 +2393,6 @@ done:
 }
 
 static void
-ohci_root_ctrl_poll(struct ohci_softc *sc)
-{
-	usb2_sw_transfer(&sc->sc_root_ctrl,
-	    &ohci_root_ctrl_done);
-}
-
-struct usb2_pipe_methods ohci_root_ctrl_methods =
-{
-	.open = ohci_root_ctrl_open,
-	.close = ohci_root_ctrl_close,
-	.enter = ohci_root_ctrl_enter,
-	.start = ohci_root_ctrl_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 0,
-};
-
-/*------------------------------------------------------------------------*
- * ohci root interrupt support
- *------------------------------------------------------------------------*/
-static void
-ohci_root_intr_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-ohci_root_intr_close(struct usb2_xfer *xfer)
-{
-	ohci_softc_t *sc = OHCI_BUS2SC(xfer->xroot->bus);
-
-	if (sc->sc_root_intr.xfer == xfer) {
-		sc->sc_root_intr.xfer = NULL;
-	}
-	ohci_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-static void
-ohci_root_intr_enter(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-ohci_root_intr_start(struct usb2_xfer *xfer)
-{
-	ohci_softc_t *sc = OHCI_BUS2SC(xfer->xroot->bus);
-
-	sc->sc_root_intr.xfer = xfer;
-}
-
-struct usb2_pipe_methods ohci_root_intr_methods =
-{
-	.open = ohci_root_intr_open,
-	.close = ohci_root_intr_close,
-	.enter = ohci_root_intr_enter,
-	.start = ohci_root_intr_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
-};
-
-static void
 ohci_xfer_setup(struct usb2_setup_params *parm)
 {
 	struct usb2_page_search page_info;
@@ -2529,7 +2422,7 @@ ohci_xfer_setup(struct usb2_setup_params *parm)
 
 		nitd = 0;
 		ntd = ((2 * xfer->nframes) + 1	/* STATUS */
-		    + (xfer->max_data_length / xfer->max_usb2_frame_size));
+		    + (xfer->max_data_length / xfer->max_hc_frame_size));
 		nqh = 1;
 
 	} else if (parm->methods == &ohci_device_bulk_methods) {
@@ -2539,7 +2432,7 @@ ohci_xfer_setup(struct usb2_setup_params *parm)
 
 		nitd = 0;
 		ntd = ((2 * xfer->nframes)
-		    + (xfer->max_data_length / xfer->max_usb2_frame_size));
+		    + (xfer->max_data_length / xfer->max_hc_frame_size));
 		nqh = 1;
 
 	} else if (parm->methods == &ohci_device_intr_methods) {
@@ -2549,7 +2442,7 @@ ohci_xfer_setup(struct usb2_setup_params *parm)
 
 		nitd = 0;
 		ntd = ((2 * xfer->nframes)
-		    + (xfer->max_data_length / xfer->max_usb2_frame_size));
+		    + (xfer->max_data_length / xfer->max_hc_frame_size));
 		nqh = 1;
 
 	} else if (parm->methods == &ohci_device_isoc_methods) {
@@ -2678,19 +2571,7 @@ ohci_pipe_init(struct usb2_device *udev, struct usb2_endpoint_descriptor *edesc,
 		/* not supported */
 		return;
 	}
-	if (udev->device_index == sc->sc_addr) {
-		switch (edesc->bEndpointAddress) {
-		case USB_CONTROL_ENDPOINT:
-			pipe->methods = &ohci_root_ctrl_methods;
-			break;
-		case UE_DIR_IN | OHCI_INTR_ENDPT:
-			pipe->methods = &ohci_root_intr_methods;
-			break;
-		default:
-			/* do nothing */
-			break;
-		}
-	} else {
+	if (udev->device_index != sc->sc_addr) {
 		switch (edesc->bmAttributes & UE_XFERTYPE) {
 		case UE_CONTROL:
 			pipe->methods = &ohci_device_ctrl_methods;
@@ -2849,5 +2730,5 @@ struct usb2_bus_methods ohci_bus_methods =
 	.device_resume = ohci_device_resume,
 	.device_suspend = ohci_device_suspend,
 	.set_hw_power = ohci_set_hw_power,
-	.roothub_exec = ohci_root_ctrl_task,
+	.roothub_exec = ohci_roothub_exec,
 };
diff --git a/sys/dev/usb/controller/ohci.h b/sys/dev/usb/controller/ohci.h
index 84a6afd..322324a 100644
--- a/sys/dev/usb/controller/ohci.h
+++ b/sys/dev/usb/controller/ohci.h
@@ -39,9 +39,9 @@
 #ifndef _OHCI_H_
 #define	_OHCI_H_
 
-#define	OHCI_MAX_DEVICES USB_MAX_DEVICES
+#define	OHCI_MAX_DEVICES MIN(USB_MAX_DEVICES, 128)
 
-/* PCI config registers */
+/* PCI config registers  */
 #define	PCI_CBMEM		0x10	/* configuration base memory */
 #define	PCI_INTERFACE_OHCI	0x10
 
@@ -322,8 +322,6 @@ typedef struct ohci_softc {
 	struct usb2_bus sc_bus;		/* base device */
 	struct usb2_callout sc_tmo_rhsc;
 	union ohci_hub_desc sc_hub_desc;
-	struct usb2_sw_transfer sc_root_ctrl;
-	struct usb2_sw_transfer sc_root_intr;
 
 	struct usb2_device *sc_devices[OHCI_MAX_DEVICES];
 	struct resource *sc_io_res;
diff --git a/sys/dev/usb/controller/ohci_atmelarm.c b/sys/dev/usb/controller/ohci_atmelarm.c
index 562cf3d..0f213c5 100644
--- a/sys/dev/usb/controller/ohci_atmelarm.c
+++ b/sys/dev/usb/controller/ohci_atmelarm.c
@@ -26,13 +26,11 @@
 __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb_mfunc.h>
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb.h>
 
 #include <dev/usb/usb_core.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_util.h>
 
 #include <dev/usb/usb_controller.h>
@@ -112,10 +110,10 @@ ohci_atmelarm_attach(device_t dev)
 
 #if (__FreeBSD_version >= 700031)
 	err = bus_setup_intr(dev, sc->sc_ohci.sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    NULL, (void *)ohci_interrupt, sc, &sc->sc_ohci.sc_intr_hdl);
+	    NULL, (driver_intr_t *)ohci_interrupt, sc, &sc->sc_ohci.sc_intr_hdl);
 #else
 	err = bus_setup_intr(dev, sc->sc_ohci.sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    (void *)ohci_interrupt, sc, &sc->sc_ohci.sc_intr_hdl);
+	    (driver_intr_t *)ohci_interrupt, sc, &sc->sc_ohci.sc_intr_hdl);
 #endif
 	if (err) {
 		sc->sc_ohci.sc_intr_hdl = NULL;
diff --git a/sys/dev/usb/controller/ohci_pci.c b/sys/dev/usb/controller/ohci_pci.c
index c9f586d..031be52 100644
--- a/sys/dev/usb/controller/ohci_pci.c
+++ b/sys/dev/usb/controller/ohci_pci.c
@@ -52,12 +52,10 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
-#include <dev/usb/usb_defs.h>
 
 #include <dev/usb/usb_core.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_util.h>
 
 #include <dev/usb/usb_controller.h>
@@ -290,10 +288,10 @@ ohci_pci_attach(device_t self)
 
 #if (__FreeBSD_version >= 700031)
 	err = bus_setup_intr(self, sc->sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    NULL, (void *)(void *)ohci_interrupt, sc, &sc->sc_intr_hdl);
+	    NULL, (driver_intr_t *)ohci_interrupt, sc, &sc->sc_intr_hdl);
 #else
 	err = bus_setup_intr(self, sc->sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    (void *)(void *)ohci_interrupt, sc, &sc->sc_intr_hdl);
+	    (driver_intr_t *)ohci_interrupt, sc, &sc->sc_intr_hdl);
 #endif
 	if (err) {
 		device_printf(self, "Could not setup irq, %d\n", err);
diff --git a/sys/dev/usb/controller/uhci.c b/sys/dev/usb/controller/uhci.c
index 82a9623..bc32557 100644
--- a/sys/dev/usb/controller/uhci.c
+++ b/sys/dev/usb/controller/uhci.c
@@ -33,7 +33,7 @@ __FBSDID("$FreeBSD$");
  * Handles e.g. PIIX3 and PIIX4.
  *
  * UHCI spec: http://developer.intel.com/design/USB/UHCI11D.htm
- * USB spec: http://www.usb.org/developers/docs/usbspec.zip
+ * USB spec:  http://www.usb.org/developers/docs/usbspec.zip
  * PIIXn spec: ftp://download.intel.com/design/intarch/datashts/29055002.pdf
  *             ftp://download.intel.com/design/intarch/datashts/29056201.pdf
  */
@@ -41,7 +41,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR uhcidebug
 
@@ -49,7 +48,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb_debug.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_transfer.h>
 #include <dev/usb/usb_device.h>
 #include <dev/usb/usb_hub.h>
@@ -60,8 +58,9 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/controller/uhci.h>
 
 #define	alt_next next
-#define	UHCI_BUS2SC(bus) ((uhci_softc_t *)(((uint8_t *)(bus)) - \
-   USB_P2U(&(((uhci_softc_t *)0)->sc_bus))))
+#define	UHCI_BUS2SC(bus) \
+   ((uhci_softc_t *)(((uint8_t *)(bus)) - \
+    ((uint8_t *)&(((uhci_softc_t *)0)->sc_bus))))
 
 #if USB_DEBUG
 static int uhcidebug = 0;
@@ -124,7 +123,7 @@ struct uhci_std_temp {
 	uint16_t max_frame_size;
 	uint8_t	shortpkt;
 	uint8_t	setup_alt_next;
-	uint8_t	short_frames_ok;
+	uint8_t	last_frame;
 };
 
 extern struct usb2_bus_methods uhci_bus_methods;
@@ -132,16 +131,13 @@ extern struct usb2_pipe_methods uhci_device_bulk_methods;
 extern struct usb2_pipe_methods uhci_device_ctrl_methods;
 extern struct usb2_pipe_methods uhci_device_intr_methods;
 extern struct usb2_pipe_methods uhci_device_isoc_methods;
-extern struct usb2_pipe_methods uhci_root_ctrl_methods;
-extern struct usb2_pipe_methods uhci_root_intr_methods;
 
-static void	uhci_root_ctrl_poll(struct uhci_softc *);
 static void	uhci_do_poll(struct usb2_bus *);
 static void	uhci_device_done(struct usb2_xfer *, usb2_error_t);
 static void	uhci_transfer_intr_enqueue(struct usb2_xfer *);
-static void	uhci_root_intr_check(void *);
 static void	uhci_timeout(void *);
 static uint8_t	uhci_check_transfer(struct usb2_xfer *);
+static void	uhci_root_intr(uhci_softc_t *sc);
 
 void
 uhci_iterate_hw_softc(struct usb2_bus *bus, usb2_bus_mem_sub_cb_t *cb)
@@ -317,6 +313,13 @@ done_2:
 	UWRITE4(sc, UHCI_FLBASEADDR, buf_res.physaddr);
 	UWRITE2(sc, UHCI_FRNUM, sc->sc_saved_frnum);
 	UWRITE1(sc, UHCI_SOF, sc->sc_saved_sof);
+
+	USB_BUS_UNLOCK(&sc->sc_bus);
+
+	/* stop root interrupt */
+	usb2_callout_drain(&sc->sc_root_intr);
+
+	USB_BUS_LOCK(&sc->sc_bus);
 }
 
 static void
@@ -358,7 +361,8 @@ uhci_start(uhci_softc_t *sc)
 	    "cannot start HC controller\n");
 
 done:
-	return;
+	/* start root interrupt */
+	uhci_root_intr(sc);
 }
 
 static struct uhci_qh *
@@ -408,12 +412,13 @@ uhci_init(uhci_softc_t *sc)
 
 	DPRINTF("start\n");
 
+	usb2_callout_init_mtx(&sc->sc_root_intr, &sc->sc_bus.bus_mtx, 0);
+
 #if USB_DEBUG
 	if (uhcidebug > 2) {
 		uhci_dumpregs(sc);
 	}
 #endif
-
 	sc->sc_saved_sof = 0x40;	/* default value */
 	sc->sc_saved_frnum = 0;		/* default frame number */
 
@@ -1253,30 +1258,33 @@ uhci_check_transfer_sub(struct usb2_xfer *xfer)
 	td_self = td->td_self;
 	td_alt_next = td->alt_next;
 
-	if ((td->td_token ^ td_token) & htole32(UHCI_TD_SET_DT(1))) {
+	if (xfer->flags_int.control_xfr)
+		goto skip;	/* don't touch the DT value! */
 
-		/*
-	         * The data toggle is wrong and
-	         * we need to switch it !
-	         */
+	if (!((td->td_token ^ td_token) & htole32(UHCI_TD_SET_DT(1))))
+		goto skip;	/* data toggle has correct value */
 
-		while (1) {
+	/*
+	 * The data toggle is wrong and we need to toggle it !
+	 */
+	while (1) {
 
-			td->td_token ^= htole32(UHCI_TD_SET_DT(1));
-			usb2_pc_cpu_flush(td->page_cache);
+		td->td_token ^= htole32(UHCI_TD_SET_DT(1));
+		usb2_pc_cpu_flush(td->page_cache);
 
-			if (td == xfer->td_transfer_last) {
-				/* last transfer */
-				break;
-			}
-			td = td->obj_next;
+		if (td == xfer->td_transfer_last) {
+			/* last transfer */
+			break;
+		}
+		td = td->obj_next;
 
-			if (td->alt_next != td_alt_next) {
-				/* next frame */
-				break;
-			}
+		if (td->alt_next != td_alt_next) {
+			/* next frame */
+			break;
 		}
 	}
+skip:
+
 	/* update the QH */
 	qh->qh_e_next = td_self;
 	usb2_pc_cpu_flush(qh->page_cache);
@@ -1502,7 +1510,6 @@ uhci_do_poll(struct usb2_bus *bus)
 
 	USB_BUS_LOCK(&sc->sc_bus);
 	uhci_interrupt_poll(sc);
-	uhci_root_ctrl_poll(sc);
 	USB_BUS_UNLOCK(&sc->sc_bus);
 }
 
@@ -1631,10 +1638,8 @@ restart:
 		precompute = 0;
 
 		/* setup alt next pointer, if any */
-		if (temp->short_frames_ok) {
-			if (temp->setup_alt_next) {
-				td_alt_next = td_next;
-			}
+		if (temp->last_frame) {
+			td_alt_next = NULL;
 		} else {
 			/* we use this field internally */
 			td_alt_next = td_next;
@@ -1673,8 +1678,8 @@ uhci_setup_standard_chain(struct usb2_xfer *xfer)
 
 	temp.td = NULL;
 	temp.td_next = td;
+	temp.last_frame = 0;
 	temp.setup_alt_next = xfer->flags_int.short_frames_ok;
-	temp.short_frames_ok = xfer->flags_int.short_frames_ok;
 
 	uhci_mem_layout_init(&temp.ml, xfer);
 
@@ -1707,7 +1712,14 @@ uhci_setup_standard_chain(struct usb2_xfer *xfer)
 			temp.len = xfer->frlengths[0];
 			temp.ml.buf_pc = xfer->frbuffers + 0;
 			temp.shortpkt = temp.len ? 1 : 0;
-
+			/* check for last frame */
+			if (xfer->nframes == 1) {
+				/* no STATUS stage yet, SETUP is last */
+				if (xfer->flags_int.control_act) {
+					temp.last_frame = 1;
+					temp.setup_alt_next = 0;
+				}
+			}
 			uhci_setup_standard_chain_sub(&temp);
 		}
 		x = 1;
@@ -1725,7 +1737,16 @@ uhci_setup_standard_chain(struct usb2_xfer *xfer)
 		x++;
 
 		if (x == xfer->nframes) {
-			temp.setup_alt_next = 0;
+			if (xfer->flags_int.control_xfr) {
+				/* no STATUS stage yet, DATA is last */
+				if (xfer->flags_int.control_act) {
+					temp.last_frame = 1;
+					temp.setup_alt_next = 0;
+				}
+			} else {
+				temp.last_frame = 1;
+				temp.setup_alt_next = 0;
+			}
 		}
 		/*
 		 * Keep previous data toggle,
@@ -1780,11 +1801,14 @@ uhci_setup_standard_chain(struct usb2_xfer *xfer)
 		temp.len = 0;
 		temp.ml.buf_pc = NULL;
 		temp.shortpkt = 0;
+		temp.last_frame = 1;
+		temp.setup_alt_next = 0;
 
 		uhci_setup_standard_chain_sub(&temp);
 	}
 	td = temp.td;
 
+	/* Ensure that last TD is terminating: */
 	td->td_next = htole32(UHCI_PTR_T);
 
 	/* set interrupt bit */
@@ -1915,8 +1939,6 @@ struct usb2_pipe_methods uhci_device_bulk_methods =
 	.close = uhci_device_bulk_close,
 	.enter = uhci_device_bulk_enter,
 	.start = uhci_device_bulk_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -1979,8 +2001,6 @@ struct usb2_pipe_methods uhci_device_ctrl_methods =
 	.close = uhci_device_ctrl_close,
 	.enter = uhci_device_ctrl_enter,
 	.start = uhci_device_ctrl_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -2070,8 +2090,6 @@ struct usb2_pipe_methods uhci_device_intr_methods =
 	.close = uhci_device_intr_close,
 	.enter = uhci_device_intr_enter,
 	.start = uhci_device_intr_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -2280,38 +2298,14 @@ struct usb2_pipe_methods uhci_device_isoc_methods =
 	.close = uhci_device_isoc_close,
 	.enter = uhci_device_isoc_enter,
 	.start = uhci_device_isoc_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
  * uhci root control support
  *------------------------------------------------------------------------*
- * simulate a hardware hub by handling
- * all the necessary requests
+ * Simulate a hardware hub by handling all the necessary requests.
  *------------------------------------------------------------------------*/
 
-static void
-uhci_root_ctrl_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-uhci_root_ctrl_close(struct usb2_xfer *xfer)
-{
-	uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
-
-	if (sc->sc_root_ctrl.xfer == xfer) {
-		sc->sc_root_ctrl.xfer = NULL;
-	}
-	uhci_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-/* data structures and routines
- * to emulate the root hub:
- */
-
 static const
 struct usb2_device_descriptor uhci_devd =
 {
@@ -2338,7 +2332,6 @@ static const struct uhci_config_desc uhci_confd = {
 		.bmAttributes = UC_SELF_POWERED,
 		.bMaxPower = 0		/* max power */
 	},
-
 	.ifcd = {
 		.bLength = sizeof(struct usb2_interface_descriptor),
 		.bDescriptorType = UDESC_INTERFACE,
@@ -2347,7 +2340,6 @@ static const struct uhci_config_desc uhci_confd = {
 		.bInterfaceSubClass = UISUBCLASS_HUB,
 		.bInterfaceProtocol = UIPROTO_FSHUB,
 	},
-
 	.endpd = {
 		.bLength = sizeof(struct usb2_endpoint_descriptor),
 		.bDescriptorType = UDESC_ENDPOINT,
@@ -2491,34 +2483,10 @@ done:
 }
 
 static void
-uhci_root_ctrl_enter(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-uhci_root_ctrl_start(struct usb2_xfer *xfer)
-{
-	uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
-
-	DPRINTF("\n");
-
-	sc->sc_root_ctrl.xfer = xfer;
-
-	usb2_bus_roothub_exec(xfer->xroot->bus);
-}
-
-static void
-uhci_root_ctrl_task(struct usb2_bus *bus)
-{
-	uhci_root_ctrl_poll(UHCI_BUS2SC(bus));
-}
-
-static void
-uhci_root_ctrl_done(struct usb2_xfer *xfer,
-    struct usb2_sw_transfer *std)
+uhci_roothub_exec(struct usb2_bus *bus)
 {
-	uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
+	uhci_softc_t *sc = UHCI_BUS2SC(bus);
+	struct usb2_sw_transfer *std = &sc->sc_bus.roothub_req;
 	char *ptr;
 	uint16_t x;
 	uint16_t port;
@@ -2529,13 +2497,6 @@ uhci_root_ctrl_done(struct usb2_xfer *xfer,
 
 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 
-	if (std->state != USB_SW_TR_SETUP) {
-		if (std->state == USB_SW_TR_PRE_CALLBACK) {
-			/* transfer transferred */
-			uhci_device_done(xfer, std->err);
-		}
-		goto done;
-	}
 	/* buffer reset */
 	std->ptr = sc->sc_hub_desc.temp;
 	std->len = 0;
@@ -2626,7 +2587,7 @@ uhci_root_ctrl_done(struct usb2_xfer *xfer,
 		USETW(sc->sc_hub_desc.stat.wStatus, 0);
 		break;
 	case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
-		if (value >= USB_MAX_DEVICES) {
+		if (value >= UHCI_MAX_DEVICES) {
 			std->err = USB_ERR_IOERROR;
 			goto done;
 		}
@@ -2836,92 +2797,13 @@ done:
 	return;
 }
 
-static void
-uhci_root_ctrl_poll(struct uhci_softc *sc)
-{
-	usb2_sw_transfer(&sc->sc_root_ctrl,
-	    &uhci_root_ctrl_done);
-}
-
-struct usb2_pipe_methods uhci_root_ctrl_methods =
-{
-	.open = uhci_root_ctrl_open,
-	.close = uhci_root_ctrl_close,
-	.enter = uhci_root_ctrl_enter,
-	.start = uhci_root_ctrl_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 0,
-};
-
-/*------------------------------------------------------------------------*
- * uhci root interrupt support
- *------------------------------------------------------------------------*/
-static void
-uhci_root_intr_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-uhci_root_intr_close(struct usb2_xfer *xfer)
-{
-	uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
-
-	if (sc->sc_root_intr.xfer == xfer) {
-		sc->sc_root_intr.xfer = NULL;
-	}
-	uhci_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-static void
-uhci_root_intr_enter(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-uhci_root_intr_start(struct usb2_xfer *xfer)
-{
-	uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
-
-	sc->sc_root_intr.xfer = xfer;
-
-	usb2_transfer_timeout_ms(xfer,
-	    &uhci_root_intr_check, xfer->interval);
-}
-
-static void
-uhci_root_intr_done(struct usb2_xfer *xfer,
-    struct usb2_sw_transfer *std)
-{
-	uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
-
-	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
-
-	if (std->state != USB_SW_TR_PRE_DATA) {
-		if (std->state == USB_SW_TR_PRE_CALLBACK) {
-			/* transfer is transferred */
-			uhci_device_done(xfer, std->err);
-		}
-		goto done;
-	}
-	/* setup buffer */
-	std->ptr = sc->sc_hub_idata;
-	std->len = sizeof(sc->sc_hub_idata);
-done:
-	return;
-}
-
 /*
- * this routine is executed periodically and simulates interrupts
- * from the root controller interrupt pipe for port status change
+ * This routine is executed periodically and simulates interrupts from
+ * the root controller interrupt pipe for port status change:
  */
 static void
-uhci_root_intr_check(void *arg)
+uhci_root_intr(uhci_softc_t *sc)
 {
-	struct usb2_xfer *xfer = arg;
-	uhci_softc_t *sc = UHCI_BUS2SC(xfer->xroot->bus);
-
 	DPRINTFN(21, "\n");
 
 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
@@ -2936,27 +2818,17 @@ uhci_root_intr_check(void *arg)
 	    UHCI_PORTSC_OCIC | UHCI_PORTSC_RD)) {
 		sc->sc_hub_idata[0] |= 1 << 2;
 	}
-	if (sc->sc_hub_idata[0] == 0) {
-		/*
-		 * no change or controller not running, try again in a while
-		 */
-		uhci_root_intr_start(xfer);
-	} else {
-		usb2_sw_transfer(&sc->sc_root_intr,
-		    &uhci_root_intr_done);
+
+	/* restart timer */
+	usb2_callout_reset(&sc->sc_root_intr, hz,
+	    (void *)&uhci_root_intr, sc);
+
+	if (sc->sc_hub_idata[0] != 0) {
+		uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
+		    sizeof(sc->sc_hub_idata));
 	}
 }
 
-struct usb2_pipe_methods uhci_root_intr_methods =
-{
-	.open = uhci_root_intr_open,
-	.close = uhci_root_intr_close,
-	.enter = uhci_root_intr_enter,
-	.start = uhci_root_intr_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
-};
-
 static void
 uhci_xfer_setup(struct usb2_setup_params *parm)
 {
@@ -3168,19 +3040,7 @@ uhci_pipe_init(struct usb2_device *udev, struct usb2_endpoint_descriptor *edesc,
 		/* not supported */
 		return;
 	}
-	if (udev->device_index == sc->sc_addr) {
-		switch (edesc->bEndpointAddress) {
-		case USB_CONTROL_ENDPOINT:
-			pipe->methods = &uhci_root_ctrl_methods;
-			break;
-		case UE_DIR_IN | UHCI_INTR_ENDPT:
-			pipe->methods = &uhci_root_intr_methods;
-			break;
-		default:
-			/* do nothing */
-			break;
-		}
-	} else {
+	if (udev->device_index != sc->sc_addr) {
 		switch (edesc->bmAttributes & UE_XFERTYPE) {
 		case UE_CONTROL:
 			pipe->methods = &uhci_device_ctrl_methods;
@@ -3354,5 +3214,5 @@ struct usb2_bus_methods uhci_bus_methods =
 	.device_resume = uhci_device_resume,
 	.device_suspend = uhci_device_suspend,
 	.set_hw_power = uhci_set_hw_power,
-	.roothub_exec = uhci_root_ctrl_task,
+	.roothub_exec = uhci_roothub_exec,
 };
diff --git a/sys/dev/usb/controller/uhci.h b/sys/dev/usb/controller/uhci.h
index 9365a4c..64a80cf 100644
--- a/sys/dev/usb/controller/uhci.h
+++ b/sys/dev/usb/controller/uhci.h
@@ -39,9 +39,9 @@
 #ifndef _UHCI_H_
 #define	_UHCI_H_
 
-#define	UHCI_MAX_DEVICES USB_MAX_DEVICES
+#define	UHCI_MAX_DEVICES MIN(USB_MAX_DEVICES, 128)
 
-/* PCI config registers */
+/* PCI config registers  */
 #define	PCI_USBREV		0x60	/* USB protocol revision */
 #define	PCI_USB_REV_MASK		0xff
 #define	PCI_USB_REV_PRE_1_0	0x00
@@ -271,19 +271,19 @@ typedef struct uhci_softc {
 	struct uhci_hw_softc sc_hw;
 	struct usb2_bus sc_bus;		/* base device */
 	union uhci_hub_desc sc_hub_desc;
-	struct usb2_sw_transfer sc_root_ctrl;
-	struct usb2_sw_transfer sc_root_intr;
+	struct usb2_callout sc_root_intr;
 
 	struct usb2_device *sc_devices[UHCI_MAX_DEVICES];
-	struct uhci_td *sc_isoc_p_last[UHCI_VFRAMELIST_COUNT];	/* pointer to last TD
-								 * for isochronous */
-	struct uhci_qh *sc_intr_p_last[UHCI_IFRAMELIST_COUNT];	/* pointer to last QH
-								 * for interrupt */
-	struct uhci_qh *sc_ls_ctl_p_last;	/* pointer to last QH for low
-						 * speed control */
-	struct uhci_qh *sc_fs_ctl_p_last;	/* pointer to last QH for full
-						 * speed control */
-	struct uhci_qh *sc_bulk_p_last;	/* pointer to last QH for bulk */
+	/* pointer to last TD for isochronous */
+	struct uhci_td *sc_isoc_p_last[UHCI_VFRAMELIST_COUNT];
+	/* pointer to last QH for interrupt */
+	struct uhci_qh *sc_intr_p_last[UHCI_IFRAMELIST_COUNT];
+	/* pointer to last QH for low speed control */
+	struct uhci_qh *sc_ls_ctl_p_last;
+	/* pointer to last QH for full speed control */
+	struct uhci_qh *sc_fs_ctl_p_last;
+	/* pointer to last QH for bulk */
+	struct uhci_qh *sc_bulk_p_last;
 	struct uhci_qh *sc_reclaim_qh_p;
 	struct uhci_qh *sc_last_qh_p;
 	struct uhci_td *sc_last_td_p;
diff --git a/sys/dev/usb/controller/uhci_pci.c b/sys/dev/usb/controller/uhci_pci.c
index a61693b..061a178 100644
--- a/sys/dev/usb/controller/uhci_pci.c
+++ b/sys/dev/usb/controller/uhci_pci.c
@@ -49,13 +49,11 @@ __FBSDID("$FreeBSD$");
  */
 
 #include <dev/usb/usb_mfunc.h>
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb.h>
 
 #include <dev/usb/usb_core.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_util.h>
 #include <dev/usb/usb_debug.h>
 
@@ -324,10 +322,10 @@ uhci_pci_attach(device_t self)
 
 #if (__FreeBSD_version >= 700031)
 	err = bus_setup_intr(self, sc->sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    NULL, (void *)(void *)uhci_interrupt, sc, &sc->sc_intr_hdl);
+	    NULL, (driver_intr_t *)uhci_interrupt, sc, &sc->sc_intr_hdl);
 #else
 	err = bus_setup_intr(self, sc->sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    (void *)(void *)uhci_interrupt, sc, &sc->sc_intr_hdl);
+	    (driver_intr_t *)uhci_interrupt, sc, &sc->sc_intr_hdl);
 #endif
 
 	if (err) {
diff --git a/sys/dev/usb/controller/usb_controller.c b/sys/dev/usb/controller/usb_controller.c
index f835ec1..474cbe5 100644
--- a/sys/dev/usb/controller/usb_controller.c
+++ b/sys/dev/usb/controller/usb_controller.c
@@ -25,7 +25,6 @@
  */
 
 #include <dev/usb/usb_mfunc.h>
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb_error.h>
 #include <dev/usb/usb.h>
 
@@ -42,7 +41,7 @@
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
 
-/* function prototypes */
+/* function prototypes  */
 
 static device_probe_t usb2_probe;
 static device_attach_t usb2_attach;
@@ -50,16 +49,6 @@ static device_detach_t usb2_detach;
 
 static void	usb2_attach_sub(device_t, struct usb2_bus *);
 static void	usb2_post_init(void *);
-static void	usb2_bus_mem_flush_all_cb(struct usb2_bus *,
-		    struct usb2_page_cache *, struct usb2_page *, uint32_t,
-		    uint32_t);
-static void	usb2_bus_mem_alloc_all_cb(struct usb2_bus *,
-		    struct usb2_page_cache *, struct usb2_page *, uint32_t,
-		    uint32_t);
-static void	usb2_bus_mem_free_all_cb(struct usb2_bus *,
-		    struct usb2_page_cache *, struct usb2_page *, uint32_t,
-		    uint32_t);
-static void	usb2_bus_roothub(struct usb2_proc_msg *pm);
 
 /* static variables */
 
@@ -176,10 +165,6 @@ usb2_detach(device_t dev)
 	usb2_proc_free(&bus->giant_callback_proc);
 	usb2_proc_free(&bus->non_giant_callback_proc);
 
-	/* Get rid of USB roothub process */
-
-	usb2_proc_free(&bus->roothub_proc);
-
 	/* Get rid of USB explore process */
 
 	usb2_proc_free(&bus->explore_proc);
@@ -219,7 +204,6 @@ usb2_bus_explore(struct usb2_proc_msg *pm)
 		 * First update the USB power state!
 		 */
 		usb2_bus_powerd(bus);
-
 		/*
 		 * Explore the Root USB HUB. This call can sleep,
 		 * exiting Giant, which is actually Giant.
@@ -264,8 +248,8 @@ usb2_bus_detach(struct usb2_proc_msg *pm)
 	 * Free USB Root device, but not any sub-devices, hence they
 	 * are freed by the caller of this function:
 	 */
-	usb2_detach_device(udev, USB_IFACE_INDEX_ANY, 0);
-	usb2_free_device(udev);
+	usb2_free_device(udev,
+	    USB_UNCFG_FLAG_FREE_EP0);
 
 	mtx_unlock(&Giant);
 	USB_BUS_LOCK(bus);
@@ -287,7 +271,7 @@ usb2_power_wdog(void *arg)
 
 	usb2_bus_power_update(bus);
 
-	return;
+	USB_BUS_LOCK(bus);
 }
 
 /*------------------------------------------------------------------------*
@@ -338,6 +322,20 @@ usb2_bus_attach(struct usb2_proc_msg *pm)
 	USB_BUS_UNLOCK(bus);
 	mtx_lock(&Giant);		/* XXX not required by USB */
 
+	/* default power_mask value */
+	bus->hw_power_state =
+	  USB_HW_POWER_CONTROL |
+	  USB_HW_POWER_BULK |
+	  USB_HW_POWER_INTERRUPT |
+	  USB_HW_POWER_ISOC |
+	  USB_HW_POWER_NON_ROOT_HUB;
+
+	/* make sure power is set at least once */
+
+	if (bus->methods->set_hw_power != NULL) {
+		(bus->methods->set_hw_power) (bus);
+	}
+
 	/* Allocate the Root USB device */
 
 	child = usb2_alloc_device(bus->bdev, bus, NULL, 0, 0, 1,
@@ -366,11 +364,8 @@ usb2_bus_attach(struct usb2_proc_msg *pm)
 	/* set softc - we are ready */
 	device_set_softc(dev, bus);
 
-	/* start watchdog - this function will unlock the BUS lock ! */
+	/* start watchdog */
 	usb2_power_wdog(bus);
-
-	/* need to return locked */
-	USB_BUS_LOCK(bus);
 }
 
 /*------------------------------------------------------------------------*
@@ -403,12 +398,7 @@ usb2_attach_sub(device_t dev, struct usb2_bus *bus)
 	bus->attach_msg[1].hdr.pm_callback = &usb2_bus_attach;
 	bus->attach_msg[1].bus = bus;
 
-	bus->roothub_msg[0].hdr.pm_callback = &usb2_bus_roothub;
-	bus->roothub_msg[0].bus = bus;
-	bus->roothub_msg[1].hdr.pm_callback = &usb2_bus_roothub;
-	bus->roothub_msg[1].bus = bus;
-
-	/* Create USB explore, roothub and callback processes */
+	/* Create USB explore and callback processes */
 
 	if (usb2_proc_create(&bus->giant_callback_proc,
 	    &bus->bus_mtx, pname, USB_PRI_MED)) {
@@ -418,10 +408,6 @@ usb2_attach_sub(device_t dev, struct usb2_bus *bus)
 	    &bus->bus_mtx, pname, USB_PRI_HIGH)) {
 		printf("WARNING: Creation of USB non-Giant "
 		    "callback process failed.\n");
-	} else if (usb2_proc_create(&bus->roothub_proc,
-	    &bus->bus_mtx, pname, USB_PRI_HIGH)) {
-		printf("WARNING: Creation of USB roothub "
-		    "process failed.\n");
 	} else if (usb2_proc_create(&bus->explore_proc,
 	    &bus->bus_mtx, pname, USB_PRI_MED)) {
 		printf("WARNING: Creation of USB explore "
@@ -488,16 +474,19 @@ SYSUNINIT(usb2_bus_unload, SI_SUB_KLD, SI_ORDER_ANY, usb2_bus_unload, NULL);
 /*------------------------------------------------------------------------*
  *	usb2_bus_mem_flush_all_cb
  *------------------------------------------------------------------------*/
+#if USB_HAVE_BUSDMA
 static void
 usb2_bus_mem_flush_all_cb(struct usb2_bus *bus, struct usb2_page_cache *pc,
     struct usb2_page *pg, uint32_t size, uint32_t align)
 {
 	usb2_pc_cpu_flush(pc);
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *	usb2_bus_mem_flush_all - factored out code
  *------------------------------------------------------------------------*/
+#if USB_HAVE_BUSDMA
 void
 usb2_bus_mem_flush_all(struct usb2_bus *bus, usb2_bus_mem_cb_t *cb)
 {
@@ -505,10 +494,12 @@ usb2_bus_mem_flush_all(struct usb2_bus *bus, usb2_bus_mem_cb_t *cb)
 		cb(bus, &usb2_bus_mem_flush_all_cb);
 	}
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *	usb2_bus_mem_alloc_all_cb
  *------------------------------------------------------------------------*/
+#if USB_HAVE_BUSDMA
 static void
 usb2_bus_mem_alloc_all_cb(struct usb2_bus *bus, struct usb2_page_cache *pc,
     struct usb2_page *pg, uint32_t size, uint32_t align)
@@ -520,6 +511,7 @@ usb2_bus_mem_alloc_all_cb(struct usb2_bus *bus, struct usb2_page_cache *pc,
 		bus->alloc_failed = 1;
 	}
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *	usb2_bus_mem_alloc_all - factored out code
@@ -538,13 +530,14 @@ usb2_bus_mem_alloc_all(struct usb2_bus *bus, bus_dma_tag_t dmat,
 	    NULL, MTX_DEF | MTX_RECURSE);
 
 	usb2_callout_init_mtx(&bus->power_wdog,
-	    &bus->bus_mtx, CALLOUT_RETURNUNLOCKED);
+	    &bus->bus_mtx, 0);
 
 	TAILQ_INIT(&bus->intr_q.head);
 
+#if USB_HAVE_BUSDMA
 	usb2_dma_tag_setup(bus->dma_parent_tag, bus->dma_tags,
-	    dmat, &bus->bus_mtx, NULL, NULL, 32, USB_BUS_DMA_TAG_MAX);
-
+	    dmat, &bus->bus_mtx, NULL, 32, USB_BUS_DMA_TAG_MAX);
+#endif
 	if ((bus->devices_max > USB_MAX_DEVICES) ||
 	    (bus->devices_max < USB_MIN_DEVICES) ||
 	    (bus->devices == NULL)) {
@@ -552,9 +545,11 @@ usb2_bus_mem_alloc_all(struct usb2_bus *bus, bus_dma_tag_t dmat,
 		    "initialised properly!\n");
 		bus->alloc_failed = 1;		/* failure */
 	}
+#if USB_HAVE_BUSDMA
 	if (cb) {
 		cb(bus, &usb2_bus_mem_alloc_all_cb);
 	}
+#endif
 	if (bus->alloc_failed) {
 		usb2_bus_mem_free_all(bus, cb);
 	}
@@ -564,12 +559,14 @@ usb2_bus_mem_alloc_all(struct usb2_bus *bus, bus_dma_tag_t dmat,
 /*------------------------------------------------------------------------*
  *	usb2_bus_mem_free_all_cb
  *------------------------------------------------------------------------*/
+#if USB_HAVE_BUSDMA
 static void
 usb2_bus_mem_free_all_cb(struct usb2_bus *bus, struct usb2_page_cache *pc,
     struct usb2_page *pg, uint32_t size, uint32_t align)
 {
 	usb2_pc_free_mem(pc);
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *	usb2_bus_mem_free_all - factored out code
@@ -577,45 +574,12 @@ usb2_bus_mem_free_all_cb(struct usb2_bus *bus, struct usb2_page_cache *pc,
 void
 usb2_bus_mem_free_all(struct usb2_bus *bus, usb2_bus_mem_cb_t *cb)
 {
+#if USB_HAVE_BUSDMA
 	if (cb) {
 		cb(bus, &usb2_bus_mem_free_all_cb);
 	}
 	usb2_dma_tag_unsetup(bus->dma_parent_tag);
+#endif
 
 	mtx_destroy(&bus->bus_mtx);
 }
-
-/*------------------------------------------------------------------------*
- *	usb2_bus_roothub
- *
- * This function is used to execute roothub control requests on the
- * roothub and is called from the roothub process.
- *------------------------------------------------------------------------*/
-static void
-usb2_bus_roothub(struct usb2_proc_msg *pm)
-{
-	struct usb2_bus *bus;
-
-	bus = ((struct usb2_bus_msg *)pm)->bus;
-
-	USB_BUS_LOCK_ASSERT(bus, MA_OWNED);
-
-	(bus->methods->roothub_exec) (bus);
-}
-
-/*------------------------------------------------------------------------*
- *	usb2_bus_roothub_exec
- *
- * This function is used to schedule the "roothub_done" bus callback
- * method. The bus lock must be locked when calling this function.
- *------------------------------------------------------------------------*/
-void
-usb2_bus_roothub_exec(struct usb2_bus *bus)
-{
-	USB_BUS_LOCK_ASSERT(bus, MA_OWNED);
-
-	if (usb2_proc_msignal(&bus->roothub_proc,
-	    &bus->roothub_msg[0], &bus->roothub_msg[1])) {
-		/* ignore */
-	}
-}
diff --git a/sys/dev/usb/controller/uss820dci.c b/sys/dev/usb/controller/uss820dci.c
index 5fc8e98..9582869 100644
--- a/sys/dev/usb/controller/uss820dci.c
+++ b/sys/dev/usb/controller/uss820dci.c
@@ -29,14 +29,13 @@
  * This file contains the driver for the USS820 series USB Device
  * Controller
  *
- * NOTE: The datasheet does not document everything!
+ * NOTE: The datasheet does not document everything.
  */
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_revision.h>
 #include <dev/usb/usb_error.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR uss820dcidebug
 
@@ -44,7 +43,6 @@
 #include <dev/usb/usb_debug.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_transfer.h>
 #include <dev/usb/usb_device.h>
 #include <dev/usb/usb_hub.h>
@@ -56,10 +54,10 @@
 
 #define	USS820_DCI_BUS2SC(bus) \
    ((struct uss820dci_softc *)(((uint8_t *)(bus)) - \
-   USB_P2U(&(((struct uss820dci_softc *)0)->sc_bus))))
+    ((uint8_t *)&(((struct uss820dci_softc *)0)->sc_bus))))
 
 #define	USS820_DCI_PC2SC(pc) \
-   USS820_DCI_BUS2SC((pc)->tag_parent->info->bus)
+   USS820_DCI_BUS2SC(USB_DMATAG_TO_XROOT((pc)->tag_parent)->bus)
 
 #if USB_DEBUG
 static int uss820dcidebug = 0;
@@ -78,8 +76,6 @@ struct usb2_pipe_methods uss820dci_device_bulk_methods;
 struct usb2_pipe_methods uss820dci_device_ctrl_methods;
 struct usb2_pipe_methods uss820dci_device_intr_methods;
 struct usb2_pipe_methods uss820dci_device_isoc_fs_methods;
-struct usb2_pipe_methods uss820dci_root_ctrl_methods;
-struct usb2_pipe_methods uss820dci_root_intr_methods;
 
 static uss820dci_cmd_t uss820dci_setup_rx;
 static uss820dci_cmd_t uss820dci_data_rx;
@@ -87,14 +83,11 @@ static uss820dci_cmd_t uss820dci_data_tx;
 static uss820dci_cmd_t uss820dci_data_tx_sync;
 static void	uss820dci_device_done(struct usb2_xfer *, usb2_error_t);
 static void	uss820dci_do_poll(struct usb2_bus *);
-static void	uss820dci_root_ctrl_poll(struct uss820dci_softc *);
 static void	uss820dci_standard_done(struct usb2_xfer *);
 static void	uss820dci_intr_set(struct usb2_xfer *, uint8_t);
 static void	uss820dci_update_shared_1(struct uss820dci_softc *, uint8_t,
 		    uint8_t, uint8_t);
-
-static usb2_sw_transfer_func_t uss820dci_root_intr_done;
-static usb2_sw_transfer_func_t uss820dci_root_ctrl_done;
+static void	uss820dci_root_intr(struct uss820dci_softc *);
 
 /*
  * Here is a list of what the USS820D chip can support. The main
@@ -249,19 +242,11 @@ uss820dci_setup_rx(struct uss820dci_td *td)
 	DPRINTFN(5, "rx_stat=0x%02x rem=%u\n", rx_stat, td->remainder);
 
 	if (!(rx_stat & USS820_RXSTAT_RXSETUP)) {
-		/* abort any ongoing transfer */
-		if (!td->did_stall) {
-			DPRINTFN(5, "stalling\n");
-
-			/* set stall */
-
-			uss820dci_update_shared_1(sc, USS820_EPCON, 0xFF,
-			    (USS820_EPCON_TXSTL | USS820_EPCON_RXSTL));
-
-			td->did_stall = 1;
-		}
 		goto not_complete;
 	}
+	/* clear did stall */
+	td->did_stall = 0;
+
 	/* clear stall and all I/O */
 	uss820dci_update_shared_1(sc, USS820_EPCON,
 	    0xFF ^ (USS820_EPCON_TXSTL |
@@ -284,12 +269,12 @@ uss820dci_setup_rx(struct uss820dci_td *td)
 	if (count != td->remainder) {
 		DPRINTFN(0, "Invalid SETUP packet "
 		    "length, %d bytes\n", count);
-		goto not_complete;
+		goto setup_not_complete;
 	}
 	if (count != sizeof(req)) {
 		DPRINTFN(0, "Unsupported SETUP packet "
 		    "length, %d bytes\n", count);
-		goto not_complete;
+		goto setup_not_complete;
 	}
 	/* receive data */
 	bus_space_read_multi_1(td->io_tag, td->io_hdl,
@@ -332,7 +317,28 @@ uss820dci_setup_rx(struct uss820dci_td *td)
 	}
 	return (0);			/* complete */
 
+setup_not_complete:
+
+	/* set RXFFRC bit */
+	temp = bus_space_read_1(td->io_tag, td->io_hdl,
+	    td->rx_cntl_reg);
+	temp |= USS820_RXCON_RXFFRC;
+	bus_space_write_1(td->io_tag, td->io_hdl,
+	    td->rx_cntl_reg, temp);
+
+	/* FALLTHROUGH */
+
 not_complete:
+	/* abort any ongoing transfer */
+	if (!td->did_stall) {
+		DPRINTFN(5, "stalling\n");
+		/* set stall */
+		uss820dci_update_shared_1(sc, USS820_EPCON, 0xFF,
+		    (USS820_EPCON_TXSTL | USS820_EPCON_RXSTL));
+
+		td->did_stall = 1;
+	}
+
 	/* clear end overwrite flag, if any */
 	if (rx_stat & USS820_RXSTAT_RXSETUP) {
 		uss820dci_update_shared_1(sc, USS820_RXSTAT,
@@ -774,9 +780,7 @@ uss820dci_interrupt(struct uss820dci_softc *sc)
 			DPRINTF("real bus interrupt 0x%02x\n", ssr);
 
 			/* complete root HUB interrupt endpoint */
-
-			usb2_sw_transfer(&sc->sc_root_intr,
-			    &uss820dci_root_intr_done);
+			uss820dci_root_intr(sc);
 		}
 	}
 	/* acknowledge all SBI interrupts */
@@ -837,8 +841,8 @@ uss820dci_setup_standard_chain(struct usb2_xfer *xfer)
 
 	temp.td = NULL;
 	temp.td_next = xfer->td_start[0];
-	temp.setup_alt_next = xfer->flags_int.short_frames_ok;
 	temp.offset = 0;
+	temp.setup_alt_next = xfer->flags_int.short_frames_ok;
 
 	sc = USS820_DCI_BUS2SC(xfer->xroot->bus);
 	ep_no = (xfer->endpoint & UE_ADDR);
@@ -852,6 +856,12 @@ uss820dci_setup_standard_chain(struct usb2_xfer *xfer)
 			temp.len = xfer->frlengths[0];
 			temp.pc = xfer->frbuffers + 0;
 			temp.short_pkt = temp.len ? 1 : 0;
+			/* check for last frame */
+			if (xfer->nframes == 1) {
+				/* no STATUS stage yet, SETUP is last */
+				if (xfer->flags_int.control_act)
+					temp.setup_alt_next = 0;
+			}
 
 			uss820dci_setup_standard_chain_sub(&temp);
 		}
@@ -879,7 +889,13 @@ uss820dci_setup_standard_chain(struct usb2_xfer *xfer)
 		x++;
 
 		if (x == xfer->nframes) {
-			temp.setup_alt_next = 0;
+			if (xfer->flags_int.control_xfr) {
+				if (xfer->flags_int.control_act) {
+					temp.setup_alt_next = 0;
+				}
+			} else {
+				temp.setup_alt_next = 0;
+			}
 		}
 		if (temp.len == 0) {
 
@@ -904,37 +920,39 @@ uss820dci_setup_standard_chain(struct usb2_xfer *xfer)
 		}
 	}
 
-	/* always setup a valid "pc" pointer for status and sync */
-	temp.pc = xfer->frbuffers + 0;
-
-	/* check if we should append a status stage */
-
-	if (xfer->flags_int.control_xfr &&
-	    !xfer->flags_int.control_act) {
+	/* check for control transfer */
+	if (xfer->flags_int.control_xfr) {
 		uint8_t need_sync;
 
-		/*
-		 * Send a DATA1 message and invert the current
-		 * endpoint direction.
-		 */
-		if (xfer->endpoint & UE_DIR_IN) {
-			temp.func = &uss820dci_data_rx;
-			need_sync = 0;
-		} else {
-			temp.func = &uss820dci_data_tx;
-			need_sync = 1;
-		}
+		/* always setup a valid "pc" pointer for status and sync */
+		temp.pc = xfer->frbuffers + 0;
 		temp.len = 0;
 		temp.short_pkt = 0;
+		temp.setup_alt_next = 0;
 
-		uss820dci_setup_standard_chain_sub(&temp);
-		if (need_sync) {
-			/* we need a SYNC point after TX */
-			temp.func = &uss820dci_data_tx_sync;
+		/* check if we should append a status stage */
+		if (!xfer->flags_int.control_act) {
+
+			/*
+			 * Send a DATA1 message and invert the current
+			 * endpoint direction.
+			 */
+			if (xfer->endpoint & UE_DIR_IN) {
+				temp.func = &uss820dci_data_rx;
+				need_sync = 0;
+			} else {
+				temp.func = &uss820dci_data_tx;
+				need_sync = 1;
+			}
 			temp.len = 0;
 			temp.short_pkt = 0;
 
 			uss820dci_setup_standard_chain_sub(&temp);
+			if (need_sync) {
+				/* we need a SYNC point after TX */
+				temp.func = &uss820dci_data_tx_sync;
+				uss820dci_setup_standard_chain_sub(&temp);
+			}
 		}
 	}
 	/* must have at least one frame! */
@@ -1021,31 +1039,17 @@ uss820dci_start_standard_chain(struct usb2_xfer *xfer)
 }
 
 static void
-uss820dci_root_intr_done(struct usb2_xfer *xfer,
-    struct usb2_sw_transfer *std)
+uss820dci_root_intr(struct uss820dci_softc *sc)
 {
-	struct uss820dci_softc *sc = USS820_DCI_BUS2SC(xfer->xroot->bus);
-
 	DPRINTFN(9, "\n");
 
 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 
-	if (std->state != USB_SW_TR_PRE_DATA) {
-		if (std->state == USB_SW_TR_PRE_CALLBACK) {
-			/* transfer transferred */
-			uss820dci_device_done(xfer, std->err);
-		}
-		goto done;
-	}
-	/* setup buffer */
-	std->ptr = sc->sc_hub_idata;
-	std->len = sizeof(sc->sc_hub_idata);
-
 	/* set port bit */
 	sc->sc_hub_idata[0] = 0x02;	/* we only have one port */
 
-done:
-	return;
+	uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
+	    sizeof(sc->sc_hub_idata));
 }
 
 static usb2_error_t
@@ -1499,7 +1503,6 @@ uss820dci_do_poll(struct usb2_bus *bus)
 
 	USB_BUS_LOCK(&sc->sc_bus);
 	uss820dci_interrupt_poll(sc);
-	uss820dci_root_ctrl_poll(sc);
 	USB_BUS_UNLOCK(&sc->sc_bus);
 }
 
@@ -1538,8 +1541,6 @@ struct usb2_pipe_methods uss820dci_device_bulk_methods =
 	.close = uss820dci_device_bulk_close,
 	.enter = uss820dci_device_bulk_enter,
 	.start = uss820dci_device_bulk_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -1577,8 +1578,6 @@ struct usb2_pipe_methods uss820dci_device_ctrl_methods =
 	.close = uss820dci_device_ctrl_close,
 	.enter = uss820dci_device_ctrl_enter,
 	.start = uss820dci_device_ctrl_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -1616,8 +1615,6 @@ struct usb2_pipe_methods uss820dci_device_intr_methods =
 	.close = uss820dci_device_intr_close,
 	.enter = uss820dci_device_intr_enter,
 	.start = uss820dci_device_intr_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
@@ -1700,38 +1697,14 @@ struct usb2_pipe_methods uss820dci_device_isoc_fs_methods =
 	.close = uss820dci_device_isoc_fs_close,
 	.enter = uss820dci_device_isoc_fs_enter,
 	.start = uss820dci_device_isoc_fs_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
 };
 
 /*------------------------------------------------------------------------*
  * at91dci root control support
  *------------------------------------------------------------------------*
- * simulate a hardware HUB by handling
- * all the necessary requests
+ * Simulate a hardware HUB by handling all the necessary requests.
  *------------------------------------------------------------------------*/
 
-static void
-uss820dci_root_ctrl_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-uss820dci_root_ctrl_close(struct usb2_xfer *xfer)
-{
-	struct uss820dci_softc *sc = USS820_DCI_BUS2SC(xfer->xroot->bus);
-
-	if (sc->sc_root_ctrl.xfer == xfer) {
-		sc->sc_root_ctrl.xfer = NULL;
-	}
-	uss820dci_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-/*
- * USB descriptors for the virtual Root HUB:
- */
-
 static const struct usb2_device_descriptor uss820dci_devd = {
 	.bLength = sizeof(struct usb2_device_descriptor),
 	.bDescriptorType = UDESC_DEVICE,
@@ -1816,44 +1789,15 @@ USB_MAKE_STRING_DESC(STRING_VENDOR, uss820dci_vendor);
 USB_MAKE_STRING_DESC(STRING_PRODUCT, uss820dci_product);
 
 static void
-uss820dci_root_ctrl_enter(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-uss820dci_root_ctrl_start(struct usb2_xfer *xfer)
-{
-	struct uss820dci_softc *sc = USS820_DCI_BUS2SC(xfer->xroot->bus);
-
-	sc->sc_root_ctrl.xfer = xfer;
-
-	usb2_bus_roothub_exec(xfer->xroot->bus);
-}
-
-static void
-uss820dci_root_ctrl_task(struct usb2_bus *bus)
-{
-	uss820dci_root_ctrl_poll(USS820_DCI_BUS2SC(bus));
-}
-
-static void
-uss820dci_root_ctrl_done(struct usb2_xfer *xfer,
-    struct usb2_sw_transfer *std)
+uss820dci_roothub_exec(struct usb2_bus *bus)
 {
-	struct uss820dci_softc *sc = USS820_DCI_BUS2SC(xfer->xroot->bus);
+	struct uss820dci_softc *sc = USS820_DCI_BUS2SC(bus);
+	struct usb2_sw_transfer *std = &sc->sc_bus.roothub_req;
 	uint16_t value;
 	uint16_t index;
 
 	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
 
-	if (std->state != USB_SW_TR_SETUP) {
-		if (std->state == USB_SW_TR_PRE_CALLBACK) {
-			/* transfer transferred */
-			uss820dci_device_done(xfer, std->err);
-		}
-		goto done;
-	}
 	/* buffer reset */
 	std->ptr = USB_ADD_BYTES(&sc->sc_hub_temp, 0);
 	std->len = 0;
@@ -2227,67 +2171,6 @@ done:
 }
 
 static void
-uss820dci_root_ctrl_poll(struct uss820dci_softc *sc)
-{
-	usb2_sw_transfer(&sc->sc_root_ctrl,
-	    &uss820dci_root_ctrl_done);
-}
-
-struct usb2_pipe_methods uss820dci_root_ctrl_methods =
-{
-	.open = uss820dci_root_ctrl_open,
-	.close = uss820dci_root_ctrl_close,
-	.enter = uss820dci_root_ctrl_enter,
-	.start = uss820dci_root_ctrl_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 0,
-};
-
-/*------------------------------------------------------------------------*
- * at91dci root interrupt support
- *------------------------------------------------------------------------*/
-static void
-uss820dci_root_intr_open(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-uss820dci_root_intr_close(struct usb2_xfer *xfer)
-{
-	struct uss820dci_softc *sc = USS820_DCI_BUS2SC(xfer->xroot->bus);
-
-	if (sc->sc_root_intr.xfer == xfer) {
-		sc->sc_root_intr.xfer = NULL;
-	}
-	uss820dci_device_done(xfer, USB_ERR_CANCELLED);
-}
-
-static void
-uss820dci_root_intr_enter(struct usb2_xfer *xfer)
-{
-	return;
-}
-
-static void
-uss820dci_root_intr_start(struct usb2_xfer *xfer)
-{
-	struct uss820dci_softc *sc = USS820_DCI_BUS2SC(xfer->xroot->bus);
-
-	sc->sc_root_intr.xfer = xfer;
-}
-
-struct usb2_pipe_methods uss820dci_root_intr_methods =
-{
-	.open = uss820dci_root_intr_open,
-	.close = uss820dci_root_intr_close,
-	.enter = uss820dci_root_intr_enter,
-	.start = uss820dci_root_intr_start,
-	.enter_is_cancelable = 1,
-	.start_is_cancelable = 1,
-};
-
-static void
 uss820dci_xfer_setup(struct usb2_setup_params *parm)
 {
 	const struct usb2_hw_ep_profile *pf;
@@ -2426,24 +2309,7 @@ uss820dci_pipe_init(struct usb2_device *udev, struct usb2_endpoint_descriptor *e
 	    edesc->bEndpointAddress, udev->flags.usb2_mode,
 	    sc->sc_rt_addr);
 
-	if (udev->device_index == sc->sc_rt_addr) {
-
-		if (udev->flags.usb2_mode != USB_MODE_HOST) {
-			/* not supported */
-			return;
-		}
-		switch (edesc->bEndpointAddress) {
-		case USB_CONTROL_ENDPOINT:
-			pipe->methods = &uss820dci_root_ctrl_methods;
-			break;
-		case UE_DIR_IN | USS820_DCI_INTR_ENDPT:
-			pipe->methods = &uss820dci_root_intr_methods;
-			break;
-		default:
-			/* do nothing */
-			break;
-		}
-	} else {
+	if (udev->device_index != sc->sc_rt_addr) {
 
 		if (udev->flags.usb2_mode != USB_MODE_DEVICE) {
 			/* not supported */
@@ -2481,5 +2347,5 @@ struct usb2_bus_methods uss820dci_bus_methods =
 	.get_hw_ep_profile = &uss820dci_get_hw_ep_profile,
 	.set_stall = &uss820dci_set_stall,
 	.clear_stall = &uss820dci_clear_stall,
-	.roothub_exec = &uss820dci_root_ctrl_task,
+	.roothub_exec = &uss820dci_roothub_exec,
 };
diff --git a/sys/dev/usb/controller/uss820dci.h b/sys/dev/usb/controller/uss820dci.h
index fcddb4a..637f562 100644
--- a/sys/dev/usb/controller/uss820dci.h
+++ b/sys/dev/usb/controller/uss820dci.h
@@ -345,8 +345,6 @@ struct uss820dci_softc {
 	struct usb2_bus sc_bus;
 	union uss820_hub_temp sc_hub_temp;
 	LIST_HEAD(, usb2_xfer) sc_interrupt_list_head;
-	struct usb2_sw_transfer sc_root_ctrl;
-	struct usb2_sw_transfer sc_root_intr;
 
 	struct usb2_device *sc_devices[USS820_MAX_DEVICES];
 	struct resource *sc_io_res;
diff --git a/sys/dev/usb/controller/uss820dci_atmelarm.c b/sys/dev/usb/controller/uss820dci_atmelarm.c
index 5691d4b..c01c2c5 100644
--- a/sys/dev/usb/controller/uss820dci_atmelarm.c
+++ b/sys/dev/usb/controller/uss820dci_atmelarm.c
@@ -28,13 +28,11 @@ __FBSDID("$FreeBSD$");
  */
 
 #include <dev/usb/usb_mfunc.h>
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb.h>
 
 #include <dev/usb/usb_core.h>
 #include <dev/usb/usb_busdma.h>
 #include <dev/usb/usb_process.h>
-#include <dev/usb/usb_sw_transfer.h>
 #include <dev/usb/usb_util.h>
 
 #include <dev/usb/usb_controller.h>
@@ -171,10 +169,10 @@ uss820_atmelarm_attach(device_t dev)
 
 #if (__FreeBSD_version >= 700031)
 	err = bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    NULL, (void *)uss820dci_interrupt, sc, &sc->sc_intr_hdl);
+	    NULL, (driver_intr_t *)uss820dci_interrupt, sc, &sc->sc_intr_hdl);
 #else
 	err = bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
-	    (void *)uss820dci_interrupt, sc, &sc->sc_intr_hdl);
+	    (driver_intr_t *)uss820dci_interrupt, sc, &sc->sc_intr_hdl);
 #endif
 	if (err) {
 		sc->sc_intr_hdl = NULL;
diff --git a/sys/dev/usb/image/uscanner.c b/sys/dev/usb/image/uscanner.c
deleted file mode 100644
index 916295e..0000000
--- a/sys/dev/usb/image/uscanner.c
+++ /dev/null
@@ -1,641 +0,0 @@
-/*	$NetBSD: uscanner.c,v 1.30 2002/07/11 21:14:36 augustss Exp$	*/
-
-/* Also already merged from NetBSD:
- *	$NetBSD: uscanner.c,v 1.33 2002/09/23 05:51:24 simonb Exp $
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-/*-
- * Copyright (c) 2000 The NetBSD Foundation, Inc.
- * All rights reserved.
- *
- * This code is derived from software contributed to The NetBSD Foundation
- * by Lennart Augustsson (lennart@augustsson.net) at
- * Carlstedt Research & Technology
- * and Nick Hibma (n_hibma@qubesoft.com).
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *        This product includes software developed by the NetBSD
- *        Foundation, Inc. and its contributors.
- * 4. Neither the name of The NetBSD Foundation nor the names of its
- *    contributors may be used to endorse or promote products derived
- *    from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
- * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
- * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "usbdevs.h"
-#include <dev/usb/usb.h>
-#include <dev/usb/usb_mfunc.h>
-#include <dev/usb/usb_error.h>
-
-#define	USB_DEBUG_VAR uscanner_debug
-
-#include <dev/usb/usb_core.h>
-#include <dev/usb/usb_debug.h>
-#include <dev/usb/usb_process.h>
-#include <dev/usb/usb_request.h>
-#include <dev/usb/usb_lookup.h>
-#include <dev/usb/usb_util.h>
-#include <dev/usb/usb_busdma.h>
-#include <dev/usb/usb_mbuf.h>
-#include <dev/usb/usb_dev.h>
-
-#if USB_DEBUG
-static int uscanner_debug = 0;
-
-SYSCTL_NODE(_hw_usb2, OID_AUTO, uscanner, CTLFLAG_RW, 0, "USB uscanner");
-SYSCTL_INT(_hw_usb2_uscanner, OID_AUTO, debug, CTLFLAG_RW, &uscanner_debug,
-    0, "uscanner debug level");
-#endif
-
-/*
- * uscanner transfers macros definition.
- */
-#define	USCANNER_BSIZE			(1 << 15)
-#define	USCANNER_IFQ_MAXLEN		2
-
-/*
- * Transfers stallings handling flags definition.
- */
-#define	USCANNER_FLAG_READ_STALL	0x01
-#define	USCANNER_FLAG_WRITE_STALL	0x02
-
-/*
- * uscanner_info flags definition.
- */
-#define	USCANNER_FLAG_KEEP_OPEN		0x04
-
-enum {
-	USCANNER_BULK_DT_WR,
-	USCANNER_BULK_DT_RD,
-	USCANNER_BULK_CS_WR,
-	USCANNER_BULK_CS_RD,
-	USCANNER_N_TRANSFER = 4,
-};
-
-struct uscanner_softc {
-	struct usb2_fifo_sc sc_fifo;
-	struct mtx sc_mtx;
-
-	struct usb2_xfer *sc_xfer[USCANNER_N_TRANSFER];
-
-	uint8_t	sc_flags;		/* Used to prevent stalls */
-};
-
-/*
- * Prototypes for driver handling routines (sorted by use).
- */
-static device_probe_t uscanner_probe;
-static device_attach_t uscanner_attach;
-static device_detach_t uscanner_detach;
-
-/*
- * Prototypes for xfer transfer callbacks.
- */
-static usb2_callback_t uscanner_read_callback;
-static usb2_callback_t uscanner_read_clear_stall_callback;
-static usb2_callback_t uscanner_write_callback;
-static usb2_callback_t uscanner_write_clear_stall_callback;
-
-/*
- * Prototypes for the character device handling routines.
- */
-static usb2_fifo_close_t uscanner_close;
-static usb2_fifo_cmd_t uscanner_start_read;
-static usb2_fifo_cmd_t uscanner_start_write;
-static usb2_fifo_cmd_t uscanner_stop_read;
-static usb2_fifo_cmd_t uscanner_stop_write;
-static usb2_fifo_open_t uscanner_open;
-
-static struct usb2_fifo_methods uscanner_fifo_methods = {
-	.f_close = &uscanner_close,
-	.f_open = &uscanner_open,
-	.f_start_read = &uscanner_start_read,
-	.f_start_write = &uscanner_start_write,
-	.f_stop_read = &uscanner_stop_read,
-	.f_stop_write = &uscanner_stop_write,
-	.basename[0] = "uscanner",
-};
-
-/*
- * xfer transfers array.  Resolve-stalling callbacks are marked as control
- * transfers.
- */
-static const struct usb2_config uscanner_config[USCANNER_N_TRANSFER] = {
-	[USCANNER_BULK_DT_WR] = {
-		.type = UE_BULK,
-		.endpoint = UE_ADDR_ANY,
-		.direction = UE_DIR_OUT,
-		.mh.bufsize = USCANNER_BSIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,.proxy_buffer = 1,.force_short_xfer = 1,},
-		.mh.callback = &uscanner_write_callback,
-	},
-
-	[USCANNER_BULK_DT_RD] = {
-		.type = UE_BULK,
-		.endpoint = UE_ADDR_ANY,
-		.direction = UE_DIR_IN,
-		.mh.bufsize = USCANNER_BSIZE,
-		.mh.flags = {.pipe_bof = 1,.proxy_buffer = 1,.short_xfer_ok = 1,},
-		.mh.callback = &uscanner_read_callback,
-	},
-
-	[USCANNER_BULK_CS_WR] = {
-		.type = UE_CONTROL,
-		.endpoint = 0x00,	/* Control pipe */
-		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &uscanner_write_clear_stall_callback,
-		.mh.timeout = 1000,
-		.mh.interval = 50,	/* 50ms */
-	},
-
-	[USCANNER_BULK_CS_RD] = {
-		.type = UE_CONTROL,
-		.endpoint = 0x00,
-		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &uscanner_read_clear_stall_callback,
-		.mh.timeout = 1000,
-		.mh.interval = 50,	/* 50ms */
-	},
-};
-
-static devclass_t uscanner_devclass;
-
-static device_method_t uscanner_methods[] = {
-	DEVMETHOD(device_probe, uscanner_probe),
-	DEVMETHOD(device_attach, uscanner_attach),
-	DEVMETHOD(device_detach, uscanner_detach),
-	{0, 0}
-};
-
-static driver_t uscanner_driver = {
-	.name = "uscanner",
-	.methods = uscanner_methods,
-	.size = sizeof(struct uscanner_softc),
-};
-
-DRIVER_MODULE(uscanner, uhub, uscanner_driver, uscanner_devclass, NULL, 0);
-MODULE_DEPEND(uscanner, usb, 1, 1, 1);
-
-/*
- * USB scanners device IDs
- */
-static const struct usb2_device_id uscanner_devs[] = {
-	/* Acer */
-	{USB_VPI(USB_VENDOR_ACERP, USB_PRODUCT_ACERP_ACERSCAN_320U, 0)},
-	{USB_VPI(USB_VENDOR_ACERP, USB_PRODUCT_ACERP_ACERSCAN_640U, 0)},
-	{USB_VPI(USB_VENDOR_ACERP, USB_PRODUCT_ACERP_ACERSCAN_640BT, 0)},
-	{USB_VPI(USB_VENDOR_ACERP, USB_PRODUCT_ACERP_ACERSCAN_620U, 0)},
-	{USB_VPI(USB_VENDOR_ACERP, USB_PRODUCT_ACERP_ACERSCAN_1240U, 0)},
-	{USB_VPI(USB_VENDOR_ACERP, USB_PRODUCT_ACERP_ACERSCAN_C310U, 0)},
-	{USB_VPI(USB_VENDOR_ACERP, USB_PRODUCT_ACERP_ACERSCAN_4300U, 0)},
-	/* AGFA */
-	{USB_VPI(USB_VENDOR_AGFA, USB_PRODUCT_AGFA_SNAPSCAN1236U, 0)},
-	{USB_VPI(USB_VENDOR_AGFA, USB_PRODUCT_AGFA_SNAPSCAN1212U, 0)},
-	{USB_VPI(USB_VENDOR_AGFA, USB_PRODUCT_AGFA_SNAPSCAN1212U2, 0)},
-	{USB_VPI(USB_VENDOR_AGFA, USB_PRODUCT_AGFA_SNAPSCANTOUCH, 0)},
-	{USB_VPI(USB_VENDOR_AGFA, USB_PRODUCT_AGFA_SNAPSCANE40, 0)},
-	{USB_VPI(USB_VENDOR_AGFA, USB_PRODUCT_AGFA_SNAPSCANE50, 0)},
-	{USB_VPI(USB_VENDOR_AGFA, USB_PRODUCT_AGFA_SNAPSCANE20, 0)},
-	{USB_VPI(USB_VENDOR_AGFA, USB_PRODUCT_AGFA_SNAPSCANE25, 0)},
-	{USB_VPI(USB_VENDOR_AGFA, USB_PRODUCT_AGFA_SNAPSCANE26, 0)},
-	{USB_VPI(USB_VENDOR_AGFA, USB_PRODUCT_AGFA_SNAPSCANE52, 0)},
-	/* Avision */
-	{USB_VPI(USB_VENDOR_AVISION, USB_PRODUCT_AVISION_1200U, 0)},
-	/* Canon */
-	{USB_VPI(USB_VENDOR_CANON, USB_PRODUCT_CANON_N656U, 0)},
-	{USB_VPI(USB_VENDOR_CANON, USB_PRODUCT_CANON_N676U, 0)},
-	{USB_VPI(USB_VENDOR_CANON, USB_PRODUCT_CANON_N1220U, 0)},
-	{USB_VPI(USB_VENDOR_CANON, USB_PRODUCT_CANON_D660U, 0)},
-	{USB_VPI(USB_VENDOR_CANON, USB_PRODUCT_CANON_N1240U, 0)},
-	{USB_VPI(USB_VENDOR_CANON, USB_PRODUCT_CANON_LIDE25, 0)},
-	/* Epson */
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_636, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_610, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_1200, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_1240, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_1250, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_1270, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_1600, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_1640, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_640U, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_1650, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_1660, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_1670, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_1260, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_RX425, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_3200, USCANNER_FLAG_KEEP_OPEN)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_GT9700F, USCANNER_FLAG_KEEP_OPEN)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_CX5400, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_DX7400, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_GT9300UF, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_2480, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_3500, USCANNER_FLAG_KEEP_OPEN)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_3590, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_4200, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_4800, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_4990, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_5000, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_6000, 0)},
-	{USB_VPI(USB_VENDOR_EPSON, USB_PRODUCT_EPSON_DX8400, 0)},
-	/* HP */
-	{USB_VPI(USB_VENDOR_HP, USB_PRODUCT_HP_2200C, 0)},
-	{USB_VPI(USB_VENDOR_HP, USB_PRODUCT_HP_3300C, 0)},
-	{USB_VPI(USB_VENDOR_HP, USB_PRODUCT_HP_3400CSE, 0)},
-	{USB_VPI(USB_VENDOR_HP, USB_PRODUCT_HP_4100C, 0)},
-	{USB_VPI(USB_VENDOR_HP, USB_PRODUCT_HP_4200C, 0)},
-	{USB_VPI(USB_VENDOR_HP, USB_PRODUCT_HP_4300C, 0)},
-	{USB_VPI(USB_VENDOR_HP, USB_PRODUCT_HP_4470C, 0)},
-	{USB_VPI(USB_VENDOR_HP, USB_PRODUCT_HP_4670V, 0)},
-	{USB_VPI(USB_VENDOR_HP, USB_PRODUCT_HP_S20, 0)},
-	{USB_VPI(USB_VENDOR_HP, USB_PRODUCT_HP_5200C, 0)},
-	{USB_VPI(USB_VENDOR_HP, USB_PRODUCT_HP_5300C, 0)},
-	{USB_VPI(USB_VENDOR_HP, USB_PRODUCT_HP_5400C, 0)},
-	{USB_VPI(USB_VENDOR_HP, USB_PRODUCT_HP_6200C, 0)},
-	{USB_VPI(USB_VENDOR_HP, USB_PRODUCT_HP_6300C, 0)},
-	{USB_VPI(USB_VENDOR_HP, USB_PRODUCT_HP_82x0C, 0)},
-	/* Kye */
-	{USB_VPI(USB_VENDOR_KYE, USB_PRODUCT_KYE_VIVIDPRO, 0)},
-	/* Microtek */
-	{USB_VPI(USB_VENDOR_MICROTEK, USB_PRODUCT_MICROTEK_X6U, 0)},
-	{USB_VPI(USB_VENDOR_MICROTEK, USB_PRODUCT_MICROTEK_336CX, 0)},
-	{USB_VPI(USB_VENDOR_MICROTEK, USB_PRODUCT_MICROTEK_336CX2, 0)},
-	{USB_VPI(USB_VENDOR_MICROTEK, USB_PRODUCT_MICROTEK_C6, 0)},
-	{USB_VPI(USB_VENDOR_MICROTEK, USB_PRODUCT_MICROTEK_V6USL, 0)},
-	{USB_VPI(USB_VENDOR_MICROTEK, USB_PRODUCT_MICROTEK_V6USL2, 0)},
-	{USB_VPI(USB_VENDOR_MICROTEK, USB_PRODUCT_MICROTEK_V6UL, 0)},
-	/* Minolta */
-	{USB_VPI(USB_VENDOR_MINOLTA, USB_PRODUCT_MINOLTA_5400, 0)},
-	/* Mustek */
-	{USB_VPI(USB_VENDOR_MUSTEK, USB_PRODUCT_MUSTEK_1200CU, 0)},
-	{USB_VPI(USB_VENDOR_MUSTEK, USB_PRODUCT_MUSTEK_BEARPAW1200F, 0)},
-	{USB_VPI(USB_VENDOR_MUSTEK, USB_PRODUCT_MUSTEK_BEARPAW1200TA, 0)},
-	{USB_VPI(USB_VENDOR_MUSTEK, USB_PRODUCT_MUSTEK_600USB, 0)},
-	{USB_VPI(USB_VENDOR_MUSTEK, USB_PRODUCT_MUSTEK_600CU, 0)},
-	{USB_VPI(USB_VENDOR_MUSTEK, USB_PRODUCT_MUSTEK_1200USB, 0)},
-	{USB_VPI(USB_VENDOR_MUSTEK, USB_PRODUCT_MUSTEK_1200UB, 0)},
-	{USB_VPI(USB_VENDOR_MUSTEK, USB_PRODUCT_MUSTEK_1200USBPLUS, 0)},
-	{USB_VPI(USB_VENDOR_MUSTEK, USB_PRODUCT_MUSTEK_1200CUPLUS, 0)},
-	/* National */
-	{USB_VPI(USB_VENDOR_NATIONAL, USB_PRODUCT_NATIONAL_BEARPAW1200, 0)},
-	{USB_VPI(USB_VENDOR_NATIONAL, USB_PRODUCT_NATIONAL_BEARPAW2400, 0)},
-	/* Nikon */
-	{USB_VPI(USB_VENDOR_NIKON, USB_PRODUCT_NIKON_LS40, 0)},
-	/* Primax */
-	{USB_VPI(USB_VENDOR_PRIMAX, USB_PRODUCT_PRIMAX_G2X300, 0)},
-	{USB_VPI(USB_VENDOR_PRIMAX, USB_PRODUCT_PRIMAX_G2E300, 0)},
-	{USB_VPI(USB_VENDOR_PRIMAX, USB_PRODUCT_PRIMAX_G2300, 0)},
-	{USB_VPI(USB_VENDOR_PRIMAX, USB_PRODUCT_PRIMAX_G2E3002, 0)},
-	{USB_VPI(USB_VENDOR_PRIMAX, USB_PRODUCT_PRIMAX_9600, 0)},
-	{USB_VPI(USB_VENDOR_PRIMAX, USB_PRODUCT_PRIMAX_600U, 0)},
-	{USB_VPI(USB_VENDOR_PRIMAX, USB_PRODUCT_PRIMAX_6200, 0)},
-	{USB_VPI(USB_VENDOR_PRIMAX, USB_PRODUCT_PRIMAX_19200, 0)},
-	{USB_VPI(USB_VENDOR_PRIMAX, USB_PRODUCT_PRIMAX_1200U, 0)},
-	{USB_VPI(USB_VENDOR_PRIMAX, USB_PRODUCT_PRIMAX_G600, 0)},
-	{USB_VPI(USB_VENDOR_PRIMAX, USB_PRODUCT_PRIMAX_636I, 0)},
-	{USB_VPI(USB_VENDOR_PRIMAX, USB_PRODUCT_PRIMAX_G2600, 0)},
-	{USB_VPI(USB_VENDOR_PRIMAX, USB_PRODUCT_PRIMAX_G2E600, 0)},
-	/* Scanlogic */
-	{USB_VPI(USB_VENDOR_SCANLOGIC, USB_PRODUCT_SCANLOGIC_336CX, 0)},
-	/* Ultima */
-	{USB_VPI(USB_VENDOR_ULTIMA, USB_PRODUCT_ULTIMA_1200UBPLUS, 0)},
-	/* UMAX */
-	{USB_VPI(USB_VENDOR_UMAX, USB_PRODUCT_UMAX_ASTRA1220U, 0)},
-	{USB_VPI(USB_VENDOR_UMAX, USB_PRODUCT_UMAX_ASTRA1236U, 0)},
-	{USB_VPI(USB_VENDOR_UMAX, USB_PRODUCT_UMAX_ASTRA2000U, 0)},
-	{USB_VPI(USB_VENDOR_UMAX, USB_PRODUCT_UMAX_ASTRA2100U, 0)},
-	{USB_VPI(USB_VENDOR_UMAX, USB_PRODUCT_UMAX_ASTRA2200U, 0)},
-	{USB_VPI(USB_VENDOR_UMAX, USB_PRODUCT_UMAX_ASTRA3400, 0)},
-	/* Visioneer */
-	{USB_VPI(USB_VENDOR_VISIONEER, USB_PRODUCT_VISIONEER_3000, 0)},
-	{USB_VPI(USB_VENDOR_VISIONEER, USB_PRODUCT_VISIONEER_5300, 0)},
-	{USB_VPI(USB_VENDOR_VISIONEER, USB_PRODUCT_VISIONEER_7600, 0)},
-	{USB_VPI(USB_VENDOR_VISIONEER, USB_PRODUCT_VISIONEER_6100, 0)},
-	{USB_VPI(USB_VENDOR_VISIONEER, USB_PRODUCT_VISIONEER_6200, 0)},
-	{USB_VPI(USB_VENDOR_VISIONEER, USB_PRODUCT_VISIONEER_8100, 0)},
-	{USB_VPI(USB_VENDOR_VISIONEER, USB_PRODUCT_VISIONEER_8600, 0)}
-};
-
-/*
- * uscanner device probing method.
- */
-static int
-uscanner_probe(device_t dev)
-{
-	struct usb2_attach_arg *uaa;
-
-	DPRINTFN(11, "\n");
-
-	uaa = device_get_ivars(dev);
-	if (uaa->usb2_mode != USB_MODE_HOST) {
-		return (ENXIO);
-	}
-	/* Give other class drivers a chance for multifunctional scanners. */
-	if (uaa->use_generic == 0) {
-		return (ENXIO);
-	}
-	return (usb2_lookup_id_by_uaa(uscanner_devs, sizeof(uscanner_devs), uaa));
-}
-
-/*
- * uscanner device attaching method.
- */
-static int
-uscanner_attach(device_t dev)
-{
-	struct usb2_attach_arg *uaa;
-	struct uscanner_softc *sc;
-	int unit;
-	int error;
-
-	uaa = device_get_ivars(dev);
-	sc = device_get_softc(dev);
-	unit = device_get_unit(dev);
-
-	/*
-	 * A first path softc structure filling.  sc_fifo and
-	 * sc_xfer are initialised later.
-	 */
-	sc->sc_flags = USB_GET_DRIVER_INFO(uaa);
-	mtx_init(&sc->sc_mtx, "uscanner mutex", NULL, MTX_DEF | MTX_RECURSE);
-
-	/*
-	 * Announce the device:
-	 */
-	device_set_usb2_desc(dev);
-
-	/*
-	 * Setup the transfer.
-	 */
-	if ((error = usb2_transfer_setup(uaa->device, &uaa->info.bIfaceIndex, sc->sc_xfer,
-	    uscanner_config, USCANNER_N_TRANSFER, sc, &sc->sc_mtx))) {
-		device_printf(dev, "could not setup transfers, "
-		    "error=%s\n", usb2_errstr(error));
-		goto detach;
-	}
-
-	error = usb2_fifo_attach(uaa->device, sc, &sc->sc_mtx,
-	    &uscanner_fifo_methods, &sc->sc_fifo,
-	    unit, 0 - 1, uaa->info.bIfaceIndex,
-	    UID_ROOT, GID_OPERATOR, 0644);
-	if (error) {
-		goto detach;
-	}
-	return (0);
-
-detach:
-	uscanner_detach(dev);
-	return (ENOMEM);
-}
-
-/*
- * uscanner device detaching method.
- */
-static int
-uscanner_detach(device_t dev)
-{
-	struct uscanner_softc *sc;
-
-	sc = device_get_softc(dev);
-
-	usb2_fifo_detach(&sc->sc_fifo);
-	usb2_transfer_unsetup(sc->sc_xfer, USCANNER_N_TRANSFER);
-	mtx_destroy(&sc->sc_mtx);
-
-	return (0);
-}
-
-/*
- * Reading callback.  Implemented as an "in" bulk transfer.
- */
-static void
-uscanner_read_callback(struct usb2_xfer *xfer)
-{
-	struct uscanner_softc *sc;
-	struct usb2_fifo *f;
-
-	sc = xfer->priv_sc;
-	f = sc->sc_fifo.fp[USB_FIFO_RX];
-
-	switch (USB_GET_STATE(xfer)) {
-	case USB_ST_TRANSFERRED:
-		usb2_fifo_put_data(f, xfer->frbuffers, 0,
-		    xfer->actlen, 1);
-
-	case USB_ST_SETUP:
-		/*
-		 * If reading is in stall, just jump to clear stall callback and
-		 * solve the situation.
-		 */
-		if (sc->sc_flags & USCANNER_FLAG_READ_STALL) {
-			usb2_transfer_start(sc->sc_xfer[USCANNER_BULK_CS_RD]);
-			break;
-		}
-		if (usb2_fifo_put_bytes_max(f) != 0) {
-			xfer->frlengths[0] = xfer->max_data_length;
-			usb2_start_hardware(xfer);
-		}
-		break;
-
-	default:			/* Error */
-		if (xfer->error != USB_ERR_CANCELLED) {
-			sc->sc_flags |= USCANNER_FLAG_READ_STALL;
-			usb2_transfer_start(sc->sc_xfer[USCANNER_BULK_CS_RD]);
-		}
-		break;
-	}
-}
-
-/*
- * Removing stall on reading callback.
- */
-static void
-uscanner_read_clear_stall_callback(struct usb2_xfer *xfer)
-{
-	struct uscanner_softc *sc = xfer->priv_sc;
-	struct usb2_xfer *xfer_other = sc->sc_xfer[USCANNER_BULK_DT_RD];
-
-	if (usb2_clear_stall_callback(xfer, xfer_other)) {
-		DPRINTF("stall cleared\n");
-		sc->sc_flags &= ~USCANNER_FLAG_READ_STALL;
-		usb2_transfer_start(xfer_other);
-	}
-}
-
-/*
- * Writing callback.  Implemented as an "out" bulk transfer.
- */
-static void
-uscanner_write_callback(struct usb2_xfer *xfer)
-{
-	struct uscanner_softc *sc;
-	struct usb2_fifo *f;
-	uint32_t actlen;
-
-	sc = xfer->priv_sc;
-	f = sc->sc_fifo.fp[USB_FIFO_TX];
-
-	switch (USB_GET_STATE(xfer)) {
-	case USB_ST_SETUP:
-	case USB_ST_TRANSFERRED:
-		/*
-		 * If writing is in stall, just jump to clear stall callback and
-		 * solve the situation.
-		 */
-		if (sc->sc_flags & USCANNER_FLAG_WRITE_STALL) {
-			usb2_transfer_start(sc->sc_xfer[USCANNER_BULK_CS_WR]);
-			break;
-		}
-		/*
-		 * Write datas, setup and perform hardware transfer.
-		 */
-		if (usb2_fifo_get_data(f, xfer->frbuffers, 0,
-		    xfer->max_data_length, &actlen, 0)) {
-			xfer->frlengths[0] = actlen;
-			usb2_start_hardware(xfer);
-		}
-		break;
-
-	default:			/* Error */
-		if (xfer->error != USB_ERR_CANCELLED) {
-			sc->sc_flags |= USCANNER_FLAG_WRITE_STALL;
-			usb2_transfer_start(sc->sc_xfer[USCANNER_BULK_CS_WR]);
-		}
-		break;
-	}
-}
-
-/*
- * Removing stall on writing callback.
- */
-static void
-uscanner_write_clear_stall_callback(struct usb2_xfer *xfer)
-{
-	struct uscanner_softc *sc = xfer->priv_sc;
-	struct usb2_xfer *xfer_other = sc->sc_xfer[USCANNER_BULK_DT_WR];
-
-	if (usb2_clear_stall_callback(xfer, xfer_other)) {
-		DPRINTF("stall cleared\n");
-		sc->sc_flags &= ~USCANNER_FLAG_WRITE_STALL;
-		usb2_transfer_start(xfer_other);
-	}
-}
-
-/*
- * uscanner character device opening method.
- */
-static int
-uscanner_open(struct usb2_fifo *fifo, int fflags)
-{
-	struct uscanner_softc *sc;
-
-	sc = fifo->priv_sc0;
-
-	if (!(sc->sc_flags & USCANNER_FLAG_KEEP_OPEN)) {
-		if (fflags & FWRITE) {
-			sc->sc_flags |= USCANNER_FLAG_WRITE_STALL;
-		}
-		if (fflags & FREAD) {
-			sc->sc_flags |= USCANNER_FLAG_READ_STALL;
-		}
-	}
-	if (fflags & FREAD) {
-		if (usb2_fifo_alloc_buffer(fifo,
-		    sc->sc_xfer[USCANNER_BULK_DT_RD]->max_data_length,
-		    USCANNER_IFQ_MAXLEN)) {
-			return (ENOMEM);
-		}
-	}
-	if (fflags & FWRITE) {
-		if (usb2_fifo_alloc_buffer(fifo,
-		    sc->sc_xfer[USCANNER_BULK_DT_WR]->max_data_length,
-		    USCANNER_IFQ_MAXLEN)) {
-			return (ENOMEM);
-		}
-	}
-	return (0);
-}
-
-static void
-uscanner_close(struct usb2_fifo *fifo, int fflags)
-{
-	if (fflags & (FREAD | FWRITE)) {
-		usb2_fifo_free_buffer(fifo);
-	}
-}
-
-/*
- * uscanner character device start reading method.
- */
-static void
-uscanner_start_read(struct usb2_fifo *fifo)
-{
-	struct uscanner_softc *sc;
-
-	sc = fifo->priv_sc0;
-	usb2_transfer_start(sc->sc_xfer[USCANNER_BULK_DT_RD]);
-}
-
-/*
- * uscanner character device start writing method.
- */
-static void
-uscanner_start_write(struct usb2_fifo *fifo)
-{
-	struct uscanner_softc *sc;
-
-	sc = fifo->priv_sc0;
-	usb2_transfer_start(sc->sc_xfer[USCANNER_BULK_DT_WR]);
-}
-
-/*
- * uscanner character device stop reading method.
- */
-static void
-uscanner_stop_read(struct usb2_fifo *fifo)
-{
-	struct uscanner_softc *sc;
-
-	sc = fifo->priv_sc0;
-	usb2_transfer_stop(sc->sc_xfer[USCANNER_BULK_CS_RD]);
-	usb2_transfer_stop(sc->sc_xfer[USCANNER_BULK_DT_RD]);
-}
-
-/*
- * uscanner character device stop writing method.
- */
-static void
-uscanner_stop_write(struct usb2_fifo *fifo)
-{
-	struct uscanner_softc *sc;
-
-	sc = fifo->priv_sc0;
-	usb2_transfer_stop(sc->sc_xfer[USCANNER_BULK_CS_WR]);
-	usb2_transfer_stop(sc->sc_xfer[USCANNER_BULK_DT_WR]);
-}
diff --git a/sys/dev/usb/input/uhid.c b/sys/dev/usb/input/uhid.c
index 99e4b5d..78931bd 100644
--- a/sys/dev/usb/input/uhid.c
+++ b/sys/dev/usb/input/uhid.c
@@ -316,27 +316,27 @@ static const struct usb2_config uhid_config[UHID_N_TRANSFER] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = UHID_BSIZE,
-		.mh.callback = &uhid_intr_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = UHID_BSIZE,
+		.callback = &uhid_intr_callback,
 	},
 
 	[UHID_CTRL_DT_WR] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request) + UHID_BSIZE,
-		.mh.callback = &uhid_write_callback,
-		.mh.timeout = 1000,	/* 1 second */
+		.bufsize = sizeof(struct usb2_device_request) + UHID_BSIZE,
+		.callback = &uhid_write_callback,
+		.timeout = 1000,	/* 1 second */
 	},
 
 	[UHID_CTRL_DT_RD] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request) + UHID_BSIZE,
-		.mh.callback = &uhid_read_callback,
-		.mh.timeout = 1000,	/* 1 second */
+		.bufsize = sizeof(struct usb2_device_request) + UHID_BSIZE,
+		.callback = &uhid_read_callback,
+		.timeout = 1000,	/* 1 second */
 	},
 };
 
@@ -668,8 +668,8 @@ uhid_attach(device_t dev)
 			 * feature report ID 2 before it'll start
 			 * returning digitizer data.
 			 */
-			error = usb2_req_set_report
-			    (uaa->device, &Giant, reportbuf, sizeof(reportbuf),
+			error = usb2_req_set_report(uaa->device, NULL,
+			    reportbuf, sizeof(reportbuf),
 			    uaa->info.bIfaceIndex, UHID_FEATURE_REPORT, 2);
 
 			if (error) {
@@ -691,16 +691,16 @@ uhid_attach(device_t dev)
 	}
 	if (sc->sc_repdesc_ptr == NULL) {
 
-		error = usb2_req_get_hid_desc
-		    (uaa->device, &Giant, &sc->sc_repdesc_ptr,
-		    &sc->sc_repdesc_size, M_USBDEV, uaa->info.bIfaceIndex);
+		error = usb2_req_get_hid_desc(uaa->device, NULL,
+		    &sc->sc_repdesc_ptr, &sc->sc_repdesc_size,
+		    M_USBDEV, uaa->info.bIfaceIndex);
 
 		if (error) {
 			device_printf(dev, "no report descriptor\n");
 			goto detach;
 		}
 	}
-	error = usb2_req_set_idle(uaa->device, &Giant,
+	error = usb2_req_set_idle(uaa->device, NULL,
 	    uaa->info.bIfaceIndex, 0, 0);
 
 	if (error) {
diff --git a/sys/dev/usb/input/ukbd.c b/sys/dev/usb/input/ukbd.c
index df81ad9..58e8151 100644
--- a/sys/dev/usb/input/ukbd.c
+++ b/sys/dev/usb/input/ukbd.c
@@ -562,28 +562,28 @@ static const struct usb2_config ukbd_config[UKBD_N_TRANSFER] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.callback = &ukbd_intr_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.callback = &ukbd_intr_callback,
 	},
 
 	[UKBD_INTR_CS] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.callback = &ukbd_clear_stall_callback,
-		.mh.timeout = 1000,	/* 1 second */
-		.mh.interval = 50,	/* 50ms */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &ukbd_clear_stall_callback,
+		.timeout = 1000,	/* 1 second */
+		.interval = 50,	/* 50ms */
 	},
 
 	[UKBD_CTRL_LED] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request) + 1,
-		.mh.callback = &ukbd_set_leds_callback,
-		.mh.timeout = 1000,	/* 1 second */
+		.bufsize = sizeof(struct usb2_device_request) + 1,
+		.callback = &ukbd_set_leds_callback,
+		.timeout = 1000,	/* 1 second */
 	},
 };
 
diff --git a/sys/dev/usb/input/ums.c b/sys/dev/usb/input/ums.c
index 5b884e8..6ed030d 100644
--- a/sys/dev/usb/input/ums.c
+++ b/sys/dev/usb/input/ums.c
@@ -84,27 +84,20 @@ SYSCTL_INT(_hw_usb2_ums, OID_AUTO, debug, CTLFLAG_RW,
 #define	UMS_IFQ_MAXLEN   50		/* units */
 #define	UMS_BUTTON_MAX   31		/* exclusive, must be less than 32 */
 #define	UMS_BUT(i) ((i) < 3 ? (((i) + 2) % 3) : (i))
+#define	UMS_INFO_MAX	  2		/* maximum number of HID sets */
 
 enum {
 	UMS_INTR_DT,
 	UMS_N_TRANSFER,
 };
 
-struct ums_softc {
-	struct usb2_fifo_sc sc_fifo;
-	struct mtx sc_mtx;
-	struct usb2_callout sc_callout;
+struct ums_info {
 	struct hid_location sc_loc_w;
 	struct hid_location sc_loc_x;
 	struct hid_location sc_loc_y;
 	struct hid_location sc_loc_z;
 	struct hid_location sc_loc_t;
 	struct hid_location sc_loc_btn[UMS_BUTTON_MAX];
-	mousehw_t sc_hw;
-	mousemode_t sc_mode;
-	mousestatus_t sc_status;
-
-	struct usb2_xfer *sc_xfer[UMS_N_TRANSFER];
 
 	uint32_t sc_flags;
 #define	UMS_FLAG_X_AXIS     0x0001
@@ -115,14 +108,29 @@ struct ums_softc {
 #define	UMS_FLAG_REVZ	    0x0020	/* Z-axis is reversed */
 #define	UMS_FLAG_W_AXIS     0x0040
 
-	uint8_t	sc_buttons;
-	uint8_t	sc_iid;
 	uint8_t	sc_iid_w;
 	uint8_t	sc_iid_x;
 	uint8_t	sc_iid_y;
 	uint8_t	sc_iid_z;
 	uint8_t	sc_iid_t;
 	uint8_t	sc_iid_btn[UMS_BUTTON_MAX];
+	uint8_t	sc_buttons;
+};
+
+struct ums_softc {
+	struct usb2_fifo_sc sc_fifo;
+	struct mtx sc_mtx;
+	struct usb2_callout sc_callout;
+	struct ums_info sc_info[UMS_INFO_MAX];
+
+	mousehw_t sc_hw;
+	mousemode_t sc_mode;
+	mousestatus_t sc_status;
+
+	struct usb2_xfer *sc_xfer[UMS_N_TRANSFER];
+
+	uint8_t	sc_buttons;
+	uint8_t	sc_iid;
 	uint8_t	sc_temp[64];
 };
 
@@ -165,14 +173,15 @@ static void
 ums_intr_callback(struct usb2_xfer *xfer)
 {
 	struct ums_softc *sc = xfer->priv_sc;
+	struct ums_info *info = &sc->sc_info[0];
 	uint8_t *buf = sc->sc_temp;
 	uint16_t len = xfer->actlen;
 	int32_t buttons = 0;
-	int32_t dw;
-	int32_t dx;
-	int32_t dy;
-	int32_t dz;
-	int32_t dt;
+	int32_t dw = 0;
+	int32_t dx = 0;
+	int32_t dy = 0;
+	int32_t dz = 0;
+	int32_t dt = 0;
 	uint8_t i;
 	uint8_t id;
 
@@ -205,49 +214,50 @@ ums_intr_callback(struct usb2_xfer *xfer)
 
 		} else {
 			id = 0;
-			if (sc->sc_flags & UMS_FLAG_SBU) {
+			if (sc->sc_info[0].sc_flags & UMS_FLAG_SBU) {
 				if ((*buf == 0x14) || (*buf == 0x15)) {
 					goto tr_setup;
 				}
 			}
 		}
 
-		if ((sc->sc_flags & UMS_FLAG_W_AXIS) && (id == sc->sc_iid_w))
-			dw = hid_get_data(buf, len, &sc->sc_loc_w);
-		else
-			dw = 0;
-
-		if ((sc->sc_flags & UMS_FLAG_X_AXIS) && (id == sc->sc_iid_x))
-			dx = hid_get_data(buf, len, &sc->sc_loc_x);
-		else
-			dx = 0;
-
-		if ((sc->sc_flags & UMS_FLAG_Y_AXIS) && (id == sc->sc_iid_y))
-			dy = -hid_get_data(buf, len, &sc->sc_loc_y);
-		else
-			dy = 0;
-
-		if ((sc->sc_flags & UMS_FLAG_Z_AXIS) && (id == sc->sc_iid_z))
-			dz = -hid_get_data(buf, len, &sc->sc_loc_z);
-		else
-			dz = 0;
-
-		if (sc->sc_flags & UMS_FLAG_REVZ)
-			dz = -dz;
+	repeat:
+		if ((info->sc_flags & UMS_FLAG_W_AXIS) &&
+		    (id == info->sc_iid_w))
+			dw += hid_get_data(buf, len, &info->sc_loc_w);
+
+		if ((info->sc_flags & UMS_FLAG_X_AXIS) && 
+		    (id == info->sc_iid_x))
+			dx += hid_get_data(buf, len, &info->sc_loc_x);
+
+		if ((info->sc_flags & UMS_FLAG_Y_AXIS) &&
+		    (id == info->sc_iid_y))
+			dy = -hid_get_data(buf, len, &info->sc_loc_y);
+
+		if ((info->sc_flags & UMS_FLAG_Z_AXIS) &&
+		    (id == info->sc_iid_z)) {
+			int32_t temp;
+			temp = hid_get_data(buf, len, &info->sc_loc_z);
+			if (info->sc_flags & UMS_FLAG_REVZ)
+				temp = -temp;
+			dz -= temp;
+		}
 
-		if ((sc->sc_flags & UMS_FLAG_T_AXIS) && (id == sc->sc_iid_t))
-			dt = -hid_get_data(buf, len, &sc->sc_loc_t);
-		else
-			dt = 0;
+		if ((info->sc_flags & UMS_FLAG_T_AXIS) &&
+		    (id == info->sc_iid_t))
+			dt -= hid_get_data(buf, len, &info->sc_loc_t);
 
-		for (i = 0; i < sc->sc_buttons; i++) {
-			if (id != sc->sc_iid_btn[i])
+		for (i = 0; i < info->sc_buttons; i++) {
+			if (id != info->sc_iid_btn[i])
 				continue;
-			if (hid_get_data(buf, len, &sc->sc_loc_btn[i])) {
+			if (hid_get_data(buf, len, &info->sc_loc_btn[i])) {
 				buttons |= (1 << UMS_BUT(i));
 			}
 		}
 
+		if (++info != &sc->sc_info[UMS_INFO_MAX])
+			goto repeat;
+
 		if (dx || dy || dz || dt || dw ||
 		    (buttons != sc->sc_status.button)) {
 
@@ -273,7 +283,7 @@ ums_intr_callback(struct usb2_xfer *xfer)
 		         * to the queue.  In any other case we delete
 		         * the timeout event.
 		         */
-			if ((sc->sc_flags & UMS_FLAG_SBU) &&
+			if ((sc->sc_info[0].sc_flags & UMS_FLAG_SBU) &&
 			    (dx == 0) && (dy == 0) && (dz == 0) && (dt == 0) &&
 			    (dw == 0) && (buttons == 0)) {
 
@@ -312,9 +322,9 @@ static const struct usb2_config ums_config[UMS_N_TRANSFER] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.callback = &ums_intr_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.callback = &ums_intr_callback,
 	},
 };
 
@@ -338,7 +348,7 @@ ums_probe(device_t dev)
 	    (id->bInterfaceClass != UICLASS_HID))
 		return (ENXIO);
 
-	error = usb2_req_get_hid_desc(uaa->device, &Giant,
+	error = usb2_req_get_hid_desc(uaa->device, NULL,
 	    &d_ptr, &d_len, M_TEMP, uaa->info.bIfaceIndex);
 
 	if (error)
@@ -357,90 +367,56 @@ ums_probe(device_t dev)
 	return (error);
 }
 
-static int
-ums_attach(device_t dev)
+static void
+ums_hid_parse(struct ums_softc *sc, device_t dev, const uint8_t *buf,
+    uint16_t len, uint8_t index)
 {
-	struct usb2_attach_arg *uaa = device_get_ivars(dev);
-	struct ums_softc *sc = device_get_softc(dev);
-	void *d_ptr = NULL;
-	int unit = device_get_unit(dev);
-	int isize;
-	int isizebits;
-	int err;
+	struct ums_info *info = &sc->sc_info[index];
 	uint32_t flags;
-	uint16_t d_len;
 	uint8_t i;
 
-	DPRINTFN(11, "sc=%p\n", sc);
-
-	device_set_usb2_desc(dev);
-
-	mtx_init(&sc->sc_mtx, "ums lock", NULL, MTX_DEF | MTX_RECURSE);
-
-	usb2_callout_init_mtx(&sc->sc_callout, &sc->sc_mtx, 0);
-
-	/*
-         * Force the report (non-boot) protocol.
-         *
-         * Mice without boot protocol support may choose not to implement
-         * Set_Protocol at all; Ignore any error.
-         */
-	err = usb2_req_set_protocol(uaa->device, NULL, uaa->info.bIfaceIndex, 1);
-
-	err = usb2_transfer_setup(uaa->device,
-	    &uaa->info.bIfaceIndex, sc->sc_xfer, ums_config,
-	    UMS_N_TRANSFER, sc, &sc->sc_mtx);
-
-	if (err) {
-		DPRINTF("error=%s\n", usb2_errstr(err));
-		goto detach;
-	}
-	err = usb2_req_get_hid_desc
-	    (uaa->device, &Giant, &d_ptr,
-	    &d_len, M_TEMP, uaa->info.bIfaceIndex);
-
-	if (err) {
-		device_printf(dev, "error reading report description\n");
-		goto detach;
-	}
-	if (hid_locate(d_ptr, d_len, HID_USAGE2(HUP_GENERIC_DESKTOP, HUG_X),
-	    hid_input, &sc->sc_loc_x, &flags, &sc->sc_iid_x)) {
+	if (hid_locate(buf, len, HID_USAGE2(HUP_GENERIC_DESKTOP, HUG_X),
+	    hid_input, index, &info->sc_loc_x, &flags, &info->sc_iid_x)) {
 
 		if ((flags & MOUSE_FLAGS_MASK) == MOUSE_FLAGS) {
-			sc->sc_flags |= UMS_FLAG_X_AXIS;
+			info->sc_flags |= UMS_FLAG_X_AXIS;
 		}
 	}
-	if (hid_locate(d_ptr, d_len, HID_USAGE2(HUP_GENERIC_DESKTOP, HUG_Y),
-	    hid_input, &sc->sc_loc_y, &flags, &sc->sc_iid_y)) {
+	if (hid_locate(buf, len, HID_USAGE2(HUP_GENERIC_DESKTOP, HUG_Y),
+	    hid_input, index, &info->sc_loc_y, &flags, &info->sc_iid_y)) {
 
 		if ((flags & MOUSE_FLAGS_MASK) == MOUSE_FLAGS) {
-			sc->sc_flags |= UMS_FLAG_Y_AXIS;
+			info->sc_flags |= UMS_FLAG_Y_AXIS;
 		}
 	}
 	/* Try the wheel first as the Z activator since it's tradition. */
-	if (hid_locate(d_ptr, d_len, HID_USAGE2(HUP_GENERIC_DESKTOP,
-	    HUG_WHEEL), hid_input, &sc->sc_loc_z, &flags, &sc->sc_iid_z) ||
-	    hid_locate(d_ptr, d_len, HID_USAGE2(HUP_GENERIC_DESKTOP,
-	    HUG_TWHEEL), hid_input, &sc->sc_loc_z, &flags, &sc->sc_iid_z)) {
+	if (hid_locate(buf, len, HID_USAGE2(HUP_GENERIC_DESKTOP,
+	    HUG_WHEEL), hid_input, index, &info->sc_loc_z, &flags,
+	    &info->sc_iid_z) ||
+	    hid_locate(buf, len, HID_USAGE2(HUP_GENERIC_DESKTOP,
+	    HUG_TWHEEL), hid_input, index, &info->sc_loc_z, &flags,
+	    &info->sc_iid_z)) {
 		if ((flags & MOUSE_FLAGS_MASK) == MOUSE_FLAGS) {
-			sc->sc_flags |= UMS_FLAG_Z_AXIS;
+			info->sc_flags |= UMS_FLAG_Z_AXIS;
 		}
 		/*
 		 * We might have both a wheel and Z direction, if so put
 		 * put the Z on the W coordinate.
 		 */
-		if (hid_locate(d_ptr, d_len, HID_USAGE2(HUP_GENERIC_DESKTOP,
-		    HUG_Z), hid_input, &sc->sc_loc_w, &flags, &sc->sc_iid_w)) {
+		if (hid_locate(buf, len, HID_USAGE2(HUP_GENERIC_DESKTOP,
+		    HUG_Z), hid_input, index, &info->sc_loc_w, &flags,
+		    &info->sc_iid_w)) {
 
 			if ((flags & MOUSE_FLAGS_MASK) == MOUSE_FLAGS) {
-				sc->sc_flags |= UMS_FLAG_W_AXIS;
+				info->sc_flags |= UMS_FLAG_W_AXIS;
 			}
 		}
-	} else if (hid_locate(d_ptr, d_len, HID_USAGE2(HUP_GENERIC_DESKTOP,
-	    HUG_Z), hid_input, &sc->sc_loc_z, &flags, &sc->sc_iid_z)) {
+	} else if (hid_locate(buf, len, HID_USAGE2(HUP_GENERIC_DESKTOP,
+	    HUG_Z), hid_input, index, &info->sc_loc_z, &flags, 
+	    &info->sc_iid_z)) {
 
 		if ((flags & MOUSE_FLAGS_MASK) == MOUSE_FLAGS) {
-			sc->sc_flags |= UMS_FLAG_Z_AXIS;
+			info->sc_flags |= UMS_FLAG_Z_AXIS;
 		}
 	}
 	/*
@@ -450,25 +426,89 @@ ums_attach(device_t dev)
 	 * There are no other HID axis descriptors other than X,Y and
 	 * TWHEEL
 	 */
-	if (hid_locate(d_ptr, d_len, HID_USAGE2(HUP_GENERIC_DESKTOP, HUG_TWHEEL),
-	    hid_input, &sc->sc_loc_t, &flags, &sc->sc_iid_t)) {
+	if (hid_locate(buf, len, HID_USAGE2(HUP_GENERIC_DESKTOP,
+	    HUG_TWHEEL), hid_input, index, &info->sc_loc_t, 
+	    &flags, &info->sc_iid_t)) {
 
-		sc->sc_loc_t.pos += 8;
+		info->sc_loc_t.pos += 8;
 
 		if ((flags & MOUSE_FLAGS_MASK) == MOUSE_FLAGS) {
-			sc->sc_flags |= UMS_FLAG_T_AXIS;
+			info->sc_flags |= UMS_FLAG_T_AXIS;
 		}
 	}
 	/* figure out the number of buttons */
 
 	for (i = 0; i < UMS_BUTTON_MAX; i++) {
-		if (!hid_locate(d_ptr, d_len, HID_USAGE2(HUP_BUTTON, (i + 1)),
-			hid_input, &sc->sc_loc_btn[i], NULL, &sc->sc_iid_btn[i])) {
+		if (!hid_locate(buf, len, HID_USAGE2(HUP_BUTTON, (i + 1)),
+		    hid_input, index, &info->sc_loc_btn[i], NULL, 
+		    &info->sc_iid_btn[i])) {
 			break;
 		}
 	}
+	info->sc_buttons = i;
+
+	if (i > sc->sc_buttons)
+		sc->sc_buttons = i;
+
+	if (info->sc_flags == 0)
+		return;
+
+	/* announce information about the mouse */
+	device_printf(dev, "%d buttons and [%s%s%s%s%s] coordinates ID=%u\n",
+	    (info->sc_buttons),
+	    (info->sc_flags & UMS_FLAG_X_AXIS) ? "X" : "",
+	    (info->sc_flags & UMS_FLAG_Y_AXIS) ? "Y" : "",
+	    (info->sc_flags & UMS_FLAG_Z_AXIS) ? "Z" : "",
+	    (info->sc_flags & UMS_FLAG_T_AXIS) ? "T" : "",
+	    (info->sc_flags & UMS_FLAG_W_AXIS) ? "W" : "",
+	    info->sc_iid_x);
+}
+
+static int
+ums_attach(device_t dev)
+{
+	struct usb2_attach_arg *uaa = device_get_ivars(dev);
+	struct ums_softc *sc = device_get_softc(dev);
+	struct ums_info *info;
+	void *d_ptr = NULL;
+	int isize;
+	int err;
+	uint16_t d_len;
+	uint8_t i;
+	uint8_t j;
+
+	DPRINTFN(11, "sc=%p\n", sc);
+
+	device_set_usb2_desc(dev);
+
+	mtx_init(&sc->sc_mtx, "ums lock", NULL, MTX_DEF | MTX_RECURSE);
+
+	usb2_callout_init_mtx(&sc->sc_callout, &sc->sc_mtx, 0);
+
+	/*
+         * Force the report (non-boot) protocol.
+         *
+         * Mice without boot protocol support may choose not to implement
+         * Set_Protocol at all; Ignore any error.
+         */
+	err = usb2_req_set_protocol(uaa->device, NULL,
+	    uaa->info.bIfaceIndex, 1);
+
+	err = usb2_transfer_setup(uaa->device,
+	    &uaa->info.bIfaceIndex, sc->sc_xfer, ums_config,
+	    UMS_N_TRANSFER, sc, &sc->sc_mtx);
+
+	if (err) {
+		DPRINTF("error=%s\n", usb2_errstr(err));
+		goto detach;
+	}
+	err = usb2_req_get_hid_desc(uaa->device, NULL, &d_ptr,
+	    &d_len, M_TEMP, uaa->info.bIfaceIndex);
 
-	sc->sc_buttons = i;
+	if (err) {
+		device_printf(dev, "error reading report description\n");
+		goto detach;
+	}
 
 	isize = hid_report_size(d_ptr, d_len, hid_input, &sc->sc_iid);
 
@@ -479,83 +519,66 @@ ums_attach(device_t dev)
 	 * it has two addional buttons and a tilt wheel.
 	 */
 	if (usb2_test_quirk(uaa, UQ_MS_BAD_CLASS)) {
-		sc->sc_flags = (UMS_FLAG_X_AXIS |
+		info = &sc->sc_info[0];
+		info->sc_flags = (UMS_FLAG_X_AXIS |
 		    UMS_FLAG_Y_AXIS |
 		    UMS_FLAG_Z_AXIS |
 		    UMS_FLAG_SBU);
-		sc->sc_buttons = 3;
+		info->sc_buttons = 3;
 		isize = 5;
-		sc->sc_iid = 0;
-		sc->sc_iid_x = 0;
-		sc->sc_iid_y = 0;
-		sc->sc_iid_z = 0;
-		sc->sc_iid_btn[0] = 0;
-		sc->sc_iid_btn[1] = 0;
-		sc->sc_iid_btn[2] = 0;
 		/* 1st byte of descriptor report contains garbage */
-		sc->sc_loc_x.pos = 16;
-		sc->sc_loc_y.pos = 24;
-		sc->sc_loc_z.pos = 32;
-		sc->sc_loc_btn[0].pos = 8;
-		sc->sc_loc_btn[1].pos = 9;
-		sc->sc_loc_btn[2].pos = 10;
-	}
+		info->sc_loc_x.pos = 16;
+		info->sc_loc_y.pos = 24;
+		info->sc_loc_z.pos = 32;
+		info->sc_loc_btn[0].pos = 8;
+		info->sc_loc_btn[1].pos = 9;
+		info->sc_loc_btn[2].pos = 10;
 
-	/*
-	 * Some Microsoft devices have incorrectly high location
-	 * positions. Correct this:
-	 */
-	isizebits = isize * 8;
-	if ((sc->sc_iid != 0) && (isizebits > 8)) {
-		isizebits -= 8;	/* remove size of report ID */
-		sc->sc_loc_w.pos %= isizebits;
-		sc->sc_loc_x.pos %= isizebits;
-		sc->sc_loc_y.pos %= isizebits;
-		sc->sc_loc_z.pos %= isizebits;
-		sc->sc_loc_t.pos %= isizebits;
-		for (i = 0; i != UMS_BUTTON_MAX; i++)
-			sc->sc_loc_btn[i].pos %= isizebits;
+		/* Announce device */
+		device_printf(dev, "3 buttons and [XYZ] "
+		    "coordinates ID=0\n");
+
+	} else {
+		/* Search the HID descriptor and announce device */
+		for (i = 0; i < UMS_INFO_MAX; i++) {
+			ums_hid_parse(sc, dev, d_ptr, d_len, i);
+		}
 	}
 
 	if (usb2_test_quirk(uaa, UQ_MS_REVZ)) {
+		info = &sc->sc_info[0];
 		/* Some wheels need the Z axis reversed. */
-		sc->sc_flags |= UMS_FLAG_REVZ;
+		info->sc_flags |= UMS_FLAG_REVZ;
 	}
 	if (isize > sc->sc_xfer[UMS_INTR_DT]->max_frame_size) {
 		DPRINTF("WARNING: report size, %d bytes, is larger "
 		    "than interrupt size, %d bytes!\n",
 		    isize, sc->sc_xfer[UMS_INTR_DT]->max_frame_size);
 	}
-	/* announce information about the mouse */
-
-	device_printf(dev, "%d buttons and [%s%s%s%s%s] coordinates\n",
-	    (sc->sc_buttons),
-	    (sc->sc_flags & UMS_FLAG_X_AXIS) ? "X" : "",
-	    (sc->sc_flags & UMS_FLAG_Y_AXIS) ? "Y" : "",
-	    (sc->sc_flags & UMS_FLAG_Z_AXIS) ? "Z" : "",
-	    (sc->sc_flags & UMS_FLAG_T_AXIS) ? "T" : "",
-	    (sc->sc_flags & UMS_FLAG_W_AXIS) ? "W" : "");
-
 	free(d_ptr, M_TEMP);
 	d_ptr = NULL;
 
 #if USB_DEBUG
-	DPRINTF("sc=%p\n", sc);
-	DPRINTF("X\t%d/%d id=%d\n", sc->sc_loc_x.pos,
-	    sc->sc_loc_x.size, sc->sc_iid_x);
-	DPRINTF("Y\t%d/%d id=%d\n", sc->sc_loc_y.pos,
-	    sc->sc_loc_y.size, sc->sc_iid_y);
-	DPRINTF("Z\t%d/%d id=%d\n", sc->sc_loc_z.pos,
-	    sc->sc_loc_z.size, sc->sc_iid_z);
-	DPRINTF("T\t%d/%d id=%d\n", sc->sc_loc_t.pos,
-	    sc->sc_loc_t.size, sc->sc_iid_t);
-	DPRINTF("W\t%d/%d id=%d\n", sc->sc_loc_w.pos,
-	    sc->sc_loc_w.size, sc->sc_iid_w);
-
-	for (i = 0; i < sc->sc_buttons; i++) {
-		DPRINTF("B%d\t%d/%d id=%d\n",
-		    i + 1, sc->sc_loc_btn[i].pos,
-		    sc->sc_loc_btn[i].size, sc->sc_iid_btn[i]);
+	for (j = 0; j < UMS_INFO_MAX; j++) {
+		info = &sc->sc_info[j];
+
+		DPRINTF("sc=%p, index=%d\n", sc, j);
+		DPRINTF("X\t%d/%d id=%d\n", info->sc_loc_x.pos,
+		    info->sc_loc_x.size, info->sc_iid_x);
+		DPRINTF("Y\t%d/%d id=%d\n", info->sc_loc_y.pos,
+		    info->sc_loc_y.size, info->sc_iid_y);
+		DPRINTF("Z\t%d/%d id=%d\n", info->sc_loc_z.pos,
+		    info->sc_loc_z.size, info->sc_iid_z);
+		DPRINTF("T\t%d/%d id=%d\n", info->sc_loc_t.pos,
+		    info->sc_loc_t.size, info->sc_iid_t);
+		DPRINTF("W\t%d/%d id=%d\n", info->sc_loc_w.pos,
+		    info->sc_loc_w.size, info->sc_iid_w);
+
+		for (i = 0; i < info->sc_buttons; i++) {
+			DPRINTF("B%d\t%d/%d id=%d\n",
+			    i + 1, info->sc_loc_btn[i].pos,
+			    info->sc_loc_btn[i].size, info->sc_iid_btn[i]);
+		}
 	}
 	DPRINTF("size=%d, id=%d\n", isize, sc->sc_iid);
 #endif
@@ -579,16 +602,9 @@ ums_attach(device_t dev)
 	sc->sc_mode.syncmask[0] = MOUSE_MSC_SYNCMASK;
 	sc->sc_mode.syncmask[1] = MOUSE_MSC_SYNC;
 
-	sc->sc_status.flags = 0;
-	sc->sc_status.button = 0;
-	sc->sc_status.obutton = 0;
-	sc->sc_status.dx = 0;
-	sc->sc_status.dy = 0;
-	sc->sc_status.dz = 0;
-
 	err = usb2_fifo_attach(uaa->device, sc, &sc->sc_mtx,
 	    &ums_fifo_methods, &sc->sc_fifo,
-	    unit, 0 - 1, uaa->info.bIfaceIndex,
+	    device_get_unit(dev), 0 - 1, uaa->info.bIfaceIndex,
   	    UID_ROOT, GID_OPERATOR, 0644);
 	if (err) {
 		goto detach;
diff --git a/sys/dev/usb/misc/udbp.c b/sys/dev/usb/misc/udbp.c
index 2b40ff1..e8acba8 100644
--- a/sys/dev/usb/misc/udbp.c
+++ b/sys/dev/usb/misc/udbp.c
@@ -192,41 +192,39 @@ static const struct usb2_config udbp_config[UDBP_T_MAX] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = UDBP_BUFFERSIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &udbp_bulk_write_callback,
-		.mh.timeout = UDBP_TIMEOUT,
+		.bufsize = UDBP_BUFFERSIZE,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &udbp_bulk_write_callback,
+		.timeout = UDBP_TIMEOUT,
 	},
 
 	[UDBP_T_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = UDBP_BUFFERSIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &udbp_bulk_read_callback,
+		.bufsize = UDBP_BUFFERSIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &udbp_bulk_read_callback,
 	},
 
 	[UDBP_T_WR_CS] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &udbp_bulk_write_clear_stall_callback,
-		.mh.timeout = 1000,	/* 1 second */
-		.mh.interval = 50,	/* 50ms */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &udbp_bulk_write_clear_stall_callback,
+		.timeout = 1000,	/* 1 second */
+		.interval = 50,	/* 50ms */
 	},
 
 	[UDBP_T_RD_CS] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &udbp_bulk_read_clear_stall_callback,
-		.mh.timeout = 1000,	/* 1 second */
-		.mh.interval = 50,	/* 50ms */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &udbp_bulk_read_clear_stall_callback,
+		.timeout = 1000,	/* 1 second */
+		.interval = 50,	/* 50ms */
 	},
 };
 
diff --git a/sys/dev/usb/net/if_aue.c b/sys/dev/usb/net/if_aue.c
index b014b01..ca72948 100644
--- a/sys/dev/usb/net/if_aue.c
+++ b/sys/dev/usb/net/if_aue.c
@@ -209,28 +209,28 @@ static const struct usb2_config aue_config[AUE_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = (MCLBYTES + 2),
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = aue_bulk_write_callback,
-		.mh.timeout = 10000,	/* 10 seconds */
+		.bufsize = (MCLBYTES + 2),
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = aue_bulk_write_callback,
+		.timeout = 10000,	/* 10 seconds */
 	},
 
 	[AUE_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = (MCLBYTES + 4 + ETHER_CRC_LEN),
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = aue_bulk_read_callback,
+		.bufsize = (MCLBYTES + 4 + ETHER_CRC_LEN),
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = aue_bulk_read_callback,
 	},
 
 	[AUE_INTR_DT_RD] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.callback = aue_intr_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.callback = aue_intr_callback,
 	},
 };
 
diff --git a/sys/dev/usb/net/if_axe.c b/sys/dev/usb/net/if_axe.c
index 43ac676..c0db84b 100644
--- a/sys/dev/usb/net/if_axe.c
+++ b/sys/dev/usb/net/if_axe.c
@@ -182,10 +182,10 @@ static const struct usb2_config axe_config[AXE_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = AXE_BULK_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = axe_bulk_write_callback,
-		.mh.timeout = 10000,	/* 10 seconds */
+		.bufsize = AXE_BULK_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = axe_bulk_write_callback,
+		.timeout = 10000,	/* 10 seconds */
 	},
 
 	[AXE_BULK_DT_RD] = {
@@ -195,19 +195,19 @@ static const struct usb2_config axe_config[AXE_N_TRANSFER] = {
 #if (MCLBYTES < 2048)
 #error "(MCLBYTES < 2048)"
 #endif
-		.mh.bufsize = MCLBYTES,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = axe_bulk_read_callback,
-		.mh.timeout = 0,	/* no timeout */
+		.bufsize = MCLBYTES,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = axe_bulk_read_callback,
+		.timeout = 0,	/* no timeout */
 	},
 
 	[AXE_INTR_DT_RD] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.callback = axe_intr_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.callback = axe_intr_callback,
 	},
 };
 
diff --git a/sys/dev/usb/net/if_cdce.c b/sys/dev/usb/net/if_cdce.c
index a54ddc0..9a9cb37 100644
--- a/sys/dev/usb/net/if_cdce.c
+++ b/sys/dev/usb/net/if_cdce.c
@@ -48,7 +48,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
 #include <dev/usb/usb_cdc.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR cdce_debug
 
@@ -97,59 +96,54 @@ SYSCTL_INT(_hw_usb2_cdce, OID_AUTO, debug, CTLFLAG_RW, &cdce_debug, 0,
 
 static const struct usb2_config cdce_config[CDCE_N_TRANSFER] = {
 
-	[CDCE_BULK_A] = {
+	[CDCE_BULK_RX] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
-		.direction = UE_DIR_OUT,
+		.direction = UE_DIR_RX,
 		.if_index = 0,
-		/* Host Mode */
-		.mh.frames = CDCE_FRAMES_MAX,
-		.mh.bufsize = (CDCE_FRAMES_MAX * MCLBYTES),
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,.ext_buffer = 1,},
-		.mh.callback = cdce_bulk_write_callback,
-		.mh.timeout = 10000,	/* 10 seconds */
-		/* Device Mode */
-		.md.frames = CDCE_FRAMES_MAX,
-		.md.bufsize = (CDCE_FRAMES_MAX * MCLBYTES),
-		.md.flags = {.pipe_bof = 1,.short_frames_ok = 1,.short_xfer_ok = 1,.ext_buffer = 1,},
-		.md.callback = cdce_bulk_read_callback,
-		.md.timeout = 0,	/* no timeout */
+		.frames = CDCE_FRAMES_MAX,
+		.bufsize = (CDCE_FRAMES_MAX * MCLBYTES),
+		.flags = {.pipe_bof = 1,.short_frames_ok = 1,.short_xfer_ok = 1,.ext_buffer = 1,},
+		.callback = cdce_bulk_read_callback,
+		.timeout = 0,	/* no timeout */
+		.usb_mode = USB_MODE_MAX,	/* both modes */
 	},
 
-	[CDCE_BULK_B] = {
+	[CDCE_BULK_TX] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
-		.direction = UE_DIR_IN,
+		.direction = UE_DIR_TX,
 		.if_index = 0,
-		/* Host Mode */
-		.mh.frames = CDCE_FRAMES_MAX,
-		.mh.bufsize = (CDCE_FRAMES_MAX * MCLBYTES),
-		.mh.flags = {.pipe_bof = 1,.short_frames_ok = 1,.short_xfer_ok = 1,.ext_buffer = 1,},
-		.mh.callback = cdce_bulk_read_callback,
-		.mh.timeout = 0,	/* no timeout */
-		/* Device Mode */
-		.md.frames = CDCE_FRAMES_MAX,
-		.md.bufsize = (CDCE_FRAMES_MAX * MCLBYTES),
-		.md.flags = {.pipe_bof = 1,.force_short_xfer = 1,.ext_buffer = 1,},
-		.md.callback = cdce_bulk_write_callback,
-		.md.timeout = 10000,	/* 10 seconds */
+		.frames = CDCE_FRAMES_MAX,
+		.bufsize = (CDCE_FRAMES_MAX * MCLBYTES),
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,.ext_buffer = 1,},
+		.callback = cdce_bulk_write_callback,
+		.timeout = 10000,	/* 10 seconds */
+		.usb_mode = USB_MODE_MAX,	/* both modes */
 	},
 
-	[CDCE_INTR] = {
+	[CDCE_INTR_RX] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
-		.direction = UE_DIR_IN,
+		.direction = UE_DIR_RX,
 		.if_index = 1,
-		/* Host Mode */
-		.mh.bufsize = CDCE_IND_SIZE_MAX,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,.no_pipe_ok = 1,},
-		.mh.callback = cdce_intr_read_callback,
-		.mh.timeout = 0,
-		/* Device Mode */
-		.md.bufsize = CDCE_IND_SIZE_MAX,
-		.md.flags = {.pipe_bof = 1,.force_short_xfer = 1,.no_pipe_ok = 1,},
-		.md.callback = cdce_intr_write_callback,
-		.md.timeout = 10000,	/* 10 seconds */
+		.bufsize = CDCE_IND_SIZE_MAX,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,.no_pipe_ok = 1,},
+		.callback = cdce_intr_read_callback,
+		.timeout = 0,
+		.usb_mode = USB_MODE_HOST,
+	},
+
+	[CDCE_INTR_TX] = {
+		.type = UE_INTERRUPT,
+		.endpoint = UE_ADDR_ANY,
+		.direction = UE_DIR_TX,
+		.if_index = 1,
+		.bufsize = CDCE_IND_SIZE_MAX,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,.no_pipe_ok = 1,},
+		.callback = cdce_intr_write_callback,
+		.timeout = 10000,	/* 10 seconds */
+		.usb_mode = USB_MODE_DEVICE,
 	},
 };
 
@@ -417,8 +411,8 @@ cdce_start(struct usb2_ether *ue)
 	/*
 	 * Start the USB transfers, if not already started:
 	 */
-	usb2_transfer_start(sc->sc_xfer[CDCE_BULK_B]);
-	usb2_transfer_start(sc->sc_xfer[CDCE_BULK_A]);
+	usb2_transfer_start(sc->sc_xfer[CDCE_BULK_TX]);
+	usb2_transfer_start(sc->sc_xfer[CDCE_BULK_RX]);
 }
 
 static void
@@ -558,13 +552,11 @@ cdce_init(struct usb2_ether *ue)
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
 
 	/* start interrupt transfer */
-	usb2_transfer_start(sc->sc_xfer[CDCE_INTR]);
+	usb2_transfer_start(sc->sc_xfer[CDCE_INTR_RX]);
+	usb2_transfer_start(sc->sc_xfer[CDCE_INTR_TX]);
 
 	/* stall data write direction, which depends on USB mode */
-	if (usb2_get_mode(sc->sc_ue.ue_udev) == USB_MODE_HOST)
-		usb2_transfer_set_stall(sc->sc_xfer[CDCE_BULK_A]);
-	else
-		usb2_transfer_set_stall(sc->sc_xfer[CDCE_BULK_B]);
+	usb2_transfer_set_stall(sc->sc_xfer[CDCE_BULK_TX]);
 
 	/* start data transfers */
 	cdce_start(ue);
@@ -583,9 +575,10 @@ cdce_stop(struct usb2_ether *ue)
 	/*
 	 * stop all the transfers, if not already stopped:
 	 */
-	usb2_transfer_stop(sc->sc_xfer[CDCE_BULK_A]);
-	usb2_transfer_stop(sc->sc_xfer[CDCE_BULK_B]);
-	usb2_transfer_stop(sc->sc_xfer[CDCE_INTR]);
+	usb2_transfer_stop(sc->sc_xfer[CDCE_BULK_RX]);
+	usb2_transfer_stop(sc->sc_xfer[CDCE_BULK_TX]);
+	usb2_transfer_stop(sc->sc_xfer[CDCE_INTR_RX]);
+	usb2_transfer_stop(sc->sc_xfer[CDCE_INTR_TX]);
 }
 
 static void
diff --git a/sys/dev/usb/net/if_cdcereg.h b/sys/dev/usb/net/if_cdcereg.h
index dac5121..0bcdde7 100644
--- a/sys/dev/usb/net/if_cdcereg.h
+++ b/sys/dev/usb/net/if_cdcereg.h
@@ -39,9 +39,10 @@
 #define	CDCE_IND_SIZE_MAX 32            /* bytes */
 
 enum {
-	CDCE_BULK_A,
-	CDCE_BULK_B,
-	CDCE_INTR,
+	CDCE_BULK_RX,
+	CDCE_BULK_TX,
+	CDCE_INTR_RX,
+	CDCE_INTR_TX,
 	CDCE_N_TRANSFER,
 };
 
diff --git a/sys/dev/usb/net/if_cue.c b/sys/dev/usb/net/if_cue.c
index 54bff1d..10fd038 100644
--- a/sys/dev/usb/net/if_cue.c
+++ b/sys/dev/usb/net/if_cue.c
@@ -121,19 +121,19 @@ static const struct usb2_config cue_config[CUE_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = (MCLBYTES + 2),
-		.mh.flags = {.pipe_bof = 1,},
-		.mh.callback = cue_bulk_write_callback,
-		.mh.timeout = 10000,	/* 10 seconds */
+		.bufsize = (MCLBYTES + 2),
+		.flags = {.pipe_bof = 1,},
+		.callback = cue_bulk_write_callback,
+		.timeout = 10000,	/* 10 seconds */
 	},
 
 	[CUE_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = (MCLBYTES + 2),
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = cue_bulk_read_callback,
+		.bufsize = (MCLBYTES + 2),
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = cue_bulk_read_callback,
 	},
 };
 
diff --git a/sys/dev/usb/net/if_kue.c b/sys/dev/usb/net/if_kue.c
index 8a648a4..3056692 100644
--- a/sys/dev/usb/net/if_kue.c
+++ b/sys/dev/usb/net/if_kue.c
@@ -163,20 +163,20 @@ static const struct usb2_config kue_config[KUE_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = (MCLBYTES + 2 + 64),
-		.mh.flags = {.pipe_bof = 1,},
-		.mh.callback = kue_bulk_write_callback,
-		.mh.timeout = 10000,	/* 10 seconds */
+		.bufsize = (MCLBYTES + 2 + 64),
+		.flags = {.pipe_bof = 1,},
+		.callback = kue_bulk_write_callback,
+		.timeout = 10000,	/* 10 seconds */
 	},
 
 	[KUE_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = (MCLBYTES + 2),
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = kue_bulk_read_callback,
-		.mh.timeout = 0,	/* no timeout */
+		.bufsize = (MCLBYTES + 2),
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = kue_bulk_read_callback,
+		.timeout = 0,	/* no timeout */
 	},
 };
 
diff --git a/sys/dev/usb/net/if_rue.c b/sys/dev/usb/net/if_rue.c
index 73958d1..dc7a31f 100644
--- a/sys/dev/usb/net/if_rue.c
+++ b/sys/dev/usb/net/if_rue.c
@@ -141,29 +141,29 @@ static const struct usb2_config rue_config[RUE_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = MCLBYTES,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = rue_bulk_write_callback,
-		.mh.timeout = 10000,	/* 10 seconds */
+		.bufsize = MCLBYTES,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = rue_bulk_write_callback,
+		.timeout = 10000,	/* 10 seconds */
 	},
 
 	[RUE_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = (MCLBYTES + 4),
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = rue_bulk_read_callback,
-		.mh.timeout = 0,	/* no timeout */
+		.bufsize = (MCLBYTES + 4),
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = rue_bulk_read_callback,
+		.timeout = 0,	/* no timeout */
 	},
 
 	[RUE_INTR_DT_RD] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.callback = rue_intr_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.callback = rue_intr_callback,
 	},
 };
 
diff --git a/sys/dev/usb/net/if_udav.c b/sys/dev/usb/net/if_udav.c
index 76cdef4..5962d19 100644
--- a/sys/dev/usb/net/if_udav.c
+++ b/sys/dev/usb/net/if_udav.c
@@ -101,29 +101,29 @@ static const struct usb2_config udav_config[UDAV_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = (MCLBYTES + 2),
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = udav_bulk_write_callback,
-		.mh.timeout = 10000,	/* 10 seconds */
+		.bufsize = (MCLBYTES + 2),
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = udav_bulk_write_callback,
+		.timeout = 10000,	/* 10 seconds */
 	},
 
 	[UDAV_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = (MCLBYTES + 3),
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = udav_bulk_read_callback,
-		.mh.timeout = 0,	/* no timeout */
+		.bufsize = (MCLBYTES + 3),
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = udav_bulk_read_callback,
+		.timeout = 0,	/* no timeout */
 	},
 
 	[UDAV_INTR_DT_RD] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.callback = udav_intr_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.callback = udav_intr_callback,
 	},
 };
 
diff --git a/sys/dev/usb/serial/u3g.c b/sys/dev/usb/serial/u3g.c
index 4b79aae..19009c6 100644
--- a/sys/dev/usb/serial/u3g.c
+++ b/sys/dev/usb/serial/u3g.c
@@ -34,7 +34,6 @@
 #include <dev/usb/usb.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR u3g_debug
 
@@ -117,18 +116,18 @@ static const struct usb2_config u3g_config[U3G_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = U3G_BSIZE,/* bytes */
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &u3g_write_callback,
+		.bufsize = U3G_BSIZE,/* bytes */
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &u3g_write_callback,
 	},
 
 	[U3G_BULK_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = U3G_BSIZE,/* bytes */
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &u3g_read_callback,
+		.bufsize = U3G_BSIZE,/* bytes */
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &u3g_read_callback,
 	},
 };
 
diff --git a/sys/dev/usb/serial/uark.c b/sys/dev/usb/serial/uark.c
index e0f9db7a5..bc0bace 100644
--- a/sys/dev/usb/serial/uark.c
+++ b/sys/dev/usb/serial/uark.c
@@ -106,18 +106,18 @@ static const struct usb2_config
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = UARK_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &uark_bulk_write_callback,
+		.bufsize = UARK_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &uark_bulk_write_callback,
 	},
 
 	[UARK_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = UARK_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &uark_bulk_read_callback,
+		.bufsize = UARK_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &uark_bulk_read_callback,
 	},
 };
 
diff --git a/sys/dev/usb/serial/ubsa.c b/sys/dev/usb/serial/ubsa.c
index 244ca65..a52c828 100644
--- a/sys/dev/usb/serial/ubsa.c
+++ b/sys/dev/usb/serial/ubsa.c
@@ -188,27 +188,27 @@ static const struct usb2_config ubsa_config[UBSA_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = UBSA_BSIZE,	/* bytes */
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &ubsa_write_callback,
+		.bufsize = UBSA_BSIZE,	/* bytes */
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &ubsa_write_callback,
 	},
 
 	[UBSA_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = UBSA_BSIZE,	/* bytes */
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &ubsa_read_callback,
+		.bufsize = UBSA_BSIZE,	/* bytes */
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &ubsa_read_callback,
 	},
 
 	[UBSA_INTR_DT_RD] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.callback = &ubsa_intr_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.callback = &ubsa_intr_callback,
 	},
 };
 
diff --git a/sys/dev/usb/serial/ubser.c b/sys/dev/usb/serial/ubser.c
index 3aaad78..6ec67ae 100644
--- a/sys/dev/usb/serial/ubser.c
+++ b/sys/dev/usb/serial/ubser.c
@@ -80,7 +80,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
 #include <dev/usb/usb_cdc.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR ubser_debug
 
@@ -157,18 +156,18 @@ static const struct usb2_config ubser_config[UBSER_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &ubser_write_callback,
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &ubser_write_callback,
 	},
 
 	[UBSER_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &ubser_read_callback,
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &ubser_read_callback,
 	},
 };
 
@@ -244,8 +243,8 @@ ubser_attach(device_t dev)
 	req.wIndex[0] = sc->sc_iface_no;
 	req.wIndex[1] = 0;
 	USETW(req.wLength, 1);
-	error = usb2_do_request_flags
-	    (uaa->device, &Giant, &req, &sc->sc_numser,
+	error = usb2_do_request_flags(uaa->device, NULL,
+	    &req, &sc->sc_numser,
 	    0, NULL, USB_DEFAULT_TIMEOUT);
 
 	if (error || (sc->sc_numser == 0)) {
diff --git a/sys/dev/usb/serial/uchcom.c b/sys/dev/usb/serial/uchcom.c
index b99f048..413fb12 100644
--- a/sys/dev/usb/serial/uchcom.c
+++ b/sys/dev/usb/serial/uchcom.c
@@ -233,27 +233,27 @@ static const struct usb2_config uchcom_config_data[UCHCOM_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = UCHCOM_BULK_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &uchcom_write_callback,
+		.bufsize = UCHCOM_BULK_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &uchcom_write_callback,
 	},
 
 	[UCHCOM_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = UCHCOM_BULK_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &uchcom_read_callback,
+		.bufsize = UCHCOM_BULK_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &uchcom_read_callback,
 	},
 
 	[UCHCOM_INTR_DT_RD] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.callback = &uchcom_intr_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.callback = &uchcom_intr_callback,
 	},
 };
 
diff --git a/sys/dev/usb/serial/ucycom.c b/sys/dev/usb/serial/ucycom.c
index ff965ca..a044c00 100644
--- a/sys/dev/usb/serial/ucycom.c
+++ b/sys/dev/usb/serial/ucycom.c
@@ -120,19 +120,18 @@ static const struct usb2_config ucycom_config[UCYCOM_N_TRANSFER] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = (sizeof(struct usb2_device_request) + UCYCOM_MAX_IOLEN),
-		.mh.flags = {},
-		.mh.callback = &ucycom_ctrl_write_callback,
-		.mh.timeout = 1000,	/* 1 second */
+		.bufsize = (sizeof(struct usb2_device_request) + UCYCOM_MAX_IOLEN),
+		.callback = &ucycom_ctrl_write_callback,
+		.timeout = 1000,	/* 1 second */
 	},
 
 	[UCYCOM_INTR_RD] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = UCYCOM_MAX_IOLEN,
-		.mh.callback = &ucycom_intr_read_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = UCYCOM_MAX_IOLEN,
+		.callback = &ucycom_intr_read_callback,
 	},
 };
 
@@ -222,8 +221,7 @@ ucycom_attach(device_t dev)
 
 	/* get report descriptor */
 
-	error = usb2_req_get_hid_desc
-	    (uaa->device, &Giant,
+	error = usb2_req_get_hid_desc(uaa->device, NULL,
 	    &urd_ptr, &urd_len, M_USBDEV,
 	    UCYCOM_IFACE_INDEX);
 
diff --git a/sys/dev/usb/serial/ufoma.c b/sys/dev/usb/serial/ufoma.c
index f29e44c..62471e7 100644
--- a/sys/dev/usb/serial/ufoma.c
+++ b/sys/dev/usb/serial/ufoma.c
@@ -232,29 +232,28 @@ static const struct usb2_config
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = sizeof(struct usb2_cdc_notification),
-		.mh.callback = &ufoma_intr_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = sizeof(struct usb2_cdc_notification),
+		.callback = &ufoma_intr_callback,
 	},
 
 	[UFOMA_CTRL_ENDPT_READ] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = (sizeof(struct usb2_device_request) + UFOMA_CMD_BUF_SIZE),
-		.mh.flags = {.short_xfer_ok = 1,},
-		.mh.callback = &ufoma_ctrl_read_callback,
-		.mh.timeout = 1000,	/* 1 second */
+		.bufsize = (sizeof(struct usb2_device_request) + UFOMA_CMD_BUF_SIZE),
+		.flags = {.short_xfer_ok = 1,},
+		.callback = &ufoma_ctrl_read_callback,
+		.timeout = 1000,	/* 1 second */
 	},
 
 	[UFOMA_CTRL_ENDPT_WRITE] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = (sizeof(struct usb2_device_request) + 1),
-		.mh.flags = {},
-		.mh.callback = &ufoma_ctrl_write_callback,
-		.mh.timeout = 1000,	/* 1 second */
+		.bufsize = (sizeof(struct usb2_device_request) + 1),
+		.callback = &ufoma_ctrl_write_callback,
+		.timeout = 1000,	/* 1 second */
 	},
 };
 
@@ -265,18 +264,18 @@ static const struct usb2_config
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = UFOMA_BULK_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &ufoma_bulk_write_callback,
+		.bufsize = UFOMA_BULK_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &ufoma_bulk_write_callback,
 	},
 
 	[UFOMA_BULK_ENDPT_READ] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = UFOMA_BULK_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &ufoma_bulk_read_callback,
+		.bufsize = UFOMA_BULK_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &ufoma_bulk_read_callback,
 	},
 };
 
diff --git a/sys/dev/usb/serial/uftdi.c b/sys/dev/usb/serial/uftdi.c
index d03638f..c075b06 100644
--- a/sys/dev/usb/serial/uftdi.c
+++ b/sys/dev/usb/serial/uftdi.c
@@ -150,18 +150,18 @@ static const struct usb2_config uftdi_config[UFTDI_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = UFTDI_OBUFSIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &uftdi_write_callback,
+		.bufsize = UFTDI_OBUFSIZE,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &uftdi_write_callback,
 	},
 
 	[UFTDI_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = UFTDI_IBUFSIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &uftdi_read_callback,
+		.bufsize = UFTDI_IBUFSIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &uftdi_read_callback,
 	},
 };
 
diff --git a/sys/dev/usb/serial/ugensa.c b/sys/dev/usb/serial/ugensa.c
index c85e57d..f370e1b 100644
--- a/sys/dev/usb/serial/ugensa.c
+++ b/sys/dev/usb/serial/ugensa.c
@@ -47,7 +47,6 @@
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
 #include <dev/usb/usb_cdc.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR usb2_debug
 
@@ -107,18 +106,18 @@ static const struct usb2_config
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = UGENSA_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &ugensa_bulk_write_callback,
+		.bufsize = UGENSA_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &ugensa_bulk_write_callback,
 	},
 
 	[UGENSA_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = UGENSA_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &ugensa_bulk_read_callback,
+		.bufsize = UGENSA_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &ugensa_bulk_read_callback,
 	},
 };
 
diff --git a/sys/dev/usb/serial/uipaq.c b/sys/dev/usb/serial/uipaq.c
index daebe8b..24dbd25 100644
--- a/sys/dev/usb/serial/uipaq.c
+++ b/sys/dev/usb/serial/uipaq.c
@@ -115,18 +115,18 @@ static const struct usb2_config uipaq_config_data[UIPAQ_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = UIPAQ_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &uipaq_write_callback,
+		.bufsize = UIPAQ_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &uipaq_write_callback,
 	},
 
 	[UIPAQ_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = UIPAQ_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &uipaq_read_callback,
+		.bufsize = UIPAQ_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &uipaq_read_callback,
 	},
 };
 
diff --git a/sys/dev/usb/serial/ulpt.c b/sys/dev/usb/serial/ulpt.c
index 2ecce99..024325e 100644
--- a/sys/dev/usb/serial/ulpt.c
+++ b/sys/dev/usb/serial/ulpt.c
@@ -200,7 +200,8 @@ ulpt_write_callback(struct usb2_xfer *xfer)
 		DPRINTF("no FIFO\n");
 		return;
 	}
-	DPRINTF("state=0x%x\n", USB_GET_STATE(xfer));
+	DPRINTF("state=0x%x actlen=%u\n",
+	    USB_GET_STATE(xfer), xfer->actlen);
 
 	switch (USB_GET_STATE(xfer)) {
 	case USB_ST_TRANSFERRED:
@@ -208,7 +209,6 @@ ulpt_write_callback(struct usb2_xfer *xfer)
 tr_setup:
 		if (usb2_fifo_get_data(f, xfer->frbuffers,
 		    0, xfer->max_data_length, &actlen, 0)) {
-
 			xfer->frlengths[0] = actlen;
 			usb2_start_hardware(xfer);
 		}
@@ -338,27 +338,27 @@ static const struct usb2_config ulpt_config[ULPT_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = ULPT_BSIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,.proxy_buffer = 1},
-		.mh.callback = &ulpt_write_callback,
+		.bufsize = ULPT_BSIZE,
+		.flags = {.pipe_bof = 1,.proxy_buffer = 1},
+		.callback = &ulpt_write_callback,
 	},
 
 	[ULPT_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = ULPT_BSIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,.proxy_buffer = 1},
-		.mh.callback = &ulpt_read_callback,
+		.bufsize = ULPT_BSIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,.proxy_buffer = 1},
+		.callback = &ulpt_read_callback,
 	},
 
 	[ULPT_INTR_DT_RD] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request) + 1,
-		.mh.callback = &ulpt_status_callback,
-		.mh.timeout = 1000,	/* 1 second */
+		.bufsize = sizeof(struct usb2_device_request) + 1,
+		.callback = &ulpt_status_callback,
+		.timeout = 1000,	/* 1 second */
 	},
 };
 
diff --git a/sys/dev/usb/serial/umct.c b/sys/dev/usb/serial/umct.c
index dba38d2..db3ddb4 100644
--- a/sys/dev/usb/serial/umct.c
+++ b/sys/dev/usb/serial/umct.c
@@ -49,7 +49,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
 #include <dev/usb/usb_cdc.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR usb2_debug
 
@@ -138,18 +137,18 @@ static const struct usb2_config umct_config[UMCT_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &umct_write_callback,
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &umct_write_callback,
 	},
 
 	[UMCT_BULK_DT_RD] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.callback = &umct_read_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.callback = &umct_read_callback,
 		.ep_index = 0,		/* first interrupt endpoint */
 	},
 
@@ -157,9 +156,9 @@ static const struct usb2_config umct_config[UMCT_N_TRANSFER] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.callback = &umct_intr_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.callback = &umct_intr_callback,
 		.ep_index = 1,		/* second interrupt endpoint */
 	},
 };
diff --git a/sys/dev/usb/serial/umodem.c b/sys/dev/usb/serial/umodem.c
index 9960635..35186fd 100644
--- a/sys/dev/usb/serial/umodem.c
+++ b/sys/dev/usb/serial/umodem.c
@@ -86,7 +86,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb_error.h>
 #include <dev/usb/usb_cdc.h>
 #include <dev/usb/usb_ioctl.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR umodem_debug
 
@@ -190,9 +189,9 @@ static const struct usb2_config umodem_config[UMODEM_N_TRANSFER] = {
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
 		.if_index = 0,
-		.mh.bufsize = UMODEM_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &umodem_write_callback,
+		.bufsize = UMODEM_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &umodem_write_callback,
 	},
 
 	[UMODEM_BULK_RD] = {
@@ -200,9 +199,9 @@ static const struct usb2_config umodem_config[UMODEM_N_TRANSFER] = {
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
 		.if_index = 0,
-		.mh.bufsize = UMODEM_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &umodem_read_callback,
+		.bufsize = UMODEM_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &umodem_read_callback,
 	},
 
 	[UMODEM_INTR_RD] = {
@@ -210,9 +209,9 @@ static const struct usb2_config umodem_config[UMODEM_N_TRANSFER] = {
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
 		.if_index = 1,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,.no_pipe_ok = 1,},
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.callback = &umodem_intr_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,.no_pipe_ok = 1,},
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.callback = &umodem_intr_callback,
 	},
 };
 
@@ -769,7 +768,7 @@ umodem_set_comm_feature(struct usb2_device *udev, uint8_t iface_no,
 	USETW(req.wLength, UCDC_ABSTRACT_STATE_LENGTH);
 	USETW(ast.wState, state);
 
-	return (usb2_do_request(udev, &Giant, &req, &ast));
+	return (usb2_do_request(udev, NULL, &req, &ast));
 }
 
 static int
diff --git a/sys/dev/usb/serial/umoscom.c b/sys/dev/usb/serial/umoscom.c
index 8ced0c8..f2820ae 100644
--- a/sys/dev/usb/serial/umoscom.c
+++ b/sys/dev/usb/serial/umoscom.c
@@ -204,27 +204,27 @@ static const struct usb2_config umoscom_config_data[UMOSCOM_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = UMOSCOM_BUFSIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &umoscom_write_callback,
+		.bufsize = UMOSCOM_BUFSIZE,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &umoscom_write_callback,
 	},
 
 	[UMOSCOM_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = UMOSCOM_BUFSIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &umoscom_read_callback,
+		.bufsize = UMOSCOM_BUFSIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &umoscom_read_callback,
 	},
 
 	[UMOSCOM_INTR_DT_RD] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.callback = &umoscom_intr_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.callback = &umoscom_intr_callback,
 	},
 };
 
diff --git a/sys/dev/usb/serial/uplcom.c b/sys/dev/usb/serial/uplcom.c
index 44e676b..24a4b36 100644
--- a/sys/dev/usb/serial/uplcom.c
+++ b/sys/dev/usb/serial/uplcom.c
@@ -187,9 +187,9 @@ static const struct usb2_config uplcom_config_data[UPLCOM_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = UPLCOM_BULK_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &uplcom_write_callback,
+		.bufsize = UPLCOM_BULK_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &uplcom_write_callback,
 		.if_index = 0,
 	},
 
@@ -197,9 +197,9 @@ static const struct usb2_config uplcom_config_data[UPLCOM_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = UPLCOM_BULK_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &uplcom_read_callback,
+		.bufsize = UPLCOM_BULK_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &uplcom_read_callback,
 		.if_index = 0,
 	},
 
@@ -207,9 +207,9 @@ static const struct usb2_config uplcom_config_data[UPLCOM_N_TRANSFER] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.callback = &uplcom_intr_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.callback = &uplcom_intr_callback,
 		.if_index = 1,
 	},
 };
@@ -441,7 +441,7 @@ uplcom_reset(struct uplcom_softc *sc, struct usb2_device *udev)
 	req.wIndex[1] = 0;
 	USETW(req.wLength, 0);
 
-	return (usb2_do_request(udev, &Giant, &req, NULL));
+	return (usb2_do_request(udev, NULL, &req, NULL));
 }
 
 struct pl2303x_init {
@@ -485,7 +485,7 @@ uplcom_pl2303x_init(struct usb2_device *udev)
 		USETW(req.wIndex, pl2303x[i].index);
 		USETW(req.wLength, pl2303x[i].length);
 
-		err = usb2_do_request(udev, &Giant, &req, buf);
+		err = usb2_do_request(udev, NULL, &req, buf);
 		if (err) {
 			DPRINTF("error=%s\n", usb2_errstr(err));
 			return (EIO);
diff --git a/sys/dev/usb/serial/usb_serial.c b/sys/dev/usb/serial/usb_serial.c
index 38fd818..6f5558a 100644
--- a/sys/dev/usb/serial/usb_serial.c
+++ b/sys/dev/usb/serial/usb_serial.c
@@ -412,6 +412,12 @@ usb2_com_queue_command(struct usb2_com_softc *sc,
 	if (fn == usb2_com_cfg_close)
 		usb2_proc_mwait(&ssc->sc_tq, t0, t1);
 
+	/*
+	 * In case of multiple configure requests,
+	 * keep track of the last one!
+	 */
+	if (fn == usb2_com_cfg_start_transfers)
+		sc->sc_last_start_xfer = &task->hdr;
 }
 
 static void
@@ -458,7 +464,9 @@ usb2_com_cfg_start_transfers(struct usb2_proc_msg *_task)
 		/* TTY device closed */
 		return;
 	}
-	sc->sc_flag |= UCOM_FLAG_GP_DATA;
+
+	if (_task == sc->sc_last_start_xfer)
+		sc->sc_flag |= UCOM_FLAG_GP_DATA;
 
 	if (sc->sc_callback->usb2_com_start_read) {
 		(sc->sc_callback->usb2_com_start_read) (sc);
diff --git a/sys/dev/usb/serial/usb_serial.h b/sys/dev/usb/serial/usb_serial.h
index c7d57a0..7b5837a 100644
--- a/sys/dev/usb/serial/usb_serial.h
+++ b/sys/dev/usb/serial/usb_serial.h
@@ -152,6 +152,8 @@ struct usb2_com_softc {
 	struct usb2_com_cfg_task	sc_status_task[2];
 	struct usb2_com_param_task	sc_param_task[2];
 	struct cv sc_cv;
+	/* Used to set "UCOM_FLAG_GP_DATA" flag: */
+	struct usb2_proc_msg	*sc_last_start_xfer;
 	const struct usb2_com_callback *sc_callback;
 	struct usb2_com_super_softc *sc_super;
 	struct tty *sc_tty;
diff --git a/sys/dev/usb/serial/uslcom.c b/sys/dev/usb/serial/uslcom.c
index d1351aa..5df0abb 100644
--- a/sys/dev/usb/serial/uslcom.c
+++ b/sys/dev/usb/serial/uslcom.c
@@ -128,18 +128,18 @@ static const struct usb2_config uslcom_config[USLCOM_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = USLCOM_BULK_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &uslcom_write_callback,
+		.bufsize = USLCOM_BULK_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &uslcom_write_callback,
 	},
 
 	[USLCOM_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = USLCOM_BULK_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &uslcom_read_callback,
+		.bufsize = USLCOM_BULK_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &uslcom_read_callback,
 	},
 };
 
diff --git a/sys/dev/usb/serial/uvisor.c b/sys/dev/usb/serial/uvisor.c
index 0fb0658..54de2e2 100644
--- a/sys/dev/usb/serial/uvisor.c
+++ b/sys/dev/usb/serial/uvisor.c
@@ -196,18 +196,18 @@ static const struct usb2_config uvisor_config[UVISOR_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = UVISOR_BUFSIZE,	/* bytes */
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &uvisor_write_callback,
+		.bufsize = UVISOR_BUFSIZE,	/* bytes */
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &uvisor_write_callback,
 	},
 
 	[UVISOR_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = UVISOR_BUFSIZE,	/* bytes */
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &uvisor_read_callback,
+		.bufsize = UVISOR_BUFSIZE,	/* bytes */
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &uvisor_read_callback,
 	},
 };
 
@@ -373,8 +373,8 @@ uvisor_init(struct uvisor_softc *sc, struct usb2_device *udev, struct usb2_confi
 		USETW(req.wValue, 0);
 		USETW(req.wIndex, 0);
 		USETW(req.wLength, UVISOR_CONNECTION_INFO_SIZE);
-		err = usb2_do_request_flags
-		    (udev, &Giant, &req, &coninfo, USB_SHORT_XFER_OK,
+		err = usb2_do_request_flags(udev, NULL,
+		    &req, &coninfo, USB_SHORT_XFER_OK,
 		    &actlen, USB_DEFAULT_TIMEOUT);
 
 		if (err) {
@@ -427,7 +427,7 @@ uvisor_init(struct uvisor_softc *sc, struct usb2_device *udev, struct usb2_confi
 		USETW(req.wLength, UVISOR_GET_PALM_INFORMATION_LEN);
 
 		err = usb2_do_request_flags
-		    (udev, &Giant, &req, &pconinfo, USB_SHORT_XFER_OK,
+		    (udev, NULL, &req, &pconinfo, USB_SHORT_XFER_OK,
 		    &actlen, USB_DEFAULT_TIMEOUT);
 
 		if (err) {
@@ -468,7 +468,7 @@ uvisor_init(struct uvisor_softc *sc, struct usb2_device *udev, struct usb2_confi
 		USETW(req.wIndex, 0);
 		USETW(req.wLength, 1);
 
-		err = usb2_do_request(udev, &Giant, &req, buffer);
+		err = usb2_do_request(udev, NULL, &req, buffer);
 		if (err) {
 			goto done;
 		}
@@ -479,7 +479,7 @@ uvisor_init(struct uvisor_softc *sc, struct usb2_device *udev, struct usb2_confi
 		USETW(req.wValue, 0);
 		USETW(req.wIndex, 0);
 		USETW(req.wLength, 1);
-		err = usb2_do_request(udev, &Giant, &req, buffer);
+		err = usb2_do_request(udev, NULL, &req, buffer);
 		if (err) {
 			goto done;
 		}
@@ -490,7 +490,7 @@ uvisor_init(struct uvisor_softc *sc, struct usb2_device *udev, struct usb2_confi
 	USETW(req.wValue, 0);
 	USETW(req.wIndex, 5);
 	USETW(req.wLength, sizeof(wAvail));
-	err = usb2_do_request(udev, &Giant, &req, &wAvail);
+	err = usb2_do_request(udev, NULL, &req, &wAvail);
 	if (err) {
 		goto done;
 	}
diff --git a/sys/dev/usb/serial/uvscom.c b/sys/dev/usb/serial/uvscom.c
index 6de4f5b..cfda9f8 100644
--- a/sys/dev/usb/serial/uvscom.c
+++ b/sys/dev/usb/serial/uvscom.c
@@ -179,27 +179,27 @@ static const struct usb2_config uvscom_config[UVSCOM_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = UVSCOM_BULK_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = &uvscom_write_callback,
+		.bufsize = UVSCOM_BULK_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = &uvscom_write_callback,
 	},
 
 	[UVSCOM_BULK_DT_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = UVSCOM_BULK_BUF_SIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = &uvscom_read_callback,
+		.bufsize = UVSCOM_BULK_BUF_SIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = &uvscom_read_callback,
 	},
 
 	[UVSCOM_INTR_DT_RD] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.callback = &uvscom_intr_callback,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.callback = &uvscom_intr_callback,
 	},
 };
 
diff --git a/sys/dev/usb/storage/umass.c b/sys/dev/usb/storage/umass.c
index 8a80300..2f2a163 100644
--- a/sys/dev/usb/storage/umass.c
+++ b/sys/dev/usb/storage/umass.c
@@ -458,6 +458,7 @@ static const struct umass_devdescr umass_devdescr[] = {
 	{USB_VENDOR_GENESYS, USB_PRODUCT_GENESYS_GL641USB2IDE, RID_WILDCARD,
 		UMASS_PROTO_SCSI | UMASS_PROTO_BBB,
 		FORCE_SHORT_INQUIRY | NO_START_STOP | IGNORE_RESIDUE
+		    | NO_SYNCHRONIZE_CACHE
 	},
 	{USB_VENDOR_GENESYS, USB_PRODUCT_GENESYS_GL641USB2IDE_2, RID_WILDCARD,
 		UMASS_PROTO_ATAPI | UMASS_PROTO_BBB,
@@ -609,7 +610,7 @@ static const struct umass_devdescr umass_devdescr[] = {
 	},
 	{USB_VENDOR_MYSON, USB_PRODUCT_MYSON_HEDEN, RID_WILDCARD,
 		UMASS_PROTO_SCSI | UMASS_PROTO_BBB,
-		NO_INQUIRY | IGNORE_RESIDUE
+		IGNORE_RESIDUE | NO_SYNCHRONIZE_CACHE
 	},
 	{USB_VENDOR_MYSON, USB_PRODUCT_MYSON_STARREADER, RID_WILDCARD,
 		UMASS_PROTO_SCSI | UMASS_PROTO_BBB,
@@ -847,6 +848,10 @@ static const struct umass_devdescr umass_devdescr[] = {
 		UMASS_PROTO_SCSI | UMASS_PROTO_BBB,
 		NO_QUIRKS
 	},
+	{USB_VENDOR_SUPERTOP, USB_PRODUCT_SUPERTOP_IDE, RID_WILDCARD,
+		UMASS_PROTO_SCSI | UMASS_PROTO_BBB,
+		IGNORE_RESIDUE | NO_SYNCHRONIZE_CACHE
+	},
 	{USB_VENDOR_TAUGA, USB_PRODUCT_TAUGA_CAMERAMATE, RID_WILDCARD,
 		UMASS_PROTO_SCSI,
 		NO_QUIRKS
@@ -1058,93 +1063,87 @@ struct usb2_config umass_bbb_config[UMASS_T_BBB_MAX] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &umass_t_bbb_reset1_callback,
-		.mh.timeout = 5000,	/* 5 seconds */
-		.mh.interval = 500,	/* 500 milliseconds */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &umass_t_bbb_reset1_callback,
+		.timeout = 5000,	/* 5 seconds */
+		.interval = 500,	/* 500 milliseconds */
 	},
 
 	[UMASS_T_BBB_RESET2] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &umass_t_bbb_reset2_callback,
-		.mh.timeout = 5000,	/* 5 seconds */
-		.mh.interval = 50,	/* 50 milliseconds */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &umass_t_bbb_reset2_callback,
+		.timeout = 5000,	/* 5 seconds */
+		.interval = 50,	/* 50 milliseconds */
 	},
 
 	[UMASS_T_BBB_RESET3] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &umass_t_bbb_reset3_callback,
-		.mh.timeout = 5000,	/* 5 seconds */
-		.mh.interval = 50,	/* 50 milliseconds */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &umass_t_bbb_reset3_callback,
+		.timeout = 5000,	/* 5 seconds */
+		.interval = 50,	/* 50 milliseconds */
 	},
 
 	[UMASS_T_BBB_COMMAND] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = sizeof(umass_bbb_cbw_t),
-		.mh.flags = {},
-		.mh.callback = &umass_t_bbb_command_callback,
-		.mh.timeout = 5000,	/* 5 seconds */
+		.bufsize = sizeof(umass_bbb_cbw_t),
+		.callback = &umass_t_bbb_command_callback,
+		.timeout = 5000,	/* 5 seconds */
 	},
 
 	[UMASS_T_BBB_DATA_READ] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = UMASS_BULK_SIZE,
-		.mh.flags = {.proxy_buffer = 1,.short_xfer_ok = 1, UMASS_USB_FLAGS},
-		.mh.callback = &umass_t_bbb_data_read_callback,
-		.mh.timeout = 0,	/* overwritten later */
+		.bufsize = UMASS_BULK_SIZE,
+		.flags = {.proxy_buffer = 1,.short_xfer_ok = 1, UMASS_USB_FLAGS},
+		.callback = &umass_t_bbb_data_read_callback,
+		.timeout = 0,	/* overwritten later */
 	},
 
 	[UMASS_T_BBB_DATA_RD_CS] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &umass_t_bbb_data_rd_cs_callback,
-		.mh.timeout = 5000,	/* 5 seconds */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &umass_t_bbb_data_rd_cs_callback,
+		.timeout = 5000,	/* 5 seconds */
 	},
 
 	[UMASS_T_BBB_DATA_WRITE] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = UMASS_BULK_SIZE,
-		.mh.flags = {.proxy_buffer = 1,.short_xfer_ok = 1, UMASS_USB_FLAGS},
-		.mh.callback = &umass_t_bbb_data_write_callback,
-		.mh.timeout = 0,	/* overwritten later */
+		.bufsize = UMASS_BULK_SIZE,
+		.flags = {.proxy_buffer = 1,.short_xfer_ok = 1, UMASS_USB_FLAGS},
+		.callback = &umass_t_bbb_data_write_callback,
+		.timeout = 0,	/* overwritten later */
 	},
 
 	[UMASS_T_BBB_DATA_WR_CS] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &umass_t_bbb_data_wr_cs_callback,
-		.mh.timeout = 5000,	/* 5 seconds */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &umass_t_bbb_data_wr_cs_callback,
+		.timeout = 5000,	/* 5 seconds */
 	},
 
 	[UMASS_T_BBB_STATUS] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = sizeof(umass_bbb_csw_t),
-		.mh.flags = {.short_xfer_ok = 1,},
-		.mh.callback = &umass_t_bbb_status_callback,
-		.mh.timeout = 5000,	/* ms */
+		.bufsize = sizeof(umass_bbb_csw_t),
+		.flags = {.short_xfer_ok = 1,},
+		.callback = &umass_t_bbb_status_callback,
+		.timeout = 5000,	/* ms */
 	},
 };
 
@@ -1154,105 +1153,98 @@ struct usb2_config umass_cbi_config[UMASS_T_CBI_MAX] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = (sizeof(struct usb2_device_request) +
+		.bufsize = (sizeof(struct usb2_device_request) +
 		    UMASS_CBI_DIAGNOSTIC_CMDLEN),
-		.mh.flags = {},
-		.mh.callback = &umass_t_cbi_reset1_callback,
-		.mh.timeout = 5000,	/* 5 seconds */
-		.mh.interval = 500,	/* 500 milliseconds */
+		.callback = &umass_t_cbi_reset1_callback,
+		.timeout = 5000,	/* 5 seconds */
+		.interval = 500,	/* 500 milliseconds */
 	},
 
 	[UMASS_T_CBI_RESET2] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &umass_t_cbi_reset2_callback,
-		.mh.timeout = 5000,	/* 5 seconds */
-		.mh.interval = 50,	/* 50 milliseconds */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &umass_t_cbi_reset2_callback,
+		.timeout = 5000,	/* 5 seconds */
+		.interval = 50,	/* 50 milliseconds */
 	},
 
 	[UMASS_T_CBI_RESET3] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &umass_t_cbi_reset3_callback,
-		.mh.timeout = 5000,	/* 5 seconds */
-		.mh.interval = 50,	/* 50 milliseconds */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &umass_t_cbi_reset3_callback,
+		.timeout = 5000,	/* 5 seconds */
+		.interval = 50,	/* 50 milliseconds */
 	},
 
 	[UMASS_T_CBI_COMMAND] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = (sizeof(struct usb2_device_request) +
+		.bufsize = (sizeof(struct usb2_device_request) +
 		    UMASS_MAX_CMDLEN),
-		.mh.flags = {},
-		.mh.callback = &umass_t_cbi_command_callback,
-		.mh.timeout = 5000,	/* 5 seconds */
+		.callback = &umass_t_cbi_command_callback,
+		.timeout = 5000,	/* 5 seconds */
 	},
 
 	[UMASS_T_CBI_DATA_READ] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = UMASS_BULK_SIZE,
-		.mh.flags = {.proxy_buffer = 1,.short_xfer_ok = 1, UMASS_USB_FLAGS},
-		.mh.callback = &umass_t_cbi_data_read_callback,
-		.mh.timeout = 0,	/* overwritten later */
+		.bufsize = UMASS_BULK_SIZE,
+		.flags = {.proxy_buffer = 1,.short_xfer_ok = 1, UMASS_USB_FLAGS},
+		.callback = &umass_t_cbi_data_read_callback,
+		.timeout = 0,	/* overwritten later */
 	},
 
 	[UMASS_T_CBI_DATA_RD_CS] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &umass_t_cbi_data_rd_cs_callback,
-		.mh.timeout = 5000,	/* 5 seconds */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &umass_t_cbi_data_rd_cs_callback,
+		.timeout = 5000,	/* 5 seconds */
 	},
 
 	[UMASS_T_CBI_DATA_WRITE] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = UMASS_BULK_SIZE,
-		.mh.flags = {.proxy_buffer = 1,.short_xfer_ok = 1, UMASS_USB_FLAGS},
-		.mh.callback = &umass_t_cbi_data_write_callback,
-		.mh.timeout = 0,	/* overwritten later */
+		.bufsize = UMASS_BULK_SIZE,
+		.flags = {.proxy_buffer = 1,.short_xfer_ok = 1, UMASS_USB_FLAGS},
+		.callback = &umass_t_cbi_data_write_callback,
+		.timeout = 0,	/* overwritten later */
 	},
 
 	[UMASS_T_CBI_DATA_WR_CS] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &umass_t_cbi_data_wr_cs_callback,
-		.mh.timeout = 5000,	/* 5 seconds */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &umass_t_cbi_data_wr_cs_callback,
+		.timeout = 5000,	/* 5 seconds */
 	},
 
 	[UMASS_T_CBI_STATUS] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.flags = {.short_xfer_ok = 1,},
-		.mh.bufsize = sizeof(umass_cbi_sbl_t),
-		.mh.callback = &umass_t_cbi_status_callback,
-		.mh.timeout = 5000,	/* ms */
+		.flags = {.short_xfer_ok = 1,},
+		.bufsize = sizeof(umass_cbi_sbl_t),
+		.callback = &umass_t_cbi_status_callback,
+		.timeout = 5000,	/* ms */
 	},
 
 	[UMASS_T_CBI_RESET4] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &umass_t_cbi_reset4_callback,
-		.mh.timeout = 5000,	/* ms */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &umass_t_cbi_reset4_callback,
+		.timeout = 5000,	/* ms */
 	},
 };
 
@@ -1579,7 +1571,7 @@ umass_attach(device_t dev)
 	 * some devices need a delay after that the configuration value is
 	 * set to function properly:
 	 */
-	usb2_pause_mtx(&Giant, hz);
+	usb2_pause_mtx(NULL, hz);
 
 	/* register the SIM */
 	err = umass_cam_attach_sim(sc);
@@ -1638,7 +1630,7 @@ umass_init_shuttle(struct umass_softc *sc)
 	req.wIndex[0] = sc->sc_iface_no;
 	req.wIndex[1] = 0;
 	USETW(req.wLength, sizeof(status));
-	err = usb2_do_request(sc->sc_udev, &Giant, &req, &status);
+	err = usb2_do_request(sc->sc_udev, NULL, &req, &status);
 
 	DPRINTF(sc, UDMASS_GEN, "Shuttle init returned 0x%02x%02x\n",
 	    status[0], status[1]);
@@ -2022,7 +2014,7 @@ umass_t_bbb_status_callback(struct usb2_xfer *xfer)
 
 		residue = UGETDW(sc->csw.dCSWDataResidue);
 
-		if (!residue) {
+		if ((!residue) || (sc->sc_quirks & IGNORE_RESIDUE)) {
 			residue = (sc->sc_transfer.data_len -
 			    sc->sc_transfer.actlen);
 		}
@@ -2157,7 +2149,7 @@ umass_bbb_get_max_lun(struct umass_softc *sc)
 	req.wIndex[1] = 0;
 	USETW(req.wLength, 1);
 
-	err = usb2_do_request(sc->sc_udev, &Giant, &req, &buf);
+	err = usb2_do_request(sc->sc_udev, NULL, &req, &buf);
 	if (err) {
 		buf = 0;
 
diff --git a/sys/dev/usb/storage/urio.c b/sys/dev/usb/storage/urio.c
index e87a7cd..0c27aa0 100644
--- a/sys/dev/usb/storage/urio.c
+++ b/sys/dev/usb/storage/urio.c
@@ -130,40 +130,38 @@ static const struct usb2_config urio_config[URIO_T_MAX] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = URIO_BSIZE,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,.proxy_buffer = 1,},
-		.mh.callback = &urio_write_callback,
+		.bufsize = URIO_BSIZE,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,.proxy_buffer = 1,},
+		.callback = &urio_write_callback,
 	},
 
 	[URIO_T_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = URIO_BSIZE,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,.proxy_buffer = 1,},
-		.mh.callback = &urio_read_callback,
+		.bufsize = URIO_BSIZE,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,.proxy_buffer = 1,},
+		.callback = &urio_read_callback,
 	},
 
 	[URIO_T_WR_CS] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &urio_write_clear_stall_callback,
-		.mh.timeout = 1000,	/* 1 second */
-		.mh.interval = 50,	/* 50ms */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &urio_write_clear_stall_callback,
+		.timeout = 1000,	/* 1 second */
+		.interval = 50,	/* 50ms */
 	},
 
 	[URIO_T_RD_CS] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &urio_read_clear_stall_callback,
-		.mh.timeout = 1000,	/* 1 second */
-		.mh.interval = 50,	/* 50ms */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &urio_read_clear_stall_callback,
+		.timeout = 1000,	/* 1 second */
+		.interval = 50,	/* 50ms */
 	},
 };
 
diff --git a/sys/dev/usb/storage/ustorage_fs.c b/sys/dev/usb/storage/ustorage_fs.c
index e119232..efea84c 100644
--- a/sys/dev/usb/storage/ustorage_fs.c
+++ b/sys/dev/usb/storage/ustorage_fs.c
@@ -39,7 +39,6 @@
 #include <dev/usb/usb.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR ustorage_fs_debug
 
@@ -60,10 +59,38 @@ SYSCTL_INT(_hw_usb2_ustorage_fs, OID_AUTO, debug, CTLFLAG_RW,
 
 /* Define some limits */
 
-#define	USTORAGE_FS_BULK_SIZE (1 << 17)
-#define	USTORAGE_FS_MAX_LUN 8
-#define	USTORAGE_FS_RELEASE 0x0101
-#define	USTORAGE_FS_RAM_SECT (1 << 13)
+#ifndef USTORAGE_FS_BULK_SIZE 
+#define	USTORAGE_FS_BULK_SIZE (1UL << 17)	/* bytes */
+#endif
+
+#ifndef	USTORAGE_FS_MAX_LUN
+#define	USTORAGE_FS_MAX_LUN	8	/* units */
+#endif
+
+#ifndef USTORAGE_QDATA_MAX
+#define	USTORAGE_QDATA_MAX	40	/* bytes */
+#endif
+
+#define sc_cmd_data sc_cbw.CBWCDB
+
+/*
+ * The SCSI ID string must be exactly 28 characters long
+ * exluding the terminating zero.
+ */
+#ifndef USTORAGE_FS_ID_STRING
+#define	USTORAGE_FS_ID_STRING \
+	"FreeBSD " /* 8 */ \
+	"File-Stor Gadget" /* 16 */ \
+	"0101" /* 4 */
+#endif
+
+/*
+ * The following macro defines the number of
+ * sectors to be allocated for the RAM disk:
+ */
+#ifndef USTORAGE_FS_RAM_SECT
+#define	USTORAGE_FS_RAM_SECT (1UL << 13)
+#endif
 
 static uint8_t *ustorage_fs_ramdisk;
 
@@ -120,7 +147,7 @@ typedef struct {
 
 struct ustorage_fs_lun {
 
-	void   *memory_image;
+	uint8_t	*memory_image;
 
 	uint32_t num_sectors;
 	uint32_t sense_data;
@@ -153,7 +180,6 @@ struct ustorage_fs_softc {
 		uint8_t	cbw_dir;
 		uint8_t	cmd_dir;
 		uint8_t	lun;
-		uint8_t	cmd_data[CBWCDBLENGTH];
 		uint8_t	cmd_len;
 		uint8_t	data_short:1;
 		uint8_t	data_error:1;
@@ -163,13 +189,10 @@ struct ustorage_fs_softc {
 	struct usb2_device *sc_udev;
 	struct usb2_xfer *sc_xfer[USTORAGE_FS_T_BBB_MAX];
 
-	uint32_t sc_unit;
-
-	uint8_t	sc_name[16];
 	uint8_t	sc_iface_no;		/* interface number */
 	uint8_t	sc_last_lun;
 	uint8_t	sc_last_xfer_index;
-	uint8_t	sc_qdata[1024];
+	uint8_t	sc_qdata[USTORAGE_QDATA_MAX];
 };
 
 /* prototypes */
@@ -239,45 +262,50 @@ struct usb2_config ustorage_fs_bbb_config[USTORAGE_FS_T_BBB_MAX] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.md.bufsize = sizeof(ustorage_fs_bbb_cbw_t),
-		.md.flags = {.ext_buffer = 1,},
-		.md.callback = &ustorage_fs_t_bbb_command_callback,
+		.bufsize = sizeof(ustorage_fs_bbb_cbw_t),
+		.flags = {.ext_buffer = 1,},
+		.callback = &ustorage_fs_t_bbb_command_callback,
+		.usb_mode = USB_MODE_DEVICE,
 	},
 
 	[USTORAGE_FS_T_BBB_DATA_DUMP] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.md.bufsize = 0,	/* use wMaxPacketSize */
-		.md.flags = {.proxy_buffer = 1,.short_xfer_ok = 1,},
-		.md.callback = &ustorage_fs_t_bbb_data_dump_callback,
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.flags = {.proxy_buffer = 1,.short_xfer_ok = 1,},
+		.callback = &ustorage_fs_t_bbb_data_dump_callback,
+		.usb_mode = USB_MODE_DEVICE,
 	},
 
 	[USTORAGE_FS_T_BBB_DATA_READ] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.md.bufsize = USTORAGE_FS_BULK_SIZE,
-		.md.flags = {.proxy_buffer = 1,.short_xfer_ok = 1,.ext_buffer = 1},
-		.md.callback = &ustorage_fs_t_bbb_data_read_callback,
+		.bufsize = USTORAGE_FS_BULK_SIZE,
+		.flags = {.proxy_buffer = 1,.short_xfer_ok = 1,.ext_buffer = 1},
+		.callback = &ustorage_fs_t_bbb_data_read_callback,
+		.usb_mode = USB_MODE_DEVICE,
 	},
 
 	[USTORAGE_FS_T_BBB_DATA_WRITE] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.md.bufsize = USTORAGE_FS_BULK_SIZE,
-		.md.flags = {.proxy_buffer = 1,.short_xfer_ok = 1,.ext_buffer = 1},
-		.md.callback = &ustorage_fs_t_bbb_data_write_callback,
+		.bufsize = USTORAGE_FS_BULK_SIZE,
+		.flags = {.proxy_buffer = 1,.short_xfer_ok = 1,.ext_buffer = 1},
+		.callback = &ustorage_fs_t_bbb_data_write_callback,
+		.usb_mode = USB_MODE_DEVICE,
 	},
 
 	[USTORAGE_FS_T_BBB_STATUS] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.md.bufsize = sizeof(ustorage_fs_bbb_csw_t),
-		.md.flags = {.short_xfer_ok = 1,.ext_buffer = 1,},
-		.md.callback = &ustorage_fs_t_bbb_status_callback,
+		.bufsize = sizeof(ustorage_fs_bbb_csw_t),
+		.flags = {.short_xfer_ok = 1,.ext_buffer = 1,},
+		.callback = &ustorage_fs_t_bbb_status_callback,
+		.usb_mode = USB_MODE_DEVICE,
 	},
 };
 
@@ -316,6 +344,7 @@ ustorage_fs_attach(device_t dev)
 	struct usb2_attach_arg *uaa = device_get_ivars(dev);
 	struct usb2_interface_descriptor *id;
 	int err;
+	int unit;
 
 	/*
 	 * NOTE: the softc struct is bzero-ed in device_set_driver.
@@ -325,9 +354,9 @@ ustorage_fs_attach(device_t dev)
 
 	sc->sc_dev = dev;
 	sc->sc_udev = uaa->device;
-	sc->sc_unit = device_get_unit(dev);
+	unit = device_get_unit(dev);
 
-	if (sc->sc_unit == 0) {
+	if (unit == 0) {
 		if (ustorage_fs_ramdisk == NULL) {
 			/*
 			 * allocate a memory image for our ramdisk until
@@ -343,8 +372,6 @@ ustorage_fs_attach(device_t dev)
 		sc->sc_lun[0].num_sectors = USTORAGE_FS_RAM_SECT;
 		sc->sc_lun[0].removable = 1;
 	}
-	snprintf(sc->sc_name, sizeof(sc->sc_name),
-	    "%s", device_get_nameunit(dev));
 
 	device_set_usb2_desc(dev);
 
@@ -525,11 +552,6 @@ ustorage_fs_t_bbb_command_callback(struct usb2_xfer *xfer)
 			    sc->sc_transfer.cmd_len);
 			break;
 		}
-		bcopy(sc->sc_cbw.CBWCDB, sc->sc_transfer.cmd_data,
-		    sc->sc_transfer.cmd_len);
-
-		bzero(sc->sc_cbw.CBWCDB + sc->sc_transfer.cmd_len,
-		    sizeof(sc->sc_cbw.CBWCDB) - sc->sc_transfer.cmd_len);
 
 		error = ustorage_fs_do_cmd(sc);
 		if (error) {
@@ -908,17 +930,17 @@ ustorage_fs_verify(struct ustorage_fs_softc *sc)
 	/*
 	 * Get the starting Logical Block Address
 	 */
-	lba = get_be32(&sc->sc_transfer.cmd_data[2]);
+	lba = get_be32(&sc->sc_cmd_data[2]);
 
 	/*
 	 * We allow DPO (Disable Page Out = don't save data in the cache)
 	 * but we don't implement it.
 	 */
-	if ((sc->sc_transfer.cmd_data[1] & ~0x10) != 0) {
+	if ((sc->sc_cmd_data[1] & ~0x10) != 0) {
 		currlun->sense_data = SS_INVALID_FIELD_IN_CDB;
 		return (1);
 	}
-	vlen = get_be16(&sc->sc_transfer.cmd_data[7]);
+	vlen = get_be16(&sc->sc_cmd_data[7]);
 	if (vlen == 0) {
 		goto done;
 	}
@@ -955,8 +977,6 @@ static uint8_t
 ustorage_fs_inquiry(struct ustorage_fs_softc *sc)
 {
 	uint8_t *buf = sc->sc_transfer.data_ptr;
-	static const char vendor_id[] = "FreeBSD ";
-	static const char product_id[] = "File-Stor Gadget";
 
 	struct ustorage_fs_lun *currlun = sc->sc_transfer.currlun;
 
@@ -978,12 +998,12 @@ ustorage_fs_inquiry(struct ustorage_fs_softc *sc)
 	buf[4] = 31;
 	/* Additional length */
 	/* No special options */
-	/*
-	 * NOTE: We are writing an extra zero here, that is not
-	 * transferred to the peer:
-	 */
-	snprintf(buf + 8, 28 + 1, "%-8s%-16s%04x", vendor_id, product_id,
-	    USTORAGE_FS_RELEASE);
+	/* Copy in ID string */
+	memcpy(buf + 8, USTORAGE_FS_ID_STRING, 28);
+
+#if (USTORAGE_QDATA_MAX < 36)
+#error "(USTORAGE_QDATA_MAX < 36)"
+#endif
 	return (ustorage_fs_min_len(sc, 36, 0 - 1));
 }
 
@@ -1049,10 +1069,13 @@ ustorage_fs_request_sense(struct ustorage_fs_softc *sc)
 	/* Additional sense length */
 	buf[12] = ASC(sd);
 	buf[13] = ASCQ(sd);
+
+#if (USTORAGE_QDATA_MAX < 18)
+#error "(USTORAGE_QDATA_MAX < 18)"
+#endif
 	return (ustorage_fs_min_len(sc, 18, 0 - 1));
 }
 
-
 /*------------------------------------------------------------------------*
  *	ustorage_fs_read_capacity
  *
@@ -1065,22 +1088,25 @@ ustorage_fs_read_capacity(struct ustorage_fs_softc *sc)
 {
 	uint8_t *buf = sc->sc_transfer.data_ptr;
 	struct ustorage_fs_lun *currlun = sc->sc_transfer.currlun;
-	uint32_t lba = get_be32(&sc->sc_transfer.cmd_data[2]);
-	uint8_t pmi = sc->sc_transfer.cmd_data[8];
+	uint32_t lba = get_be32(&sc->sc_cmd_data[2]);
+	uint8_t pmi = sc->sc_cmd_data[8];
 
 	/* Check the PMI and LBA fields */
 	if ((pmi > 1) || ((pmi == 0) && (lba != 0))) {
 		currlun->sense_data = SS_INVALID_FIELD_IN_CDB;
 		return (1);
 	}
-	put_be32(&buf[0], currlun->num_sectors - 1);
 	/* Max logical block */
-	put_be32(&buf[4], 512);
+	put_be32(&buf[0], currlun->num_sectors - 1);
 	/* Block length */
+	put_be32(&buf[4], 512);
+
+#if (USTORAGE_QDATA_MAX < 8)
+#error "(USTORAGE_QDATA_MAX < 8)"
+#endif
 	return (ustorage_fs_min_len(sc, 8, 0 - 1));
 }
 
-
 /*------------------------------------------------------------------------*
  *	ustorage_fs_mode_sense
  *
@@ -1096,7 +1122,7 @@ ustorage_fs_mode_sense(struct ustorage_fs_softc *sc)
 	uint8_t *buf0;
 	uint16_t len;
 	uint16_t limit;
-	uint8_t mscmnd = sc->sc_transfer.cmd_data[0];
+	uint8_t mscmnd = sc->sc_cmd_data[0];
 	uint8_t pc;
 	uint8_t page_code;
 	uint8_t changeable_values;
@@ -1104,13 +1130,13 @@ ustorage_fs_mode_sense(struct ustorage_fs_softc *sc)
 
 	buf0 = buf;
 
-	if ((sc->sc_transfer.cmd_data[1] & ~0x08) != 0) {
+	if ((sc->sc_cmd_data[1] & ~0x08) != 0) {
 		/* Mask away DBD */
 		currlun->sense_data = SS_INVALID_FIELD_IN_CDB;
 		return (1);
 	}
-	pc = sc->sc_transfer.cmd_data[2] >> 6;
-	page_code = sc->sc_transfer.cmd_data[2] & 0x3f;
+	pc = sc->sc_cmd_data[2] >> 6;
+	page_code = sc->sc_cmd_data[2] & 0x3f;
 	if (pc == 3) {
 		currlun->sense_data = SS_SAVING_PARAMETERS_NOT_SUPPORTED;
 		return (1);
@@ -1181,6 +1207,10 @@ ustorage_fs_mode_sense(struct ustorage_fs_softc *sc)
 		buf0[0] = len - 1;
 	else
 		put_be16(buf0, len - 2);
+
+#if (USTORAGE_QDATA_MAX < 24)
+#error "(USTORAGE_QDATA_MAX < 24)"
+#endif
 	return (ustorage_fs_min_len(sc, len, 0 - 1));
 }
 
@@ -1203,9 +1233,9 @@ ustorage_fs_start_stop(struct ustorage_fs_softc *sc)
 		currlun->sense_data = SS_INVALID_COMMAND;
 		return (1);
 	}
-	immed = sc->sc_transfer.cmd_data[1] & 0x01;
-	loej = sc->sc_transfer.cmd_data[4] & 0x02;
-	start = sc->sc_transfer.cmd_data[4] & 0x01;
+	immed = sc->sc_cmd_data[1] & 0x01;
+	loej = sc->sc_cmd_data[4] & 0x02;
+	start = sc->sc_cmd_data[4] & 0x01;
 
 	if (immed || loej || start) {
 		/* compile fix */
@@ -1230,8 +1260,8 @@ ustorage_fs_prevent_allow(struct ustorage_fs_softc *sc)
 		currlun->sense_data = SS_INVALID_COMMAND;
 		return (1);
 	}
-	prevent = sc->sc_transfer.cmd_data[4] & 0x01;
-	if ((sc->sc_transfer.cmd_data[4] & ~0x01) != 0) {
+	prevent = sc->sc_cmd_data[4] & 0x01;
+	if ((sc->sc_cmd_data[4] & ~0x01) != 0) {
 		/* Mask away Prevent */
 		currlun->sense_data = SS_INVALID_FIELD_IN_CDB;
 		return (1);
@@ -1261,12 +1291,16 @@ ustorage_fs_read_format_capacities(struct ustorage_fs_softc *sc)
 	/* Only the Current / Maximum Capacity Descriptor */
 	buf += 4;
 
-	put_be32(&buf[0], currlun->num_sectors);
 	/* Number of blocks */
-	put_be32(&buf[4], 512);
+	put_be32(&buf[0], currlun->num_sectors);
 	/* Block length */
-	buf[4] = 0x02;
+	put_be32(&buf[4], 512);
 	/* Current capacity */
+	buf[4] = 0x02;
+
+#if (USTORAGE_QDATA_MAX < 12)
+#error "(USTORAGE_QDATA_MAX < 12)"
+#endif
 	return (ustorage_fs_min_len(sc, 12, 0 - 1));
 }
 
@@ -1331,18 +1365,18 @@ ustorage_fs_read(struct ustorage_fs_softc *sc)
 	 * Get the starting Logical Block Address and check that it's not
 	 * too big
 	 */
-	if (sc->sc_transfer.cmd_data[0] == SC_READ_6) {
-		lba = (sc->sc_transfer.cmd_data[1] << 16) |
-		    get_be16(&sc->sc_transfer.cmd_data[2]);
+	if (sc->sc_cmd_data[0] == SC_READ_6) {
+		lba = (((uint32_t)sc->sc_cmd_data[1]) << 16) |
+		    get_be16(&sc->sc_cmd_data[2]);
 	} else {
-		lba = get_be32(&sc->sc_transfer.cmd_data[2]);
+		lba = get_be32(&sc->sc_cmd_data[2]);
 
 		/*
 		 * We allow DPO (Disable Page Out = don't save data in the
 		 * cache) and FUA (Force Unit Access = don't read from the
 		 * cache), but we don't implement them.
 		 */
-		if ((sc->sc_transfer.cmd_data[1] & ~0x18) != 0) {
+		if ((sc->sc_cmd_data[1] & ~0x18) != 0) {
 			currlun->sense_data = SS_INVALID_FIELD_IN_CDB;
 			return (1);
 		}
@@ -1359,8 +1393,7 @@ ustorage_fs_read(struct ustorage_fs_softc *sc)
 	file_offset = lba;
 	file_offset <<= 9;
 
-	sc->sc_transfer.data_ptr =
-	    USB_ADD_BYTES(currlun->memory_image, (uint32_t)file_offset);
+	sc->sc_transfer.data_ptr = currlun->memory_image + file_offset;
 
 	return (0);
 }
@@ -1390,11 +1423,11 @@ ustorage_fs_write(struct ustorage_fs_softc *sc)
 	 * Get the starting Logical Block Address and check that it's not
 	 * too big.
 	 */
-	if (sc->sc_transfer.cmd_data[0] == SC_WRITE_6)
-		lba = (sc->sc_transfer.cmd_data[1] << 16) |
-		    get_be16(&sc->sc_transfer.cmd_data[2]);
+	if (sc->sc_cmd_data[0] == SC_WRITE_6)
+		lba = (((uint32_t)sc->sc_cmd_data[1]) << 16) |
+		    get_be16(&sc->sc_cmd_data[2]);
 	else {
-		lba = get_be32(&sc->sc_transfer.cmd_data[2]);
+		lba = get_be32(&sc->sc_cmd_data[2]);
 
 		/*
 		 * We allow DPO (Disable Page Out = don't save data in the
@@ -1402,11 +1435,11 @@ ustorage_fs_write(struct ustorage_fs_softc *sc)
 		 * medium).  We don't implement DPO; we implement FUA by
 		 * performing synchronous output.
 		 */
-		if ((sc->sc_transfer.cmd_data[1] & ~0x18) != 0) {
+		if ((sc->sc_cmd_data[1] & ~0x18) != 0) {
 			currlun->sense_data = SS_INVALID_FIELD_IN_CDB;
 			return (1);
 		}
-		if (sc->sc_transfer.cmd_data[1] & 0x08) {
+		if (sc->sc_cmd_data[1] & 0x08) {
 			/* FUA */
 			/* XXX set SYNC flag here */
 		}
@@ -1424,8 +1457,7 @@ ustorage_fs_write(struct ustorage_fs_softc *sc)
 	file_offset = lba;
 	file_offset <<= 9;
 
-	sc->sc_transfer.data_ptr =
-	    USB_ADD_BYTES(currlun->memory_image, (uint32_t)file_offset);
+	sc->sc_transfer.data_ptr = currlun->memory_image + file_offset;
 
 	return (0);
 }
@@ -1483,7 +1515,7 @@ ustorage_fs_check_cmd(struct ustorage_fs_softc *sc, uint8_t min_cmd_size,
     uint16_t mask, uint8_t needs_medium)
 {
 	struct ustorage_fs_lun *currlun;
-	uint8_t lun = (sc->sc_transfer.cmd_data[1] >> 5);
+	uint8_t lun = (sc->sc_cmd_data[1] >> 5);
 	uint8_t i;
 
 	/* Verify the length of the command itself */
@@ -1494,7 +1526,7 @@ ustorage_fs_check_cmd(struct ustorage_fs_softc *sc, uint8_t min_cmd_size,
 		return (1);
 	}
 	/* Mask away the LUN */
-	sc->sc_transfer.cmd_data[1] &= 0x1f;
+	sc->sc_cmd_data[1] &= 0x1f;
 
 	/* Check if LUN is correct */
 	if (lun != sc->sc_transfer.lun) {
@@ -1504,7 +1536,7 @@ ustorage_fs_check_cmd(struct ustorage_fs_softc *sc, uint8_t min_cmd_size,
 	if (sc->sc_transfer.lun <= sc->sc_last_lun) {
 		sc->sc_transfer.currlun = currlun =
 		    sc->sc_lun + sc->sc_transfer.lun;
-		if (sc->sc_transfer.cmd_data[0] != SC_REQUEST_SENSE) {
+		if (sc->sc_cmd_data[0] != SC_REQUEST_SENSE) {
 			currlun->sense_data = SS_NO_SENSE;
 			currlun->sense_data_info = 0;
 			currlun->info_valid = 0;
@@ -1515,8 +1547,8 @@ ustorage_fs_check_cmd(struct ustorage_fs_softc *sc, uint8_t min_cmd_size,
 		 * else must fail!
 		 */
 		if ((currlun->unit_attention_data != SS_NO_SENSE) &&
-		    (sc->sc_transfer.cmd_data[0] != SC_INQUIRY) &&
-		    (sc->sc_transfer.cmd_data[0] != SC_REQUEST_SENSE)) {
+		    (sc->sc_cmd_data[0] != SC_INQUIRY) &&
+		    (sc->sc_cmd_data[0] != SC_REQUEST_SENSE)) {
 			currlun->sense_data = currlun->unit_attention_data;
 			currlun->unit_attention_data = SS_NO_SENSE;
 			return (1);
@@ -1528,8 +1560,8 @@ ustorage_fs_check_cmd(struct ustorage_fs_softc *sc, uint8_t min_cmd_size,
 		 * INQUIRY and REQUEST SENSE commands are explicitly allowed
 		 * to use unsupported LUNs; all others may not.
 		 */
-		if ((sc->sc_transfer.cmd_data[0] != SC_INQUIRY) &&
-		    (sc->sc_transfer.cmd_data[0] != SC_REQUEST_SENSE)) {
+		if ((sc->sc_cmd_data[0] != SC_INQUIRY) &&
+		    (sc->sc_cmd_data[0] != SC_REQUEST_SENSE)) {
 			return (1);
 		}
 	}
@@ -1539,7 +1571,7 @@ ustorage_fs_check_cmd(struct ustorage_fs_softc *sc, uint8_t min_cmd_size,
 	 * non-zero.
 	 */
 	for (i = 0; i != min_cmd_size; i++) {
-		if (sc->sc_transfer.cmd_data[i] && !(mask & (1 << i))) {
+		if (sc->sc_cmd_data[i] && !(mask & (1UL << i))) {
 			if (currlun) {
 				currlun->sense_data = SS_INVALID_FIELD_IN_CDB;
 			}
@@ -1570,22 +1602,24 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 {
 	uint8_t error = 1;
 	uint8_t i;
+	uint32_t temp;
+	const uint32_t mask9 = (0xFFFFFFFFUL >> 9) << 9;
 
 	/* set default data transfer pointer */
 	sc->sc_transfer.data_ptr = sc->sc_qdata;
 
 	DPRINTF("cmd_data[0]=0x%02x, data_rem=0x%08x\n",
-	    sc->sc_transfer.cmd_data[0], sc->sc_transfer.data_rem);
+	    sc->sc_cmd_data[0], sc->sc_transfer.data_rem);
 
-	switch (sc->sc_transfer.cmd_data[0]) {
+	switch (sc->sc_cmd_data[0]) {
 	case SC_INQUIRY:
 		sc->sc_transfer.cmd_dir = DIR_WRITE;
-		error = ustorage_fs_min_len(sc, sc->sc_transfer.cmd_data[4], 0 - 1);
+		error = ustorage_fs_min_len(sc, sc->sc_cmd_data[4], 0 - 1);
 		if (error) {
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 6,
-		    (1 << 4) | 1, 0);
+		    (1UL << 4) | 1, 0);
 		if (error) {
 			break;
 		}
@@ -1595,12 +1629,12 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 
 	case SC_MODE_SELECT_6:
 		sc->sc_transfer.cmd_dir = DIR_READ;
-		error = ustorage_fs_min_len(sc, sc->sc_transfer.cmd_data[4], 0 - 1);
+		error = ustorage_fs_min_len(sc, sc->sc_cmd_data[4], 0 - 1);
 		if (error) {
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 6,
-		    (1 << 1) | (1 << 4) | 1, 0);
+		    (1UL << 1) | (1UL << 4) | 1, 0);
 		if (error) {
 			break;
 		}
@@ -1611,12 +1645,12 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 	case SC_MODE_SELECT_10:
 		sc->sc_transfer.cmd_dir = DIR_READ;
 		error = ustorage_fs_min_len(sc,
-		    get_be16(&sc->sc_transfer.cmd_data[7]), 0 - 1);
+		    get_be16(&sc->sc_cmd_data[7]), 0 - 1);
 		if (error) {
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 10,
-		    (1 << 1) | (3 << 7) | 1, 0);
+		    (1UL << 1) | (3UL << 7) | 1, 0);
 		if (error) {
 			break;
 		}
@@ -1626,12 +1660,12 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 
 	case SC_MODE_SENSE_6:
 		sc->sc_transfer.cmd_dir = DIR_WRITE;
-		error = ustorage_fs_min_len(sc, sc->sc_transfer.cmd_data[4], 0 - 1);
+		error = ustorage_fs_min_len(sc, sc->sc_cmd_data[4], 0 - 1);
 		if (error) {
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 6,
-		    (1 << 1) | (1 << 2) | (1 << 4) | 1, 0);
+		    (1UL << 1) | (1UL << 2) | (1UL << 4) | 1, 0);
 		if (error) {
 			break;
 		}
@@ -1642,12 +1676,12 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 	case SC_MODE_SENSE_10:
 		sc->sc_transfer.cmd_dir = DIR_WRITE;
 		error = ustorage_fs_min_len(sc,
-		    get_be16(&sc->sc_transfer.cmd_data[7]), 0 - 1);
+		    get_be16(&sc->sc_cmd_data[7]), 0 - 1);
 		if (error) {
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 10,
-		    (1 << 1) | (1 << 2) | (3 << 7) | 1, 0);
+		    (1UL << 1) | (1UL << 2) | (3UL << 7) | 1, 0);
 		if (error) {
 			break;
 		}
@@ -1661,7 +1695,7 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 6,
-		    (1 << 4) | 1, 0);
+		    (1UL << 4) | 1, 0);
 		if (error) {
 			break;
 		}
@@ -1670,15 +1704,15 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 		break;
 
 	case SC_READ_6:
-		i = sc->sc_transfer.cmd_data[4];
+		i = sc->sc_cmd_data[4];
 		sc->sc_transfer.cmd_dir = DIR_WRITE;
-		error = ustorage_fs_min_len(sc,
-		    ((i == 0) ? 256 : i) << 9, 0 - (1 << 9));
+		temp = ((i == 0) ? 256UL : i);
+		error = ustorage_fs_min_len(sc, temp << 9, mask9);
 		if (error) {
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 6,
-		    (7 << 1) | (1 << 4) | 1, 1);
+		    (7UL << 1) | (1UL << 4) | 1, 1);
 		if (error) {
 			break;
 		}
@@ -1688,13 +1722,13 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 
 	case SC_READ_10:
 		sc->sc_transfer.cmd_dir = DIR_WRITE;
-		error = ustorage_fs_min_len(sc,
-		    get_be16(&sc->sc_transfer.cmd_data[7]) << 9, 0 - (1 << 9));
+		temp = get_be16(&sc->sc_cmd_data[7]);
+		error = ustorage_fs_min_len(sc, temp << 9, mask9);
 		if (error) {
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 10,
-		    (1 << 1) | (0xf << 2) | (3 << 7) | 1, 1);
+		    (1UL << 1) | (0xfUL << 2) | (3UL << 7) | 1, 1);
 		if (error) {
 			break;
 		}
@@ -1704,13 +1738,19 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 
 	case SC_READ_12:
 		sc->sc_transfer.cmd_dir = DIR_WRITE;
-		error = ustorage_fs_min_len(sc,
-		    get_be32(&sc->sc_transfer.cmd_data[6]) << 9, 0 - (1 << 9));
+		temp = get_be32(&sc->sc_cmd_data[6]);
+		if (temp >= (1UL << (32 - 9))) {
+			/* numerical overflow */
+			sc->sc_csw.bCSWStatus = CSWSTATUS_FAILED;
+			error = 1;
+			break;
+		}
+		error = ustorage_fs_min_len(sc, temp << 9, mask9);
 		if (error) {
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 12,
-		    (1 << 1) | (0xf << 2) | (0xf << 6) | 1, 1);
+		    (1UL << 1) | (0xfUL << 2) | (0xfUL << 6) | 1, 1);
 		if (error) {
 			break;
 		}
@@ -1721,7 +1761,7 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 	case SC_READ_CAPACITY:
 		sc->sc_transfer.cmd_dir = DIR_WRITE;
 		error = ustorage_fs_check_cmd(sc, 10,
-		    (0xf << 2) | (1 << 8) | 1, 1);
+		    (0xfUL << 2) | (1UL << 8) | 1, 1);
 		if (error) {
 			break;
 		}
@@ -1732,12 +1772,12 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 	case SC_READ_FORMAT_CAPACITIES:
 		sc->sc_transfer.cmd_dir = DIR_WRITE;
 		error = ustorage_fs_min_len(sc,
-		    get_be16(&sc->sc_transfer.cmd_data[7]), 0 - 1);
+		    get_be16(&sc->sc_cmd_data[7]), 0 - 1);
 		if (error) {
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 10,
-		    (3 << 7) | 1, 1);
+		    (3UL << 7) | 1, 1);
 		if (error) {
 			break;
 		}
@@ -1747,12 +1787,12 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 
 	case SC_REQUEST_SENSE:
 		sc->sc_transfer.cmd_dir = DIR_WRITE;
-		error = ustorage_fs_min_len(sc, sc->sc_transfer.cmd_data[4], 0 - 1);
+		error = ustorage_fs_min_len(sc, sc->sc_cmd_data[4], 0 - 1);
 		if (error) {
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 6,
-		    (1 << 4) | 1, 0);
+		    (1UL << 4) | 1, 0);
 		if (error) {
 			break;
 		}
@@ -1766,7 +1806,7 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 6,
-		    (1 << 1) | (1 << 4) | 1, 0);
+		    (1UL << 1) | (1UL << 4) | 1, 0);
 		if (error) {
 			break;
 		}
@@ -1780,7 +1820,7 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 10,
-		    (0xf << 2) | (3 << 7) | 1, 1);
+		    (0xfUL << 2) | (3UL << 7) | 1, 1);
 		if (error) {
 			break;
 		}
@@ -1807,7 +1847,7 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 10,
-		    (1 << 1) | (0xf << 2) | (3 << 7) | 1, 1);
+		    (1UL << 1) | (0xfUL << 2) | (3UL << 7) | 1, 1);
 		if (error) {
 			break;
 		}
@@ -1816,15 +1856,15 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 		break;
 
 	case SC_WRITE_6:
-		i = sc->sc_transfer.cmd_data[4];
+		i = sc->sc_cmd_data[4];
 		sc->sc_transfer.cmd_dir = DIR_READ;
-		error = ustorage_fs_min_len(sc,
-		    ((i == 0) ? 256 : i) << 9, 0 - (1 << 9));
+		temp = ((i == 0) ? 256UL : i);
+		error = ustorage_fs_min_len(sc, temp << 9, mask9);
 		if (error) {
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 6,
-		    (7 << 1) | (1 << 4) | 1, 1);
+		    (7UL << 1) | (1UL << 4) | 1, 1);
 		if (error) {
 			break;
 		}
@@ -1834,13 +1874,13 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 
 	case SC_WRITE_10:
 		sc->sc_transfer.cmd_dir = DIR_READ;
-		error = ustorage_fs_min_len(sc,
-		    get_be16(&sc->sc_transfer.cmd_data[7]) << 9, 0 - (1 << 9));
+		temp = get_be16(&sc->sc_cmd_data[7]);
+		error = ustorage_fs_min_len(sc, temp << 9, mask9);
 		if (error) {
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 10,
-		    (1 << 1) | (0xf << 2) | (3 << 7) | 1, 1);
+		    (1UL << 1) | (0xfUL << 2) | (3UL << 7) | 1, 1);
 		if (error) {
 			break;
 		}
@@ -1850,13 +1890,19 @@ ustorage_fs_do_cmd(struct ustorage_fs_softc *sc)
 
 	case SC_WRITE_12:
 		sc->sc_transfer.cmd_dir = DIR_READ;
-		error = ustorage_fs_min_len(sc,
-		    get_be32(&sc->sc_transfer.cmd_data[6]) << 9, 0 - (1 << 9));
+		temp = get_be32(&sc->sc_cmd_data[6]);
+		if (temp > (mask9 >> 9)) {
+			/* numerical overflow */
+			sc->sc_csw.bCSWStatus = CSWSTATUS_FAILED;
+			error = 1;
+			break;
+		}
+		error = ustorage_fs_min_len(sc, temp << 9, mask9);
 		if (error) {
 			break;
 		}
 		error = ustorage_fs_check_cmd(sc, 12,
-		    (1 << 1) | (0xf << 2) | (0xf << 6) | 1, 1);
+		    (1UL << 1) | (0xfUL << 2) | (0xfUL << 6) | 1, 1);
 		if (error) {
 			break;
 		}
diff --git a/sys/dev/usb/template/usb_template.c b/sys/dev/usb/template/usb_template.c
index 31c853a..dd1a536 100644
--- a/sys/dev/usb/template/usb_template.c
+++ b/sys/dev/usb/template/usb_template.c
@@ -32,7 +32,6 @@
 #include <dev/usb/usb.h>
 #include <dev/usb/usb_cdc.h>
 #include <dev/usb/usb_mfunc.h>
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb_error.h>
 
 #define	USB_DEBUG_VAR usb2_debug
diff --git a/sys/dev/usb/usb.h b/sys/dev/usb/usb.h
index fa5e5dd..3d9dcc3 100644
--- a/sys/dev/usb/usb.h
+++ b/sys/dev/usb/usb.h
@@ -461,8 +461,10 @@ struct usb2_endpoint_descriptor {
 	uByte	bEndpointAddress;
 #define	UE_GET_DIR(a)	((a) & 0x80)
 #define	UE_SET_DIR(a,d)	((a) | (((d)&1) << 7))
-#define	UE_DIR_IN	0x80
-#define	UE_DIR_OUT	0x00
+#define	UE_DIR_IN	0x80		/* IN-token endpoint, fixed */
+#define	UE_DIR_OUT	0x00		/* OUT-token endpoint, fixed */
+#define	UE_DIR_RX	0xfd		/* for internal use only! */
+#define	UE_DIR_TX	0xfe		/* for internal use only! */
 #define	UE_DIR_ANY	0xff		/* for internal use only! */
 #define	UE_ADDR		0x0f
 #define	UE_ADDR_ANY	0xff		/* for internal use only! */
diff --git a/sys/dev/usb/usb_bus.h b/sys/dev/usb/usb_bus.h
index d2fecb9..43cdf35 100644
--- a/sys/dev/usb/usb_bus.h
+++ b/sys/dev/usb/usb_bus.h
@@ -28,8 +28,8 @@
 #define	_USB2_BUS_H_
 
 /*
- * The following structure defines the USB explore message sent to the
- * USB explore process.
+ * The following structure defines the USB explore message sent to the USB
+ * explore process.
  */
 
 struct usb2_bus_msg {
@@ -45,6 +45,17 @@ struct usb2_bus_stat {
 };
 
 /*
+ * The following structure is used to keep the state of a standard
+ * root transfer.
+ */
+struct usb2_sw_transfer {
+	struct usb2_device_request req;
+	uint8_t *ptr;
+	uint16_t len;
+	usb2_error_t err;
+};
+
+/*
  * The following structure defines an USB BUS. There is one USB BUS
  * for every Host or Device controller.
  */
@@ -52,7 +63,7 @@ struct usb2_bus {
 	struct usb2_bus_stat stats_err;
 	struct usb2_bus_stat stats_ok;
 	struct usb2_process explore_proc;
-	struct usb2_process roothub_proc;
+	struct usb2_sw_transfer roothub_req;
 	struct root_hold_token *bus_roothold;
 	/*
 	 * There are two callback processes. One for Giant locked
@@ -64,7 +75,6 @@ struct usb2_bus {
 	struct usb2_bus_msg explore_msg[2];
 	struct usb2_bus_msg detach_msg[2];
 	struct usb2_bus_msg attach_msg[2];
-	struct usb2_bus_msg roothub_msg[2];
 	/*
 	 * This mutex protects the USB hardware:
 	 */
@@ -75,15 +85,16 @@ struct usb2_bus {
 	device_t parent;
 	device_t bdev;			/* filled by HC driver */
 
+#if USB_HAVE_BUSDMA
 	struct usb2_dma_parent_tag dma_parent_tag[1];
 	struct usb2_dma_tag dma_tags[USB_BUS_DMA_TAG_MAX];
-
+#endif
 	struct usb2_bus_methods *methods;	/* filled by HC driver */
 	struct usb2_device **devices;
 
-	uint32_t hw_power_state;	/* see USB_HW_POWER_XXX */
-	uint32_t uframe_usage[USB_HS_MICRO_FRAMES_MAX];
-	uint32_t transfer_count[4];
+	usb2_power_mask_t hw_power_state;	/* see USB_HW_POWER_XXX */
+	usb2_size_t uframe_usage[USB_HS_MICRO_FRAMES_MAX];
+
 	uint16_t isoc_time_last;	/* in milliseconds */
 
 	uint8_t	alloc_failed;		/* Set if memory allocation failed. */
diff --git a/sys/dev/usb/usb_busdma.c b/sys/dev/usb/usb_busdma.c
index 809c3bf..c8f30ed 100644
--- a/sys/dev/usb/usb_busdma.c
+++ b/sys/dev/usb/usb_busdma.c
@@ -27,7 +27,6 @@
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
 #include <dev/usb/usb.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR usb2_debug
 
@@ -42,20 +41,20 @@
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
 
-static void	usb2_dma_tag_create(struct usb2_dma_tag *, uint32_t, uint32_t);
+#if USB_HAVE_BUSDMA
+static void	usb2_dma_tag_create(struct usb2_dma_tag *, usb2_size_t, usb2_size_t);
 static void	usb2_dma_tag_destroy(struct usb2_dma_tag *);
+#endif
 
-#ifdef __FreeBSD__
+#if USB_HAVE_BUSDMA && defined(__FreeBSD__)
 static void	usb2_dma_lock_cb(void *, bus_dma_lock_op_t);
-static int32_t	usb2_m_copy_in_cb(void *, void *, uint32_t);
 static void	usb2_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int);
 static void	usb2_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int);
 static void	usb2_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int,
 		    uint8_t);
 #endif
 
-#ifdef __NetBSD__
-static int32_t	usb2_m_copy_in_cb(void *, caddr_t, uint32_t);
+#if USB_HAVE_BUSDMA && defined(__NetBSD__)
 static void	usb2_pc_common_mem_cb(struct usb2_page_cache *,
 		    bus_dma_segment_t *, int, int, uint8_t);
 #endif
@@ -67,11 +66,12 @@ static void	usb2_pc_common_mem_cb(struct usb2_page_cache *,
  * been properly initialized !
  *------------------------------------------------------------------------*/
 void
-usb2_get_page(struct usb2_page_cache *pc, uint32_t offset,
+usb2_get_page(struct usb2_page_cache *pc, usb2_frlength_t offset,
     struct usb2_page_search *res)
 {
 	struct usb2_page *page;
 
+#if USB_HAVE_BUSDMA
 	if (pc->page_start) {
 
 		/* Case 1 - something has been loaded into DMA */
@@ -106,22 +106,24 @@ usb2_get_page(struct usb2_page_cache *pc, uint32_t offset,
 
 			res->buffer = USB_ADD_BYTES(page->buffer, offset);
 		}
-	} else {
-
-		/* Case 2 - Plain PIO */
-
-		res->buffer = USB_ADD_BYTES(pc->buffer, offset);
-		res->length = 0 - 1;
-		res->physaddr = 0;
+		return;
 	}
+#endif
+	/* Case 2 - Plain PIO */
+
+	res->buffer = USB_ADD_BYTES(pc->buffer, offset);
+	res->length = 0 - 1;
+#if USB_HAVE_BUSDMA
+	res->physaddr = 0;
+#endif
 }
 
 /*------------------------------------------------------------------------*
  *  usb2_copy_in - copy directly to DMA-able memory
  *------------------------------------------------------------------------*/
 void
-usb2_copy_in(struct usb2_page_cache *cache, uint32_t offset,
-    const void *ptr, uint32_t len)
+usb2_copy_in(struct usb2_page_cache *cache, usb2_frlength_t offset,
+    const void *ptr, usb2_frlength_t len)
 {
 	struct usb2_page_search buf_res;
 
@@ -147,9 +149,10 @@ usb2_copy_in(struct usb2_page_cache *cache, uint32_t offset,
  *    0: Success
  * Else: Failure
  *------------------------------------------------------------------------*/
+#if USB_HAVE_USER_IO
 int
-usb2_copy_in_user(struct usb2_page_cache *cache, uint32_t offset,
-    const void *ptr, uint32_t len)
+usb2_copy_in_user(struct usb2_page_cache *cache, usb2_frlength_t offset,
+    const void *ptr, usb2_frlength_t len)
 {
 	struct usb2_page_search buf_res;
 	int error;
@@ -171,16 +174,18 @@ usb2_copy_in_user(struct usb2_page_cache *cache, uint32_t offset,
 	}
 	return (0);			/* success */
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *  usb2_m_copy_in - copy a mbuf chain directly into DMA-able memory
  *------------------------------------------------------------------------*/
+#if USB_HAVE_MBUF
 struct usb2_m_copy_in_arg {
 	struct usb2_page_cache *cache;
-	uint32_t dst_offset;
+	usb2_frlength_t dst_offset;
 };
 
-static int32_t
+static int
 #ifdef __FreeBSD__
 usb2_m_copy_in_cb(void *arg, void *src, uint32_t count)
 #else
@@ -195,21 +200,23 @@ usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count)
 }
 
 void
-usb2_m_copy_in(struct usb2_page_cache *cache, uint32_t dst_offset,
-    struct mbuf *m, uint32_t src_offset, uint32_t src_len)
+usb2_m_copy_in(struct usb2_page_cache *cache, usb2_frlength_t dst_offset,
+    struct mbuf *m, usb2_size_t src_offset, usb2_frlength_t src_len)
 {
 	struct usb2_m_copy_in_arg arg = {cache, dst_offset};
-	register int error;
+	int error;
 
 	error = m_apply(m, src_offset, src_len, &usb2_m_copy_in_cb, &arg);
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *  usb2_uiomove - factored out code
  *------------------------------------------------------------------------*/
+#if USB_HAVE_USER_IO
 int
 usb2_uiomove(struct usb2_page_cache *pc, struct uio *uio,
-    uint32_t pc_offset, uint32_t len)
+    usb2_frlength_t pc_offset, usb2_frlength_t len)
 {
 	struct usb2_page_search res;
 	int error = 0;
@@ -235,13 +242,14 @@ usb2_uiomove(struct usb2_page_cache *pc, struct uio *uio,
 	}
 	return (error);
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *  usb2_copy_out - copy directly from DMA-able memory
  *------------------------------------------------------------------------*/
 void
-usb2_copy_out(struct usb2_page_cache *cache, uint32_t offset,
-    void *ptr, uint32_t len)
+usb2_copy_out(struct usb2_page_cache *cache, usb2_frlength_t offset,
+    void *ptr, usb2_frlength_t len)
 {
 	struct usb2_page_search res;
 
@@ -267,9 +275,10 @@ usb2_copy_out(struct usb2_page_cache *cache, uint32_t offset,
  *    0: Success
  * Else: Failure
  *------------------------------------------------------------------------*/
+#if USB_HAVE_USER_IO
 int
-usb2_copy_out_user(struct usb2_page_cache *cache, uint32_t offset,
-    void *ptr, uint32_t len)
+usb2_copy_out_user(struct usb2_page_cache *cache, usb2_frlength_t offset,
+    void *ptr, usb2_frlength_t len)
 {
 	struct usb2_page_search res;
 	int error;
@@ -291,12 +300,14 @@ usb2_copy_out_user(struct usb2_page_cache *cache, uint32_t offset,
 	}
 	return (0);			/* success */
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *  usb2_bzero - zero DMA-able memory
  *------------------------------------------------------------------------*/
 void
-usb2_bzero(struct usb2_page_cache *cache, uint32_t offset, uint32_t len)
+usb2_bzero(struct usb2_page_cache *cache, usb2_frlength_t offset,
+    usb2_frlength_t len)
 {
 	struct usb2_page_search res;
 
@@ -314,8 +325,7 @@ usb2_bzero(struct usb2_page_cache *cache, uint32_t offset, uint32_t len)
 	}
 }
 
-
-#ifdef __FreeBSD__
+#if USB_HAVE_BUSDMA && defined(__FreeBSD__)
 
 /*------------------------------------------------------------------------*
  *	usb2_dma_lock_cb - dummy callback
@@ -334,7 +344,7 @@ usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
  *------------------------------------------------------------------------*/
 static void
 usb2_dma_tag_create(struct usb2_dma_tag *udt,
-    uint32_t size, uint32_t align)
+    usb2_size_t size, usb2_size_t align)
 {
 	bus_dma_tag_t tag;
 
@@ -399,7 +409,7 @@ usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
 	struct usb2_dma_parent_tag *uptag;
 	struct usb2_page_cache *pc;
 	struct usb2_page *pg;
-	uint32_t rem;
+	usb2_size_t rem;
 	uint8_t owned;
 
 	pc = arg;
@@ -462,7 +472,7 @@ done:
  *------------------------------------------------------------------------*/
 uint8_t
 usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg,
-    uint32_t size, uint32_t align)
+    usb2_size_t size, usb2_size_t align)
 {
 	struct usb2_dma_parent_tag *uptag;
 	struct usb2_dma_tag *utag;
@@ -585,7 +595,7 @@ usb2_pc_free_mem(struct usb2_page_cache *pc)
  * Else: Error
  *------------------------------------------------------------------------*/
 uint8_t
-usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync)
+usb2_pc_load_mem(struct usb2_page_cache *pc, usb2_size_t size, uint8_t sync)
 {
 	/* setup page cache */
 	pc->page_offset_buf = 0;
@@ -686,13 +696,13 @@ usb2_pc_cpu_flush(struct usb2_page_cache *pc)
  * Else: Failure
  *------------------------------------------------------------------------*/
 uint8_t
-usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size)
+usb2_pc_dmamap_create(struct usb2_page_cache *pc, usb2_size_t size)
 {
 	struct usb2_xfer_root *info;
 	struct usb2_dma_tag *utag;
 
 	/* get info */
-	info = pc->tag_parent->info;
+	info = USB_DMATAG_TO_XROOT(pc->tag_parent);
 
 	/* sanity check */
 	if (info == NULL) {
@@ -732,7 +742,7 @@ usb2_pc_dmamap_destroy(struct usb2_page_cache *pc)
 
 #endif
 
-#ifdef __NetBSD__
+#if USB_HAVE_BUSDMA && defined(__NetBSD__)
 
 /*------------------------------------------------------------------------*
  *	usb2_dma_tag_create - allocate a DMA tag
@@ -742,9 +752,9 @@ usb2_pc_dmamap_destroy(struct usb2_page_cache *pc)
  *------------------------------------------------------------------------*/
 static void
 usb2_dma_tag_create(struct usb2_dma_tag *udt,
-    uint32_t size, uint32_t align)
+    usb2_size_t size, usb2_size_t align)
 {
-	uint32_t nseg;
+	usb2_size_t nseg;
 
 	if (align == 1) {
 		nseg = (2 + (size / USB_PAGE_SIZE));
@@ -780,7 +790,7 @@ usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs,
 {
 	struct usb2_dma_parent_tag *uptag;
 	struct usb2_page *pg;
-	uint32_t rem;
+	usb2_size_t rem;
 	uint8_t ext_seg;		/* extend last segment */
 
 	uptag = pc->tag_parent;
@@ -846,7 +856,7 @@ done:
  *------------------------------------------------------------------------*/
 uint8_t
 usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg,
-    uint32_t size, uint32_t align)
+    usb2_size_t size, usb2_size_t align)
 {
 	struct usb2_dma_parent_tag *uptag;
 	struct usb2_dma_tag *utag;
@@ -969,7 +979,7 @@ usb2_pc_free_mem(struct usb2_page_cache *pc)
  * Else: Error
  *------------------------------------------------------------------------*/
 uint8_t
-usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync)
+usb2_pc_load_mem(struct usb2_page_cache *pc, usb2_size_t size, uint8_t sync)
 {
 	int error;
 
@@ -1019,7 +1029,7 @@ usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync)
 void
 usb2_pc_cpu_invalidate(struct usb2_page_cache *pc)
 {
-	uint32_t len;
+	usb2_size_t len;
 
 	len = pc->page_offset_end - pc->page_offset_buf;
 
@@ -1037,7 +1047,7 @@ usb2_pc_cpu_invalidate(struct usb2_page_cache *pc)
 void
 usb2_pc_cpu_flush(struct usb2_page_cache *pc)
 {
-	uint32_t len;
+	usb2_size_t len;
 
 	len = pc->page_offset_end - pc->page_offset_buf;
 
@@ -1057,13 +1067,13 @@ usb2_pc_cpu_flush(struct usb2_page_cache *pc)
  * Else: Failure
  *------------------------------------------------------------------------*/
 uint8_t
-usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size)
+usb2_pc_dmamap_create(struct usb2_page_cache *pc, usb2_size_t size)
 {
 	struct usb2_xfer_root *info;
 	struct usb2_dma_tag *utag;
 
 	/* get info */
-	info = pc->tag_parent->info;
+	info = USB_DMATAG_TO_XROOT(pc->tag_parent);
 
 	/* sanity check */
 	if (info == NULL) {
@@ -1107,12 +1117,14 @@ usb2_pc_dmamap_destroy(struct usb2_page_cache *pc)
 
 #endif
 
+#if USB_HAVE_BUSDMA
+
 /*------------------------------------------------------------------------*
  *	usb2_dma_tag_find - factored out code
  *------------------------------------------------------------------------*/
 struct usb2_dma_tag *
 usb2_dma_tag_find(struct usb2_dma_parent_tag *udpt,
-    uint32_t size, uint32_t align)
+    usb2_size_t size, usb2_size_t align)
 {
 	struct usb2_dma_tag *udt;
 	uint8_t nudt;
@@ -1149,8 +1161,7 @@ void
 usb2_dma_tag_setup(struct usb2_dma_parent_tag *udpt,
     struct usb2_dma_tag *udt, bus_dma_tag_t dmat,
     struct mtx *mtx, usb2_dma_callback_t *func,
-    struct usb2_xfer_root *info, uint8_t ndmabits,
-    uint8_t nudt)
+    uint8_t ndmabits, uint8_t nudt)
 {
 	bzero(udpt, sizeof(*udpt));
 
@@ -1168,7 +1179,6 @@ usb2_dma_tag_setup(struct usb2_dma_parent_tag *udpt,
 
 	/* store some information */
 	udpt->mtx = mtx;
-	udpt->info = info;
 	udpt->func = func;
 	udpt->tag = dmat;
 	udpt->utag_first = udt;
@@ -1223,7 +1233,7 @@ usb2_bdma_work_loop(struct usb2_xfer_queue *pq)
 {
 	struct usb2_xfer_root *info;
 	struct usb2_xfer *xfer;
-	uint32_t nframes;
+	usb2_frcount_t nframes;
 
 	xfer = pq->curr;
 	info = xfer->xroot;
@@ -1239,7 +1249,7 @@ usb2_bdma_work_loop(struct usb2_xfer_queue *pq)
 	}
 	if (!xfer->flags_int.bdma_setup) {
 		struct usb2_page *pg;
-		uint32_t frlength_0;
+		usb2_frlength_t frlength_0;
 		uint8_t isread;
 
 		xfer->flags_int.bdma_setup = 1;
@@ -1350,7 +1360,7 @@ usb2_bdma_done_event(struct usb2_dma_parent_tag *udpt)
 {
 	struct usb2_xfer_root *info;
 
-	info = udpt->info;
+	info = USB_DMATAG_TO_XROOT(udpt);
 
 	mtx_assert(info->xfer_mtx, MA_OWNED);
 
@@ -1372,7 +1382,7 @@ void
 usb2_bdma_pre_sync(struct usb2_xfer *xfer)
 {
 	struct usb2_page_cache *pc;
-	uint32_t nframes;
+	usb2_frcount_t nframes;
 
 	if (xfer->flags_int.isochronous_xfr) {
 		/* only one frame buffer */
@@ -1405,7 +1415,7 @@ void
 usb2_bdma_post_sync(struct usb2_xfer *xfer)
 {
 	struct usb2_page_cache *pc;
-	uint32_t nframes;
+	usb2_frcount_t nframes;
 
 	if (xfer->flags_int.isochronous_xfr) {
 		/* only one frame buffer */
@@ -1424,3 +1434,5 @@ usb2_bdma_post_sync(struct usb2_xfer *xfer)
 		pc++;
 	}
 }
+
+#endif
diff --git a/sys/dev/usb/usb_busdma.h b/sys/dev/usb/usb_busdma.h
index 3c1600b..9653006 100644
--- a/sys/dev/usb/usb_busdma.h
+++ b/sys/dev/usb/usb_busdma.h
@@ -48,6 +48,7 @@
 
 struct usb2_xfer_root;
 struct usb2_dma_parent_tag;
+struct usb2_dma_tag;
 
 /*
  * The following typedef defines the USB DMA load done callback.
@@ -60,8 +61,10 @@ typedef void (usb2_dma_callback_t)(struct usb2_dma_parent_tag *udpt);
  * address of a memory page having size USB_PAGE_SIZE.
  */
 struct usb2_page {
+#if USB_HAVE_BUSDMA
 	bus_size_t physaddr;
 	void   *buffer;			/* non Kernel Virtual Address */
+#endif
 };
 
 /*
@@ -71,8 +74,10 @@ struct usb2_page {
  */
 struct usb2_page_search {
 	void   *buffer;
+#if USB_HAVE_BUSDMA
 	bus_size_t physaddr;
-	uint32_t length;
+#endif
+	usb2_size_t length;
 };
 
 /*
@@ -81,100 +86,108 @@ struct usb2_page_search {
  */
 struct usb2_page_cache {
 
-#ifdef __FreeBSD__
+#if USB_HAVE_BUSDMA && defined(__FreeBSD__)
 	bus_dma_tag_t tag;
 	bus_dmamap_t map;
 #endif
-#ifdef __NetBSD__
+#if USB_HAVE_BUSDMA && defined(__NetBSD__)
 	bus_dma_tag_t tag;
 	bus_dmamap_t map;
 	bus_dma_segment_t *p_seg;
 #endif
+#if USB_HAVE_BUSDMA
 	struct usb2_page *page_start;
+#endif
 	struct usb2_dma_parent_tag *tag_parent;	/* always set */
 	void   *buffer;			/* virtual buffer pointer */
-#ifdef __NetBSD__
+#if USB_HAVE_BUSDMA && defined(_NetBSD__)
 	int	n_seg;
 #endif
-	uint32_t page_offset_buf;
-	uint32_t page_offset_end;
+#if USB_HAVE_BUSDMA
+	usb2_size_t page_offset_buf;
+	usb2_size_t page_offset_end;
 	uint8_t	isread:1;		/* set if we are currently reading
 					 * from the memory. Else write. */
 	uint8_t	ismultiseg:1;		/* set if we can have multiple
 					 * segments */
+#endif
 };
 
 /*
  * The following structure describes the parent USB DMA tag.
  */
+#if USB_HAVE_BUSDMA
 struct usb2_dma_parent_tag {
-#ifdef __FreeBSD__
+#if defined(__FreeBSD__)
 	struct cv cv[1];		/* internal condition variable */
 #endif
-
 	bus_dma_tag_t tag;		/* always set */
 
 	struct mtx *mtx;		/* private mutex, always set */
-	struct usb2_xfer_root *info;	/* used by the callback function */
 	usb2_dma_callback_t *func;	/* load complete callback function */
 	struct usb2_dma_tag *utag_first;/* pointer to first USB DMA tag */
-
 	uint8_t	dma_error;		/* set if DMA load operation failed */
 	uint8_t	dma_bits;		/* number of DMA address lines */
 	uint8_t	utag_max;		/* number of USB DMA tags */
 };
+#else
+struct usb2_dma_parent_tag {};		/* empty struct */
+#endif
 
 /*
  * The following structure describes an USB DMA tag.
  */
+#if USB_HAVE_BUSDMA
 struct usb2_dma_tag {
-#ifdef __NetBSD__
+#if defined(__NetBSD__)
 	bus_dma_segment_t *p_seg;
 #endif
 	struct usb2_dma_parent_tag *tag_parent;
 	bus_dma_tag_t tag;
 
-	uint32_t align;
-	uint32_t size;
-#ifdef __NetBSD__
-	uint32_t n_seg;
+	usb2_size_t align;
+	usb2_size_t size;
+#if defined(__NetBSD__)
+	usb2_size_t n_seg;
 #endif
 };
+#else
+struct usb2_dma_tag {};			/* empty struct */
+#endif
 
 /* function prototypes */
 
 int	usb2_uiomove(struct usb2_page_cache *pc, struct uio *uio,
-	    uint32_t pc_offset, uint32_t len);
+	    usb2_frlength_t pc_offset, usb2_frlength_t len);
 struct usb2_dma_tag *usb2_dma_tag_find(struct usb2_dma_parent_tag *udpt,
-	    uint32_t size, uint32_t align);
+	    usb2_size_t size, usb2_size_t align);
 uint8_t	usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg,
-	    uint32_t size, uint32_t align);
-uint8_t	usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size);
-uint8_t	usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size,
+	    usb2_size_t size, usb2_size_t align);
+uint8_t	usb2_pc_dmamap_create(struct usb2_page_cache *pc, usb2_size_t size);
+uint8_t	usb2_pc_load_mem(struct usb2_page_cache *pc, usb2_size_t size,
 	    uint8_t sync);
 void	usb2_bdma_done_event(struct usb2_dma_parent_tag *udpt);
 void	usb2_bdma_post_sync(struct usb2_xfer *xfer);
 void	usb2_bdma_pre_sync(struct usb2_xfer *xfer);
 void	usb2_bdma_work_loop(struct usb2_xfer_queue *pq);
-void	usb2_bzero(struct usb2_page_cache *cache, uint32_t offset,
-	    uint32_t len);
-void	usb2_copy_in(struct usb2_page_cache *cache, uint32_t offset,
-	    const void *ptr, uint32_t len);
-int	usb2_copy_in_user(struct usb2_page_cache *cache, uint32_t offset,
-	    const void *ptr, uint32_t len);
-void	usb2_copy_out(struct usb2_page_cache *cache, uint32_t offset,
-	    void *ptr, uint32_t len);
-int	usb2_copy_out_user(struct usb2_page_cache *cache, uint32_t offset,
-	    void *ptr, uint32_t len);
+void	usb2_bzero(struct usb2_page_cache *cache, usb2_frlength_t offset,
+	    usb2_frlength_t len);
+void	usb2_copy_in(struct usb2_page_cache *cache, usb2_frlength_t offset,
+	    const void *ptr, usb2_frlength_t len);
+int	usb2_copy_in_user(struct usb2_page_cache *cache, usb2_frlength_t offset,
+	    const void *ptr, usb2_frlength_t len);
+void	usb2_copy_out(struct usb2_page_cache *cache, usb2_frlength_t offset,
+	    void *ptr, usb2_frlength_t len);
+int	usb2_copy_out_user(struct usb2_page_cache *cache, usb2_frlength_t offset,
+	    void *ptr, usb2_frlength_t len);
 void	usb2_dma_tag_setup(struct usb2_dma_parent_tag *udpt,
 	    struct usb2_dma_tag *udt, bus_dma_tag_t dmat, struct mtx *mtx,
-	    usb2_dma_callback_t *func, struct usb2_xfer_root *info,
-	    uint8_t ndmabits, uint8_t nudt);
+	    usb2_dma_callback_t *func, uint8_t ndmabits, uint8_t nudt);
 void	usb2_dma_tag_unsetup(struct usb2_dma_parent_tag *udpt);
-void	usb2_get_page(struct usb2_page_cache *pc, uint32_t offset,
+void	usb2_get_page(struct usb2_page_cache *pc, usb2_frlength_t offset,
 	    struct usb2_page_search *res);
-void	usb2_m_copy_in(struct usb2_page_cache *cache, uint32_t dst_offset,
-	    struct mbuf *m, uint32_t src_offset, uint32_t src_len);
+void	usb2_m_copy_in(struct usb2_page_cache *cache, usb2_frlength_t dst_offset,
+	    struct mbuf *m, usb2_size_t src_offset, usb2_frlength_t src_len);
 void	usb2_pc_cpu_flush(struct usb2_page_cache *pc);
 void	usb2_pc_cpu_invalidate(struct usb2_page_cache *pc);
 void	usb2_pc_dmamap_destroy(struct usb2_page_cache *pc);
diff --git a/sys/dev/usb/usb_compat_linux.c b/sys/dev/usb/usb_compat_linux.c
index 6d6c857..eed72ce 100644
--- a/sys/dev/usb/usb_compat_linux.c
+++ b/sys/dev/usb/usb_compat_linux.c
@@ -25,7 +25,6 @@
  * SUCH DAMAGE.
  */
 
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb.h>
 #include <dev/usb/usb_error.h>
@@ -68,7 +67,7 @@ static usb2_callback_t usb_linux_non_isoc_callback;
 static usb_complete_t usb_linux_wait_complete;
 
 static uint16_t	usb_max_isoc_frames(struct usb_device *);
-static int	usb_start_wait_urb(struct urb *, uint32_t, uint16_t *);
+static int	usb_start_wait_urb(struct urb *, usb2_timeout_t, uint16_t *);
 static const struct usb_device_id *usb_linux_lookup_id(
 		    const struct usb_device_id *, struct usb2_attach_arg *);
 static struct	usb_driver *usb_linux_get_usb_driver(struct usb_linux_softc *);
@@ -565,7 +564,7 @@ usb_clear_halt(struct usb_device *dev, struct usb_host_endpoint *uhe)
  * Linux USB transfers.
  *------------------------------------------------------------------------*/
 static int
-usb_start_wait_urb(struct urb *urb, uint32_t timeout, uint16_t *p_actlen)
+usb_start_wait_urb(struct urb *urb, usb2_timeout_t timeout, uint16_t *p_actlen)
 {
 	int err;
 
@@ -621,7 +620,7 @@ int
 usb_control_msg(struct usb_device *dev, struct usb_host_endpoint *uhe,
     uint8_t request, uint8_t requesttype,
     uint16_t value, uint16_t index, void *data,
-    uint16_t size, uint32_t timeout)
+    uint16_t size, usb2_timeout_t timeout)
 {
 	struct usb2_device_request req;
 	struct urb *urb;
@@ -742,7 +741,7 @@ usb_set_interface(struct usb_device *dev, uint8_t iface_no, uint8_t alt_index)
  *------------------------------------------------------------------------*/
 int
 usb_setup_endpoint(struct usb_device *dev,
-    struct usb_host_endpoint *uhe, uint32_t bufsize)
+    struct usb_host_endpoint *uhe, usb2_size_t bufsize)
 {
 	struct usb2_config cfg[2];
 	uint8_t type = uhe->desc.bmAttributes & UE_XFERTYPE;
@@ -771,10 +770,10 @@ usb_setup_endpoint(struct usb_device *dev,
 		cfg[0].type = type;
 		cfg[0].endpoint = addr & UE_ADDR;
 		cfg[0].direction = addr & (UE_DIR_OUT | UE_DIR_IN);
-		cfg[0].mh.callback = &usb_linux_isoc_callback;
-		cfg[0].mh.bufsize = 0;	/* use wMaxPacketSize */
-		cfg[0].mh.frames = usb_max_isoc_frames(dev);
-		cfg[0].mh.flags.proxy_buffer = 1;
+		cfg[0].callback = &usb_linux_isoc_callback;
+		cfg[0].bufsize = 0;	/* use wMaxPacketSize */
+		cfg[0].frames = usb_max_isoc_frames(dev);
+		cfg[0].flags.proxy_buffer = 1;
 #if 0
 		/*
 		 * The Linux USB API allows non back-to-back
@@ -783,9 +782,9 @@ usb_setup_endpoint(struct usb_device *dev,
 		 * do a copy, and then we need a buffer for
 		 * that. Enable this at your own risk.
 		 */
-		cfg[0].mh.flags.ext_buffer = 1;
+		cfg[0].flags.ext_buffer = 1;
 #endif
-		cfg[0].mh.flags.short_xfer_ok = 1;
+		cfg[0].flags.short_xfer_ok = 1;
 
 		bcopy(cfg, cfg + 1, sizeof(*cfg));
 
@@ -805,11 +804,11 @@ usb_setup_endpoint(struct usb_device *dev,
 		cfg[0].type = type;
 		cfg[0].endpoint = addr & UE_ADDR;
 		cfg[0].direction = addr & (UE_DIR_OUT | UE_DIR_IN);
-		cfg[0].mh.callback = &usb_linux_non_isoc_callback;
-		cfg[0].mh.bufsize = bufsize;
-		cfg[0].mh.flags.ext_buffer = 1;	/* enable zero-copy */
-		cfg[0].mh.flags.proxy_buffer = 1;
-		cfg[0].mh.flags.short_xfer_ok = 1;
+		cfg[0].callback = &usb_linux_non_isoc_callback;
+		cfg[0].bufsize = bufsize;
+		cfg[0].flags.ext_buffer = 1;	/* enable zero-copy */
+		cfg[0].flags.proxy_buffer = 1;
+		cfg[0].flags.short_xfer_ok = 1;
 
 		if (usb2_transfer_setup(dev->bsd_udev, &uhe->bsd_iface_index,
 		    uhe->bsd_xfer, cfg, 1, uhe, &Giant)) {
@@ -837,7 +836,7 @@ usb_linux_create_usb_device(struct usb2_device *udev, device_t dev)
 	struct usb_interface *p_ui = NULL;
 	struct usb_host_interface *p_uhi = NULL;
 	struct usb_host_endpoint *p_uhe = NULL;
-	uint32_t size;
+	usb2_size_t size;
 	uint16_t niface_total;
 	uint16_t nedesc;
 	uint16_t iface_no_curr;
@@ -972,7 +971,7 @@ struct urb *
 usb_alloc_urb(uint16_t iso_packets, uint16_t mem_flags)
 {
 	struct urb *urb;
-	uint32_t size;
+	usb2_size_t size;
 
 	if (iso_packets == 0xFFFF) {
 		/*
@@ -1103,7 +1102,7 @@ usb_ifnum_to_if(struct usb_device *dev, uint8_t iface_no)
  *	usb_buffer_alloc
  *------------------------------------------------------------------------*/
 void   *
-usb_buffer_alloc(struct usb_device *dev, uint32_t size, uint16_t mem_flags, uint8_t *dma_addr)
+usb_buffer_alloc(struct usb_device *dev, usb2_size_t size, uint16_t mem_flags, uint8_t *dma_addr)
 {
 	return (malloc(size, M_USBDEV, M_WAITOK | M_ZERO));
 }
@@ -1194,7 +1193,7 @@ usb_linux_free_device(struct usb_device *dev)
  *	usb_buffer_free
  *------------------------------------------------------------------------*/
 void
-usb_buffer_free(struct usb_device *dev, uint32_t size,
+usb_buffer_free(struct usb_device *dev, usb2_size_t size,
     void *addr, uint8_t dma_addr)
 {
 	free(addr, M_USBDEV);
@@ -1327,9 +1326,9 @@ usb_linux_complete(struct usb2_xfer *xfer)
 static void
 usb_linux_isoc_callback(struct usb2_xfer *xfer)
 {
-	uint32_t max_frame = xfer->max_frame_size;
-	uint32_t offset;
-	uint16_t x;
+	usb2_frlength_t max_frame = xfer->max_frame_size;
+	usb2_frlength_t offset;
+	usb2_frcount_t x;
 	struct urb *urb = xfer->priv_fifo;
 	struct usb_host_endpoint *uhe = xfer->priv_sc;
 	struct usb_iso_packet_descriptor *uipd;
@@ -1501,7 +1500,7 @@ usb_linux_non_isoc_callback(struct usb2_xfer *xfer)
 	struct urb *urb = xfer->priv_fifo;
 	struct usb_host_endpoint *uhe = xfer->priv_sc;
 	uint8_t *ptr;
-	uint32_t max_bulk = xfer->max_data_length;
+	usb2_frlength_t max_bulk = xfer->max_data_length;
 	uint8_t data_frame = xfer->flags_int.control_xfr ? 1 : 0;
 
 	DPRINTF("\n");
diff --git a/sys/dev/usb/usb_compat_linux.h b/sys/dev/usb/usb_compat_linux.h
index 8ebb7e3..c63d82e 100644
--- a/sys/dev/usb/usb_compat_linux.h
+++ b/sys/dev/usb/usb_compat_linux.h
@@ -321,7 +321,7 @@ struct usb_host_endpoint {
 
 	uint8_t *extra;			/* Extra descriptors */
 
-	uint32_t fbsd_buf_size;
+	usb2_frlength_t fbsd_buf_size;
 
 	uint16_t extralen;
 
@@ -406,10 +406,10 @@ struct urb {
 	void   *context;		/* (in) context for completion */
 	usb_complete_t *complete;	/* (in) completion routine */
 
-	uint32_t transfer_buffer_length;/* (in) data buffer length */
-	uint32_t actual_length;		/* (return) actual transfer length */
-	uint32_t bsd_length_rem;
-	uint32_t timeout;		/* FreeBSD specific */
+	usb2_size_t transfer_buffer_length;/* (in) data buffer length */
+	usb2_size_t bsd_length_rem;
+	usb2_size_t actual_length;	/* (return) actual transfer length */
+	usb2_timeout_t timeout;		/* FreeBSD specific */
 
 	uint16_t transfer_flags;	/* (in) */
 #define	URB_SHORT_NOT_OK	0x0001	/* report short transfers like errors */
@@ -420,8 +420,8 @@ struct urb {
 #define	URB_WAIT_WAKEUP		0x0010	/* custom flags */
 #define	URB_IS_SLEEPING		0x0020	/* custom flags */
 
-	uint16_t start_frame;		/* (modify) start frame (ISO) */
-	uint16_t number_of_packets;	/* (in) number of ISO packets */
+	usb2_frcount_t start_frame;	/* (modify) start frame (ISO) */
+	usb2_frcount_t number_of_packets;	/* (in) number of ISO packets */
 	uint16_t interval;		/* (modify) transfer interval
 					 * (INT/ISO) */
 	uint16_t error_count;		/* (return) number of ISO errors */
@@ -441,11 +441,11 @@ int	usb_unlink_urb(struct urb *urb);
 int	usb_clear_halt(struct usb_device *dev, struct usb_host_endpoint *uhe);
 int	usb_control_msg(struct usb_device *dev, struct usb_host_endpoint *pipe,
 	    uint8_t request, uint8_t requesttype, uint16_t value,
-	    uint16_t index, void *data, uint16_t size, uint32_t timeout);
+	    uint16_t index, void *data, uint16_t size, usb2_timeout_t timeout);
 int	usb_set_interface(struct usb_device *dev, uint8_t ifnum,
 	    uint8_t alternate);
 int	usb_setup_endpoint(struct usb_device *dev,
-	    struct usb_host_endpoint *uhe, uint32_t bufsize);
+	    struct usb_host_endpoint *uhe, usb2_frlength_t bufsize);
 
 struct usb_host_endpoint *usb_find_host_endpoint(struct usb_device *dev,
 	    uint8_t type, uint8_t ep);
@@ -454,11 +454,11 @@ struct usb_host_interface *usb_altnum_to_altsetting(
 	    const struct usb_interface *intf, uint8_t alt_index);
 struct usb_interface *usb_ifnum_to_if(struct usb_device *dev, uint8_t iface_no);
 
-void   *usb_buffer_alloc(struct usb_device *dev, uint32_t size,
+void   *usb_buffer_alloc(struct usb_device *dev, usb2_size_t size,
 	    uint16_t mem_flags, uint8_t *dma_addr);
 void   *usb_get_intfdata(struct usb_interface *intf);
 
-void	usb_buffer_free(struct usb_device *dev, uint32_t size, void *addr, uint8_t dma_addr);
+void	usb_buffer_free(struct usb_device *dev, usb2_size_t size, void *addr, uint8_t dma_addr);
 void	usb_free_urb(struct urb *urb);
 void	usb_init_urb(struct urb *urb);
 void	usb_kill_urb(struct urb *urb);
diff --git a/sys/dev/usb/usb_controller.h b/sys/dev/usb/usb_controller.h
index ed329d8..08be1c9 100644
--- a/sys/dev/usb/usb_controller.h
+++ b/sys/dev/usb/usb_controller.h
@@ -31,7 +31,7 @@
 
 #define	USB_BUS_DMA_TAG_MAX 8
 
-/* structure prototypes */
+/* structure prototypes  */
 
 struct usb2_bus;
 struct usb2_page;
@@ -45,7 +45,7 @@ struct usb2_endpoint_descriptor;
 
 /* typedefs */
 
-typedef void (usb2_bus_mem_sub_cb_t)(struct usb2_bus *bus, struct usb2_page_cache *pc, struct usb2_page *pg, uint32_t size, uint32_t align);
+typedef void (usb2_bus_mem_sub_cb_t)(struct usb2_bus *bus, struct usb2_page_cache *pc, struct usb2_page *pg, usb2_size_t size, usb2_size_t align);
 typedef void (usb2_bus_mem_cb_t)(struct usb2_bus *bus, usb2_bus_mem_sub_cb_t *scb);
 
 /*
@@ -108,20 +108,15 @@ struct usb2_pipe_methods {
 
 	/* Mandatory USB Device and Host mode callbacks: */
 
-	void    (*open) (struct usb2_xfer *xfer);
-	void    (*close) (struct usb2_xfer *xfer);
+	usb2_callback_t *open;
+	usb2_callback_t *close;
 
-	void    (*enter) (struct usb2_xfer *xfer);
-	void    (*start) (struct usb2_xfer *xfer);
+	usb2_callback_t *enter;
+	usb2_callback_t *start;
 
 	/* Optional */
 
 	void   *info;
-
-	/* Flags */
-
-	uint8_t	enter_is_cancelable:1;
-	uint8_t	start_is_cancelable:1;
 };
 
 /*
@@ -175,7 +170,7 @@ struct usb2_hw_ep_scratch {
  */
 struct usb2_temp_setup {
 	void   *buf;
-	uint32_t size;
+	usb2_size_t size;
 	uint8_t	usb2_speed;
 	uint8_t	self_powered;
 	uint8_t	bNumEndpoints;
@@ -190,7 +185,6 @@ struct usb2_temp_setup {
 void	usb2_bus_mem_flush_all(struct usb2_bus *bus, usb2_bus_mem_cb_t *cb);
 uint8_t	usb2_bus_mem_alloc_all(struct usb2_bus *bus, bus_dma_tag_t dmat, usb2_bus_mem_cb_t *cb);
 void	usb2_bus_mem_free_all(struct usb2_bus *bus, usb2_bus_mem_cb_t *cb);
-void	usb2_bus_roothub_exec(struct usb2_bus *bus);
 uint16_t usb2_isoc_time_expand(struct usb2_bus *bus, uint16_t isoc_time_curr);
 uint16_t usb2_fs_isoc_schedule_isoc_time_expand(struct usb2_device *udev, struct usb2_fs_isoc_schedule **pp_start, struct usb2_fs_isoc_schedule **pp_end, uint16_t isoc_time);
 uint8_t	usb2_fs_isoc_schedule_alloc(struct usb2_fs_isoc_schedule *fss, uint8_t *pstart, uint16_t len);
diff --git a/sys/dev/usb/usb_core.h b/sys/dev/usb/usb_core.h
index 6067815..cb08476 100644
--- a/sys/dev/usb/usb_core.h
+++ b/sys/dev/usb/usb_core.h
@@ -25,23 +25,116 @@
  */
 
 /*
- * Including this file is mandatory for all USB related c-files in the
- * kernel.
+ * Including this file is mandatory for all USB related c-files in the kernel.
  */
 
 #ifndef _USB2_CORE_H_
 #define	_USB2_CORE_H_
 
+#define	USB_STACK_VERSION 2000		/* 2.0 */
+
+/* Allow defines in "opt_usb.h" to override configuration */
+
+#include "opt_usb.h"
+#include "opt_bus.h"
+
 /* Default USB configuration */
 
-#ifndef USB_USE_CONDVAR
-#define	USB_USE_CONDVAR 0
+/*
+ * The following macro defines if the code shall use cv_xxx() instead
+ * of msleep() and wakeup().
+ */
+#ifndef USB_HAVE_CONDVAR
+#define	USB_HAVE_CONDVAR 0
 #endif
 
+/*
+ * The following macro defines if the code shall support
+ * /dev/usb/x.y.z.
+ */
 #ifndef USB_HAVE_UGEN
 #define	USB_HAVE_UGEN 1
 #endif
 
+/*
+ * The following macro defines if the code shall support any forms of
+ * ASCII strings.
+ */
+#ifndef USB_HAVE_STRINGS
+#define	USB_HAVE_STRINGS 1
+#endif
+
+/*
+ * The following macro defines if the code shall support BUS-DMA.
+ */
+#ifndef USB_HAVE_BUSDMA
+#define	USB_HAVE_BUSDMA 1
+#endif
+
+/*
+ * The following macro defines if the code shall support the Linux
+ * compatibility layer.
+ */
+#ifndef USB_HAVE_COMPAT_LINUX
+#define	USB_HAVE_COMPAT_LINUX 1
+#endif
+
+/*
+ * The following macro defines if the code shall support
+ * userland data transfer via copyin() and copyout()
+ */
+#ifndef USB_HAVE_USER_IO
+#define	USB_HAVE_USER_IO 1
+#endif
+
+/*
+ * The following macro defines if the code shall support copy in via
+ * bsd-mbufs to USB.
+ */
+#ifndef USB_HAVE_MBUF
+#define	USB_HAVE_MBUF 1
+#endif
+
+/*
+ * The following macro defines if the code shall compile a table
+ * describing USB vendor and product IDs.
+ */
+#ifndef USB_VERBOSE
+#define	USB_VERBOSE 1
+#endif
+
+/*
+ * The following macro defines if USB debugging support shall be
+ * compiled for the USB core and all drivers.
+ */
+#ifndef USB_DEBUG
+#define	USB_DEBUG 1
+#endif
+
+/*
+ * The following macro defines if USB transaction translator support
+ * shall be supported for the USB HUB and USB controller drivers.
+ */
+#ifndef	USB_HAVE_TT_SUPPORT
+#define	USB_HAVE_TT_SUPPORT 1
+#endif
+
+/*
+ * The following macro defines if the USB power daemon shall
+ * be supported in the USB core.
+ */
+#ifndef	USB_HAVE_POWERD
+#define	USB_HAVE_POWERD 1
+#endif
+
+/*
+ * The following macro defines if the USB autoinstall detection shall
+ * be supported in the USB core.
+ */
+#ifndef USB_HAVE_MSCTEST
+#define	USB_HAVE_MSCTEST 1
+#endif
+
 #ifndef USB_TD_GET_PROC
 #define	USB_TD_GET_PROC(td) (td)->td_proc
 #endif
@@ -72,34 +165,57 @@
 #include <sys/malloc.h>
 #include <sys/priv.h>
 
-#include <dev/usb/usb_mfunc.h>
+#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb_revision.h>
 
 #include "usb_if.h"
-#include "opt_usb.h"
-#include "opt_bus.h"
-
-#define	USB_STACK_VERSION 2000		/* 2.0 */
 
+#ifndef USB_HOST_ALIGN
 #define	USB_HOST_ALIGN    8		/* bytes, must be power of two */
+#endif
 
-#define	USB_ISOC_TIME_MAX 128		/* ms */
+#ifndef USB_FS_ISOC_UFRAME_MAX
 #define	USB_FS_ISOC_UFRAME_MAX 4	/* exclusive unit */
+#endif
 
 #if (USB_FS_ISOC_UFRAME_MAX > 6)
 #error "USB_FS_ISOC_UFRAME_MAX cannot be set higher than 6"
 #endif
 
+#ifndef USB_BUS_MAX
+#define	USB_BUS_MAX 256			/* units */
+#endif
+
+#ifndef USB_MAX_DEVICES
+#define	USB_MAX_DEVICES 128		/* units */
+#endif
+
+#if (USB_MAX_DEVICES < USB_MIN_DEVICES)
+#error "Minimum number of devices is greater than maximum number of devices."
+#endif
+
+#ifndef USB_IFACE_MAX
+#define	USB_IFACE_MAX 32		/* units */
+#endif
+
+#ifndef USB_FIFO_MAX
+#define	USB_FIFO_MAX 128		/* units */
+#endif
+
+#if (USB_FIFO_MAX & 1)
+#error "Number of FIFOs must be odd."
+#endif
+
 #define	USB_MAX_FS_ISOC_FRAMES_PER_XFER (120)	/* units */
 #define	USB_MAX_HS_ISOC_FRAMES_PER_XFER (8*120)	/* units */
 
-#define	USB_MAX_IPACKET		8	/* maximum size of the initial USB
-					 * data packet */
-#ifndef USB_VERBOSE
-#define	USB_VERBOSE 1
+#ifndef USB_HUB_MAX_DEPTH
+#define	USB_HUB_MAX_DEPTH	5
 #endif
 
-#define	USB_HUB_MAX_DEPTH 5
+#ifndef USB_EP0_BUFSIZE
+#define	USB_EP0_BUFSIZE		1024	/* bytes */
+#endif
 
 /* USB transfer states */
 
@@ -157,10 +273,36 @@ struct usb2_xfer_root;
 
 /* typedefs */
 
-typedef uint8_t usb2_error_t;
-
 typedef void (usb2_callback_t)(struct usb2_xfer *);
 
+#ifndef USB_HAVE_USB_ERROR_T
+typedef uint8_t usb2_error_t;		/* see "USB_ERR_XXX" */
+#endif
+
+#ifndef USB_HAVE_TIMEOUT_T
+typedef uint32_t usb2_timeout_t;	/* milliseconds */
+#endif
+
+#ifndef USB_HAVE_FRLENGTH_T
+typedef uint32_t usb2_frlength_t;	/* bytes */
+#endif
+
+#ifndef USB_HAVE_FRCOUNT_T
+typedef uint32_t usb2_frcount_t;	/* units */
+#endif
+
+#ifndef USB_HAVE_SIZE_T
+typedef uint32_t usb2_size_t;		/* bytes */
+#endif
+
+#ifndef USB_HAVE_TICKS_T
+typedef uint32_t usb2_ticks_t;		/* system defined */
+#endif
+
+#ifndef USB_HAVE_POWER_MASK_T
+typedef uint16_t usb2_power_mask_t;	/* see "USB_HW_POWER_XXX" */
+#endif
+
 /* structures */
 
 /*
@@ -226,12 +368,14 @@ struct usb2_xfer_flags_int {
 
 	uint8_t	short_frames_ok:1;	/* filtered version */
 	uint8_t	short_xfer_ok:1;	/* filtered version */
+#if USB_HAVE_BUSDMA
 	uint8_t	bdma_enable:1;		/* filtered version (only set if
 					 * hardware supports DMA) */
 	uint8_t	bdma_no_post_sync:1;	/* set if the USB callback wrapper
 					 * should not do the BUS-DMA post sync
 					 * operation */
 	uint8_t	bdma_setup:1;		/* set if BUS-DMA has been setup */
+#endif
 	uint8_t	isochronous_xfr:1;	/* set if isochronous transfer */
 	uint8_t	usb2_mode:1;		/* shadow copy of "udev->usb2_mode" */
 	uint8_t	curr_dma_set:1;		/* used by USB HC/DC driver */
@@ -240,31 +384,24 @@ struct usb2_xfer_flags_int {
 };
 
 /*
- * The following structure defines the symmetric part of an USB config
- * structure.
- */
-struct usb2_config_sub {
-	usb2_callback_t *callback;	/* USB transfer callback */
-	uint32_t bufsize;		/* total pipe buffer size in bytes */
-	uint32_t frames;		/* maximum number of USB frames */
-	uint16_t interval;		/* interval in milliseconds */
-#define	USB_DEFAULT_INTERVAL	0
-	uint16_t timeout;		/* transfer timeout in milliseconds */
-	struct usb2_xfer_flags flags;	/* transfer flags */
-};
-
-/*
  * The following structure define an USB configuration, that basically
  * is used when setting up an USB transfer.
  */
 struct usb2_config {
-	struct usb2_config_sub mh;	/* parameters for USB_MODE_HOST */
-	struct usb2_config_sub md;	/* parameters for USB_MODE_DEVICE */
+	usb2_callback_t *callback;	/* USB transfer callback */
+	usb2_frlength_t bufsize;	/* total pipe buffer size in bytes */
+	usb2_frcount_t frames;		/* maximum number of USB frames */
+	usb2_timeout_t interval;	/* interval in milliseconds */
+#define	USB_DEFAULT_INTERVAL	0
+	usb2_timeout_t timeout;		/* transfer timeout in milliseconds */
+	struct usb2_xfer_flags flags;	/* transfer flags */
 	uint8_t	type;			/* pipe type */
 	uint8_t	endpoint;		/* pipe number */
 	uint8_t	direction;		/* pipe direction */
 	uint8_t	ep_index;		/* pipe index match to use */
 	uint8_t	if_index;		/* "ifaces" index to use */
+	uint8_t usb_mode;		/* see "USB_MODE_XXX", 
+					 * "USB_MODE_MAX" means any mode! */
 };
 
 /*
@@ -288,29 +425,29 @@ struct usb2_xfer {
 	void   *priv_sc;		/* device driver data pointer 1 */
 	void   *priv_fifo;		/* device driver data pointer 2 */
 	void   *local_buffer;
-	uint32_t *frlengths;
+	usb2_frlength_t *frlengths;
 	struct usb2_page_cache *frbuffers;
 	usb2_callback_t *callback;
 
-	uint32_t max_usb2_frame_size;
-	uint32_t max_data_length;
-	uint32_t sumlen;		/* sum of all lengths in bytes */
-	uint32_t actlen;		/* actual length in bytes */
-	uint32_t timeout;		/* milliseconds */
+	usb2_frlength_t max_hc_frame_size;
+	usb2_frlength_t max_data_length;
+	usb2_frlength_t sumlen;		/* sum of all lengths in bytes */
+	usb2_frlength_t actlen;		/* actual length in bytes */
+	usb2_timeout_t timeout;		/* milliseconds */
 #define	USB_NO_TIMEOUT 0
 #define	USB_DEFAULT_TIMEOUT 5000	/* 5000 ms = 5 seconds */
 
-	uint32_t max_frame_count;	/* initial value of "nframes" after
+	usb2_frcount_t max_frame_count;	/* initial value of "nframes" after
 					 * setup */
-	uint32_t nframes;		/* number of USB frames to transfer */
-	uint32_t aframes;		/* actual number of USB frames
+	usb2_frcount_t nframes;		/* number of USB frames to transfer */
+	usb2_frcount_t aframes;		/* actual number of USB frames
 					 * transferred */
 
 	uint16_t max_packet_size;
 	uint16_t max_frame_size;
 	uint16_t qh_pos;
 	uint16_t isoc_time_complete;	/* in ms */
-	uint16_t interval;		/* milliseconds */
+	usb2_timeout_t interval;	/* milliseconds */
 
 	uint8_t	address;		/* physical USB address */
 	uint8_t	endpoint;		/* physical USB endpoint */
@@ -395,9 +532,9 @@ usb2_error_t usb2_transfer_setup(struct usb2_device *udev,
 	    const struct usb2_config *setup_start, uint16_t n_setup,
 	    void *priv_sc, struct mtx *priv_mtx);
 void	usb2_set_frame_data(struct usb2_xfer *xfer, void *ptr,
-	    uint32_t frindex);
-void	usb2_set_frame_offset(struct usb2_xfer *xfer, uint32_t offset,
-	    uint32_t frindex);
+	    usb2_frcount_t frindex);
+void	usb2_set_frame_offset(struct usb2_xfer *xfer, usb2_frlength_t offset,
+	    usb2_frcount_t frindex);
 void	usb2_start_hardware(struct usb2_xfer *xfer);
 void	usb2_transfer_clear_stall(struct usb2_xfer *xfer);
 void	usb2_transfer_drain(struct usb2_xfer *xfer);
diff --git a/sys/dev/usb/usb_debug.c b/sys/dev/usb/usb_debug.c
index b7eeea7..da317fe 100644
--- a/sys/dev/usb/usb_debug.c
+++ b/sys/dev/usb/usb_debug.c
@@ -25,7 +25,6 @@
  */
 
 #include <dev/usb/usb.h>
-#include <dev/usb/usb_defs.h>
 
 #include <dev/usb/usb_core.h>
 #include <dev/usb/usb_debug.h>
diff --git a/sys/dev/usb/usb_debug.h b/sys/dev/usb/usb_debug.h
index 92dcbd5..3d1f872 100644
--- a/sys/dev/usb/usb_debug.h
+++ b/sys/dev/usb/usb_debug.h
@@ -35,11 +35,6 @@ SYSCTL_DECL(_hw_usb2);
 /* Declare global USB debug variable. */
 extern int usb2_debug;
 
-/* Force debugging until further */
-#ifndef USB_DEBUG
-#define	USB_DEBUG 1
-#endif
-
 /* Check if USB debugging is enabled. */
 #ifdef USB_DEBUG_VAR
 #if (USB_DEBUG != 0)
diff --git a/sys/dev/usb/usb_defs.h b/sys/dev/usb/usb_defs.h
index 64caf39..4f72ea5 100644
--- a/sys/dev/usb/usb_defs.h
+++ b/sys/dev/usb/usb_defs.h
@@ -27,22 +27,16 @@
 #ifndef _USB2_DEFS_H_
 #define	_USB2_DEFS_H_
 
-/* Definition of some USB constants */
+/* Definition of some hardcoded USB constants. */
+
+#define	USB_MAX_IPACKET		8	/* initial USB packet size */
 
-#define	USB_BUS_MAX 256			/* units */
-#define	USB_DEV_MAX 128			/* units */
-#define	USB_IFACE_MAX 32		/* units */
 #define	USB_EP_MAX (2*16)		/* hardcoded */
-#define	USB_FIFO_MAX (4 * USB_EP_MAX)
 
 #define	USB_ROOT_HUB_ADDR 1		/* index */
 
 #define	USB_MIN_DEVICES 2		/* unused + root HUB */
 
-#define	USB_MAX_DEVICES USB_DEV_MAX	/* including virtual root HUB and
-					 * address zero */
-#define	USB_MAX_ENDPOINTS USB_EP_MAX	/* 2 directions on 16 endpoints */
-
 #define	USB_UNCONFIG_INDEX 0xFF		/* internal use only */
 #define	USB_IFACE_INDEX_ANY 0xFF	/* internal use only */
 
@@ -57,20 +51,10 @@
 #define	USB_FS_BYTES_PER_HS_UFRAME 188	/* bytes */
 #define	USB_HS_MICRO_FRAMES_MAX 8	/* units */
 
+#define	USB_ISOC_TIME_MAX 128		/* ms */
+
 /* sanity checks */
 
-#if (USB_FIFO_MAX < USB_EP_MAX)
-#error "There cannot be less FIFOs than USB endpoints."
-#endif
-#if (USB_FIFO_MAX & 1)
-#error "Number of FIFOs must be odd."
-#endif
-#if (USB_EP_MAX < (2*16))
-#error "Number of hardware USB endpoints cannot be less than 32."
-#endif
-#if (USB_MAX_DEVICES < USB_MIN_DEVICES)
-#error "Minimum number of devices is greater than maximum number of devices."
-#endif
 #if (USB_ROOT_HUB_ADDR >= USB_MIN_DEVICES)
 #error "The root hub address must be less than USB_MIN_DEVICES."
 #endif
diff --git a/sys/dev/usb/usb_dev.c b/sys/dev/usb/usb_dev.c
index 512b1f8..d9d81e0 100644
--- a/sys/dev/usb/usb_dev.c
+++ b/sys/dev/usb/usb_dev.c
@@ -29,7 +29,6 @@
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usb_ioctl.h>
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
 
@@ -55,6 +54,8 @@
 
 #include <machine/stdarg.h>
 
+#if USB_HAVE_UGEN
+
 #if USB_DEBUG
 static int usb2_fifo_debug = 0;
 
@@ -72,7 +73,8 @@ SYSCTL_INT(_hw_usb2_dev, OID_AUTO, debug, CTLFLAG_RW,
 
 /* prototypes */
 
-static int	usb2_fifo_open(struct usb2_fifo *, int);
+static int	usb2_fifo_open(struct usb2_cdev_privdata *, 
+		    struct usb2_fifo *, int);
 static void	usb2_fifo_close(struct usb2_fifo *, int);
 static void	usb2_dev_init(void *);
 static void	usb2_dev_init_post(void *);
@@ -87,7 +89,7 @@ static void	usb2_loc_fill(struct usb2_fs_privdata *,
 		    struct usb2_cdev_privdata *);
 static void	usb2_close(void *);
 static usb2_error_t usb2_ref_device(struct usb2_cdev_privdata *, int);
-static usb2_error_t usb2_uref_location(struct usb2_cdev_privdata *);
+static usb2_error_t usb2_usb_ref_device(struct usb2_cdev_privdata *);
 static void	usb2_unref_device(struct usb2_cdev_privdata *);
 
 static d_open_t usb2_open;
@@ -160,7 +162,6 @@ usb2_ref_device(struct usb2_cdev_privdata* cpd, int need_uref)
 {
 	struct usb2_fifo **ppf;
 	struct usb2_fifo *f;
-	int dev_ep_index;
 
 	DPRINTFN(2, "usb2_ref_device, cpd=%p need uref=%d\n", cpd, need_uref);
 
@@ -179,8 +180,23 @@ usb2_ref_device(struct usb2_cdev_privdata* cpd, int need_uref)
 		DPRINTFN(2, "no dev ref\n");
 		goto error;
 	}
+	if (need_uref) {
+		DPRINTFN(2, "ref udev - needed\n");
+		cpd->udev->refcount++;
+		cpd->is_uref = 1;
+
+		mtx_unlock(&usb2_ref_lock);
+
+		/*
+		 * We need to grab the sx-lock before grabbing the
+		 * FIFO refs to avoid deadlock at detach!
+		 */
+		sx_xlock(cpd->udev->default_sx + 1);
+
+		mtx_lock(&usb2_ref_lock);
+	}
+
 	/* check if we are doing an open */
-	dev_ep_index = cpd->ep_addr;
 	if (cpd->fflags == 0) {
 		/* set defaults */
 		cpd->txfifo = NULL;
@@ -200,15 +216,12 @@ usb2_ref_device(struct usb2_cdev_privdata* cpd, int need_uref)
 			cpd->is_write = 1;	/* ref */
 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
 				goto error;
+			if (f->curr_cpd != cpd)
+				goto error;
 			/* check if USB-FS is active */
 			if (f->fs_ep_max != 0) {
 				cpd->is_usbfs = 1;
 			}
-			/*
-			 * Get real endpoint index associated with
-			 * this FIFO:
-			 */
-			dev_ep_index = f->dev_ep_index;
 		} else {
 			cpd->txfifo = NULL;
 			cpd->is_write = 0;	/* no ref */
@@ -222,15 +235,12 @@ usb2_ref_device(struct usb2_cdev_privdata* cpd, int need_uref)
 			cpd->is_read = 1;	/* ref */
 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
 				goto error;
+			if (f->curr_cpd != cpd)
+				goto error;
 			/* check if USB-FS is active */
 			if (f->fs_ep_max != 0) {
 				cpd->is_usbfs = 1;
 			}
-			/*
-			 * Get real endpoint index associated with
-			 * this FIFO:
-			 */
-			dev_ep_index = f->dev_ep_index;
 		} else {
 			cpd->rxfifo = NULL;
 			cpd->is_read = 0;	/* no ref */
@@ -246,31 +256,28 @@ usb2_ref_device(struct usb2_cdev_privdata* cpd, int need_uref)
 		DPRINTFN(2, "ref read\n");
 		cpd->rxfifo->refcount++;
 	}
-	if (need_uref) {
-		DPRINTFN(2, "ref udev - needed\n");
-		cpd->udev->refcount++;
-		cpd->is_uref = 1;
-	}
 	mtx_unlock(&usb2_ref_lock);
 
 	if (cpd->is_uref) {
-		/*
-		 * We are about to alter the bus-state. Apply the
-		 * required locks.
-		 */
-		sx_xlock(cpd->udev->default_sx + 1);
 		mtx_lock(&Giant);	/* XXX */
 	}
 	return (0);
 
 error:
+	if (cpd->is_uref) {
+		sx_unlock(cpd->udev->default_sx + 1);
+		if (--(cpd->udev->refcount) == 0) {
+			usb2_cv_signal(cpd->udev->default_cv + 1);
+		}
+		cpd->is_uref = 0;
+	}
 	mtx_unlock(&usb2_ref_lock);
 	DPRINTFN(2, "fail\n");
 	return (USB_ERR_INVAL);
 }
 
 /*------------------------------------------------------------------------*
- *	usb2_uref_location
+ *	usb2_usb_ref_device
  *
  * This function is used to upgrade an USB reference to include the
  * USB device reference on a USB location.
@@ -280,46 +287,21 @@ error:
  *  Else: Failure.
  *------------------------------------------------------------------------*/
 static usb2_error_t
-usb2_uref_location(struct usb2_cdev_privdata *cpd)
+usb2_usb_ref_device(struct usb2_cdev_privdata *cpd)
 {
 	/*
 	 * Check if we already got an USB reference on this location:
 	 */
-	if (cpd->is_uref) {
+	if (cpd->is_uref)
 		return (0);		/* success */
-	}
-	mtx_lock(&usb2_ref_lock);
-	if (cpd->bus != devclass_get_softc(usb2_devclass_ptr, cpd->bus_index)) {
-		DPRINTFN(2, "bus changed at %u\n", cpd->bus_index);
-		goto error;
-	}
-	if (cpd->udev != cpd->bus->devices[cpd->dev_index]) {
-		DPRINTFN(2, "device changed at %u\n", cpd->dev_index);
-		goto error;
-	}
-	if (cpd->udev->refcount == USB_DEV_REF_MAX) {
-		DPRINTFN(2, "no dev ref\n");
-		goto error;
-	}
-	DPRINTFN(2, "ref udev\n");
-	cpd->udev->refcount++;
-	mtx_unlock(&usb2_ref_lock);
-
-	/* set "uref" */
-	cpd->is_uref = 1;
 
 	/*
-	 * We are about to alter the bus-state. Apply the
-	 * required locks.
+	 * To avoid deadlock at detach we need to drop the FIFO ref
+	 * and re-acquire a new ref!
 	 */
-	sx_xlock(cpd->udev->default_sx + 1);
-	mtx_lock(&Giant);		/* XXX */
-	return (0);
+	usb2_unref_device(cpd);
 
-error:
-	mtx_unlock(&usb2_ref_lock);
-	DPRINTFN(2, "fail\n");
-	return (USB_ERR_INVAL);
+	return (usb2_ref_device(cpd, 1 /* need uref */));
 }
 
 /*------------------------------------------------------------------------*
@@ -434,7 +416,7 @@ usb2_fifo_create(struct usb2_cdev_privdata *cpd)
 					/* wrong endpoint index */
 					continue;
 				}
-				if (f->opened) {
+				if (f->curr_cpd != NULL) {
 					/* FIFO is opened */
 					is_busy = 1;
 					continue;
@@ -451,7 +433,7 @@ usb2_fifo_create(struct usb2_cdev_privdata *cpd)
 					/* wrong endpoint index */
 					continue;
 				}
-				if (f->opened) {
+				if (f->curr_cpd != NULL) {
 					/* FIFO is opened */
 					is_busy = 1;
 					continue;
@@ -466,10 +448,20 @@ usb2_fifo_create(struct usb2_cdev_privdata *cpd)
 	if (no_null == 0) {
 		if (ep >= (USB_EP_MAX / 2)) {
 			/* we don't create any endpoints in this range */
-			DPRINTFN(5, "dev_ep_index out of range\n");
+			DPRINTFN(5, "ep out of range\n");
 			return (is_busy ? EBUSY : EINVAL);
 		}
 	}
+
+	if ((ep != 0) && is_busy) {
+		/*
+		 * Only the default control endpoint is allowed to be
+		 * opened multiple times!
+		 */
+		DPRINTFN(5, "busy\n");
+		return (EBUSY);
+	}
+
 	/* Check TX FIFO */
 	if (is_tx &&
 	    (udev->fifo[n + USB_FIFO_TX] == NULL)) {
@@ -639,7 +631,8 @@ usb2_dev_get_pipe(struct usb2_device *udev, uint8_t ep_index, uint8_t dir)
  * Else: Failure
  *------------------------------------------------------------------------*/
 static int
-usb2_fifo_open(struct usb2_fifo *f, int fflags)
+usb2_fifo_open(struct usb2_cdev_privdata *cpd, 
+    struct usb2_fifo *f, int fflags)
 {
 	int err;
 
@@ -660,11 +653,14 @@ usb2_fifo_open(struct usb2_fifo *f, int fflags)
 
 	/* check if we are already opened */
 	/* we don't need any locks when checking this variable */
-	if (f->opened) {
+	if (f->curr_cpd != NULL) {
 		err = EBUSY;
 		goto done;
 	}
 
+	/* reset short flag before open */
+	f->flag_short = 0;
+
 	/* call open method */
 	err = (f->methods->f_open) (f, fflags);
 	if (err) {
@@ -690,9 +686,9 @@ usb2_fifo_open(struct usb2_fifo *f, int fflags)
 	/* reset ASYNC proc flag */
 	f->async_p = NULL;
 
-	/* flag the fifo as opened to prevent others */
 	mtx_lock(&usb2_ref_lock);
-	f->opened = 1;
+	/* flag the fifo as opened to prevent others */
+	f->curr_cpd = cpd;
 	mtx_unlock(&usb2_ref_lock);
 
 	/* reset queue */
@@ -733,14 +729,14 @@ usb2_fifo_close(struct usb2_fifo *f, int fflags)
 	int err;
 
 	/* check if we are not opened */
-	if (!f->opened) {
+	if (f->curr_cpd == NULL) {
 		/* nothing to do - already closed */
 		return;
 	}
 	mtx_lock(f->priv_mtx);
 
-	/* clear current file flag */
-	f->opened = 0;
+	/* clear current cdev private data pointer */
+	f->curr_cpd = NULL;
 
 	/* check if we are selected */
 	if (f->flag_isselect) {
@@ -813,7 +809,7 @@ usb2_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
 	struct usb2_cdev_privdata *cpd;
 	int err, ep;
 
-	DPRINTFN(2, "fflags=0x%08x\n", fflags);
+	DPRINTFN(2, "%s fflags=0x%08x\n", dev->si_name, fflags);
 
 	KASSERT(fflags & (FREAD|FWRITE), ("invalid open flags"));
 	if (((fflags & FREAD) && !(pd->mode & FREAD)) ||
@@ -834,21 +830,6 @@ usb2_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
 	}
 	cpd->fflags = fflags;	/* access mode for open lifetime */
 
-	/* Check if the endpoint is already open, we always allow EP0 */
-	if (ep > 0) {
-		if ((fflags & FREAD && cpd->udev->ep_rd_opened & (1 << ep)) ||
-		    (fflags & FWRITE && cpd->udev->ep_wr_opened & (1 << ep))) {
-			DPRINTFN(2, "endpoint already open\n");
-			usb2_unref_device(cpd);
-			free(cpd, M_USBDEV);
-			return (EBUSY);
-		}
-		if (fflags & FREAD)
-			cpd->udev->ep_rd_opened |= (1 << ep);
-		if (fflags & FWRITE)
-			cpd->udev->ep_wr_opened |= (1 << ep);
-	}
-
 	/* create FIFOs, if any */
 	err = usb2_fifo_create(cpd);
 	/* check for error */
@@ -859,7 +840,7 @@ usb2_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
 		return (err);
 	}
 	if (fflags & FREAD) {
-		err = usb2_fifo_open(cpd->rxfifo, fflags);
+		err = usb2_fifo_open(cpd, cpd->rxfifo, fflags);
 		if (err) {
 			DPRINTFN(2, "read open failed\n");
 			usb2_unref_device(cpd);
@@ -868,7 +849,7 @@ usb2_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
 		}
 	}
 	if (fflags & FWRITE) {
-		err = usb2_fifo_open(cpd->txfifo, fflags);
+		err = usb2_fifo_open(cpd, cpd->txfifo, fflags);
 		if (err) {
 			DPRINTFN(2, "write open failed\n");
 			if (fflags & FREAD) {
@@ -892,27 +873,20 @@ static void
 usb2_close(void *arg)
 {
 	struct usb2_cdev_privdata *cpd = arg;
-	struct usb2_device *udev;
 	int err;
 
-	DPRINTFN(2, "usb2_close, cpd=%p\n", cpd);
+	DPRINTFN(2, "cpd=%p\n", cpd);
 
 	err = usb2_ref_device(cpd, 1);
 	if (err) {
 		free(cpd, M_USBDEV);
 		return;
 	}
-
-	udev = cpd->udev;
 	if (cpd->fflags & FREAD) {
 		usb2_fifo_close(cpd->rxfifo, cpd->fflags);
-		/* clear read bitmask */
-		udev->ep_rd_opened &= ~(1 << cpd->ep_addr);
 	}
 	if (cpd->fflags & FWRITE) {
 		usb2_fifo_close(cpd->txfifo, cpd->fflags);
-		/* clear write bitmask */
-		udev->ep_wr_opened &= ~(1 << cpd->ep_addr);
 	}
 
 	usb2_unref_device(cpd);
@@ -1004,6 +978,7 @@ usb2_ioctl_f_sub(struct usb2_fifo *f, u_long cmd, void *addr,
 	default:
 		return (ENOIOCTL);
 	}
+	DPRINTFN(3, "cmd 0x%lx = %d\n", cmd, error);
 	return (error);
 }
 
@@ -1018,6 +993,8 @@ usb2_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int fflag, struct thread*
 	int fflags;
 	int err;
 
+	DPRINTFN(2, "cmd=0x%lx\n", cmd);
+
 	err = devfs_get_cdevpriv((void **)&cpd);
 	if (err != 0)
 		return (err);
@@ -1033,8 +1010,6 @@ usb2_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int fflag, struct thread*
 	}
 	fflags = cpd->fflags;
 
-	DPRINTFN(2, "fflags=%u, cmd=0x%lx\n", fflags, cmd);
-
 	f = NULL;			/* set default value */
 	err = ENOIOCTL;			/* set default value */
 
@@ -1049,12 +1024,14 @@ usb2_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int fflag, struct thread*
 	KASSERT(f != NULL, ("fifo not found"));
 	if (err == ENOIOCTL) {
 		err = (f->methods->f_ioctl) (f, cmd, addr, fflags);
+		DPRINTFN(2, "f_ioctl cmd 0x%lx = %d\n", cmd, err);
 		if (err == ENOIOCTL) {
-			if (usb2_uref_location(cpd)) {
+			if (usb2_usb_ref_device(cpd)) {
 				err = ENXIO;
 				goto done;
 			}
 			err = (f->methods->f_ioctl_post) (f, cmd, addr, fflags);
+			DPRINTFN(2, "f_ioctl_post cmd 0x%lx = %d\n", cmd, err);
 		}
 	}
 	if (err == ENOIOCTL) {
@@ -1621,7 +1598,7 @@ usb2_fifo_attach(struct usb2_device *udev, void *priv_sc,
 	/* initialise FIFO structures */
 
 	f_tx->fifo_index = n + USB_FIFO_TX;
-	f_tx->dev_ep_index = (n / 2) + (USB_EP_MAX / 2);
+	f_tx->dev_ep_index = -1;
 	f_tx->priv_mtx = priv_mtx;
 	f_tx->priv_sc0 = priv_sc;
 	f_tx->methods = pm;
@@ -1629,7 +1606,7 @@ usb2_fifo_attach(struct usb2_device *udev, void *priv_sc,
 	f_tx->udev = udev;
 
 	f_rx->fifo_index = n + USB_FIFO_RX;
-	f_rx->dev_ep_index = (n / 2) + (USB_EP_MAX / 2);
+	f_rx->dev_ep_index = -1;
 	f_rx->priv_mtx = priv_mtx;
 	f_rx->priv_sc0 = priv_sc;
 	f_rx->methods = pm;
@@ -1684,12 +1661,13 @@ usb2_fifo_attach(struct usb2_device *udev, void *priv_sc,
 		pd->bus_index = device_get_unit(udev->bus->bdev);
 		pd->dev_index = udev->device_index;
 		pd->ep_addr = -1;	/* not an endpoint */
-		pd->fifo_index = f_tx->fifo_index;
+		pd->fifo_index = f_tx->fifo_index & f_rx->fifo_index;
 		pd->mode = FREAD|FWRITE;
 
 		/* Now, create the device itself */
 		f_sc->dev = make_dev(&usb2_devsw, 0, uid, gid, mode,
 		    devname);
+		/* XXX setting si_drv1 and creating the device is not atomic! */
 		f_sc->dev->si_drv1 = pd;
 	}
 
@@ -1705,7 +1683,7 @@ usb2_fifo_attach(struct usb2_device *udev, void *priv_sc,
  * Else failure
  *------------------------------------------------------------------------*/
 int
-usb2_fifo_alloc_buffer(struct usb2_fifo *f, uint32_t bufsize,
+usb2_fifo_alloc_buffer(struct usb2_fifo *f, usb2_size_t bufsize,
     uint16_t nbuf)
 {
 	usb2_fifo_free_buffer(f);
@@ -1762,17 +1740,19 @@ usb2_fifo_detach(struct usb2_fifo_sc *f_sc)
 	f_sc->fp[USB_FIFO_RX] = NULL;
 
 	if (f_sc->dev != NULL) {
-		destroy_dev_sched_cb(f_sc->dev, usb2_fifo_cleanup, f_sc->dev->si_drv1);
+		destroy_dev_sched_cb(f_sc->dev, 
+		    usb2_fifo_cleanup, f_sc->dev->si_drv1);
+		f_sc->dev = NULL;
 	}
 
 	DPRINTFN(2, "detached %p\n", f_sc);
 }
 
-uint32_t
+usb2_size_t
 usb2_fifo_put_bytes_max(struct usb2_fifo *f)
 {
 	struct usb2_mbuf *m;
-	uint32_t len;
+	usb2_size_t len;
 
 	USB_IF_POLL(&f->free_q, m);
 
@@ -1793,10 +1773,10 @@ usb2_fifo_put_bytes_max(struct usb2_fifo *f)
  *------------------------------------------------------------------------*/
 void
 usb2_fifo_put_data(struct usb2_fifo *f, struct usb2_page_cache *pc,
-    uint32_t offset, uint32_t len, uint8_t what)
+    usb2_frlength_t offset, usb2_frlength_t len, uint8_t what)
 {
 	struct usb2_mbuf *m;
-	uint32_t io_len;
+	usb2_frlength_t io_len;
 
 	while (len || (what == 1)) {
 
@@ -1831,10 +1811,10 @@ usb2_fifo_put_data(struct usb2_fifo *f, struct usb2_page_cache *pc,
 
 void
 usb2_fifo_put_data_linear(struct usb2_fifo *f, void *ptr,
-    uint32_t len, uint8_t what)
+    usb2_size_t len, uint8_t what)
 {
 	struct usb2_mbuf *m;
-	uint32_t io_len;
+	usb2_size_t io_len;
 
 	while (len || (what == 1)) {
 
@@ -1868,7 +1848,7 @@ usb2_fifo_put_data_linear(struct usb2_fifo *f, void *ptr,
 }
 
 uint8_t
-usb2_fifo_put_data_buffer(struct usb2_fifo *f, void *ptr, uint32_t len)
+usb2_fifo_put_data_buffer(struct usb2_fifo *f, void *ptr, usb2_size_t len)
 {
 	struct usb2_mbuf *m;
 
@@ -1904,11 +1884,11 @@ usb2_fifo_put_data_error(struct usb2_fifo *f)
  *------------------------------------------------------------------------*/
 uint8_t
 usb2_fifo_get_data(struct usb2_fifo *f, struct usb2_page_cache *pc,
-    uint32_t offset, uint32_t len, uint32_t *actlen,
+    usb2_frlength_t offset, usb2_frlength_t len, usb2_frlength_t *actlen,
     uint8_t what)
 {
 	struct usb2_mbuf *m;
-	uint32_t io_len;
+	usb2_frlength_t io_len;
 	uint8_t tr_data = 0;
 
 	actlen[0] = 0;
@@ -1949,6 +1929,13 @@ usb2_fifo_get_data(struct usb2_fifo *f, struct usb2_page_cache *pc,
 				break;
 			}
 			if (f->flag_flushing) {
+				/* check if we should send a short packet */
+				if (f->flag_short != 0) {
+					f->flag_short = 0;
+					tr_data = 1;
+					break;
+				}
+				/* flushing complete */
 				f->flag_flushing = 0;
 				usb2_fifo_wakeup(f);
 			}
@@ -1963,10 +1950,10 @@ usb2_fifo_get_data(struct usb2_fifo *f, struct usb2_page_cache *pc,
 
 uint8_t
 usb2_fifo_get_data_linear(struct usb2_fifo *f, void *ptr,
-    uint32_t len, uint32_t *actlen, uint8_t what)
+    usb2_size_t len, usb2_size_t *actlen, uint8_t what)
 {
 	struct usb2_mbuf *m;
-	uint32_t io_len;
+	usb2_size_t io_len;
 	uint8_t tr_data = 0;
 
 	actlen[0] = 0;
@@ -2007,6 +1994,13 @@ usb2_fifo_get_data_linear(struct usb2_fifo *f, void *ptr,
 				break;
 			}
 			if (f->flag_flushing) {
+				/* check if we should send a short packet */
+				if (f->flag_short != 0) {
+					f->flag_short = 0;
+					tr_data = 1;
+					break;
+				}
+				/* flushing complete */
 				f->flag_flushing = 0;
 				usb2_fifo_wakeup(f);
 			}
@@ -2020,7 +2014,7 @@ usb2_fifo_get_data_linear(struct usb2_fifo *f, void *ptr,
 }
 
 uint8_t
-usb2_fifo_get_data_buffer(struct usb2_fifo *f, void **pptr, uint32_t *plen)
+usb2_fifo_get_data_buffer(struct usb2_fifo *f, void **pptr, usb2_size_t *plen)
 {
 	struct usb2_mbuf *m;
 
@@ -2186,3 +2180,14 @@ usb2_read_symlink(uint8_t *user_ptr, uint32_t startentry, uint32_t user_len)
 	sx_unlock(&usb2_sym_lock);
 	return (error);
 }
+
+void
+usb2_fifo_set_close_zlp(struct usb2_fifo *f, uint8_t onoff)
+{
+	if (f == NULL)
+		return;
+
+	/* send a Zero Length Packet, ZLP, before close */
+	f->flag_short = onoff;
+}
+#endif	/* USB_HAVE_UGEN */
diff --git a/sys/dev/usb/usb_dev.h b/sys/dev/usb/usb_dev.h
index 92be7af..af252c5 100644
--- a/sys/dev/usb/usb_dev.h
+++ b/sys/dev/usb/usb_dev.h
@@ -93,12 +93,12 @@ struct usb2_cdev_privdata {
 	int			bus_index;	/* bus index */
 	int			dev_index;	/* device index */
 	int			ep_addr;	/* endpoint address */
+	int			fflags;
 	uint8_t			fifo_index;	/* FIFO index */
 	uint8_t			is_read;	/* location has read access */
 	uint8_t			is_write;	/* location has write access */
 	uint8_t			is_uref;	/* USB refcount decr. needed */
 	uint8_t			is_usbfs;	/* USB-FS is active */
-	int			fflags;
 };
 
 struct usb2_fs_privdata {
@@ -130,13 +130,14 @@ struct usb2_fifo {
 	struct usb2_xfer *xfer[2];
 	struct usb2_xfer **fs_xfer;
 	struct mtx *priv_mtx;		/* client data */
-	int    opened;			/* set if FIFO is opened by a FILE */
+	/* set if FIFO is opened by a FILE: */
+	struct usb2_cdev_privdata *curr_cpd;
 	void   *priv_sc0;		/* client data */
 	void   *priv_sc1;		/* client data */
 	void   *queue_data;
-	uint32_t timeout;		/* timeout in milliseconds */
-	uint32_t bufsize;		/* BULK and INTERRUPT buffer size */
-	uint16_t nframes;		/* for isochronous mode */
+	usb2_timeout_t timeout;		/* timeout in milliseconds */
+	usb2_frlength_t bufsize;		/* BULK and INTERRUPT buffer size */
+	usb2_frcount_t nframes;		/* for isochronous mode */
 	uint16_t dev_ep_index;		/* our device endpoint index */
 	uint8_t	flag_sleeping;		/* set if FIFO is sleeping */
 	uint8_t	flag_iscomplete;	/* set if a USB transfer is complete */
@@ -174,17 +175,18 @@ int	usb2_fifo_attach(struct usb2_device *udev, void *priv_sc,
 void	usb2_fifo_detach(struct usb2_fifo_sc *f_sc);
 uint32_t usb2_fifo_put_bytes_max(struct usb2_fifo *fifo);
 void	usb2_fifo_put_data(struct usb2_fifo *fifo, struct usb2_page_cache *pc,
-	    uint32_t offset, uint32_t len, uint8_t what);
+	    usb2_frlength_t offset, usb2_frlength_t len, uint8_t what);
 void	usb2_fifo_put_data_linear(struct usb2_fifo *fifo, void *ptr,
-	    uint32_t len, uint8_t what);
-uint8_t	usb2_fifo_put_data_buffer(struct usb2_fifo *f, void *ptr, uint32_t len);
+	    usb2_size_t len, uint8_t what);
+uint8_t	usb2_fifo_put_data_buffer(struct usb2_fifo *f, void *ptr, usb2_size_t len);
 void	usb2_fifo_put_data_error(struct usb2_fifo *fifo);
 uint8_t	usb2_fifo_get_data(struct usb2_fifo *fifo, struct usb2_page_cache *pc,
-	    uint32_t offset, uint32_t len, uint32_t *actlen, uint8_t what);
+	    usb2_frlength_t offset, usb2_frlength_t len, usb2_frlength_t *actlen,
+	    uint8_t what);
 uint8_t	usb2_fifo_get_data_linear(struct usb2_fifo *fifo, void *ptr,
-	    uint32_t len, uint32_t *actlen, uint8_t what);
+	    usb2_size_t len, usb2_size_t *actlen, uint8_t what);
 uint8_t	usb2_fifo_get_data_buffer(struct usb2_fifo *f, void **pptr,
-	    uint32_t *plen);
+	    usb2_size_t *plen);
 void	usb2_fifo_get_data_error(struct usb2_fifo *fifo);
 uint8_t	usb2_fifo_opened(struct usb2_fifo *fifo);
 void	usb2_fifo_free(struct usb2_fifo *f);
@@ -194,5 +196,6 @@ struct usb2_symlink *usb2_alloc_symlink(const char *target);
 void	usb2_free_symlink(struct usb2_symlink *ps);
 int	usb2_read_symlink(uint8_t *user_ptr, uint32_t startentry,
 	    uint32_t user_len);
+void	usb2_fifo_set_close_zlp(struct usb2_fifo *, uint8_t);
 
 #endif					/* _USB2_DEV_H_ */
diff --git a/sys/dev/usb/usb_device.c b/sys/dev/usb/usb_device.c
index 8bd7326..f50a29d 100644
--- a/sys/dev/usb/usb_device.c
+++ b/sys/dev/usb/usb_device.c
@@ -24,7 +24,6 @@
  * SUCH DAMAGE.
  */
 
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
 #include <dev/usb/usb.h>
@@ -45,21 +44,23 @@
 #include <dev/usb/usb_hub.h>
 #include <dev/usb/usb_util.h>
 #include <dev/usb/usb_mbuf.h>
-#include <dev/usb/usb_dev.h>
 #include <dev/usb/usb_msctest.h>
+#if USB_HAVE_UGEN
+#include <dev/usb/usb_dev.h>
 #include <dev/usb/usb_generic.h>
+#endif
 
 #include <dev/usb/quirk/usb_quirk.h>
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
 
-/* function prototypes */
+/* function prototypes  */
 
-static void	usb2_fill_pipe_data(struct usb2_device *, uint8_t,
+static void	usb2_init_pipe(struct usb2_device *, uint8_t,
 		    struct usb2_endpoint_descriptor *, struct usb2_pipe *);
-static void	usb2_free_pipe_data(struct usb2_device *, uint8_t, uint8_t);
-static void	usb2_free_iface_data(struct usb2_device *);
+static void	usb2_unconfigure(struct usb2_device *, uint8_t);
+static void	usb2_detach_device(struct usb2_device *, uint8_t, uint8_t);
 static void	usb2_detach_device_sub(struct usb2_device *, device_t *,
 		    uint8_t);
 static uint8_t	usb2_probe_and_attach_sub(struct usb2_device *,
@@ -69,12 +70,12 @@ static void	usb2_init_attach_arg(struct usb2_device *,
 static void	usb2_suspend_resume_sub(struct usb2_device *, device_t,
 		    uint8_t);
 static void	usb2_clear_stall_proc(struct usb2_proc_msg *_pm);
+usb2_error_t	usb2_config_parse(struct usb2_device *, uint8_t, uint8_t);
+#if USB_HAVE_STRINGS
 static void	usb2_check_strings(struct usb2_device *);
-static usb2_error_t usb2_fill_iface_data(struct usb2_device *, uint8_t,
-		    uint8_t);
-static void	usb2_notify_addq(const char *type, struct usb2_device *);
-
+#endif
 #if USB_HAVE_UGEN
+static void	usb2_notify_addq(const char *type, struct usb2_device *);
 static void	usb2_fifo_free_wrap(struct usb2_device *, uint8_t, uint8_t);
 static struct cdev *usb2_make_dev(struct usb2_device *, int, int);
 static void	usb2_cdev_create(struct usb2_device *);
@@ -104,7 +105,7 @@ struct usb2_pipe *
 usb2_get_pipe_by_addr(struct usb2_device *udev, uint8_t ea_val)
 {
 	struct usb2_pipe *pipe = udev->pipes;
-	struct usb2_pipe *pipe_end = udev->pipes + USB_EP_MAX;
+	struct usb2_pipe *pipe_end = udev->pipes + udev->pipes_max;
 	enum {
 		EA_MASK = (UE_DIR_IN | UE_DIR_OUT | UE_ADDR),
 	};
@@ -159,7 +160,7 @@ usb2_get_pipe(struct usb2_device *udev, uint8_t iface_index,
     const struct usb2_config *setup)
 {
 	struct usb2_pipe *pipe = udev->pipes;
-	struct usb2_pipe *pipe_end = udev->pipes + USB_EP_MAX;
+	struct usb2_pipe *pipe_end = udev->pipes + udev->pipes_max;
 	uint8_t index = setup->ep_index;
 	uint8_t ea_mask;
 	uint8_t ea_val;
@@ -171,9 +172,25 @@ usb2_get_pipe(struct usb2_device *udev, uint8_t iface_index,
 	    udev, iface_index, setup->endpoint,
 	    setup->type, setup->direction, setup->ep_index);
 
+	/* check USB mode */
+
+	if ((setup->usb_mode != USB_MODE_MAX) &&
+	    (udev->flags.usb2_mode != setup->usb_mode)) {
+		/* wrong mode - no pipe */
+		return (NULL);
+	}
+
 	/* setup expected endpoint direction mask and value */
 
-	if (setup->direction == UE_DIR_ANY) {
+	if (setup->direction == UE_DIR_RX) {
+		ea_mask = (UE_DIR_IN | UE_DIR_OUT);
+		ea_val = (udev->flags.usb2_mode == USB_MODE_DEVICE) ?
+		    UE_DIR_OUT : UE_DIR_IN;
+	} else if (setup->direction == UE_DIR_TX) {
+		ea_mask = (UE_DIR_IN | UE_DIR_OUT);
+		ea_val = (udev->flags.usb2_mode == USB_MODE_DEVICE) ?
+		    UE_DIR_IN : UE_DIR_OUT;
+	} else if (setup->direction == UE_DIR_ANY) {
 		/* match any endpoint direction */
 		ea_mask = 0;
 		ea_val = 0;
@@ -266,71 +283,47 @@ usb2_interface_count(struct usb2_device *udev, uint8_t *count)
 		*count = 0;
 		return (USB_ERR_NOT_CONFIGURED);
 	}
-	*count = udev->cdesc->bNumInterface;
+	*count = udev->ifaces_max;
 	return (USB_ERR_NORMAL_COMPLETION);
 }
 
 
 /*------------------------------------------------------------------------*
- *	usb2_fill_pipe_data
+ *	usb2_init_pipe
  *
  * This function will initialise the USB pipe structure pointed to by
- * the "pipe" argument.
+ * the "pipe" argument. The structure pointed to by "pipe" must be
+ * zeroed before calling this function.
  *------------------------------------------------------------------------*/
 static void
-usb2_fill_pipe_data(struct usb2_device *udev, uint8_t iface_index,
+usb2_init_pipe(struct usb2_device *udev, uint8_t iface_index,
     struct usb2_endpoint_descriptor *edesc, struct usb2_pipe *pipe)
 {
+	struct usb2_bus_methods *methods;
 
-	bzero(pipe, sizeof(*pipe));
+	methods = udev->bus->methods;
 
-	(udev->bus->methods->pipe_init) (udev, edesc, pipe);
+	(methods->pipe_init) (udev, edesc, pipe);
 
-	if (pipe->methods == NULL) {
-		/* the pipe is invalid: just return */
-		return;
-	}
 	/* initialise USB pipe structure */
 	pipe->edesc = edesc;
 	pipe->iface_index = iface_index;
 	TAILQ_INIT(&pipe->pipe_q.head);
 	pipe->pipe_q.command = &usb2_pipe_start;
 
+	/* the pipe is not supported by the hardware */
+ 	if (pipe->methods == NULL)
+		return;
+
 	/* clear stall, if any */
-	if (udev->bus->methods->clear_stall) {
+	if (methods->clear_stall != NULL) {
 		USB_BUS_LOCK(udev->bus);
-		(udev->bus->methods->clear_stall) (udev, pipe);
+		(methods->clear_stall) (udev, pipe);
 		USB_BUS_UNLOCK(udev->bus);
 	}
 }
 
-/*------------------------------------------------------------------------*
- *	usb2_free_pipe_data
- *
- * This function will free USB pipe data for the given interface
- * index. Hence we do not have any dynamic allocations we simply clear
- * "pipe->edesc" to indicate that the USB pipe structure can be
- * reused. The pipes belonging to the given interface should not be in
- * use when this function is called and no check is performed to
- * prevent this.
- *------------------------------------------------------------------------*/
-static void
-usb2_free_pipe_data(struct usb2_device *udev,
-    uint8_t iface_index, uint8_t iface_mask)
-{
-	struct usb2_pipe *pipe = udev->pipes;
-	struct usb2_pipe *pipe_end = udev->pipes + USB_EP_MAX;
-
-	while (pipe != pipe_end) {
-		if ((pipe->iface_index & iface_mask) == iface_index) {
-			/* free pipe */
-			pipe->edesc = NULL;
-		}
-		pipe++;
-	}
-}
-
-/*------------------------------------------------------------------------*
+/*-----------------------------------------------------------------------*
  *	usb2_pipe_foreach
  *
  * This function will iterate all the USB endpoints except the control
@@ -343,7 +336,7 @@ usb2_free_pipe_data(struct usb2_device *udev,
 struct usb2_pipe *
 usb2_pipe_foreach(struct usb2_device *udev, struct usb2_pipe *pipe)
 {
-	struct usb2_pipe *pipe_end = udev->pipes + USB_EP_MAX;
+	struct usb2_pipe *pipe_end = udev->pipes + udev->pipes_max;
 
 	/* be NULL safe */
 	if (udev == NULL)
@@ -365,148 +358,62 @@ usb2_pipe_foreach(struct usb2_device *udev, struct usb2_pipe *pipe)
 }
 
 /*------------------------------------------------------------------------*
- *	usb2_fill_iface_data
+ *	usb2_unconfigure
  *
- * This function will fill in interface data and allocate USB pipes
- * for all the endpoints that belong to the given interface. This
- * function is typically called when setting the configuration or when
- * setting an alternate interface.
+ * This function will free all USB interfaces and USB pipes belonging
+ * to an USB device.
+ *
+ * Flag values, see "USB_UNCFG_FLAG_XXX".
  *------------------------------------------------------------------------*/
-static usb2_error_t
-usb2_fill_iface_data(struct usb2_device *udev,
-    uint8_t iface_index, uint8_t alt_index)
+static void
+usb2_unconfigure(struct usb2_device *udev, uint8_t flag)
 {
-	struct usb2_interface *iface = usb2_get_iface(udev, iface_index);
-	struct usb2_pipe *pipe;
-	struct usb2_pipe *pipe_end;
-	struct usb2_interface_descriptor *id;
-	struct usb2_endpoint_descriptor *ed = NULL;
-	struct usb2_descriptor *desc;
-	uint8_t nendpt;
+	uint8_t do_unlock;
 
-	if (iface == NULL) {
-		return (USB_ERR_INVAL);
+	/* automatic locking */
+	if (sx_xlocked(udev->default_sx + 1)) {
+		do_unlock = 0;
+	} else {
+		do_unlock = 1;
+		sx_xlock(udev->default_sx + 1);
 	}
-	DPRINTFN(5, "iface_index=%d alt_index=%d\n",
-	    iface_index, alt_index);
 
-	sx_assert(udev->default_sx + 1, SA_LOCKED);
-
-	pipe = udev->pipes;
-	pipe_end = udev->pipes + USB_EP_MAX;
-
-	/*
-	 * Check if any USB pipes on the given USB interface are in
-	 * use:
-	 */
-	while (pipe != pipe_end) {
-		if ((pipe->edesc != NULL) &&
-		    (pipe->iface_index == iface_index) &&
-		    (pipe->refcount != 0)) {
-			return (USB_ERR_IN_USE);
-		}
-		pipe++;
-	}
+	/* detach all interface drivers */
+	usb2_detach_device(udev, USB_IFACE_INDEX_ANY, flag);
 
-	pipe = &udev->pipes[0];
+#if USB_HAVE_UGEN
+	/* free all FIFOs except control endpoint FIFOs */
+	usb2_fifo_free_wrap(udev, USB_IFACE_INDEX_ANY, flag);
 
-	id = usb2_find_idesc(udev->cdesc, iface_index, alt_index);
-	if (id == NULL) {
-		return (USB_ERR_INVAL);
-	}
 	/*
-	 * Free old pipes after we know that an interface descriptor exists,
-	 * if any.
+	 * Free all cdev's, if any.
 	 */
-	usb2_free_pipe_data(udev, iface_index, 0 - 1);
-
-	/* Setup USB interface structure */
-	iface->idesc = id;
-	iface->alt_index = alt_index;
-	iface->parent_iface_index = USB_IFACE_INDEX_ANY;
-
-	nendpt = id->bNumEndpoints;
-	DPRINTFN(5, "found idesc nendpt=%d\n", nendpt);
-
-	desc = (void *)id;
-
-	while (nendpt--) {
-		DPRINTFN(11, "endpt=%d\n", nendpt);
-
-		while ((desc = usb2_desc_foreach(udev->cdesc, desc))) {
-			if ((desc->bDescriptorType == UDESC_ENDPOINT) &&
-			    (desc->bLength >= sizeof(*ed))) {
-				goto found;
-			}
-			if (desc->bDescriptorType == UDESC_INTERFACE) {
-				break;
-			}
-		}
-		goto error;
-
-found:
-		ed = (void *)desc;
-
-		/* find a free pipe */
-		while (pipe != pipe_end) {
-			if (pipe->edesc == NULL) {
-				/* pipe is free */
-				usb2_fill_pipe_data(udev, iface_index, ed, pipe);
-				break;
-			}
-			pipe++;
-		}
-	}
-	return (USB_ERR_NORMAL_COMPLETION);
-
-error:
-	/* passed end, or bad desc */
-	DPRINTFN(0, "%s: bad descriptor(s), addr=%d!\n",
-	    __FUNCTION__, udev->address);
-
-	/* free old pipes if any */
-	usb2_free_pipe_data(udev, iface_index, 0 - 1);
-	return (USB_ERR_INVAL);
-}
-
-/*------------------------------------------------------------------------*
- *	usb2_free_iface_data
- *
- * This function will free all USB interfaces and USB pipes belonging
- * to an USB device.
- *------------------------------------------------------------------------*/
-static void
-usb2_free_iface_data(struct usb2_device *udev)
-{
-	struct usb2_interface *iface = udev->ifaces;
-	struct usb2_interface *iface_end = udev->ifaces + USB_IFACE_MAX;
-
-	/* mtx_assert() */
+	usb2_cdev_free(udev);
+#endif
 
+#if USB_HAVE_COMPAT_LINUX
 	/* free Linux compat device, if any */
 	if (udev->linux_dev) {
 		usb_linux_free_device(udev->linux_dev);
 		udev->linux_dev = NULL;
 	}
-	/* free all pipes, if any */
-	usb2_free_pipe_data(udev, 0, 0);
+#endif
 
-	/* free all interfaces, if any */
-	while (iface != iface_end) {
-		iface->idesc = NULL;
-		iface->alt_index = 0;
-		iface->parent_iface_index = USB_IFACE_INDEX_ANY;
-		iface++;
-	}
+	usb2_config_parse(udev, USB_IFACE_INDEX_ANY, USB_CFG_FREE);
 
-	/* free "cdesc" after "ifaces", if any */
-	if (udev->cdesc) {
-		free(udev->cdesc, M_USB);
+	/* free "cdesc" after "ifaces" and "pipes", if any */
+	if (udev->cdesc != NULL) {
+		if (udev->flags.usb2_mode != USB_MODE_DEVICE)
+			free(udev->cdesc, M_USB);
 		udev->cdesc = NULL;
 	}
 	/* set unconfigured state */
 	udev->curr_config_no = USB_UNCONFIG_NO;
 	udev->curr_config_index = USB_UNCONFIG_INDEX;
+
+	if (do_unlock) {
+		sx_unlock(udev->default_sx + 1);
+	}
 }
 
 /*------------------------------------------------------------------------*
@@ -524,11 +431,9 @@ usb2_error_t
 usb2_set_config_index(struct usb2_device *udev, uint8_t index)
 {
 	struct usb2_status ds;
-	struct usb2_hub_descriptor hd;
 	struct usb2_config_descriptor *cdp;
 	uint16_t power;
 	uint16_t max_power;
-	uint8_t nifc;
 	uint8_t selfpowered;
 	uint8_t do_unlock;
 	usb2_error_t err;
@@ -543,30 +448,26 @@ usb2_set_config_index(struct usb2_device *udev, uint8_t index)
 		sx_xlock(udev->default_sx + 1);
 	}
 
-	/* detach all interface drivers */
-	usb2_detach_device(udev, USB_IFACE_INDEX_ANY, 1);
-
-#if USB_HAVE_UGEN
-	/* free all FIFOs except control endpoint FIFOs */
-	usb2_fifo_free_wrap(udev, USB_IFACE_INDEX_ANY, 0);
-
-	/* free all configuration data structures */
-	usb2_cdev_free(udev);
-#endif
-	usb2_free_iface_data(udev);
+	usb2_unconfigure(udev, USB_UNCFG_FLAG_FREE_SUBDEV);
 
 	if (index == USB_UNCONFIG_INDEX) {
 		/*
 		 * Leave unallocated when unconfiguring the
-		 * device. "usb2_free_iface_data()" will also reset
+		 * device. "usb2_unconfigure()" will also reset
 		 * the current config number and index.
 		 */
 		err = usb2_req_set_config(udev, NULL, USB_UNCONFIG_NO);
 		goto done;
 	}
 	/* get the full config descriptor */
-	err = usb2_req_get_config_desc_full(udev,
-	    NULL, &cdp, M_USB, index);
+	if (udev->flags.usb2_mode == USB_MODE_DEVICE) {
+		/* save some memory */
+		err = usb2_req_get_config_desc_ptr(udev, &cdp, index);
+	} else {
+		/* normal request */
+		err = usb2_req_get_config_desc_full(udev,
+		    NULL, &cdp, M_USB, index);
+	}
 	if (err) {
 		goto done;
 	}
@@ -574,10 +475,6 @@ usb2_set_config_index(struct usb2_device *udev, uint8_t index)
 
 	udev->cdesc = cdp;
 
-	if (cdp->bNumInterface > USB_IFACE_MAX) {
-		DPRINTFN(0, "too many interfaces: %d\n", cdp->bNumInterface);
-		cdp->bNumInterface = USB_IFACE_MAX;
-	}
 	/* Figure out if the device is self or bus powered. */
 	selfpowered = 0;
 	if ((!udev->flags.uq_bus_powered) &&
@@ -586,45 +483,23 @@ usb2_set_config_index(struct usb2_device *udev, uint8_t index)
 		/* May be self powered. */
 		if (cdp->bmAttributes & UC_BUS_POWERED) {
 			/* Must ask device. */
-			if (udev->flags.uq_power_claim) {
-				/*
-				 * HUB claims to be self powered, but isn't.
-				 * It seems that the power status can be
-				 * determined by the HUB characteristics.
-				 */
-				err = usb2_req_get_hub_descriptor
-				    (udev, NULL, &hd, 1);
-				if (err) {
-					DPRINTFN(0, "could not read "
-					    "HUB descriptor: %s\n",
-					    usb2_errstr(err));
-
-				} else if (UGETW(hd.wHubCharacteristics) &
-				    UHD_PWR_INDIVIDUAL) {
-					selfpowered = 1;
-				}
-				DPRINTF("characteristics=0x%04x\n",
-				    UGETW(hd.wHubCharacteristics));
-			} else {
-				err = usb2_req_get_device_status
-				    (udev, NULL, &ds);
-				if (err) {
-					DPRINTFN(0, "could not read "
-					    "device status: %s\n",
-					    usb2_errstr(err));
-				} else if (UGETW(ds.wStatus) & UDS_SELF_POWERED) {
-					selfpowered = 1;
-				}
-				DPRINTF("status=0x%04x \n",
-				    UGETW(ds.wStatus));
+			err = usb2_req_get_device_status(udev, NULL, &ds);
+			if (err) {
+				DPRINTFN(0, "could not read "
+				    "device status: %s\n",
+				    usb2_errstr(err));
+			} else if (UGETW(ds.wStatus) & UDS_SELF_POWERED) {
+				selfpowered = 1;
 			}
+			DPRINTF("status=0x%04x \n",
+				UGETW(ds.wStatus));
 		} else
 			selfpowered = 1;
 	}
 	DPRINTF("udev=%p cdesc=%p (addr %d) cno=%d attr=0x%02x, "
 	    "selfpowered=%d, power=%d\n",
 	    udev, cdp,
-	    cdp->bConfigurationValue, udev->address, cdp->bmAttributes,
+	    udev->address, cdp->bConfigurationValue, cdp->bmAttributes,
 	    selfpowered, cdp->bMaxPower * 2);
 
 	/* Check if we have enough power. */
@@ -654,14 +529,17 @@ usb2_set_config_index(struct usb2_device *udev, uint8_t index)
 	if (err) {
 		goto done;
 	}
-	/* Allocate and fill interface data. */
-	nifc = cdp->bNumInterface;
-	while (nifc--) {
-		err = usb2_fill_iface_data(udev, nifc, 0);
-		if (err) {
-			goto done;
-		}
+
+	err = usb2_config_parse(udev, USB_IFACE_INDEX_ANY, USB_CFG_ALLOC);
+	if (err) {
+		goto done;
+	}
+
+	err = usb2_config_parse(udev, USB_IFACE_INDEX_ANY, USB_CFG_INIT);
+	if (err) {
+		goto done;
 	}
+
 #if USB_HAVE_UGEN
 	/* create device nodes for each endpoint */
 	usb2_cdev_create(udev);
@@ -670,10 +548,7 @@ usb2_set_config_index(struct usb2_device *udev, uint8_t index)
 done:
 	DPRINTF("error=%s\n", usb2_errstr(err));
 	if (err) {
-#if USB_HAVE_UGEN
-		usb2_cdev_free(udev);
-#endif
-		usb2_free_iface_data(udev);
+		usb2_unconfigure(udev, USB_UNCFG_FLAG_FREE_SUBDEV);
 	}
 	if (do_unlock) {
 		sx_unlock(udev->default_sx + 1);
@@ -682,6 +557,205 @@ done:
 }
 
 /*------------------------------------------------------------------------*
+ *	usb2_config_parse
+ *
+ * This function will allocate and free USB interfaces and USB pipes,
+ * parse the USB configuration structure and initialise the USB pipes
+ * and interfaces. If "iface_index" is not equal to
+ * "USB_IFACE_INDEX_ANY" then the "cmd" parameter is the
+ * alternate_setting to be selected for the given interface. Else the
+ * "cmd" parameter is defined by "USB_CFG_XXX". "iface_index" can be
+ * "USB_IFACE_INDEX_ANY" or a valid USB interface index. This function
+ * is typically called when setting the configuration or when setting
+ * an alternate interface.
+ *
+ * Returns:
+ *    0: Success
+ * Else: Failure
+ *------------------------------------------------------------------------*/
+usb2_error_t
+usb2_config_parse(struct usb2_device *udev, uint8_t iface_index, uint8_t cmd)
+{
+	struct usb2_idesc_parse_state ips;
+	struct usb2_interface_descriptor *id;
+	struct usb2_endpoint_descriptor *ed;
+	struct usb2_interface *iface;
+	struct usb2_pipe *pipe;
+	usb2_error_t err;
+	uint8_t ep_curr;
+	uint8_t ep_max;
+	uint8_t temp;
+	uint8_t do_init;
+	uint8_t alt_index;
+
+	if (iface_index != USB_IFACE_INDEX_ANY) {
+		/* parameter overload */
+		alt_index = cmd;
+		cmd = USB_CFG_INIT;
+	} else {
+		/* not used */
+		alt_index = 0;
+	}
+
+	err = 0;
+
+	DPRINTFN(5, "iface_index=%d cmd=%d\n",
+	    iface_index, cmd);
+
+	if (cmd == USB_CFG_FREE)
+		goto cleanup;
+
+	if (cmd == USB_CFG_INIT) {
+		sx_assert(udev->default_sx + 1, SA_LOCKED);
+
+		/* check for in-use pipes */
+
+		pipe = udev->pipes;
+		ep_max = udev->pipes_max;
+		while (ep_max--) {
+			/* look for matching pipes */
+			if ((iface_index == USB_IFACE_INDEX_ANY) ||
+			    (iface_index == pipe->iface_index)) {
+				if (pipe->refcount != 0) {
+					/*
+					 * This typically indicates a
+					 * more serious error.
+					 */
+					err = USB_ERR_IN_USE;
+				} else {
+					/* reset pipe */
+					memset(pipe, 0, sizeof(*pipe));
+					/* make sure we don't zero the pipe again */
+					pipe->iface_index = USB_IFACE_INDEX_ANY;
+				}
+			}
+			pipe++;
+		}
+
+		if (err)
+			return (err);
+	}
+
+	memset(&ips, 0, sizeof(ips));
+
+	ep_curr = 0;
+	ep_max = 0;
+
+	while ((id = usb2_idesc_foreach(udev->cdesc, &ips))) {
+
+		/* check for interface overflow */
+		if (ips.iface_index == USB_IFACE_MAX)
+			break;			/* crazy */
+
+		iface = udev->ifaces + ips.iface_index;
+
+		/* check for specific interface match */
+
+		if (cmd == USB_CFG_INIT) {
+			if ((iface_index != USB_IFACE_INDEX_ANY) && 
+			    (iface_index != ips.iface_index)) {
+				/* wrong interface */
+				do_init = 0;
+			} else if (alt_index != ips.iface_index_alt) {
+				/* wrong alternate setting */
+				do_init = 0;
+			} else {
+				/* initialise interface */
+				do_init = 1;
+			}
+		} else
+			do_init = 0;
+
+		/* check for new interface */
+		if (ips.iface_index_alt == 0) {
+			/* update current number of endpoints */
+			ep_curr = ep_max;
+		}
+		/* check for init */
+		if (do_init) {
+			/* setup the USB interface structure */
+			iface->idesc = id;
+			/* default setting */
+			iface->parent_iface_index = USB_IFACE_INDEX_ANY;
+			/* set alternate index */
+			iface->alt_index = alt_index;
+		}
+
+		DPRINTFN(5, "found idesc nendpt=%d\n", id->bNumEndpoints);
+
+		ed = (struct usb2_endpoint_descriptor *)id;
+
+		temp = ep_curr;
+
+		/* iterate all the endpoint descriptors */
+		while ((ed = usb2_edesc_foreach(udev->cdesc, ed))) {
+
+			if (temp == USB_EP_MAX)
+				break;			/* crazy */
+
+			pipe = udev->pipes + temp;
+
+			if (do_init) {
+				usb2_init_pipe(udev, 
+				    ips.iface_index, ed, pipe);
+			}
+
+			temp ++;
+
+			/* find maximum number of endpoints */
+			if (ep_max < temp)
+				ep_max = temp;
+
+			/* optimalisation */
+			id = (struct usb2_interface_descriptor *)ed;
+		}
+	}
+
+	/* NOTE: It is valid to have no interfaces and no endpoints! */
+
+	if (cmd == USB_CFG_ALLOC) {
+		udev->ifaces_max = ips.iface_index;
+		udev->ifaces = NULL;
+		if (udev->ifaces_max != 0) {
+			udev->ifaces = malloc(sizeof(*iface) * udev->ifaces_max,
+			        M_USB, M_WAITOK | M_ZERO);
+			if (udev->ifaces == NULL) {
+				err = USB_ERR_NOMEM;
+				goto done;
+			}
+		}
+		udev->pipes_max = ep_max;
+		udev->pipes = NULL;
+		if (udev->pipes_max != 0) {
+			udev->pipes = malloc(sizeof(*pipe) * udev->pipes_max,
+			        M_USB, M_WAITOK | M_ZERO);
+			if (udev->pipes == NULL) {
+				err = USB_ERR_NOMEM;
+				goto done;
+			}
+		}
+	}
+
+done:
+	if (err) {
+		if (cmd == USB_CFG_ALLOC) {
+cleanup:
+			/* cleanup */
+			if (udev->ifaces != NULL)
+				free(udev->ifaces, M_USB);
+			if (udev->pipes != NULL)
+				free(udev->pipes, M_USB);
+
+			udev->ifaces = NULL;
+			udev->pipes = NULL;
+			udev->ifaces_max = 0;
+			udev->pipes_max = 0;
+		}
+	}
+	return (err);
+}
+
+/*------------------------------------------------------------------------*
  *	usb2_set_alt_interface_index
  *
  * This function will select an alternate interface index for the
@@ -715,7 +789,8 @@ usb2_set_alt_interface_index(struct usb2_device *udev,
 		goto done;
 	}
 	if (udev->flags.usb2_mode == USB_MODE_DEVICE) {
-		usb2_detach_device(udev, iface_index, 1);
+		usb2_detach_device(udev, iface_index,
+		    USB_UNCFG_FLAG_FREE_SUBDEV);
 	} else {
 		if (iface->alt_index == alt_index) {
 			/* 
@@ -734,7 +809,7 @@ usb2_set_alt_interface_index(struct usb2_device *udev,
 	usb2_fifo_free_wrap(udev, iface_index, 0);
 #endif
 
-	err = usb2_fill_iface_data(udev, iface_index, alt_index);
+	err = usb2_config_parse(udev, iface_index, alt_index);
 	if (err) {
 		goto done;
 	}
@@ -842,7 +917,7 @@ usb2_reset_iface_endpoints(struct usb2_device *udev, uint8_t iface_index)
 	usb2_error_t err;
 
 	pipe = udev->pipes;
-	pipe_end = udev->pipes + USB_EP_MAX;
+	pipe_end = udev->pipes + udev->pipes_max;
 
 	for (; pipe != pipe_end; pipe++) {
 
@@ -864,15 +939,17 @@ usb2_reset_iface_endpoints(struct usb2_device *udev, uint8_t iface_index)
  *
  * This function will try to detach an USB device. If it fails a panic
  * will result.
+ *
+ * Flag values, see "USB_UNCFG_FLAG_XXX".
  *------------------------------------------------------------------------*/
 static void
 usb2_detach_device_sub(struct usb2_device *udev, device_t *ppdev,
-    uint8_t free_subdev)
+    uint8_t flag)
 {
 	device_t dev;
 	int err;
 
-	if (!free_subdev) {
+	if (!(flag & USB_UNCFG_FLAG_FREE_SUBDEV)) {
 
 		*ppdev = NULL;
 
@@ -918,14 +995,15 @@ error:
  *
  * The following function will detach the matching interfaces.
  * This function is NULL safe.
+ *
+ * Flag values, see "USB_UNCFG_FLAG_XXX".
  *------------------------------------------------------------------------*/
 void
 usb2_detach_device(struct usb2_device *udev, uint8_t iface_index,
-    uint8_t free_subdev)
+    uint8_t flag)
 {
 	struct usb2_interface *iface;
 	uint8_t i;
-	uint8_t do_unlock;
 
 	if (udev == NULL) {
 		/* nothing to do */
@@ -933,13 +1011,7 @@ usb2_detach_device(struct usb2_device *udev, uint8_t iface_index,
 	}
 	DPRINTFN(4, "udev=%p\n", udev);
 
-	/* automatic locking */
-	if (sx_xlocked(udev->default_sx + 1)) {
-		do_unlock = 0;
-	} else {
-		do_unlock = 1;
-		sx_xlock(udev->default_sx + 1);
-	}
+	sx_assert(udev->default_sx + 1, SA_LOCKED);
 
 	/*
 	 * First detach the child to give the child's detach routine a
@@ -964,11 +1036,7 @@ usb2_detach_device(struct usb2_device *udev, uint8_t iface_index,
 			/* looks like the end of the USB interfaces */
 			break;
 		}
-		usb2_detach_device_sub(udev, &iface->subdev, free_subdev);
-	}
-
-	if (do_unlock) {
-		sx_unlock(udev->default_sx + 1);
+		usb2_detach_device_sub(udev, &iface->subdev, flag);
 	}
 }
 
@@ -1373,8 +1441,6 @@ usb2_alloc_device(device_t parent_dev, struct usb2_bus *bus,
 	usb2_cv_init(udev->default_cv, "WCTRL");
 	usb2_cv_init(udev->default_cv + 1, "UGONE");
 
-	LIST_INIT(&udev->pd_list);
-
 	/* initialise our mutex */
 	mtx_init(udev->default_mtx, "USB device mutex", NULL, MTX_DEF);
 
@@ -1392,7 +1458,7 @@ usb2_alloc_device(device_t parent_dev, struct usb2_bus *bus,
 	udev->depth = depth;
 	udev->bus = bus;
 	udev->address = USB_START_ADDR;	/* default value */
-	udev->plugtime = (uint32_t)ticks;
+	udev->plugtime = (usb2_ticks_t)ticks;
 	/*
 	 * We need to force the power mode to "on" because there are plenty
 	 * of USB devices out there that do not work very well with
@@ -1400,7 +1466,6 @@ usb2_alloc_device(device_t parent_dev, struct usb2_bus *bus,
 	 */
 	udev->power_mode = USB_POWER_MODE_ON;
 	udev->pwr_save.last_xfer_time = ticks;
-
 	/* we are not ready yet */
 	udev->refcount = 1;
 
@@ -1417,10 +1482,6 @@ usb2_alloc_device(device_t parent_dev, struct usb2_bus *bus,
 	udev->speed = speed;
 	udev->flags.usb2_mode = usb2_mode;
 
-	/* speed combination should be checked by the parent HUB */
-
-	hub = udev->parent_hub;
-
 	/* search for our High Speed USB HUB, if any */
 
 	adev = udev;
@@ -1437,25 +1498,26 @@ usb2_alloc_device(device_t parent_dev, struct usb2_bus *bus,
 	}
 
 	/* init the default pipe */
-	usb2_fill_pipe_data(udev, 0,
+	usb2_init_pipe(udev, 0,
 	    &udev->default_ep_desc,
 	    &udev->default_pipe);
 
 	/* set device index */
 	udev->device_index = device_index;
 
+#if USB_HAVE_UGEN
 	/* Create ugen name */
 	snprintf(udev->ugen_name, sizeof(udev->ugen_name),
 	    USB_GENERIC_NAME "%u.%u", device_get_unit(bus->bdev),
 	    device_index);
-#if USB_HAVE_UGEN
+	LIST_INIT(&udev->pd_list);
+
 	/* Create the control endpoint device */
 	udev->default_dev = usb2_make_dev(udev, 0, FREAD|FWRITE);
 
 	/* Create a link from /dev/ugenX.X to the default endpoint */
 	make_dev_alias(udev->default_dev, udev->ugen_name);
 #endif
-
 	if (udev->flags.usb2_mode == USB_MODE_HOST) {
 
 		err = usb2_req_set_address(udev, NULL, device_index);
@@ -1475,7 +1537,8 @@ usb2_alloc_device(device_t parent_dev, struct usb2_bus *bus,
 		 */
 		if (err) {
 			DPRINTFN(0, "set address %d failed "
-			    "(ignored)\n", udev->address);
+			    "(%s, ignored)\n", udev->address, 
+			    usb2_errstr(err));
 		}
 		/* allow device time to set new address */
 		usb2_pause_mtx(NULL, 
@@ -1511,7 +1574,8 @@ usb2_alloc_device(device_t parent_dev, struct usb2_bus *bus,
 	    USB_MAX_IPACKET, USB_MAX_IPACKET, 0, UDESC_DEVICE, 0, 0);
 	if (err) {
 		DPRINTFN(0, "getting device descriptor "
-		    "at addr %d failed!\n", udev->address);
+		    "at addr %d failed, %s!\n", udev->address,
+		    usb2_errstr(err));
 		/* XXX try to re-enumerate the device */
 		err = usb2_req_re_enumerate(udev, NULL);
 		if (err) {
@@ -1544,9 +1608,6 @@ usb2_alloc_device(device_t parent_dev, struct usb2_bus *bus,
 	if (usb2_test_quirk(&uaa, UQ_BUS_POWERED)) {
 		udev->flags.uq_bus_powered = 1;
 	}
-	if (usb2_test_quirk(&uaa, UQ_POWER_CLAIM)) {
-		udev->flags.uq_power_claim = 1;
-	}
 	if (usb2_test_quirk(&uaa, UQ_NO_STRINGS)) {
 		udev->flags.no_strings = 1;
 	}
@@ -1585,6 +1646,7 @@ usb2_alloc_device(device_t parent_dev, struct usb2_bus *bus,
 	/* assume 100mA bus powered for now. Changed when configured. */
 	udev->power = USB_MIN_POWER;
 
+#if USB_HAVE_STRINGS
 	/* get serial number string */
 	err = usb2_req_get_string_any
 	    (udev, NULL, (char *)scratch_ptr,
@@ -1608,6 +1670,7 @@ usb2_alloc_device(device_t parent_dev, struct usb2_bus *bus,
 
 	/* finish up all the strings */
 	usb2_check_strings(udev);
+#endif
 
 	if (udev->flags.usb2_mode == USB_MODE_HOST) {
 		uint8_t config_index;
@@ -1643,9 +1706,7 @@ repeat_set_config:
 		DPRINTF("setting config %u\n", config_index);
 
 		/* get the USB device configured */
-		sx_xlock(udev->default_sx + 1);
 		err = usb2_set_config_index(udev, config_index);
-		sx_unlock(udev->default_sx + 1);
 		if (err) {
 			if (udev->ddesc.bNumConfigurations != 0) {
 				if (!set_config_failed) {
@@ -1673,7 +1734,8 @@ repeat_set_config:
 		} else if ((config_index + 1) < udev->ddesc.bNumConfigurations) {
 
 			if ((udev->cdesc->bNumInterface < 2) &&
-			    (usb2_get_no_endpoints(udev->cdesc) == 0)) {
+			    (usb2_get_no_descriptors(udev->cdesc,
+			    UDESC_ENDPOINT) == 0)) {
 				DPRINTFN(0, "Found no endpoints "
 				    "(trying next config)!\n");
 				config_index++;
@@ -1706,18 +1768,22 @@ repeat_set_config:
 	usb2_bus_port_set_device(bus, parent_hub ?
 	    parent_hub->hub->ports + port_index : NULL, udev, device_index);
 
-	/* Link and announce the ugen device name */
 #if USB_HAVE_UGEN
+	/* Symlink the ugen device name */
 	udev->ugen_symlink = usb2_alloc_symlink(udev->ugen_name);
-#endif
+
+	/* Announce device */
 	printf("%s: <%s> at %s\n", udev->ugen_name, udev->manufacturer,
 	    device_get_nameunit(udev->bus->bdev));
 
 	usb2_notify_addq("+", udev);
+#endif
 done:
 	if (err) {
 		/* free device  */
-		usb2_free_device(udev);
+		usb2_free_device(udev,
+		    USB_UNCFG_FLAG_FREE_SUBDEV |
+		    USB_UNCFG_FLAG_FREE_EP0);
 		udev = NULL;
 	}
 	return (udev);
@@ -1817,7 +1883,6 @@ usb2_cdev_free(struct usb2_device *udev)
 
 	while ((pd = LIST_FIRST(&udev->pd_list)) != NULL) {
 		KASSERT(pd->cdev->si_drv1 == pd, ("privdata corrupt"));
-		KASSERT(pd->ep_addr > 0, ("freeing EP0"));
 
 		destroy_dev_sched_cb(pd->cdev, usb2_cdev_cleanup, pd);
 		pd->cdev = NULL;
@@ -1836,14 +1901,22 @@ usb2_cdev_cleanup(void* arg)
  *	usb2_free_device
  *
  * This function is NULL safe and will free an USB device.
+ *
+ * Flag values, see "USB_UNCFG_FLAG_XXX".
  *------------------------------------------------------------------------*/
 void
-usb2_free_device(struct usb2_device *udev)
+usb2_free_device(struct usb2_device *udev, uint8_t flag)
 {
-	struct usb2_bus *bus = udev->bus;;
+	struct usb2_bus *bus;
+
+	if (udev == NULL)
+		return;		/* already freed */
 
 	DPRINTFN(4, "udev=%p port=%d\n", udev, udev->port_no);
 
+	bus = udev->bus;;
+
+#if USB_HAVE_UGEN
 	usb2_notify_addq("-", udev);
 
 	printf("%s: <%s> at %s (disconnected)\n", udev->ugen_name,
@@ -1851,11 +1924,10 @@ usb2_free_device(struct usb2_device *udev)
 
 	/* Destroy UGEN symlink, if any */
 	if (udev->ugen_symlink) {
-#if USB_HAVE_UGEN
 		usb2_free_symlink(udev->ugen_symlink);
-#endif
 		udev->ugen_symlink = NULL;
 	}
+#endif
 	/*
 	 * Unregister our device first which will prevent any further
 	 * references:
@@ -1866,33 +1938,24 @@ usb2_free_device(struct usb2_device *udev)
 
 #if USB_HAVE_UGEN
 	/* wait for all pending references to go away: */
-
 	mtx_lock(&usb2_ref_lock);
 	udev->refcount--;
 	while (udev->refcount != 0) {
 		usb2_cv_wait(udev->default_cv + 1, &usb2_ref_lock);
 	}
 	mtx_unlock(&usb2_ref_lock);
+
+	destroy_dev_sched_cb(udev->default_dev, usb2_cdev_cleanup,
+	    udev->default_dev->si_drv1);
 #endif
 
 	if (udev->flags.usb2_mode == USB_MODE_DEVICE) {
 		/* stop receiving any control transfers (Device Side Mode) */
 		usb2_transfer_unsetup(udev->default_xfer, USB_DEFAULT_XFER_MAX);
 	}
-#if USB_HAVE_UGEN
-	/* free all FIFOs */
-	usb2_fifo_free_wrap(udev, USB_IFACE_INDEX_ANY, 1);
 
-	/*
-	 * Free all interface related data and FIFOs, if any.
-	 */
-	usb2_cdev_free(udev);
-#endif
-	usb2_free_iface_data(udev);
-#if USB_HAVE_UGEN
-	destroy_dev_sched_cb(udev->default_dev, usb2_cdev_cleanup,
-	    udev->default_dev->si_drv1);
-#endif
+	/* the following will get the device unconfigured in software */
+	usb2_unconfigure(udev, flag);
 
 	/* unsetup any leftover default USB transfers */
 	usb2_transfer_unsetup(udev->default_xfer, USB_DEFAULT_XFER_MAX);
@@ -1916,7 +1979,9 @@ usb2_free_device(struct usb2_device *udev)
 	usb2_cv_destroy(udev->default_cv + 1);
 
 	mtx_destroy(udev->default_mtx);
+#if USB_HAVE_UGEN
 	KASSERT(LIST_FIRST(&udev->pd_list) == NULL, ("leaked cdev entries"));
+#endif
 
 	/* free device */
 	free(udev, M_USB);
@@ -1937,12 +2002,8 @@ usb2_get_iface(struct usb2_device *udev, uint8_t iface_index)
 {
 	struct usb2_interface *iface = udev->ifaces + iface_index;
 
-	if ((iface < udev->ifaces) ||
-	    (iface_index >= USB_IFACE_MAX) ||
-	    (udev->cdesc == NULL) ||
-	    (iface_index >= udev->cdesc->bNumInterface)) {
+	if (iface_index >= udev->ifaces_max)
 		return (NULL);
-	}
 	return (iface);
 }
 
@@ -2019,20 +2080,31 @@ usb2_devinfo(struct usb2_device *udev, char *dst_ptr, uint16_t dst_len)
 
 	if (udd->bDeviceClass != 0xFF) {
 		snprintf(dst_ptr, dst_len, "%s %s, class %d/%d, rev %x.%02x/"
-		    "%x.%02x, addr %d", udev->manufacturer, udev->product,
+		    "%x.%02x, addr %d",
+#if USB_HAVE_STRINGS
+		    udev->manufacturer, udev->product,
+#else
+		    "-", "-",
+#endif
 		    udd->bDeviceClass, udd->bDeviceSubClass,
 		    (bcdUSB >> 8), bcdUSB & 0xFF,
 		    (bcdDevice >> 8), bcdDevice & 0xFF,
 		    udev->address);
 	} else {
 		snprintf(dst_ptr, dst_len, "%s %s, rev %x.%02x/"
-		    "%x.%02x, addr %d", udev->manufacturer, udev->product,
+		    "%x.%02x, addr %d",
+#if USB_HAVE_STRINGS
+		    udev->manufacturer, udev->product,
+#else
+		    "-", "-",
+#endif
 		    (bcdUSB >> 8), bcdUSB & 0xFF,
 		    (bcdDevice >> 8), bcdDevice & 0xFF,
 		    udev->address);
 	}
 }
 
+#if USB_HAVE_STRINGS
 #if USB_VERBOSE
 /*
  * Descriptions of of known vendors and devices ("products").
@@ -2129,6 +2201,7 @@ usb2_check_strings(struct usb2_device *udev)
 		    sizeof(udev->product), "product 0x%04x", product_id);
 	}
 }
+#endif
 
 /*
  * Returns:
@@ -2221,6 +2294,7 @@ usb2_get_device_index(struct usb2_device *udev)
 	return (udev->device_index);
 }
 
+#if USB_HAVE_UGEN
 /*------------------------------------------------------------------------*
  *	usb2_notify_addq
  *
@@ -2243,65 +2317,48 @@ usb2_notify_addq(const char *type, struct usb2_device *udev)
 		return;
 
 	/* String it all together. */
-	if (udev->parent_hub) {
-		snprintf(data, 1024,
-		    "%s"
-		    "%s "
-		    "vendor=0x%04x "
-		    "product=0x%04x "
-		    "devclass=0x%02x "
-		    "devsubclass=0x%02x "
-		    "sernum=\"%s\" "
-		    "at "
-		    "port=%u "
-		    "on "
-		    "%s\n",
-		    type,
-		    udev->ugen_name,
-		    UGETW(udev->ddesc.idVendor),
-		    UGETW(udev->ddesc.idProduct),
-		    udev->ddesc.bDeviceClass,
-		    udev->ddesc.bDeviceSubClass,
-		    udev->serial,
-		    udev->port_no,
-		    udev->parent_hub->ugen_name);
-	} else {
-		snprintf(data, 1024,
-		    "%s"
-		    "%s "
-		    "vendor=0x%04x "
-		    "product=0x%04x "
-		    "devclass=0x%02x "
-		    "devsubclass=0x%02x "
-		    "sernum=\"%s\" "
-		    "at port=%u "
-		    "on "
-		    "%s\n",
-		    type,
-		    udev->ugen_name,
-		    UGETW(udev->ddesc.idVendor),
-		    UGETW(udev->ddesc.idProduct),
-		    udev->ddesc.bDeviceClass,
-		    udev->ddesc.bDeviceSubClass,
-		    udev->serial,
-		    udev->port_no,
-		    device_get_nameunit(device_get_parent(udev->bus->bdev)));
-	}
+	snprintf(data, 1024,
+	    "%s"
+	    "%s "
+	    "vendor=0x%04x "
+	    "product=0x%04x "
+	    "devclass=0x%02x "
+	    "devsubclass=0x%02x "
+	    "sernum=\"%s\" "
+	    "at "
+	    "port=%u "
+	    "on "
+	    "%s\n",
+	    type,
+	    udev->ugen_name,
+	    UGETW(udev->ddesc.idVendor),
+	    UGETW(udev->ddesc.idProduct),
+	    udev->ddesc.bDeviceClass,
+	    udev->ddesc.bDeviceSubClass,
+#if USB_HAVE_STRINGS
+	    udev->serial,
+#else
+	    "",
+#endif
+	    udev->port_no,
+	    udev->parent_hub != NULL ?
+		udev->parent_hub->ugen_name :
+		device_get_nameunit(device_get_parent(udev->bus->bdev)));
+
 	devctl_queue_data(data);
 }
 
-#if USB_HAVE_UGEN
 /*------------------------------------------------------------------------*
  *	usb2_fifo_free_wrap
  *
  * This function will free the FIFOs.
  *
- * Flag values, if "iface_index" is equal to "USB_IFACE_INDEX_ANY".
- * 0: Free all FIFOs except generic control endpoints.
- * 1: Free all FIFOs.
- *
- * Flag values, if "iface_index" is not equal to "USB_IFACE_INDEX_ANY".
- * Not used.
+ * Description of "flag" argument: If the USB_UNCFG_FLAG_FREE_EP0 flag
+ * is set and "iface_index" is set to "USB_IFACE_INDEX_ANY", we free
+ * all FIFOs. If the USB_UNCFG_FLAG_FREE_EP0 flag is not set and
+ * "iface_index" is set to "USB_IFACE_INDEX_ANY", we free all non
+ * control endpoint FIFOs. If "iface_index" is not set to
+ * "USB_IFACE_INDEX_ANY" the flag has no effect.
  *------------------------------------------------------------------------*/
 static void
 usb2_fifo_free_wrap(struct usb2_device *udev,
@@ -2334,7 +2391,8 @@ usb2_fifo_free_wrap(struct usb2_device *udev,
 			}
 		} else if (iface_index == USB_IFACE_INDEX_ANY) {
 			if ((f->methods == &usb2_ugen_methods) &&
-			    (f->dev_ep_index == 0) && (flag == 0) &&
+			    (f->dev_ep_index == 0) &&
+			    (!(flag & USB_UNCFG_FLAG_FREE_EP0)) &&
 			    (f->fs_xfer == NULL)) {
 				/* no need to free this FIFO */
 				continue;
diff --git a/sys/dev/usb/usb_device.h b/sys/dev/usb/usb_device.h
index 8ebf9e5..5ae9779 100644
--- a/sys/dev/usb/usb_device.h
+++ b/sys/dev/usb/usb_device.h
@@ -27,10 +27,23 @@
 #ifndef _USB2_DEVICE_H_
 #define	_USB2_DEVICE_H_
 
-struct usb2_symlink;
+struct usb2_symlink;		/* UGEN */
+struct usb_device;		/* linux compat */
 
 #define	USB_DEFAULT_XFER_MAX 2
 
+/* "usb2_parse_config()" commands */
+
+#define	USB_CFG_ALLOC 0
+#define	USB_CFG_FREE 1
+#define	USB_CFG_INIT 2
+
+/* "usb2_unconfigure()" flags */
+
+#define	USB_UNCFG_FLAG_NONE 0x00
+#define	USB_UNCFG_FLAG_FREE_SUBDEV 0x01		/* subdevices are freed */
+#define	USB_UNCFG_FLAG_FREE_EP0	0x02		/* endpoint zero is freed */
+
 struct usb2_clear_stall_msg {
 	struct usb2_proc_msg hdr;
 	struct usb2_device *udev;
@@ -43,7 +56,6 @@ struct usb2_clear_stall_msg {
 struct usb2_pipe {
 	struct usb2_xfer_queue pipe_q;	/* queue of USB transfers */
 
-	struct usb2_xfer *xfer_block;	/* blocking USB transfer */
 	struct usb2_endpoint_descriptor *edesc;
 	struct usb2_pipe_methods *methods;	/* set by HC driver */
 
@@ -78,7 +90,6 @@ struct usb2_device_flags {
 					 * strings */
 	uint8_t	remote_wakeup:1;	/* set if remote wakeup is enabled */
 	uint8_t	uq_bus_powered:1;	/* set if BUS powered quirk is present */
-	uint8_t	uq_power_claim:1;	/* set if power claim quirk is present */
 };
 
 /*
@@ -86,10 +97,10 @@ struct usb2_device_flags {
  * in this structure is protected by the USB BUS lock.
  */
 struct usb2_power_save {
-	int	last_xfer_time;		/* copy of "ticks" */
-	uint32_t type_refs[4];		/* transfer reference count */
-	uint32_t read_refs;		/* data read references */
-	uint32_t write_refs;		/* data write references */
+	usb2_ticks_t last_xfer_time;	/* copy of "ticks" */
+	usb2_size_t type_refs[4];	/* transfer reference count */
+	usb2_size_t read_refs;		/* data read references */
+	usb2_size_t write_refs;		/* data write references */
 	uint8_t	suspended;		/* set if USB device is suspended */
 };
 
@@ -103,32 +114,30 @@ struct usb2_device {
 	struct sx default_sx[2];
 	struct mtx default_mtx[1];
 	struct cv default_cv[2];
-	struct usb2_interface ifaces[USB_IFACE_MAX];
+	struct usb2_interface *ifaces;
 	struct usb2_pipe default_pipe;	/* Control Endpoint 0 */
-	struct cdev *default_dev;	/* Control Endpoint 0 device node */
-	struct usb2_pipe pipes[USB_EP_MAX];
+	struct usb2_pipe *pipes;
 	struct usb2_power_save pwr_save;/* power save data */
-
 	struct usb2_bus *bus;		/* our USB BUS */
 	device_t parent_dev;		/* parent device */
 	struct usb2_device *parent_hub;
 	struct usb2_config_descriptor *cdesc;	/* full config descr */
 	struct usb2_hub *hub;		/* only if this is a hub */
+#if USB_HAVE_COMPAT_LINUX
 	struct usb_device *linux_dev;
+#endif
 	struct usb2_xfer *default_xfer[USB_DEFAULT_XFER_MAX];
 	struct usb2_temp_data *usb2_template_ptr;
 	struct usb2_pipe *pipe_curr;	/* current clear stall pipe */
+#if USB_HAVE_UGEN
 	struct usb2_fifo *fifo[USB_FIFO_MAX];
-
-	char ugen_name[20];			/* name of ugenX.X device */
 	struct usb2_symlink *ugen_symlink;	/* our generic symlink */
-
+	struct cdev *default_dev;	/* Control Endpoint 0 device node */
 	LIST_HEAD(,usb2_fs_privdata) pd_list;
+	char	ugen_name[20];		/* name of ugenX.X device */
+#endif
+	usb2_ticks_t plugtime;		/* copy of "ticks" */
 
-	uint32_t plugtime;		/* copy of "ticks" */
-
-	uint16_t ep_rd_opened;		/* bitmask of endpoints opened */
-	uint16_t ep_wr_opened;		/*  from the device nodes. */
 	uint16_t refcount;
 #define	USB_DEV_REF_MAX 0xffff
 
@@ -147,6 +156,8 @@ struct usb2_device {
 	uint8_t	hs_port_no;		/* high-speed HUB port number */
 	uint8_t	driver_added_refcount;	/* our driver added generation count */
 	uint8_t	power_mode;		/* see USB_POWER_XXX */
+	uint8_t ifaces_max;		/* number of interfaces present */
+	uint8_t pipes_max;		/* number of pipes present */
 
 	/* the "flags" field is write-protected by "bus->mtx" */
 
@@ -155,9 +166,11 @@ struct usb2_device {
 	struct usb2_endpoint_descriptor default_ep_desc;	/* for pipe 0 */
 	struct usb2_device_descriptor ddesc;	/* device descriptor */
 
+#if USB_HAVE_STRINGS
 	char	serial[64];		/* serial number */
 	char	manufacturer[64];	/* manufacturer string */
 	char	product[64];		/* product string */
+#endif
 };
 
 /* globals */
@@ -183,10 +196,8 @@ usb2_error_t	usb2_set_endpoint_stall(struct usb2_device *udev,
 		    struct usb2_pipe *pipe, uint8_t do_stall);
 usb2_error_t	usb2_suspend_resume(struct usb2_device *udev,
 		    uint8_t do_suspend);
-void	usb2_detach_device(struct usb2_device *udev, uint8_t iface_index,
-	    uint8_t free_subdev);
 void	usb2_devinfo(struct usb2_device *udev, char *dst_ptr, uint16_t dst_len);
-void	usb2_free_device(struct usb2_device *udev);
+void	usb2_free_device(struct usb2_device *, uint8_t);
 void	*usb2_find_descriptor(struct usb2_device *udev, void *id,
 	    uint8_t iface_index, uint8_t type, uint8_t type_mask,
 	    uint8_t subtype, uint8_t subtype_mask);
diff --git a/sys/dev/usb/usb_dynamic.c b/sys/dev/usb/usb_dynamic.c
index 6983f64..8c0a644 100644
--- a/sys/dev/usb/usb_dynamic.c
+++ b/sys/dev/usb/usb_dynamic.c
@@ -24,7 +24,6 @@
  * SUCH DAMAGE.
  */
 
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
 #include <dev/usb/usb.h>
diff --git a/sys/dev/usb/usb_endian.h b/sys/dev/usb/usb_endian.h
index 2f49008..f7d2244 100644
--- a/sys/dev/usb/usb_endian.h
+++ b/sys/dev/usb/usb_endian.h
@@ -48,19 +48,19 @@ typedef uint8_t uQWord[8];
 
 #define	UGETW(w)			\
   ((w)[0] |				\
-  ((w)[1] << 8))
+  (((uint16_t)((w)[1])) << 8))
 
 #define	UGETDW(w)			\
   ((w)[0] |				\
-  ((w)[1] << 8) |			\
-  ((w)[2] << 16) |			\
-  ((w)[3] << 24))
+  (((uint16_t)((w)[1])) << 8) |		\
+  (((uint32_t)((w)[2])) << 16) |	\
+  (((uint32_t)((w)[3])) << 24))
 
 #define	UGETQW(w)			\
   ((w)[0] |				\
-  ((w)[1] << 8) |			\
-  ((w)[2] << 16) |			\
-  ((w)[3] << 24) |			\
+  (((uint16_t)((w)[1])) << 8) |		\
+  (((uint32_t)((w)[2])) << 16) |	\
+  (((uint32_t)((w)[3])) << 24) |	\
   (((uint64_t)((w)[4])) << 32) |	\
   (((uint64_t)((w)[5])) << 40) |	\
   (((uint64_t)((w)[6])) << 48) |	\
diff --git a/sys/dev/usb/usb_generic.c b/sys/dev/usb/usb_generic.c
index c7cf056..27de49b 100644
--- a/sys/dev/usb/usb_generic.c
+++ b/sys/dev/usb/usb_generic.c
@@ -24,7 +24,6 @@
  * SUCH DAMAGE.
  */
 
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb.h>
 #include <dev/usb/usb_ioctl.h>
@@ -48,6 +47,8 @@
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
 
+#if USB_HAVE_UGEN
+
 /* defines */
 
 #define	UGEN_BULK_FS_BUFFER_SIZE	(64*32)	/* bytes */
@@ -221,28 +222,29 @@ ugen_open_pipe_write(struct usb2_fifo *f)
 	usb2_config[1].type = UE_CONTROL;
 	usb2_config[1].endpoint = 0;
 	usb2_config[1].direction = UE_DIR_ANY;
-	usb2_config[1].mh.timeout = 1000;	/* 1 second */
-	usb2_config[1].mh.interval = 50;/* 50 milliseconds */
-	usb2_config[1].mh.bufsize = sizeof(struct usb2_device_request);
-	usb2_config[1].mh.callback = &ugen_write_clear_stall_callback;
+	usb2_config[1].timeout = 1000;	/* 1 second */
+	usb2_config[1].interval = 50;/* 50 milliseconds */
+	usb2_config[1].bufsize = sizeof(struct usb2_device_request);
+	usb2_config[1].callback = &ugen_write_clear_stall_callback;
+	usb2_config[1].usb_mode = USB_MODE_HOST;
 
 	usb2_config[0].type = ed->bmAttributes & UE_XFERTYPE;
 	usb2_config[0].endpoint = ed->bEndpointAddress & UE_ADDR;
-	usb2_config[0].direction = ed->bEndpointAddress & (UE_DIR_OUT | UE_DIR_IN);
-	usb2_config[0].mh.interval = USB_DEFAULT_INTERVAL;
-	usb2_config[0].mh.flags.proxy_buffer = 1;
+	usb2_config[0].direction = UE_DIR_TX;
+	usb2_config[0].interval = USB_DEFAULT_INTERVAL;
+	usb2_config[0].flags.proxy_buffer = 1;
+	usb2_config[0].usb_mode = USB_MODE_MAX;		/* both modes */
 
 	switch (ed->bmAttributes & UE_XFERTYPE) {
 	case UE_INTERRUPT:
 	case UE_BULK:
 		if (f->flag_short) {
-			usb2_config[0].mh.flags.force_short_xfer = 1;
+			usb2_config[0].flags.force_short_xfer = 1;
 		}
-		usb2_config[0].mh.callback = &ugen_default_write_callback;
-		usb2_config[0].mh.timeout = f->timeout;
-		usb2_config[0].mh.frames = 1;
-		usb2_config[0].mh.bufsize = f->bufsize;
-		usb2_config[0].md = usb2_config[0].mh;	/* symmetric config */
+		usb2_config[0].callback = &ugen_default_write_callback;
+		usb2_config[0].timeout = f->timeout;
+		usb2_config[0].frames = 1;
+		usb2_config[0].bufsize = f->bufsize;
 		if (ugen_transfer_setup(f, usb2_config, 2)) {
 			return (EIO);
 		}
@@ -251,12 +253,11 @@ ugen_open_pipe_write(struct usb2_fifo *f)
 		break;
 
 	case UE_ISOCHRONOUS:
-		usb2_config[0].mh.flags.short_xfer_ok = 1;
-		usb2_config[0].mh.bufsize = 0;	/* use default */
-		usb2_config[0].mh.frames = f->nframes;
-		usb2_config[0].mh.callback = &ugen_isoc_write_callback;
-		usb2_config[0].mh.timeout = 0;
-		usb2_config[0].md = usb2_config[0].mh;	/* symmetric config */
+		usb2_config[0].flags.short_xfer_ok = 1;
+		usb2_config[0].bufsize = 0;	/* use default */
+		usb2_config[0].frames = f->nframes;
+		usb2_config[0].callback = &ugen_isoc_write_callback;
+		usb2_config[0].timeout = 0;
 
 		/* clone configuration */
 		usb2_config[1] = usb2_config[0];
@@ -289,28 +290,29 @@ ugen_open_pipe_read(struct usb2_fifo *f)
 	usb2_config[1].type = UE_CONTROL;
 	usb2_config[1].endpoint = 0;
 	usb2_config[1].direction = UE_DIR_ANY;
-	usb2_config[1].mh.timeout = 1000;	/* 1 second */
-	usb2_config[1].mh.interval = 50;/* 50 milliseconds */
-	usb2_config[1].mh.bufsize = sizeof(struct usb2_device_request);
-	usb2_config[1].mh.callback = &ugen_read_clear_stall_callback;
+	usb2_config[1].timeout = 1000;	/* 1 second */
+	usb2_config[1].interval = 50;/* 50 milliseconds */
+	usb2_config[1].bufsize = sizeof(struct usb2_device_request);
+	usb2_config[1].callback = &ugen_read_clear_stall_callback;
+	usb2_config[1].usb_mode = USB_MODE_HOST;
 
 	usb2_config[0].type = ed->bmAttributes & UE_XFERTYPE;
 	usb2_config[0].endpoint = ed->bEndpointAddress & UE_ADDR;
-	usb2_config[0].direction = UE_DIR_IN;
-	usb2_config[0].mh.interval = USB_DEFAULT_INTERVAL;
-	usb2_config[0].mh.flags.proxy_buffer = 1;
+	usb2_config[0].direction = UE_DIR_RX;
+	usb2_config[0].interval = USB_DEFAULT_INTERVAL;
+	usb2_config[0].flags.proxy_buffer = 1;
+	usb2_config[0].usb_mode = USB_MODE_MAX;		/* both modes */
 
 	switch (ed->bmAttributes & UE_XFERTYPE) {
 	case UE_INTERRUPT:
 	case UE_BULK:
 		if (f->flag_short) {
-			usb2_config[0].mh.flags.short_xfer_ok = 1;
+			usb2_config[0].flags.short_xfer_ok = 1;
 		}
-		usb2_config[0].mh.timeout = f->timeout;
-		usb2_config[0].mh.frames = 1;
-		usb2_config[0].mh.callback = &ugen_default_read_callback;
-		usb2_config[0].mh.bufsize = f->bufsize;
-		usb2_config[0].md = usb2_config[0].mh;	/* symmetric config */
+		usb2_config[0].timeout = f->timeout;
+		usb2_config[0].frames = 1;
+		usb2_config[0].callback = &ugen_default_read_callback;
+		usb2_config[0].bufsize = f->bufsize;
 
 		if (ugen_transfer_setup(f, usb2_config, 2)) {
 			return (EIO);
@@ -320,12 +322,11 @@ ugen_open_pipe_read(struct usb2_fifo *f)
 		break;
 
 	case UE_ISOCHRONOUS:
-		usb2_config[0].mh.flags.short_xfer_ok = 1;
-		usb2_config[0].mh.bufsize = 0;	/* use default */
-		usb2_config[0].mh.frames = f->nframes;
-		usb2_config[0].mh.callback = &ugen_isoc_read_callback;
-		usb2_config[0].mh.timeout = 0;
-		usb2_config[0].md = usb2_config[0].mh;	/* symmetric config */
+		usb2_config[0].flags.short_xfer_ok = 1;
+		usb2_config[0].bufsize = 0;	/* use default */
+		usb2_config[0].frames = f->nframes;
+		usb2_config[0].callback = &ugen_isoc_read_callback;
+		usb2_config[0].timeout = 0;
 
 		/* clone configuration */
 		usb2_config[1] = usb2_config[0];
@@ -417,6 +418,8 @@ ugen_default_read_callback(struct usb2_xfer *xfer)
 
 	default:			/* Error */
 		if (xfer->error != USB_ERR_CANCELLED) {
+			/* send a zero length packet to userland */
+			usb2_fifo_put_data(f, xfer->frbuffers, 0, 0, 1);
 			f->flag_stall = 1;
 			f->fifo_zlp = 0;
 			usb2_transfer_start(f->xfer[1]);
@@ -429,7 +432,7 @@ static void
 ugen_default_write_callback(struct usb2_xfer *xfer)
 {
 	struct usb2_fifo *f = xfer->priv_sc;
-	uint32_t actlen;
+	usb2_frlength_t actlen;
 
 	DPRINTFN(4, "actlen=%u, aframes=%u\n", xfer->actlen, xfer->aframes);
 
@@ -501,8 +504,8 @@ static void
 ugen_isoc_read_callback(struct usb2_xfer *xfer)
 {
 	struct usb2_fifo *f = xfer->priv_sc;
-	uint32_t offset;
-	uint16_t n;
+	usb2_frlength_t offset;
+	usb2_frcount_t n;
 
 	DPRINTFN(4, "actlen=%u, aframes=%u\n", xfer->actlen, xfer->aframes);
 
@@ -540,9 +543,9 @@ static void
 ugen_isoc_write_callback(struct usb2_xfer *xfer)
 {
 	struct usb2_fifo *f = xfer->priv_sc;
-	uint32_t actlen;
-	uint32_t offset;
-	uint16_t n;
+	usb2_frlength_t actlen;
+	usb2_frlength_t offset;
+	usb2_frcount_t n;
 
 	DPRINTFN(4, "actlen=%u, aframes=%u\n", xfer->actlen, xfer->aframes);
 
@@ -798,12 +801,14 @@ usb2_gen_fill_deviceinfo(struct usb2_fifo *f, struct usb2_device_info *di)
 	di->udi_bus = device_get_unit(udev->bus->bdev);
 	di->udi_addr = udev->address;
 	di->udi_index = udev->device_index;
+#if USB_HAVE_STRINGS
 	strlcpy(di->udi_serial, udev->serial,
 	    sizeof(di->udi_serial));
 	strlcpy(di->udi_vendor, udev->manufacturer,
 	    sizeof(di->udi_vendor));
 	strlcpy(di->udi_product, udev->product,
 	    sizeof(di->udi_product));
+#endif
 	usb2_printBCD(di->udi_release, sizeof(di->udi_release),
 	    UGETW(udev->ddesc.bcdDevice));
 	di->udi_vendorNo = UGETW(udev->ddesc.idVendor);
@@ -1021,10 +1026,10 @@ ugen_fs_copy_in(struct usb2_fifo *f, uint8_t ep_index)
 	struct usb2_fs_endpoint fs_ep;
 	void *uaddr;			/* userland pointer */
 	void *kaddr;
-	uint32_t offset;
+	usb2_frlength_t offset;
+	usb2_frlength_t rem;
+	usb2_frcount_t n;
 	uint32_t length;
-	uint32_t n;
-	uint32_t rem;
 	int error;
 	uint8_t isread;
 
@@ -1198,21 +1203,21 @@ ugen_fs_copy_out(struct usb2_fifo *f, uint8_t ep_index)
 	struct usb2_fs_endpoint *fs_ep_uptr;	/* userland ptr */
 	void *uaddr;			/* userland ptr */
 	void *kaddr;
-	uint32_t offset;
+	usb2_frlength_t offset;
+	usb2_frlength_t rem;
+	usb2_frcount_t n;
 	uint32_t length;
 	uint32_t temp;
-	uint32_t n;
-	uint32_t rem;
 	int error;
 	uint8_t isread;
 
-	if (ep_index >= f->fs_ep_max) {
+	if (ep_index >= f->fs_ep_max)
 		return (EINVAL);
-	}
+
 	xfer = f->fs_xfer[ep_index];
-	if (xfer == NULL) {
+	if (xfer == NULL)
 		return (EINVAL);
-	}
+
 	mtx_lock(f->priv_mtx);
 	if (usb2_transfer_pending(xfer)) {
 		mtx_unlock(f->priv_mtx);
@@ -1458,13 +1463,13 @@ ugen_ioctl(struct usb2_fifo *f, u_long cmd, void *addr, int fflags)
 		usb2_config[0].type = ed->bmAttributes & UE_XFERTYPE;
 		usb2_config[0].endpoint = ed->bEndpointAddress & UE_ADDR;
 		usb2_config[0].direction = ed->bEndpointAddress & (UE_DIR_OUT | UE_DIR_IN);
-		usb2_config[0].mh.interval = USB_DEFAULT_INTERVAL;
-		usb2_config[0].mh.flags.proxy_buffer = 1;
-		usb2_config[0].mh.callback = &ugen_default_fs_callback;
-		usb2_config[0].mh.timeout = 0;	/* no timeout */
-		usb2_config[0].mh.frames = u.popen->max_frames;
-		usb2_config[0].mh.bufsize = u.popen->max_bufsize;
-		usb2_config[0].md = usb2_config[0].mh;	/* symmetric config */
+		usb2_config[0].interval = USB_DEFAULT_INTERVAL;
+		usb2_config[0].flags.proxy_buffer = 1;
+		usb2_config[0].callback = &ugen_default_fs_callback;
+		usb2_config[0].timeout = 0;	/* no timeout */
+		usb2_config[0].frames = u.popen->max_frames;
+		usb2_config[0].bufsize = u.popen->max_bufsize;
+		usb2_config[0].usb_mode = USB_MODE_MAX;		/* both modes */
 
 		if (usb2_config[0].type == UE_CONTROL) {
 			if (f->udev->flags.usb2_mode != USB_MODE_HOST) {
@@ -1615,10 +1620,10 @@ ugen_get_frame_size(struct usb2_fifo *f, void *addr)
 static int
 ugen_set_buffer_size(struct usb2_fifo *f, void *addr)
 {
-	uint32_t t;
+	usb2_frlength_t t;
 
-	if (*(int *)addr < 1024)
-		t = 1024;
+	if (*(int *)addr < 0)
+		t = 0;		/* use "wMaxPacketSize" */
 	else if (*(int *)addr < (256 * 1024))
 		t = *(int *)addr;
 	else
@@ -2182,3 +2187,4 @@ ugen_default_fs_callback(struct usb2_xfer *xfer)
 		break;
 	}
 }
+#endif	/* USB_HAVE_UGEN */
diff --git a/sys/dev/usb/usb_handle_request.c b/sys/dev/usb/usb_handle_request.c
index 3d2425a..7a9a697 100644
--- a/sys/dev/usb/usb_handle_request.c
+++ b/sys/dev/usb/usb_handle_request.c
@@ -24,7 +24,6 @@
  * SUCH DAMAGE.
  */
 
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
 #include <dev/usb/usb.h>
@@ -212,7 +211,7 @@ tr_repeat:
 	    (iface->subdev != NULL) &&
 	    device_is_attached(iface->subdev)) {
 #if 0
-		DEVMETHOD(usb2_handle_request, NULL);	/* dummy */
+		DEVMETHOD(usb_handle_request, NULL);	/* dummy */
 #endif
 		error = USB_HANDLE_REQUEST(iface->subdev,
 		    &req, ppdata, plen,
@@ -399,9 +398,7 @@ usb2_handle_remote_wakeup(struct usb2_xfer *xfer, uint8_t is_on)
 	USB_BUS_UNLOCK(bus);
 
 	/* In case we are out of sync, update the power state. */
-
 	usb2_bus_power_update(udev->bus);
-
 	return (0);			/* success */
 }
 
diff --git a/sys/dev/usb/usb_hid.c b/sys/dev/usb/usb_hid.c
index 5ac1930..555d324 100644
--- a/sys/dev/usb/usb_hid.c
+++ b/sys/dev/usb/usb_hid.c
@@ -43,7 +43,6 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usbhid.h>
 
 #define	USB_DEBUG_VAR usb2_debug
@@ -106,7 +105,7 @@ hid_clear_local(struct hid_item *c)
  *	hid_start_parse
  *------------------------------------------------------------------------*/
 struct hid_data *
-hid_start_parse(const void *d, int len, int kindset)
+hid_start_parse(const void *d, usb2_size_t len, int kindset)
 {
 	struct hid_data *s;
 
@@ -441,7 +440,7 @@ hid_get_item(struct hid_data *s, struct hid_item *h)
 
 				/* sanity check */
 				if ((s->nusage < MAXUSAGE) &&
-				    (c->usage_minimum < c->usage_maximum)) {
+				    (c->usage_minimum <= c->usage_maximum)) {
 					/* add usage range */
 					s->usages_min[s->nusage] = 
 					    c->usage_minimum;
@@ -491,7 +490,7 @@ hid_get_item(struct hid_data *s, struct hid_item *h)
  *	hid_report_size
  *------------------------------------------------------------------------*/
 int
-hid_report_size(const void *buf, int len, enum hid_kind k, uint8_t *id)
+hid_report_size(const void *buf, usb2_size_t len, enum hid_kind k, uint8_t *id)
 {
 	struct hid_data *d;
 	struct hid_item h;
@@ -544,14 +543,16 @@ hid_report_size(const void *buf, int len, enum hid_kind k, uint8_t *id)
  *	hid_locate
  *------------------------------------------------------------------------*/
 int
-hid_locate(const void *desc, int size, uint32_t u, enum hid_kind k,
-    struct hid_location *loc, uint32_t *flags, uint8_t *id)
+hid_locate(const void *desc, usb2_size_t size, uint32_t u, enum hid_kind k,
+    uint8_t index, struct hid_location *loc, uint32_t *flags, uint8_t *id)
 {
 	struct hid_data *d;
 	struct hid_item h;
 
 	for (d = hid_start_parse(desc, size, 1 << k); hid_get_item(d, &h);) {
 		if (h.kind == k && !(h.flags & HIO_CONST) && h.usage == u) {
+			if (index--)
+				continue;
 			if (loc != NULL)
 				*loc = h.loc;
 			if (flags != NULL)
@@ -576,7 +577,7 @@ hid_locate(const void *desc, int size, uint32_t u, enum hid_kind k,
  *	hid_get_data
  *------------------------------------------------------------------------*/
 uint32_t
-hid_get_data(const uint8_t *buf, uint32_t len, struct hid_location *loc)
+hid_get_data(const uint8_t *buf, usb2_size_t len, struct hid_location *loc)
 {
 	uint32_t hpos = loc->pos;
 	uint32_t hsize = loc->size;
@@ -619,7 +620,7 @@ hid_get_data(const uint8_t *buf, uint32_t len, struct hid_location *loc)
  *	hid_is_collection
  *------------------------------------------------------------------------*/
 int
-hid_is_collection(const void *desc, int size, uint32_t usage)
+hid_is_collection(const void *desc, usb2_size_t size, uint32_t usage)
 {
 	struct hid_data *hd;
 	struct hid_item hi;
diff --git a/sys/dev/usb/usb_hid.h b/sys/dev/usb/usb_hid.h
index 0711c47..8febc4e 100644
--- a/sys/dev/usb/usb_hid.h
+++ b/sys/dev/usb/usb_hid.h
@@ -74,17 +74,17 @@ struct hid_item {
 
 /* prototypes from "usb2_hid.c" */
 
-struct hid_data *hid_start_parse(const void *d, int len, int kindset);
+struct hid_data *hid_start_parse(const void *d, usb2_size_t len, int kindset);
 void	hid_end_parse(struct hid_data *s);
 int	hid_get_item(struct hid_data *s, struct hid_item *h);
-int	hid_report_size(const void *buf, int len, enum hid_kind k,
+int	hid_report_size(const void *buf, usb2_size_t len, enum hid_kind k,
 	    uint8_t *id);
-int	hid_locate(const void *desc, int size, uint32_t usage,
-	    enum hid_kind kind, struct hid_location *loc,
+int	hid_locate(const void *desc, usb2_size_t size, uint32_t usage,
+	    enum hid_kind kind, uint8_t index, struct hid_location *loc,
 	    uint32_t *flags, uint8_t *id);
-uint32_t hid_get_data(const uint8_t *buf, uint32_t len,
+uint32_t hid_get_data(const uint8_t *buf, usb2_size_t len,
 	    struct hid_location *loc);
-int	hid_is_collection(const void *desc, int size, uint32_t usage);
+int	hid_is_collection(const void *desc, usb2_size_t size, uint32_t usage);
 struct usb2_hid_descriptor *hid_get_descriptor_from_usb(
 	    struct usb2_config_descriptor *cd,
 	    struct usb2_interface_descriptor *id);
diff --git a/sys/dev/usb/usb_hub.c b/sys/dev/usb/usb_hub.c
index 67190ed..627d8a7 100644
--- a/sys/dev/usb/usb_hub.c
+++ b/sys/dev/usb/usb_hub.c
@@ -27,10 +27,9 @@
  */
 
 /*
- * USB spec: http://www.usb.org/developers/docs/usbspec.zip
+ * USB spec: http://www.usb.org/developers/docs/usbspec.zip 
  */
 
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
 #include <dev/usb/usb.h>
@@ -63,10 +62,12 @@ SYSCTL_INT(_hw_usb2_uhub, OID_AUTO, debug, CTLFLAG_RW, &uhub_debug, 0,
     "Debug level");
 #endif
 
+#if USB_HAVE_POWERD
 static int usb2_power_timeout = 30;	/* seconds */
 
 SYSCTL_INT(_hw_usb2, OID_AUTO, power_timeout, CTLFLAG_RW,
     &usb2_power_timeout, 0, "USB power timeout");
+#endif
 
 struct uhub_current_state {
 	uint16_t port_change;
@@ -110,11 +111,11 @@ static const struct usb2_config uhub_config[UHUB_N_TRANSFER] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_ANY,
-		.mh.timeout = 0,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.bufsize = 0,	/* use wMaxPacketSize */
-		.mh.callback = &uhub_intr_callback,
-		.mh.interval = UHUB_INTR_INTERVAL,
+		.timeout = 0,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.bufsize = 0,	/* use wMaxPacketSize */
+		.callback = &uhub_intr_callback,
+		.interval = UHUB_INTR_INTERVAL,
 	},
 };
 
@@ -124,23 +125,24 @@ static const struct usb2_config uhub_config[UHUB_N_TRANSFER] = {
  */
 static devclass_t uhub_devclass;
 
-static driver_t uhub_driver =
-{
+static device_method_t uhub_methods[] = {
+	DEVMETHOD(device_probe, uhub_probe),
+	DEVMETHOD(device_attach, uhub_attach),
+	DEVMETHOD(device_detach, uhub_detach),
+
+	DEVMETHOD(device_suspend, uhub_suspend),
+	DEVMETHOD(device_resume, uhub_resume),
+	DEVMETHOD(device_shutdown, bus_generic_shutdown),
+
+	DEVMETHOD(bus_child_location_str, uhub_child_location_string),
+	DEVMETHOD(bus_child_pnpinfo_str, uhub_child_pnpinfo_string),
+	DEVMETHOD(bus_driver_added, uhub_driver_added),
+	{0, 0}
+};
+
+static driver_t uhub_driver = {
 	.name = "uhub",
-	.methods = (device_method_t[]){
-		DEVMETHOD(device_probe, uhub_probe),
-		DEVMETHOD(device_attach, uhub_attach),
-		DEVMETHOD(device_detach, uhub_detach),
-
-		DEVMETHOD(device_suspend, uhub_suspend),
-		DEVMETHOD(device_resume, uhub_resume),
-		DEVMETHOD(device_shutdown, bus_generic_shutdown),
-
-		DEVMETHOD(bus_child_location_str, uhub_child_location_string),
-		DEVMETHOD(bus_child_pnpinfo_str, uhub_child_pnpinfo_string),
-		DEVMETHOD(bus_driver_added, uhub_driver_added),
-		{0, 0}
-	},
+	.methods = uhub_methods,
 	.size = sizeof(struct uhub_softc)
 };
 
@@ -298,8 +300,9 @@ repeat:
 	/* detach any existing devices */
 
 	if (child) {
-		usb2_detach_device(child, USB_IFACE_INDEX_ANY, 1);
-		usb2_free_device(child);
+		usb2_free_device(child,
+		    USB_UNCFG_FLAG_FREE_SUBDEV |
+		    USB_UNCFG_FLAG_FREE_EP0);
 		child = NULL;
 	}
 	/* get fresh status */
@@ -418,8 +421,9 @@ repeat:
 
 error:
 	if (child) {
-		usb2_detach_device(child, USB_IFACE_INDEX_ANY, 1);
-		usb2_free_device(child);
+		usb2_free_device(child,
+		    USB_UNCFG_FLAG_FREE_SUBDEV |
+		    USB_UNCFG_FLAG_FREE_EP0);
 		child = NULL;
 	}
 	if (err == 0) {
@@ -500,6 +504,21 @@ done:
 }
 
 /*------------------------------------------------------------------------*
+ *	uhub_root_interrupt
+ *
+ * This function is called when a Root HUB interrupt has
+ * happened. "ptr" and "len" makes up the Root HUB interrupt
+ * packet. This function is called having the "bus_mtx" locked.
+ *------------------------------------------------------------------------*/
+void
+uhub_root_intr(struct usb2_bus *bus, const uint8_t *ptr, uint8_t len)
+{
+	USB_BUS_LOCK_ASSERT(bus, MA_OWNED);
+
+	usb2_needs_explore(bus, 0);
+}
+
+/*------------------------------------------------------------------------*
  *	uhub_explore
  *
  * Returns:
@@ -711,9 +730,10 @@ uhub_attach(device_t dev)
 	}
 	udev->hub = hub;
 
+#if USB_HAVE_TT_SUPPORT
 	/* init FULL-speed ISOCHRONOUS schedule */
 	usb2_fs_isoc_schedule_init_all(hub->fs_isoc_schedule);
-
+#endif
 	/* initialize HUB structure */
 	hub->hubsoftc = sc;
 	hub->explore = &uhub_explore;
@@ -729,8 +749,14 @@ uhub_attach(device_t dev)
 
 	/* set up interrupt pipe */
 	iface_index = 0;
-	err = usb2_transfer_setup(udev, &iface_index, sc->sc_xfer,
-	    uhub_config, UHUB_N_TRANSFER, sc, &Giant);
+	if (udev->parent_hub == NULL) {
+		/* root HUB is special */
+		err = 0;
+	} else {
+		/* normal HUB */
+		err = usb2_transfer_setup(udev, &iface_index, sc->sc_xfer,
+		    uhub_config, UHUB_N_TRANSFER, sc, &Giant);
+	}
 	if (err) {
 		DPRINTFN(0, "cannot setup interrupt transfer, "
 		    "errstr=%s!\n", usb2_errstr(err));
@@ -802,11 +828,13 @@ uhub_attach(device_t dev)
 	    "removable, %s powered\n", nports, (nports != 1) ? "s" : "",
 	    removable, udev->flags.self_powered ? "self" : "bus");
 
-	/* start the interrupt endpoint */
+	/* Start the interrupt endpoint, if any */
 
-	USB_XFER_LOCK(sc->sc_xfer[0]);
-	usb2_transfer_start(sc->sc_xfer[0]);
-	USB_XFER_UNLOCK(sc->sc_xfer[0]);
+	if (sc->sc_xfer[0] != NULL) {
+		USB_XFER_LOCK(sc->sc_xfer[0]);
+		usb2_transfer_start(sc->sc_xfer[0]);
+		USB_XFER_UNLOCK(sc->sc_xfer[0]);
+	}
 
 	/* Enable automatic power save on all USB HUBs */
 
@@ -853,9 +881,8 @@ uhub_detach(device_t dev)
 		 * Subdevices are not freed, because the caller of
 		 * uhub_detach() will do that.
 		 */
-		usb2_detach_device(child, USB_IFACE_INDEX_ANY, 0);
-		usb2_free_device(child);
-		child = NULL;
+		usb2_free_device(child,
+		    USB_UNCFG_FLAG_FREE_EP0);
 	}
 
 	usb2_transfer_unsetup(sc->sc_xfer, UHUB_N_TRANSFER);
@@ -979,7 +1006,11 @@ uhub_child_pnpinfo_string(device_t parent, device_t child,
 		    UGETW(res.udev->ddesc.idProduct),
 		    res.udev->ddesc.bDeviceClass,
 		    res.udev->ddesc.bDeviceSubClass,
+#if USB_HAVE_STRINGS
 		    res.udev->serial,
+#else
+		    "",
+#endif
 		    iface->idesc->bInterfaceClass,
 		    iface->idesc->bInterfaceSubClass);
 	} else {
@@ -1031,9 +1062,9 @@ done:
  *   The best Transaction Translation slot for an interrupt endpoint.
  *------------------------------------------------------------------------*/
 static uint8_t
-usb2_intr_find_best_slot(uint32_t *ptr, uint8_t start, uint8_t end)
+usb2_intr_find_best_slot(usb2_size_t *ptr, uint8_t start, uint8_t end)
 {
-	uint32_t max = 0xffffffff;
+	usb2_size_t max = 0 - 1;
 	uint8_t x;
 	uint8_t y;
 
@@ -1111,6 +1142,7 @@ usb2_intr_schedule_adjust(struct usb2_device *udev, int16_t len, uint8_t slot)
  * This function initialises an USB FULL speed isochronous schedule
  * entry.
  *------------------------------------------------------------------------*/
+#if USB_HAVE_TT_SUPPORT
 static void
 usb2_fs_isoc_schedule_init_sub(struct usb2_fs_isoc_schedule *fss)
 {
@@ -1119,6 +1151,7 @@ usb2_fs_isoc_schedule_init_sub(struct usb2_fs_isoc_schedule *fss)
 	fss->frame_bytes = (USB_FS_BYTES_PER_HS_UFRAME);
 	fss->frame_slot = 0;
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *	usb2_fs_isoc_schedule_init_all
@@ -1126,6 +1159,7 @@ usb2_fs_isoc_schedule_init_sub(struct usb2_fs_isoc_schedule *fss)
  * This function will reset the complete USB FULL speed isochronous
  * bandwidth schedule.
  *------------------------------------------------------------------------*/
+#if USB_HAVE_TT_SUPPORT
 void
 usb2_fs_isoc_schedule_init_all(struct usb2_fs_isoc_schedule *fss)
 {
@@ -1136,6 +1170,7 @@ usb2_fs_isoc_schedule_init_all(struct usb2_fs_isoc_schedule *fss)
 		fss++;
 	}
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *	usb2_isoc_time_expand
@@ -1183,6 +1218,7 @@ usb2_isoc_time_expand(struct usb2_bus *bus, uint16_t isoc_time_curr)
  * NOTE: This function depends on being called regularly with
  * intervals less than "USB_ISOC_TIME_MAX".
  *------------------------------------------------------------------------*/
+#if USB_HAVE_TT_SUPPORT
 uint16_t
 usb2_fs_isoc_schedule_isoc_time_expand(struct usb2_device *udev,
     struct usb2_fs_isoc_schedule **pp_start,
@@ -1229,6 +1265,7 @@ usb2_fs_isoc_schedule_isoc_time_expand(struct usb2_device *udev,
 	}
 	return (isoc_time);
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *	usb2_fs_isoc_schedule_alloc
@@ -1243,6 +1280,7 @@ usb2_fs_isoc_schedule_isoc_time_expand(struct usb2_device *udev,
  *    0: Success
  * Else: Error
  *------------------------------------------------------------------------*/
+#if USB_HAVE_TT_SUPPORT
 uint8_t
 usb2_fs_isoc_schedule_alloc(struct usb2_fs_isoc_schedule *fss,
     uint8_t *pstart, uint16_t len)
@@ -1275,6 +1313,7 @@ usb2_fs_isoc_schedule_alloc(struct usb2_fs_isoc_schedule *fss,
 	*pstart = slot;
 	return (0);			/* success */
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *	usb2_bus_port_get_device
@@ -1346,13 +1385,25 @@ usb2_bus_port_set_device(struct usb2_bus *bus, struct usb2_port *up,
 void
 usb2_needs_explore(struct usb2_bus *bus, uint8_t do_probe)
 {
+	uint8_t do_unlock;
+
 	DPRINTF("\n");
 
 	if (bus == NULL) {
 		DPRINTF("No bus pointer!\n");
 		return;
 	}
-	USB_BUS_LOCK(bus);
+	if ((bus->devices == NULL) ||
+	    (bus->devices[USB_ROOT_HUB_ADDR] == NULL)) {
+		DPRINTF("No root HUB\n");
+		return;
+	}
+	if (mtx_owned(&bus->bus_mtx)) {
+		do_unlock = 0;
+	} else {
+		USB_BUS_LOCK(bus);
+		do_unlock = 1;
+	}
 	if (do_probe) {
 		bus->do_probe = 1;
 	}
@@ -1360,7 +1411,9 @@ usb2_needs_explore(struct usb2_bus *bus, uint8_t do_probe)
 	    &bus->explore_msg[0], &bus->explore_msg[1])) {
 		/* ignore */
 	}
-	USB_BUS_UNLOCK(bus);
+	if (do_unlock) {
+		USB_BUS_UNLOCK(bus);
+	}
 }
 
 /*------------------------------------------------------------------------*
@@ -1407,11 +1460,13 @@ usb2_needs_explore_all(void)
  * properly suspended or resumed according to the device transfer
  * state.
  *------------------------------------------------------------------------*/
+#if USB_HAVE_POWERD
 void
 usb2_bus_power_update(struct usb2_bus *bus)
 {
 	usb2_needs_explore(bus, 0 /* no probe */ );
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *	usb2_transfer_power_ref
@@ -1420,10 +1475,11 @@ usb2_bus_power_update(struct usb2_bus *bus)
  * wakeup the USB device associated with the given USB transfer, if
  * needed.
  *------------------------------------------------------------------------*/
+#if USB_HAVE_POWERD
 void
 usb2_transfer_power_ref(struct usb2_xfer *xfer, int val)
 {
-	static const uint32_t power_mask[4] = {
+	static const usb2_power_mask_t power_mask[4] = {
 		[UE_CONTROL] = USB_HW_POWER_CONTROL,
 		[UE_BULK] = USB_HW_POWER_BULK,
 		[UE_INTERRUPT] = USB_HW_POWER_INTERRUPT,
@@ -1489,8 +1545,8 @@ usb2_transfer_power_ref(struct usb2_xfer *xfer, int val)
 			(udev->bus->methods->set_hw_power) (udev->bus);
 		}
 	}
-	return;
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *	usb2_bus_powerd
@@ -1498,14 +1554,15 @@ usb2_transfer_power_ref(struct usb2_xfer *xfer, int val)
  * This function implements the USB power daemon and is called
  * regularly from the USB explore thread.
  *------------------------------------------------------------------------*/
+#if USB_HAVE_POWERD
 void
 usb2_bus_powerd(struct usb2_bus *bus)
 {
 	struct usb2_device *udev;
-	unsigned int temp;
-	unsigned int limit;
-	unsigned int mintime;
-	uint32_t type_refs[5];
+	usb2_ticks_t temp;
+	usb2_ticks_t limit;
+	usb2_ticks_t mintime;
+	usb2_size_t type_refs[5];
 	uint8_t x;
 	uint8_t rem_wakeup;
 
@@ -1621,6 +1678,7 @@ usb2_bus_powerd(struct usb2_bus *bus)
 	}
 	return;
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *	usb2_dev_resume_peer
@@ -1680,6 +1738,7 @@ usb2_dev_resume_peer(struct usb2_device *udev)
 	USB_BUS_LOCK(bus);
 	/* set that this device is now resumed */
 	udev->pwr_save.suspended = 0;
+#if USB_HAVE_POWERD
 	/* make sure that we don't go into suspend right away */
 	udev->pwr_save.last_xfer_time = ticks;
 
@@ -1692,6 +1751,7 @@ usb2_dev_resume_peer(struct usb2_device *udev)
 		bus->hw_power_state |= USB_HW_POWER_INTERRUPT;
 	if (udev->pwr_save.type_refs[UE_ISOCHRONOUS] != 0)
 		bus->hw_power_state |= USB_HW_POWER_ISOC;
+#endif
 	USB_BUS_UNLOCK(bus);
 
 	if (bus->methods->set_hw_power != NULL) {
@@ -1728,7 +1788,6 @@ usb2_dev_suspend_peer(struct usb2_device *udev)
 {
 	struct usb2_device *hub;
 	struct usb2_device *child;
-	uint32_t temp;
 	int err;
 	uint8_t x;
 	uint8_t nports;
@@ -1801,6 +1860,7 @@ repeat:
 	USB_BUS_UNLOCK(udev->bus);
 
 	if (udev->bus->methods->device_suspend != NULL) {
+		usb2_timeout_t temp;
 
 		/* suspend device on the USB controller */
 		(udev->bus->methods->device_suspend) (udev);
@@ -1840,7 +1900,7 @@ usb2_set_power_mode(struct usb2_device *udev, uint8_t power_mode)
 	}
 	udev->power_mode = power_mode;	/* update copy of power mode */
 
+#if USB_HAVE_POWERD
 	usb2_bus_power_update(udev->bus);
-
-	return;
+#endif
 }
diff --git a/sys/dev/usb/usb_hub.h b/sys/dev/usb/usb_hub.h
index 87d85b3..0c8905d 100644
--- a/sys/dev/usb/usb_hub.h
+++ b/sys/dev/usb/usb_hub.h
@@ -28,7 +28,7 @@
 #define	_USB2_HUB_H_
 
 /*
- * The following structure defines an USB port.
+ * The following structure defines an USB port. 
  */
 struct usb2_port {
 	uint8_t	restartcnt;
@@ -52,11 +52,13 @@ struct usb2_fs_isoc_schedule {
  * The following structure defines an USB HUB.
  */
 struct usb2_hub {
+#if USB_HAVE_TT_SUPPORT
 	struct usb2_fs_isoc_schedule fs_isoc_schedule[USB_ISOC_TIME_MAX];
+#endif
 	struct usb2_device *hubudev;	/* the HUB device */
 	usb2_error_t (*explore) (struct usb2_device *hub);
 	void   *hubsoftc;
-	uint32_t uframe_usage[USB_HS_MICRO_FRAMES_MAX];
+	usb2_size_t uframe_usage[USB_HS_MICRO_FRAMES_MAX];
 	uint16_t portpower;		/* mA per USB port */
 	uint8_t	isoc_last_time;
 	uint8_t	nports;
@@ -76,5 +78,6 @@ void	usb2_needs_explore(struct usb2_bus *bus, uint8_t do_probe);
 void	usb2_needs_explore_all(void);
 void	usb2_bus_power_update(struct usb2_bus *bus);
 void	usb2_bus_powerd(struct usb2_bus *bus);
+void	uhub_root_intr(struct usb2_bus *, const uint8_t *, uint8_t);
 
 #endif					/* _USB2_HUB_H_ */
diff --git a/sys/dev/usb/usb_lookup.c b/sys/dev/usb/usb_lookup.c
index a8fa271..6c707ce 100644
--- a/sys/dev/usb/usb_lookup.c
+++ b/sys/dev/usb/usb_lookup.c
@@ -42,7 +42,7 @@
  * Else: Pointer to matching entry.
  *------------------------------------------------------------------------*/
 const struct usb2_device_id *
-usb2_lookup_id_by_info(const struct usb2_device_id *id, uint32_t sizeof_id,
+usb2_lookup_id_by_info(const struct usb2_device_id *id, usb2_size_t sizeof_id,
     const struct usb2_lookup_info *info)
 {
 	const struct usb2_device_id *id_end;
@@ -121,7 +121,7 @@ done:
  * Else: Failure
  *------------------------------------------------------------------------*/
 int
-usb2_lookup_id_by_uaa(const struct usb2_device_id *id, uint32_t sizeof_id,
+usb2_lookup_id_by_uaa(const struct usb2_device_id *id, usb2_size_t sizeof_id,
     struct usb2_attach_arg *uaa)
 {
 	id = usb2_lookup_id_by_info(id, sizeof_id, &uaa->info);
diff --git a/sys/dev/usb/usb_lookup.h b/sys/dev/usb/usb_lookup.h
index e447292..74d02e5 100644
--- a/sys/dev/usb/usb_lookup.h
+++ b/sys/dev/usb/usb_lookup.h
@@ -114,9 +114,9 @@ struct usb2_device_id {
   (((const uint8_t *)((did)->driver_info)) - ((const uint8_t *)0))
 
 const struct usb2_device_id *usb2_lookup_id_by_info(
-	    const struct usb2_device_id *id, uint32_t sizeof_id,
+	    const struct usb2_device_id *id, usb2_size_t sizeof_id,
 	    const struct usb2_lookup_info *info);
 int	usb2_lookup_id_by_uaa(const struct usb2_device_id *id,
-	    uint32_t sizeof_id, struct usb2_attach_arg *uaa);
+	    usb2_size_t sizeof_id, struct usb2_attach_arg *uaa);
 
 #endif					/* _USB2_LOOKUP_H_ */
diff --git a/sys/dev/usb/usb_mbuf.c b/sys/dev/usb/usb_mbuf.c
index 3ae6ee6..e4d92b6 100644
--- a/sys/dev/usb/usb_mbuf.c
+++ b/sys/dev/usb/usb_mbuf.c
@@ -36,12 +36,12 @@
  *------------------------------------------------------------------------*/
 void   *
 usb2_alloc_mbufs(struct malloc_type *type, struct usb2_ifqueue *ifq,
-    uint32_t block_size, uint16_t nblocks)
+    usb2_size_t block_size, uint16_t nblocks)
 {
 	struct usb2_mbuf *m_ptr;
 	uint8_t *data_ptr;
 	void *free_ptr = NULL;
-	uint32_t alloc_size;
+	usb2_size_t alloc_size;
 
 	/* align data */
 	block_size += ((-block_size) & (USB_HOST_ALIGN - 1));
diff --git a/sys/dev/usb/usb_mbuf.h b/sys/dev/usb/usb_mbuf.h
index 109340c..c748dbc 100644
--- a/sys/dev/usb/usb_mbuf.h
+++ b/sys/dev/usb/usb_mbuf.h
@@ -37,8 +37,8 @@ struct usb2_mbuf {
 	struct usb2_mbuf *usb2_nextpkt;
 	struct usb2_mbuf *usb2_next;
 
-	uint32_t cur_data_len;
-	uint32_t max_data_len;
+	usb2_size_t cur_data_len;
+	usb2_size_t max_data_len;
 	uint8_t last_packet:1;
 	uint8_t unused:7;
 };
@@ -51,8 +51,8 @@ struct usb2_ifqueue {
 	struct usb2_mbuf *ifq_head;
 	struct usb2_mbuf *ifq_tail;
 
-	uint32_t ifq_len;
-	uint32_t ifq_maxlen;
+	usb2_size_t ifq_len;
+	usb2_size_t ifq_maxlen;
 };
 
 #define	USB_IF_ENQUEUE(ifq, m) do {		\
@@ -97,6 +97,6 @@ struct usb2_ifqueue {
 
 /* prototypes */
 void   *usb2_alloc_mbufs(struct malloc_type *type, struct usb2_ifqueue *ifq,
-	    uint32_t block_size, uint16_t nblocks);
+	    usb2_size_t block_size, uint16_t nblocks);
 
 #endif					/* _USB2_MBUF_H_ */
diff --git a/sys/dev/usb/usb_msctest.c b/sys/dev/usb/usb_msctest.c
index 35b71ec..960cd95 100644
--- a/sys/dev/usb/usb_msctest.c
+++ b/sys/dev/usb/usb_msctest.c
@@ -32,7 +32,6 @@
  * mass storage quirks for not supported SCSI commands!
  */
 
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
 #include <dev/usb/usb.h>
@@ -110,10 +109,10 @@ struct bbb_transfer {
 
 	uint8_t *data_ptr;
 
-	uint32_t data_len;		/* bytes */
-	uint32_t data_rem;		/* bytes */
-	uint32_t data_timeout;		/* ms */
-	uint32_t actlen;		/* bytes */
+	usb2_size_t data_len;		/* bytes */
+	usb2_size_t data_rem;		/* bytes */
+	usb2_timeout_t data_timeout;	/* ms */
+	usb2_frlength_t actlen;		/* bytes */
 
 	uint8_t	cmd_len;		/* bytes */
 	uint8_t	dir;
@@ -138,60 +137,57 @@ static const struct usb2_config bbb_config[ST_MAX] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = sizeof(struct bbb_cbw),
-		.mh.flags = {},
-		.mh.callback = &bbb_command_callback,
-		.mh.timeout = 4 * USB_MS_HZ,	/* 4 seconds */
+		.bufsize = sizeof(struct bbb_cbw),
+		.callback = &bbb_command_callback,
+		.timeout = 4 * USB_MS_HZ,	/* 4 seconds */
 	},
 
 	[ST_DATA_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = BULK_SIZE,
-		.mh.flags = {.proxy_buffer = 1,.short_xfer_ok = 1,},
-		.mh.callback = &bbb_data_read_callback,
-		.mh.timeout = 4 * USB_MS_HZ,	/* 4 seconds */
+		.bufsize = BULK_SIZE,
+		.flags = {.proxy_buffer = 1,.short_xfer_ok = 1,},
+		.callback = &bbb_data_read_callback,
+		.timeout = 4 * USB_MS_HZ,	/* 4 seconds */
 	},
 
 	[ST_DATA_RD_CS] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &bbb_data_rd_cs_callback,
-		.mh.timeout = 1 * USB_MS_HZ,	/* 1 second  */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &bbb_data_rd_cs_callback,
+		.timeout = 1 * USB_MS_HZ,	/* 1 second  */
 	},
 
 	[ST_DATA_WR] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = BULK_SIZE,
-		.mh.flags = {.proxy_buffer = 1,},
-		.mh.callback = &bbb_data_write_callback,
-		.mh.timeout = 4 * USB_MS_HZ,	/* 4 seconds */
+		.bufsize = BULK_SIZE,
+		.flags = {.proxy_buffer = 1,},
+		.callback = &bbb_data_write_callback,
+		.timeout = 4 * USB_MS_HZ,	/* 4 seconds */
 	},
 
 	[ST_DATA_WR_CS] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &bbb_data_wr_cs_callback,
-		.mh.timeout = 1 * USB_MS_HZ,	/* 1 second  */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &bbb_data_wr_cs_callback,
+		.timeout = 1 * USB_MS_HZ,	/* 1 second  */
 	},
 
 	[ST_STATUS] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = sizeof(struct bbb_csw),
-		.mh.flags = {.short_xfer_ok = 1,},
-		.mh.callback = &bbb_status_callback,
-		.mh.timeout = 1 * USB_MS_HZ,	/* 1 second  */
+		.bufsize = sizeof(struct bbb_csw),
+		.flags = {.short_xfer_ok = 1,},
+		.callback = &bbb_status_callback,
+		.timeout = 1 * USB_MS_HZ,	/* 1 second  */
 	},
 };
 
@@ -266,7 +262,7 @@ bbb_command_callback(struct usb2_xfer *xfer)
 		tag = UGETDW(sc->cbw.dCBWTag) + 1;
 		USETDW(sc->cbw.dCBWSignature, CBWSIGNATURE);
 		USETDW(sc->cbw.dCBWTag, tag);
-		USETDW(sc->cbw.dCBWDataTransferLength, sc->data_len);
+		USETDW(sc->cbw.dCBWDataTransferLength, (uint32_t)sc->data_len);
 		sc->cbw.bCBWFlags = ((sc->dir == DIR_IN) ? CBWFLAGS_IN : CBWFLAGS_OUT);
 		sc->cbw.bCBWLUN = sc->lun;
 		sc->cbw.bCDBLength = sc->cmd_len;
@@ -290,7 +286,7 @@ static void
 bbb_data_read_callback(struct usb2_xfer *xfer)
 {
 	struct bbb_transfer *sc = xfer->priv_sc;
-	uint32_t max_bulk = xfer->max_data_length;
+	usb2_frlength_t max_bulk = xfer->max_data_length;
 
 	switch (USB_GET_STATE(xfer)) {
 	case USB_ST_TRANSFERRED:
@@ -341,7 +337,7 @@ static void
 bbb_data_write_callback(struct usb2_xfer *xfer)
 {
 	struct bbb_transfer *sc = xfer->priv_sc;
-	uint32_t max_bulk = xfer->max_data_length;
+	usb2_frlength_t max_bulk = xfer->max_data_length;
 
 	switch (USB_GET_STATE(xfer)) {
 	case USB_ST_TRANSFERRED:
@@ -439,8 +435,8 @@ bbb_status_callback(struct usb2_xfer *xfer)
  *------------------------------------------------------------------------*/
 static uint8_t
 bbb_command_start(struct bbb_transfer *sc, uint8_t dir, uint8_t lun,
-    void *data_ptr, uint32_t data_len, uint8_t cmd_len,
-    uint32_t data_timeout)
+    void *data_ptr, usb2_size_t data_len, uint8_t cmd_len,
+    usb2_timeout_t data_timeout)
 {
 	sc->lun = lun;
 	sc->dir = data_len ? dir : DIR_NONE;
diff --git a/sys/dev/usb/usb_parse.c b/sys/dev/usb/usb_parse.c
index 381f546..710d8fa 100644
--- a/sys/dev/usb/usb_parse.c
+++ b/sys/dev/usb/usb_parse.c
@@ -84,113 +84,109 @@ usb2_desc_foreach(struct usb2_config_descriptor *cd,
 }
 
 /*------------------------------------------------------------------------*
- *	usb2_find_idesc
+ *	usb2_idesc_foreach
  *
- * This function will return the interface descriptor, if any, that
- * has index "iface_index" and alternate index "alt_index".
+ * This function will iterate the interface descriptors in the config
+ * descriptor. The parse state structure should be zeroed before
+ * calling this function the first time.
  *
  * Return values:
  *   NULL: End of descriptors
  *   Else: A valid interface descriptor
  *------------------------------------------------------------------------*/
 struct usb2_interface_descriptor *
-usb2_find_idesc(struct usb2_config_descriptor *cd,
-    uint8_t iface_index, uint8_t alt_index)
+usb2_idesc_foreach(struct usb2_config_descriptor *cd,
+    struct usb2_idesc_parse_state *ps)
 {
-	struct usb2_descriptor *desc = NULL;
 	struct usb2_interface_descriptor *id;
-	uint8_t curidx = 0;
-	uint8_t lastidx = 0;
-	uint8_t curaidx = 0;
-	uint8_t first = 1;
-
-	while ((desc = usb2_desc_foreach(cd, desc))) {
-		if ((desc->bDescriptorType == UDESC_INTERFACE) &&
-		    (desc->bLength >= sizeof(*id))) {
-			id = (void *)desc;
-
-			if (first) {
-				first = 0;
-				lastidx = id->bInterfaceNumber;
-
-			} else if (id->bInterfaceNumber != lastidx) {
+	uint8_t new_iface;
 
-				lastidx = id->bInterfaceNumber;
-				curidx++;
-				curaidx = 0;
+	/* retrieve current descriptor */
+	id = (struct usb2_interface_descriptor *)ps->desc;
+	/* default is to start a new interface */
+	new_iface = 1;
 
-			} else {
-				curaidx++;
-			}
-
-			if ((iface_index == curidx) && (alt_index == curaidx)) {
-				return (id);
-			}
+	while (1) {
+		id = (struct usb2_interface_descriptor *)
+		    usb2_desc_foreach(cd, (struct usb2_descriptor *)id);
+		if (id == NULL)
+			break;
+		if ((id->bDescriptorType == UDESC_INTERFACE) &&
+		    (id->bLength >= sizeof(*id))) {
+			if (ps->iface_no_last == id->bInterfaceNumber)
+				new_iface = 0;
+			ps->iface_no_last = id->bInterfaceNumber;
+			break;
 		}
 	}
-	return (NULL);
+
+	if (ps->desc == NULL) {
+		/* first time */
+	} else if (new_iface) {
+		/* new interface */
+		ps->iface_index ++;
+		ps->iface_index_alt = 0;
+	} else {
+		/* new alternate interface */
+		ps->iface_index_alt ++;
+	}
+
+	/* store and return current descriptor */
+	ps->desc = (struct usb2_descriptor *)id;
+	return (id);
 }
 
 /*------------------------------------------------------------------------*
- *	usb2_find_edesc
+ *	usb2_edesc_foreach
  *
- * This function will return the endpoint descriptor for the passed
- * interface index, alternate index and endpoint index.
+ * This function will iterate all the endpoint descriptors within an
+ * interface descriptor. Starting value for the "ped" argument should
+ * be a valid interface descriptor.
  *
  * Return values:
  *   NULL: End of descriptors
  *   Else: A valid endpoint descriptor
  *------------------------------------------------------------------------*/
 struct usb2_endpoint_descriptor *
-usb2_find_edesc(struct usb2_config_descriptor *cd,
-    uint8_t iface_index, uint8_t alt_index, uint8_t ep_index)
+usb2_edesc_foreach(struct usb2_config_descriptor *cd,
+    struct usb2_endpoint_descriptor *ped)
 {
-	struct usb2_descriptor *desc = NULL;
-	struct usb2_interface_descriptor *d;
-	uint8_t curidx = 0;
-
-	d = usb2_find_idesc(cd, iface_index, alt_index);
-	if (d == NULL)
-		return (NULL);
-
-	if (ep_index >= d->bNumEndpoints)	/* quick exit */
-		return (NULL);
+	struct usb2_descriptor *desc;
 
-	desc = ((void *)d);
+	desc = ((struct usb2_descriptor *)ped);
 
 	while ((desc = usb2_desc_foreach(cd, desc))) {
 		if (desc->bDescriptorType == UDESC_INTERFACE) {
 			break;
 		}
 		if (desc->bDescriptorType == UDESC_ENDPOINT) {
-			if (curidx == ep_index) {
-				if (desc->bLength <
-				    sizeof(struct usb2_endpoint_descriptor)) {
-					/* endpoint index is invalid */
-					break;
-				}
-				return ((void *)desc);
+			if (desc->bLength < sizeof(*ped)) {
+				/* endpoint index is invalid */
+				break;
 			}
-			curidx++;
+			return ((struct usb2_endpoint_descriptor *)desc);
 		}
 	}
 	return (NULL);
 }
 
 /*------------------------------------------------------------------------*
- *	usb2_get_no_endpoints
+ *	usb2_get_no_descriptors
  *
- * This function will count the total number of endpoints available.
+ * This function will count the total number of descriptors in the
+ * configuration descriptor of type "type".
  *------------------------------------------------------------------------*/
-uint16_t
-usb2_get_no_endpoints(struct usb2_config_descriptor *cd)
+uint8_t
+usb2_get_no_descriptors(struct usb2_config_descriptor *cd, uint8_t type)
 {
 	struct usb2_descriptor *desc = NULL;
-	uint16_t count = 0;
+	uint8_t count = 0;
 
 	while ((desc = usb2_desc_foreach(cd, desc))) {
-		if (desc->bDescriptorType == UDESC_ENDPOINT) {
+		if (desc->bDescriptorType == type) {
 			count++;
+			if (count == 0xFF)
+				break;			/* crazy */
 		}
 	}
 	return (count);
@@ -200,25 +196,30 @@ usb2_get_no_endpoints(struct usb2_config_descriptor *cd)
  *	usb2_get_no_alts
  *
  * Return value:
- *   Number of alternate settings for the given "ifaceno".
- *
- * NOTE: The returned can be larger than the actual number of
- * alternate settings.
+ *   Number of alternate settings for the given interface descriptor pointer.
  *------------------------------------------------------------------------*/
-uint16_t
-usb2_get_no_alts(struct usb2_config_descriptor *cd, uint8_t ifaceno)
+uint8_t
+usb2_get_no_alts(struct usb2_config_descriptor *cd,
+    struct usb2_interface_descriptor *id)
 {
-	struct usb2_descriptor *desc = NULL;
-	struct usb2_interface_descriptor *id;
-	uint16_t n = 0;
+	struct usb2_descriptor *desc;
+	uint8_t n = 0;
+	uint8_t ifaceno;
+
+	ifaceno = id->bInterfaceNumber;
+
+	desc = (struct usb2_descriptor *)id;
 
 	while ((desc = usb2_desc_foreach(cd, desc))) {
 		if ((desc->bDescriptorType == UDESC_INTERFACE) &&
 		    (desc->bLength >= sizeof(*id))) {
-			id = (void *)desc;
+			id = (struct usb2_interface_descriptor *)desc;
 			if (id->bInterfaceNumber == ifaceno) {
 				n++;
-			}
+				if (n == 0xFF)
+					break;		/* crazy */
+			} else
+				break;			/* end */
 		}
 	}
 	return (n);
diff --git a/sys/dev/usb/usb_parse.h b/sys/dev/usb/usb_parse.h
index a9e6509..b836b57 100644
--- a/sys/dev/usb/usb_parse.h
+++ b/sys/dev/usb/usb_parse.h
@@ -27,15 +27,28 @@
 #ifndef _USB2_PARSE_H_
 #define	_USB2_PARSE_H_
 
+/* structures */
+
+struct usb2_idesc_parse_state {
+	struct usb2_descriptor *desc;
+	uint8_t iface_index;		/* current interface index */
+	uint8_t iface_no_last;
+	uint8_t iface_index_alt;	/* current alternate setting */
+};
+
+/* prototypes */
+
 struct usb2_descriptor *usb2_desc_foreach(struct usb2_config_descriptor *cd,
 	    struct usb2_descriptor *desc);
-struct usb2_interface_descriptor *usb2_find_idesc(
-	    struct usb2_config_descriptor *cd, uint8_t iface_index,
-	    uint8_t alt_index);
-struct usb2_endpoint_descriptor *usb2_find_edesc(
-	    struct usb2_config_descriptor *cd, uint8_t iface_index,
-	    uint8_t alt_index, uint8_t ep_index);
-uint16_t usb2_get_no_endpoints(struct usb2_config_descriptor *cd);
-uint16_t usb2_get_no_alts(struct usb2_config_descriptor *cd, uint8_t ifaceno);
+struct usb2_interface_descriptor *usb2_idesc_foreach(
+	    struct usb2_config_descriptor *cd,
+	    struct usb2_idesc_parse_state *ps);
+struct usb2_endpoint_descriptor *usb2_edesc_foreach(
+	    struct usb2_config_descriptor *cd,
+	    struct usb2_endpoint_descriptor *ped);
+uint8_t usb2_get_no_descriptors(struct usb2_config_descriptor *cd,
+	    uint8_t type);
+uint8_t usb2_get_no_alts(struct usb2_config_descriptor *cd,
+	    struct usb2_interface_descriptor *id);
 
 #endif					/* _USB2_PARSE_H_ */
diff --git a/sys/dev/usb/usb_process.c b/sys/dev/usb/usb_process.c
index 0354284..5ec46d1 100644
--- a/sys/dev/usb/usb_process.c
+++ b/sys/dev/usb/usb_process.c
@@ -243,7 +243,7 @@ usb2_proc_msignal(struct usb2_process *up, void *_pm0, void *_pm1)
 	struct usb2_proc_msg *pm0 = _pm0;
 	struct usb2_proc_msg *pm1 = _pm1;
 	struct usb2_proc_msg *pm2;
-	uint32_t d;
+	usb2_size_t d;
 	uint8_t t;
 
 	/* check if gone, return dummy value */
diff --git a/sys/dev/usb/usb_process.h b/sys/dev/usb/usb_process.h
index 756b929..c151f66 100644
--- a/sys/dev/usb/usb_process.h
+++ b/sys/dev/usb/usb_process.h
@@ -51,7 +51,7 @@ typedef void (usb2_proc_callback_t)(struct usb2_proc_msg *hdr);
 struct usb2_proc_msg {
 	TAILQ_ENTRY(usb2_proc_msg) pm_qentry;
 	usb2_proc_callback_t *pm_callback;
-	uint32_t pm_num;
+	usb2_size_t pm_num;
 };
 
 /*
@@ -66,7 +66,7 @@ struct usb2_process {
 	struct thread *up_curtd;
 	struct mtx *up_mtx;
 
-	uint32_t up_msg_num;
+	usb2_size_t up_msg_num;
 
 	uint8_t	up_prio;
 	uint8_t	up_gone;
diff --git a/sys/dev/usb/usb_request.c b/sys/dev/usb/usb_request.c
index 47f7ec7..4a1425c 100644
--- a/sys/dev/usb/usb_request.c
+++ b/sys/dev/usb/usb_request.c
@@ -24,9 +24,8 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
- */
+ */ 
 
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
 #include <dev/usb/usb.h>
@@ -98,7 +97,7 @@ usb2_do_clear_stall_callback(struct usb2_xfer *xfer)
 	struct usb2_pipe *pipe;
 	struct usb2_pipe *pipe_end;
 	struct usb2_pipe *pipe_first;
-	uint8_t to = USB_EP_MAX;
+	uint8_t to;
 
 	udev = xfer->xroot->udev;
 
@@ -107,8 +106,9 @@ usb2_do_clear_stall_callback(struct usb2_xfer *xfer)
 	/* round robin pipe clear stall */
 
 	pipe = udev->pipe_curr;
-	pipe_end = udev->pipes + USB_EP_MAX;
+	pipe_end = udev->pipes + udev->pipes_max;
 	pipe_first = udev->pipes;
+	to = udev->pipes_max;
 	if (pipe == NULL) {
 		pipe = pipe_first;
 	}
@@ -229,15 +229,15 @@ tr_setup:
  *------------------------------------------------------------------------*/
 usb2_error_t
 usb2_do_request_flags(struct usb2_device *udev, struct mtx *mtx,
-    struct usb2_device_request *req, void *data, uint32_t flags,
-    uint16_t *actlen, uint32_t timeout)
+    struct usb2_device_request *req, void *data, uint16_t flags,
+    uint16_t *actlen, usb2_timeout_t timeout)
 {
 	struct usb2_xfer *xfer;
 	const void *desc;
 	int err = 0;
-	uint32_t start_ticks;
-	uint32_t delta_ticks;
-	uint32_t max_ticks;
+	usb2_ticks_t start_ticks;
+	usb2_ticks_t delta_ticks;
+	usb2_ticks_t max_ticks;
 	uint16_t length;
 	uint16_t temp;
 
@@ -262,29 +262,33 @@ usb2_do_request_flags(struct usb2_device *udev, struct mtx *mtx,
 	 * Set "actlen" to a known value in case the caller does not
 	 * check the return value:
 	 */
-	if (actlen) {
+	if (actlen)
 		*actlen = 0;
-	}
+
+#if (USB_HAVE_USER_IO == 0)
+	if (flags & USB_USER_DATA_PTR)
+		return (USB_ERR_INVAL);
+#endif
 	if (udev->flags.usb2_mode == USB_MODE_DEVICE) {
 		DPRINTF("USB device mode\n");
 		(usb2_temp_get_desc_p) (udev, req, &desc, &temp);
 		if (length > temp) {
-			if (!(flags & USB_SHORT_XFER_OK)) {
+			if (!(flags & USB_SHORT_XFER_OK))
 				return (USB_ERR_SHORT_XFER);
-			}
 			length = temp;
 		}
-		if (actlen) {
+		if (actlen)
 			*actlen = length;
-		}
+
 		if (length > 0) {
+#if USB_HAVE_USER_IO
 			if (flags & USB_USER_DATA_PTR) {
 				if (copyout(desc, data, length)) {
 					return (USB_ERR_INVAL);
 				}
-			} else {
+			} else
+#endif
 				bcopy(desc, data, length);
-			}
 		}
 		return (0);		/* success */
 	}
@@ -301,6 +305,59 @@ usb2_do_request_flags(struct usb2_device *udev, struct mtx *mtx,
 
 	sx_xlock(udev->default_sx);
 
+	if (udev->parent_hub == NULL) {
+		struct usb2_sw_transfer *std = &udev->bus->roothub_req;
+
+		/* root HUB code - stripped down */
+
+		if (req->bmRequestType & UT_READ) {
+			std->ptr = NULL;
+		} else {
+			if (length != 0) {
+			    DPRINTFN(1, "Root HUB does not support "
+				"writing data!\n");
+			    err = USB_ERR_INVAL;
+			    goto done;
+			}
+		}
+		/* setup request */
+		std->req = *req;
+		std->err = 0;
+		std->len = 0;
+
+		USB_BUS_LOCK(udev->bus);
+		(udev->bus->methods->roothub_exec) (udev->bus);
+		USB_BUS_UNLOCK(udev->bus);
+
+		err = std->err;
+		if (err)
+			goto done;
+
+		if (length > std->len) {
+			length = std->len;
+			if (!(flags & USB_SHORT_XFER_OK)) {
+				err = USB_ERR_SHORT_XFER;
+				goto done;
+			}
+		}
+
+		if (actlen)
+			*actlen = length;
+
+		if (length > 0) {
+#if USB_HAVE_USER_IO
+			if (flags & USB_USER_DATA_PTR) {
+				if (copyout(std->ptr, data, length)) {
+					err = USB_ERR_INVAL;
+					goto done;
+				}
+			} else
+#endif
+				bcopy(std->ptr, data, length);
+		}
+		goto done;
+	}
+
 	/*
 	 * Setup a new USB transfer or use the existing one, if any:
 	 */
@@ -314,11 +371,15 @@ usb2_do_request_flags(struct usb2_device *udev, struct mtx *mtx,
 	}
 	USB_XFER_LOCK(xfer);
 
-	if (flags & USB_DELAY_STATUS_STAGE) {
+	if (flags & USB_DELAY_STATUS_STAGE)
 		xfer->flags.manual_status = 1;
-	} else {
+	else
 		xfer->flags.manual_status = 0;
-	}
+
+	if (flags & USB_SHORT_XFER_OK)
+		xfer->flags.short_xfer_ok = 1;
+	else
+		xfer->flags.short_xfer_ok = 0;
 
 	xfer->timeout = timeout;
 
@@ -340,6 +401,7 @@ usb2_do_request_flags(struct usb2_device *udev, struct mtx *mtx,
 
 		if (temp > 0) {
 			if (!(req->bmRequestType & UT_READ)) {
+#if USB_HAVE_USER_IO
 				if (flags & USB_USER_DATA_PTR) {
 					USB_XFER_UNLOCK(xfer);
 					err = usb2_copy_in_user(xfer->frbuffers + 1,
@@ -349,9 +411,10 @@ usb2_do_request_flags(struct usb2_device *udev, struct mtx *mtx,
 						err = USB_ERR_INVAL;
 						break;
 					}
-				} else {
-					usb2_copy_in(xfer->frbuffers + 1, 0, data, temp);
-				}
+				} else
+#endif
+					usb2_copy_in(xfer->frbuffers + 1,
+					    0, data, temp);
 			}
 			xfer->nframes = 2;
 		} else {
@@ -402,13 +465,11 @@ usb2_do_request_flags(struct usb2_device *udev, struct mtx *mtx,
 
 		if (temp > xfer->actlen) {
 			temp = xfer->actlen;
-			if (!(flags & USB_SHORT_XFER_OK)) {
-				err = USB_ERR_SHORT_XFER;
-			}
 			length = temp;
 		}
 		if (temp > 0) {
 			if (req->bmRequestType & UT_READ) {
+#if USB_HAVE_USER_IO
 				if (flags & USB_USER_DATA_PTR) {
 					USB_XFER_UNLOCK(xfer);
 					err = usb2_copy_out_user(xfer->frbuffers + 1,
@@ -418,10 +479,10 @@ usb2_do_request_flags(struct usb2_device *udev, struct mtx *mtx,
 						err = USB_ERR_INVAL;
 						break;
 					}
-				} else {
+				} else
+#endif
 					usb2_copy_out(xfer->frbuffers + 1,
 					    0, data, temp);
-				}
 			}
 		}
 		/*
@@ -480,8 +541,8 @@ done:
  *------------------------------------------------------------------------*/
 usb2_error_t
 usb2_do_request_proc(struct usb2_device *udev, struct usb2_process *pproc,
-    struct usb2_device_request *req, void *data, uint32_t flags,
-    uint16_t *actlen, uint32_t timeout)
+    struct usb2_device_request *req, void *data, uint16_t flags,
+    uint16_t *actlen, usb2_timeout_t timeout)
 {
 	usb2_error_t err;
 	uint16_t len;
@@ -826,6 +887,40 @@ usb2_req_get_string_desc(struct usb2_device *udev, struct mtx *mtx, void *sdesc,
 }
 
 /*------------------------------------------------------------------------*
+ *	usb2_req_get_config_desc_ptr
+ *
+ * This function is used in device side mode to retrieve the pointer
+ * to the generated config descriptor. This saves allocating space for
+ * an additional config descriptor when setting the configuration.
+ *
+ * Returns:
+ *    0: Success
+ * Else: Failure
+ *------------------------------------------------------------------------*/
+usb2_error_t
+usb2_req_get_config_desc_ptr(struct usb2_device *udev,
+    struct usb2_config_descriptor **ppcd, uint8_t config_index)
+{
+	uint16_t len;
+
+	struct usb2_device_request req;
+
+	if (udev->flags.usb2_mode != USB_MODE_DEVICE)
+		return (USB_ERR_INVAL);
+
+	req.bmRequestType = UT_READ_DEVICE;
+	req.bRequest = UR_GET_DESCRIPTOR;
+	USETW2(req.wValue, UDESC_CONFIG, config_index);
+	USETW(req.wIndex, 0);
+	USETW(req.wLength, 0);
+
+	(usb2_temp_get_desc_p) (udev, &req, 
+	    __DECONST(const void **, ppcd), &len);
+
+	return (*ppcd ? USB_ERR_NORMAL_COMPLETION : USB_ERR_INVAL);
+}
+
+/*------------------------------------------------------------------------*
  *	usb2_req_get_config_desc
  *
  * Returns:
@@ -1392,7 +1487,8 @@ usb2_req_re_enumerate(struct usb2_device *udev, struct mtx *mtx)
 retry:
 	err = usb2_req_reset_port(parent_hub, mtx, udev->port_no);
 	if (err) {
-		DPRINTFN(0, "addr=%d, port reset failed\n", old_addr);
+		DPRINTFN(0, "addr=%d, port reset failed, %s\n", 
+		    old_addr, usb2_errstr(err));
 		goto done;
 	}
 	/*
@@ -1410,8 +1506,8 @@ retry:
 	err = usb2_req_set_address(udev, mtx, old_addr);
 	if (err) {
 		/* XXX ignore any errors! */
-		DPRINTFN(0, "addr=%d, set address failed! (ignored)\n",
-		    old_addr);
+		DPRINTFN(0, "addr=%d, set address failed! (%s, ignored)\n",
+		    old_addr, usb2_errstr(err));
 	}
 	/* restore device address */
 	udev->address = old_addr;
@@ -1424,14 +1520,16 @@ retry:
 	    USB_MAX_IPACKET, USB_MAX_IPACKET, 0, UDESC_DEVICE, 0, 0);
 	if (err) {
 		DPRINTFN(0, "getting device descriptor "
-		    "at addr %d failed!\n", udev->address);
+		    "at addr %d failed, %s!\n", udev->address,
+		    usb2_errstr(err));
 		goto done;
 	}
 	/* get the full device descriptor */
 	err = usb2_req_get_device_desc(udev, mtx, &udev->ddesc);
 	if (err) {
 		DPRINTFN(0, "addr=%d, getting device "
-		    "descriptor failed!\n", old_addr);
+		    "descriptor failed, %s!\n", old_addr, 
+		    usb2_errstr(err));
 		goto done;
 	}
 done:
diff --git a/sys/dev/usb/usb_request.h b/sys/dev/usb/usb_request.h
index 402daa0..6d30fd4 100644
--- a/sys/dev/usb/usb_request.h
+++ b/sys/dev/usb/usb_request.h
@@ -30,11 +30,11 @@
 struct usb2_process;
 
 usb2_error_t usb2_do_request_flags(struct usb2_device *udev, struct mtx *mtx,
-		    struct usb2_device_request *req, void *data, uint32_t flags,
-		    uint16_t *actlen, uint32_t timeout);
+		    struct usb2_device_request *req, void *data, uint16_t flags,
+		    uint16_t *actlen, usb2_timeout_t timeout);
 usb2_error_t usb2_do_request_proc(struct usb2_device *udev, struct usb2_process *pproc,
-		    struct usb2_device_request *req, void *data, uint32_t flags,
-		    uint16_t *actlen, uint32_t timeout);
+		    struct usb2_device_request *req, void *data, uint16_t flags,
+		    uint16_t *actlen, usb2_timeout_t timeout);
 usb2_error_t usb2_req_clear_hub_feature(struct usb2_device *udev,
 		    struct mtx *mtx, uint16_t sel);
 usb2_error_t usb2_req_clear_port_feature(struct usb2_device *udev,
@@ -44,6 +44,8 @@ usb2_error_t usb2_req_get_alt_interface_no(struct usb2_device *udev,
 		    uint8_t iface_index);
 usb2_error_t usb2_req_get_config(struct usb2_device *udev, struct mtx *mtx,
 		    uint8_t *pconf);
+usb2_error_t usb2_req_get_config_desc_ptr(struct usb2_device *udev,
+		    struct usb2_config_descriptor **ppcd, uint8_t config_index);
 usb2_error_t usb2_req_get_config_desc(struct usb2_device *udev, struct mtx *mtx,
 		    struct usb2_config_descriptor *d, uint8_t conf_index);
 usb2_error_t usb2_req_get_config_desc_full(struct usb2_device *udev,
diff --git a/sys/dev/usb/usb_sw_transfer.c b/sys/dev/usb/usb_sw_transfer.c
deleted file mode 100644
index 984c233..0000000
--- a/sys/dev/usb/usb_sw_transfer.c
+++ /dev/null
@@ -1,170 +0,0 @@
-/* $FreeBSD$ */
-/*-
- * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <dev/usb/usb_mfunc.h>
-#include <dev/usb/usb.h>
-#include <dev/usb/usb_error.h>
-#include <dev/usb/usb_defs.h>
-
-#define	USB_DEBUG_VAR usb2_debug
-
-#include <dev/usb/usb_core.h>
-#include <dev/usb/usb_process.h>
-#include <dev/usb/usb_busdma.h>
-#include <dev/usb/usb_transfer.h>
-#include <dev/usb/usb_sw_transfer.h>
-#include <dev/usb/usb_device.h>
-#include <dev/usb/usb_debug.h>
-
-#include <dev/usb/usb_controller.h>
-#include <dev/usb/usb_bus.h>
-
-/*------------------------------------------------------------------------*
- *	usb2_sw_transfer - factored out code
- *
- * This function is basically used for the Virtual Root HUB, and can
- * emulate control, bulk and interrupt endpoints. Data is exchanged
- * using the "std->ptr" and "std->len" fields, that allows kernel
- * virtual memory to be transferred. All state is kept in the
- * structure pointed to by the "std" argument passed to this
- * function. The "func" argument points to a function that is called
- * back in the various states, so that the application using this
- * function can get a chance to select the outcome. The "func"
- * function is allowed to sleep, exiting all mutexes. If this function
- * will sleep the "enter" and "start" methods must be marked
- * non-cancelable, hence there is no extra cancelled checking in this
- * function.
- *------------------------------------------------------------------------*/
-void
-usb2_sw_transfer(struct usb2_sw_transfer *std,
-    usb2_sw_transfer_func_t *func)
-{
-	struct usb2_xfer *xfer;
-	uint32_t len;
-	uint8_t shortpkt = 0;
-
-	xfer = std->xfer;
-	if (xfer == NULL) {
-		/* the transfer is gone */
-		DPRINTF("xfer gone\n");
-		return;
-	}
-	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
-
-	std->xfer = NULL;
-
-	/* check for control transfer */
-	if (xfer->flags_int.control_xfr) {
-		/* check if we are transferring the SETUP packet */
-		if (xfer->flags_int.control_hdr) {
-
-			/* copy out the USB request */
-
-			if (xfer->frlengths[0] == sizeof(std->req)) {
-				usb2_copy_out(xfer->frbuffers, 0,
-				    &std->req, sizeof(std->req));
-			} else {
-				std->err = USB_ERR_INVAL;
-				goto done;
-			}
-
-			xfer->aframes = 1;
-
-			std->err = 0;
-			std->state = USB_SW_TR_SETUP;
-
-			(func) (xfer, std);
-
-			if (std->err) {
-				goto done;
-			}
-		} else {
-			/* skip the first frame in this case */
-			xfer->aframes = 1;
-		}
-	}
-	std->err = 0;
-	std->state = USB_SW_TR_PRE_DATA;
-
-	(func) (xfer, std);
-
-	if (std->err) {
-		goto done;
-	}
-	/* Transfer data. Iterate accross all frames. */
-	while (xfer->aframes != xfer->nframes) {
-
-		len = xfer->frlengths[xfer->aframes];
-
-		if (len > std->len) {
-			len = std->len;
-			shortpkt = 1;
-		}
-		if (len > 0) {
-			if ((xfer->endpoint & (UE_DIR_IN | UE_DIR_OUT)) == UE_DIR_IN) {
-				usb2_copy_in(xfer->frbuffers + xfer->aframes, 0,
-				    std->ptr, len);
-			} else {
-				usb2_copy_out(xfer->frbuffers + xfer->aframes, 0,
-				    std->ptr, len);
-			}
-		}
-		std->ptr += len;
-		std->len -= len;
-		xfer->frlengths[xfer->aframes] = len;
-		xfer->aframes++;
-
-		if (shortpkt) {
-			break;
-		}
-	}
-
-	std->err = 0;
-	std->state = USB_SW_TR_POST_DATA;
-
-	(func) (xfer, std);
-
-	if (std->err) {
-		goto done;
-	}
-	/* check if the control transfer is complete */
-	if (xfer->flags_int.control_xfr &&
-	    !xfer->flags_int.control_act) {
-
-		std->err = 0;
-		std->state = USB_SW_TR_STATUS;
-
-		(func) (xfer, std);
-
-		if (std->err) {
-			goto done;
-		}
-	}
-done:
-	DPRINTF("done err=%s\n", usb2_errstr(std->err));
-	std->state = USB_SW_TR_PRE_CALLBACK;
-	(func) (xfer, std);
-}
diff --git a/sys/dev/usb/usb_transfer.c b/sys/dev/usb/usb_transfer.c
index 71bbb08..41b07f1 100644
--- a/sys/dev/usb/usb_transfer.c
+++ b/sys/dev/usb/usb_transfer.c
@@ -22,12 +22,11 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
- */
+ */ 
 
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
 #include <dev/usb/usb.h>
-#include <dev/usb/usb_defs.h>
 
 #define	USB_DEBUG_VAR usb2_debug
 
@@ -51,45 +50,7 @@ struct usb2_std_packet_size {
 	uint16_t fixed[4];
 };
 
-/*
- * This table stores the all the allowed packet sizes based on
- * endpoint type and USB speed:
- */
-static const struct usb2_std_packet_size
-	usb2_std_packet_size[4][USB_SPEED_MAX] = {
-
-	[UE_INTERRUPT] = {
-		[USB_SPEED_LOW] = {.range = {0, 8}},
-		[USB_SPEED_FULL] = {.range = {0, 64}},
-		[USB_SPEED_HIGH] = {.range = {0, 1024}},
-		[USB_SPEED_VARIABLE] = {.range = {0, 1024}},
-		[USB_SPEED_SUPER] = {.range = {0, 1024}},
-	},
-
-	[UE_CONTROL] = {
-		[USB_SPEED_LOW] = {.fixed = {8, 8, 8, 8}},
-		[USB_SPEED_FULL] = {.fixed = {8, 16, 32, 64}},
-		[USB_SPEED_HIGH] = {.fixed = {64, 64, 64, 64}},
-		[USB_SPEED_VARIABLE] = {.fixed = {512, 512, 512, 512}},
-		[USB_SPEED_SUPER] = {.fixed = {512, 512, 512, 512}},
-	},
-
-	[UE_BULK] = {
-		[USB_SPEED_LOW] = {.fixed = {0, 0, 0, 0}},	/* invalid */
-		[USB_SPEED_FULL] = {.fixed = {8, 16, 32, 64}},
-		[USB_SPEED_HIGH] = {.fixed = {512, 512, 512, 512}},
-		[USB_SPEED_VARIABLE] = {.fixed = {512, 512, 1024, 1536}},
-		[USB_SPEED_SUPER] = {.fixed = {1024, 1024, 1024, 1024}},
-	},
-
-	[UE_ISOCHRONOUS] = {
-		[USB_SPEED_LOW] = {.fixed = {0, 0, 0, 0}},	/* invalid */
-		[USB_SPEED_FULL] = {.range = {0, 1023}},
-		[USB_SPEED_HIGH] = {.range = {0, 1024}},
-		[USB_SPEED_VARIABLE] = {.range = {0, 3584}},
-		[USB_SPEED_SUPER] = {.range = {0, 1024}},
-	},
-};
+static usb2_callback_t usb2_request_callback;
 
 static const struct usb2_config usb2_control_ep_cfg[USB_DEFAULT_XFER_MAX] = {
 
@@ -99,12 +60,10 @@ static const struct usb2_config usb2_control_ep_cfg[USB_DEFAULT_XFER_MAX] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control endpoint */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = 1024,	/* bytes */
-		.mh.flags = {.proxy_buffer = 1,.short_xfer_ok = 1,},
-		.mh.callback = &usb2_do_request_callback,
-		.md.bufsize = 1024,	/* bytes */
-		.md.flags = {.proxy_buffer = 1,.short_xfer_ok = 0,},
-		.md.callback = &usb2_handle_request_callback,
+		.bufsize = USB_EP0_BUFSIZE,	/* bytes */
+		.flags = {.proxy_buffer = 1,},
+		.callback = &usb2_request_callback,
+		.usb_mode = USB_MODE_MAX,	/* both modes */
 	},
 
 	/* This transfer is used for generic clear stall only */
@@ -113,11 +72,11 @@ static const struct usb2_config usb2_control_ep_cfg[USB_DEFAULT_XFER_MAX] = {
 		.type = UE_CONTROL,
 		.endpoint = 0x00,	/* Control pipe */
 		.direction = UE_DIR_ANY,
-		.mh.bufsize = sizeof(struct usb2_device_request),
-		.mh.flags = {},
-		.mh.callback = &usb2_do_clear_stall_callback,
-		.mh.timeout = 1000,	/* 1 second */
-		.mh.interval = 50,	/* 50ms */
+		.bufsize = sizeof(struct usb2_device_request),
+		.callback = &usb2_do_clear_stall_callback,
+		.timeout = 1000,	/* 1 second */
+		.interval = 50,	/* 50ms */
+		.usb_mode = USB_MODE_HOST,
 	},
 };
 
@@ -133,6 +92,20 @@ static void	usb2_callback_wrapper(struct usb2_xfer_queue *);
 static void	usb2_dma_delay_done_cb(void *);
 static void	usb2_transfer_start_cb(void *);
 static uint8_t	usb2_callback_wrapper_sub(struct usb2_xfer *);
+static void	usb2_get_std_packet_size(struct usb2_std_packet_size *ptr, 
+		    uint8_t type, uint8_t usb_speed);
+
+/*------------------------------------------------------------------------*
+ *	usb2_request_callback
+ *------------------------------------------------------------------------*/
+static void
+usb2_request_callback(struct usb2_xfer *xfer)
+{
+	if (xfer->flags_int.usb2_mode == USB_MODE_DEVICE)
+		usb2_handle_request_callback(xfer);
+	else
+		usb2_do_request_callback(xfer);
+}
 
 /*------------------------------------------------------------------------*
  *	usb2_update_max_frame_size
@@ -164,7 +137,7 @@ usb2_update_max_frame_size(struct usb2_xfer *xfer)
  *    0: no DMA delay required
  * Else: milliseconds of DMA delay
  *------------------------------------------------------------------------*/
-uint32_t
+usb2_timeout_t
 usb2_get_dma_delay(struct usb2_bus *bus)
 {
 	uint32_t temp = 0;
@@ -192,20 +165,21 @@ usb2_get_dma_delay(struct usb2_bus *bus)
  *    0: Success
  * Else: Failure
  *------------------------------------------------------------------------*/
+#if USB_HAVE_BUSDMA
 uint8_t
 usb2_transfer_setup_sub_malloc(struct usb2_setup_params *parm,
-    struct usb2_page_cache **ppc, uint32_t size, uint32_t align,
-    uint32_t count)
+    struct usb2_page_cache **ppc, usb2_size_t size, usb2_size_t align,
+    usb2_size_t count)
 {
 	struct usb2_page_cache *pc;
 	struct usb2_page *pg;
 	void *buf;
-	uint32_t n_dma_pc;
-	uint32_t n_obj;
-	uint32_t x;
-	uint32_t y;
-	uint32_t r;
-	uint32_t z;
+	usb2_size_t n_dma_pc;
+	usb2_size_t n_obj;
+	usb2_size_t x;
+	usb2_size_t y;
+	usb2_size_t r;
+	usb2_size_t z;
 
 	USB_ASSERT(align > 1, ("Invalid alignment, 0x%08x!\n",
 	    align));
@@ -302,6 +276,7 @@ usb2_transfer_setup_sub_malloc(struct usb2_setup_params *parm,
 	parm->dma_page_ptr = pg;
 	return (0);
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *	usb2_transfer_setup_sub - transfer setup subroutine
@@ -320,12 +295,12 @@ usb2_transfer_setup_sub(struct usb2_setup_params *parm)
 		MIN_PKT = 8,
 	};
 	struct usb2_xfer *xfer = parm->curr_xfer;
-	const struct usb2_config_sub *setup_sub = parm->curr_setup_sub;
+	const struct usb2_config *setup = parm->curr_setup;
 	struct usb2_endpoint_descriptor *edesc;
 	struct usb2_std_packet_size std_size;
-	uint32_t n_frlengths;
-	uint32_t n_frbuffers;
-	uint32_t x;
+	usb2_frcount_t n_frlengths;
+	usb2_frcount_t n_frbuffers;
+	usb2_frcount_t x;
 	uint8_t type;
 	uint8_t zmps;
 
@@ -343,18 +318,18 @@ usb2_transfer_setup_sub(struct usb2_setup_params *parm)
 
 	type = (edesc->bmAttributes & UE_XFERTYPE);
 
-	xfer->flags = setup_sub->flags;
-	xfer->nframes = setup_sub->frames;
-	xfer->timeout = setup_sub->timeout;
-	xfer->callback = setup_sub->callback;
-	xfer->interval = setup_sub->interval;
+	xfer->flags = setup->flags;
+	xfer->nframes = setup->frames;
+	xfer->timeout = setup->timeout;
+	xfer->callback = setup->callback;
+	xfer->interval = setup->interval;
 	xfer->endpoint = edesc->bEndpointAddress;
 	xfer->max_packet_size = UGETW(edesc->wMaxPacketSize);
 	xfer->max_packet_count = 1;
 	/* make a shadow copy: */
 	xfer->flags_int.usb2_mode = parm->udev->flags.usb2_mode;
 
-	parm->bufsize = setup_sub->bufsize;
+	parm->bufsize = setup->bufsize;
 
 	if (parm->speed == USB_SPEED_HIGH) {
 		xfer->max_packet_count += (xfer->max_packet_size >> 11) & 3;
@@ -373,7 +348,7 @@ usb2_transfer_setup_sub(struct usb2_setup_params *parm)
 	}
 	/* filter "wMaxPacketSize" according to standard sizes */
 
-	std_size = usb2_std_packet_size[type][parm->speed];
+	usb2_get_std_packet_size(&std_size, type, parm->speed);
 
 	if (std_size.range.min || std_size.range.max) {
 
@@ -405,7 +380,7 @@ usb2_transfer_setup_sub(struct usb2_setup_params *parm)
 
 	if (type == UE_ISOCHRONOUS) {
 
-		uint32_t frame_limit;
+		uint16_t frame_limit;
 
 		xfer->interval = 0;	/* not used, must be zero */
 		xfer->flags_int.isochronous_xfr = 1;	/* set flag */
@@ -644,6 +619,7 @@ usb2_transfer_setup_sub(struct usb2_setup_params *parm)
 	if (parm->bufsize_max < parm->bufsize) {
 		parm->bufsize_max = parm->bufsize;
 	}
+#if USB_HAVE_BUSDMA
 	if (xfer->flags_int.bdma_enable) {
 		/*
 		 * Setup "dma_page_ptr".
@@ -671,17 +647,18 @@ usb2_transfer_setup_sub(struct usb2_setup_params *parm)
 		parm->dma_page_ptr += (2 * n_frbuffers);
 		parm->dma_page_ptr += (parm->bufsize / USB_PAGE_SIZE);
 	}
+#endif
 	if (zmps) {
 		/* correct maximum data length */
 		xfer->max_data_length = 0;
 	}
 	/* subtract USB frame remainder from "hc_max_frame_size" */
 
-	xfer->max_usb2_frame_size =
+	xfer->max_hc_frame_size =
 	    (parm->hc_max_frame_size -
 	    (parm->hc_max_frame_size % xfer->max_frame_size));
 
-	if (xfer->max_usb2_frame_size == 0) {
+	if (xfer->max_hc_frame_size == 0) {
 		parm->err = USB_ERR_INVAL;
 		goto done;
 	}
@@ -695,7 +672,7 @@ usb2_transfer_setup_sub(struct usb2_setup_params *parm)
 		for (x = 0; x != n_frbuffers; x++) {
 			xfer->frbuffers[x].tag_parent =
 			    &xfer->xroot->dma_parent_tag;
-
+#if USB_HAVE_BUSDMA
 			if (xfer->flags_int.bdma_enable &&
 			    (parm->bufsize_max > 0)) {
 
@@ -706,6 +683,7 @@ usb2_transfer_setup_sub(struct usb2_setup_params *parm)
 					goto done;
 				}
 			}
+#endif
 		}
 	}
 done:
@@ -713,7 +691,7 @@ done:
 		/*
 		 * Set some dummy values so that we avoid division by zero:
 		 */
-		xfer->max_usb2_frame_size = 1;
+		xfer->max_hc_frame_size = 1;
 		xfer->max_frame_size = 1;
 		xfer->max_packet_size = 1;
 		xfer->max_data_length = 0;
@@ -776,13 +754,11 @@ usb2_transfer_setup(struct usb2_device *udev,
 	/* sanity checks */
 	for (setup = setup_start, n = 0;
 	    setup != setup_end; setup++, n++) {
-		if ((setup->mh.bufsize == 0xffffffff) ||
-		    (setup->md.bufsize == 0xffffffff)) {
+		if (setup->bufsize == (usb2_frlength_t)-1) {
 			parm.err = USB_ERR_BAD_BUFSIZE;
 			DPRINTF("invalid bufsize\n");
 		}
-		if ((setup->mh.callback == NULL) &&
-		    (setup->md.callback == NULL)) {
+		if (setup->callback == NULL) {
 			parm.err = USB_ERR_NO_CALLBACK;
 			DPRINTF("no callback\n");
 		}
@@ -816,28 +792,31 @@ usb2_transfer_setup(struct usb2_device *udev,
 			info->memory_base = buf;
 			info->memory_size = parm.size[0];
 
+#if USB_HAVE_BUSDMA
 			info->dma_page_cache_start = USB_ADD_BYTES(buf, parm.size[4]);
 			info->dma_page_cache_end = USB_ADD_BYTES(buf, parm.size[5]);
+#endif
 			info->xfer_page_cache_start = USB_ADD_BYTES(buf, parm.size[5]);
 			info->xfer_page_cache_end = USB_ADD_BYTES(buf, parm.size[2]);
 
 			usb2_cv_init(&info->cv_drain, "WDRAIN");
 
 			info->xfer_mtx = xfer_mtx;
-
+#if USB_HAVE_BUSDMA
 			usb2_dma_tag_setup(&info->dma_parent_tag,
 			    parm.dma_tag_p, udev->bus->dma_parent_tag[0].tag,
-			    xfer_mtx, &usb2_bdma_done_event, info, 32, parm.dma_tag_max);
+			    xfer_mtx, &usb2_bdma_done_event, 32, parm.dma_tag_max);
+#endif
 
 			info->bus = udev->bus;
 			info->udev = udev;
 
 			TAILQ_INIT(&info->done_q.head);
 			info->done_q.command = &usb2_callback_wrapper;
-
+#if USB_HAVE_BUSDMA
 			TAILQ_INIT(&info->dma_q.head);
 			info->dma_q.command = &usb2_bdma_work_loop;
-
+#endif
 			info->done_m[0].hdr.pm_callback = &usb2_callback_proc;
 			info->done_m[0].xroot = info;
 			info->done_m[1].hdr.pm_callback = &usb2_callback_proc;
@@ -859,33 +838,30 @@ usb2_transfer_setup(struct usb2_device *udev,
 		for (setup = setup_start, n = 0;
 		    setup != setup_end; setup++, n++) {
 
-			/* select mode specific structure */
-			if (udev->flags.usb2_mode == USB_MODE_HOST) {
-				parm.curr_setup_sub = &setup->mh;
-			} else {
-				parm.curr_setup_sub = &setup->md;
-			}
 			/* skip USB transfers without callbacks: */
-			if (parm.curr_setup_sub->callback == NULL) {
+			if (setup->callback == NULL) {
 				continue;
 			}
 			/* see if there is a matching endpoint */
 			pipe = usb2_get_pipe(udev,
 			    ifaces[setup->if_index], setup);
 
-			if (!pipe) {
-				if (parm.curr_setup_sub->flags.no_pipe_ok) {
+			if ((pipe == NULL) || (pipe->methods == NULL)) {
+				if (setup->flags.no_pipe_ok)
+					continue;
+				if ((setup->usb_mode != USB_MODE_MAX) &&
+				    (setup->usb_mode != udev->flags.usb2_mode))
 					continue;
-				}
 				parm.err = USB_ERR_NO_PIPE;
 				goto done;
 			}
-			/* store current setup pointer */
-			parm.curr_setup = setup;
 
 			/* align data properly */
 			parm.size[0] += ((-parm.size[0]) & (USB_HOST_ALIGN - 1));
 
+			/* store current setup pointer */
+			parm.curr_setup = setup;
+
 			if (buf) {
 				/*
 				 * Common initialization of the
@@ -1068,13 +1044,13 @@ static void
 usb2_transfer_unsetup_sub(struct usb2_xfer_root *info, uint8_t needs_delay)
 {
 	struct usb2_page_cache *pc;
-	uint32_t temp;
 
 	USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED);
 
 	/* wait for any outstanding DMA operations */
 
 	if (needs_delay) {
+		usb2_timeout_t temp;
 		temp = usb2_get_dma_delay(info->bus);
 		usb2_pause_mtx(&info->bus->bus_mtx,
 		    USB_MS_TO_TICKS(temp));
@@ -1085,6 +1061,7 @@ usb2_transfer_unsetup_sub(struct usb2_xfer_root *info, uint8_t needs_delay)
 
 	USB_BUS_UNLOCK(info->bus);
 
+#if USB_HAVE_BUSDMA
 	/* free DMA'able memory, if any */
 	pc = info->dma_page_cache_start;
 	while (pc != info->dma_page_cache_end) {
@@ -1101,6 +1078,7 @@ usb2_transfer_unsetup_sub(struct usb2_xfer_root *info, uint8_t needs_delay)
 
 	/* free all DMA tags */
 	usb2_dma_tag_unsetup(&info->dma_parent_tag);
+#endif
 
 	usb2_cv_destroy(&info->cv_drain);
 
@@ -1162,9 +1140,10 @@ usb2_transfer_unsetup(struct usb2_xfer **pxfer, uint16_t n_setup)
 
 		usb2_transfer_drain(xfer);
 
+#if USB_HAVE_BUSDMA
 		if (xfer->flags_int.bdma_enable)
 			needs_delay = 1;
-
+#endif
 		/*
 		 * NOTE: default pipe does not have an
 		 * interface, even if pipe->iface_index == 0
@@ -1234,13 +1213,27 @@ usb2_control_transfer_init(struct usb2_xfer *xfer)
 static uint8_t
 usb2_start_hardware_sub(struct usb2_xfer *xfer)
 {
-	uint32_t len;
+	usb2_frlength_t len;
 
 	/* Check for control endpoint stall */
 	if (xfer->flags.stall_pipe) {
 		/* no longer active */
 		xfer->flags_int.control_act = 0;
 	}
+
+	/* Check for invalid number of frames */
+	if (xfer->nframes > 2) {
+		/*
+		 * If you need to split a control transfer, you
+		 * have to do one part at a time. Only with
+		 * non-control transfers you can do multiple
+		 * parts a time.
+		 */
+		DPRINTFN(0, "Too many frames: %u\n",
+		    (unsigned int)xfer->nframes);
+		goto error;
+	}
+
 	/*
          * Check if there is a control
          * transfer in progress:
@@ -1363,7 +1356,7 @@ error:
 void
 usb2_start_hardware(struct usb2_xfer *xfer)
 {
-	uint32_t x;
+	usb2_frcount_t x;
 
 	DPRINTF("xfer=%p, pipe=%p, nframes=%d, dir=%s\n",
 	    xfer, xfer->pipe, xfer->nframes, USB_GET_DATA_ISREAD(xfer) ?
@@ -1395,9 +1388,10 @@ usb2_start_hardware(struct usb2_xfer *xfer)
 	/* set "transferring" flag */
 	xfer->flags_int.transferring = 1;
 
+#if USB_HAVE_POWERD
 	/* increment power reference */
 	usb2_transfer_power_ref(xfer, 1);
-
+#endif
 	/*
 	 * Check if the transfer is waiting on a queue, most
 	 * frequently the "done_q":
@@ -1413,9 +1407,10 @@ usb2_start_hardware(struct usb2_xfer *xfer)
 	/* clear "did_close" flag */
 	xfer->flags_int.did_close = 0;
 
+#if USB_HAVE_BUSDMA
 	/* clear "bdma_setup" flag */
 	xfer->flags_int.bdma_setup = 0;
-
+#endif
 	/* by default we cannot cancel any USB transfer immediately */
 	xfer->flags_int.can_cancel_immed = 0;
 
@@ -1484,23 +1479,28 @@ usb2_start_hardware(struct usb2_xfer *xfer)
 	 */
 	if (USB_GET_DATA_ISREAD(xfer)) {
 
-		if (xfer->flags_int.control_xfr) {
-
-			/*
-			 * Control transfers do not support reception
-			 * of multiple short USB frames !
-			 */
-
-			if (xfer->flags.short_xfer_ok) {
-				xfer->flags_int.short_xfer_ok = 1;
-			}
-		} else {
+		if (xfer->flags.short_frames_ok) {
+			xfer->flags_int.short_xfer_ok = 1;
+			xfer->flags_int.short_frames_ok = 1;
+		} else if (xfer->flags.short_xfer_ok) {
+			xfer->flags_int.short_xfer_ok = 1;
 
-			if (xfer->flags.short_frames_ok) {
-				xfer->flags_int.short_xfer_ok = 1;
+			/* check for control transfer */
+			if (xfer->flags_int.control_xfr) {
+				/*
+				 * 1) Control transfers do not support
+				 * reception of multiple short USB
+				 * frames in host mode and device side
+				 * mode, with exception of:
+				 *
+				 * 2) Due to sometimes buggy device
+				 * side firmware we need to do a
+				 * STATUS stage in case of short
+				 * control transfers in USB host mode.
+				 * The STATUS stage then becomes the
+				 * "alt_next" to the DATA stage.
+				 */
 				xfer->flags_int.short_frames_ok = 1;
-			} else if (xfer->flags.short_xfer_ok) {
-				xfer->flags_int.short_xfer_ok = 1;
 			}
 		}
 	}
@@ -1508,11 +1508,13 @@ usb2_start_hardware(struct usb2_xfer *xfer)
 	 * Check if BUS-DMA support is enabled and try to load virtual
 	 * buffers into DMA, if any:
 	 */
+#if USB_HAVE_BUSDMA
 	if (xfer->flags_int.bdma_enable) {
 		/* insert the USB transfer last in the BUS-DMA queue */
 		usb2_command_wrapper(&xfer->xroot->dma_q, xfer);
 		return;
 	}
+#endif
 	/*
 	 * Enter the USB transfer into the Host Controller or
 	 * Device Controller schedule:
@@ -1539,18 +1541,14 @@ usb2_pipe_enter(struct usb2_xfer *xfer)
 	/* enter the transfer */
 	(pipe->methods->enter) (xfer);
 
-	/* check cancelability */
-	if (pipe->methods->enter_is_cancelable) {
-		xfer->flags_int.can_cancel_immed = 1;
-		/* check for transfer error */
-		if (xfer->error) {
-			/* some error has happened */
-			usb2_transfer_done(xfer, 0);
-			USB_BUS_UNLOCK(xfer->xroot->bus);
-			return;
-		}
-	} else {
-		xfer->flags_int.can_cancel_immed = 0;
+	xfer->flags_int.can_cancel_immed = 1;
+
+	/* check for transfer error */
+	if (xfer->error) {
+		/* some error has happened */
+		usb2_transfer_done(xfer, 0);
+		USB_BUS_UNLOCK(xfer->xroot->bus);
+		return;
 	}
 
 	/* start the transfer */
@@ -1762,7 +1760,7 @@ usb2_transfer_drain(struct usb2_xfer *xfer)
  * than zero gives undefined results!
  *------------------------------------------------------------------------*/
 void
-usb2_set_frame_data(struct usb2_xfer *xfer, void *ptr, uint32_t frindex)
+usb2_set_frame_data(struct usb2_xfer *xfer, void *ptr, usb2_frcount_t frindex)
 {
 	/* set virtual address to load and length */
 	xfer->frbuffers[frindex].buffer = ptr;
@@ -1775,8 +1773,8 @@ usb2_set_frame_data(struct usb2_xfer *xfer, void *ptr, uint32_t frindex)
  * of the USB DMA buffer allocated for this USB transfer.
  *------------------------------------------------------------------------*/
 void
-usb2_set_frame_offset(struct usb2_xfer *xfer, uint32_t offset,
-    uint32_t frindex)
+usb2_set_frame_offset(struct usb2_xfer *xfer, usb2_frlength_t offset,
+    usb2_frcount_t frindex)
 {
 	USB_ASSERT(!xfer->flags.ext_buffer, ("Cannot offset data frame "
 	    "when the USB buffer is external!\n"));
@@ -1864,8 +1862,8 @@ usb2_callback_wrapper(struct usb2_xfer_queue *pq)
 	struct usb2_xfer *xfer = pq->curr;
 	struct usb2_xfer_root *info = xfer->xroot;
 
-	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
-	if (!mtx_owned(xfer->xroot->xfer_mtx)) {
+	USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED);
+	if (!mtx_owned(info->xfer_mtx)) {
 		/*
 	       	 * Cases that end up here:
 		 *
@@ -1896,27 +1894,28 @@ usb2_callback_wrapper(struct usb2_xfer_queue *pq)
 	/* get next USB transfer in the queue */
 	info->done_q.curr = NULL;
 
-	USB_BUS_UNLOCK(xfer->xroot->bus);
-	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_NOTOWNED);
+	USB_BUS_UNLOCK(info->bus);
+	USB_BUS_LOCK_ASSERT(info->bus, MA_NOTOWNED);
 
 	/* set correct USB state for callback */
 	if (!xfer->flags_int.transferring) {
 		xfer->usb2_state = USB_ST_SETUP;
 		if (!xfer->flags_int.started) {
 			/* we got stopped before we even got started */
-			USB_BUS_LOCK(xfer->xroot->bus);
+			USB_BUS_LOCK(info->bus);
 			goto done;
 		}
 	} else {
 
 		if (usb2_callback_wrapper_sub(xfer)) {
 			/* the callback has been deferred */
-			USB_BUS_LOCK(xfer->xroot->bus);
+			USB_BUS_LOCK(info->bus);
 			goto done;
 		}
+#if USB_HAVE_POWERD
 		/* decrement power reference */
 		usb2_transfer_power_ref(xfer, -1);
-
+#endif
 		xfer->flags_int.transferring = 0;
 
 		if (xfer->error) {
@@ -1924,12 +1923,13 @@ usb2_callback_wrapper(struct usb2_xfer_queue *pq)
 		} else {
 			/* set transferred state */
 			xfer->usb2_state = USB_ST_TRANSFERRED;
-
+#if USB_HAVE_BUSDMA
 			/* sync DMA memory, if any */
 			if (xfer->flags_int.bdma_enable &&
 			    (!xfer->flags_int.bdma_no_post_sync)) {
 				usb2_bdma_post_sync(xfer);
 			}
+#endif
 		}
 	}
 
@@ -1937,7 +1937,7 @@ usb2_callback_wrapper(struct usb2_xfer_queue *pq)
 	(xfer->callback) (xfer);
 
 	/* pickup the USB mutex again */
-	USB_BUS_LOCK(xfer->xroot->bus);
+	USB_BUS_LOCK(info->bus);
 
 	/*
 	 * Check if we got started after that we got cancelled, but
@@ -1959,7 +1959,7 @@ done:
 	    (!xfer->flags_int.transferring)) {
 		/* "usb2_transfer_drain()" is waiting for end of transfer */
 		xfer->flags_int.draining = 0;
-		usb2_cv_broadcast(&xfer->xroot->cv_drain);
+		usb2_cv_broadcast(&info->cv_drain);
 	}
 
 	/* do the next callback, if any */
@@ -2043,8 +2043,6 @@ usb2_transfer_enqueue(struct usb2_xfer_queue *pq, struct usb2_xfer *xfer)
 void
 usb2_transfer_done(struct usb2_xfer *xfer, usb2_error_t error)
 {
-	struct usb2_xfer_queue *pq;
-
 	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
 
 	DPRINTF("err=%s\n", usb2_errstr(error));
@@ -2071,7 +2069,10 @@ usb2_transfer_done(struct usb2_xfer *xfer, usb2_error_t error)
 	 */
 	usb2_transfer_dequeue(xfer);
 
+#if USB_HAVE_BUSDMA
 	if (mtx_owned(xfer->xroot->xfer_mtx)) {
+		struct usb2_xfer_queue *pq;
+
 		/*
 		 * If the private USB lock is not locked, then we assume
 		 * that the BUS-DMA load stage has been passed:
@@ -2083,6 +2084,7 @@ usb2_transfer_done(struct usb2_xfer *xfer, usb2_error_t error)
 			usb2_command_wrapper(pq, NULL);
 		}
 	}
+#endif
 	/* keep some statistics */
 	if (xfer->error) {
 		xfer->xroot->bus->stats_err.uds_requests
@@ -2116,15 +2118,12 @@ usb2_transfer_start_cb(void *arg)
 	/* start the transfer */
 	(pipe->methods->start) (xfer);
 
-	/* check cancelability */
-	if (pipe->methods->start_is_cancelable) {
-		xfer->flags_int.can_cancel_immed = 1;
-		if (xfer->error) {
-			/* some error has happened */
-			usb2_transfer_done(xfer, 0);
-		}
-	} else {
-		xfer->flags_int.can_cancel_immed = 0;
+	xfer->flags_int.can_cancel_immed = 1;
+
+	/* check for error */
+	if (xfer->error) {
+		/* some error has happened */
+		usb2_transfer_done(xfer, 0);
 	}
 }
 
@@ -2270,15 +2269,12 @@ usb2_pipe_start(struct usb2_xfer_queue *pq)
 	/* start USB transfer */
 	(pipe->methods->start) (xfer);
 
-	/* check cancelability */
-	if (pipe->methods->start_is_cancelable) {
-		xfer->flags_int.can_cancel_immed = 1;
-		if (xfer->error) {
-			/* some error has happened */
-			usb2_transfer_done(xfer, 0);
-		}
-	} else {
-		xfer->flags_int.can_cancel_immed = 0;
+	xfer->flags_int.can_cancel_immed = 1;
+
+	/* check for error */
+	if (xfer->error) {
+		/* some error has happened */
+		usb2_transfer_done(xfer, 0);
 	}
 }
 
@@ -2291,7 +2287,7 @@ usb2_pipe_start(struct usb2_xfer_queue *pq)
  *------------------------------------------------------------------------*/
 void
 usb2_transfer_timeout_ms(struct usb2_xfer *xfer,
-    void (*cb) (void *arg), uint32_t ms)
+    void (*cb) (void *arg), usb2_timeout_t ms)
 {
 	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
 
@@ -2321,7 +2317,7 @@ static uint8_t
 usb2_callback_wrapper_sub(struct usb2_xfer *xfer)
 {
 	struct usb2_pipe *pipe;
-	uint32_t x;
+	usb2_frcount_t x;
 
 	if ((!xfer->flags_int.open) &&
 	    (!xfer->flags_int.did_close)) {
@@ -2341,7 +2337,7 @@ usb2_callback_wrapper_sub(struct usb2_xfer *xfer)
 	    (xfer->error == USB_ERR_TIMEOUT)) &&
 	    (!xfer->flags_int.did_dma_delay)) {
 
-		uint32_t temp;
+		usb2_timeout_t temp;
 
 		/* only delay once */
 		xfer->flags_int.did_dma_delay = 1;
@@ -2541,6 +2537,9 @@ usb2_default_transfer_setup(struct usb2_device *udev)
 	uint8_t no_resetup;
 	uint8_t iface_index;
 
+	/* check for root HUB */
+	if (udev->parent_hub == NULL)
+		return;
 repeat:
 
 	xfer = udev->default_xfer[0];
@@ -2635,9 +2634,9 @@ usb2_clear_data_toggle(struct usb2_device *udev, struct usb2_pipe *pipe)
  *	.direction = UE_DIR_ANY,
  *	.interval = 50, //50 milliseconds
  *	.bufsize = sizeof(struct usb2_device_request),
- *	.mh.timeout = 1000, //1.000 seconds
- *	.mh.flags = { },
- *	.mh.callback = &my_clear_stall_callback, // **
+ *	.timeout = 1000, //1.000 seconds
+ *	.callback = &my_clear_stall_callback, // **
+ *	.usb_mode = USB_MODE_HOST,
  * };
  *
  * ** "my_clear_stall_callback" calls "usb2_clear_stall_callback"
@@ -2716,3 +2715,78 @@ usb2_do_poll(struct usb2_xfer **ppxfer, uint16_t max)
 		    "not supported!\n");
 	}
 }
+
+static void
+usb2_get_std_packet_size(struct usb2_std_packet_size *ptr, 
+    uint8_t type, uint8_t usb_speed)
+{
+	static const uint16_t intr_range_max[USB_SPEED_MAX] = {
+		[USB_SPEED_LOW] = 8,
+		[USB_SPEED_FULL] = 64,
+		[USB_SPEED_HIGH] = 1024,
+		[USB_SPEED_VARIABLE] = 1024,
+		[USB_SPEED_SUPER] = 1024,
+	};
+
+	static const uint16_t isoc_range_max[USB_SPEED_MAX] = {
+		[USB_SPEED_LOW] = 0,	/* invalid */
+		[USB_SPEED_FULL] = 1023,
+		[USB_SPEED_HIGH] = 1024,
+		[USB_SPEED_VARIABLE] = 3584,
+		[USB_SPEED_SUPER] = 1024,
+	};
+
+	static const uint16_t control_min[USB_SPEED_MAX] = {
+		[USB_SPEED_LOW] = 8,
+		[USB_SPEED_FULL] = 8,
+		[USB_SPEED_HIGH] = 64,
+		[USB_SPEED_VARIABLE] = 512,
+		[USB_SPEED_SUPER] = 512,
+	};
+
+	static const uint16_t bulk_min[USB_SPEED_MAX] = {
+		[USB_SPEED_LOW] = 0,	/* not supported */
+		[USB_SPEED_FULL] = 8,
+		[USB_SPEED_HIGH] = 512,
+		[USB_SPEED_VARIABLE] = 512,
+		[USB_SPEED_SUPER] = 1024,
+	};
+
+	uint16_t temp;
+
+	memset(ptr, 0, sizeof(*ptr));
+
+	switch (type) {
+	case UE_INTERRUPT:
+		ptr->range.max = intr_range_max[usb_speed];
+		break;
+	case UE_ISOCHRONOUS:
+		ptr->range.max = isoc_range_max[usb_speed];
+		break;
+	default:
+		if (type == UE_BULK)
+			temp = bulk_min[usb_speed];
+		else /* UE_CONTROL */
+			temp = control_min[usb_speed];
+
+		/* default is fixed */
+		ptr->fixed[0] = temp;
+		ptr->fixed[1] = temp;
+		ptr->fixed[2] = temp;
+		ptr->fixed[3] = temp;
+
+		if (usb_speed == USB_SPEED_FULL) {
+			/* multiple sizes */
+			ptr->fixed[1] = 16;
+			ptr->fixed[2] = 32;
+			ptr->fixed[3] = 64;
+		}
+		if ((usb_speed == USB_SPEED_VARIABLE) &&
+		    (type == UE_BULK)) {
+			/* multiple sizes */
+			ptr->fixed[2] = 1024;
+			ptr->fixed[3] = 1536;
+		}
+		break;
+	}
+}
diff --git a/sys/dev/usb/usb_transfer.h b/sys/dev/usb/usb_transfer.h
index 34124c5..a15e8a3 100644
--- a/sys/dev/usb/usb_transfer.h
+++ b/sys/dev/usb/usb_transfer.h
@@ -36,36 +36,46 @@ struct usb2_done_msg {
 	struct usb2_xfer_root *xroot;
 };
 
+#define	USB_DMATAG_TO_XROOT(dpt)				\
+  ((struct usb2_xfer_root *)(					\
+   ((uint8_t *)(dpt)) -						\
+   ((uint8_t *)&((struct usb2_xfer_root *)0)->dma_parent_tag)))
+
 /*
  * The following structure is used to keep information about memory
  * that should be automatically freed at the moment all USB transfers
  * have been freed.
  */
 struct usb2_xfer_root {
+	struct usb2_dma_parent_tag dma_parent_tag;
+#if USB_HAVE_BUSDMA
 	struct usb2_xfer_queue dma_q;
+#endif
 	struct usb2_xfer_queue done_q;
 	struct usb2_done_msg done_m[2];
 	struct cv cv_drain;
-	struct usb2_dma_parent_tag dma_parent_tag;
 
 	struct usb2_process *done_p;	/* pointer to callback process */
 	void   *memory_base;
 	struct mtx *xfer_mtx;	/* cannot be changed during operation */
+#if USB_HAVE_BUSDMA
 	struct usb2_page_cache *dma_page_cache_start;
 	struct usb2_page_cache *dma_page_cache_end;
+#endif
 	struct usb2_page_cache *xfer_page_cache_start;
 	struct usb2_page_cache *xfer_page_cache_end;
 	struct usb2_bus *bus;		/* pointer to USB bus (cached) */
 	struct usb2_device *udev;	/* pointer to USB device */
 
-	uint32_t memory_size;
-	uint32_t setup_refcount;
-	uint32_t page_size;
-	uint32_t dma_nframes;		/* number of page caches to load */
-	uint32_t dma_currframe;		/* currect page cache number */
-	uint32_t dma_frlength_0;	/* length of page cache zero */
+	usb2_size_t memory_size;
+	usb2_size_t setup_refcount;
+#if USB_HAVE_BUSDMA
+	usb2_frcount_t dma_nframes;	/* number of page caches to load */
+	usb2_frcount_t dma_currframe;	/* currect page cache number */
+	usb2_frlength_t dma_frlength_0;	/* length of page cache zero */
 	uint8_t	dma_error;		/* set if virtual memory could not be
 					 * loaded */
+#endif
 	uint8_t	done_sleep;		/* set if done thread is sleeping */
 };
 
@@ -83,16 +93,15 @@ struct usb2_setup_params {
 	struct usb2_device *udev;
 	struct usb2_xfer *curr_xfer;
 	const struct usb2_config *curr_setup;
-	const struct usb2_config_sub *curr_setup_sub;
 	const struct usb2_pipe_methods *methods;
 	void   *buf;
-	uint32_t *xfer_length_ptr;
+	usb2_frlength_t *xfer_length_ptr;
 
-	uint32_t size[7];
-	uint32_t bufsize;
-	uint32_t bufsize_max;
-	uint32_t hc_max_frame_size;
+	usb2_size_t size[7];
+	usb2_frlength_t bufsize;
+	usb2_frlength_t bufsize_max;
 
+	uint16_t hc_max_frame_size;
 	uint16_t hc_max_packet_size;
 	uint8_t	hc_max_packet_count;
 	uint8_t	speed;
@@ -103,8 +112,8 @@ struct usb2_setup_params {
 /* function prototypes */
 
 uint8_t	usb2_transfer_setup_sub_malloc(struct usb2_setup_params *parm,
-	    struct usb2_page_cache **ppc, uint32_t size, uint32_t align,
-	    uint32_t count);
+	    struct usb2_page_cache **ppc, usb2_size_t size, usb2_size_t align,
+	    usb2_size_t count);
 void	usb2_command_wrapper(struct usb2_xfer_queue *pq,
 	    struct usb2_xfer *xfer);
 void	usb2_pipe_enter(struct usb2_xfer *xfer);
@@ -122,8 +131,8 @@ usb2_callback_t usb2_do_request_callback;
 usb2_callback_t usb2_handle_request_callback;
 usb2_callback_t usb2_do_clear_stall_callback;
 void	usb2_transfer_timeout_ms(struct usb2_xfer *xfer,
-	    void (*cb) (void *arg), uint32_t ms);
-uint32_t usb2_get_dma_delay(struct usb2_bus *bus);
+	    void (*cb) (void *arg), usb2_timeout_t ms);
+usb2_timeout_t usb2_get_dma_delay(struct usb2_bus *bus);
 void	usb2_transfer_power_ref(struct usb2_xfer *xfer, int val);
 
 #endif					/* _USB2_TRANSFER_H_ */
diff --git a/sys/dev/usb/usb_util.c b/sys/dev/usb/usb_util.c
index 541cd55..5eba73c 100644
--- a/sys/dev/usb/usb_util.c
+++ b/sys/dev/usb/usb_util.c
@@ -24,7 +24,6 @@
  * SUCH DAMAGE.
  */
 
-#include <dev/usb/usb_defs.h>
 #include <dev/usb/usb_mfunc.h>
 #include <dev/usb/usb_error.h>
 #include <dev/usb/usb.h>
@@ -40,7 +39,7 @@
 #include <dev/usb/usb_bus.h>
 
 /* function prototypes */
-#if (USB_USE_CONDVAR == 0)
+#if (USB_HAVE_CONDVAR == 0)
 static int usb2_msleep(void *chan, struct mtx *mtx, int priority, const char *wmesg, int timo);
 
 #endif
@@ -48,6 +47,7 @@ static int usb2_msleep(void *chan, struct mtx *mtx, int priority, const char *wm
 /*------------------------------------------------------------------------*
  * device_delete_all_children - delete all children of a device
  *------------------------------------------------------------------------*/
+#ifndef device_delete_all_children
 int
 device_delete_all_children(device_t dev)
 {
@@ -67,6 +67,7 @@ device_delete_all_children(device_t dev)
 	}
 	return (error);
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *	device_set_usb2_desc
@@ -77,6 +78,7 @@ device_delete_all_children(device_t dev)
 void
 device_set_usb2_desc(device_t dev)
 {
+#if USB_HAVE_STRINGS
 	struct usb2_attach_arg *uaa;
 	struct usb2_device *udev;
 	struct usb2_interface *iface;
@@ -119,6 +121,7 @@ device_set_usb2_desc(device_t dev)
 	device_set_desc_copy(dev, temp_p);
 	device_printf(dev, "<%s> on %s\n", temp_p,
 	    device_get_nameunit(udev->bus->bdev));
+#endif
 }
 
 /*------------------------------------------------------------------------*
@@ -178,6 +181,7 @@ usb2_printBCD(char *p, uint16_t p_len, uint16_t bcd)
  * This function removes spaces at the beginning and the end of the string
  * pointed to by the "p" argument.
  *------------------------------------------------------------------------*/
+#if USB_HAVE_STRINGS
 void
 usb2_trim_spaces(char *p)
 {
@@ -194,20 +198,7 @@ usb2_trim_spaces(char *p)
 			e = p;
 	*e = 0;				/* kill trailing spaces */
 }
-
-/*------------------------------------------------------------------------*
- *	usb2_get_devid
- *
- * This function returns the USB Vendor and Product ID like a 32-bit
- * unsigned integer.
- *------------------------------------------------------------------------*/
-uint32_t
-usb2_get_devid(device_t dev)
-{
-	struct usb2_attach_arg *uaa = device_get_ivars(dev);
-
-	return ((uaa->info.idVendor << 16) | (uaa->info.idProduct));
-}
+#endif
 
 /*------------------------------------------------------------------------*
  *	usb2_make_str_desc - convert an ASCII string into a UNICODE string
@@ -247,7 +238,7 @@ usb2_make_str_desc(void *ptr, uint16_t max_len, const char *s)
 	return (totlen);
 }
 
-#if (USB_USE_CONDVAR == 0)
+#if (USB_HAVE_CONDVAR == 0)
 
 /*------------------------------------------------------------------------*
  *	usb2_cv_init - wrapper function
diff --git a/sys/dev/usb/usb_util.h b/sys/dev/usb/usb_util.h
index e081c31..a265a44 100644
--- a/sys/dev/usb/usb_util.h
+++ b/sys/dev/usb/usb_util.h
@@ -28,14 +28,13 @@
 #define	_USB2_UTIL_H_
 
 int	device_delete_all_children(device_t dev);
-uint32_t usb2_get_devid(device_t dev);
 uint8_t	usb2_make_str_desc(void *ptr, uint16_t max_len, const char *s);
 void	device_set_usb2_desc(device_t dev);
 void	usb2_pause_mtx(struct mtx *mtx, int _ticks);
 void	usb2_printBCD(char *p, uint16_t p_len, uint16_t bcd);
 void	usb2_trim_spaces(char *p);
 
-#if (USB_USE_CONDVAR == 0)
+#if (USB_HAVE_CONDVAR == 0)
 void	usb2_cv_init(struct cv *cv, const char *desc);
 void	usb2_cv_destroy(struct cv *cv);
 void	usb2_cv_wait(struct cv *cv, struct mtx *mtx);
diff --git a/sys/dev/usb/usbdevs b/sys/dev/usb/usbdevs
index bec012e..2406ad4 100644
--- a/sys/dev/usb/usbdevs
+++ b/sys/dev/usb/usbdevs
@@ -593,6 +593,7 @@ vendor SILICOM		0x1485	Silicom
 vendor RALINK		0x148f	Ralink Technology
 vendor IMAGINATION	0x149a	Imagination Technologies
 vendor CONCEPTRONIC2	0x14b2	Conceptronic
+vendor SUPERTOP		0x14cd	Super Top
 vendor PLANEX3		0x14ea	Planex Communications
 vendor SILICONPORTALS	0x1527	Silicon Portals
 vendor UBIQUAM		0x1529	UBIQUAM Co., Ltd.
@@ -1632,6 +1633,7 @@ product LOGITECH QUICKCAMWEB	0x0801	QuickCam Web
 product LOGITECH QUICKCAMPRO	0x0810	QuickCam Pro
 product LOGITECH QUICKCAMEXP	0x0840	QuickCam Express
 product LOGITECH QUICKCAM	0x0850	QuickCam
+product LOGITECH QUICKCAMPRO3	0x0990	QuickCam Pro 9000
 product LOGITECH N43		0xc000	N43
 product LOGITECH N48		0xc001	N48 mouse
 product LOGITECH MBA47		0xc002	M-BA47 mouse
@@ -2025,6 +2027,7 @@ product PUTERCOM UPA100		0x047e	USB-1284 BRIDGE
 /* Qcom products */
 product QCOM RT2573		0x6196	RT2573
 product QCOM RT2573_2		0x6229	RT2573
+product QCOM RT2573_3		0x6238	RT2573
 
 /* Qualcomm products */
 product QUALCOMM CDMA_MSM	0x6000	CDMA Technologies MSM phone
@@ -2308,9 +2311,14 @@ product SUNTAC AS64LX		0x000b	SUNTAC U-Cable type A3
 product SUNTAC AS144L4		0x0011	SUNTAC U-Cable type A4
 
 /* Sun Microsystems products */
-product SUN KEYBOARD		0x0005	Type 6 USB keyboard
+product SUN KEYBOARD_TYPE_6	0x0005	Type 6 USB keyboard
+product SUN KEYBOARD_TYPE_7	0x00a2	Type 7 USB keyboard
 /* XXX The above is a North American PC style keyboard possibly */
 product SUN MOUSE		0x0100	Type 6 USB mouse
+product SUN KBD_HUB		0x100e	Kbd Hub
+
+/* Super Top products */
+product	SUPERTOP IDE		0x6600	USB-IDE
 
 /* Supra products */
 product DIAMOND2 SUPRAEXPRESS56K 0x07da	Supra Express 56K modem
diff --git a/sys/dev/usb/wlan/if_rum.c b/sys/dev/usb/wlan/if_rum.c
index 659010c..856ed38 100644
--- a/sys/dev/usb/wlan/if_rum.c
+++ b/sys/dev/usb/wlan/if_rum.c
@@ -95,6 +95,7 @@ static const struct usb2_device_id rum_devs[] = {
     { USB_VP(USB_VENDOR_PLANEX2,	USB_PRODUCT_PLANEX2_GWUSMM) },
     { USB_VP(USB_VENDOR_QCOM,		USB_PRODUCT_QCOM_RT2573) },
     { USB_VP(USB_VENDOR_QCOM,		USB_PRODUCT_QCOM_RT2573_2) },
+    { USB_VP(USB_VENDOR_QCOM,		USB_PRODUCT_QCOM_RT2573_3) },
     { USB_VP(USB_VENDOR_RALINK,		USB_PRODUCT_RALINK_RT2573) },
     { USB_VP(USB_VENDOR_RALINK,		USB_PRODUCT_RALINK_RT2573_2) },
     { USB_VP(USB_VENDOR_RALINK,		USB_PRODUCT_RALINK_RT2671) },
@@ -362,18 +363,18 @@ static const struct usb2_config rum_config[RUM_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = (MCLBYTES + RT2573_TX_DESC_SIZE + 8),
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = rum_bulk_write_callback,
-		.mh.timeout = 5000,	/* ms */
+		.bufsize = (MCLBYTES + RT2573_TX_DESC_SIZE + 8),
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = rum_bulk_write_callback,
+		.timeout = 5000,	/* ms */
 	},
 	[RUM_BULK_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = (MCLBYTES + RT2573_RX_DESC_SIZE),
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = rum_bulk_read_callback,
+		.bufsize = (MCLBYTES + RT2573_RX_DESC_SIZE),
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = rum_bulk_read_callback,
 	},
 };
 
@@ -494,7 +495,6 @@ rum_attach_post(struct usb2_proc_msg *pm)
 
 	ic->ic_ifp = ifp;
 	ic->ic_phytype = IEEE80211_T_OFDM;	/* not only, but not used */
-	IEEE80211_ADDR_COPY(ic->ic_myaddr, sc->sc_bssid);
 
 	/* set device capabilities */
 	ic->ic_caps =
@@ -516,7 +516,7 @@ rum_attach_post(struct usb2_proc_msg *pm)
 		setbit(&bands, IEEE80211_MODE_11A);
 	ieee80211_init_channels(ic, NULL, &bands);
 
-	ieee80211_ifattach(ic);
+	ieee80211_ifattach(ic, sc->sc_bssid);
 	ic->ic_update_promisc = rum_update_promisc;
 	ic->ic_newassoc = rum_newassoc;
 	ic->ic_raw_xmit = rum_raw_xmit;
@@ -528,8 +528,6 @@ rum_attach_post(struct usb2_proc_msg *pm)
 	ic->ic_vap_create = rum_vap_create;
 	ic->ic_vap_delete = rum_vap_delete;
 
-	sc->sc_rates = ieee80211_get_ratetable(ic->ic_curchan);
-
 	bpfattach(ifp, DLT_IEEE802_11_RADIO,
 	    sizeof (struct ieee80211_frame) + sizeof(sc->sc_txtap));
 
@@ -1068,7 +1066,7 @@ rum_setup_tx_desc(struct rum_softc *sc, struct rum_tx_desc *desc,
 	desc->plcp_service = 4;
 
 	len += IEEE80211_CRC_LEN;
-	if (ieee80211_rate2phytype(sc->sc_rates, rate) == IEEE80211_T_OFDM) {
+	if (ieee80211_rate2phytype(ic->ic_rt, rate) == IEEE80211_T_OFDM) {
 		desc->flags |= htole32(RT2573_TX_OFDM);
 
 		plcp_length = len & 0xfff;
@@ -1107,16 +1105,16 @@ rum_sendprot(struct rum_softc *sc,
 	wh = mtod(m, const struct ieee80211_frame *);
 	pktlen = m->m_pkthdr.len + IEEE80211_CRC_LEN;
 
-	protrate = ieee80211_ctl_rate(sc->sc_rates, rate);
-	ackrate = ieee80211_ack_rate(sc->sc_rates, rate);
+	protrate = ieee80211_ctl_rate(ic->ic_rt, rate);
+	ackrate = ieee80211_ack_rate(ic->ic_rt, rate);
 
 	isshort = (ic->ic_flags & IEEE80211_F_SHPREAMBLE) != 0;
-	dur = ieee80211_compute_duration(sc->sc_rates, pktlen, rate, isshort);
-	    + ieee80211_ack_duration(sc->sc_rates, rate, isshort);
+	dur = ieee80211_compute_duration(ic->ic_rt, pktlen, rate, isshort);
+	    + ieee80211_ack_duration(ic->ic_rt, rate, isshort);
 	flags = RT2573_TX_MORE_FRAG;
 	if (prot == IEEE80211_PROT_RTSCTS) {
 		/* NB: CTS is the same size as an ACK */
-		dur += ieee80211_ack_duration(sc->sc_rates, rate, isshort);
+		dur += ieee80211_ack_duration(ic->ic_rt, rate, isshort);
 		flags |= RT2573_TX_NEED_ACK;
 		mprot = ieee80211_alloc_rts(ic, wh->i_addr1, wh->i_addr2, dur);
 	} else {
@@ -1175,7 +1173,7 @@ rum_tx_mgt(struct rum_softc *sc, struct mbuf *m0, struct ieee80211_node *ni)
 	if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) {
 		flags |= RT2573_TX_NEED_ACK;
 
-		dur = ieee80211_ack_duration(sc->sc_rates, tp->mgmtrate, 
+		dur = ieee80211_ack_duration(ic->ic_rt, tp->mgmtrate, 
 		    ic->ic_flags & IEEE80211_F_SHPREAMBLE);
 		*(uint16_t *)wh->i_dur = htole16(dur);
 
@@ -1295,7 +1293,7 @@ rum_tx_data(struct rum_softc *sc, struct mbuf *m0, struct ieee80211_node *ni)
 		if (m0->m_pkthdr.len + IEEE80211_CRC_LEN > vap->iv_rtsthreshold)
 			prot = IEEE80211_PROT_RTSCTS;
 		else if ((ic->ic_flags & IEEE80211_F_USEPROT) &&
-		    ieee80211_rate2phytype(sc->sc_rates, rate) == IEEE80211_T_OFDM)
+		    ieee80211_rate2phytype(ic->ic_rt, rate) == IEEE80211_T_OFDM)
 			prot = ic->ic_protmode;
 		if (prot != IEEE80211_PROT_NONE) {
 			error = rum_sendprot(sc, m0, ni, prot, rate);
@@ -1319,7 +1317,7 @@ rum_tx_data(struct rum_softc *sc, struct mbuf *m0, struct ieee80211_node *ni)
 		flags |= RT2573_TX_NEED_ACK;
 		flags |= RT2573_TX_MORE_FRAG;
 
-		dur = ieee80211_ack_duration(sc->sc_rates, rate, 
+		dur = ieee80211_ack_duration(ic->ic_rt, rate, 
 		    ic->ic_flags & IEEE80211_F_SHPREAMBLE);
 		*(uint16_t *)wh->i_dur = htole16(dur);
 	}
@@ -1357,12 +1355,6 @@ rum_start(struct ifnet *ifp)
 			break;
 		}
 		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
-		m = ieee80211_encap(ni, m);
-		if (m == NULL) {
-			ieee80211_free_node(ni);
-			ifp->if_oerrors++;
-			continue;
-		}
 		if (rum_tx_data(sc, m, ni) != 0) {
 			ieee80211_free_node(ni);
 			ifp->if_oerrors++;
@@ -2064,8 +2056,7 @@ rum_init_task(struct usb2_proc_msg *pm)
 	/* clear STA registers */
 	rum_read_multi(sc, RT2573_STA_CSR0, sc->sta, sizeof sc->sta);
 
-	IEEE80211_ADDR_COPY(ic->ic_myaddr, IF_LLADDR(ifp));
-	rum_set_macaddr(sc, ic->ic_myaddr);
+	rum_set_macaddr(sc, IF_LLADDR(ifp));
 
 	/* initialize ASIC */
 	rum_write(sc, RT2573_MAC_CSR1, 4);
@@ -2369,7 +2360,6 @@ rum_set_channel(struct ieee80211com *ic)
 	RUM_LOCK(sc);
 	/* do it in a process context */
 	sc->sc_scan_action = RUM_SET_CHANNEL;
-	sc->sc_rates = ieee80211_get_ratetable(ic->ic_curchan);
 	rum_queue_command(sc, rum_scantask,
 	    &sc->sc_scantask[0].hdr, &sc->sc_scantask[1].hdr);
 	RUM_UNLOCK(sc);
diff --git a/sys/dev/usb/wlan/if_rumvar.h b/sys/dev/usb/wlan/if_rumvar.h
index ed08e36..d6b8137 100644
--- a/sys/dev/usb/wlan/if_rumvar.h
+++ b/sys/dev/usb/wlan/if_rumvar.h
@@ -101,7 +101,6 @@ struct rum_softc {
 	struct usb2_device		*sc_udev;
 	struct usb2_process		sc_tq;
 
-	const struct ieee80211_rate_table *sc_rates;
 	struct usb2_xfer		*sc_xfer[RUM_N_TRANSFER];
 	struct rum_task			*sc_last_task;
 
diff --git a/sys/dev/usb/wlan/if_uath.c b/sys/dev/usb/wlan/if_uath.c
new file mode 100644
index 0000000..eb1c31c
--- /dev/null
+++ b/sys/dev/usb/wlan/if_uath.c
@@ -0,0 +1,2862 @@
+/*-
+ * Copyright (c) 2006 Sam Leffler, Errno Consulting
+ * Copyright (c) 2008-2009 Weongyo Jeong <weongyo@freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
+ *    redistribution must be conditioned upon including a substantially
+ *    similar Disclaimer requirement for further binary redistribution.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
+ * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGES.
+ */
+
+/*
+ * This driver is distantly derived from a driver of the same name
+ * by Damien Bergamini.  The original copyright is included below:
+ *
+ * Copyright (c) 2006
+ *	Damien Bergamini <damien.bergamini@free.fr>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*-
+ * Driver for Atheros AR5523 USB parts.
+ *
+ * The driver requires firmware to be loaded into the device.  This
+ * is done on device discovery from a user application (uathload)
+ * that is launched by devd when a device with suitable product ID
+ * is recognized.  Once firmware has been loaded the device will
+ * reset the USB port and re-attach with the original product ID+1
+ * and this driver will be attached.  The firmware is licensed for
+ * general use (royalty free) and may be incorporated in products.
+ * Note that the firmware normally packaged with the NDIS drivers
+ * for these devices does not work in this way and so does not work
+ * with this driver.
+ */
+#include <sys/param.h>
+#include <sys/sockio.h>
+#include <sys/sysctl.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/mbuf.h>
+#include <sys/kernel.h>
+#include <sys/socket.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/kdb.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <sys/rman.h>
+
+#include <net/bpf.h>
+#include <net/if.h>
+#include <net/if_arp.h>
+#include <net/ethernet.h>
+#include <net/if_dl.h>
+#include <net/if_media.h>
+#include <net/if_types.h>
+
+#ifdef INET
+#include <netinet/in.h>
+#include <netinet/in_systm.h>
+#include <netinet/in_var.h>
+#include <netinet/if_ether.h>
+#include <netinet/ip.h>
+#endif
+
+#include <net80211/ieee80211_var.h>
+#include <net80211/ieee80211_regdomain.h>
+#include <net80211/ieee80211_radiotap.h>
+
+#include <dev/usb/usb.h>
+#include <dev/usb/usb_core.h>
+#include <dev/usb/usb_busdma.h>
+#include <dev/usb/usb_debug.h>
+#include <dev/usb/usb_error.h>
+#include <dev/usb/usb_lookup.h>
+#include <dev/usb/usb_util.h>
+#include "usbdevs.h"
+
+#include <dev/usb/wlan/if_uathreg.h>
+#include <dev/usb/wlan/if_uathvar.h>
+
+SYSCTL_NODE(_hw_usb2, OID_AUTO, uath, CTLFLAG_RW, 0, "USB Atheros");
+
+static	int uath_countrycode = CTRY_DEFAULT;	/* country code */
+SYSCTL_INT(_hw_usb2_uath, OID_AUTO, countrycode, CTLFLAG_RW, &uath_countrycode,
+    0, "country code");
+TUNABLE_INT("hw.usb2.uath.countrycode", &uath_countrycode);
+static	int uath_regdomain = 0;			/* regulatory domain */
+SYSCTL_INT(_hw_usb2_uath, OID_AUTO, regdomain, CTLFLAG_RD, &uath_regdomain,
+    0, "regulatory domain");
+
+#ifdef UATH_DEBUG
+int uath_debug = 0;
+SYSCTL_INT(_hw_usb2_uath, OID_AUTO, debug, CTLFLAG_RW, &uath_debug, 0,
+    "uath debug level");
+TUNABLE_INT("hw.usb.uath.debug", &uath_debug);
+enum {
+	UATH_DEBUG_XMIT		= 0x00000001,	/* basic xmit operation */
+	UATH_DEBUG_XMIT_DUMP	= 0x00000002,	/* xmit dump */
+	UATH_DEBUG_RECV		= 0x00000004,	/* basic recv operation */
+	UATH_DEBUG_TX_PROC	= 0x00000008,	/* tx ISR proc */
+	UATH_DEBUG_RX_PROC	= 0x00000010,	/* rx ISR proc */
+	UATH_DEBUG_RECV_ALL	= 0x00000020,	/* trace all frames (beacons) */
+	UATH_DEBUG_INIT		= 0x00000040,	/* initialization of dev */
+	UATH_DEBUG_DEVCAP	= 0x00000080,	/* dev caps */
+	UATH_DEBUG_CMDS		= 0x00000100,	/* commands */
+	UATH_DEBUG_CMDS_DUMP	= 0x00000200,	/* command buffer dump */
+	UATH_DEBUG_RESET	= 0x00000400,	/* reset processing */
+	UATH_DEBUG_STATE	= 0x00000800,	/* 802.11 state transitions */
+	UATH_DEBUG_MULTICAST	= 0x00001000,	/* multicast */
+	UATH_DEBUG_WME		= 0x00002000,	/* WME */
+	UATH_DEBUG_CHANNEL	= 0x00004000,	/* channel */
+	UATH_DEBUG_RATES	= 0x00008000,	/* rates */
+	UATH_DEBUG_CRYPTO	= 0x00010000,	/* crypto */
+	UATH_DEBUG_LED		= 0x00020000,	/* LED */
+	UATH_DEBUG_ANY		= 0xffffffff
+};
+#define	DPRINTF(sc, m, fmt, ...) do {				\
+	if (sc->sc_debug & (m))					\
+		printf(fmt, __VA_ARGS__);			\
+} while (0)
+#else
+#define	DPRINTF(sc, m, fmt, ...) do {				\
+	(void) sc;						\
+} while (0)
+#endif
+
+/* unaligned little endian access */
+#define LE_READ_2(p)							\
+	((u_int16_t)							\
+	 ((((u_int8_t *)(p))[0]      ) | (((u_int8_t *)(p))[1] <<  8)))
+#define LE_READ_4(p)							\
+	((u_int32_t)							\
+	 ((((u_int8_t *)(p))[0]      ) | (((u_int8_t *)(p))[1] <<  8) |	\
+	  (((u_int8_t *)(p))[2] << 16) | (((u_int8_t *)(p))[3] << 24)))
+
+/* recognized device vendors/products */
+static const struct usb2_device_id uath_devs[] = {
+#define	UATH_DEV(v,p) { USB_VP(USB_VENDOR_##v, USB_PRODUCT_##v##_##p) }
+	UATH_DEV(ATHEROS,		AR5523),
+	UATH_DEV(ATHEROS2,		AR5523_1),
+	UATH_DEV(ATHEROS2,		AR5523_2),
+	UATH_DEV(ATHEROS2,		AR5523_3),
+	UATH_DEV(CONCEPTRONIC,		AR5523_1),
+	UATH_DEV(CONCEPTRONIC,		AR5523_2),
+	UATH_DEV(DLINK,			DWLAG122),
+	UATH_DEV(DLINK,			DWLAG132),
+	UATH_DEV(DLINK,			DWLG132),
+	UATH_DEV(GIGASET,		AR5523),
+	UATH_DEV(GIGASET,		SMCWUSBTG),
+	UATH_DEV(GLOBALSUN,		AR5523_1),
+	UATH_DEV(GLOBALSUN,		AR5523_2),
+	UATH_DEV(NETGEAR,		WG111U),
+	UATH_DEV(NETGEAR3,		WG111T),
+	UATH_DEV(NETGEAR3,		WPN111),
+	UATH_DEV(UMEDIA,		TEW444UBEU),
+	UATH_DEV(UMEDIA,		AR5523_2),
+	UATH_DEV(WISTRONNEWEB,		AR5523_1),
+	UATH_DEV(WISTRONNEWEB,		AR5523_2),
+	UATH_DEV(ZCOM,			AR5523)
+#undef UATH_DEV
+};
+
+static usb2_callback_t uath_intr_rx_callback;
+static usb2_callback_t uath_intr_tx_callback;
+static usb2_callback_t uath_bulk_rx_callback;
+static usb2_callback_t uath_bulk_tx_callback;
+
+static const struct usb2_config uath_usbconfig[UATH_N_XFERS] = {
+	[UATH_INTR_RX] = {
+		.type = UE_BULK,
+		.endpoint = 0x1,
+		.direction = UE_DIR_IN,
+		.bufsize = UATH_MAX_CMDSZ,
+		.flags = {
+			.pipe_bof = 1,
+			.short_xfer_ok = 1
+		},
+		.callback = uath_intr_rx_callback
+	},
+	[UATH_INTR_TX] = {
+		.type = UE_BULK,
+		.endpoint = 0x1,
+		.direction = UE_DIR_OUT,
+		.bufsize = UATH_MAX_CMDSZ,
+		.flags = {
+			.ext_buffer = 1,
+			.force_short_xfer = 1,
+			.pipe_bof = 1,
+		},
+		.callback = uath_intr_tx_callback,
+		.timeout = UATH_CMD_TIMEOUT
+	},
+	[UATH_BULK_RX] = {
+		.type = UE_BULK,
+		.endpoint = 0x2,
+		.direction = UE_DIR_IN,
+		.bufsize = MCLBYTES,
+		.flags = {
+			.ext_buffer = 1,
+			.pipe_bof = 1,
+			.short_xfer_ok = 1
+		},
+		.callback = uath_bulk_rx_callback
+	},
+	[UATH_BULK_TX] = {
+		.type = UE_BULK,
+		.endpoint = 0x2,
+		.direction = UE_DIR_OUT,
+		.bufsize = UATH_MAX_TXBUFSZ,
+		.flags = {
+			.ext_buffer = 1,
+			.force_short_xfer = 1,
+			.pipe_bof = 1
+		},
+		.callback = uath_bulk_tx_callback,
+		.timeout = UATH_DATA_TIMEOUT
+	}
+};
+
+static struct ieee80211vap *uath_vap_create(struct ieee80211com *,
+		    const char name[IFNAMSIZ], int unit, int opmode,
+		    int flags, const uint8_t bssid[IEEE80211_ADDR_LEN],
+		    const uint8_t mac[IEEE80211_ADDR_LEN]);
+static void	uath_vap_delete(struct ieee80211vap *);
+static int	uath_alloc_cmd_list(struct uath_softc *, struct uath_cmd [],
+		    int, int);
+static void	uath_free_cmd_list(struct uath_softc *, struct uath_cmd [],
+		    int);
+static int	uath_host_available(struct uath_softc *);
+static int	uath_get_capability(struct uath_softc *, uint32_t, uint32_t *);
+static int	uath_get_devcap(struct uath_softc *);
+static struct uath_cmd *
+		uath_get_cmdbuf(struct uath_softc *);
+static int	uath_cmd_read(struct uath_softc *, uint32_t, const void *,
+		    int, void *, int, int);
+static int	uath_cmd_write(struct uath_softc *, uint32_t, const void *,
+		    int, int);
+static void	uath_stat(void *);
+#ifdef UATH_DEBUG
+static void	uath_dump_cmd(const uint8_t *, int, char);
+static const char *
+		uath_codename(int);
+#endif
+static int	uath_get_devstatus(struct uath_softc *,
+		    uint8_t macaddr[IEEE80211_ADDR_LEN]);
+static int	uath_get_status(struct uath_softc *, uint32_t, void *, int);
+static int	uath_alloc_rx_data_list(struct uath_softc *);
+static int	uath_alloc_tx_data_list(struct uath_softc *);
+static void	uath_free_rx_data_list(struct uath_softc *);
+static void	uath_free_tx_data_list(struct uath_softc *);
+static int	uath_init_locked(void *);
+static void	uath_init(void *);
+static void	uath_stop_locked(struct ifnet *);
+static void	uath_stop(struct ifnet *);
+static int	uath_ioctl(struct ifnet *, u_long, caddr_t);
+static void	uath_start(struct ifnet *);
+static int	uath_raw_xmit(struct ieee80211_node *, struct mbuf *,
+		    const struct ieee80211_bpf_params *);
+static void	uath_scan_start(struct ieee80211com *);
+static void	uath_scan_end(struct ieee80211com *);
+static void	uath_set_channel(struct ieee80211com *);
+static void	uath_update_mcast(struct ifnet *);
+static void	uath_update_promisc(struct ifnet *);
+static int	uath_config(struct uath_softc *, uint32_t, uint32_t);
+static int	uath_config_multi(struct uath_softc *, uint32_t, const void *,
+		    int);
+static int	uath_switch_channel(struct uath_softc *,
+		    struct ieee80211_channel *);
+static int	uath_set_rxfilter(struct uath_softc *, uint32_t, uint32_t);
+static void	uath_watchdog(void *);
+static void	uath_abort_xfers(struct uath_softc *);
+static int	uath_dataflush(struct uath_softc *);
+static int	uath_cmdflush(struct uath_softc *);
+static int	uath_flush(struct uath_softc *);
+static int	uath_set_ledstate(struct uath_softc *, int);
+static int	uath_set_chan(struct uath_softc *, struct ieee80211_channel *);
+static int	uath_reset_tx_queues(struct uath_softc *);
+static int	uath_wme_init(struct uath_softc *);
+static struct uath_data *
+		uath_getbuf(struct uath_softc *);
+static int	uath_newstate(struct ieee80211vap *, enum ieee80211_state,
+		    int);
+static int	uath_set_key(struct uath_softc *,
+		    const struct ieee80211_key *, int);
+static int	uath_set_keys(struct uath_softc *, struct ieee80211vap *);
+static void	uath_sysctl_node(struct uath_softc *);
+
+static int
+uath_match(device_t dev)
+{
+	struct usb2_attach_arg *uaa = device_get_ivars(dev);
+
+	if (uaa->usb2_mode != USB_MODE_HOST)
+		return (ENXIO);
+	if (uaa->info.bConfigIndex != UATH_CONFIG_INDEX)
+		return (ENXIO);
+	if (uaa->info.bIfaceIndex != UATH_IFACE_INDEX)
+		return (ENXIO);
+
+	return (usb2_lookup_id_by_uaa(uath_devs, sizeof(uath_devs), uaa));
+}
+
+static int
+uath_attach(device_t dev)
+{
+	struct uath_softc *sc = device_get_softc(dev);
+	struct usb2_attach_arg *uaa = device_get_ivars(dev);
+	struct ieee80211com *ic;
+	struct ifnet *ifp;
+	uint8_t bands, iface_index = UATH_IFACE_INDEX;		/* XXX */
+	usb2_error_t error;
+	uint8_t macaddr[IEEE80211_ADDR_LEN];
+
+	sc->sc_dev = dev;
+	sc->sc_udev = uaa->device;
+#ifdef UATH_DEBUG
+	sc->sc_debug = uath_debug;
+#endif
+
+	/*
+	 * Only post-firmware devices here.
+	 */
+	mtx_init(&sc->sc_mtx, device_get_nameunit(sc->sc_dev), MTX_NETWORK_LOCK,
+	    MTX_DEF);
+	callout_init(&sc->stat_ch, 0);
+	callout_init_mtx(&sc->watchdog_ch, &sc->sc_mtx, 0);
+
+	/*
+	 * Allocate xfers for firmware commands.
+	 */
+	error = uath_alloc_cmd_list(sc, sc->sc_cmd, UATH_CMD_LIST_COUNT,
+	    UATH_MAX_CMDSZ);
+	if (error != 0) {
+		device_printf(sc->sc_dev,
+		    "could not allocate Tx command list\n");
+		goto fail;
+	}
+
+	error = usb2_transfer_setup(uaa->device, &iface_index, sc->sc_xfer,
+	    uath_usbconfig, UATH_N_XFERS, sc, &sc->sc_mtx);
+	if (error) {
+		device_printf(dev, "could not allocate USB transfers, "
+		    "err=%s\n", usb2_errstr(error));
+		goto fail1;
+	}
+
+	/*
+	 * We're now ready to send+receive firmware commands.
+	 */
+	UATH_LOCK(sc);
+	error = uath_host_available(sc);
+	if (error != 0) {
+		device_printf(sc->sc_dev, "could not initialize adapter\n");
+		goto fail3;
+	}
+	error = uath_get_devcap(sc);
+	if (error != 0) {
+		device_printf(sc->sc_dev,
+		    "could not get device capabilities\n");
+		goto fail3;
+	}
+	UATH_UNLOCK(sc);
+
+	/* Create device sysctl node. */
+	uath_sysctl_node(sc);
+
+	ifp = sc->sc_ifp = if_alloc(IFT_IEEE80211);
+	if (ifp == NULL) {
+		device_printf(sc->sc_dev, "can not allocate ifnet\n");
+		error = ENXIO;
+		goto fail2;
+	}
+
+	UATH_LOCK(sc);
+	error = uath_get_devstatus(sc, macaddr);
+	if (error != 0) {
+		device_printf(sc->sc_dev, "could not get device status\n");
+		goto fail4;
+	}
+
+	/*
+	 * Allocate xfers for Rx/Tx data pipes.
+	 */
+	error = uath_alloc_rx_data_list(sc);
+	if (error != 0) {
+		device_printf(sc->sc_dev, "could not allocate Rx data list\n");
+		goto fail4;
+	}
+	error = uath_alloc_tx_data_list(sc);
+	if (error != 0) {
+		device_printf(sc->sc_dev, "could not allocate Tx data list\n");
+		goto fail4;
+	}
+	UATH_UNLOCK(sc);
+
+	ifp->if_softc = sc;
+	if_initname(ifp, "uath", device_get_unit(sc->sc_dev));
+	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
+	ifp->if_init = uath_init;
+	ifp->if_ioctl = uath_ioctl;
+	ifp->if_start = uath_start;
+	/* XXX UATH_TX_DATA_LIST_COUNT */
+	IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN);
+	ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN;
+	IFQ_SET_READY(&ifp->if_snd);
+
+	ic = ifp->if_l2com;
+	ic->ic_ifp = ifp;
+	ic->ic_phytype = IEEE80211_T_OFDM;	/* not only, but not used */
+	ic->ic_opmode = IEEE80211_M_STA;	/* default to BSS mode */
+
+	/* set device capabilities */
+	ic->ic_caps =
+	    IEEE80211_C_STA |		/* station mode */
+	    IEEE80211_C_MONITOR |	/* monitor mode supported */
+	    IEEE80211_C_TXPMGT |	/* tx power management */
+	    IEEE80211_C_SHPREAMBLE |	/* short preamble supported */
+	    IEEE80211_C_SHSLOT |	/* short slot time supported */
+	    IEEE80211_C_WPA |		/* 802.11i */
+	    IEEE80211_C_BGSCAN |	/* capable of bg scanning */
+	    IEEE80211_C_TXFRAG;		/* handle tx frags */
+
+	/* put a regulatory domain to reveal informations.  */
+	uath_regdomain = sc->sc_devcap.regDomain;
+
+	bands = 0;
+	setbit(&bands, IEEE80211_MODE_11B);
+	setbit(&bands, IEEE80211_MODE_11G);
+	if ((sc->sc_devcap.analog5GhzRevision & 0xf0) == 0x30)
+		setbit(&bands, IEEE80211_MODE_11A);
+	/* XXX turbo */
+	ieee80211_init_channels(ic, NULL, &bands);
+
+	ieee80211_ifattach(ic, macaddr);
+	ic->ic_raw_xmit = uath_raw_xmit;
+	ic->ic_scan_start = uath_scan_start;
+	ic->ic_scan_end = uath_scan_end;
+	ic->ic_set_channel = uath_set_channel;
+
+	ic->ic_vap_create = uath_vap_create;
+	ic->ic_vap_delete = uath_vap_delete;
+	ic->ic_update_mcast = uath_update_mcast;
+	ic->ic_update_promisc = uath_update_promisc;
+
+	bpfattach(ifp, DLT_IEEE802_11_RADIO,
+	    sizeof (struct ieee80211_frame) + sizeof(sc->sc_txtap));
+
+	sc->sc_rxtap_len = sizeof sc->sc_rxtap;
+	sc->sc_rxtap.wr_ihdr.it_len = htole16(sc->sc_rxtap_len);
+	sc->sc_rxtap.wr_ihdr.it_present = htole32(UATH_RX_RADIOTAP_PRESENT);
+
+	sc->sc_txtap_len = sizeof sc->sc_txtap;
+	sc->sc_txtap.wt_ihdr.it_len = htole16(sc->sc_txtap_len);
+	sc->sc_txtap.wt_ihdr.it_present = htole32(UATH_TX_RADIOTAP_PRESENT);
+
+	if (bootverbose)
+		ieee80211_announce(ic);
+
+	return (0);
+
+fail4:	if_free(ifp);
+fail3:	UATH_UNLOCK(sc);
+fail2:	usb2_transfer_unsetup(sc->sc_xfer, UATH_N_XFERS);
+fail1:	uath_free_cmd_list(sc, sc->sc_cmd, UATH_CMD_LIST_COUNT);
+fail:
+	return (error);
+}
+
+static int
+uath_detach(device_t dev)
+{
+	struct uath_softc *sc = device_get_softc(dev);
+	struct ifnet *ifp = sc->sc_ifp;
+	struct ieee80211com *ic = ifp->if_l2com;
+
+	if (!device_is_attached(dev))
+		return (0);
+
+	sc->sc_flags |= UATH_FLAG_INVALID;
+	uath_stop(ifp);
+	ieee80211_ifdetach(ic);
+
+	callout_drain(&sc->stat_ch);
+	callout_drain(&sc->watchdog_ch);
+
+	usb2_transfer_unsetup(sc->sc_xfer, UATH_N_XFERS);
+
+	/* free buffers */
+	UATH_LOCK(sc);
+	uath_free_rx_data_list(sc);
+	uath_free_tx_data_list(sc);
+	uath_free_cmd_list(sc, sc->sc_cmd, UATH_CMD_LIST_COUNT);
+	UATH_UNLOCK(sc);
+
+	bpfdetach(ifp);
+	if_free(ifp);
+	mtx_destroy(&sc->sc_mtx);
+	return (0);
+}
+
+static void
+uath_free_cmd_list(struct uath_softc *sc, struct uath_cmd cmds[], int ncmd)
+{
+	int i;
+
+	for (i = 0; i < ncmd; i++)
+		if (cmds[i].buf != NULL)
+			free(cmds[i].buf, M_USBDEV);
+}
+
+static int
+uath_alloc_cmd_list(struct uath_softc *sc, struct uath_cmd cmds[],
+	int ncmd, int maxsz)
+{
+	int i, error;
+
+	STAILQ_INIT(&sc->sc_cmd_active);
+	STAILQ_INIT(&sc->sc_cmd_pending);
+	STAILQ_INIT(&sc->sc_cmd_waiting);
+	STAILQ_INIT(&sc->sc_cmd_inactive);
+
+	for (i = 0; i < ncmd; i++) {
+		struct uath_cmd *cmd = &cmds[i];
+
+		cmd->sc = sc;	/* backpointer for callbacks */
+		cmd->msgid = i;
+		cmd->buf = malloc(maxsz, M_USBDEV, M_NOWAIT);
+		if (cmd->buf == NULL) {
+			device_printf(sc->sc_dev,
+			    "could not allocate xfer buffer\n");
+			error = ENOMEM;
+			goto fail;
+		}
+		STAILQ_INSERT_TAIL(&sc->sc_cmd_inactive, cmd, next);
+		UATH_STAT_INC(sc, st_cmd_inactive);
+	}
+	return (0);
+
+fail:	uath_free_cmd_list(sc, cmds, ncmd);
+	return (error);
+}
+
+static int
+uath_host_available(struct uath_softc *sc)
+{
+	struct uath_cmd_host_available setup;
+
+	UATH_ASSERT_LOCKED(sc);
+
+	/* inform target the host is available */
+	setup.sw_ver_major = htobe32(ATH_SW_VER_MAJOR);
+	setup.sw_ver_minor = htobe32(ATH_SW_VER_MINOR);
+	setup.sw_ver_patch = htobe32(ATH_SW_VER_PATCH);
+	setup.sw_ver_build = htobe32(ATH_SW_VER_BUILD);
+	return uath_cmd_read(sc, WDCMSG_HOST_AVAILABLE,
+		&setup, sizeof setup, NULL, 0, 0);
+}
+
+#ifdef UATH_DEBUG
+static void
+uath_dump_cmd(const uint8_t *buf, int len, char prefix)
+{
+	const char *sep = "";
+	int i;
+
+	for (i = 0; i < len; i++) {
+		if ((i % 16) == 0) {
+			printf("%s%c ", sep, prefix);
+			sep = "\n";
+		}
+		else if ((i % 4) == 0)
+			printf(" ");
+		printf("%02x", buf[i]);
+	}
+	printf("\n");
+}
+
+static const char *
+uath_codename(int code)
+{
+#define	N(a)	(sizeof(a)/sizeof(a[0]))
+	static const char *names[] = {
+	    "0x00",
+	    "HOST_AVAILABLE",
+	    "BIND",
+	    "TARGET_RESET",
+	    "TARGET_GET_CAPABILITY",
+	    "TARGET_SET_CONFIG",
+	    "TARGET_GET_STATUS",
+	    "TARGET_GET_STATS",
+	    "TARGET_START",
+	    "TARGET_STOP",
+	    "TARGET_ENABLE",
+	    "TARGET_DISABLE",
+	    "CREATE_CONNECTION",
+	    "UPDATE_CONNECT_ATTR",
+	    "DELETE_CONNECT",
+	    "SEND",
+	    "FLUSH",
+	    "STATS_UPDATE",
+	    "BMISS",
+	    "DEVICE_AVAIL",
+	    "SEND_COMPLETE",
+	    "DATA_AVAIL",
+	    "SET_PWR_MODE",
+	    "BMISS_ACK",
+	    "SET_LED_STEADY",
+	    "SET_LED_BLINK",
+	    "SETUP_BEACON_DESC",
+	    "BEACON_INIT",
+	    "RESET_KEY_CACHE",
+	    "RESET_KEY_CACHE_ENTRY",
+	    "SET_KEY_CACHE_ENTRY",
+	    "SET_DECOMP_MASK",
+	    "SET_REGULATORY_DOMAIN",
+	    "SET_LED_STATE",
+	    "WRITE_ASSOCID",
+	    "SET_STA_BEACON_TIMERS",
+	    "GET_TSF",
+	    "RESET_TSF",
+	    "SET_ADHOC_MODE",
+	    "SET_BASIC_RATE",
+	    "MIB_CONTROL",
+	    "GET_CHANNEL_DATA",
+	    "GET_CUR_RSSI",
+	    "SET_ANTENNA_SWITCH",
+	    "0x2c", "0x2d", "0x2e",
+	    "USE_SHORT_SLOT_TIME",
+	    "SET_POWER_MODE",
+	    "SETUP_PSPOLL_DESC",
+	    "SET_RX_MULTICAST_FILTER",
+	    "RX_FILTER",
+	    "PER_CALIBRATION",
+	    "RESET",
+	    "DISABLE",
+	    "PHY_DISABLE",
+	    "SET_TX_POWER_LIMIT",
+	    "SET_TX_QUEUE_PARAMS",
+	    "SETUP_TX_QUEUE",
+	    "RELEASE_TX_QUEUE",
+	};
+	static char buf[8];
+
+	if (code < N(names))
+		return names[code];
+	if (code == WDCMSG_SET_DEFAULT_KEY)
+		return "SET_DEFAULT_KEY";
+	snprintf(buf, sizeof(buf), "0x%02x", code);
+	return buf;
+#undef N
+}
+#endif
+
+/*
+ * Low-level function to send read or write commands to the firmware.
+ */
+static int
+uath_cmdsend(struct uath_softc *sc, uint32_t code, const void *idata, int ilen,
+    void *odata, int olen, int flags)
+{
+	struct uath_cmd_hdr *hdr;
+	struct uath_cmd *cmd;
+	int error;
+
+	UATH_ASSERT_LOCKED(sc);
+
+	/* grab a xfer */
+	cmd = uath_get_cmdbuf(sc);
+	if (cmd == NULL) {
+		device_printf(sc->sc_dev, "%s: empty inactive queue\n",
+		    __func__);
+		return (ENOBUFS);
+	}
+	cmd->flags = flags;
+	/* always bulk-out a multiple of 4 bytes */
+	cmd->buflen = roundup2(sizeof(struct uath_cmd_hdr) + ilen, 4);
+
+	hdr = (struct uath_cmd_hdr *)cmd->buf;
+	bzero(hdr, sizeof (struct uath_cmd_hdr));	/* XXX not needed */
+	hdr->len   = htobe32(cmd->buflen);
+	hdr->code  = htobe32(code);
+	hdr->msgid = cmd->msgid;	/* don't care about endianness */
+	hdr->magic = htobe32((cmd->flags & UATH_CMD_FLAG_MAGIC) ? 1 << 24 : 0);
+	bcopy(idata, (uint8_t *)(hdr + 1), ilen);
+
+#ifdef UATH_DEBUG
+	if (sc->sc_debug & UATH_DEBUG_CMDS) {
+		printf("%s: send  %s [flags 0x%x] olen %d\n",
+		    __func__, uath_codename(code), cmd->flags, olen);
+		if (sc->sc_debug & UATH_DEBUG_CMDS_DUMP)
+			uath_dump_cmd(cmd->buf, cmd->buflen, '+');
+	}
+#endif
+	cmd->odata = odata;
+	KASSERT(odata == NULL ||
+	    olen < UATH_MAX_CMDSZ - sizeof(*hdr) + sizeof(uint32_t),
+	    ("odata %p olen %u", odata, olen));
+	cmd->olen = olen;
+
+	STAILQ_INSERT_TAIL(&sc->sc_cmd_pending, cmd, next);
+	UATH_STAT_INC(sc, st_cmd_pending);
+	usb2_transfer_start(sc->sc_xfer[UATH_INTR_TX]);
+
+	if (cmd->flags & UATH_CMD_FLAG_READ) {
+		usb2_transfer_start(sc->sc_xfer[UATH_INTR_RX]);
+
+		/* wait at most two seconds for command reply */
+		error = mtx_sleep(cmd, &sc->sc_mtx, 0, "uathcmd", 2 * hz);
+		cmd->odata = NULL;	/* in case reply comes too late */
+		if (error != 0) {
+			device_printf(sc->sc_dev, "timeout waiting for reply "
+			    "to cmd 0x%x (%u)\n", code, code);
+		} else if (cmd->olen != olen) {
+			device_printf(sc->sc_dev, "unexpected reply data count "
+			    "to cmd 0x%x (%u), got %u, expected %u\n",
+			    code, code, cmd->olen, olen);
+			error = EINVAL;
+		}
+		return (error);
+	}
+	return (0);
+}
+
+static int
+uath_cmd_read(struct uath_softc *sc, uint32_t code, const void *idata,
+    int ilen, void *odata, int olen, int flags)
+{
+
+	flags |= UATH_CMD_FLAG_READ;
+	return uath_cmdsend(sc, code, idata, ilen, odata, olen, flags);
+}
+
+static int
+uath_cmd_write(struct uath_softc *sc, uint32_t code, const void *data, int len,
+    int flags)
+{
+
+	flags &= ~UATH_CMD_FLAG_READ;
+	return uath_cmdsend(sc, code, data, len, NULL, 0, flags);
+}
+
+static struct uath_cmd *
+uath_get_cmdbuf(struct uath_softc *sc)
+{
+	struct uath_cmd *uc;
+
+	UATH_ASSERT_LOCKED(sc);
+
+	uc = STAILQ_FIRST(&sc->sc_cmd_inactive);
+	if (uc != NULL) {
+		STAILQ_REMOVE_HEAD(&sc->sc_cmd_inactive, next);
+		UATH_STAT_DEC(sc, st_cmd_inactive);
+	} else
+		uc = NULL;
+	if (uc == NULL)
+		DPRINTF(sc, UATH_DEBUG_XMIT, "%s: %s\n", __func__,
+		    "out of command xmit buffers");
+	return (uc);
+}
+
+/*
+ * This function is called periodically (every second) when associated to
+ * query device statistics.
+ */
+static void
+uath_stat(void *arg)
+{
+	struct uath_softc *sc = arg;
+	int error;
+
+	UATH_LOCK(sc);
+	/*
+	 * Send request for statistics asynchronously. The timer will be
+	 * restarted when we'll get the stats notification.
+	 */
+	error = uath_cmd_write(sc, WDCMSG_TARGET_GET_STATS, NULL, 0,
+	    UATH_CMD_FLAG_ASYNC);
+	if (error != 0) {
+		device_printf(sc->sc_dev,
+		    "could not query stats, error %d\n", error);
+	}
+	UATH_UNLOCK(sc);
+}
+
+static int
+uath_get_capability(struct uath_softc *sc, uint32_t cap, uint32_t *val)
+{
+	int error;
+
+	cap = htobe32(cap);
+	error = uath_cmd_read(sc, WDCMSG_TARGET_GET_CAPABILITY,
+	    &cap, sizeof cap, val, sizeof(uint32_t), UATH_CMD_FLAG_MAGIC);
+	if (error != 0) {
+		device_printf(sc->sc_dev, "could not read capability %u\n",
+		    be32toh(cap));
+		return (error);
+	}
+	*val = be32toh(*val);
+	return (error);
+}
+
+static int
+uath_get_devcap(struct uath_softc *sc)
+{
+#define	GETCAP(x, v) do {				\
+	error = uath_get_capability(sc, x, &v);		\
+	if (error != 0)					\
+		return (error);				\
+	DPRINTF(sc, UATH_DEBUG_DEVCAP,			\
+	    "%s: %s=0x%08x\n", __func__, #x, v);	\
+} while (0)
+	struct uath_devcap *cap = &sc->sc_devcap;
+	int error;
+
+	/* collect device capabilities */
+	GETCAP(CAP_TARGET_VERSION, cap->targetVersion);
+	GETCAP(CAP_TARGET_REVISION, cap->targetRevision);
+	GETCAP(CAP_MAC_VERSION, cap->macVersion);
+	GETCAP(CAP_MAC_REVISION, cap->macRevision);
+	GETCAP(CAP_PHY_REVISION, cap->phyRevision);
+	GETCAP(CAP_ANALOG_5GHz_REVISION, cap->analog5GhzRevision);
+	GETCAP(CAP_ANALOG_2GHz_REVISION, cap->analog2GhzRevision);
+
+	GETCAP(CAP_REG_DOMAIN, cap->regDomain);
+	GETCAP(CAP_REG_CAP_BITS, cap->regCapBits);
+#if 0
+	/* NB: not supported in rev 1.5 */
+	GETCAP(CAP_COUNTRY_CODE, cap->countryCode);
+#endif
+	GETCAP(CAP_WIRELESS_MODES, cap->wirelessModes);
+	GETCAP(CAP_CHAN_SPREAD_SUPPORT, cap->chanSpreadSupport);
+	GETCAP(CAP_COMPRESS_SUPPORT, cap->compressSupport);
+	GETCAP(CAP_BURST_SUPPORT, cap->burstSupport);
+	GETCAP(CAP_FAST_FRAMES_SUPPORT, cap->fastFramesSupport);
+	GETCAP(CAP_CHAP_TUNING_SUPPORT, cap->chapTuningSupport);
+	GETCAP(CAP_TURBOG_SUPPORT, cap->turboGSupport);
+	GETCAP(CAP_TURBO_PRIME_SUPPORT, cap->turboPrimeSupport);
+	GETCAP(CAP_DEVICE_TYPE, cap->deviceType);
+	GETCAP(CAP_WME_SUPPORT, cap->wmeSupport);
+	GETCAP(CAP_TOTAL_QUEUES, cap->numTxQueues);
+	GETCAP(CAP_CONNECTION_ID_MAX, cap->connectionIdMax);
+
+	GETCAP(CAP_LOW_5GHZ_CHAN, cap->low5GhzChan);
+	GETCAP(CAP_HIGH_5GHZ_CHAN, cap->high5GhzChan);
+	GETCAP(CAP_LOW_2GHZ_CHAN, cap->low2GhzChan);
+	GETCAP(CAP_HIGH_2GHZ_CHAN, cap->high2GhzChan);
+	GETCAP(CAP_TWICE_ANTENNAGAIN_5G, cap->twiceAntennaGain5G);
+	GETCAP(CAP_TWICE_ANTENNAGAIN_2G, cap->twiceAntennaGain2G);
+
+	GETCAP(CAP_CIPHER_AES_CCM, cap->supportCipherAES_CCM);
+	GETCAP(CAP_CIPHER_TKIP, cap->supportCipherTKIP);
+	GETCAP(CAP_MIC_TKIP, cap->supportMicTKIP);
+
+	cap->supportCipherWEP = 1;	/* NB: always available */
+
+	return (0);
+}
+
+static int
+uath_get_devstatus(struct uath_softc *sc, uint8_t macaddr[IEEE80211_ADDR_LEN])
+{
+	int error;
+
+	/* retrieve MAC address */
+	error = uath_get_status(sc, ST_MAC_ADDR, macaddr, IEEE80211_ADDR_LEN);
+	if (error != 0) {
+		device_printf(sc->sc_dev, "could not read MAC address\n");
+		return (error);
+	}
+
+	error = uath_get_status(sc, ST_SERIAL_NUMBER,
+	    &sc->sc_serial[0], sizeof(sc->sc_serial));
+	if (error != 0) {
+		device_printf(sc->sc_dev,
+		    "could not read device serial number\n");
+		return (error);
+	}
+	return (0);
+}
+
+static int
+uath_get_status(struct uath_softc *sc, uint32_t which, void *odata, int olen)
+{
+	int error;
+
+	which = htobe32(which);
+	error = uath_cmd_read(sc, WDCMSG_TARGET_GET_STATUS,
+	    &which, sizeof(which), odata, olen, UATH_CMD_FLAG_MAGIC);
+	if (error != 0)
+		device_printf(sc->sc_dev,
+		    "could not read EEPROM offset 0x%02x\n", be32toh(which));
+	return (error);
+}
+
+static void
+uath_free_data_list(struct uath_softc *sc, struct uath_data data[], int ndata,
+    int fillmbuf)
+{
+	int i;
+
+	for (i = 0; i < ndata; i++) {
+		struct uath_data *dp = &data[i];
+
+		if (fillmbuf == 1) {
+			if (dp->m != NULL) {
+				m_freem(dp->m);
+				dp->m = NULL;
+				dp->buf = NULL;
+			}
+		} else {
+			if (dp->buf != NULL) {
+				free(dp->buf, M_USBDEV);
+				dp->buf = NULL;
+			}
+		}
+#ifdef UATH_DEBUG
+		if (dp->ni != NULL)
+			device_printf(sc->sc_dev, "Node isn't NULL\n");
+#endif
+	}
+}
+
+static int
+uath_alloc_data_list(struct uath_softc *sc, struct uath_data data[],
+	int ndata, int maxsz, int fillmbuf)
+{
+	int i, error;
+
+	for (i = 0; i < ndata; i++) {
+		struct uath_data *dp = &data[i];
+
+		dp->sc = sc;
+		if (fillmbuf) {
+			/* XXX check maxsz */
+			dp->m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
+			if (dp->m == NULL) {
+				device_printf(sc->sc_dev,
+				    "could not allocate rx mbuf\n");
+				error = ENOMEM;
+				goto fail;
+			}
+			dp->buf = mtod(dp->m, uint8_t *);
+		} else {
+			dp->m = NULL;
+			dp->buf = malloc(maxsz, M_USBDEV, M_NOWAIT);
+			if (dp->buf == NULL) {
+				device_printf(sc->sc_dev,
+				    "could not allocate buffer\n");
+				error = ENOMEM;
+				goto fail;
+			}
+		}
+		dp->ni = NULL;
+	}
+
+	return (0);
+
+fail:	uath_free_data_list(sc, data, ndata, fillmbuf);
+	return (error);
+}
+
+static int
+uath_alloc_rx_data_list(struct uath_softc *sc)
+{
+	int error, i;
+
+	/* XXX is it enough to store the RX packet with MCLBYTES bytes?  */
+	error = uath_alloc_data_list(sc,
+	    sc->sc_rx, UATH_RX_DATA_LIST_COUNT, MCLBYTES,
+	    1 /* setup mbufs */);
+	if (error != 0)
+		return (error);
+
+	STAILQ_INIT(&sc->sc_rx_active);
+	STAILQ_INIT(&sc->sc_rx_inactive);
+
+	for (i = 0; i < UATH_RX_DATA_LIST_COUNT; i++) {
+		STAILQ_INSERT_HEAD(&sc->sc_rx_inactive, &sc->sc_rx[i],
+		    next);
+		UATH_STAT_INC(sc, st_rx_inactive);
+	}
+
+	return (0);
+}
+
+static int
+uath_alloc_tx_data_list(struct uath_softc *sc)
+{
+	int error, i;
+
+	error = uath_alloc_data_list(sc,
+	    sc->sc_tx, UATH_TX_DATA_LIST_COUNT, UATH_MAX_TXBUFSZ,
+	    0 /* no mbufs */);
+	if (error != 0)
+		return (error);
+
+	STAILQ_INIT(&sc->sc_tx_active);
+	STAILQ_INIT(&sc->sc_tx_inactive);
+	STAILQ_INIT(&sc->sc_tx_pending);
+
+	for (i = 0; i < UATH_TX_DATA_LIST_COUNT; i++) {
+		STAILQ_INSERT_HEAD(&sc->sc_tx_inactive, &sc->sc_tx[i],
+		    next);
+		UATH_STAT_INC(sc, st_tx_inactive);
+	}
+
+	return (0);
+}
+
+static void
+uath_free_rx_data_list(struct uath_softc *sc)
+{
+
+	STAILQ_INIT(&sc->sc_rx_active);
+	STAILQ_INIT(&sc->sc_rx_inactive);
+
+	uath_free_data_list(sc, sc->sc_rx, UATH_RX_DATA_LIST_COUNT,
+	    1 /* free mbufs */);
+}
+
+static void
+uath_free_tx_data_list(struct uath_softc *sc)
+{
+
+	STAILQ_INIT(&sc->sc_tx_active);
+	STAILQ_INIT(&sc->sc_tx_inactive);
+	STAILQ_INIT(&sc->sc_tx_pending);
+
+	uath_free_data_list(sc, sc->sc_tx, UATH_TX_DATA_LIST_COUNT,
+	    0 /* no mbufs */);
+}
+
+static struct ieee80211vap *
+uath_vap_create(struct ieee80211com *ic,
+	const char name[IFNAMSIZ], int unit, int opmode, int flags,
+	const uint8_t bssid[IEEE80211_ADDR_LEN],
+	const uint8_t mac[IEEE80211_ADDR_LEN])
+{
+	struct uath_vap *uvp;
+	struct ieee80211vap *vap;
+
+	if (!TAILQ_EMPTY(&ic->ic_vaps))		/* only one at a time */
+		return (NULL);
+	uvp = (struct uath_vap *) malloc(sizeof(struct uath_vap),
+	    M_80211_VAP, M_NOWAIT | M_ZERO);
+	if (uvp == NULL)
+		return (NULL);
+	vap = &uvp->vap;
+	/* enable s/w bmiss handling for sta mode */
+	ieee80211_vap_setup(ic, vap, name, unit, opmode,
+	    flags | IEEE80211_CLONE_NOBEACONS, bssid, mac);
+
+	/* override state transition machine */
+	uvp->newstate = vap->iv_newstate;
+	vap->iv_newstate = uath_newstate;
+
+	/* complete setup */
+	ieee80211_vap_attach(vap, ieee80211_media_change,
+	    ieee80211_media_status);
+	ic->ic_opmode = opmode;
+	return (vap);
+}
+
+static void
+uath_vap_delete(struct ieee80211vap *vap)
+{
+	struct uath_vap *uvp = UATH_VAP(vap);
+
+	ieee80211_vap_detach(vap);
+	free(uvp, M_80211_VAP);
+}
+
+static int
+uath_init_locked(void *arg)
+{
+	struct uath_softc *sc = arg;
+	struct ifnet *ifp = sc->sc_ifp;
+	struct ieee80211com *ic = ifp->if_l2com;
+	uint32_t val;
+	int error;
+
+	UATH_ASSERT_LOCKED(sc);
+
+	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+		uath_stop_locked(ifp);
+
+	/* reset variables */
+	sc->sc_intrx_nextnum = sc->sc_msgid = 0;
+
+	val = htobe32(0);
+	uath_cmd_write(sc, WDCMSG_BIND, &val, sizeof val, 0);
+
+	/* set MAC address */
+	uath_config_multi(sc, CFG_MAC_ADDR, IF_LLADDR(ifp), IEEE80211_ADDR_LEN);
+
+	/* XXX honor net80211 state */
+	uath_config(sc, CFG_RATE_CONTROL_ENABLE, 0x00000001);
+	uath_config(sc, CFG_DIVERSITY_CTL, 0x00000001);
+	uath_config(sc, CFG_ABOLT, 0x0000003f);
+	uath_config(sc, CFG_WME_ENABLED, 0x00000001);
+
+	uath_config(sc, CFG_SERVICE_TYPE, 1);
+	uath_config(sc, CFG_TP_SCALE, 0x00000000);
+	uath_config(sc, CFG_TPC_HALF_DBM5, 0x0000003c);
+	uath_config(sc, CFG_TPC_HALF_DBM2, 0x0000003c);
+	uath_config(sc, CFG_OVERRD_TX_POWER, 0x00000000);
+	uath_config(sc, CFG_GMODE_PROTECTION, 0x00000000);
+	uath_config(sc, CFG_GMODE_PROTECT_RATE_INDEX, 0x00000003);
+	uath_config(sc, CFG_PROTECTION_TYPE, 0x00000000);
+	uath_config(sc, CFG_MODE_CTS, 0x00000002);
+
+	error = uath_cmd_read(sc, WDCMSG_TARGET_START, NULL, 0,
+	    &val, sizeof(val), UATH_CMD_FLAG_MAGIC);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "could not start target, error %d\n", error);
+		goto fail;
+	}
+	DPRINTF(sc, UATH_DEBUG_INIT, "%s returns handle: 0x%x\n",
+	    uath_codename(WDCMSG_TARGET_START), be32toh(val));
+
+	/* set default channel */
+	error = uath_switch_channel(sc, ic->ic_curchan);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "could not switch channel, error %d\n", error);
+		goto fail;
+	}
+
+	val = htobe32(TARGET_DEVICE_AWAKE);
+	uath_cmd_write(sc, WDCMSG_SET_PWR_MODE, &val, sizeof val, 0);
+	/* XXX? check */
+	uath_cmd_write(sc, WDCMSG_RESET_KEY_CACHE, NULL, 0, 0);
+
+	usb2_transfer_start(sc->sc_xfer[UATH_BULK_RX]);
+	/* enable Rx */
+	uath_set_rxfilter(sc, 0x0, UATH_FILTER_OP_INIT);
+	uath_set_rxfilter(sc,
+	    UATH_FILTER_RX_UCAST | UATH_FILTER_RX_MCAST |
+	    UATH_FILTER_RX_BCAST | UATH_FILTER_RX_BEACON,
+	    UATH_FILTER_OP_SET);
+
+	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+	ifp->if_drv_flags |= IFF_DRV_RUNNING;
+	sc->sc_flags |= UATH_FLAG_INITDONE;
+
+	callout_reset(&sc->watchdog_ch, hz, uath_watchdog, sc);
+
+	return (0);
+
+fail:
+	uath_stop_locked(ifp);
+	return (error);
+}
+
+static void
+uath_init(void *arg)
+{
+	struct uath_softc *sc = arg;
+
+	UATH_LOCK(sc);
+	(void)uath_init_locked(sc);
+	UATH_UNLOCK(sc);
+}
+
+static void
+uath_stop_locked(struct ifnet *ifp)
+{
+	struct uath_softc *sc = ifp->if_softc;
+
+	UATH_ASSERT_LOCKED(sc);
+
+	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
+	sc->sc_flags &= ~UATH_FLAG_INITDONE;
+
+	callout_stop(&sc->stat_ch);
+	callout_stop(&sc->watchdog_ch);
+	sc->sc_tx_timer = 0;
+	/* abort pending transmits  */
+	uath_abort_xfers(sc);
+	/* flush data & control requests into the target  */
+	(void)uath_flush(sc);
+	/* set a LED status to the disconnected.  */
+	uath_set_ledstate(sc, 0);
+	/* stop the target  */
+	uath_cmd_write(sc, WDCMSG_TARGET_STOP, NULL, 0, 0);
+}
+
+static void
+uath_stop(struct ifnet *ifp)
+{
+	struct uath_softc *sc = ifp->if_softc;
+
+	UATH_LOCK(sc);
+	uath_stop_locked(ifp);
+	UATH_UNLOCK(sc);
+}
+
+static int
+uath_config(struct uath_softc *sc, uint32_t reg, uint32_t val)
+{
+	struct uath_write_mac write;
+	int error;
+
+	write.reg = htobe32(reg);
+	write.len = htobe32(0);	/* 0 = single write */
+	*(uint32_t *)write.data = htobe32(val);
+
+	error = uath_cmd_write(sc, WDCMSG_TARGET_SET_CONFIG, &write,
+	    3 * sizeof (uint32_t), 0);
+	if (error != 0) {
+		device_printf(sc->sc_dev, "could not write register 0x%02x\n",
+		    reg);
+	}
+	return (error);
+}
+
+static int
+uath_config_multi(struct uath_softc *sc, uint32_t reg, const void *data,
+    int len)
+{
+	struct uath_write_mac write;
+	int error;
+
+	write.reg = htobe32(reg);
+	write.len = htobe32(len);
+	bcopy(data, write.data, len);
+
+	/* properly handle the case where len is zero (reset) */
+	error = uath_cmd_write(sc, WDCMSG_TARGET_SET_CONFIG, &write,
+	    (len == 0) ? sizeof (uint32_t) : 2 * sizeof (uint32_t) + len, 0);
+	if (error != 0) {
+		device_printf(sc->sc_dev,
+		    "could not write %d bytes to register 0x%02x\n", len, reg);
+	}
+	return (error);
+}
+
+static int
+uath_switch_channel(struct uath_softc *sc, struct ieee80211_channel *c)
+{
+	int error;
+
+	UATH_ASSERT_LOCKED(sc);
+
+	/* set radio frequency */
+	error = uath_set_chan(sc, c);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "could not set channel, error %d\n", error);
+		goto failed;
+	}
+	/* reset Tx rings */
+	error = uath_reset_tx_queues(sc);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "could not reset Tx queues, error %d\n", error);
+		goto failed;
+	}
+	/* set Tx rings WME properties */
+	error = uath_wme_init(sc);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "could not init Tx queues, error %d\n", error);
+		goto failed;
+	}
+	error = uath_set_ledstate(sc, 0);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "could not set led state, error %d\n", error);
+		goto failed;
+	}
+	error = uath_flush(sc);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "could not flush pipes, error %d\n", error);
+		goto failed;
+	}
+failed:
+	return (error);
+}
+
+static int
+uath_set_rxfilter(struct uath_softc *sc, uint32_t bits, uint32_t op)
+{
+	struct uath_cmd_rx_filter rxfilter;
+
+	rxfilter.bits = htobe32(bits);
+	rxfilter.op = htobe32(op);
+
+	DPRINTF(sc, UATH_DEBUG_RECV | UATH_DEBUG_RECV_ALL,
+	    "setting Rx filter=0x%x flags=0x%x\n", bits, op);
+	return uath_cmd_write(sc, WDCMSG_RX_FILTER, &rxfilter,
+	    sizeof rxfilter, 0);
+}
+
+static void
+uath_watchdog(void *arg)
+{
+	struct uath_softc *sc = arg;
+	struct ifnet *ifp = sc->sc_ifp;
+
+	if (sc->sc_tx_timer > 0) {
+		if (--sc->sc_tx_timer == 0) {
+			device_printf(sc->sc_dev, "device timeout\n");
+			/*uath_init(ifp); XXX needs a process context! */
+			ifp->if_oerrors++;
+			return;
+		}
+		callout_reset(&sc->watchdog_ch, hz, uath_watchdog, sc);
+	}
+}
+
+static void
+uath_abort_xfers(struct uath_softc *sc)
+{
+	int i;
+
+	UATH_ASSERT_LOCKED(sc);
+	/* abort any pending transfers */
+	for (i = 0; i < UATH_N_XFERS; i++)
+		usb2_transfer_stop(sc->sc_xfer[i]);
+}
+
+static int
+uath_flush(struct uath_softc *sc)
+{
+	int error;
+
+	error = uath_dataflush(sc);
+	if (error != 0)
+		goto failed;
+
+	error = uath_cmdflush(sc);
+	if (error != 0)
+		goto failed;
+
+failed:
+	return (error);
+}
+
+static int
+uath_cmdflush(struct uath_softc *sc)
+{
+
+	return uath_cmd_write(sc, WDCMSG_FLUSH, NULL, 0, 0);
+}
+
+static int
+uath_dataflush(struct uath_softc *sc)
+{
+	struct uath_data *data;
+	struct uath_chunk *chunk;
+	struct uath_tx_desc *desc;
+
+	UATH_ASSERT_LOCKED(sc);
+
+	data = uath_getbuf(sc);
+	if (data == NULL)
+		return (ENOBUFS);
+	data->buflen = sizeof(struct uath_chunk) + sizeof(struct uath_tx_desc);
+	data->m = NULL;
+	data->ni = NULL;
+	chunk = (struct uath_chunk *)data->buf;
+	desc = (struct uath_tx_desc *)(chunk + 1);
+
+	/* one chunk only */
+	chunk->seqnum = 0;
+	chunk->flags = UATH_CFLAGS_FINAL;
+	chunk->length = htobe16(sizeof (struct uath_tx_desc));
+
+	bzero(desc, sizeof(struct uath_tx_desc));
+	desc->msglen = htobe32(sizeof(struct uath_tx_desc));
+	desc->msgid  = (sc->sc_msgid++) + 1; /* don't care about endianness */
+	desc->type   = htobe32(WDCMSG_FLUSH);
+	desc->txqid  = htobe32(0);
+	desc->connid = htobe32(0);
+	desc->flags  = htobe32(0);
+
+#ifdef UATH_DEBUG
+	if (sc->sc_debug & UATH_DEBUG_CMDS) {
+		DPRINTF(sc, UATH_DEBUG_RESET, "send flush ix %d\n",
+		    desc->msgid);
+		if (sc->sc_debug & UATH_DEBUG_CMDS_DUMP)
+			uath_dump_cmd(data->buf, data->buflen, '+');
+	}
+#endif
+
+	STAILQ_INSERT_TAIL(&sc->sc_tx_pending, data, next);
+	UATH_STAT_INC(sc, st_tx_pending);
+	sc->sc_tx_timer = 5;
+	usb2_transfer_start(sc->sc_xfer[UATH_BULK_TX]);
+
+	return (0);
+}
+
+static struct uath_data *
+_uath_getbuf(struct uath_softc *sc)
+{
+	struct uath_data *bf;
+
+	bf = STAILQ_FIRST(&sc->sc_tx_inactive);
+	if (bf != NULL) {
+		STAILQ_REMOVE_HEAD(&sc->sc_tx_inactive, next);
+		UATH_STAT_DEC(sc, st_tx_inactive);
+	} else
+		bf = NULL;
+	if (bf == NULL)
+		DPRINTF(sc, UATH_DEBUG_XMIT, "%s: %s\n", __func__,
+		    "out of xmit buffers");
+	return (bf);
+}
+
+static struct uath_data *
+uath_getbuf(struct uath_softc *sc)
+{
+	struct uath_data *bf;
+
+	UATH_ASSERT_LOCKED(sc);
+
+	bf = _uath_getbuf(sc);
+	if (bf == NULL) {
+		struct ifnet *ifp = sc->sc_ifp;
+
+		DPRINTF(sc, UATH_DEBUG_XMIT, "%s: stop queue\n", __func__);
+		ifp->if_drv_flags |= IFF_DRV_OACTIVE;
+	}
+	return (bf);
+}
+
+static int
+uath_set_ledstate(struct uath_softc *sc, int connected)
+{
+
+	DPRINTF(sc, UATH_DEBUG_LED,
+	    "set led state %sconnected\n", connected ? "" : "!");
+	connected = htobe32(connected);
+	return uath_cmd_write(sc, WDCMSG_SET_LED_STATE,
+	     &connected, sizeof connected, 0);
+}
+
+static int
+uath_set_chan(struct uath_softc *sc, struct ieee80211_channel *c)
+{
+#ifdef UATH_DEBUG
+	struct ifnet *ifp = sc->sc_ifp;
+	struct ieee80211com *ic = ifp->if_l2com;
+#endif
+	struct uath_cmd_reset reset;
+
+	bzero(&reset, sizeof reset);
+	if (IEEE80211_IS_CHAN_2GHZ(c))
+		reset.flags |= htobe32(UATH_CHAN_2GHZ);
+	if (IEEE80211_IS_CHAN_5GHZ(c))
+		reset.flags |= htobe32(UATH_CHAN_5GHZ);
+	/* NB: 11g =>'s 11b so don't specify both OFDM and CCK */
+	if (IEEE80211_IS_CHAN_G(c))
+		reset.flags |= htobe32(UATH_CHAN_OFDM);
+	else if (IEEE80211_IS_CHAN_B(c))
+		reset.flags |= htobe32(UATH_CHAN_CCK);
+	/* turbo can be used in either 2GHz or 5GHz */
+	if (c->ic_flags & IEEE80211_CHAN_TURBO)
+		reset.flags |= htobe32(UATH_CHAN_TURBO);
+	reset.freq = htobe32(c->ic_freq);
+	reset.maxrdpower = htobe32(50);	/* XXX */
+	reset.channelchange = htobe32(1);
+	reset.keeprccontent = htobe32(0);
+
+	DPRINTF(sc, UATH_DEBUG_CHANNEL, "set channel %d, flags 0x%x freq %u\n",
+	    ieee80211_chan2ieee(ic, c),
+	    be32toh(reset.flags), be32toh(reset.freq));
+	return uath_cmd_write(sc, WDCMSG_RESET, &reset, sizeof reset, 0);
+}
+
+static int
+uath_reset_tx_queues(struct uath_softc *sc)
+{
+	int ac, error;
+
+	DPRINTF(sc, UATH_DEBUG_RESET, "%s: reset Tx queues\n", __func__);
+	for (ac = 0; ac < 4; ac++) {
+		const uint32_t qid = htobe32(ac);
+
+		error = uath_cmd_write(sc, WDCMSG_RELEASE_TX_QUEUE, &qid,
+		    sizeof qid, 0);
+		if (error != 0)
+			break;
+	}
+	return (error);
+}
+
+static int
+uath_wme_init(struct uath_softc *sc)
+{
+	/* XXX get from net80211 */
+	static const struct uath_wme_settings uath_wme_11g[4] = {
+		{ 7, 4, 10,  0, 0 },	/* Background */
+		{ 3, 4, 10,  0, 0 },	/* Best-Effort */
+		{ 3, 3,  4, 26, 0 },	/* Video */
+		{ 2, 2,  3, 47, 0 }	/* Voice */
+	};
+	struct uath_cmd_txq_setup qinfo;
+	int ac, error;
+
+	DPRINTF(sc, UATH_DEBUG_WME, "%s: setup Tx queues\n", __func__);
+	for (ac = 0; ac < 4; ac++) {
+		qinfo.qid		= htobe32(ac);
+		qinfo.len		= htobe32(sizeof(qinfo.attr));
+		qinfo.attr.priority	= htobe32(ac);	/* XXX */
+		qinfo.attr.aifs		= htobe32(uath_wme_11g[ac].aifsn);
+		qinfo.attr.logcwmin	= htobe32(uath_wme_11g[ac].logcwmin);
+		qinfo.attr.logcwmax	= htobe32(uath_wme_11g[ac].logcwmax);
+		qinfo.attr.bursttime	= htobe32(UATH_TXOP_TO_US(
+					    uath_wme_11g[ac].txop));
+		qinfo.attr.mode		= htobe32(uath_wme_11g[ac].acm);/*XXX? */
+		qinfo.attr.qflags	= htobe32(1);	/* XXX? */
+
+		error = uath_cmd_write(sc, WDCMSG_SETUP_TX_QUEUE, &qinfo,
+		    sizeof qinfo, 0);
+		if (error != 0)
+			break;
+	}
+	return (error);
+}
+
+static int
+uath_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
+{
+	struct ieee80211com *ic = ifp->if_l2com;
+	struct ifreq *ifr = (struct ifreq *) data;
+	int error = 0, startall = 0;
+
+	switch (cmd) {
+	case SIOCSIFFLAGS:
+		if (ifp->if_flags & IFF_UP) {
+			if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
+				uath_init(ifp->if_softc);
+				startall = 1;
+			}
+		} else {
+			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+				uath_stop(ifp);
+		}
+		if (startall)
+			ieee80211_start_all(ic);
+		break;
+	case SIOCGIFMEDIA:
+		error = ifmedia_ioctl(ifp, ifr, &ic->ic_media, cmd);
+		break;
+	case SIOCGIFADDR:
+		error = ether_ioctl(ifp, cmd, data);
+		break;
+	default:
+		error = EINVAL;
+		break;
+	}
+
+	return (error);
+}
+
+static int
+uath_tx_start(struct uath_softc *sc, struct mbuf *m0, struct ieee80211_node *ni,
+    struct uath_data *data)
+{
+	struct ifnet *ifp = sc->sc_ifp;
+	struct ieee80211com *ic = ifp->if_l2com;
+	struct uath_chunk *chunk;
+	struct uath_tx_desc *desc;
+	const struct ieee80211_frame *wh;
+	struct ieee80211_key *k;
+	int framelen, msglen;
+
+	UATH_ASSERT_LOCKED(sc);
+
+	data->ni = ni;
+	data->m = m0;
+	chunk = (struct uath_chunk *)data->buf;
+	desc = (struct uath_tx_desc *)(chunk + 1);
+
+	if (bpf_peers_present(ifp->if_bpf)) {
+		struct uath_tx_radiotap_header *tap = &sc->sc_txtap;
+
+		tap->wt_flags = 0;
+		if (m0->m_flags & M_FRAG)
+			tap->wt_flags |= IEEE80211_RADIOTAP_F_FRAG;
+		tap->wt_chan_freq = htole16(ic->ic_curchan->ic_freq);
+		tap->wt_chan_flags = htole16(ic->ic_curchan->ic_flags);
+
+		bpf_mtap2(ifp->if_bpf, tap, sc->sc_txtap_len, m0);
+	}
+
+	wh = mtod(m0, struct ieee80211_frame *);
+	if (wh->i_fc[1] & IEEE80211_FC1_WEP) {
+		k = ieee80211_crypto_encap(ni, m0);
+		if (k == NULL) {
+			m_freem(m0);
+			return (ENOBUFS);
+		}
+
+		/* packet header may have moved, reset our local pointer */
+		wh = mtod(m0, struct ieee80211_frame *);
+	}
+	m_copydata(m0, 0, m0->m_pkthdr.len, (uint8_t *)(desc + 1));
+
+	framelen = m0->m_pkthdr.len + IEEE80211_CRC_LEN;
+	msglen = framelen + sizeof (struct uath_tx_desc);
+	data->buflen = msglen + sizeof (struct uath_chunk);
+
+	/* one chunk only for now */
+	chunk->seqnum = sc->sc_seqnum++;
+	chunk->flags = (m0->m_flags & M_FRAG) ? 0 : UATH_CFLAGS_FINAL;
+	if (m0->m_flags & M_LASTFRAG)
+		chunk->flags |= UATH_CFLAGS_FINAL;
+	chunk->flags = UATH_CFLAGS_FINAL;
+	chunk->length = htobe16(msglen);
+
+	/* fill Tx descriptor */
+	desc->msglen = htobe32(msglen);
+	/* NB: to get UATH_TX_NOTIFY reply, `msgid' must be larger than 0  */
+	desc->msgid  = (sc->sc_msgid++) + 1; /* don't care about endianness */
+	desc->type   = htobe32(WDCMSG_SEND);
+	switch (wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) {
+	case IEEE80211_FC0_TYPE_CTL:
+	case IEEE80211_FC0_TYPE_MGT:
+		/* NB: force all management frames to highest queue */
+		if (ni->ni_flags & IEEE80211_NODE_QOS) {
+			/* NB: force all management frames to highest queue */
+			desc->txqid = htobe32(WME_AC_VO | UATH_TXQID_MINRATE);
+		} else
+			desc->txqid = htobe32(WME_AC_BE | UATH_TXQID_MINRATE);
+		break;
+	case IEEE80211_FC0_TYPE_DATA:
+		/* XXX multicast frames should honor mcastrate */
+		desc->txqid = htobe32(M_WME_GETAC(m0));
+		break;
+	default:
+		device_printf(sc->sc_dev, "bogus frame type 0x%x (%s)\n",
+			wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK, __func__);
+		m_freem(m0);
+		return (EIO);
+	}
+	if (sc->sc_state == IEEE80211_S_AUTH ||
+	    sc->sc_state == IEEE80211_S_ASSOC ||
+	    sc->sc_state == IEEE80211_S_RUN)
+		desc->connid = htobe32(UATH_ID_BSS);
+	else
+		desc->connid = htobe32(UATH_ID_INVALID);
+	desc->flags  = htobe32(0 /* no UATH_TX_NOTIFY */);
+	desc->buflen = htobe32(m0->m_pkthdr.len);
+
+#ifdef UATH_DEBUG
+	DPRINTF(sc, UATH_DEBUG_XMIT,
+	    "send frame ix %u framelen %d msglen %d connid 0x%x txqid 0x%x\n",
+	    desc->msgid, framelen, msglen, be32toh(desc->connid),
+	    be32toh(desc->txqid));
+	if (sc->sc_debug & UATH_DEBUG_XMIT_DUMP)
+		uath_dump_cmd(data->buf, data->buflen, '+');
+#endif
+
+	STAILQ_INSERT_TAIL(&sc->sc_tx_pending, data, next);
+	UATH_STAT_INC(sc, st_tx_pending);
+	usb2_transfer_start(sc->sc_xfer[UATH_BULK_TX]);
+
+	return (0);
+}
+
+/*
+ * Cleanup driver resources when we run out of buffers while processing
+ * fragments; return the tx buffers allocated and drop node references.
+ */
+static void
+uath_txfrag_cleanup(struct uath_softc *sc,
+    uath_datahead *frags, struct ieee80211_node *ni)
+{
+	struct uath_data *bf, *next;
+
+	UATH_ASSERT_LOCKED(sc);
+
+	STAILQ_FOREACH_SAFE(bf, frags, next, next) {
+		/* NB: bf assumed clean */
+		STAILQ_REMOVE_HEAD(frags, next);
+		STAILQ_INSERT_HEAD(&sc->sc_tx_inactive, bf, next);
+		UATH_STAT_INC(sc, st_tx_inactive);
+		ieee80211_node_decref(ni);
+	}
+}
+
+/*
+ * Setup xmit of a fragmented frame.  Allocate a buffer for each frag and bump
+ * the node reference count to reflect the held reference to be setup by
+ * uath_tx_start.
+ */
+static int
+uath_txfrag_setup(struct uath_softc *sc, uath_datahead *frags,
+    struct mbuf *m0, struct ieee80211_node *ni)
+{
+	struct mbuf *m;
+	struct uath_data *bf;
+
+	UATH_ASSERT_LOCKED(sc);
+	for (m = m0->m_nextpkt; m != NULL; m = m->m_nextpkt) {
+		bf = uath_getbuf(sc);
+		if (bf == NULL) {       /* out of buffers, cleanup */
+			uath_txfrag_cleanup(sc, frags, ni);
+			break;
+		}
+		ieee80211_node_incref(ni);
+		STAILQ_INSERT_TAIL(frags, bf, next);
+	}
+
+	return !STAILQ_EMPTY(frags);
+}
+
+/*
+ * Reclaim mbuf resources.  For fragmented frames we need to claim each frag
+ * chained with m_nextpkt.
+ */
+static void
+uath_freetx(struct mbuf *m)
+{
+	struct mbuf *next;
+
+	do {
+		next = m->m_nextpkt;
+		m->m_nextpkt = NULL;
+		m_freem(m);
+	} while ((m = next) != NULL);
+}
+
+static void
+uath_start(struct ifnet *ifp)
+{
+	struct uath_data *bf;
+	struct uath_softc *sc = ifp->if_softc;
+	struct ieee80211_node *ni;
+	struct mbuf *m, *next;
+	uath_datahead frags;
+
+	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
+	    (sc->sc_flags & UATH_FLAG_INVALID))
+		return;
+
+	UATH_LOCK(sc);
+	for (;;) {
+		bf = uath_getbuf(sc);
+		if (bf == NULL)
+			break;
+
+		IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
+		if (m == NULL) {
+			STAILQ_INSERT_HEAD(&sc->sc_tx_inactive, bf, next);
+			UATH_STAT_INC(sc, st_tx_inactive);
+			break;
+		}
+		ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
+		m->m_pkthdr.rcvif = NULL;
+
+		/*
+		 * Check for fragmentation.  If this frame has been broken up
+		 * verify we have enough buffers to send all the fragments
+		 * so all go out or none...
+		 */
+		STAILQ_INIT(&frags);
+		if ((m->m_flags & M_FRAG) && 
+		    !uath_txfrag_setup(sc, &frags, m, ni)) {
+			DPRINTF(sc, UATH_DEBUG_XMIT,
+			    "%s: out of txfrag buffers\n", __func__);
+			uath_freetx(m);
+			goto bad;
+		}
+		sc->sc_seqnum = 0;
+	nextfrag:
+		/*
+		 * Pass the frame to the h/w for transmission.
+		 * Fragmented frames have each frag chained together
+		 * with m_nextpkt.  We know there are sufficient uath_data's
+		 * to send all the frags because of work done by
+		 * uath_txfrag_setup.
+		 */
+		next = m->m_nextpkt;
+		if (uath_tx_start(sc, m, ni, bf) != 0) {
+	bad:
+			ifp->if_oerrors++;
+	reclaim:
+			STAILQ_INSERT_HEAD(&sc->sc_tx_inactive, bf, next);
+			UATH_STAT_INC(sc, st_tx_inactive);
+			uath_txfrag_cleanup(sc, &frags, ni);
+			ieee80211_free_node(ni);
+			continue;
+		}
+
+		if (next != NULL) {
+			/*
+			 * Beware of state changing between frags.
+			 XXX check sta power-save state?
+			*/
+			if (ni->ni_vap->iv_state != IEEE80211_S_RUN) {
+				DPRINTF(sc, UATH_DEBUG_XMIT,
+				    "%s: flush fragmented packet, state %s\n",
+				    __func__,
+				    ieee80211_state_name[ni->ni_vap->iv_state]);
+				uath_freetx(next);
+				goto reclaim;
+			}
+			m = next;
+			bf = STAILQ_FIRST(&frags);
+			KASSERT(bf != NULL, ("no buf for txfrag"));
+			STAILQ_REMOVE_HEAD(&frags, next);
+			goto nextfrag;
+		}
+
+		sc->sc_tx_timer = 5;
+	}
+	UATH_UNLOCK(sc);
+}
+
+static int
+uath_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
+    const struct ieee80211_bpf_params *params)
+{
+	struct ieee80211com *ic = ni->ni_ic;
+	struct ifnet *ifp = ic->ic_ifp;
+	struct uath_data *bf;
+	struct uath_softc *sc = ifp->if_softc;
+
+	/* prevent management frames from being sent if we're not ready */
+	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
+		m_freem(m);
+		ieee80211_free_node(ni);
+		return (ENETDOWN);
+	}
+
+	UATH_LOCK(sc);
+	/* grab a TX buffer  */
+	bf = uath_getbuf(sc);
+	if (bf == NULL) {
+		ieee80211_free_node(ni);
+		m_freem(m);
+		UATH_UNLOCK(sc);
+		return (ENOBUFS);
+	}
+
+	sc->sc_seqnum = 0;
+	if (uath_tx_start(sc, m, ni, bf) != 0) {
+		ieee80211_free_node(ni);
+		ifp->if_oerrors++;
+		STAILQ_INSERT_HEAD(&sc->sc_tx_inactive, bf, next);
+		UATH_STAT_INC(sc, st_tx_inactive);
+		UATH_UNLOCK(sc);
+		return (EIO);
+	}
+	UATH_UNLOCK(sc);
+
+	sc->sc_tx_timer = 5;
+	return (0);
+}
+
+static void
+uath_scan_start(struct ieee80211com *ic)
+{
+	/* do nothing  */
+}
+
+static void
+uath_scan_end(struct ieee80211com *ic)
+{
+	/* do nothing  */
+}
+
+static void
+uath_set_channel(struct ieee80211com *ic)
+{
+	struct ifnet *ifp = ic->ic_ifp;
+	struct uath_softc *sc = ifp->if_softc;
+
+	UATH_LOCK(sc);
+	(void)uath_switch_channel(sc, ic->ic_curchan);
+	UATH_UNLOCK(sc);
+}
+
+static int
+uath_set_rxmulti_filter(struct uath_softc *sc)
+{
+
+	return (0);
+}
+static void
+uath_update_mcast(struct ifnet *ifp)
+{
+	struct uath_softc *sc = ifp->if_softc;
+
+	/*
+	 * this is for avoiding the race condition when we're try to
+	 * connect to the AP with WPA.
+	 */
+	if (!(sc->sc_flags & UATH_FLAG_INITDONE))
+		return;
+	(void)uath_set_rxmulti_filter(sc);
+}
+
+static void
+uath_update_promisc(struct ifnet *ifp)
+{
+	struct uath_softc *sc = ifp->if_softc;
+
+	if (!(sc->sc_flags & UATH_FLAG_INITDONE))
+		return;
+	uath_set_rxfilter(sc,
+	    UATH_FILTER_RX_UCAST | UATH_FILTER_RX_MCAST |
+	    UATH_FILTER_RX_BCAST | UATH_FILTER_RX_BEACON |
+	    UATH_FILTER_RX_PROM, UATH_FILTER_OP_SET);
+}
+
+static int
+uath_create_connection(struct uath_softc *sc, uint32_t connid)
+{
+	const struct ieee80211_rateset *rs;
+	struct ieee80211com *ic = sc->sc_ifp->if_l2com;
+	struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
+	struct ieee80211_node *ni = vap->iv_bss;
+	struct uath_cmd_create_connection create;
+
+	bzero(&create, sizeof create);
+	create.connid = htobe32(connid);
+	create.bssid = htobe32(0);
+	/* XXX packed or not?  */
+	create.size = htobe32(sizeof(struct uath_cmd_rateset));
+
+	rs = &ni->ni_rates;
+	create.connattr.rateset.length = rs->rs_nrates;
+	bcopy(rs->rs_rates, &create.connattr.rateset.set[0],
+	    rs->rs_nrates);
+
+	/* XXX turbo */
+	if (IEEE80211_IS_CHAN_A(ni->ni_chan))
+		create.connattr.wlanmode = htobe32(WLAN_MODE_11a);
+	else if (IEEE80211_IS_CHAN_ANYG(ni->ni_chan))
+		create.connattr.wlanmode = htobe32(WLAN_MODE_11g);
+	else
+		create.connattr.wlanmode = htobe32(WLAN_MODE_11b);
+
+	return uath_cmd_write(sc, WDCMSG_CREATE_CONNECTION, &create,
+	    sizeof create, 0);
+}
+
+static int
+uath_set_rates(struct uath_softc *sc, const struct ieee80211_rateset *rs)
+{
+	struct uath_cmd_rates rates;
+
+	bzero(&rates, sizeof rates);
+	rates.connid = htobe32(UATH_ID_BSS);		/* XXX */
+	rates.size   = htobe32(sizeof(struct uath_cmd_rateset));
+	/* XXX bounds check rs->rs_nrates */
+	rates.rateset.length = rs->rs_nrates;
+	bcopy(rs->rs_rates, &rates.rateset.set[0], rs->rs_nrates);
+
+	DPRINTF(sc, UATH_DEBUG_RATES,
+	    "setting supported rates nrates=%d\n", rs->rs_nrates);
+	return uath_cmd_write(sc, WDCMSG_SET_BASIC_RATE,
+	    &rates, sizeof rates, 0);
+}
+
+static int
+uath_write_associd(struct uath_softc *sc)
+{
+	struct ieee80211com *ic = sc->sc_ifp->if_l2com;
+	struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
+	struct ieee80211_node *ni = vap->iv_bss;
+	struct uath_cmd_set_associd associd;
+
+	bzero(&associd, sizeof associd);
+	associd.defaultrateix = htobe32(1);	/* XXX */
+	associd.associd = htobe32(ni->ni_associd);
+	associd.timoffset = htobe32(0x3b);	/* XXX */
+	IEEE80211_ADDR_COPY(associd.bssid, ni->ni_bssid);
+	return uath_cmd_write(sc, WDCMSG_WRITE_ASSOCID, &associd,
+	    sizeof associd, 0);
+}
+
+static int
+uath_set_ledsteady(struct uath_softc *sc, int lednum, int ledmode)
+{
+	struct uath_cmd_ledsteady led;
+
+	led.lednum = htobe32(lednum);
+	led.ledmode = htobe32(ledmode);
+
+	DPRINTF(sc, UATH_DEBUG_LED, "set %s led %s (steady)\n",
+	    (lednum == UATH_LED_LINK) ? "link" : "activity",
+	    ledmode ? "on" : "off");
+	return uath_cmd_write(sc, WDCMSG_SET_LED_STEADY, &led, sizeof led, 0);
+}
+
+static int
+uath_set_ledblink(struct uath_softc *sc, int lednum, int ledmode,
+	int blinkrate, int slowmode)
+{
+	struct uath_cmd_ledblink led;
+
+	led.lednum = htobe32(lednum);
+	led.ledmode = htobe32(ledmode);
+	led.blinkrate = htobe32(blinkrate);
+	led.slowmode = htobe32(slowmode);
+
+	DPRINTF(sc, UATH_DEBUG_LED, "set %s led %s (blink)\n",
+	    (lednum == UATH_LED_LINK) ? "link" : "activity",
+	    ledmode ? "on" : "off");
+	return uath_cmd_write(sc, WDCMSG_SET_LED_BLINK, &led, sizeof led, 0);
+}
+
+static int
+uath_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
+{
+	enum ieee80211_state ostate = vap->iv_state;
+	int error;
+	struct ieee80211_node *ni = vap->iv_bss;
+	struct ieee80211com *ic = vap->iv_ic;
+	struct uath_softc *sc = ic->ic_ifp->if_softc;
+	struct uath_vap *uvp = UATH_VAP(vap);
+
+	DPRINTF(sc, UATH_DEBUG_STATE,
+	    "%s: %s -> %s\n", __func__, ieee80211_state_name[vap->iv_state],
+	    ieee80211_state_name[nstate]);
+
+	UATH_LOCK(sc);
+
+	callout_stop(&sc->stat_ch);
+	callout_stop(&sc->watchdog_ch);
+	sc->sc_state = nstate;
+
+	switch (nstate) {
+	case IEEE80211_S_INIT:
+		if (ostate == IEEE80211_S_RUN) {
+			/* turn link and activity LEDs off */
+			uath_set_ledstate(sc, 0);
+		}
+		break;
+
+	case IEEE80211_S_SCAN:
+		break;
+
+	case IEEE80211_S_AUTH:
+		/* XXX good place?  set RTS threshold  */
+		uath_config(sc, CFG_USER_RTS_THRESHOLD, vap->iv_rtsthreshold);
+		/* XXX bad place  */
+		error = uath_set_keys(sc, vap);
+		if (error != 0) {
+			device_printf(sc->sc_dev,
+			    "could not set crypto keys, error %d\n", error);
+			break;
+		}
+		if (uath_switch_channel(sc, ni->ni_chan) != 0) {
+			device_printf(sc->sc_dev, "could not switch channel\n");
+			break;
+		}
+		if (uath_create_connection(sc, UATH_ID_BSS) != 0) {
+			device_printf(sc->sc_dev,
+			    "could not create connection\n");
+			break;
+		}
+		break;
+
+	case IEEE80211_S_ASSOC:
+		if (uath_set_rates(sc, &ni->ni_rates) != 0) {
+			device_printf(sc->sc_dev,
+			    "could not set negotiated rate set\n");
+			break;
+		}
+		break;
+
+	case IEEE80211_S_RUN:
+		/* XXX monitor mode doesn't be tested  */
+		if (ic->ic_opmode == IEEE80211_M_MONITOR) {
+			uath_set_ledstate(sc, 1);
+			break;
+		}
+
+		/*
+		 * Tx rate is controlled by firmware, report the maximum
+		 * negotiated rate in ifconfig output.
+		 */
+		ni->ni_txrate = ni->ni_rates.rs_rates[ni->ni_rates.rs_nrates-1];
+
+		if (uath_write_associd(sc) != 0) {
+			device_printf(sc->sc_dev,
+			    "could not write association id\n");
+			break;
+		}
+		/* turn link LED on */
+		uath_set_ledsteady(sc, UATH_LED_LINK, UATH_LED_ON);
+		/* make activity LED blink */
+		uath_set_ledblink(sc, UATH_LED_ACTIVITY, UATH_LED_ON, 1, 2);
+		/* set state to associated */
+		uath_set_ledstate(sc, 1);
+
+		/* start statistics timer */
+		callout_reset(&sc->stat_ch, hz, uath_stat, sc);
+		break;
+	default:
+		break;
+	}
+	UATH_UNLOCK(sc);
+
+	IEEE80211_LOCK(ic);
+	uvp->newstate(vap, nstate, arg);
+	if (vap->iv_newstate_cb != NULL)
+		vap->iv_newstate_cb(vap, nstate, arg);
+	IEEE80211_UNLOCK(ic);
+
+	return (0);
+}
+
+static int
+uath_set_key(struct uath_softc *sc, const struct ieee80211_key *wk,
+    int index)
+{
+#if 0
+	struct uath_cmd_crypto crypto;
+	int i;
+
+	bzero(&crypto, sizeof crypto);
+	crypto.keyidx = htobe32(index);
+	crypto.magic1 = htobe32(1);
+	crypto.size   = htobe32(368);
+	crypto.mask   = htobe32(0xffff);
+	crypto.flags  = htobe32(0x80000068);
+	if (index != UATH_DEFAULT_KEY)
+		crypto.flags |= htobe32(index << 16);
+	memset(crypto.magic2, 0xff, sizeof crypto.magic2);
+
+	/*
+	 * Each byte of the key must be XOR'ed with 10101010 before being
+	 * transmitted to the firmware.
+	 */
+	for (i = 0; i < wk->wk_keylen; i++)
+		crypto.key[i] = wk->wk_key[i] ^ 0xaa;
+
+	DPRINTF(sc, UATH_DEBUG_CRYPTO,
+	    "setting crypto key index=%d len=%d\n", index, wk->wk_keylen);
+	return uath_cmd_write(sc, WDCMSG_SET_KEY_CACHE_ENTRY, &crypto,
+	    sizeof crypto, 0);
+#else
+	/* XXX support H/W cryto  */
+	return (0);
+#endif
+}
+
+static int
+uath_set_keys(struct uath_softc *sc, struct ieee80211vap *vap)
+{
+	int i, error;
+
+	error = 0;
+	for (i = 0; i < IEEE80211_WEP_NKID; i++) {
+		const struct ieee80211_key *wk = &vap->iv_nw_keys[i];
+
+		if (wk->wk_flags & (IEEE80211_KEY_XMIT|IEEE80211_KEY_RECV)) {
+			error = uath_set_key(sc, wk, i);
+			if (error)
+				return (error);
+		}
+	}
+	if (vap->iv_def_txkey != IEEE80211_KEYIX_NONE) {
+		error = uath_set_key(sc, &vap->iv_nw_keys[vap->iv_def_txkey],
+			UATH_DEFAULT_KEY);
+	}
+	return (error);
+}
+
+#define	UATH_SYSCTL_STAT_ADD32(c, h, n, p, d)	\
+	    SYSCTL_ADD_UINT(c, h, OID_AUTO, n, CTLFLAG_RD, p, 0, d)
+
+static void
+uath_sysctl_node(struct uath_softc *sc)
+{
+	struct sysctl_ctx_list *ctx;
+	struct sysctl_oid_list *child;
+	struct sysctl_oid *tree;
+	struct uath_stat *stats;
+
+	stats = &sc->sc_stat;
+	ctx = device_get_sysctl_ctx(sc->sc_dev);
+	child = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->sc_dev));
+
+	tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "stats", CTLFLAG_RD,
+	    NULL, "UATH statistics");
+	child = SYSCTL_CHILDREN(tree);
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "badchunkseqnum",
+	    &stats->st_badchunkseqnum, "Bad chunk sequence numbers");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "invalidlen", &stats->st_invalidlen,
+	    "Invalid length");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "multichunk", &stats->st_multichunk,
+	    "Multi chunks");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "toobigrxpkt",
+	    &stats->st_toobigrxpkt, "Too big rx packets");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "stopinprogress",
+	    &stats->st_stopinprogress, "Stop in progress");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "crcerrs", &stats->st_crcerr,
+	    "CRC errors");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "phyerr", &stats->st_phyerr,
+	    "PHY errors");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "decrypt_crcerr",
+	    &stats->st_decrypt_crcerr, "Decryption CRC errors");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "decrypt_micerr",
+	    &stats->st_decrypt_micerr, "Decryption Misc errors");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "decomperr", &stats->st_decomperr,
+	    "Decomp errors");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "keyerr", &stats->st_keyerr,
+	    "Key errors");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "err", &stats->st_err,
+	    "Unknown errors");
+
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "cmd_active",
+	    &stats->st_cmd_active, "Active numbers in Command queue");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "cmd_inactive",
+	    &stats->st_cmd_inactive, "Inactive numbers in Command queue");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "cmd_pending",
+	    &stats->st_cmd_pending, "Pending numbers in Command queue");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "cmd_waiting",
+	    &stats->st_cmd_waiting, "Waiting numbers in Command queue");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "rx_active",
+	    &stats->st_rx_active, "Active numbers in RX queue");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "rx_inactive",
+	    &stats->st_rx_inactive, "Inactive numbers in RX queue");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "tx_active",
+	    &stats->st_tx_active, "Active numbers in TX queue");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "tx_inactive",
+	    &stats->st_tx_inactive, "Inactive numbers in TX queue");
+	UATH_SYSCTL_STAT_ADD32(ctx, child, "tx_pending",
+	    &stats->st_tx_pending, "Pending numbers in TX queue");
+}
+
+#undef UATH_SYSCTL_STAT_ADD32
+
+static void
+uath_cmdeof(struct uath_softc *sc, struct uath_cmd *cmd)
+{
+	struct uath_cmd_hdr *hdr;
+	int dlen;
+
+	hdr = (struct uath_cmd_hdr *)cmd->buf;
+	/* NB: msgid is passed thru w/o byte swapping */
+#ifdef UATH_DEBUG
+	if (sc->sc_debug & UATH_DEBUG_CMDS) {
+		int len = be32toh(hdr->len);
+		printf("%s: %s [ix %u] len %u status %u\n",
+		    __func__, uath_codename(be32toh(hdr->code)),
+		    hdr->msgid, len, be32toh(hdr->magic));
+		if (sc->sc_debug & UATH_DEBUG_CMDS_DUMP)
+			uath_dump_cmd(cmd->buf,
+			    len > UATH_MAX_CMDSZ ? sizeof(*hdr) : len, '-');
+	}
+#endif
+	hdr->code = be32toh(hdr->code);
+	hdr->len = be32toh(hdr->len);
+	hdr->magic = be32toh(hdr->magic);	/* target status on return */
+
+	switch (hdr->code & 0xff) {
+	/* reply to a read command */
+	default:
+		dlen = hdr->len - sizeof(*hdr);
+		DPRINTF(sc, UATH_DEBUG_RX_PROC | UATH_DEBUG_RECV_ALL,
+		    "%s: code %d data len %u\n",
+		    __func__, hdr->code & 0xff, dlen);
+		/*
+		 * The first response from the target after the
+		 * HOST_AVAILABLE has an invalid msgid so we must
+		 * treat it specially.
+		 */
+		if (hdr->msgid < UATH_CMD_LIST_COUNT) {
+			uint32_t *rp = (uint32_t *)(hdr+1);
+			u_int olen;
+
+			if (!(sizeof(*hdr) <= hdr->len &&
+			      hdr->len < UATH_MAX_CMDSZ)) {
+				device_printf(sc->sc_dev,
+				    "%s: invalid WDC msg length %u; "
+				    "msg ignored\n", __func__, hdr->len);
+				return;
+			}
+			/*
+			 * Calculate return/receive payload size; the
+			 * first word, if present, always gives the
+			 * number of bytes--unless it's 0 in which
+			 * case a single 32-bit word should be present.
+			 */
+			if (dlen >= sizeof(uint32_t)) {
+				olen = be32toh(rp[0]);
+				dlen -= sizeof(uint32_t);
+				if (olen == 0) {
+					/* convention is 0 =>'s one word */
+					olen = sizeof(uint32_t);
+					/* XXX KASSERT(olen == dlen ) */
+				}
+			} else
+				olen = 0;
+			if (cmd->odata != NULL) {
+				/* NB: cmd->olen validated in uath_cmd */
+				if (olen > cmd->olen) {
+					/* XXX complain? */
+					device_printf(sc->sc_dev,
+					    "%s: cmd 0x%x olen %u cmd olen %u\n",
+					    __func__, hdr->code, olen,
+					    cmd->olen);
+					olen = cmd->olen;
+				}
+				if (olen > dlen) {
+					/* XXX complain, shouldn't happen */
+					device_printf(sc->sc_dev,
+					    "%s: cmd 0x%x olen %u dlen %u\n",
+					    __func__, hdr->code, olen, dlen);
+					olen = dlen;
+				}
+				/* XXX have submitter do this */
+				/* copy answer into caller's supplied buffer */
+				bcopy(&rp[1], cmd->odata, olen);
+				cmd->olen = olen;
+			}
+		}
+		wakeup_one(cmd);		/* wake up caller */
+		break;
+
+	case WDCMSG_TARGET_START:
+		if (hdr->msgid >= UATH_CMD_LIST_COUNT) {
+			/* XXX */
+			return;
+		}
+		dlen = hdr->len - sizeof(*hdr);
+		if (dlen != sizeof(uint32_t)) {
+			/* XXX something wrong */
+			return;
+		}
+		/* XXX have submitter do this */
+		/* copy answer into caller's supplied buffer */
+		bcopy(hdr+1, cmd->odata, sizeof(uint32_t));
+		cmd->olen = sizeof(uint32_t);
+		wakeup_one(cmd);		/* wake up caller */
+		break;
+
+	case WDCMSG_SEND_COMPLETE:
+		/* this notification is sent when UATH_TX_NOTIFY is set */
+		DPRINTF(sc, UATH_DEBUG_RX_PROC | UATH_DEBUG_RECV_ALL,
+		    "%s: received Tx notification\n", __func__);
+		break;
+
+	case WDCMSG_TARGET_GET_STATS:
+		DPRINTF(sc, UATH_DEBUG_RX_PROC | UATH_DEBUG_RECV_ALL,
+		    "%s: received device statistics\n", __func__);
+		callout_reset(&sc->stat_ch, hz, uath_stat, sc);
+		break;
+	}
+}
+
+static void
+uath_intr_rx_callback(struct usb2_xfer *xfer)
+{
+	struct uath_softc *sc = xfer->priv_sc;
+	struct uath_cmd *cmd;
+
+	UATH_ASSERT_LOCKED(sc);
+
+	switch (USB_GET_STATE(xfer)) {
+	case USB_ST_TRANSFERRED:
+		cmd = STAILQ_FIRST(&sc->sc_cmd_waiting);
+		if (cmd == NULL)
+			goto setup;
+		STAILQ_REMOVE_HEAD(&sc->sc_cmd_waiting, next);
+		UATH_STAT_DEC(sc, st_cmd_waiting);
+		STAILQ_INSERT_TAIL(&sc->sc_cmd_inactive, cmd, next);
+		UATH_STAT_INC(sc, st_cmd_inactive);
+
+		KASSERT(xfer->actlen >= sizeof(struct uath_cmd_hdr),
+		    ("short xfer error"));
+		usb2_copy_out(xfer->frbuffers, 0, cmd->buf, xfer->actlen);
+		uath_cmdeof(sc, cmd);
+	case USB_ST_SETUP:
+setup:
+		xfer->frlengths[0] = xfer->max_data_length;
+		usb2_start_hardware(xfer);
+		break;
+	default:
+		if (xfer->error != USB_ERR_CANCELLED) {
+			xfer->flags.stall_pipe = 1;
+			goto setup;
+		}
+		break;
+	}
+}
+
+static void
+uath_intr_tx_callback(struct usb2_xfer *xfer)
+{
+	struct uath_softc *sc = xfer->priv_sc;
+	struct uath_cmd *cmd;
+
+	UATH_ASSERT_LOCKED(sc);
+
+	switch (USB_GET_STATE(xfer)) {
+	case USB_ST_TRANSFERRED:
+		cmd = STAILQ_FIRST(&sc->sc_cmd_active);
+		if (cmd == NULL)
+			goto setup;
+		STAILQ_REMOVE_HEAD(&sc->sc_cmd_active, next);
+		UATH_STAT_DEC(sc, st_cmd_active);
+		STAILQ_INSERT_TAIL((cmd->flags & UATH_CMD_FLAG_READ) ?
+		    &sc->sc_cmd_waiting : &sc->sc_cmd_inactive, cmd, next);
+		if (cmd->flags & UATH_CMD_FLAG_READ)
+			UATH_STAT_INC(sc, st_cmd_waiting);
+		else
+			UATH_STAT_INC(sc, st_cmd_inactive);
+		/* FALLTHROUGH */
+	case USB_ST_SETUP:
+setup:
+		cmd = STAILQ_FIRST(&sc->sc_cmd_pending);
+		if (cmd == NULL) {
+			DPRINTF(sc, UATH_DEBUG_XMIT, "%s: empty pending queue\n",
+			    __func__);
+			return;
+		}
+		STAILQ_REMOVE_HEAD(&sc->sc_cmd_pending, next);
+		UATH_STAT_DEC(sc, st_cmd_pending);
+		STAILQ_INSERT_TAIL((cmd->flags & UATH_CMD_FLAG_ASYNC) ?
+		    &sc->sc_cmd_inactive : &sc->sc_cmd_active, cmd, next);
+		if (cmd->flags & UATH_CMD_FLAG_ASYNC)
+			UATH_STAT_INC(sc, st_cmd_inactive);
+		else
+			UATH_STAT_INC(sc, st_cmd_active);
+
+		usb2_set_frame_data(xfer, cmd->buf, 0);
+		xfer->frlengths[0] = cmd->buflen;
+		usb2_start_hardware(xfer);
+		break;
+	default:
+		if (xfer->error != USB_ERR_CANCELLED) {
+			xfer->flags.stall_pipe = 1;
+			goto setup;
+		}
+		break;
+	}
+}
+
+static void
+uath_update_rxstat(struct uath_softc *sc, uint32_t status)
+{
+
+	switch (status) {
+	case UATH_STATUS_STOP_IN_PROGRESS:
+		UATH_STAT_INC(sc, st_stopinprogress);
+		break;
+	case UATH_STATUS_CRC_ERR:
+		UATH_STAT_INC(sc, st_crcerr);
+		break;
+	case UATH_STATUS_PHY_ERR:
+		UATH_STAT_INC(sc, st_phyerr);
+		break;
+	case UATH_STATUS_DECRYPT_CRC_ERR:
+		UATH_STAT_INC(sc, st_decrypt_crcerr);
+		break;
+	case UATH_STATUS_DECRYPT_MIC_ERR:
+		UATH_STAT_INC(sc, st_decrypt_micerr);
+		break;
+	case UATH_STATUS_DECOMP_ERR:
+		UATH_STAT_INC(sc, st_decomperr);
+		break;
+	case UATH_STATUS_KEY_ERR:
+		UATH_STAT_INC(sc, st_keyerr);
+		break;
+	case UATH_STATUS_ERR:
+		UATH_STAT_INC(sc, st_err);
+		break;
+	default:
+		break;
+	}
+}
+
+static struct mbuf *
+uath_data_rxeof(struct usb2_xfer *xfer, struct uath_data *data,
+    struct uath_rx_desc **pdesc)
+{
+	struct uath_softc *sc = xfer->priv_sc;
+	struct ifnet *ifp = sc->sc_ifp;
+	struct ieee80211com *ic = ifp->if_l2com;
+	struct uath_chunk *chunk;
+	struct uath_rx_desc *desc;
+	struct mbuf *m = data->m, *mnew, *mp;
+	uint16_t chunklen;
+
+	if (xfer->actlen < UATH_MIN_RXBUFSZ) {
+		DPRINTF(sc, UATH_DEBUG_RECV | UATH_DEBUG_RECV_ALL,
+		    "%s: wrong xfer size (len=%d)\n", __func__, xfer->actlen);
+		ifp->if_ierrors++;
+		return (NULL);
+	}
+
+	chunk = (struct uath_chunk *)data->buf;
+	if (chunk->seqnum == 0 && chunk->flags == 0 && chunk->length == 0) {
+		device_printf(sc->sc_dev, "%s: strange response\n", __func__);
+		ifp->if_ierrors++;
+		UATH_RESET_INTRX(sc);
+		return (NULL);
+	}
+
+	if (chunk->seqnum != sc->sc_intrx_nextnum) {
+		DPRINTF(sc, UATH_DEBUG_XMIT, "invalid seqnum %d, expected %d\n",
+		    chunk->seqnum, sc->sc_intrx_nextnum);
+		UATH_STAT_INC(sc, st_badchunkseqnum);
+		if (sc->sc_intrx_head != NULL)
+			m_freem(sc->sc_intrx_head);
+		UATH_RESET_INTRX(sc);
+		return (NULL);
+	}
+
+	/* check multi-chunk frames  */
+	if ((chunk->seqnum == 0 && !(chunk->flags & UATH_CFLAGS_FINAL)) ||
+	    (chunk->seqnum != 0 && (chunk->flags & UATH_CFLAGS_FINAL)) ||
+	    chunk->flags & UATH_CFLAGS_RXMSG)
+		UATH_STAT_INC(sc, st_multichunk);
+
+	chunklen = be16toh(chunk->length);
+	if (chunk->flags & UATH_CFLAGS_FINAL)
+		chunklen -= sizeof(struct uath_rx_desc);
+
+	if (chunklen > 0 &&
+	    (!(chunk->flags & UATH_CFLAGS_FINAL) || !(chunk->seqnum == 0))) {
+		/* we should use intermediate RX buffer  */
+		if (chunk->seqnum == 0)
+			UATH_RESET_INTRX(sc);
+		if ((sc->sc_intrx_len + sizeof(struct uath_rx_desc) +
+		    chunklen) > UATH_MAX_INTRX_SIZE) {
+			UATH_STAT_INC(sc, st_invalidlen);
+			ifp->if_iqdrops++;
+			if (sc->sc_intrx_head != NULL)
+				m_freem(sc->sc_intrx_head);
+			UATH_RESET_INTRX(sc);
+			return (NULL);
+		}
+
+		m->m_len = chunklen;
+		m->m_data += sizeof(struct uath_chunk);
+
+		if (sc->sc_intrx_head == NULL) {
+			sc->sc_intrx_head = m;
+			sc->sc_intrx_tail = m;
+		} else {
+			m->m_flags &= ~M_PKTHDR;
+			sc->sc_intrx_tail->m_next = m;
+			sc->sc_intrx_tail = m;
+		}
+	}
+	sc->sc_intrx_len += chunklen;
+
+	mnew = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
+	if (mnew == NULL) {
+		DPRINTF(sc, UATH_DEBUG_RECV | UATH_DEBUG_RECV_ALL,
+		    "%s: can't get new mbuf, drop frame\n", __func__);
+		ifp->if_ierrors++;
+		if (sc->sc_intrx_head != NULL)
+			m_freem(sc->sc_intrx_head);
+		UATH_RESET_INTRX(sc);
+		return (NULL);
+	}
+
+	data->m = mnew;
+	data->buf = mtod(mnew, uint8_t *);
+
+	/* if the frame is not final continue the transfer  */
+	if (!(chunk->flags & UATH_CFLAGS_FINAL)) {
+		sc->sc_intrx_nextnum++;
+		UATH_RESET_INTRX(sc);
+		return (NULL);
+	}
+
+	/*
+	 * if the frame is not set UATH_CFLAGS_RXMSG, then rx descriptor is
+	 * located at the end, 32-bit aligned
+	 */
+	desc = (chunk->flags & UATH_CFLAGS_RXMSG) ?
+		(struct uath_rx_desc *)(chunk + 1) :
+		(struct uath_rx_desc *)(((uint8_t *)chunk) + 
+		    sizeof(struct uath_chunk) + be16toh(chunk->length) -
+		    sizeof(struct uath_rx_desc));
+	*pdesc = desc;
+
+	DPRINTF(sc, UATH_DEBUG_RECV | UATH_DEBUG_RECV_ALL,
+	    "%s: frame len %u code %u status %u rate %u antenna %u "
+	    "rssi %d channel %u phyerror %u connix %u decrypterror %u "
+	    "keycachemiss %u\n", __func__, be32toh(desc->framelen)
+	    , be32toh(desc->code), be32toh(desc->status), be32toh(desc->rate)
+	    , be32toh(desc->antenna), be32toh(desc->rssi), be32toh(desc->channel)
+	    , be32toh(desc->phyerror), be32toh(desc->connix)
+	    , be32toh(desc->decrypterror), be32toh(desc->keycachemiss));
+
+	if (be32toh(desc->len) > MCLBYTES) {
+		DPRINTF(sc, UATH_DEBUG_RECV | UATH_DEBUG_RECV_ALL,
+		    "%s: bad descriptor (len=%d)\n", __func__,
+		    be32toh(desc->len));
+		ifp->if_iqdrops++;
+		UATH_STAT_INC(sc, st_toobigrxpkt);
+		if (sc->sc_intrx_head != NULL)
+			m_freem(sc->sc_intrx_head);
+		UATH_RESET_INTRX(sc);
+		return (NULL);
+	}
+
+	uath_update_rxstat(sc, be32toh(desc->status));
+
+	/* finalize mbuf */
+	if (sc->sc_intrx_head == NULL) {
+		m->m_pkthdr.rcvif = ifp;
+		m->m_pkthdr.len = m->m_len =
+			be32toh(desc->framelen) - UATH_RX_DUMMYSIZE;
+		m->m_data += sizeof(struct uath_chunk);
+	} else {
+		mp = sc->sc_intrx_head;
+		mp->m_pkthdr.rcvif = ifp;
+		mp->m_flags |= M_PKTHDR;
+		mp->m_pkthdr.len = sc->sc_intrx_len;
+		m = mp;
+	}
+
+	/* there are a lot more fields in the RX descriptor */
+	if (bpf_peers_present(ifp->if_bpf)) {
+		struct uath_rx_radiotap_header *tap = &sc->sc_rxtap;
+
+		tap->wr_chan_freq = htole16(be32toh(desc->channel));
+		tap->wr_chan_flags = htole16(ic->ic_curchan->ic_flags);
+		tap->wr_dbm_antsignal = (int8_t)be32toh(desc->rssi);
+
+		bpf_mtap2(ifp->if_bpf, tap, sc->sc_rxtap_len, m);
+	}
+
+	ifp->if_ipackets++;
+	UATH_RESET_INTRX(sc);
+
+	return (m);
+}
+
+static void
+uath_bulk_rx_callback(struct usb2_xfer *xfer)
+{
+	struct uath_softc *sc = xfer->priv_sc;
+	struct ifnet *ifp = sc->sc_ifp;
+	struct ieee80211com *ic = ifp->if_l2com;
+	struct ieee80211_frame *wh;
+	struct ieee80211_node *ni;
+	struct mbuf *m = NULL;
+	struct uath_data *data;
+	struct uath_rx_desc *desc = NULL;
+	int8_t nf;
+
+	UATH_ASSERT_LOCKED(sc);
+
+	switch (USB_GET_STATE(xfer)) {
+	case USB_ST_TRANSFERRED:
+		data = STAILQ_FIRST(&sc->sc_rx_active);
+		if (data == NULL)
+			goto setup;
+		STAILQ_REMOVE_HEAD(&sc->sc_rx_active, next);
+		UATH_STAT_DEC(sc, st_rx_active);
+		m = uath_data_rxeof(xfer, data, &desc);
+		STAILQ_INSERT_TAIL(&sc->sc_rx_inactive, data, next);
+		UATH_STAT_INC(sc, st_rx_inactive);
+		/* FALLTHROUGH */
+	case USB_ST_SETUP:
+setup:
+		data = STAILQ_FIRST(&sc->sc_rx_inactive);
+		if (data == NULL)
+			return;
+		STAILQ_REMOVE_HEAD(&sc->sc_rx_inactive, next);
+		UATH_STAT_DEC(sc, st_rx_inactive);
+		STAILQ_INSERT_TAIL(&sc->sc_rx_active, data, next);
+		UATH_STAT_INC(sc, st_rx_active);
+		usb2_set_frame_data(xfer, data->buf, 0);
+		xfer->frlengths[0] = xfer->max_data_length;
+		usb2_start_hardware(xfer);
+
+		/*
+		 * To avoid LOR we should unlock our private mutex here to call
+		 * ieee80211_input() because here is at the end of a USB
+		 * callback and safe to unlock.
+		 */
+		UATH_UNLOCK(sc);
+		if (m != NULL && desc != NULL) {
+			wh = mtod(m, struct ieee80211_frame *);
+			ni = ieee80211_find_rxnode(ic,
+			    (struct ieee80211_frame_min *)wh);
+			nf = -95;	/* XXX */
+			if (ni != NULL) {
+				(void) ieee80211_input(ni, m,
+				    (int)be32toh(desc->rssi), nf, 0);
+				/* node is no longer needed */
+				ieee80211_free_node(ni);
+			} else
+				(void) ieee80211_input_all(ic, m,
+				    (int)be32toh(desc->rssi), nf, 0);
+			m = NULL;
+			desc = NULL;
+		}
+		UATH_LOCK(sc);
+		break;
+	default:
+		/* needs it to the inactive queue due to a error.  */
+		data = STAILQ_FIRST(&sc->sc_rx_active);
+		if (data != NULL) {
+			STAILQ_REMOVE_HEAD(&sc->sc_rx_active, next);
+			UATH_STAT_DEC(sc, st_rx_active);
+			STAILQ_INSERT_TAIL(&sc->sc_rx_inactive, data, next);
+			UATH_STAT_INC(sc, st_rx_inactive);
+		}
+		if (xfer->error != USB_ERR_CANCELLED) {
+			xfer->flags.stall_pipe = 1;
+			ifp->if_ierrors++;
+			goto setup;
+		}
+		break;
+	}
+}
+
+static void
+uath_data_txeof(struct usb2_xfer *xfer, struct uath_data *data)
+{
+	struct uath_softc *sc = xfer->priv_sc;
+	struct ifnet *ifp = sc->sc_ifp;
+	struct mbuf *m;
+
+	UATH_ASSERT_LOCKED(sc);
+
+	/*
+	 * Do any tx complete callback.  Note this must be done before releasing
+	 * the node reference.
+	 */
+	if (data->m) {
+		m = data->m;
+		if (m->m_flags & M_TXCB) {
+			/* XXX status? */
+			ieee80211_process_callback(data->ni, m, 0);
+		}
+		m_freem(m);
+		data->m = NULL;
+	}
+	if (data->ni) {
+		ieee80211_free_node(data->ni);
+		data->ni = NULL;
+	}
+	sc->sc_tx_timer = 0;
+	ifp->if_opackets++;
+	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+}
+
+static void
+uath_bulk_tx_callback(struct usb2_xfer *xfer)
+{
+	struct uath_softc *sc = xfer->priv_sc;
+	struct ifnet *ifp = sc->sc_ifp;
+	struct uath_data *data;
+
+	UATH_ASSERT_LOCKED(sc);
+
+	switch (USB_GET_STATE(xfer)) {
+	case USB_ST_TRANSFERRED:
+		data = STAILQ_FIRST(&sc->sc_tx_active);
+		if (data == NULL)
+			goto setup;
+		STAILQ_REMOVE_HEAD(&sc->sc_tx_active, next);
+		UATH_STAT_DEC(sc, st_tx_active);
+		uath_data_txeof(xfer, data);
+		STAILQ_INSERT_TAIL(&sc->sc_tx_inactive, data, next);
+		UATH_STAT_INC(sc, st_tx_inactive);
+		/* FALLTHROUGH */
+	case USB_ST_SETUP:
+setup:
+		data = STAILQ_FIRST(&sc->sc_tx_pending);
+		if (data == NULL) {
+			DPRINTF(sc, UATH_DEBUG_XMIT, "%s: empty pending queue\n",
+			    __func__);
+			return;
+		}
+		STAILQ_REMOVE_HEAD(&sc->sc_tx_pending, next);
+		UATH_STAT_DEC(sc, st_tx_pending);
+		STAILQ_INSERT_TAIL(&sc->sc_tx_active, data, next);
+		UATH_STAT_INC(sc, st_tx_active);
+
+		usb2_set_frame_data(xfer, data->buf, 0);
+		xfer->frlengths[0] = data->buflen;
+		usb2_start_hardware(xfer);
+
+		UATH_UNLOCK(sc);
+		uath_start(ifp);
+		UATH_LOCK(sc);
+		break;
+	default:
+		data = STAILQ_FIRST(&sc->sc_tx_active);
+		if (data == NULL)
+			goto setup;
+		if (data->ni != NULL) {
+			ieee80211_free_node(data->ni);
+			data->ni = NULL;
+			ifp->if_oerrors++;
+		}
+		if (xfer->error != USB_ERR_CANCELLED) {
+			xfer->flags.stall_pipe = 1;
+			goto setup;
+		}
+		break;
+	}
+}
+
+static device_method_t uath_methods[] = {
+	DEVMETHOD(device_probe, uath_match),
+	DEVMETHOD(device_attach, uath_attach),
+	DEVMETHOD(device_detach, uath_detach),
+	{ 0, 0 }
+};
+static driver_t uath_driver = {
+	"uath",
+	uath_methods,
+	sizeof(struct uath_softc)
+};
+static devclass_t uath_devclass;
+
+DRIVER_MODULE(uath, uhub, uath_driver, uath_devclass, NULL, 0);
+MODULE_DEPEND(uath, wlan, 1, 1, 1);
+MODULE_DEPEND(uath, usb, 1, 1, 1);
diff --git a/sys/dev/usb/wlan/if_uathreg.h b/sys/dev/usb/wlan/if_uathreg.h
new file mode 100644
index 0000000..1e7929b
--- /dev/null
+++ b/sys/dev/usb/wlan/if_uathreg.h
@@ -0,0 +1,601 @@
+/*	$OpenBSD: if_uathreg.h,v 1.2 2006/09/18 16:34:23 damien Exp $	*/
+/*	$FreeBSD$	*/
+
+/*-
+ * Copyright (c) 2006
+ *	Damien Bergamini <damien.bergamini@free.fr>
+ * Copyright (c) 2006 Sam Leffler, Errno Consulting
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#define UATH_CONFIG_INDEX	0
+#define UATH_IFACE_INDEX	0
+
+/* all fields are big endian */
+struct uath_fwblock {
+	uint32_t	flags;
+#define UATH_WRITE_BLOCK	(1 << 4)
+
+	uint32_t	len;
+#define UATH_MAX_FWBLOCK_SIZE	2048
+
+	uint32_t	total;
+	uint32_t	remain;
+	uint32_t	rxtotal;
+	uint32_t	pad[123];
+} __packed;
+
+#define UATH_MAX_CMDSZ		512
+
+/*
+ * Messages are passed in Target Endianness.  All fixed-size
+ * fields of a WDS Control Message are treated as 32-bit
+ * values and Control Msgs are guaranteed to be 32-bit aligned.
+ *
+ * The format of a WDS Control Message is as follows:
+ *    Message Length	32 bits
+ *    Message Opcode	32 bits
+ *    Message ID	32 bits
+ *    parameter 1
+ *    parameter 2
+ *       ...
+ *
+ * A variable-length parameter, or a parmeter that is larger than
+ * 32 bits is passed as <length, data> pair, where length is a
+ * 32-bit quantity and data is padded to 32 bits.
+ */
+struct uath_cmd_hdr {
+	uint32_t	len;		/* msg length including header */
+	uint32_t	code;		/* operation code */
+/* NB: these are defined for rev 1.5 firmware; rev 1.6 is different */
+/* messages from Host -> Target */
+#define	WDCMSG_HOST_AVAILABLE		0x01
+#define WDCMSG_BIND			0x02
+#define WDCMSG_TARGET_RESET		0x03
+#define WDCMSG_TARGET_GET_CAPABILITY	0x04
+#define WDCMSG_TARGET_SET_CONFIG	0x05
+#define WDCMSG_TARGET_GET_STATUS	0x06
+#define WDCMSG_TARGET_GET_STATS		0x07
+#define WDCMSG_TARGET_START		0x08
+#define WDCMSG_TARGET_STOP		0x09
+#define WDCMSG_TARGET_ENABLE		0x0a
+#define WDCMSG_TARGET_DISABLE		0x0b
+#define	WDCMSG_CREATE_CONNECTION	0x0c
+#define WDCMSG_UPDATE_CONNECT_ATTR	0x0d
+#define	WDCMSG_DELETE_CONNECT		0x0e
+#define	WDCMSG_SEND			0x0f
+#define WDCMSG_FLUSH			0x10
+/* messages from Target -> Host */
+#define	WDCMSG_STATS_UPDATE		0x11
+#define	WDCMSG_BMISS			0x12
+#define	WDCMSG_DEVICE_AVAIL		0x13
+#define	WDCMSG_SEND_COMPLETE		0x14
+#define	WDCMSG_DATA_AVAIL		0x15
+#define	WDCMSG_SET_PWR_MODE		0x16
+#define	WDCMSG_BMISS_ACK		0x17
+#define	WDCMSG_SET_LED_STEADY		0x18
+#define	WDCMSG_SET_LED_BLINK		0x19
+/* more messages */
+#define	WDCMSG_SETUP_BEACON_DESC	0x1a
+#define	WDCMSG_BEACON_INIT		0x1b
+#define	WDCMSG_RESET_KEY_CACHE		0x1c
+#define	WDCMSG_RESET_KEY_CACHE_ENTRY	0x1d
+#define	WDCMSG_SET_KEY_CACHE_ENTRY	0x1e
+#define	WDCMSG_SET_DECOMP_MASK		0x1f
+#define	WDCMSG_SET_REGULATORY_DOMAIN	0x20
+#define	WDCMSG_SET_LED_STATE		0x21
+#define	WDCMSG_WRITE_ASSOCID		0x22
+#define	WDCMSG_SET_STA_BEACON_TIMERS	0x23
+#define	WDCMSG_GET_TSF			0x24
+#define	WDCMSG_RESET_TSF		0x25
+#define	WDCMSG_SET_ADHOC_MODE		0x26
+#define	WDCMSG_SET_BASIC_RATE		0x27
+#define	WDCMSG_MIB_CONTROL		0x28
+#define	WDCMSG_GET_CHANNEL_DATA		0x29
+#define	WDCMSG_GET_CUR_RSSI		0x2a
+#define	WDCMSG_SET_ANTENNA_SWITCH	0x2b
+#define	WDCMSG_USE_SHORT_SLOT_TIME	0x2f
+#define	WDCMSG_SET_POWER_MODE		0x30
+#define	WDCMSG_SETUP_PSPOLL_DESC	0x31
+#define	WDCMSG_SET_RX_MULTICAST_FILTER	0x32
+#define	WDCMSG_RX_FILTER		0x33
+#define	WDCMSG_PER_CALIBRATION		0x34
+#define	WDCMSG_RESET			0x35
+#define	WDCMSG_DISABLE			0x36
+#define	WDCMSG_PHY_DISABLE		0x37
+#define	WDCMSG_SET_TX_POWER_LIMIT	0x38
+#define	WDCMSG_SET_TX_QUEUE_PARAMS	0x39
+#define	WDCMSG_SETUP_TX_QUEUE		0x3a
+#define	WDCMSG_RELEASE_TX_QUEUE		0x3b
+#define	WDCMSG_SET_DEFAULT_KEY		0x43
+	uint32_t	msgid;		/* msg id (supplied by host) */
+	uint32_t	magic;		/* response desired/target status */
+	uint32_t	debug[4];	/* debug data area */
+	/* msg data follows */
+} __packed;
+
+struct uath_chunk {
+	uint8_t		seqnum;		/* sequence number for ordering */
+	uint8_t		flags;
+#define	UATH_CFLAGS_FINAL	0x01	/* final chunk of a msg */
+#define	UATH_CFLAGS_RXMSG	0x02	/* chunk contains rx completion */
+#define	UATH_CFLAGS_DEBUG	0x04	/* for debugging */
+	uint16_t	length;		/* chunk size in bytes */
+	/* chunk data follows */
+} __packed;
+
+#define	UATH_RX_DUMMYSIZE		4
+
+/*
+ * Message format for a WDCMSG_DATA_AVAIL message from Target to Host.
+ */
+struct uath_rx_desc {
+	uint32_t	len;		/* msg length including header */
+	uint32_t	code;		/* WDCMSG_DATA_AVAIL */
+	uint32_t	gennum;		/* generation number */
+	uint32_t	status;		/* start of RECEIVE_INFO */
+#define	UATH_STATUS_OK			0
+#define	UATH_STATUS_STOP_IN_PROGRESS	1
+#define	UATH_STATUS_CRC_ERR		2
+#define	UATH_STATUS_PHY_ERR		3
+#define	UATH_STATUS_DECRYPT_CRC_ERR	4
+#define	UATH_STATUS_DECRYPT_MIC_ERR	5
+#define	UATH_STATUS_DECOMP_ERR		6
+#define	UATH_STATUS_KEY_ERR		7
+#define	UATH_STATUS_ERR			8
+	uint32_t	tstamp_low;	/* low-order 32-bits of rx timestamp */
+	uint32_t	tstamp_high;	/* high-order 32-bits of rx timestamp */
+	uint32_t	framelen;	/* frame length */
+	uint32_t	rate;		/* rx rate code */
+	uint32_t	antenna;
+	int32_t		rssi;
+	uint32_t	channel;
+	uint32_t	phyerror;
+	uint32_t	connix;		/* key table ix for bss traffic */
+	uint32_t	decrypterror;
+	uint32_t	keycachemiss;
+	uint32_t	pad;		/* XXX? */
+} __packed;
+
+struct uath_tx_desc {
+	uint32_t	msglen;
+	uint32_t	msgid;		/* msg id (supplied by host) */
+	uint32_t	type;		/* opcode: WDMSG_SEND or WDCMSG_FLUSH */
+	uint32_t	txqid;		/* tx queue id and flags */
+#define	UATH_TXQID_MASK		0x0f
+#define	UATH_TXQID_MINRATE	0x10	/* use min tx rate */
+#define	UATH_TXQID_FF		0x20	/* content is fast frame */
+	uint32_t	connid;		/* tx connection id */
+#define UATH_ID_INVALID	0xffffffff	/* for sending prior to connection */
+	uint32_t	flags;		/* non-zero if response desired */
+#define UATH_TX_NOTIFY	(1 << 24)	/* f/w will send a UATH_NOTIF_TX */
+	uint32_t	buflen;		/* payload length */
+} __packed;
+
+struct uath_cmd_host_available {
+	uint32_t	sw_ver_major;
+	uint32_t	sw_ver_minor;
+	uint32_t	sw_ver_patch;
+	uint32_t	sw_ver_build;
+} __packed;
+#define	ATH_SW_VER_MAJOR	1
+#define	ATH_SW_VER_MINOR	5
+#define	ATH_SW_VER_PATCH	0
+#define	ATH_SW_VER_BUILD	9999
+
+struct uath_cmd_bind {
+	uint32_t	targethandle;
+	uint32_t	hostapiversion;
+} __packed;
+
+/* structure for command WDCMSG_RESET */
+struct uath_cmd_reset {
+	uint32_t	flags;		/* channel flags */
+#define	UATH_CHAN_TURBO	0x0100
+#define	UATH_CHAN_CCK	0x0200
+#define	UATH_CHAN_OFDM	0x0400
+#define	UATH_CHAN_2GHZ	0x1000
+#define	UATH_CHAN_5GHZ	0x2000
+	uint32_t	freq;		/* channel frequency */
+	uint32_t	maxrdpower;
+	uint32_t	cfgctl;
+	uint32_t	twiceantennareduction;
+	uint32_t	channelchange;
+	uint32_t	keeprccontent;
+} __packed;
+
+/* structure for commands UATH_CMD_READ_MAC and UATH_CMD_READ_EEPROM */
+struct uath_read_mac {
+	uint32_t	len;
+	uint8_t		data[32];
+} __packed;
+
+/* structure for command UATH_CMD_WRITE_MAC */
+struct uath_write_mac {
+	uint32_t	reg;
+	uint32_t	len;
+	uint8_t		data[32];
+} __packed;
+
+/* structure for command UATH_CMD_STA_JOIN */
+struct uath_cmd_join_bss {
+	uint32_t	bssid;		/* NB: use zero */
+	uint32_t	bssmac[2];	/* bssid mac address */
+	uint32_t	bsstype;
+	uint32_t	wlanmode;
+	uint32_t	beaconinterval;
+	uint32_t	dtiminterval;
+	uint32_t	cfpinterval;
+	uint32_t	atimwindow;
+	uint32_t	defaultrateix;
+	uint32_t	shortslottime11g;
+	uint32_t	sleepduration;
+	uint32_t	bmissthreshold;
+	uint32_t	tcppowerlimit;
+	uint32_t	quietduration;
+	uint32_t	quietoffset;
+	uint32_t	quietackctsallow;
+	uint32_t	bssdefaultkey;	/* XXX? */
+} __packed;
+
+struct uath_cmd_assoc_bss {
+	uint32_t	bssid;
+	uint32_t	associd;
+} __packed;
+
+struct uath_cmd_start_bss {
+	uint32_t	bssid;
+} __packed;
+
+/* structure for command UATH_CMD_0C */
+struct uath_cmd_0c {
+	uint32_t	magic1;
+	uint32_t	magic2;
+	uint32_t	magic3;
+} __packed;
+
+struct uath_cmd_ledsteady {		/* WDCMSG_SET_LED_STEADY */
+	uint32_t	lednum;
+#define UATH_LED_LINK		0
+#define UATH_LED_ACTIVITY	1
+	uint32_t	ledmode;
+#define UATH_LED_OFF	0
+#define UATH_LED_ON	1
+} __packed;
+
+struct uath_cmd_ledblink {		/* WDCMSG_SET_LED_BLINK */
+	uint32_t	lednum;
+	uint32_t	ledmode;
+	uint32_t	blinkrate;
+	uint32_t	slowmode;
+} __packed;
+
+struct uath_cmd_ledstate {		/* WDCMSG_SET_LED_STATE */
+	uint32_t	connected;
+} __packed;
+
+struct uath_connkey_rec {
+	uint8_t		bssid[IEEE80211_ADDR_LEN];
+	uint32_t	keyiv;
+	uint32_t	extkeyiv;
+	uint16_t	keyflags;
+	uint16_t	keylen;
+	uint16_t	keytype;	/* WEP, TKIP or AES */
+	/* As far as I know, MIPS 4Kp is 32-bit processor  */
+	uint32_t	priv;
+	uint8_t		keyval[32];
+	uint16_t	aes_keylen;
+	uint8_t		aes_keyval[16];
+	uint8_t		mic_txkeyval[8];
+	uint8_t		mic_rxkeyval[8];
+	int64_t		keyrsc[17];
+	int32_t		keytsc[17];
+	int32_t		keyexttsc[17];
+} __packed;
+
+/* structure for command UATH_CMD_CRYPTO */
+struct uath_cmd_crypto {
+	uint32_t		keyidx;
+#define UATH_DEFAULT_KEY	6
+	uint32_t		xorkey;
+	uint32_t		size;
+	struct uath_connkey_rec	rec;
+} __packed;
+
+struct uath_cmd_rateset {
+	uint8_t		length;
+#define UATH_MAX_NRATES	32
+	uint8_t		set[UATH_MAX_NRATES];
+};
+
+/* structure for command WDCMSG_SET_BASIC_RATE */
+struct uath_cmd_rates {
+	uint32_t	connid;
+	uint32_t	keeprccontent;
+	uint32_t	size;
+	struct uath_cmd_rateset rateset;
+} __packed;
+
+enum {
+	WLAN_MODE_NONE = 0,
+	WLAN_MODE_11b,
+	WLAN_MODE_11a,
+	WLAN_MODE_11g,
+	WLAN_MODE_11a_TURBO,
+	WLAN_MODE_11g_TURBO,
+	WLAN_MODE_11a_TURBO_PRIME,
+	WLAN_MODE_11g_TURBO_PRIME,
+	WLAN_MODE_11a_XR,
+	WLAN_MODE_11g_XR,
+};
+
+struct uath_cmd_connection_attr {
+	uint32_t	longpreambleonly;
+	struct uath_cmd_rateset	rateset;
+	uint32_t	wlanmode;
+} __packed;
+
+/* structure for command WDCMSG_CREATE_CONNECTION */
+struct uath_cmd_create_connection {
+	uint32_t	connid;
+	uint32_t	bssid;
+	uint32_t	size;
+	struct uath_cmd_connection_attr	connattr;
+} __packed;
+
+struct uath_cmd_txq_setparams {		/* WDCMSG_SET_TX_QUEUE_PARAMS */
+	uint32_t	qnum;
+	uint32_t	aifs;
+	uint32_t	logcwmin;
+	uint32_t	logcwmax;
+	uint32_t	bursttime;
+	uint32_t	qflags;
+} __packed;
+
+struct uath_cmd_txq_attr {
+	uint32_t	priority;
+	uint32_t	aifs;
+	uint32_t	logcwmin;
+	uint32_t	logcwmax;
+	uint32_t	bursttime;
+	uint32_t	mode;
+	uint32_t	qflags;
+} __packed;
+
+struct uath_cmd_txq_setup {		/* WDCMSG_SETUP_TX_QUEUE */
+	uint32_t	qid;
+	uint32_t	len;
+	struct uath_cmd_txq_attr attr;
+} __packed;
+
+struct uath_cmd_stoptxdma {		/* WDCMSG_STOP_TX_DMA */
+	uint32_t	qnum;
+	uint32_t	msec;
+} __packed;
+
+/* structure for command UATH_CMD_31 */
+struct uath_cmd_31 {
+	uint32_t	magic1;
+	uint32_t	magic2;
+} __packed;
+
+struct uath_cmd_rx_filter {		/* WDCMSG_RX_FILTER */
+	uint32_t	bits;
+#define UATH_FILTER_RX_UCAST		0x00000001
+#define UATH_FILTER_RX_MCAST		0x00000002
+#define UATH_FILTER_RX_BCAST		0x00000004
+#define UATH_FILTER_RX_CONTROL		0x00000008
+#define UATH_FILTER_RX_BEACON		0x00000010	/* beacon frames */
+#define UATH_FILTER_RX_PROM		0x00000020	/* promiscuous mode */
+#define UATH_FILTER_RX_PHY_ERR		0x00000040	/* phy errors */
+#define UATH_FILTER_RX_PHY_RADAR	0x00000080	/* radar phy errors */
+#define UATH_FILTER_RX_XR_POOL		0x00000400	/* XR group polls */
+#define UATH_FILTER_RX_PROBE_REQ	0x00000800
+	uint32_t	op;
+#define UATH_FILTER_OP_INIT		0x0
+#define UATH_FILTER_OP_SET		0x1
+#define UATH_FILTER_OP_CLEAR		0x2
+#define UATH_FILTER_OP_TEMP		0x3
+#define UATH_FILTER_OP_RESTORE		0x4
+} __packed;
+
+struct uath_cmd_rx_mcast_filter {	/* WDCMSG_SET_RX_MCAST_FILTER */
+	uint32_t	filter0;
+	uint32_t	filter1;
+} __packed;
+
+struct uath_cmd_set_associd {		/* WDCMSG_WRITE_ASSOCID */
+	uint32_t	defaultrateix;
+	uint32_t	associd;
+	uint32_t	timoffset;
+	uint32_t	turboprime;
+	uint32_t	bssid[2];
+} __packed;
+
+struct uath_cmd_set_stabeacon_timers {	/* WDCMSG_SET_STA_BEACON_TIMERS */
+	uint32_t	nexttbtt;
+	uint32_t	nextdtim;
+	uint32_t	nextcfp;
+	uint32_t	beaconperiod;
+	uint32_t	dtimperiod;
+	uint32_t	cfpperiod;
+	uint32_t	cfpduration;
+	uint32_t	sleepduration;
+	uint32_t	bsmissthreshold;
+} __packed;
+
+enum {
+	CFG_NONE,			/* Sentinal to indicate "no config" */
+	CFG_REG_DOMAIN,			/* Regulatory Domain */
+	CFG_RATE_CONTROL_ENABLE,
+	CFG_DEF_XMIT_DATA_RATE,		/* NB: if rate control is not enabled */
+	CFG_HW_TX_RETRIES,
+	CFG_SW_TX_RETRIES,
+	CFG_SLOW_CLOCK_ENABLE,
+	CFG_COMP_PROC,
+	CFG_USER_RTS_THRESHOLD,
+	CFG_XR2NORM_RATE_THRESHOLD,
+	CFG_XRMODE_SWITCH_COUNT,
+	CFG_PROTECTION_TYPE,
+	CFG_BURST_SEQ_THRESHOLD,
+	CFG_ABOLT,
+	CFG_IQ_LOG_COUNT_MAX,
+	CFG_MODE_CTS,
+	CFG_WME_ENABLED,
+	CFG_GPRS_CBR_PERIOD,
+	CFG_SERVICE_TYPE,
+	/* MAC Address to use.  Overrides EEPROM */
+	CFG_MAC_ADDR,
+	CFG_DEBUG_EAR,
+	CFG_INIT_REGS,
+	/* An ID for use in error & debug messages */
+	CFG_DEBUG_ID,
+	CFG_COMP_WIN_SZ,
+	CFG_DIVERSITY_CTL,
+	CFG_TP_SCALE,
+	CFG_TPC_HALF_DBM5,
+	CFG_TPC_HALF_DBM2,
+	CFG_OVERRD_TX_POWER,
+	CFG_USE_32KHZ_CLOCK,
+	CFG_GMODE_PROTECTION,
+	CFG_GMODE_PROTECT_RATE_INDEX,
+	CFG_GMODE_NON_ERP_PREAMBLE,
+	CFG_WDC_TRANSPORT_CHUNK_SIZE,
+};
+
+enum {
+	/* Sentinal to indicate "no capability" */
+	CAP_NONE,
+	CAP_ALL,			/* ALL capabilities */
+	CAP_TARGET_VERSION,
+	CAP_TARGET_REVISION,
+	CAP_MAC_VERSION,
+	CAP_MAC_REVISION,
+	CAP_PHY_REVISION,
+	CAP_ANALOG_5GHz_REVISION,
+	CAP_ANALOG_2GHz_REVISION,
+	/* Target supports WDC message debug features */
+	CAP_DEBUG_WDCMSG_SUPPORT,
+
+	CAP_REG_DOMAIN,
+	CAP_COUNTRY_CODE,
+	CAP_REG_CAP_BITS,
+
+	CAP_WIRELESS_MODES,
+	CAP_CHAN_SPREAD_SUPPORT,
+	CAP_SLEEP_AFTER_BEACON_BROKEN,
+	CAP_COMPRESS_SUPPORT,
+	CAP_BURST_SUPPORT,
+	CAP_FAST_FRAMES_SUPPORT,
+	CAP_CHAP_TUNING_SUPPORT,
+	CAP_TURBOG_SUPPORT,
+	CAP_TURBO_PRIME_SUPPORT,
+	CAP_DEVICE_TYPE,
+	CAP_XR_SUPPORT,
+	CAP_WME_SUPPORT,
+	CAP_TOTAL_QUEUES,
+	CAP_CONNECTION_ID_MAX,		/* Should absorb CAP_KEY_CACHE_SIZE */
+
+	CAP_LOW_5GHZ_CHAN,
+	CAP_HIGH_5GHZ_CHAN,
+	CAP_LOW_2GHZ_CHAN,
+	CAP_HIGH_2GHZ_CHAN,
+
+	CAP_MIC_AES_CCM,
+	CAP_MIC_CKIP,
+	CAP_MIC_TKIP,
+	CAP_MIC_TKIP_WME,
+	CAP_CIPHER_AES_CCM,
+	CAP_CIPHER_CKIP,
+	CAP_CIPHER_TKIP,
+
+	CAP_TWICE_ANTENNAGAIN_5G,
+	CAP_TWICE_ANTENNAGAIN_2G,
+};
+
+enum {
+	ST_NONE,                    /* Sentinal to indicate "no status" */
+	ST_ALL,
+	ST_SERVICE_TYPE,
+	ST_WLAN_MODE,
+	ST_FREQ,
+	ST_BAND,
+	ST_LAST_RSSI,
+	ST_PS_FRAMES_DROPPED,
+	ST_CACHED_DEF_ANT,
+	ST_COUNT_OTHER_RX_ANT,
+	ST_USE_FAST_DIVERSITY,
+	ST_MAC_ADDR,
+	ST_RX_GENERATION_NUM,
+	ST_TX_QUEUE_DEPTH,
+	ST_SERIAL_NUMBER,
+	ST_WDC_TRANSPORT_CHUNK_SIZE,
+};
+
+enum {
+	BSS_ATTR_BEACON_INTERVAL,
+	BSS_ATTR_DTIM_INTERVAL,
+	BSS_ATTR_CFP_INTERVAL,
+	BSS_ATTR_CFP_MAX_DURATION,
+	BSS_ATTR_ATIM_WINDOW,
+	BSS_ATTR_DEFAULT_RATE_INDEX,
+	BSS_ATTR_SHORT_SLOT_TIME_11g,
+	BSS_ATTR_SLEEP_DURATION,
+	BSS_ATTR_BMISS_THRESHOLD,
+	BSS_ATTR_TPC_POWER_LIMIT,
+	BSS_ATTR_BSS_KEY_UPDATE,
+};
+
+struct uath_cmd_update_bss_attribute {
+	uint32_t	bssid;
+	uint32_t	attribute;	/* BSS_ATTR_BEACON_INTERVAL, et al. */
+	uint32_t	cfgsize;	/* should be zero 0 */
+	uint32_t	cfgdata;
+};
+
+struct uath_cmd_update_bss_attribute_key {
+	uint32_t	bssid;
+	uint32_t	attribute;	/* BSS_ATTR_BSS_KEY_UPDATE */
+	uint32_t	cfgsize;	/* size of remaining data */
+	uint32_t	bsskeyix;
+	uint32_t	isdefaultkey;
+	uint32_t	keyiv;		/* IV generation control */
+	uint32_t	extkeyiv;	/* extended IV for TKIP & CCM */
+	uint32_t	keyflags;
+	uint32_t	keytype;
+	uint32_t	initvalue;	/* XXX */
+	uint32_t	keyval[4];
+	uint32_t	mictxkeyval[2];
+	uint32_t	micrxkeyval[2];
+	uint32_t	keyrsc[2];
+};
+
+enum {
+	TARGET_DEVICE_AWAKE,
+	TARGET_DEVICE_SLEEP,
+	TARGET_DEVICE_PWRDN,
+	TARGET_DEVICE_PWRSAVE,
+	TARGET_DEVICE_SUSPEND,
+	TARGET_DEVICE_RESUME,
+};
+
+#define UATH_MAX_TXBUFSZ						\
+	(sizeof(struct uath_chunk) + sizeof(struct uath_tx_desc) +	\
+	IEEE80211_MAX_LEN)
+
+/*
+ * it's not easy to measure how the chunk is passed into the host if the target
+ * passed the multi-chunks so just we check a minimal size we can imagine.
+ */
+#define UATH_MIN_RXBUFSZ	(sizeof(struct uath_chunk))
diff --git a/sys/dev/usb/wlan/if_uathvar.h b/sys/dev/usb/wlan/if_uathvar.h
new file mode 100644
index 0000000..2717399
--- /dev/null
+++ b/sys/dev/usb/wlan/if_uathvar.h
@@ -0,0 +1,237 @@
+/*	$OpenBSD: if_uathvar.h,v 1.3 2006/09/20 19:47:17 damien Exp $	*/
+/*	$FreeBSD$	*/
+
+/*-
+ * Copyright (c) 2006
+ *	Damien Bergamini <damien.bergamini@free.fr>
+ * Copyright (c) 2006 Sam Leffler, Errno Consulting
+ * Copyright (c) 2008-2009 Weongyo Jeong <weongyo@freebsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+enum {
+	UATH_INTR_RX,
+	UATH_INTR_TX,
+	UATH_BULK_RX,
+	UATH_BULK_TX,
+	UATH_N_XFERS = 4,
+};
+
+#define	UATH_ID_BSS		2	/* Connection ID  */
+
+#define	UATH_RX_DATA_LIST_COUNT	128
+#define	UATH_TX_DATA_LIST_COUNT	16
+#define	UATH_CMD_LIST_COUNT	60
+
+#define	UATH_DATA_TIMEOUT	10000
+#define	UATH_CMD_TIMEOUT	1000
+
+/* flags for sending firmware commands */
+#define	UATH_CMD_FLAG_ASYNC	(1 << 0)
+#define	UATH_CMD_FLAG_READ	(1 << 1)
+#define	UATH_CMD_FLAG_MAGIC	(1 << 2)
+
+struct uath_rx_radiotap_header {
+	struct ieee80211_radiotap_header wr_ihdr;
+	uint8_t				wr_flags;
+	uint16_t			wr_chan_freq;
+	uint16_t			wr_chan_flags;
+	int8_t				wr_dbm_antsignal;
+} __packed;
+
+#define	UATH_RX_RADIOTAP_PRESENT					\
+	((1 << IEEE80211_RADIOTAP_FLAGS) |				\
+	 (1 << IEEE80211_RADIOTAP_CHANNEL) |				\
+	 (1 << IEEE80211_RADIOTAP_DBM_ANTSIGNAL))
+
+struct uath_tx_radiotap_header {
+	struct ieee80211_radiotap_header wt_ihdr;
+	uint8_t				wt_flags;
+	uint16_t			wt_chan_freq;
+	uint16_t			wt_chan_flags;
+} __packed;
+
+#define	UATH_TX_RADIOTAP_PRESENT					\
+	((1 << IEEE80211_RADIOTAP_FLAGS) |				\
+	 (1 << IEEE80211_RADIOTAP_CHANNEL))
+
+struct uath_data {
+	struct uath_softc		*sc;
+	uint8_t				*buf;
+	uint16_t			buflen;
+	struct mbuf			*m;
+	struct ieee80211_node		*ni;		/* NB: tx only */
+	STAILQ_ENTRY(uath_data)		next;
+};
+typedef STAILQ_HEAD(, uath_data) uath_datahead;
+
+struct uath_cmd {
+	struct uath_softc		*sc;
+	uint32_t			flags;
+	uint32_t			msgid;
+	uint8_t				*buf;
+	uint16_t			buflen;
+	void				*odata;		/* NB: tx only */
+	int				olen;		/* space in odata */
+	STAILQ_ENTRY(uath_cmd)		next;
+};
+typedef STAILQ_HEAD(, uath_cmd) uath_cmdhead;
+
+struct uath_wme_settings {
+	uint8_t				aifsn;
+	uint8_t				logcwmin;
+	uint8_t				logcwmax;
+	uint16_t			txop;
+#define	UATH_TXOP_TO_US(txop)		((txop) << 5)
+	uint8_t				acm;
+};
+
+struct uath_devcap {
+	uint32_t			targetVersion;
+	uint32_t			targetRevision;
+	uint32_t			macVersion;
+	uint32_t			macRevision;
+	uint32_t			phyRevision;
+	uint32_t			analog5GhzRevision;
+	uint32_t			analog2GhzRevision;
+	uint32_t			regDomain;
+	uint32_t			regCapBits;
+	uint32_t			countryCode;
+	uint32_t			keyCacheSize;
+	uint32_t			numTxQueues;
+	uint32_t			connectionIdMax;
+	uint32_t			wirelessModes;
+#define	UATH_WIRELESS_MODE_11A		0x01
+#define	UATH_WIRELESS_MODE_TURBO	0x02
+#define	UATH_WIRELESS_MODE_11B		0x04
+#define	UATH_WIRELESS_MODE_11G		0x08
+#define	UATH_WIRELESS_MODE_108G		0x10
+	uint32_t			chanSpreadSupport;
+	uint32_t			compressSupport;
+	uint32_t			burstSupport;
+	uint32_t			fastFramesSupport;
+	uint32_t			chapTuningSupport;
+	uint32_t			turboGSupport;
+	uint32_t			turboPrimeSupport;
+	uint32_t			deviceType;
+	uint32_t			wmeSupport;
+	uint32_t			low2GhzChan;
+	uint32_t			high2GhzChan;
+	uint32_t			low5GhzChan;
+	uint32_t			high5GhzChan;
+	uint32_t			supportCipherWEP;
+	uint32_t			supportCipherAES_CCM;
+	uint32_t			supportCipherTKIP;
+	uint32_t			supportCipherMicAES_CCM;
+	uint32_t			supportMicTKIP;
+	uint32_t			twiceAntennaGain5G;
+	uint32_t			twiceAntennaGain2G;
+};
+
+struct uath_stat {
+	uint32_t			st_badchunkseqnum;
+	uint32_t			st_invalidlen;
+	uint32_t			st_multichunk;
+	uint32_t			st_toobigrxpkt;
+	uint32_t			st_stopinprogress;
+	uint32_t			st_crcerr;
+	uint32_t			st_phyerr;
+	uint32_t			st_decrypt_crcerr;
+	uint32_t			st_decrypt_micerr;
+	uint32_t			st_decomperr;
+	uint32_t			st_keyerr;
+	uint32_t			st_err;
+	/* CMD/RX/TX queues */
+	uint32_t			st_cmd_active;
+	uint32_t			st_cmd_inactive;
+	uint32_t			st_cmd_pending;
+	uint32_t			st_cmd_waiting;
+	uint32_t			st_rx_active;
+	uint32_t			st_rx_inactive;
+	uint32_t			st_tx_active;
+	uint32_t			st_tx_inactive;
+	uint32_t			st_tx_pending;
+};
+#define	UATH_STAT_INC(sc, var)		(sc)->sc_stat.var++
+#define	UATH_STAT_DEC(sc, var)		(sc)->sc_stat.var--
+
+struct uath_vap {
+	struct ieee80211vap		vap;
+	int				(*newstate)(struct ieee80211vap *,
+					    enum ieee80211_state, int);
+};
+#define	UATH_VAP(vap)			((struct uath_vap *)(vap))
+
+struct uath_softc {
+	struct ifnet			*sc_ifp;
+	device_t			sc_dev;
+	struct usb2_device		*sc_udev;
+	struct mtx			sc_mtx;
+	uint32_t			sc_debug;
+
+	struct uath_stat		sc_stat;
+	int				(*sc_newstate)(struct ieee80211com *,
+					    enum ieee80211_state, int);
+	enum ieee80211_state		sc_state;
+
+	struct usb2_xfer		*sc_xfer[UATH_N_XFERS];
+	struct uath_cmd			sc_cmd[UATH_CMD_LIST_COUNT];
+	uath_cmdhead			sc_cmd_active;
+	uath_cmdhead			sc_cmd_inactive;
+	uath_cmdhead			sc_cmd_pending;
+	uath_cmdhead			sc_cmd_waiting;
+	struct uath_data		sc_rx[UATH_RX_DATA_LIST_COUNT];
+	uath_datahead			sc_rx_active;
+	uath_datahead			sc_rx_inactive;
+	struct uath_data		sc_tx[UATH_TX_DATA_LIST_COUNT];
+	uath_datahead			sc_tx_active;
+	uath_datahead			sc_tx_inactive;
+	uath_datahead			sc_tx_pending;
+
+	uint32_t			sc_msgid;
+	uint32_t			sc_seqnum;
+	int				sc_tx_timer;
+	struct callout			watchdog_ch;
+	struct callout			stat_ch;
+	/* multi-chunked support  */
+	struct mbuf			*sc_intrx_head;
+	struct mbuf			*sc_intrx_tail;
+	uint8_t				sc_intrx_nextnum;
+	uint32_t			sc_intrx_len;
+#define	UATH_MAX_INTRX_SIZE		3616
+
+	struct uath_devcap		sc_devcap;
+	uint8_t				sc_serial[16];
+
+	/* unsorted  */
+	uint32_t			sc_flags;
+#define	UATH_FLAG_INVALID		(1 << 1)
+#define	UATH_FLAG_INITDONE		(1 << 2)
+
+	struct	uath_rx_radiotap_header	sc_rxtap;
+	int				sc_rxtap_len;
+	struct	uath_tx_radiotap_header	sc_txtap;
+	int				sc_txtap_len;
+};
+
+#define	UATH_LOCK(sc)			mtx_lock(&(sc)->sc_mtx)
+#define	UATH_UNLOCK(sc)			mtx_unlock(&(sc)->sc_mtx)
+#define	UATH_ASSERT_LOCKED(sc)		mtx_assert(&(sc)->sc_mtx, MA_OWNED)
+
+#define	UATH_RESET_INTRX(sc) do {		\
+	(sc)->sc_intrx_head = NULL;		\
+	(sc)->sc_intrx_tail = NULL;		\
+	(sc)->sc_intrx_nextnum = 0;		\
+	(sc)->sc_intrx_len = 0;			\
+} while (0)
diff --git a/sys/dev/usb/wlan/if_ural.c b/sys/dev/usb/wlan/if_ural.c
index 2844f2c..919b66d 100644
--- a/sys/dev/usb/wlan/if_ural.c
+++ b/sys/dev/usb/wlan/if_ural.c
@@ -339,18 +339,18 @@ static const struct usb2_config ural_config[URAL_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = (RAL_FRAME_SIZE + RAL_TX_DESC_SIZE + 4),
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = ural_bulk_write_callback,
-		.mh.timeout = 5000,	/* ms */
+		.bufsize = (RAL_FRAME_SIZE + RAL_TX_DESC_SIZE + 4),
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = ural_bulk_write_callback,
+		.timeout = 5000,	/* ms */
 	},
 	[URAL_BULK_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = (RAL_FRAME_SIZE + RAL_RX_DESC_SIZE),
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = ural_bulk_read_callback,
+		.bufsize = (RAL_FRAME_SIZE + RAL_RX_DESC_SIZE),
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = ural_bulk_read_callback,
 	},
 };
 
@@ -485,7 +485,6 @@ ural_attach_post(struct usb2_proc_msg *pm)
 
 	ic->ic_ifp = ifp;
 	ic->ic_phytype = IEEE80211_T_OFDM; /* not only, but not used */
-	IEEE80211_ADDR_COPY(ic->ic_myaddr, sc->sc_bssid);
 
 	/* set device capabilities */
 	ic->ic_caps =
@@ -507,7 +506,7 @@ ural_attach_post(struct usb2_proc_msg *pm)
 		setbit(&bands, IEEE80211_MODE_11A);
 	ieee80211_init_channels(ic, NULL, &bands);
 
-	ieee80211_ifattach(ic);
+	ieee80211_ifattach(ic, sc->sc_bssid);
 	ic->ic_update_promisc = ural_update_promisc;
 	ic->ic_newassoc = ural_newassoc;
 	ic->ic_raw_xmit = ural_raw_xmit;
@@ -519,8 +518,6 @@ ural_attach_post(struct usb2_proc_msg *pm)
 	ic->ic_vap_create = ural_vap_create;
 	ic->ic_vap_delete = ural_vap_delete;
 
-	sc->sc_rates = ieee80211_get_ratetable(ic->ic_curchan);
-
 	bpfattach(ifp, DLT_IEEE802_11_RADIO,
 	    sizeof (struct ieee80211_frame) + sizeof(sc->sc_txtap));
 
@@ -1109,7 +1106,7 @@ ural_setup_tx_desc(struct ural_softc *sc, struct ural_tx_desc *desc,
 	desc->plcp_service = 4;
 
 	len += IEEE80211_CRC_LEN;
-	if (ieee80211_rate2phytype(sc->sc_rates, rate) == IEEE80211_T_OFDM) {
+	if (ieee80211_rate2phytype(ic->ic_rt, rate) == IEEE80211_T_OFDM) {
 		desc->flags |= htole32(RAL_TX_OFDM);
 
 		plcp_length = len & 0xfff;
@@ -1210,7 +1207,7 @@ ural_tx_mgt(struct ural_softc *sc, struct mbuf *m0, struct ieee80211_node *ni)
 	if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) {
 		flags |= RAL_TX_ACK;
 
-		dur = ieee80211_ack_duration(sc->sc_rates, tp->mgmtrate, 
+		dur = ieee80211_ack_duration(ic->ic_rt, tp->mgmtrate, 
 		    ic->ic_flags & IEEE80211_F_SHPREAMBLE);
 		*(uint16_t *)wh->i_dur = htole16(dur);
 
@@ -1250,16 +1247,16 @@ ural_sendprot(struct ural_softc *sc,
 	wh = mtod(m, const struct ieee80211_frame *);
 	pktlen = m->m_pkthdr.len + IEEE80211_CRC_LEN;
 
-	protrate = ieee80211_ctl_rate(sc->sc_rates, rate);
-	ackrate = ieee80211_ack_rate(sc->sc_rates, rate);
+	protrate = ieee80211_ctl_rate(ic->ic_rt, rate);
+	ackrate = ieee80211_ack_rate(ic->ic_rt, rate);
 
 	isshort = (ic->ic_flags & IEEE80211_F_SHPREAMBLE) != 0;
-	dur = ieee80211_compute_duration(sc->sc_rates, pktlen, rate, isshort);
-	    + ieee80211_ack_duration(sc->sc_rates, rate, isshort);
+	dur = ieee80211_compute_duration(ic->ic_rt, pktlen, rate, isshort);
+	    + ieee80211_ack_duration(ic->ic_rt, rate, isshort);
 	flags = RAL_TX_RETRY(7);
 	if (prot == IEEE80211_PROT_RTSCTS) {
 		/* NB: CTS is the same size as an ACK */
-		dur += ieee80211_ack_duration(sc->sc_rates, rate, isshort);
+		dur += ieee80211_ack_duration(ic->ic_rt, rate, isshort);
 		flags |= RAL_TX_ACK;
 		mprot = ieee80211_alloc_rts(ic, wh->i_addr1, wh->i_addr2, dur);
 	} else {
@@ -1377,7 +1374,7 @@ ural_tx_data(struct ural_softc *sc, struct mbuf *m0, struct ieee80211_node *ni)
 		if (m0->m_pkthdr.len + IEEE80211_CRC_LEN > vap->iv_rtsthreshold)
 			prot = IEEE80211_PROT_RTSCTS;
 		else if ((ic->ic_flags & IEEE80211_F_USEPROT) &&
-		    ieee80211_rate2phytype(sc->sc_rates, rate) == IEEE80211_T_OFDM)
+		    ieee80211_rate2phytype(ic->ic_rt, rate) == IEEE80211_T_OFDM)
 			prot = ic->ic_protmode;
 		if (prot != IEEE80211_PROT_NONE) {
 			error = ural_sendprot(sc, m0, ni, prot, rate);
@@ -1401,7 +1398,7 @@ ural_tx_data(struct ural_softc *sc, struct mbuf *m0, struct ieee80211_node *ni)
 		flags |= RAL_TX_ACK;
 		flags |= RAL_TX_RETRY(7);
 
-		dur = ieee80211_ack_duration(sc->sc_rates, rate, 
+		dur = ieee80211_ack_duration(ic->ic_rt, rate, 
 		    ic->ic_flags & IEEE80211_F_SHPREAMBLE);
 		*(uint16_t *)wh->i_dur = htole16(dur);
 	}
@@ -1439,12 +1436,6 @@ ural_start(struct ifnet *ifp)
 			break;
 		}
 		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
-		m = ieee80211_encap(ni, m);
-		if (m == NULL) {
-			ieee80211_free_node(ni);
-			ifp->if_oerrors++;
-			continue;
-		}
 		if (ural_tx_data(sc, m, ni) != 0) {
 			ieee80211_free_node(ni);
 			ifp->if_oerrors++;
@@ -1743,8 +1734,6 @@ ural_set_channel(struct ieee80211com *ic)
 	sc->sc_scan_action = URAL_SET_CHANNEL;
 	ural_queue_command(sc, ural_scantask,
 	    &sc->sc_scantask[0].hdr, &sc->sc_scantask[1].hdr);
-
-	sc->sc_rates = ieee80211_get_ratetable(ic->ic_curchan);
 	RAL_UNLOCK(sc);
 }
 
@@ -2216,8 +2205,7 @@ ural_init_task(struct usb2_proc_msg *pm)
 	ural_set_txantenna(sc, sc->tx_ant);
 	ural_set_rxantenna(sc, sc->rx_ant);
 
-	IEEE80211_ADDR_COPY(ic->ic_myaddr, IF_LLADDR(ifp));
-	ural_set_macaddr(sc, ic->ic_myaddr);
+	ural_set_macaddr(sc, IF_LLADDR(ifp));
 
 	/*
 	 * Allocate Tx and Rx xfer queues.
diff --git a/sys/dev/usb/wlan/if_uralvar.h b/sys/dev/usb/wlan/if_uralvar.h
index 5e72339..13a3df8 100644
--- a/sys/dev/usb/wlan/if_uralvar.h
+++ b/sys/dev/usb/wlan/if_uralvar.h
@@ -106,8 +106,6 @@ struct ural_softc {
 	struct usb2_device		*sc_udev;
 	struct usb2_process		sc_tq;
 
-	const struct ieee80211_rate_table *sc_rates;
-
 	uint32_t			asic_rev;
 	uint8_t				rf_rev;
 
diff --git a/sys/dev/usb/wlan/if_zyd.c b/sys/dev/usb/wlan/if_zyd.c
index fa9580a..6858d55 100644
--- a/sys/dev/usb/wlan/if_zyd.c
+++ b/sys/dev/usb/wlan/if_zyd.c
@@ -232,38 +232,38 @@ static const struct usb2_config zyd_config[ZYD_N_TRANSFER] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = ZYD_MAX_TXBUFSZ,
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = zyd_bulk_write_callback,
+		.bufsize = ZYD_MAX_TXBUFSZ,
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = zyd_bulk_write_callback,
 		.ep_index = 0,
-		.mh.timeout = 10000,	/* 10 seconds */
+		.timeout = 10000,	/* 10 seconds */
 	},
 	[ZYD_BULK_RD] = {
 		.type = UE_BULK,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = ZYX_MAX_RXBUFSZ,
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = zyd_bulk_read_callback,
+		.bufsize = ZYX_MAX_RXBUFSZ,
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = zyd_bulk_read_callback,
 		.ep_index = 0,
 	},
 	[ZYD_INTR_WR] = {
 		.type = UE_BULK_INTR,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_OUT,
-		.mh.bufsize = sizeof(struct zyd_cmd),
-		.mh.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
-		.mh.callback = zyd_intr_write_callback,
-		.mh.timeout = 1000,	/* 1 second */
+		.bufsize = sizeof(struct zyd_cmd),
+		.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+		.callback = zyd_intr_write_callback,
+		.timeout = 1000,	/* 1 second */
 		.ep_index = 1,
 	},
 	[ZYD_INTR_RD] = {
 		.type = UE_INTERRUPT,
 		.endpoint = UE_ADDR_ANY,
 		.direction = UE_DIR_IN,
-		.mh.bufsize = sizeof(struct zyd_cmd),
-		.mh.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
-		.mh.callback = zyd_intr_read_callback,
+		.bufsize = sizeof(struct zyd_cmd),
+		.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+		.callback = zyd_intr_read_callback,
 	},
 };
 #define zyd_read16_m(sc, val, data)	do {				\
@@ -396,7 +396,6 @@ zyd_attach_post(struct usb2_proc_msg *pm)
 	ic->ic_ifp = ifp;
 	ic->ic_phytype = IEEE80211_T_OFDM;	/* not only, but not used */
 	ic->ic_opmode = IEEE80211_M_STA;
-	IEEE80211_ADDR_COPY(ic->ic_myaddr, sc->sc_bssid);
 
 	/* set device capabilities */
 	ic->ic_caps =
@@ -413,7 +412,7 @@ zyd_attach_post(struct usb2_proc_msg *pm)
 	setbit(&bands, IEEE80211_MODE_11G);
 	ieee80211_init_channels(ic, NULL, &bands);
 
-	ieee80211_ifattach(ic);
+	ieee80211_ifattach(ic, sc->sc_bssid);
 	ic->ic_newassoc = zyd_newassoc;
 	ic->ic_raw_xmit = zyd_raw_xmit;
 	ic->ic_node_alloc = zyd_node_alloc;
@@ -2698,12 +2697,6 @@ zyd_start(struct ifnet *ifp)
 			break;
 		}
 		ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
-		m = ieee80211_encap(ni, m);
-		if (m == NULL) {
-			ieee80211_free_node(ni);
-			ifp->if_oerrors++;
-			continue;
-		}
 		if (zyd_tx_data(sc, m, ni) != 0) {
 			ieee80211_free_node(ni);
 			ifp->if_oerrors++;
@@ -2859,10 +2852,9 @@ zyd_init_task(struct usb2_proc_msg *pm)
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 		zyd_stop_task(pm);
 
-	IEEE80211_ADDR_COPY(ic->ic_myaddr, IF_LLADDR(ifp));
-	DPRINTF(sc, ZYD_DEBUG_INIT, "setting MAC address to %s\n",
-	    ether_sprintf(ic->ic_myaddr));
-	error = zyd_set_macaddr(sc, ic->ic_myaddr);
+	DPRINTF(sc, ZYD_DEBUG_INIT, "setting MAC address to %6D\n",
+	    IF_LLADDR(ifp), ":");
+	error = zyd_set_macaddr(sc, IF_LLADDR(ifp));
 	if (error != 0)
 		return;
 
diff --git a/sys/dev/usb/wlan/usb_wlan.h b/sys/dev/usb/wlan/usb_wlan.h
index 9db120e..aa0f1c4 100644
--- a/sys/dev/usb/wlan/usb_wlan.h
+++ b/sys/dev/usb/wlan/usb_wlan.h
@@ -46,7 +46,6 @@
 #include <net80211/ieee80211_radiotap.h>
 #include <net80211/ieee80211_amrr.h>
 #include <net80211/ieee80211_regdomain.h>
-#include <net80211/ieee80211_phy.h>
 
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
diff --git a/sys/dev/wi/if_wi.c b/sys/dev/wi/if_wi.c
index 62b7114..a4a74e4 100644
--- a/sys/dev/wi/if_wi.c
+++ b/sys/dev/wi/if_wi.c
@@ -89,6 +89,7 @@ __FBSDID("$FreeBSD$");
 #include <net/if_arp.h>
 #include <net/ethernet.h>
 #include <net/if_dl.h>
+#include <net/if_llc.h>
 #include <net/if_media.h>
 #include <net/if_types.h>
 
@@ -245,6 +246,7 @@ wi_attach(device_t dev)
 		0x00, 0x00, 0x00, 0x00, 0x00, 0x00
 	};
 	int error;
+	uint8_t macaddr[IEEE80211_ADDR_LEN];
 
 	ifp = sc->sc_ifp = if_alloc(IFT_IEEE80211);
 	if (ifp == NULL) {
@@ -312,12 +314,12 @@ wi_attach(device_t dev)
 	 * the probe to fail.
 	 */
 	buflen = IEEE80211_ADDR_LEN;
-	error = wi_read_rid(sc, WI_RID_MAC_NODE, ic->ic_myaddr, &buflen);
+	error = wi_read_rid(sc, WI_RID_MAC_NODE, macaddr, &buflen);
 	if (error != 0) {
 		buflen = IEEE80211_ADDR_LEN;
-		error = wi_read_rid(sc, WI_RID_MAC_NODE, ic->ic_myaddr, &buflen);
+		error = wi_read_rid(sc, WI_RID_MAC_NODE, macaddr, &buflen);
 	}
-	if (error || IEEE80211_ADDR_EQ(ic->ic_myaddr, empty_macaddr)) {
+	if (error || IEEE80211_ADDR_EQ(macaddr, empty_macaddr)) {
 		if (error != 0)
 			device_printf(dev, "mac read failed %d\n", error);
 		else {
@@ -451,7 +453,7 @@ wi_attach(device_t dev)
 	sc->sc_portnum = WI_DEFAULT_PORT;
 	TASK_INIT(&sc->sc_oor_task, 0, wi_status_oor, ic);
 
-	ieee80211_ifattach(ic);
+	ieee80211_ifattach(ic, macaddr);
 	ic->ic_raw_xmit = wi_raw_xmit;
 	ic->ic_scan_start = wi_scan_start;
 	ic->ic_scan_end = wi_scan_end;
@@ -690,7 +692,6 @@ static void
 wi_init_locked(struct wi_softc *sc)
 {
 	struct ifnet *ifp = sc->sc_ifp;
-	struct ieee80211com *ic = ifp->if_l2com;
 	int wasenabled;
 
 	WI_LOCK_ASSERT(sc);
@@ -699,8 +700,7 @@ wi_init_locked(struct wi_softc *sc)
 	if (wasenabled)
 		wi_stop_locked(sc, 1);
 
-	IEEE80211_ADDR_COPY(ic->ic_myaddr, IF_LLADDR(ifp));
-	if (wi_setup_locked(sc, sc->sc_porttype, 3, ic->ic_myaddr) != 0) {
+	if (wi_setup_locked(sc, sc->sc_porttype, 3, IF_LLADDR(ifp)) != 0) {
 		if_printf(ifp, "interface not running\n");
 		wi_stop_locked(sc, 1);
 		return;
@@ -979,6 +979,7 @@ wi_start_locked(struct ifnet *ifp)
 	struct mbuf *m0;
 	struct ieee80211_key *k;
 	struct wi_frame frmhdr;
+	const struct llc *llc;
 	int cur;
 
 	WI_LOCK_ASSERT(sc);
@@ -997,19 +998,33 @@ wi_start_locked(struct ifnet *ifp)
 			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 			break;
 		}
-		/* NB: copy before 802.11 header is prepended */
-		m_copydata(m0, 0, ETHER_HDR_LEN, 
-		    (caddr_t)&frmhdr.wi_ehdr);
-
 		ni = (struct ieee80211_node *) m0->m_pkthdr.rcvif;
-		m0 = ieee80211_encap(ni, m0);
-		if (m0 == NULL) {
-			ifp->if_oerrors++;
-			ieee80211_free_node(ni);
-			continue;
-		}
 
+		/* reconstruct 802.3 header */
 		wh = mtod(m0, struct ieee80211_frame *);
+		switch (wh->i_fc[1]) {
+		case IEEE80211_FC1_DIR_TODS:
+			IEEE80211_ADDR_COPY(frmhdr.wi_ehdr.ether_shost,
+			    wh->i_addr2);
+			IEEE80211_ADDR_COPY(frmhdr.wi_ehdr.ether_dhost,
+			    wh->i_addr3);
+			break;
+		case IEEE80211_FC1_DIR_NODS:
+			IEEE80211_ADDR_COPY(frmhdr.wi_ehdr.ether_shost,
+			    wh->i_addr2);
+			IEEE80211_ADDR_COPY(frmhdr.wi_ehdr.ether_dhost,
+			    wh->i_addr1);
+			break;
+		case IEEE80211_FC1_DIR_FROMDS:
+			IEEE80211_ADDR_COPY(frmhdr.wi_ehdr.ether_shost,
+			    wh->i_addr3);
+			IEEE80211_ADDR_COPY(frmhdr.wi_ehdr.ether_dhost,
+			    wh->i_addr1);
+			break;
+		}
+		llc = (const struct llc *)(
+		    mtod(m0, const uint8_t *) + ieee80211_hdrsize(wh));
+		frmhdr.wi_ehdr.ether_type = llc->llc_snap.ether_type;
 		frmhdr.wi_tx_ctl = htole16(WI_ENC_TX_802_11|WI_TXCNTL_TX_EX);
 		if (wh->i_fc[1] & IEEE80211_FC1_WEP) {
 			k = ieee80211_crypto_encap(ni, m0);
@@ -1504,6 +1519,7 @@ wi_status_connected(void *arg, int pending)
 	struct ieee80211com *ic = vap->iv_ic;
 
 	IEEE80211_LOCK(ic);
+	vap->iv_bss->ni_associd = 1 | 0xc000;	/* NB: anything will do */
 	WI_VAP(vap)->wv_newstate(vap, IEEE80211_S_RUN, 0);
 	if (vap->iv_newstate_cb != NULL)
 		vap->iv_newstate_cb(vap, IEEE80211_S_RUN, 0);
@@ -1516,6 +1532,7 @@ wi_status_disconnected(void *arg, int pending)
 	struct ieee80211vap *vap = arg;
 
 	if (vap->iv_state == IEEE80211_S_RUN) {
+		vap->iv_bss->ni_associd = 0;
 		vap->iv_stats.is_rx_deauth++;
 		ieee80211_new_state(vap, IEEE80211_S_SCAN, 0);
 	}
diff --git a/sys/dev/wpi/if_wpi.c b/sys/dev/wpi/if_wpi.c
index 776d089..1e980aa 100644
--- a/sys/dev/wpi/if_wpi.c
+++ b/sys/dev/wpi/if_wpi.c
@@ -211,7 +211,8 @@ static void	wpi_set_channel(struct ieee80211com *);
 static void	wpi_scan_curchan(struct ieee80211_scan_state *, unsigned long);
 static void	wpi_scan_mindwell(struct ieee80211_scan_state *);
 static int	wpi_ioctl(struct ifnet *, u_long, caddr_t);
-static void	wpi_read_eeprom(struct wpi_softc *);
+static void	wpi_read_eeprom(struct wpi_softc *,
+		    uint8_t macaddr[IEEE80211_ADDR_LEN]);
 static void	wpi_read_eeprom_channels(struct wpi_softc *, int);
 static void	wpi_read_eeprom_group(struct wpi_softc *, int);
 static int	wpi_cmd(struct wpi_softc *, int, const void *, int, int);
@@ -493,6 +494,7 @@ wpi_attach(device_t dev)
 	int ac, error, supportsa = 1;
 	uint32_t tmp;
 	const struct wpi_ident *ident;
+	uint8_t macaddr[IEEE80211_ADDR_LEN];
 
 	sc->sc_dev = dev;
 
@@ -650,7 +652,7 @@ wpi_attach(device_t dev)
 	 * Read in the eeprom and also setup the channels for
 	 * net80211. We don't set the rates as net80211 does this for us
 	 */
-	wpi_read_eeprom(sc);
+	wpi_read_eeprom(sc, macaddr);
 
 	if (bootverbose || WPI_DEBUG_SET) {
 	    device_printf(sc->sc_dev, "Regulatory Domain: %.4s\n", sc->domain);
@@ -675,7 +677,7 @@ wpi_attach(device_t dev)
 	ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN;
 	IFQ_SET_READY(&ifp->if_snd);
 
-	ieee80211_ifattach(ic);
+	ieee80211_ifattach(ic, macaddr);
 	/* override default methods */
 	ic->ic_node_alloc = wpi_node_alloc;
 	ic->ic_newassoc = wpi_newassoc;
@@ -1473,6 +1475,20 @@ wpi_rx_intr(struct wpi_softc *sc, struct wpi_rx_desc *desc,
 	    le16toh(head->len), (int8_t)stat->rssi, head->rate, head->chan,
 	    (uintmax_t)le64toh(tail->tstamp)));
 
+	/* discard Rx frames with bad CRC early */
+	if ((le32toh(tail->flags) & WPI_RX_NOERROR) != WPI_RX_NOERROR) {
+		DPRINTFN(WPI_DEBUG_RX, ("%s: rx flags error %x\n", __func__,
+		    le32toh(tail->flags)));
+		ifp->if_ierrors++;
+		return;
+	}
+	if (le16toh(head->len) < sizeof (struct ieee80211_frame)) {
+		DPRINTFN(WPI_DEBUG_RX, ("%s: frame too short: %d\n", __func__,
+		    le16toh(head->len)));
+		ifp->if_ierrors++;
+		return;
+	}
+
 	/* XXX don't need mbuf, just dma buffer */
 	mnew = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR, MJUMPAGESIZE);
 	if (mnew == NULL) {
@@ -1573,7 +1589,7 @@ wpi_tx_intr(struct wpi_softc *sc, struct wpi_rx_desc *desc)
 	 */
 	wn->amn.amn_txcnt++;
 	if (stat->ntries > 0) {
-		DPRINTFN(3, ("%d retries\n", stat->ntries));
+		DPRINTFN(WPI_DEBUG_TX, ("%d retries\n", stat->ntries));
 		wn->amn.amn_retrycnt++;
 	}
 
@@ -2029,10 +2045,9 @@ wpi_start_locked(struct ifnet *ifp)
 		return;
 
 	for (;;) {
-		IFQ_POLL(&ifp->if_snd, m);
+		IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
 		if (m == NULL)
 			break;
-		/* no QoS encapsulation for EAPOL frames */
 		ac = M_WME_GETAC(m);
 		if (sc->txq[ac].queued > sc->txq[ac].count - 8) {
 			/* there is no place left in this ring */
@@ -2040,14 +2055,7 @@ wpi_start_locked(struct ifnet *ifp)
 			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 			break;
 		}
-		IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
 		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
-		m = ieee80211_encap(ni, m);
-		if (m == NULL) {
-			ieee80211_free_node(ni);
-			ifp->if_oerrors++;
-			continue;
-		}
 		if (wpi_tx_data(sc, m, ni, ac) != 0) {
 			ieee80211_free_node(ni);
 			ifp->if_oerrors++;
@@ -2137,10 +2145,8 @@ wpi_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
  * Extract various information from EEPROM.
  */
 static void
-wpi_read_eeprom(struct wpi_softc *sc)
+wpi_read_eeprom(struct wpi_softc *sc, uint8_t macaddr[IEEE80211_ADDR_LEN])
 {
-	struct ifnet *ifp = sc->sc_ifp;
-	struct ieee80211com *ic = ifp->if_l2com;
 	int i;
 
 	/* read the hardware capabilities, revision and SKU type */
@@ -2152,7 +2158,7 @@ wpi_read_eeprom(struct wpi_softc *sc)
 	wpi_read_prom_data(sc, WPI_EEPROM_DOMAIN, sc->domain, 4);
 
 	/* read in the hw MAC address */
-	wpi_read_prom_data(sc, WPI_EEPROM_MAC, ic->ic_myaddr, 6);
+	wpi_read_prom_data(sc, WPI_EEPROM_MAC, macaddr, 6);
 
 	/* read the list of authorized channels */
 	for (i = 0; i < WPI_CHAN_BANDS_COUNT; i++)
@@ -2625,7 +2631,7 @@ wpi_scan(struct wpi_softc *sc)
 		IEEE80211_FC0_SUBTYPE_PROBE_REQ;
 	wh->i_fc[1] = IEEE80211_FC1_DIR_NODS;
 	IEEE80211_ADDR_COPY(wh->i_addr1, ifp->if_broadcastaddr);
-	IEEE80211_ADDR_COPY(wh->i_addr2, ic->ic_myaddr);
+	IEEE80211_ADDR_COPY(wh->i_addr2, IF_LLADDR(ifp));
 	IEEE80211_ADDR_COPY(wh->i_addr3, ifp->if_broadcastaddr);
 	*(u_int16_t *)&wh->i_dur[0] = 0;	/* filled by h/w */
 	*(u_int16_t *)&wh->i_seq[0] = 0;	/* filled by h/w */
@@ -2797,7 +2803,7 @@ wpi_config(struct wpi_softc *sc)
 
 	/* configure adapter */
 	memset(&sc->config, 0, sizeof (struct wpi_config));
-	IEEE80211_ADDR_COPY(sc->config.myaddr, ic->ic_myaddr);
+	IEEE80211_ADDR_COPY(sc->config.myaddr, IF_LLADDR(ifp));
 	/*set default channel*/
 	sc->config.chan = htole16(ieee80211_chan2ieee(ic, ic->ic_curchan));
 	sc->config.flags = htole32(WPI_CONFIG_TSF);
diff --git a/sys/dev/wpi/if_wpireg.h b/sys/dev/wpi/if_wpireg.h
index 60692a2..2ccc21f 100644
--- a/sys/dev/wpi/if_wpireg.h
+++ b/sys/dev/wpi/if_wpireg.h
@@ -235,12 +235,10 @@ struct wpi_rx_head {
 
 struct wpi_rx_tail {
 	uint32_t	flags;
-#if 0
 #define WPI_RX_NO_CRC_ERR	(1 << 0)
 #define WPI_RX_NO_OVFL_ERR	(1 << 1)
 /* shortcut for the above */
 #define WPI_RX_NOERROR		(WPI_RX_NO_CRC_ERR | WPI_RX_NO_OVFL_ERR)
-#endif
 	uint64_t	tstamp;
 	uint32_t	tbeacon;
 } __packed;
diff --git a/sys/dev/xen/balloon/balloon.c b/sys/dev/xen/balloon/balloon.c
index c23433c..6948173 100644
--- a/sys/dev/xen/balloon/balloon.c
+++ b/sys/dev/xen/balloon/balloon.c
@@ -167,6 +167,8 @@ minimum_target(void)
 {
 #ifdef XENHVM
 #define max_pfn physmem
+#else
+#define max_pfn HYPERVISOR_shared_info->arch.max_pfn
 #endif
 	unsigned long min_pages, curr_pages = current_target();
 
@@ -256,6 +258,7 @@ increase_reservation(unsigned long nr_pages)
 
 		set_phys_to_machine(pfn, frame_list[i]);
 
+#if 0
 #ifndef XENHVM
 		/* Link back into the page tables if not highmem. */
 		if (pfn < max_low_pfn) {
@@ -268,6 +271,7 @@ increase_reservation(unsigned long nr_pages)
 			    ("HYPERVISOR_update_va_mapping failed"));
 		}
 #endif
+#endif
 
 		/* Relinquish the page back to the allocator. */
 		vm_page_unwire(page, 0);
@@ -447,6 +451,9 @@ balloon_init(void *arg)
 {
 #ifndef XENHVM
 	vm_page_t page;
+	unsigned long pfn;
+
+#define max_pfn HYPERVISOR_shared_info->arch.max_pfn
 #endif
 
 	if (!is_running_on_xen())
@@ -477,6 +484,7 @@ balloon_init(void *arg)
 		page = PHYS_TO_VM_PAGE(pfn << PAGE_SHIFT);
 		balloon_append(page);
 	}
+#undef max_pfn
 #endif
 
 	target_watch.callback = watch_target;
diff --git a/sys/dev/xen/console/console.c b/sys/dev/xen/console/console.c
index 0634dad..ac8ebac 100644
--- a/sys/dev/xen/console/console.c
+++ b/sys/dev/xen/console/console.c
@@ -225,7 +225,6 @@ static int
 xc_attach(device_t dev) 
 {
 	int error;
-	struct xc_softc *sc = (struct xc_softc *)device_get_softc(dev);
 
 	if (xen_start_info->flags & SIF_INITDOMAIN) {
 		xc_consdev.cn_putc = xccnputc_dom0;
@@ -247,7 +246,7 @@ xc_attach(device_t dev)
 				 0,
 				 "console",
 				 NULL,
-				 xencons_priv_interrupt,
+				 xencons_priv_interrupt, NULL,
 				 INTR_TYPE_TTY, NULL);
 		
 				KASSERT(error >= 0, ("can't register console interrupt"));
diff --git a/sys/fs/devfs/devfs_vnops.c b/sys/fs/devfs/devfs_vnops.c
index b890da7..215f943 100644
--- a/sys/fs/devfs/devfs_vnops.c
+++ b/sys/fs/devfs/devfs_vnops.c
@@ -1549,7 +1549,6 @@ static struct vop_vector devfs_specops = {
 	.vop_create =		VOP_PANIC,
 	.vop_fsync =		devfs_fsync,
 	.vop_getattr =		devfs_getattr,
-	.vop_lease =		VOP_NULL,
 	.vop_link =		VOP_PANIC,
 	.vop_mkdir =		VOP_PANIC,
 	.vop_mknod =		VOP_PANIC,
diff --git a/sys/fs/fifofs/fifo_vnops.c b/sys/fs/fifofs/fifo_vnops.c
index c8bc5ab4..66963bc 100644
--- a/sys/fs/fifofs/fifo_vnops.c
+++ b/sys/fs/fifofs/fifo_vnops.c
@@ -118,7 +118,6 @@ struct vop_vector fifo_specops = {
 	.vop_getattr =		VOP_EBADF,
 	.vop_ioctl =		fifo_ioctl,
 	.vop_kqfilter =		fifo_kqfilter,
-	.vop_lease =		VOP_NULL,
 	.vop_link =		VOP_PANIC,
 	.vop_mkdir =		VOP_PANIC,
 	.vop_mknod =		VOP_PANIC,
diff --git a/sys/fs/nullfs/null_vnops.c b/sys/fs/nullfs/null_vnops.c
index 38da43f..3434deb 100644
--- a/sys/fs/nullfs/null_vnops.c
+++ b/sys/fs/nullfs/null_vnops.c
@@ -614,14 +614,6 @@ null_unlock(struct vop_unlock_args *ap)
 	return (error);
 }
 
-static int
-null_islocked(struct vop_islocked_args *ap)
-{
-	struct vnode *vp = ap->a_vp;
-
-	return (lockstatus(vp->v_vnlock));
-}
-
 /*
  * There is no way to tell that someone issued remove/rmdir operation
  * on the underlying filesystem. For now we just have to release lowervp
@@ -732,7 +724,7 @@ struct vop_vector null_vnodeops = {
 	.vop_getattr =		null_getattr,
 	.vop_getwritemount =	null_getwritemount,
 	.vop_inactive =		null_inactive,
-	.vop_islocked =		null_islocked,
+	.vop_islocked =		vop_stdislocked,
 	.vop_lock1 =		null_lock,
 	.vop_lookup =		null_lookup,
 	.vop_open =		null_open,
diff --git a/sys/fs/pseudofs/pseudofs_vnops.c b/sys/fs/pseudofs/pseudofs_vnops.c
index be86f4e..cf2d3aa 100644
--- a/sys/fs/pseudofs/pseudofs_vnops.c
+++ b/sys/fs/pseudofs/pseudofs_vnops.c
@@ -52,6 +52,20 @@ __FBSDID("$FreeBSD$");
 #include <fs/pseudofs/pseudofs.h>
 #include <fs/pseudofs/pseudofs_internal.h>
 
+#define KASSERT_PN_IS_DIR(pn)						\
+	KASSERT((pn)->pn_type == pfstype_root ||			\
+	    (pn)->pn_type == pfstype_dir ||				\
+	    (pn)->pn_type == pfstype_procdir,				\
+	    ("%s(): VDIR vnode refers to non-directory pfs_node", __func__))
+
+#define KASSERT_PN_IS_FILE(pn)						\
+	KASSERT((pn)->pn_type == pfstype_file,				\
+	    ("%s(): VREG vnode refers to non-file pfs_node", __func__))
+
+#define KASSERT_PN_IS_LINK(pn)						\
+	KASSERT((pn)->pn_type == pfstype_symlink,			\
+	    ("%s(): VLNK vnode refers to non-link pfs_node", __func__))
+
 /*
  * Returns the fileno, adjusted for target pid
  */
@@ -257,6 +271,7 @@ pfs_ioctl(struct vop_ioctl_args *va)
 
 	if (vn->v_type != VREG)
 		PFS_RETURN (EINVAL);
+	KASSERT_PN_IS_FILE(pn);
 
 	if (pn->pn_ioctl == NULL)
 		PFS_RETURN (ENOTTY);
@@ -411,6 +426,7 @@ pfs_lookup(struct vop_cachedlookup_args *va)
 
 	if (vn->v_type != VDIR)
 		PFS_RETURN (ENOTDIR);
+	KASSERT_PN_IS_DIR(pd);
 
 	error = VOP_ACCESS(vn, VEXEC, cnp->cn_cred, cnp->cn_thread);
 	if (error)
@@ -565,6 +581,7 @@ pfs_read(struct vop_read_args *va)
 
 	if (vn->v_type != VREG)
 		PFS_RETURN (EINVAL);
+	KASSERT_PN_IS_FILE(pn);
 
 	if (!(pn->pn_flags & PFS_RD))
 		PFS_RETURN (EBADF);
@@ -707,6 +724,7 @@ pfs_readdir(struct vop_readdir_args *va)
 
 	if (vn->v_type != VDIR)
 		PFS_RETURN (ENOTDIR);
+	KASSERT_PN_IS_DIR(pd);
 	uio = va->a_uio;
 
 	/* only allow reading entire entries */
@@ -815,6 +833,7 @@ pfs_readlink(struct vop_readlink_args *va)
 
 	if (vn->v_type != VLNK)
 		PFS_RETURN (EINVAL);
+	KASSERT_PN_IS_LINK(pn);
 
 	if (pn->pn_fill == NULL)
 		PFS_RETURN (EIO);
@@ -900,8 +919,7 @@ pfs_write(struct vop_write_args *va)
 
 	if (vn->v_type != VREG)
 		PFS_RETURN (EINVAL);
-	KASSERT(pn->pn_type != pfstype_file,
-	    ("%s(): VREG vnode refers to non-file pfs_node", __func__));
+	KASSERT_PN_IS_FILE(pn);
 
 	if (!(pn->pn_flags & PFS_WR))
 		PFS_RETURN (EBADF);
@@ -921,9 +939,7 @@ pfs_write(struct vop_write_args *va)
 	}
 
 	if (pn->pn_flags & PFS_RAWWR) {
-		pfs_lock(pn);
 		error = pn_fill(curthread, proc, pn, NULL, uio);
-		pfs_unlock(pn);
 		if (proc != NULL)
 			PRELE(proc);
 		PFS_RETURN (error);
diff --git a/sys/fs/unionfs/union_subr.c b/sys/fs/unionfs/union_subr.c
index beaef2f..69165e3 100644
--- a/sys/fs/unionfs/union_subr.c
+++ b/sys/fs/unionfs/union_subr.c
@@ -805,10 +805,6 @@ unionfs_mkshadowdir(struct unionfs_mount *ump, struct vnode *udvp,
 
 	if ((error = vn_start_write(udvp, &mp, V_WAIT | PCATCH)))
 		goto unionfs_mkshadowdir_free_out;
-	if ((error = VOP_LEASE(udvp, td, cn.cn_cred, LEASE_WRITE))) {
-		vn_finished_write(mp);
-		goto unionfs_mkshadowdir_free_out;
-	}
 	unionfs_create_uppervattr_core(ump, &lva, &va, td);
 
 	error = VOP_MKDIR(udvp, &uvp, &cn, &va);
@@ -874,8 +870,7 @@ unionfs_mkwhiteout(struct vnode *dvp, struct componentname *cnp,
 
 	if ((error = vn_start_write(dvp, &mp, V_WAIT | PCATCH)))
 		goto unionfs_mkwhiteout_free_out;
-	if (!(error = VOP_LEASE(dvp, td, td->td_ucred, LEASE_WRITE)))
-		error = VOP_WHITEOUT(dvp, &cn, CREATE);
+	error = VOP_WHITEOUT(dvp, &cn, CREATE);
 
 	vn_finished_write(mp);
 
@@ -949,9 +944,6 @@ unionfs_vn_create_on_upper(struct vnode **vpp, struct vnode *udvp,
 		goto unionfs_vn_create_on_upper_free_out1;
 	}
 
-	if ((error = VOP_LEASE(udvp, td, cred, LEASE_WRITE)) != 0)
-		goto unionfs_vn_create_on_upper_free_out1;
-
 	if ((error = VOP_CREATE(udvp, &vp, &cn, uvap)) != 0)
 		goto unionfs_vn_create_on_upper_free_out1;
 
@@ -999,10 +991,6 @@ unionfs_copyfile_core(struct vnode *lvp, struct vnode *uvp,
 	uio.uio_segflg = UIO_SYSSPACE;
 	uio.uio_offset = 0;
 
-	if ((error = VOP_LEASE(lvp, td, cred, LEASE_READ)) != 0)
-		return (error);
-	if ((error = VOP_LEASE(uvp, td, cred, LEASE_WRITE)) != 0)
-		return (error);
 	buf = malloc(MAXBSIZE, M_TEMP, M_WAITOK);
 
 	while (error == 0) {
diff --git a/sys/fs/unionfs/union_vnops.c b/sys/fs/unionfs/union_vnops.c
index d16851d..8505cac 100644
--- a/sys/fs/unionfs/union_vnops.c
+++ b/sys/fs/unionfs/union_vnops.c
@@ -902,27 +902,6 @@ unionfs_write(struct vop_write_args *ap)
 }
 
 static int
-unionfs_lease(struct vop_lease_args *ap)
-{
-	int error;
-	struct unionfs_node *unp;
-	struct vnode   *vp;
-
-	UNIONFS_INTERNAL_DEBUG("unionfs_lease: enter\n");
-
-	KASSERT_UNIONFS_VNODE(ap->a_vp);
-
-	unp = VTOUNIONFS(ap->a_vp);
-	vp = (unp->un_uppervp != NULLVP ? unp->un_uppervp : unp->un_lowervp);
-
-	error = VOP_LEASE(vp, ap->a_td, ap->a_cred, ap->a_flag);
-
-	UNIONFS_INTERNAL_DEBUG("unionfs_lease: lease (%d)\n", error);
-
-	return (error);
-}
-
-static int
 unionfs_ioctl(struct vop_ioctl_args *ap)
 {
 	int error;
@@ -947,7 +926,7 @@ unionfs_ioctl(struct vop_ioctl_args *ap)
 	error = VOP_IOCTL(ovp, ap->a_command, ap->a_data, ap->a_fflag,
 	    ap->a_cred, ap->a_td);
 
-	UNIONFS_INTERNAL_DEBUG("unionfs_ioctl: lease (%d)\n", error);
+	UNIONFS_INTERNAL_DEBUG("unionfs_ioctl: leave (%d)\n", error);
 
 	return (error);
 }
@@ -2461,7 +2440,6 @@ struct vop_vector unionfs_vnodeops = {
 	.vop_getwritemount =	unionfs_getwritemount,
 	.vop_inactive =		unionfs_inactive,
 	.vop_ioctl =		unionfs_ioctl,
-	.vop_lease =		unionfs_lease,
 	.vop_link =		unionfs_link,
 	.vop_listextattr =	unionfs_listextattr,
 	.vop_lock1 =		unionfs_lock,
diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c
index 596b455..2a405c5b 100644
--- a/sys/geom/eli/g_eli.c
+++ b/sys/geom/eli/g_eli.c
@@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/bio.h>
 #include <sys/sysctl.h>
 #include <sys/malloc.h>
+#include <sys/eventhandler.h>
 #include <sys/kthread.h>
 #include <sys/proc.h>
 #include <sys/sched.h>
@@ -81,8 +82,12 @@ TUNABLE_INT("kern.geom.eli.batch", &g_eli_batch);
 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, batch, CTLFLAG_RW, &g_eli_batch, 0,
     "Use crypto operations batching");
 
+static eventhandler_tag g_eli_pre_sync = NULL;
+
 static int g_eli_destroy_geom(struct gctl_req *req, struct g_class *mp,
     struct g_geom *gp);
+static void g_eli_init(struct g_class *mp);
+static void g_eli_fini(struct g_class *mp);
 
 static g_taste_t g_eli_taste;
 static g_dumpconf_t g_eli_dumpconf;
@@ -92,7 +97,9 @@ struct g_class g_eli_class = {
 	.version = G_VERSION,
 	.ctlreq = g_eli_config,
 	.taste = g_eli_taste,
-	.destroy_geom = g_eli_destroy_geom
+	.destroy_geom = g_eli_destroy_geom,
+	.init = g_eli_init,
+	.fini = g_eli_fini
 };
 
 
@@ -996,7 +1003,6 @@ g_eli_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 	/*
 	 * We have correct key, let's attach provider.
 	 */
-	md.md_flags |= G_ELI_FLAG_WO_DETACH;
 	gp = g_eli_create(NULL, mp, pp, &md, mkey, nkey);
 	bzero(mkey, sizeof(mkey));
 	bzero(&md, sizeof(md));
@@ -1076,5 +1082,52 @@ g_eli_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
 	    g_eli_algo2str(sc->sc_ealgo));
 }
 
+static void
+g_eli_shutdown_pre_sync(void *arg, int howto)
+{
+	struct g_class *mp;
+	struct g_geom *gp, *gp2;
+	struct g_provider *pp;
+	struct g_eli_softc *sc;
+	int error;
+
+	mp = arg;
+	DROP_GIANT();
+	g_topology_lock();
+	LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
+		sc = gp->softc;
+		if (sc == NULL)
+			continue;
+		pp = LIST_FIRST(&gp->provider);
+		KASSERT(pp != NULL, ("No provider? gp=%p (%s)", gp, gp->name));
+		if (pp->acr + pp->acw + pp->ace == 0)
+			error = g_eli_destroy(sc, 1);
+		else {
+			sc->sc_flags |= G_ELI_FLAG_RW_DETACH;
+			gp->access = g_eli_access;
+		}
+	}
+	g_topology_unlock();
+	PICKUP_GIANT();
+}
+
+static void
+g_eli_init(struct g_class *mp)
+{
+
+	g_eli_pre_sync = EVENTHANDLER_REGISTER(shutdown_pre_sync,
+	    g_eli_shutdown_pre_sync, mp, SHUTDOWN_PRI_FIRST);
+	if (g_eli_pre_sync == NULL)
+		G_ELI_DEBUG(0, "Warning! Cannot register shutdown event.");
+}
+
+static void
+g_eli_fini(struct g_class *mp)
+{
+
+	if (g_eli_pre_sync != NULL)
+		EVENTHANDLER_DEREGISTER(shutdown_pre_sync, g_eli_pre_sync);
+}
+
 DECLARE_GEOM_CLASS(g_eli_class, g_eli);
 MODULE_DEPEND(g_eli, crypto, 1, 1, 1);
diff --git a/sys/geom/label/g_label.c b/sys/geom/label/g_label.c
index a6685e9..288489a 100644
--- a/sys/geom/label/g_label.c
+++ b/sys/geom/label/g_label.c
@@ -77,7 +77,8 @@ struct g_class g_label_class = {
  * 6. Add your file system to manual page sbin/geom/class/label/glabel.8.
  */
 const struct g_label_desc *g_labels[] = {
-	&g_label_ufs,
+	&g_label_ufs_id,
+	&g_label_ufs_volume,
 	&g_label_iso9660,
 	&g_label_msdosfs,
 	&g_label_ext2fs,
diff --git a/sys/geom/label/g_label.h b/sys/geom/label/g_label.h
index e2d7a5e..e3541a9 100644
--- a/sys/geom/label/g_label.h
+++ b/sys/geom/label/g_label.h
@@ -64,7 +64,8 @@ struct g_label_desc {
 };
 
 /* Supported labels. */
-extern const struct g_label_desc g_label_ufs;
+extern const struct g_label_desc g_label_ufs_id;
+extern const struct g_label_desc g_label_ufs_volume;
 extern const struct g_label_desc g_label_iso9660;
 extern const struct g_label_desc g_label_msdosfs;
 extern const struct g_label_desc g_label_ext2fs;
diff --git a/sys/geom/label/g_label_ufs.c b/sys/geom/label/g_label_ufs.c
index a1801d5..8510fc0 100644
--- a/sys/geom/label/g_label_ufs.c
+++ b/sys/geom/label/g_label_ufs.c
@@ -39,12 +39,16 @@ __FBSDID("$FreeBSD$");
 #include <geom/geom.h>
 #include <geom/label/g_label.h>
 
-#define G_LABEL_UFS_DIR	"ufs"
+#define G_LABEL_UFS_VOLUME_DIR	"ufs"
+#define G_LABEL_UFS_ID_DIR	"ufsid"
+
+#define	G_LABEL_UFS_VOLUME	0
+#define	G_LABEL_UFS_ID		1
 
 static const int superblocks[] = SBLOCKSEARCH;
 
 static void
-g_label_ufs_taste(struct g_consumer *cp, char *label, size_t size)
+g_label_ufs_taste_common(struct g_consumer *cp, char *label, size_t size, int what)
 {
 	struct g_provider *pp;
 	int sb, superblock;
@@ -96,18 +100,50 @@ g_label_ufs_taste(struct g_consumer *cp, char *label, size_t size)
 		}
 		G_LABEL_DEBUG(1, "%s file system detected on %s.",
 		    fs->fs_magic == FS_UFS1_MAGIC ? "UFS1" : "UFS2", pp->name);
-		/* Check for volume label */
-		if (fs->fs_volname[0] == '\0') {
-			g_free(fs);
-			continue;
+		switch (what) {
+		case G_LABEL_UFS_VOLUME:
+			/* Check for volume label */
+			if (fs->fs_volname[0] == '\0') {
+				g_free(fs);
+				continue;
+			}
+			strlcpy(label, fs->fs_volname, size);
+			break;
+		case G_LABEL_UFS_ID:
+			if (fs->fs_id[0] == 0 && fs->fs_id[1] == 0) {
+				g_free(fs);
+				continue;
+			}
+			snprintf(label, size, "%08x%08x", fs->fs_id[0],
+			    fs->fs_id[1]);
+			break;
 		}
-		strlcpy(label, fs->fs_volname, size);
 		g_free(fs);
 		break;
 	}
 }
 
-const struct g_label_desc g_label_ufs = {
-	.ld_taste = g_label_ufs_taste,
-	.ld_dir = G_LABEL_UFS_DIR
+static void
+g_label_ufs_volume_taste(struct g_consumer *cp, char *label, size_t size)
+{
+
+	g_label_ufs_taste_common(cp, label, size, G_LABEL_UFS_VOLUME);
+}
+
+static void
+g_label_ufs_id_taste(struct g_consumer *cp, char *label, size_t size)
+{
+
+	g_label_ufs_taste_common(cp, label, size, G_LABEL_UFS_ID);
+}
+
+
+const struct g_label_desc g_label_ufs_volume = {
+	.ld_taste = g_label_ufs_volume_taste,
+	.ld_dir = G_LABEL_UFS_VOLUME_DIR
+};
+
+const struct g_label_desc g_label_ufs_id = {
+	.ld_taste = g_label_ufs_id_taste,
+	.ld_dir = G_LABEL_UFS_ID_DIR
 };
diff --git a/sys/geom/part/g_part.c b/sys/geom/part/g_part.c
index 2e8bb98..447221a 100644
--- a/sys/geom/part/g_part.c
+++ b/sys/geom/part/g_part.c
@@ -49,6 +49,10 @@ __FBSDID("$FreeBSD$");
 
 #include "g_part_if.h"
 
+#ifndef _PATH_DEV
+#define _PATH_DEV "/dev/"
+#endif
+
 static kobj_method_t g_part_null_methods[] = {
 	{ 0, 0 }
 };
@@ -273,12 +277,17 @@ g_part_new_provider(struct g_geom *gp, struct g_part_table *table,
 }
 
 static int
-g_part_parm_geom(const char *p, struct g_geom **v)
+g_part_parm_geom(const char *rawname, struct g_geom **v)
 {
 	struct g_geom *gp;
+	const char *pname;
 
+	if (strncmp(rawname, _PATH_DEV, strlen(_PATH_DEV)) == 0)
+		pname = rawname + strlen(_PATH_DEV);
+	else
+		pname = rawname;
 	LIST_FOREACH(gp, &g_part_class.geom, geom) {
-		if (!strcmp(p, gp->name))
+		if (!strcmp(pname, gp->name))
 			break;
 	}
 	if (gp == NULL)
@@ -288,11 +297,14 @@ g_part_parm_geom(const char *p, struct g_geom **v)
 }
 
 static int
-g_part_parm_provider(const char *p, struct g_provider **v)
+g_part_parm_provider(const char *pname, struct g_provider **v)
 {
 	struct g_provider *pp;
 
-	pp = g_provider_by_name(p);
+	if (strncmp(pname, _PATH_DEV, strlen(_PATH_DEV)) == 0)
+		pp = g_provider_by_name(pname + strlen(_PATH_DEV));
+	else
+		pp = g_provider_by_name(pname);
 	if (pp == NULL)
 		return (EINVAL);
 	*v = pp;
@@ -1714,7 +1726,10 @@ g_part_start(struct bio *bp)
 			 * used.
 			 */
 			if (!G_PART_DUMPTO(table, entry)) {
-				g_io_deliver(bp, ENXIO);
+				g_io_deliver(bp, ENODEV);
+				printf("GEOM_PART: Partition '%s' not suitable"
+				    " for kernel dumps (wrong type?)\n",
+				    pp->name);
 				return;
 			}
 			gkd = (struct g_kerneldump *)bp->bio_data;
diff --git a/sys/geom/part/g_part_apm.c b/sys/geom/part/g_part_apm.c
index e205da1..030a0e6 100644
--- a/sys/geom/part/g_part_apm.c
+++ b/sys/geom/part/g_part_apm.c
@@ -216,6 +216,11 @@ g_part_apm_create(struct g_part_table *basetable, struct g_part_parms *gpp)
 {
 	struct g_provider *pp;
 	struct g_part_apm_table *table;
+	uint32_t last;
+
+	/* We don't nest, which means that our depth should be 0. */
+	if (basetable->gpt_depth != 0)
+		return (ENXIO);
 
 	table = (struct g_part_apm_table *)basetable;
 	pp = gpp->gpp_provider;
@@ -223,12 +228,15 @@ g_part_apm_create(struct g_part_table *basetable, struct g_part_parms *gpp)
 	    pp->mediasize < (2 + 2 * basetable->gpt_entries) * pp->sectorsize)
 		return (ENOSPC);
 
+	/* APM uses 32-bit LBAs. */
+	last = MIN(pp->mediasize / pp->sectorsize, 0xffffffff) - 1;
+
 	basetable->gpt_first = 2 + basetable->gpt_entries;
-	basetable->gpt_last = (pp->mediasize / pp->sectorsize) - 1;
+	basetable->gpt_last = last;
 
 	table->ddr.ddr_sig = APM_DDR_SIG;
 	table->ddr.ddr_blksize = pp->sectorsize;
-	table->ddr.ddr_blkcount = basetable->gpt_last + 1;
+	table->ddr.ddr_blkcount = last + 1;
 
 	table->self.ent_sig = APM_ENT_SIG;
 	table->self.ent_pmblkcnt = basetable->gpt_entries + 1;
diff --git a/sys/geom/part/g_part_bsd.c b/sys/geom/part/g_part_bsd.c
index 34548b4..b476bfd 100644
--- a/sys/geom/part/g_part_bsd.c
+++ b/sys/geom/part/g_part_bsd.c
@@ -192,8 +192,7 @@ g_part_bsd_create(struct g_part_table *basetable, struct g_part_parms *gpp)
 	struct g_part_bsd_entry *entry;
 	struct g_part_bsd_table *table;
 	u_char *ptr;
-	uint64_t msize;
-	uint32_t ncyls, secpercyl;
+	uint32_t msize, ncyls, secpercyl;
 
 	pp = gpp->gpp_provider;
 	cp = LIST_FIRST(&pp->consumers);
@@ -203,7 +202,7 @@ g_part_bsd_create(struct g_part_table *basetable, struct g_part_parms *gpp)
 	if (BBSIZE % pp->sectorsize)
 		return (ENOTBLK);
 
-	msize = pp->mediasize / pp->sectorsize;
+	msize = MIN(pp->mediasize / pp->sectorsize, 0xffffffff);
 	secpercyl = basetable->gpt_sectors * basetable->gpt_heads;
 	ncyls = msize / secpercyl;
 
diff --git a/sys/geom/part/g_part_ebr.c b/sys/geom/part/g_part_ebr.c
index c307330..0d0b527 100644
--- a/sys/geom/part/g_part_ebr.c
+++ b/sys/geom/part/g_part_ebr.c
@@ -237,7 +237,7 @@ g_part_ebr_create(struct g_part_table *basetable, struct g_part_parms *gpp)
 	char psn[8];
 	struct g_consumer *cp;
 	struct g_provider *pp;
-	uint64_t msize;
+	uint32_t msize;
 	int error;
 
 	pp = gpp->gpp_provider;
@@ -257,10 +257,11 @@ g_part_ebr_create(struct g_part_table *basetable, struct g_part_parms *gpp)
 	if (strcmp(psn, "MBR"))
 		return (ENXIO);
 
-	msize = pp->mediasize / pp->sectorsize;
-	basetable->gpt_entries = msize / basetable->gpt_sectors;
+	msize = MIN(pp->mediasize / pp->sectorsize, 0xffffffff);
+	msize -= msize % basetable->gpt_sectors;
 	basetable->gpt_first = 0;
-	basetable->gpt_last = msize - (msize % basetable->gpt_sectors) - 1;
+	basetable->gpt_last = msize - 1;
+	basetable->gpt_entries = msize / basetable->gpt_sectors;
 	return (0);
 }
 
@@ -343,7 +344,7 @@ g_part_ebr_name(struct g_part_table *table, struct g_part_entry *entry,
     char *buf, size_t bufsz)
 {
 
-	snprintf(buf, bufsz, ".%08u", entry->gpe_index);
+	snprintf(buf, bufsz, "+%08u", entry->gpe_index);
 	return (buf);
 }
 
@@ -402,9 +403,14 @@ g_part_ebr_probe(struct g_part_table *table, struct g_consumer *cp)
 	if (magic != DOSMAGIC)
 		goto out;
 
-	/* The sector is all zeroes, except for the partition entries. */
+	/*
+	 * The sector is all zeroes, except for the partition entries
+	 * and some signatures or disk serial number. Those can be
+	 * found in the 9 bytes immediately in front of the partition
+	 * table.
+	 */
 	sum = 0;
-	for (index = 0; index < DOSPARTOFF; index++)
+	for (index = 0; index < DOSPARTOFF - 9; index++)
 		sum += buf[index];
 	if (sum != 0)
 		goto out;
@@ -420,7 +426,7 @@ g_part_ebr_probe(struct g_part_table *table, struct g_consumer *cp)
 			goto out;
 	}
 
-	res = G_PART_PROBE_PRI_HIGH;
+	res = G_PART_PROBE_PRI_NORM;
 
  out:
 	g_free(buf);
diff --git a/sys/geom/part/g_part_gpt.c b/sys/geom/part/g_part_gpt.c
index db6ebdc..cfbd897 100644
--- a/sys/geom/part/g_part_gpt.c
+++ b/sys/geom/part/g_part_gpt.c
@@ -393,6 +393,10 @@ g_part_gpt_create(struct g_part_table *basetable, struct g_part_parms *gpp)
 	quad_t last;
 	size_t tblsz;
 
+	/* We don't nest, which means that our depth should be 0. */
+	if (basetable->gpt_depth != 0)
+		return (ENXIO);
+
 	table = (struct g_part_gpt_table *)basetable;
 	pp = gpp->gpp_provider;
 	tblsz = (basetable->gpt_entries * sizeof(struct gpt_ent) +
diff --git a/sys/geom/part/g_part_mbr.c b/sys/geom/part/g_part_mbr.c
index e058b05..72d0ecb 100644
--- a/sys/geom/part/g_part_mbr.c
+++ b/sys/geom/part/g_part_mbr.c
@@ -230,7 +230,7 @@ g_part_mbr_create(struct g_part_table *basetable, struct g_part_parms *gpp)
 	struct g_consumer *cp;
 	struct g_provider *pp;
 	struct g_part_mbr_table *table;
-	uint64_t msize;
+	uint32_t msize;
 
 	pp = gpp->gpp_provider;
 	cp = LIST_FIRST(&pp->consumers);
@@ -238,7 +238,7 @@ g_part_mbr_create(struct g_part_table *basetable, struct g_part_parms *gpp)
 	if (pp->sectorsize < MBRSIZE)
 		return (ENOSPC);
 
-	msize = pp->mediasize / pp->sectorsize;
+	msize = MIN(pp->mediasize / pp->sectorsize, 0xffffffff);
 	basetable->gpt_first = basetable->gpt_sectors;
 	basetable->gpt_last = msize - (msize % basetable->gpt_sectors) - 1;
 
diff --git a/sys/geom/part/g_part_pc98.c b/sys/geom/part/g_part_pc98.c
index f9be0cc..8857f9b 100644
--- a/sys/geom/part/g_part_pc98.c
+++ b/sys/geom/part/g_part_pc98.c
@@ -218,8 +218,7 @@ g_part_pc98_create(struct g_part_table *basetable, struct g_part_parms *gpp)
 	struct g_consumer *cp;
 	struct g_provider *pp;
 	struct g_part_pc98_table *table;
-	uint64_t msize;
-	uint32_t cyl;
+	uint32_t cyl, msize;
 
 	pp = gpp->gpp_provider;
 	cp = LIST_FIRST(&pp->consumers);
@@ -231,7 +230,7 @@ g_part_pc98_create(struct g_part_table *basetable, struct g_part_parms *gpp)
 
 	cyl = basetable->gpt_heads * basetable->gpt_sectors;
 
-	msize = pp->mediasize / SECSIZE;
+	msize = MIN(pp->mediasize / SECSIZE, 0xffffffff);
 	basetable->gpt_first = cyl;
 	basetable->gpt_last = msize - (msize % cyl) - 1;
 
diff --git a/sys/geom/vinum/geom_vinum.c b/sys/geom/vinum/geom_vinum.c
index 1a915ed..4a6760a 100644
--- a/sys/geom/vinum/geom_vinum.c
+++ b/sys/geom/vinum/geom_vinum.c
@@ -1,5 +1,6 @@
 /*-
- *  Copyright (c) 2004 Lukas Ertl
+ *  Copyright (c) 2004, 2007 Lukas Ertl
+ *  Copyright (c) 2007, 2009 Ulf Lilleengen
  *  All rights reserved.
  * 
  * Redistribution and use in source and binary forms, with or without
@@ -31,6 +32,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/bio.h>
 #include <sys/kernel.h>
+#include <sys/kthread.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/module.h>
@@ -41,7 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <geom/geom.h>
 #include <geom/vinum/geom_vinum_var.h>
 #include <geom/vinum/geom_vinum.h>
-#include <geom/vinum/geom_vinum_share.h>
+#include <geom/vinum/geom_vinum_raid5.h>
 
 SYSCTL_DECL(_kern_geom);
 SYSCTL_NODE(_kern_geom, OID_AUTO, vinum, CTLFLAG_RW, 0, "GEOM_VINUM stuff");
@@ -50,14 +52,18 @@ TUNABLE_INT("kern.geom.vinum.debug", &g_vinum_debug);
 SYSCTL_UINT(_kern_geom_vinum, OID_AUTO, debug, CTLFLAG_RW, &g_vinum_debug, 0,
     "Debug level");
 
-int	gv_create(struct g_geom *, struct gctl_req *);
+static int	gv_create(struct g_geom *, struct gctl_req *);
+static void	gv_attach(struct gv_softc *, struct gctl_req *);
+static void	gv_detach(struct gv_softc *, struct gctl_req *);
+static void	gv_parityop(struct gv_softc *, struct gctl_req *);
+
 
 static void
 gv_orphan(struct g_consumer *cp)
 {
 	struct g_geom *gp;
 	struct gv_softc *sc;
-	int error;
+	struct gv_drive *d;
 	
 	g_topology_assert();
 
@@ -65,59 +71,90 @@ gv_orphan(struct g_consumer *cp)
 	gp = cp->geom;
 	KASSERT(gp != NULL, ("gv_orphan: null gp"));
 	sc = gp->softc;
+	KASSERT(sc != NULL, ("gv_orphan: null sc"));
+	d = cp->private;
+	KASSERT(d != NULL, ("gv_orphan: null d"));
 
 	g_trace(G_T_TOPOLOGY, "gv_orphan(%s)", gp->name);
 
-	if (cp->acr != 0 || cp->acw != 0 || cp->ace != 0)
-		g_access(cp, -cp->acr, -cp->acw, -cp->ace);
-	error = cp->provider->error;
-	if (error == 0)
-		error = ENXIO;
-	g_detach(cp);
-	g_destroy_consumer(cp);
-	if (!LIST_EMPTY(&gp->consumer))
-		return;
-	g_free(sc);
-	g_wither_geom(gp, error);
+	gv_post_event(sc, GV_EVENT_DRIVE_LOST, d, NULL, 0, 0);
 }
 
-static void
+void
 gv_start(struct bio *bp)
 {
-	struct bio *bp2;
 	struct g_geom *gp;
+	struct gv_softc *sc;
 	
 	gp = bp->bio_to->geom;
-	switch(bp->bio_cmd) {
+	sc = gp->softc;
+
+	switch (bp->bio_cmd) {
 	case BIO_READ:
 	case BIO_WRITE:
 	case BIO_DELETE:
-		bp2 = g_clone_bio(bp);
-		if (bp2 == NULL)
-			g_io_deliver(bp, ENOMEM);
-		else {
-			bp2->bio_done = g_std_done;
-			g_io_request(bp2, LIST_FIRST(&gp->consumer));
-		}
-		return;
+		break;
+	case BIO_GETATTR:
 	default:
 		g_io_deliver(bp, EOPNOTSUPP);
 		return;
 	}
+
+	mtx_lock(&sc->queue_mtx);
+	bioq_disksort(sc->bqueue, bp);
+	wakeup(sc);
+	mtx_unlock(&sc->queue_mtx);
 }
 
-static int
+void
+gv_done(struct bio *bp)
+{
+	struct g_geom *gp;
+	struct gv_softc *sc;
+	
+	KASSERT(bp != NULL, ("NULL bp"));
+
+	gp = bp->bio_from->geom;
+	sc = gp->softc;
+	bp->bio_cflags |= GV_BIO_DONE;
+
+	mtx_lock(&sc->queue_mtx);
+	bioq_disksort(sc->bqueue, bp);
+	wakeup(sc);
+	mtx_unlock(&sc->queue_mtx);
+}
+
+int
 gv_access(struct g_provider *pp, int dr, int dw, int de)
 {
 	struct g_geom *gp;
-	struct g_consumer *cp;
+	struct gv_softc *sc;
+	struct gv_drive *d, *d2;
 	int error;
 	
-	gp = pp->geom;
 	error = ENXIO;
-	cp = LIST_FIRST(&gp->consumer);
-	error = g_access(cp, dr, dw, de);
-	return (error);
+	gp = pp->geom;
+	sc = gp->softc;
+	if (dw > 0 && dr == 0)
+		dr = 1;
+	else if (dw < 0 && dr == 0)
+		dr = -1;
+	LIST_FOREACH(d, &sc->drives, drive) {
+		if (d->consumer == NULL)
+			continue;
+		error = g_access(d->consumer, dr, dw, de);
+		if (error) {
+			LIST_FOREACH(d2, &sc->drives, drive) {
+				if (d == d2)
+					break;
+				g_access(d2->consumer, -dr, -dw, -de);
+			}
+			G_VINUM_DEBUG(0, "g_access '%s' failed: %d", d->name,
+			    error);
+			return (error);
+		}
+	}
+	return (0);
 }
 
 static void
@@ -136,14 +173,132 @@ gv_init(struct g_class *mp)
 	gp->softc = g_malloc(sizeof(struct gv_softc), M_WAITOK | M_ZERO);
 	sc = gp->softc;
 	sc->geom = gp;
+	sc->bqueue = g_malloc(sizeof(struct bio_queue_head), M_WAITOK | M_ZERO);
+	bioq_init(sc->bqueue);
 	LIST_INIT(&sc->drives);
 	LIST_INIT(&sc->subdisks);
 	LIST_INIT(&sc->plexes);
 	LIST_INIT(&sc->volumes);
+	TAILQ_INIT(&sc->equeue);
+	mtx_init(&sc->config_mtx, "gv_config", NULL, MTX_DEF);
+	mtx_init(&sc->queue_mtx, "gv_queue", NULL, MTX_DEF);
+	kproc_create(gv_worker, sc, NULL, 0, 0, "gv_worker");
+}
+
+static int
+gv_unload(struct gctl_req *req, struct g_class *mp, struct g_geom *gp)
+{
+	struct gv_softc *sc;
+
+	g_trace(G_T_TOPOLOGY, "gv_unload(%p)", mp);
+
+	g_topology_assert();
+	sc = gp->softc;
+
+	if (sc != NULL) {
+		gv_post_event(sc, GV_EVENT_THREAD_EXIT, NULL, NULL, 0, 0);
+		gp->softc = NULL;
+		g_wither_geom(gp, ENXIO);
+		return (EAGAIN);
+	}
+
+	return (0);
+}
+
+/* Handle userland request of attaching object. */
+static void
+gv_attach(struct gv_softc *sc, struct gctl_req *req)
+{
+	struct gv_volume *v;
+	struct gv_plex *p;
+	struct gv_sd *s;
+	off_t *offset;
+	int *rename, type_child, type_parent;
+	char *child, *parent;
+
+	child = gctl_get_param(req, "child", NULL);
+	if (child == NULL) {
+		gctl_error(req, "no child given");
+		return;
+	}
+	parent = gctl_get_param(req, "parent", NULL);
+	if (parent == NULL) {
+		gctl_error(req, "no parent given");
+		return;
+	}
+	offset = gctl_get_paraml(req, "offset", sizeof(*offset));
+	if (offset == NULL) {
+		gctl_error(req, "no offset given");
+		return;
+	}
+	rename = gctl_get_paraml(req, "rename", sizeof(*rename));
+	if (rename == NULL) {
+		gctl_error(req, "no rename flag given");
+		return;
+	}
+
+	type_child = gv_object_type(sc, child);
+	type_parent = gv_object_type(sc, parent);
+
+	switch (type_child) {
+	case GV_TYPE_PLEX:
+		if (type_parent != GV_TYPE_VOL) {
+			gctl_error(req, "no such volume to attach to");
+			return;
+		}
+		v = gv_find_vol(sc, parent);
+		p = gv_find_plex(sc, child);
+		gv_post_event(sc, GV_EVENT_ATTACH_PLEX, p, v, *offset, *rename);
+		break;
+	case GV_TYPE_SD:
+		if (type_parent != GV_TYPE_PLEX) {
+			gctl_error(req, "no such plex to attach to");
+			return;
+		}
+		p = gv_find_plex(sc, parent);
+		s = gv_find_sd(sc, child);
+		gv_post_event(sc, GV_EVENT_ATTACH_SD, s, p, *offset, *rename);
+		break;
+	default:
+		gctl_error(req, "invalid child type");
+		break;
+	}
+}
+
+/* Handle userland request of detaching object. */
+static void
+gv_detach(struct gv_softc *sc, struct gctl_req *req)
+{
+	struct gv_plex *p;
+	struct gv_sd *s;
+	int *flags, type;
+	char *object;
+
+	object = gctl_get_param(req, "object", NULL);
+	if (object == NULL) {
+		gctl_error(req, "no argument given");
+		return;
+	}
+
+	flags = gctl_get_paraml(req, "flags", sizeof(*flags));
+	type = gv_object_type(sc, object);
+	switch (type) {
+	case GV_TYPE_PLEX:
+		p = gv_find_plex(sc, object);
+		gv_post_event(sc, GV_EVENT_DETACH_PLEX, p, NULL, *flags, 0);
+		break;
+	case GV_TYPE_SD:
+		s = gv_find_sd(sc, object);
+		gv_post_event(sc, GV_EVENT_DETACH_SD, s, NULL, *flags, 0);
+		break;
+	default:
+		gctl_error(req, "invalid object type");
+		break;
+	}
 }
 
 /* Handle userland requests for creating new objects. */
-int
+static int
 gv_create(struct g_geom *gp, struct gctl_req *req)
 {
 	struct gv_softc *sc;
@@ -151,10 +306,9 @@ gv_create(struct g_geom *gp, struct gctl_req *req)
 	struct gv_plex *p, *p2;
 	struct gv_sd *s, *s2;
 	struct gv_volume *v, *v2;
-	struct g_consumer *cp;
 	struct g_provider *pp;
-	int error, i, *drives, *plexes, *subdisks, *volumes;
-	char buf[20], errstr[ERRBUFSIZ];
+	int error, i, *drives, *flags, *plexes, *subdisks, *volumes;
+	char buf[20];
 
 	g_topology_assert();
 
@@ -170,6 +324,11 @@ gv_create(struct g_geom *gp, struct gctl_req *req)
 		gctl_error(req, "number of objects not given");
 		return (-1);
 	}
+	flags = gctl_get_paraml(req, "flags", sizeof(*flags));
+	if (flags == NULL) {
+		gctl_error(req, "flags not given");
+		return (-1);
+	}
 
 	/* First, handle drive definitions ... */
 	for (i = 0; i < *drives; i++) {
@@ -179,33 +338,33 @@ gv_create(struct g_geom *gp, struct gctl_req *req)
 			gctl_error(req, "no drive definition given");
 			return (-1);
 		}
-		d = gv_find_drive(sc, d2->name);
-		if (d != NULL) {
-			gctl_error(req, "drive '%s' is already known",
-			    d->name);
-			continue;
-		}
-
-		d = g_malloc(sizeof(*d), M_WAITOK | M_ZERO);
-		bcopy(d2, d, sizeof(*d));
-
 		/*
-		 * Make sure that the provider specified in the drive
-		 * specification is an active GEOM provider.
+		 * Make sure that the device specified in the drive config is
+		 * an active GEOM provider.
 		 */
-		pp = g_provider_by_name(d->device);
+		pp = g_provider_by_name(d2->device);
 		if (pp == NULL) {
-			gctl_error(req, "%s: drive not found", d->device);
-			g_free(d);
-			return (-1);
+			gctl_error(req, "%s: device not found", d2->device);
+			goto error;
+		}
+		if (gv_find_drive(sc, d2->name) != NULL) {
+			/* Ignore error. */
+			if (*flags & GV_FLAG_F)
+				continue;
+			gctl_error(req, "drive '%s' already exists", d2->name);
+			goto error;
+		}
+		if (gv_find_drive_device(sc, d2->device) != NULL) {
+			gctl_error(req, "device '%s' already configured in "
+			    "gvinum", d2->device);
+			goto error;
 		}
-		d->size = pp->mediasize - GV_DATA_START;
-		d->avail = d->size;
 
-		gv_config_new_drive(d);
 
-		d->flags |= GV_DRIVE_NEWBORN;
-		LIST_INSERT_HEAD(&sc->drives, d, drive);
+		d = g_malloc(sizeof(*d), M_WAITOK | M_ZERO);
+		bcopy(d2, d, sizeof(*d));
+
+		gv_post_event(sc, GV_EVENT_CREATE_DRIVE, d, NULL, 0, 0);
 	}
 
 	/* ... then volume definitions ... */
@@ -217,19 +376,18 @@ gv_create(struct g_geom *gp, struct gctl_req *req)
 			gctl_error(req, "no volume definition given");
 			return (-1);
 		}
-		v = gv_find_vol(sc, v2->name);
-		if (v != NULL) {
-			gctl_error(req, "volume '%s' is already known",
-			    v->name);
-			return (-1);
+		if (gv_find_vol(sc, v2->name) != NULL) {
+			/* Ignore error. */
+			if (*flags & GV_FLAG_F)
+				continue;
+			gctl_error(req, "volume '%s' already exists", v2->name);
+			goto error;
 		}
 
 		v = g_malloc(sizeof(*v), M_WAITOK | M_ZERO);
 		bcopy(v2, v, sizeof(*v));
 
-		v->vinumconf = sc;
-		LIST_INIT(&v->plexes);
-		LIST_INSERT_HEAD(&sc->volumes, v, volume);
+		gv_post_event(sc, GV_EVENT_CREATE_VOLUME, v, NULL, 0, 0);
 	}
 
 	/* ... then plex definitions ... */
@@ -241,35 +399,21 @@ gv_create(struct g_geom *gp, struct gctl_req *req)
 			gctl_error(req, "no plex definition given");
 			return (-1);
 		}
-		p = gv_find_plex(sc, p2->name);
-		if (p != NULL) {
-			gctl_error(req, "plex '%s' is already known", p->name);
-			return (-1);
+		if (gv_find_plex(sc, p2->name) != NULL) {
+			/* Ignore error. */
+			if (*flags & GV_FLAG_F)
+				continue;
+			gctl_error(req, "plex '%s' already exists", p2->name);
+			goto error;
 		}
 
 		p = g_malloc(sizeof(*p), M_WAITOK | M_ZERO);
 		bcopy(p2, p, sizeof(*p));
 
-		/* Find the volume this plex should be attached to. */
-		v = gv_find_vol(sc, p->volume);
-		if (v == NULL) {
-			gctl_error(req, "volume '%s' not found", p->volume);
-			g_free(p);
-			continue;
-		}
-		if (v->plexcount)
-			p->flags |= GV_PLEX_ADDED;
-		p->vol_sc = v;
-		v->plexcount++;
-		LIST_INSERT_HEAD(&v->plexes, p, in_volume);
-
-		p->vinumconf = sc;
-		p->flags |= GV_PLEX_NEWBORN;
-		LIST_INIT(&p->subdisks);
-		LIST_INSERT_HEAD(&sc->plexes, p, plex);
+		gv_post_event(sc, GV_EVENT_CREATE_PLEX, p, NULL, 0, 0);
 	}
 
-	/* ... and finally, subdisk definitions. */
+	/* ... and, finally, subdisk definitions. */
 	for (i = 0; i < *subdisks; i++) {
 		error = 0;
 		snprintf(buf, sizeof(buf), "sd%d", i);
@@ -278,122 +422,23 @@ gv_create(struct g_geom *gp, struct gctl_req *req)
 			gctl_error(req, "no subdisk definition given");
 			return (-1);
 		}
-		s = gv_find_sd(sc, s2->name);
-		if (s != NULL) {
-			gctl_error(req, "subdisk '%s' is already known",
-			    s->name);
-			return (-1);
+		if (gv_find_sd(sc, s2->name) != NULL) {
+			/* Ignore error. */
+			if (*flags & GV_FLAG_F)
+				continue;
+			gctl_error(req, "sd '%s' already exists", s2->name);
+			goto error;
 		}
 
 		s = g_malloc(sizeof(*s), M_WAITOK | M_ZERO);
 		bcopy(s2, s, sizeof(*s));
 
-		/* Find the drive where this subdisk should be put on. */
-		d = gv_find_drive(sc, s->drive);
-
-		/* drive not found - XXX */
-		if (d == NULL) {
-			gctl_error(req, "drive '%s' not found", s->drive);
-			g_free(s);
-			continue;
-		}
-
-		/* Find the plex where this subdisk belongs to. */
-		p = gv_find_plex(sc, s->plex);
-
-		/* plex not found - XXX */
-		if (p == NULL) {
-			gctl_error(req, "plex '%s' not found\n", s->plex);
-			g_free(s);
-			continue;
-		}
-
-		/*
-		 * First we give the subdisk to the drive, to handle autosized
-		 * values ...
-		 */
-		error = gv_sd_to_drive(sc, d, s, errstr, sizeof(errstr));
-		if (error) {
-			gctl_error(req, errstr);
-			g_free(s);
-			continue;
-		}
-
-		/*
-		 * Then, we give the subdisk to the plex; we check if the
-		 * given values are correct and maybe adjust them.
-		 */
-		error = gv_sd_to_plex(p, s, 1);
-		if (error) {
-			gctl_error(req, "GEOM_VINUM: couldn't give sd '%s' "
-			    "to plex '%s'\n", s->name, p->name);
-			if (s->drive_sc)
-				LIST_REMOVE(s, from_drive);
-			gv_free_sd(s);
-			g_free(s);
-			/*
-			 * If this subdisk can't be created, we won't create
-			 * the attached plex either, if it is also a new one.
-			 */
-			if (!(p->flags & GV_PLEX_NEWBORN))
-				continue;
-			LIST_FOREACH_SAFE(s, &p->subdisks, in_plex, s2) {
-				if (s->drive_sc)
-					LIST_REMOVE(s, from_drive);
-				p->sdcount--;
-				LIST_REMOVE(s, in_plex);
-				LIST_REMOVE(s, sd);
-				gv_free_sd(s);
-				g_free(s);
-			}
-			if (p->vol_sc != NULL) {
-				LIST_REMOVE(p, in_volume);
-				p->vol_sc->plexcount--;
-			}
-			LIST_REMOVE(p, plex);
-			g_free(p);
-			continue;
-		}
-		s->flags |= GV_SD_NEWBORN;
-
-		s->vinumconf = sc;
-		LIST_INSERT_HEAD(&sc->subdisks, s, sd);
+		gv_post_event(sc, GV_EVENT_CREATE_SD, s, NULL, 0, 0);
 	}
 
-	LIST_FOREACH(s, &sc->subdisks, sd)
-		gv_update_sd_state(s);
-	LIST_FOREACH(p, &sc->plexes, plex)
-		gv_update_plex_config(p);
-	LIST_FOREACH(v, &sc->volumes, volume)
-		gv_update_vol_state(v);
-
-	/*
-	 * Write out the configuration to each drive.  If the drive doesn't
-	 * have a valid geom_slice geom yet, attach it temporarily to our VINUM
-	 * geom.
-	 */
-	LIST_FOREACH(d, &sc->drives, drive) {
-		if (d->geom == NULL) {
-			/*
-			 * XXX if the provider disapears before we get a chance
-			 * to write the config out to the drive, should this
-			 * be handled any differently?
-			 */
-			pp = g_provider_by_name(d->device);
-			if (pp == NULL) {
-				G_VINUM_DEBUG(0, "%s: drive disappeared?",
-				    d->device);
-				continue;
-			}
-			cp = g_new_consumer(gp);
-			g_attach(cp, pp);
-			gv_save_config(cp, d, sc);
-			g_detach(cp);
-			g_destroy_consumer(cp);
-		} else
-			gv_save_config(NULL, d, sc);
-		d->flags &= ~GV_DRIVE_NEWBORN;
-	}
+error:
+	gv_post_event(sc, GV_EVENT_SETUP_OBJECTS, sc, NULL, 0, 0);
+	gv_post_event(sc, GV_EVENT_SAVE_CONFIG, sc, NULL, 0, 0);
 
 	return (0);
 }
@@ -411,13 +456,21 @@ gv_config(struct gctl_req *req, struct g_class *mp, char const *verb)
 	gp = LIST_FIRST(&mp->geom);
 	sc = gp->softc;
 
-	if (!strcmp(verb, "list")) {
+	if (!strcmp(verb, "attach")) {
+		gv_attach(sc, req);
+
+	} else if (!strcmp(verb, "concat")) {
+		gv_concat(gp, req);
+
+	} else if (!strcmp(verb, "detach")) {
+		gv_detach(sc, req);
+
+	} else if (!strcmp(verb, "list")) {
 		gv_list(gp, req);
 
 	/* Save our configuration back to disk. */
 	} else if (!strcmp(verb, "saveconfig")) {
-
-		gv_save_config_all(sc);
+		gv_post_event(sc, GV_EVENT_SAVE_CONFIG, sc, NULL, 0, 0);
 
 	/* Return configuration in string form. */
 	} else if (!strcmp(verb, "getconfig")) {
@@ -435,11 +488,18 @@ gv_config(struct gctl_req *req, struct g_class *mp, char const *verb)
 	} else if (!strcmp(verb, "create")) {
 		gv_create(gp, req);
 
+	} else if (!strcmp(verb, "mirror")) {
+		gv_mirror(gp, req);
+
 	} else if (!strcmp(verb, "move")) {
 		gv_move(gp, req);
 
-	} else if (!strcmp(verb, "parityop")) {
-		gv_parityop(gp, req);
+	} else if (!strcmp(verb, "raid5")) {
+		gv_raid5(gp, req);
+
+	} else if (!strcmp(verb, "rebuildparity") ||
+	    !strcmp(verb, "checkparity")) {
+		gv_parityop(sc, req);
 
 	} else if (!strcmp(verb, "remove")) {
 		gv_remove(gp, req);
@@ -448,100 +508,509 @@ gv_config(struct gctl_req *req, struct g_class *mp, char const *verb)
 		gv_rename(gp, req);
 	
 	} else if (!strcmp(verb, "resetconfig")) {
-		gv_resetconfig(gp, req);
+		gv_post_event(sc, GV_EVENT_RESET_CONFIG, sc, NULL, 0, 0);
 
 	} else if (!strcmp(verb, "start")) {
 		gv_start_obj(gp, req);
 
+	} else if (!strcmp(verb, "stripe")) {
+		gv_stripe(gp, req);
+
 	} else if (!strcmp(verb, "setstate")) {
 		gv_setstate(gp, req);
-
 	} else
 		gctl_error(req, "Unknown verb parameter");
 }
 
-#if 0
-static int
-gv_destroy_geom(struct gctl_req *req, struct g_class *mp, struct g_geom *gp)
+static void
+gv_parityop(struct gv_softc *sc, struct gctl_req *req)
 {
-	struct g_geom *gp2;
-	struct g_consumer *cp;
-	struct gv_softc *sc;
-	struct gv_drive *d, *d2;
-	struct gv_plex *p, *p2;
-	struct gv_sd *s, *s2;
-	struct gv_volume *v, *v2;
-	struct gv_freelist *fl, *fl2;
+	struct gv_plex *p;
+	int *flags, *rebuild, type;
+	char *plex;
 
-	g_trace(G_T_TOPOLOGY, "gv_destroy_geom: %s", gp->name);
-	g_topology_assert();
+	plex = gctl_get_param(req, "plex", NULL);
+	if (plex == NULL) {
+		gctl_error(req, "no plex given");
+		return;
+	}
 
-	KASSERT(gp != NULL, ("gv_destroy_geom: null gp"));
-	KASSERT(gp->softc != NULL, ("gv_destroy_geom: null sc"));
+	flags = gctl_get_paraml(req, "flags", sizeof(*flags));
+	if (flags == NULL) {
+		gctl_error(req, "no flags given");
+		return;
+	}
 
-	sc = gp->softc;
+	rebuild = gctl_get_paraml(req, "rebuild", sizeof(*rebuild));
+	if (rebuild == NULL) {
+		gctl_error(req, "no operation given");
+		return;
+	}
 
-	/*
-	 * Check if any of our drives is still open; if so, refuse destruction.
-	 */
-	LIST_FOREACH(d, &sc->drives, drive) {
-		gp2 = d->geom;
-		cp = LIST_FIRST(&gp2->consumer);
-		if (cp != NULL)
-			g_access(cp, -1, -1, -1);
-		if (gv_is_open(gp2))
-			return (EBUSY);
+	type = gv_object_type(sc, plex);
+	if (type != GV_TYPE_PLEX) {
+		gctl_error(req, "'%s' is not a plex", plex);
+		return;
 	}
+	p = gv_find_plex(sc, plex);
 
-	/* Clean up and deallocate what we allocated. */
-	LIST_FOREACH_SAFE(d, &sc->drives, drive, d2) {
-		LIST_REMOVE(d, drive);
-		g_free(d->hdr);
-		d->hdr = NULL;
-		LIST_FOREACH_SAFE(fl, &d->freelist, freelist, fl2) {
-			d->freelist_entries--;
-			LIST_REMOVE(fl, freelist);
-			g_free(fl);
-			fl = NULL;
-		}
-		d->geom->softc = NULL;
-		g_free(d);
+	if (p->state != GV_PLEX_UP) {
+		gctl_error(req, "plex %s is not completely accessible",
+		    p->name);
+		return;
 	}
 
-	LIST_FOREACH_SAFE(s, &sc->subdisks, sd, s2) {
-		LIST_REMOVE(s, sd);
-		s->drive_sc = NULL;
-		s->plex_sc = NULL;
-		s->provider = NULL;
-		s->consumer = NULL;
-		g_free(s);
+	if (p->org != GV_PLEX_RAID5) {
+		gctl_error(req, "plex %s is not a RAID5 plex", p->name);
+		return;
 	}
 
-	LIST_FOREACH_SAFE(p, &sc->plexes, plex, p2) {
-		LIST_REMOVE(p, plex);
-		gv_kill_thread(p);
-		p->vol_sc = NULL;
-		p->geom->softc = NULL;
-		p->provider = NULL;
-		p->consumer = NULL;
-		if (p->org == GV_PLEX_RAID5) {
-			mtx_destroy(&p->worklist_mtx);
-		}
-		g_free(p);
+	/* Put it in the event queue. */
+	/* XXX: The state of the plex might have changed when this event is
+	 * picked up ... We should perhaps check this afterwards. */
+	if (*rebuild)
+		gv_post_event(sc, GV_EVENT_PARITY_REBUILD, p, NULL, 0, 0);
+	else
+		gv_post_event(sc, GV_EVENT_PARITY_CHECK, p, NULL, 0, 0);
+}
+
+
+static struct g_geom *
+gv_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
+{
+	struct g_geom *gp;
+	struct g_consumer *cp;
+	struct gv_softc *sc;
+	struct gv_hdr vhdr;
+	int error;
+
+ 	g_topology_assert();
+	g_trace(G_T_TOPOLOGY, "gv_taste(%s, %s)", mp->name, pp->name);
+
+	gp = LIST_FIRST(&mp->geom);
+	if (gp == NULL) {
+		G_VINUM_DEBUG(0, "error: tasting, but not initialized?");
+		return (NULL);
 	}
+	sc = gp->softc;
 
-	LIST_FOREACH_SAFE(v, &sc->volumes, volume, v2) {
-		LIST_REMOVE(v, volume);
-		v->geom->softc = NULL;
-		g_free(v);
+	cp = g_new_consumer(gp);
+	if (g_attach(cp, pp) != 0) {
+		g_destroy_consumer(cp);
+		return (NULL);
 	}
+	if (g_access(cp, 1, 0, 0) != 0) {
+		g_detach(cp);
+		g_destroy_consumer(cp);
+		return (NULL);
+	}
+	g_topology_unlock();
 
-	gp->softc = NULL;
-	g_free(sc);
-	g_wither_geom(gp, ENXIO);
-	return (0);
+	error = gv_read_header(cp, &vhdr);
+
+	g_topology_lock();
+	g_access(cp, -1, 0, 0);
+	g_detach(cp);
+	g_destroy_consumer(cp);
+
+	/* Check if what we've been given is a valid vinum drive. */
+	if (!error)
+		gv_post_event(sc, GV_EVENT_DRIVE_TASTED, pp, NULL, 0, 0);
+
+	return (NULL);
+}
+
+void
+gv_worker(void *arg)
+{
+	struct g_provider *pp;
+	struct gv_softc *sc;
+	struct gv_event *ev;
+	struct gv_volume *v;
+	struct gv_plex *p;
+	struct gv_sd *s;
+	struct gv_drive *d;
+	struct bio *bp;
+	int newstate, flags, err, rename;
+	char *newname;
+	off_t offset;
+
+	sc = arg;
+	KASSERT(sc != NULL, ("NULL sc"));
+	mtx_lock(&sc->queue_mtx);
+	for (;;) {
+		/* Look at the events first... */
+		ev = TAILQ_FIRST(&sc->equeue);
+		if (ev != NULL) {
+			TAILQ_REMOVE(&sc->equeue, ev, events);
+			mtx_unlock(&sc->queue_mtx);
+
+			switch (ev->type) {
+			case GV_EVENT_DRIVE_TASTED:
+				G_VINUM_DEBUG(2, "event 'drive tasted'");
+				pp = ev->arg1;
+				gv_drive_tasted(sc, pp);
+				break;
+
+			case GV_EVENT_DRIVE_LOST:
+				G_VINUM_DEBUG(2, "event 'drive lost'");
+				d = ev->arg1;
+				gv_drive_lost(sc, d);
+				break;
+
+			case GV_EVENT_CREATE_DRIVE:
+				G_VINUM_DEBUG(2, "event 'create drive'");
+				d = ev->arg1;
+				gv_create_drive(sc, d);
+				break;
+
+			case GV_EVENT_CREATE_VOLUME:
+				G_VINUM_DEBUG(2, "event 'create volume'");
+				v = ev->arg1;
+				gv_create_volume(sc, v);
+				break;
+
+			case GV_EVENT_CREATE_PLEX:
+				G_VINUM_DEBUG(2, "event 'create plex'");
+				p = ev->arg1;
+				gv_create_plex(sc, p);
+				break;
+
+			case GV_EVENT_CREATE_SD:
+				G_VINUM_DEBUG(2, "event 'create sd'");
+				s = ev->arg1;
+				gv_create_sd(sc, s);
+				break;
+
+			case GV_EVENT_RM_DRIVE:
+				G_VINUM_DEBUG(2, "event 'remove drive'");
+				d = ev->arg1;
+				flags = ev->arg3;
+				gv_rm_drive(sc, d, flags);
+				/*gv_setup_objects(sc);*/
+				break;
+
+			case GV_EVENT_RM_VOLUME:
+				G_VINUM_DEBUG(2, "event 'remove volume'");
+				v = ev->arg1;
+				gv_rm_vol(sc, v);
+				/*gv_setup_objects(sc);*/
+				break;
+
+			case GV_EVENT_RM_PLEX:
+				G_VINUM_DEBUG(2, "event 'remove plex'");
+				p = ev->arg1;
+				gv_rm_plex(sc, p);
+				/*gv_setup_objects(sc);*/
+				break;
+
+			case GV_EVENT_RM_SD:
+				G_VINUM_DEBUG(2, "event 'remove sd'");
+				s = ev->arg1;
+				gv_rm_sd(sc, s);
+				/*gv_setup_objects(sc);*/
+				break;
+
+			case GV_EVENT_SAVE_CONFIG:
+				G_VINUM_DEBUG(2, "event 'save config'");
+				gv_save_config(sc);
+				break;
+
+			case GV_EVENT_SET_SD_STATE:
+				G_VINUM_DEBUG(2, "event 'setstate sd'");
+				s = ev->arg1;
+				newstate = ev->arg3;
+				flags = ev->arg4;
+				err = gv_set_sd_state(s, newstate, flags);
+				if (err)
+					G_VINUM_DEBUG(0, "error setting subdisk"
+					    " state: error code %d", err);
+				break;
+
+			case GV_EVENT_SET_DRIVE_STATE:
+				G_VINUM_DEBUG(2, "event 'setstate drive'");
+				d = ev->arg1;
+				newstate = ev->arg3;
+				flags = ev->arg4;
+				err = gv_set_drive_state(d, newstate, flags);
+				if (err)
+					G_VINUM_DEBUG(0, "error setting drive "
+					    "state: error code %d", err);
+				break;
+
+			case GV_EVENT_SET_VOL_STATE:
+				G_VINUM_DEBUG(2, "event 'setstate volume'");
+				v = ev->arg1;
+				newstate = ev->arg3;
+				flags = ev->arg4;
+				err = gv_set_vol_state(v, newstate, flags);
+				if (err)
+					G_VINUM_DEBUG(0, "error setting volume "
+					    "state: error code %d", err);
+				break;
+
+			case GV_EVENT_SET_PLEX_STATE:
+				G_VINUM_DEBUG(2, "event 'setstate plex'");
+				p = ev->arg1;
+				newstate = ev->arg3;
+				flags = ev->arg4;
+				err = gv_set_plex_state(p, newstate, flags);
+				if (err)
+					G_VINUM_DEBUG(0, "error setting plex "
+					    "state: error code %d", err);
+				break;
+
+			case GV_EVENT_SETUP_OBJECTS:
+				G_VINUM_DEBUG(2, "event 'setup objects'");
+				gv_setup_objects(sc);
+				break;
+
+			case GV_EVENT_RESET_CONFIG:
+				G_VINUM_DEBUG(2, "event 'resetconfig'");
+				err = gv_resetconfig(sc);
+				if (err)
+					G_VINUM_DEBUG(0, "error resetting "
+					    "config: error code %d", err);
+				break;
+
+			case GV_EVENT_PARITY_REBUILD:
+				/*
+				 * Start the rebuild. The gv_plex_done will
+				 * handle issuing of the remaining rebuild bio's
+				 * until it's finished. 
+				 */
+				G_VINUM_DEBUG(2, "event 'rebuild'");
+				p = ev->arg1;
+				if (p->state != GV_PLEX_UP) {
+					G_VINUM_DEBUG(0, "plex %s is not "
+					    "completely accessible", p->name);
+					break;
+				}
+				p->synced = 0;
+				g_topology_assert_not();
+				g_topology_lock();
+				err = gv_access(p->vol_sc->provider, 1, 1, 0);
+				if (err) {
+					G_VINUM_DEBUG(0, "unable to access "
+					    "provider");
+					break;
+				}
+				g_topology_unlock();
+				gv_parity_request(p, GV_BIO_CHECK |
+				    GV_BIO_PARITY, 0);
+				break;
+
+			case GV_EVENT_PARITY_CHECK:
+				/* Start parity check. */
+				G_VINUM_DEBUG(2, "event 'check'");
+				p = ev->arg1;
+				if (p->state != GV_PLEX_UP) {
+					G_VINUM_DEBUG(0, "plex %s is not "
+					    "completely accessible", p->name);
+					break;
+				}
+				p->synced = 0;
+				g_topology_assert_not();
+				g_topology_lock();
+				err = gv_access(p->vol_sc->provider, 1, 1, 0);
+				if (err) {
+					G_VINUM_DEBUG(0, "unable to access "
+					    "provider");
+					break;
+				}
+				g_topology_unlock();
+				gv_parity_request(p, GV_BIO_CHECK, 0);
+				break;
+
+			case GV_EVENT_START_PLEX:
+				G_VINUM_DEBUG(2, "event 'start' plex");
+				p = ev->arg1;
+				gv_start_plex(p);
+				break;
+
+			case GV_EVENT_START_VOLUME:
+				G_VINUM_DEBUG(2, "event 'start' volume");
+				v = ev->arg1;
+				gv_start_vol(v);
+				break;
+
+			case GV_EVENT_ATTACH_PLEX:
+				G_VINUM_DEBUG(2, "event 'attach' plex");
+				p = ev->arg1;
+				v = ev->arg2;
+				rename = ev->arg4;
+				err = gv_attach_plex(p, v, rename);
+				if (err)
+					G_VINUM_DEBUG(0, "error attaching %s to"
+					    " %s: error code %d", p->name,
+					    v->name, err);
+				break;
+
+			case GV_EVENT_ATTACH_SD:
+				G_VINUM_DEBUG(2, "event 'attach' sd");
+				s = ev->arg1;
+				p = ev->arg2;
+				offset = ev->arg3;
+				rename = ev->arg4;
+				err = gv_attach_sd(s, p, offset, rename);
+				if (err)
+					G_VINUM_DEBUG(0, "error attaching %s to"
+					    " %s: error code %d", s->name,
+					    p->name, err);
+				break;
+
+			case GV_EVENT_DETACH_PLEX:
+				G_VINUM_DEBUG(2, "event 'detach' plex");
+				p = ev->arg1;
+				flags = ev->arg3;
+				err = gv_detach_plex(p, flags);
+				if (err)
+					G_VINUM_DEBUG(0, "error detaching %s: "
+					    "error code %d", p->name, err);
+				break;
+
+			case GV_EVENT_DETACH_SD:
+				G_VINUM_DEBUG(2, "event 'detach' sd");
+				s = ev->arg1;
+				flags = ev->arg3;
+				err = gv_detach_sd(s, flags);
+				if (err)
+					G_VINUM_DEBUG(0, "error detaching %s: "
+					    "error code %d", s->name, err);
+				break;
+
+			case GV_EVENT_RENAME_VOL:
+				G_VINUM_DEBUG(2, "event 'rename' volume");
+				v = ev->arg1;
+				newname = ev->arg2;
+				flags = ev->arg3;
+				err = gv_rename_vol(sc, v, newname, flags);
+				if (err)
+					G_VINUM_DEBUG(0, "error renaming %s to "
+					    "%s: error code %d", v->name,
+					    newname, err);
+				g_free(newname);
+				/* Destroy and recreate the provider if we can. */
+				if (gv_provider_is_open(v->provider)) {
+					G_VINUM_DEBUG(0, "unable to rename "
+					    "provider to %s: provider in use",
+					    v->name);
+					break;
+				}
+				g_wither_provider(v->provider, ENOENT);
+				v->provider = NULL;
+				gv_post_event(sc, GV_EVENT_SETUP_OBJECTS, sc,
+				    NULL, 0, 0);
+				break;
+
+			case GV_EVENT_RENAME_PLEX:
+				G_VINUM_DEBUG(2, "event 'rename' plex");
+				p = ev->arg1;
+				newname = ev->arg2;
+				flags = ev->arg3;
+				err = gv_rename_plex(sc, p, newname, flags);
+				if (err)
+					G_VINUM_DEBUG(0, "error renaming %s to "
+					    "%s: error code %d", p->name,
+					    newname, err);
+				g_free(newname);
+				break;
+
+			case GV_EVENT_RENAME_SD:
+				G_VINUM_DEBUG(2, "event 'rename' sd");
+				s = ev->arg1;
+				newname = ev->arg2;
+				flags = ev->arg3;
+				err = gv_rename_sd(sc, s, newname, flags);
+				if (err)
+					G_VINUM_DEBUG(0, "error renaming %s to "
+					    "%s: error code %d", s->name,
+					    newname, err);
+				g_free(newname);
+				break;
+
+			case GV_EVENT_RENAME_DRIVE:
+				G_VINUM_DEBUG(2, "event 'rename' drive");
+				d = ev->arg1;
+				newname = ev->arg2;
+				flags = ev->arg3;
+				err = gv_rename_drive(sc, d, newname, flags);
+				if (err)
+					G_VINUM_DEBUG(0, "error renaming %s to "
+					    "%s: error code %d", d->name,
+					    newname, err);
+				g_free(newname);
+				break;
+
+			case GV_EVENT_MOVE_SD:
+				G_VINUM_DEBUG(2, "event 'move' sd");
+				s = ev->arg1;
+				d = ev->arg2;
+				flags = ev->arg3;
+				err = gv_move_sd(sc, s, d, flags);
+				if (err)
+					G_VINUM_DEBUG(0, "error moving %s to "
+					    "%s: error code %d", s->name,
+					    d->name, err);
+				break;
+
+			case GV_EVENT_THREAD_EXIT:
+				G_VINUM_DEBUG(2, "event 'thread exit'");
+				g_free(ev);
+				mtx_lock(&sc->queue_mtx);
+				gv_cleanup(sc);
+				mtx_destroy(&sc->queue_mtx);
+				g_free(sc->bqueue);
+				g_free(sc);
+				kproc_exit(ENXIO);
+				break;			/* not reached */
+
+			default:
+				G_VINUM_DEBUG(1, "unknown event %d", ev->type);
+			}
+
+			g_free(ev);
+
+			mtx_lock(&sc->queue_mtx);
+			continue;
+		}
+
+		/* ... then do I/O processing. */
+		bp = bioq_takefirst(sc->bqueue);
+		if (bp == NULL) {
+			msleep(sc, &sc->queue_mtx, PRIBIO, "-", hz/10);
+			continue;
+		}
+		mtx_unlock(&sc->queue_mtx);
+
+		/* A bio that is coming up from an underlying device. */
+		if (bp->bio_cflags & GV_BIO_DONE) {
+			gv_bio_done(sc, bp);
+		/* A bio that interfered with another bio. */
+		} else if (bp->bio_cflags & GV_BIO_ONHOLD) {
+			s = bp->bio_caller1;
+			p = s->plex_sc;
+			/* Is it still locked out? */
+			if (gv_stripe_active(p, bp)) {
+				/* Park the bio on the waiting queue. */
+				bioq_disksort(p->wqueue, bp);
+			} else {
+				bp->bio_cflags &= ~GV_BIO_ONHOLD;
+				g_io_request(bp, s->drive_sc->consumer);
+			}
+		/* A special request requireing special handling. */
+		} else if (bp->bio_cflags & GV_BIO_INTERNAL ||
+		    bp->bio_pflags & GV_BIO_INTERNAL) {
+			p = bp->bio_caller1;
+			gv_plex_start(p, bp);
+		/* A fresh bio, scheduled it down. */
+		} else {
+			gv_volume_start(sc, bp);
+		}
+
+		mtx_lock(&sc->queue_mtx);
+	}
 }
-#endif
 
 #define	VINUM_CLASS_NAME "VINUM"
 
@@ -549,8 +1018,9 @@ static struct g_class g_vinum_class	= {
 	.name = VINUM_CLASS_NAME,
 	.version = G_VERSION,
 	.init = gv_init,
-	/*.destroy_geom = gv_destroy_geom,*/
+	.taste = gv_taste,
 	.ctlreq = gv_config,
+	.destroy_geom = gv_unload,
 };
 
 DECLARE_GEOM_CLASS(g_vinum_class, g_vinum);
diff --git a/sys/geom/vinum/geom_vinum.h b/sys/geom/vinum/geom_vinum.h
index 07c0410..2db22d3 100644
--- a/sys/geom/vinum/geom_vinum.h
+++ b/sys/geom/vinum/geom_vinum.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2004 Lukas Ertl
+ * Copyright (c) 2004, 2007 Lukas Ertl
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -29,20 +29,25 @@
 #ifndef	_GEOM_VINUM_H_
 #define	_GEOM_VINUM_H_
 
-#define	ERRBUFSIZ	1024
+/* geom_vinum_create.c */
+void	gv_concat(struct g_geom *gp, struct gctl_req *);
+void	gv_mirror(struct g_geom *gp, struct gctl_req *);
+void	gv_stripe(struct g_geom *gp, struct gctl_req *);
+void	gv_raid5(struct g_geom *gp, struct gctl_req *);
+int	gv_create_drive(struct gv_softc *, struct gv_drive *);
+int	gv_create_volume(struct gv_softc *, struct gv_volume *);
+int	gv_create_plex(struct gv_softc *, struct gv_plex *);
+int	gv_create_sd(struct gv_softc *, struct gv_sd *);
 
 /* geom_vinum_drive.c */
-void	gv_config_new_drive(struct gv_drive *);
-void	gv_drive_modify(struct gv_drive *);
+void	gv_save_config(struct gv_softc *);
 int	gv_read_header(struct g_consumer *, struct gv_hdr *);
-void	gv_save_config_all(struct gv_softc *);
-void	gv_save_config(struct g_consumer *, struct gv_drive *,
-	    struct gv_softc *);
 int	gv_write_header(struct g_consumer *, struct gv_hdr *);
 
 /* geom_vinum_init.c */
-void	gv_parityop(struct g_geom *, struct gctl_req *);
 void	gv_start_obj(struct g_geom *, struct gctl_req *);
+int	gv_start_plex(struct gv_plex *);
+int	gv_start_vol(struct gv_volume *);
 
 /* geom_vinum_list.c */
 void	gv_ld(struct g_geom *, struct gctl_req *, struct sbuf *);
@@ -53,48 +58,97 @@ void	gv_list(struct g_geom *, struct gctl_req *);
 
 /* geom_vinum_move.c */
 void	gv_move(struct g_geom *, struct gctl_req *);
+int	gv_move_sd(struct gv_softc *, struct gv_sd *, struct gv_drive *, int);
 
 /* geom_vinum_rename.c */
 void	gv_rename(struct g_geom *, struct gctl_req *);
+int	gv_rename_drive(struct gv_softc *, struct gv_drive *, char *, int);
+int	gv_rename_plex(struct gv_softc *, struct gv_plex *, char *, int);
+int	gv_rename_sd(struct gv_softc *, struct gv_sd *, char *, int);
+int	gv_rename_vol(struct gv_softc *, struct gv_volume *, char *, int);
 
 /* geom_vinum_rm.c */
 void	gv_remove(struct g_geom *, struct gctl_req *);
-int	gv_resetconfig(struct g_geom *, struct gctl_req *);
-int	gv_rm_sd(struct gv_softc *sc, struct gctl_req *req,
-	    struct gv_sd *s, int flags);
+int	gv_resetconfig(struct gv_softc *);
+void	gv_rm_sd(struct gv_softc *sc, struct gv_sd *s);
+void	gv_rm_drive(struct gv_softc *, struct gv_drive *, int);
+void	gv_rm_plex(struct gv_softc *, struct gv_plex *);
+void	gv_rm_vol(struct gv_softc *, struct gv_volume *);
+
 
 /* geom_vinum_state.c */
 int	gv_sdstatemap(struct gv_plex *);
 void	gv_setstate(struct g_geom *, struct gctl_req *);
 int	gv_set_drive_state(struct gv_drive *, int, int);
 int	gv_set_sd_state(struct gv_sd *, int, int);
+int	gv_set_vol_state(struct gv_volume *, int, int);
+int	gv_set_plex_state(struct gv_plex *, int, int);
 void	gv_update_sd_state(struct gv_sd *);
 void	gv_update_plex_state(struct gv_plex *);
 void	gv_update_vol_state(struct gv_volume *);
 
 /* geom_vinum_subr.c */
-void	gv_adjust_freespace(struct gv_sd *, off_t);
-void	gv_free_sd(struct gv_sd *);
-struct g_geom	*find_vinum_geom(void);
-struct gv_drive	*gv_find_drive(struct gv_softc *, char *);
-struct gv_plex	*gv_find_plex(struct gv_softc *, char *);
-struct gv_sd	*gv_find_sd(struct gv_softc *, char *);
-struct gv_volume *gv_find_vol(struct gv_softc *, char *);
-void	gv_format_config(struct gv_softc *, struct sbuf *, int, char *);
-int	gv_is_striped(struct gv_plex *);
-int	gv_is_open(struct g_geom *);
-void	gv_kill_drive_thread(struct gv_drive *);
-void	gv_kill_plex_thread(struct gv_plex *);
-void	gv_kill_vol_thread(struct gv_volume *);
-int	gv_object_type(struct gv_softc *, char *);
-void	gv_parse_config(struct gv_softc *, u_char *, int);
-int	gv_sd_to_drive(struct gv_softc *, struct gv_drive *, struct gv_sd *,
-	    char *, int);
-int	gv_sd_to_plex(struct gv_plex *, struct gv_sd *, int);
-void	gv_update_plex_config(struct gv_plex *);
-void	gv_update_vol_size(struct gv_volume *, off_t);
-off_t	gv_vol_size(struct gv_volume *);
-off_t	gv_plex_size(struct gv_plex *);
+void		 	 gv_adjust_freespace(struct gv_sd *, off_t);
+void		 	 gv_free_sd(struct gv_sd *);
+struct gv_drive		*gv_find_drive(struct gv_softc *, char *);
+struct gv_drive		*gv_find_drive_device(struct gv_softc *, char *);
+struct gv_plex		*gv_find_plex(struct gv_softc *, char *);
+struct gv_sd		*gv_find_sd(struct gv_softc *, char *);
+struct gv_volume	*gv_find_vol(struct gv_softc *, char *);
+void			 gv_format_config(struct gv_softc *, struct sbuf *, int,
+			     char *);
+int			 gv_is_striped(struct gv_plex *);
+int			 gv_consumer_is_open(struct g_consumer *);
+int			 gv_provider_is_open(struct g_provider *);
+int			 gv_object_type(struct gv_softc *, char *);
+void			 gv_parse_config(struct gv_softc *, char *,
+			     struct gv_drive *);
+int			 gv_sd_to_drive(struct gv_sd *, struct gv_drive *);
+int			 gv_sd_to_plex(struct gv_sd *, struct gv_plex *);
+int			 gv_sdcount(struct gv_plex *, int);
+void			 gv_update_plex_config(struct gv_plex *);
+void			 gv_update_vol_size(struct gv_volume *, off_t);
+off_t			 gv_vol_size(struct gv_volume *);
+off_t			 gv_plex_size(struct gv_plex *);
+int			 gv_plexdown(struct gv_volume *);
+int			 gv_attach_plex(struct gv_plex *, struct gv_volume *,
+			     int);
+int			 gv_attach_sd(struct gv_sd *, struct gv_plex *, off_t,
+			     int);
+int			 gv_detach_plex(struct gv_plex *, int);
+int			 gv_detach_sd(struct gv_sd *, int);
+
+/* geom_vinum.c */
+void	gv_worker(void *);
+void	gv_post_event(struct gv_softc *, int, void *, void *, intmax_t,
+	    intmax_t);
+void	gv_drive_tasted(struct gv_softc *, struct g_provider *);
+void	gv_drive_lost(struct gv_softc *, struct gv_drive *);
+void	gv_setup_objects(struct gv_softc *);
+void	gv_start(struct bio *);
+int	gv_access(struct g_provider *, int, int, int);
+void	gv_cleanup(struct gv_softc *);
+
+/* geom_vinum_volume.c */
+void	gv_done(struct bio *);
+void	gv_volume_start(struct gv_softc *, struct bio *);
+void	gv_volume_flush(struct gv_volume *);
+void	gv_bio_done(struct gv_softc *, struct bio *);
+
+/* geom_vinum_plex.c */
+void	gv_plex_start(struct gv_plex *, struct bio *);
+void	gv_plex_raid5_done(struct gv_plex *, struct bio *);
+void	gv_plex_normal_done(struct gv_plex *, struct bio *);
+int	gv_grow_request(struct gv_plex *, off_t, off_t, int, caddr_t);
+void	gv_grow_complete(struct gv_plex *, struct bio *);
+void	gv_init_request(struct gv_sd *, off_t, caddr_t, off_t);
+void	gv_init_complete(struct gv_plex *, struct bio *);
+void	gv_parity_request(struct gv_plex *, int, off_t);
+void	gv_parity_complete(struct gv_plex *, struct bio *);
+void	gv_rebuild_complete(struct gv_plex *, struct bio *);
+int	gv_sync_request(struct gv_plex *, struct gv_plex *, off_t, off_t, int,
+	    caddr_t);
+int	gv_sync_complete(struct gv_plex *, struct bio *);
 
 extern	u_int	g_vinum_debug;
 
diff --git a/sys/geom/vinum/geom_vinum_create.c b/sys/geom/vinum/geom_vinum_create.c
new file mode 100644
index 0000000..3f8212c
--- /dev/null
+++ b/sys/geom/vinum/geom_vinum_create.c
@@ -0,0 +1,614 @@
+/*-
+ * Copyright (c) 2007 Lukas Ertl
+ * Copyright (c) 2007, 2009 Ulf Lilleengen
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bio.h>
+#include <sys/conf.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/systm.h>
+#include <sys/vimage.h>
+
+#include <geom/geom.h>
+#include <geom/vinum/geom_vinum_var.h>
+#include <geom/vinum/geom_vinum.h>
+
+#define DEFAULT_STRIPESIZE	262144
+
+/*
+ * Create a new drive object, either by user request, during taste of the drive
+ * itself, or because it was referenced by a subdisk during taste.
+ */
+int
+gv_create_drive(struct gv_softc *sc, struct gv_drive *d)
+{
+	struct g_geom *gp;
+	struct g_provider *pp;
+	struct g_consumer *cp, *cp2;
+	struct gv_drive *d2;
+	struct gv_hdr *hdr;
+	struct gv_freelist *fl;
+
+	KASSERT(d != NULL, ("gv_create_drive: NULL d"));
+
+	gp = sc->geom;
+
+	pp = NULL;
+	cp = cp2 = NULL;
+
+	/* The drive already has a consumer if it was tasted before. */
+	if (d->consumer != NULL) {
+		cp = d->consumer;
+		cp->private = d;
+		pp = cp->provider;
+	} else if (!(d->flags & GV_DRIVE_REFERENCED)) {
+		if (gv_find_drive(sc, d->name) != NULL) {
+			G_VINUM_DEBUG(0, "drive '%s' already exists", d->name);
+			g_free(d);
+			return (GV_ERR_CREATE);
+		}
+
+		if (gv_find_drive_device(sc, d->device) != NULL) {
+			G_VINUM_DEBUG(0, "provider '%s' already in use by "
+			    "gvinum", d->device);
+			return (GV_ERR_CREATE);
+		}
+
+		pp = g_provider_by_name(d->device);
+		if (pp == NULL) {
+			G_VINUM_DEBUG(0, "create '%s': device '%s' disappeared",
+			    d->name, d->device);
+			g_free(d);
+			return (GV_ERR_CREATE);
+		}
+
+		g_topology_lock();
+		cp = g_new_consumer(gp);
+		if (g_attach(cp, pp) != 0) {
+			g_destroy_consumer(cp);
+			g_topology_unlock();
+			G_VINUM_DEBUG(0, "create drive '%s': couldn't attach",
+			    d->name);
+			g_free(d);
+			return (GV_ERR_CREATE);
+		}
+		g_topology_unlock();
+
+		d->consumer = cp;
+		cp->private = d;
+	}
+
+	/*
+	 * If this was just a "referenced" drive, we're almost finished, but
+	 * insert this drive not on the head of the drives list, as
+	 * gv_drive_is_newer() expects a "real" drive from LIST_FIRST().
+	 */
+	if (d->flags & GV_DRIVE_REFERENCED) {
+		snprintf(d->device, sizeof(d->device), "???");
+		d2 = LIST_FIRST(&sc->drives);
+		if (d2 == NULL)
+			LIST_INSERT_HEAD(&sc->drives, d, drive);
+		else
+			LIST_INSERT_AFTER(d2, d, drive);
+		return (0);
+	}
+
+	/*
+	 * Update access counts of the new drive to those of an already
+	 * existing drive.
+	 */
+	LIST_FOREACH(d2, &sc->drives, drive) {
+		if ((d == d2) || (d2->consumer == NULL))
+			continue;
+
+		cp2 = d2->consumer;
+		g_topology_lock();
+		if ((cp2->acr || cp2->acw || cp2->ace) &&
+		    (g_access(cp, cp2->acr, cp2->acw, cp2->ace) != 0)) {
+			g_detach(cp);
+			g_destroy_consumer(cp);
+			g_topology_unlock();
+			G_VINUM_DEBUG(0, "create drive '%s': couldn't update "
+			    "access counts", d->name);
+			if (d->hdr != NULL)
+				g_free(d->hdr);
+			g_free(d);
+			return (GV_ERR_CREATE);
+		}
+		g_topology_unlock();
+		break;
+	}
+
+	d->size = pp->mediasize - GV_DATA_START;
+	d->avail = d->size;
+	d->vinumconf = sc;
+	LIST_INIT(&d->subdisks);
+	LIST_INIT(&d->freelist);
+
+	/* The header might have been set during taste. */
+	if (d->hdr == NULL) {
+		hdr = g_malloc(sizeof(*hdr), M_WAITOK | M_ZERO);
+		hdr->magic = GV_MAGIC;
+		hdr->config_length = GV_CFG_LEN;
+		mtx_lock(&hostname_mtx);
+		bcopy(G_hostname, hdr->label.sysname, GV_HOSTNAME_LEN);
+		mtx_unlock(&hostname_mtx);
+		strlcpy(hdr->label.name, d->name, sizeof(hdr->label.name));
+		microtime(&hdr->label.date_of_birth);
+		d->hdr = hdr;
+	}
+
+	/* We also need a freelist entry. */
+	fl = g_malloc(sizeof(struct gv_freelist), M_WAITOK | M_ZERO);
+	fl->offset = GV_DATA_START;
+	fl->size = d->avail;
+	LIST_INSERT_HEAD(&d->freelist, fl, freelist);
+	d->freelist_entries = 1;
+
+	if (gv_find_drive(sc, d->name) == NULL)
+		LIST_INSERT_HEAD(&sc->drives, d, drive);
+
+	gv_set_drive_state(d, GV_DRIVE_UP, 0);
+	return (0);
+}
+
+int
+gv_create_volume(struct gv_softc *sc, struct gv_volume *v)
+{
+	KASSERT(v != NULL, ("gv_create_volume: NULL v"));
+
+	v->vinumconf = sc;
+	v->flags |= GV_VOL_NEWBORN;
+	LIST_INIT(&v->plexes);
+	LIST_INSERT_HEAD(&sc->volumes, v, volume);
+	v->wqueue = g_malloc(sizeof(struct bio_queue_head), M_WAITOK | M_ZERO);
+	bioq_init(v->wqueue);
+	return (0);
+}
+
+int
+gv_create_plex(struct gv_softc *sc, struct gv_plex *p)
+{
+	struct gv_volume *v;
+
+	KASSERT(p != NULL, ("gv_create_plex: NULL p"));
+
+	/* Find the volume this plex should be attached to. */
+	v = gv_find_vol(sc, p->volume);
+	if (v == NULL) {
+		G_VINUM_DEBUG(0, "create plex '%s': volume '%s' not found",
+		    p->name, p->volume);
+		g_free(p);
+		return (GV_ERR_CREATE);
+	}
+	if (!(v->flags & GV_VOL_NEWBORN))
+		p->flags |= GV_PLEX_ADDED;
+	p->vol_sc = v;
+	v->plexcount++;
+	p->vinumconf = sc;
+	p->synced = 0;
+	p->flags |= GV_PLEX_NEWBORN;
+	LIST_INSERT_HEAD(&v->plexes, p, in_volume);
+	LIST_INIT(&p->subdisks);
+	TAILQ_INIT(&p->packets);
+	LIST_INSERT_HEAD(&sc->plexes, p, plex);
+	p->bqueue = g_malloc(sizeof(struct bio_queue_head), M_WAITOK | M_ZERO);
+	bioq_init(p->bqueue);
+	p->wqueue = g_malloc(sizeof(struct bio_queue_head), M_WAITOK | M_ZERO);
+	bioq_init(p->wqueue);
+	p->rqueue = g_malloc(sizeof(struct bio_queue_head), M_WAITOK | M_ZERO);
+	bioq_init(p->rqueue);
+	return (0);
+}
+
+int
+gv_create_sd(struct gv_softc *sc, struct gv_sd *s)
+{
+	struct gv_plex *p;
+	struct gv_drive *d;
+
+	KASSERT(s != NULL, ("gv_create_sd: NULL s"));
+
+	/* Find the drive where this subdisk should be put on. */
+	d = gv_find_drive(sc, s->drive);
+	if (d == NULL) {
+		/*
+		 * It's possible that the subdisk references a drive that
+		 * doesn't exist yet (during the taste process), so create a
+		 * practically empty "referenced" drive.
+		 */
+		if (s->flags & GV_SD_TASTED) {
+			d = g_malloc(sizeof(struct gv_drive),
+			    M_WAITOK | M_ZERO);
+			d->flags |= GV_DRIVE_REFERENCED;
+			strlcpy(d->name, s->drive, sizeof(d->name));
+			gv_create_drive(sc, d);
+		} else {
+			G_VINUM_DEBUG(0, "create sd '%s': drive '%s' not found",
+			    s->name, s->drive);
+			g_free(s);
+			return (GV_ERR_CREATE);
+		}
+	}
+
+	/* Find the plex where this subdisk belongs to. */
+	p = gv_find_plex(sc, s->plex);
+	if (p == NULL) {
+		G_VINUM_DEBUG(0, "create sd '%s': plex '%s' not found",
+		    s->name, s->plex);
+		g_free(s);
+		return (GV_ERR_CREATE);
+	}
+
+	/*
+	 * First we give the subdisk to the drive, to handle autosized
+	 * values ...
+	 */
+	if (gv_sd_to_drive(s, d) != 0) {
+		g_free(s);
+		return (GV_ERR_CREATE);
+	}
+
+	/*
+	 * Then, we give the subdisk to the plex; we check if the
+	 * given values are correct and maybe adjust them.
+	 */
+	if (gv_sd_to_plex(s, p) != 0) {
+		G_VINUM_DEBUG(0, "unable to give sd '%s' to plex '%s'",
+		    s->name, p->name);
+		if (s->drive_sc && !(s->drive_sc->flags & GV_DRIVE_REFERENCED))
+			LIST_REMOVE(s, from_drive);
+		gv_free_sd(s);
+		g_free(s);
+		/*
+		 * If this subdisk can't be created, we won't create
+		 * the attached plex either, if it is also a new one.
+		 */
+		if (!(p->flags & GV_PLEX_NEWBORN))
+			return (GV_ERR_CREATE);
+		gv_rm_plex(sc, p);
+		return (GV_ERR_CREATE);
+	}
+	s->flags |= GV_SD_NEWBORN;
+
+	s->vinumconf = sc;
+	LIST_INSERT_HEAD(&sc->subdisks, s, sd);
+
+	return (0);
+}
+
+/*
+ * Create a concatenated volume from specified drives or drivegroups.
+ */
+void
+gv_concat(struct g_geom *gp, struct gctl_req *req)
+{
+	struct gv_drive *d;
+	struct gv_sd *s;
+	struct gv_volume *v;
+	struct gv_plex *p;
+	struct gv_softc *sc;
+	char *drive, buf[30], *vol;
+	int *drives, *flags, dcount;
+
+	sc = gp->softc;
+	dcount = 0;
+	vol = gctl_get_param(req, "name", NULL);
+	if (vol == NULL) {
+		gctl_error(req, "volume names not given");	
+		return;
+	}
+
+	flags = gctl_get_paraml(req, "flags", sizeof(*flags));
+	drives = gctl_get_paraml(req, "drives", sizeof(*drives));
+
+	if (drives == NULL) { 
+		gctl_error(req, "drive names not given");
+		return;
+	}
+
+	/* First we create the volume. */
+	v = g_malloc(sizeof(*v), M_WAITOK | M_ZERO);
+	strlcpy(v->name, vol, sizeof(v->name));
+	v->state = GV_VOL_UP;
+	gv_post_event(sc, GV_EVENT_CREATE_VOLUME, v, NULL, 0, 0);
+
+	/* Then we create the plex. */
+	p = g_malloc(sizeof(*p), M_WAITOK | M_ZERO);
+	snprintf(p->name, sizeof(p->name), "%s.p%d", v->name, v->plexcount);
+	strlcpy(p->volume, v->name, sizeof(p->volume));
+	p->org = GV_PLEX_CONCAT;
+	p->stripesize = 0;
+	gv_post_event(sc, GV_EVENT_CREATE_PLEX, p, NULL, 0, 0);
+
+	/* Drives are first (right now) priority */
+	for (dcount = 0; dcount < *drives; dcount++) {
+		snprintf(buf, sizeof(buf), "drive%d", dcount);
+		drive = gctl_get_param(req, buf, NULL);
+		d = gv_find_drive(sc, drive);
+		if (d == NULL) {
+			gctl_error(req, "No such drive '%s'", drive);
+			continue;
+		}
+		s = g_malloc(sizeof(*s), M_WAITOK | M_ZERO);
+		snprintf(s->name, sizeof(s->name), "%s.s%d", p->name, dcount);
+		strlcpy(s->plex, p->name, sizeof(s->plex));
+		strlcpy(s->drive, drive, sizeof(s->drive));
+		s->plex_offset = -1;
+		s->drive_offset = -1;
+		s->size = -1;
+		gv_post_event(sc, GV_EVENT_CREATE_SD, s, NULL, 0, 0);
+	}
+	gv_post_event(sc, GV_EVENT_SETUP_OBJECTS, sc, NULL, 0, 0);
+	gv_post_event(sc, GV_EVENT_SAVE_CONFIG, sc, NULL, 0, 0);
+}
+
+/*
+ * Create a mirrored volume from specified drives or drivegroups.
+ */
+void
+gv_mirror(struct g_geom *gp, struct gctl_req *req)
+{
+	struct gv_drive *d;
+	struct gv_sd *s;
+	struct gv_volume *v;
+	struct gv_plex *p;
+	struct gv_softc *sc;
+	char *drive, buf[30], *vol;
+	int *drives, *flags, dcount, pcount, scount;
+
+	sc = gp->softc;
+	dcount = 0;
+	scount = 0;
+	pcount = 0;
+	vol = gctl_get_param(req, "name", NULL);
+	if (vol == NULL) {
+		gctl_error(req, "volume's not given");	
+		return;
+	}
+
+	flags = gctl_get_paraml(req, "flags", sizeof(*flags));
+	drives = gctl_get_paraml(req, "drives", sizeof(*drives));
+
+	if (drives == NULL) { 
+		gctl_error(req, "drives not given");
+		return;
+	}
+
+	/* We must have an even number of drives. */
+	if (*drives % 2 != 0) {
+		gctl_error(req, "mirror organization must have an even number "
+		    "of drives");
+		return;
+	}
+	if (*flags & GV_FLAG_S && *drives < 4) {
+		gctl_error(req, "must have at least 4 drives for striped plex");
+		return;
+	}
+
+	/* First we create the volume. */
+	v = g_malloc(sizeof(*v), M_WAITOK | M_ZERO);
+	strlcpy(v->name, vol, sizeof(v->name));
+	v->state = GV_VOL_UP;
+	gv_post_event(sc, GV_EVENT_CREATE_VOLUME, v, NULL, 0, 0);
+
+	/* Then we create the plexes. */
+	for (pcount = 0; pcount < 2; pcount++) {
+		p = g_malloc(sizeof(*p), M_WAITOK | M_ZERO);
+		snprintf(p->name, sizeof(p->name), "%s.p%d", v->name,
+		    pcount);
+		strlcpy(p->volume, v->name, sizeof(p->volume));
+		if (*flags & GV_FLAG_S) {
+			p->org = GV_PLEX_STRIPED;
+			p->stripesize = DEFAULT_STRIPESIZE;
+		} else {
+			p->org = GV_PLEX_CONCAT;
+			p->stripesize = -1;
+		}
+		gv_post_event(sc, GV_EVENT_CREATE_PLEX, p, NULL, 0, 0);
+
+		/*
+		 * We just gives each even drive to plex one, and each odd to
+		 * plex two.
+		 */
+		scount = 0;
+		for (dcount = pcount; dcount < *drives; dcount += 2) {
+			snprintf(buf, sizeof(buf), "drive%d", dcount);
+			drive = gctl_get_param(req, buf, NULL);
+			d = gv_find_drive(sc, drive);
+			if (d == NULL) {
+				gctl_error(req, "No such drive '%s', aborting",
+				    drive);
+				scount++;
+				break;
+			}
+			s = g_malloc(sizeof(*s), M_WAITOK | M_ZERO);
+			snprintf(s->name, sizeof(s->name), "%s.s%d", p->name,
+			    scount);
+			strlcpy(s->plex, p->name, sizeof(s->plex));
+			strlcpy(s->drive, drive, sizeof(s->drive));
+			s->plex_offset = -1;
+			s->drive_offset = -1;
+			s->size = -1;
+			gv_post_event(sc, GV_EVENT_CREATE_SD, s, NULL, 0, 0);
+			scount++;
+		}
+	}
+	gv_post_event(sc, GV_EVENT_SETUP_OBJECTS, sc, NULL, 0, 0);
+	gv_post_event(sc, GV_EVENT_SAVE_CONFIG, sc, NULL, 0, 0);
+}
+
+void
+gv_raid5(struct g_geom *gp, struct gctl_req *req)
+{
+	struct gv_softc *sc;
+	struct gv_drive *d;
+	struct gv_volume *v;
+	struct gv_plex *p;
+	struct gv_sd *s;
+	int *drives, *flags, dcount;
+	char *vol, *drive, buf[30];
+	off_t *stripesize;
+
+	dcount = 0;
+	sc = gp->softc;
+
+	vol = gctl_get_param(req, "name", NULL);
+	if (vol == NULL) {
+		gctl_error(req, "volume's not given");
+		return;
+	}
+	flags = gctl_get_paraml(req, "flags", sizeof(*flags));
+	drives = gctl_get_paraml(req, "drives", sizeof(*drives));
+	stripesize = gctl_get_paraml(req, "stripesize", sizeof(*stripesize));
+
+	if (stripesize == NULL) {
+		gctl_error(req, "no stripesize given");
+		return;
+	}
+
+	if (drives == NULL) {
+		gctl_error(req, "drives not given");
+		return;
+	}
+
+	/* We must have at least three drives. */
+	if (*drives < 3) {
+		gctl_error(req, "must have at least three drives for this "
+		    "plex organisation");
+		return;
+	}
+	/* First we create the volume. */
+	v = g_malloc(sizeof(*v), M_WAITOK | M_ZERO);
+	strlcpy(v->name, vol, sizeof(v->name));
+	v->state = GV_VOL_UP;
+	gv_post_event(sc, GV_EVENT_CREATE_VOLUME, v, NULL, 0, 0);
+
+	/* Then we create the plex. */
+	p = g_malloc(sizeof(*p), M_WAITOK | M_ZERO);
+	snprintf(p->name, sizeof(p->name), "%s.p%d", v->name, v->plexcount);
+	strlcpy(p->volume, v->name, sizeof(p->volume));
+	p->org = GV_PLEX_RAID5;
+	p->stripesize = *stripesize;
+	gv_post_event(sc, GV_EVENT_CREATE_PLEX, p, NULL, 0, 0);
+
+	/* Create subdisks on drives. */
+	for (dcount = 0; dcount < *drives; dcount++) {
+		snprintf(buf, sizeof(buf), "drive%d", dcount);
+		drive = gctl_get_param(req, buf, NULL);
+		d = gv_find_drive(sc, drive);
+		if (d == NULL) {
+			gctl_error(req, "No such drive '%s'", drive);
+			continue;
+		}
+		s = g_malloc(sizeof(*s), M_WAITOK | M_ZERO);
+		snprintf(s->name, sizeof(s->name), "%s.s%d", p->name, dcount);
+		strlcpy(s->plex, p->name, sizeof(s->plex));
+		strlcpy(s->drive, drive, sizeof(s->drive));
+		s->plex_offset = -1;
+		s->drive_offset = -1;
+		s->size = -1;
+		gv_post_event(sc, GV_EVENT_CREATE_SD, s, NULL, 0, 0);
+	}
+	gv_post_event(sc, GV_EVENT_SETUP_OBJECTS, sc, NULL, 0, 0);
+	gv_post_event(sc, GV_EVENT_SAVE_CONFIG, sc, NULL, 0, 0);
+}
+
+/*
+ * Create a striped volume from specified drives or drivegroups.
+ */
+void
+gv_stripe(struct g_geom *gp, struct gctl_req *req)
+{
+	struct gv_drive *d;
+	struct gv_sd *s;
+	struct gv_volume *v;
+	struct gv_plex *p;
+	struct gv_softc *sc;
+	char *drive, buf[30], *vol;
+	int *drives, *flags, dcount, pcount;
+
+	sc = gp->softc;
+	dcount = 0;
+	pcount = 0;
+	vol = gctl_get_param(req, "name", NULL);
+	if (vol == NULL) {
+		gctl_error(req, "volume's not given");	
+		return;
+	}
+	flags = gctl_get_paraml(req, "flags", sizeof(*flags));
+	drives = gctl_get_paraml(req, "drives", sizeof(*drives));
+
+	if (drives == NULL) { 
+		gctl_error(req, "drives not given");
+		return;
+	}
+
+	/* We must have at least two drives. */
+	if (*drives < 2) {
+		gctl_error(req, "must have at least 2 drives");
+		return;
+	}
+
+	/* First we create the volume. */
+	v = g_malloc(sizeof(*v), M_WAITOK | M_ZERO);
+	strlcpy(v->name, vol, sizeof(v->name));
+	v->state = GV_VOL_UP;
+	gv_post_event(sc, GV_EVENT_CREATE_VOLUME, v, NULL, 0, 0);
+
+	/* Then we create the plex. */
+	p = g_malloc(sizeof(*p), M_WAITOK | M_ZERO);
+	snprintf(p->name, sizeof(p->name), "%s.p%d", v->name, v->plexcount);
+	strlcpy(p->volume, v->name, sizeof(p->volume));
+	p->org = GV_PLEX_STRIPED;
+	p->stripesize = 262144;
+	gv_post_event(sc, GV_EVENT_CREATE_PLEX, p, NULL, 0, 0);
+
+	/* Create subdisks on drives. */
+	for (dcount = 0; dcount < *drives; dcount++) {
+		snprintf(buf, sizeof(buf), "drive%d", dcount);
+		drive = gctl_get_param(req, buf, NULL);
+		d = gv_find_drive(sc, drive);
+		if (d == NULL) {
+			gctl_error(req, "No such drive '%s'", drive);
+			continue;
+		}
+		s = g_malloc(sizeof(*s), M_WAITOK | M_ZERO);
+		snprintf(s->name, sizeof(s->name), "%s.s%d", p->name, dcount);
+		strlcpy(s->plex, p->name, sizeof(s->plex));
+		strlcpy(s->drive, drive, sizeof(s->drive));
+		s->plex_offset = -1;
+		s->drive_offset = -1;
+		s->size = -1;
+		gv_post_event(sc, GV_EVENT_CREATE_SD, s, NULL, 0, 0);
+	}
+	gv_post_event(sc, GV_EVENT_SETUP_OBJECTS, sc, NULL, 0, 0);
+	gv_post_event(sc, GV_EVENT_SAVE_CONFIG, sc, NULL, 0, 0);
+}
diff --git a/sys/geom/vinum/geom_vinum_drive.c b/sys/geom/vinum/geom_vinum_drive.c
index 6f2f0fc..1e9dd0b 100644
--- a/sys/geom/vinum/geom_vinum_drive.c
+++ b/sys/geom/vinum/geom_vinum_drive.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2004, 2005 Lukas Ertl
+ * Copyright (c) 2004, 2005, 2007 Lukas Ertl
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -27,35 +27,20 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <sys/param.h>
-#include <sys/bio.h>
-#include <sys/errno.h>
 #include <sys/endian.h>
-#include <sys/conf.h>
-#include <sys/kernel.h>
-#include <sys/kthread.h>
-#include <sys/libkern.h>
-#include <sys/lock.h>
 #include <sys/malloc.h>
-#include <sys/module.h>
-#include <sys/mutex.h>
-#include <sys/sbuf.h>
 #include <sys/systm.h>
-#include <sys/time.h>
 #include <sys/vimage.h>
 
 #include <geom/geom.h>
 #include <geom/vinum/geom_vinum_var.h>
 #include <geom/vinum/geom_vinum.h>
-#include <geom/vinum/geom_vinum_share.h>
 
 #define GV_LEGACY_I386	0
 #define GV_LEGACY_AMD64 1
 #define GV_LEGACY_SPARC64 2
 #define GV_LEGACY_POWERPC 3
 
-static void	gv_drive_dead(void *, int);
-static void	gv_drive_worker(void *);
 static int	gv_legacy_header_type(uint8_t *, int);
 
 /*
@@ -267,620 +252,96 @@ gv_write_header(struct g_consumer *cp, struct gv_hdr *m_hdr)
 	return (ret);
 }
 
+/* Save the vinum configuration back to each involved disk. */
 void
-gv_config_new_drive(struct gv_drive *d)
-{
-	struct gv_hdr *vhdr;
-	struct gv_freelist *fl;
-
-	KASSERT(d != NULL, ("config_new_drive: NULL d"));
-
-	vhdr = g_malloc(sizeof(*vhdr), M_WAITOK | M_ZERO);
-	vhdr->magic = GV_MAGIC;
-	vhdr->config_length = GV_CFG_LEN;
-
-	mtx_lock(&hostname_mtx);
-	bcopy(G_hostname, vhdr->label.sysname, GV_HOSTNAME_LEN);
-	mtx_unlock(&hostname_mtx);
-	strncpy(vhdr->label.name, d->name, GV_MAXDRIVENAME);
-	microtime(&vhdr->label.date_of_birth);
-
-	d->hdr = vhdr;
-
-	LIST_INIT(&d->subdisks);
-	LIST_INIT(&d->freelist);
-
-	fl = g_malloc(sizeof(struct gv_freelist), M_WAITOK | M_ZERO);
-	fl->offset = GV_DATA_START;
-	fl->size = d->avail;
-	LIST_INSERT_HEAD(&d->freelist, fl, freelist);
-	d->freelist_entries = 1;
-
-	d->bqueue = g_malloc(sizeof(struct bio_queue_head), M_WAITOK | M_ZERO);
-	bioq_init(d->bqueue);
-	mtx_init(&d->bqueue_mtx, "gv_drive", NULL, MTX_DEF);
-	kproc_create(gv_drive_worker, d, NULL, 0, 0, "gv_d %s", d->name);
-	d->flags |= GV_DRIVE_THREAD_ACTIVE;
-}
-
-void
-gv_save_config_all(struct gv_softc *sc)
+gv_save_config(struct gv_softc *sc)
 {
+	struct g_consumer *cp;
 	struct gv_drive *d;
-
-	g_topology_assert();
-
-	LIST_FOREACH(d, &sc->drives, drive) {
-		if (d->geom == NULL)
-			continue;
-		gv_save_config(NULL, d, sc);
-	}
-}
-
-/* Save the vinum configuration back to disk. */
-void
-gv_save_config(struct g_consumer *cp, struct gv_drive *d, struct gv_softc *sc)
-{
-	struct g_geom *gp;
-	struct g_consumer *cp2;
 	struct gv_hdr *vhdr, *hdr;
 	struct sbuf *sb;
+	struct timeval last_update;
 	int error;
 
-	g_topology_assert();
-
-	KASSERT(d != NULL, ("gv_save_config: null d"));
 	KASSERT(sc != NULL, ("gv_save_config: null sc"));
 
-	/*
-	 * We can't save the config on a drive that isn't up, but drives that
-	 * were just created aren't officially up yet, so we check a special
-	 * flag.
-	 */
-	if ((d->state != GV_DRIVE_UP) && !(d->flags && GV_DRIVE_NEWBORN))
-		return;
-
-	if (cp == NULL) {
-		gp = d->geom;
-		KASSERT(gp != NULL, ("gv_save_config: null gp"));
-		cp2 = LIST_FIRST(&gp->consumer);
-		KASSERT(cp2 != NULL, ("gv_save_config: null cp2"));
-	} else
-		cp2 = cp;
-
 	vhdr = g_malloc(GV_HDR_LEN, M_WAITOK | M_ZERO);
 	vhdr->magic = GV_MAGIC;
 	vhdr->config_length = GV_CFG_LEN;
-
-	hdr = d->hdr;
-	if (hdr == NULL) {
-		G_VINUM_DEBUG(0, "drive %s has NULL hdr", d->name);
-		g_free(vhdr);
-		return;
-	}
-	microtime(&hdr->label.last_update);
-	bcopy(&hdr->label, &vhdr->label, sizeof(struct gv_label));
+	microtime(&last_update);
 
 	sb = sbuf_new(NULL, NULL, GV_CFG_LEN, SBUF_FIXEDLEN);
 	gv_format_config(sc, sb, 1, NULL);
 	sbuf_finish(sb);
 
-	error = g_access(cp2, 0, 1, 0);
-	if (error) {
-		G_VINUM_DEBUG(0, "g_access failed on drive %s, errno %d",
-		    d->name, error);
-		sbuf_delete(sb);
-		g_free(vhdr);
-		return;
-	}
-	g_topology_unlock();
-
-	do {
-		error = gv_write_header(cp2, vhdr);
-		if (error) {
-			G_VINUM_DEBUG(0, "writing vhdr failed on drive %s, "
-			    "errno %d", d->name, error);
-			break;
-		}
-
-		error = g_write_data(cp2, GV_CFG_OFFSET, sbuf_data(sb),
-		    GV_CFG_LEN);
-		if (error) {
-			G_VINUM_DEBUG(0, "writing first config copy failed "
-			    "on drive %s, errno %d", d->name, error);
-			break;
-		}
-		
-		error = g_write_data(cp2, GV_CFG_OFFSET + GV_CFG_LEN,
-		    sbuf_data(sb), GV_CFG_LEN);
-		if (error)
-			G_VINUM_DEBUG(0, "writing second config copy failed "
-			    "on drive %s, errno %d", d->name, error);
-	} while (0);
-
-	g_topology_lock();
-	g_access(cp2, 0, -1, 0);
-	sbuf_delete(sb);
-	g_free(vhdr);
-
-	if (d->geom != NULL)
-		gv_drive_modify(d);
-}
-
-/* This resembles g_slice_access(). */
-static int
-gv_drive_access(struct g_provider *pp, int dr, int dw, int de)
-{
-	struct g_geom *gp;
-	struct g_consumer *cp;
-	struct g_provider *pp2;
-	struct gv_drive *d;
-	struct gv_sd *s, *s2;
-	int error;
-
-	gp = pp->geom;
-	cp = LIST_FIRST(&gp->consumer);
-	if (cp == NULL)
-		return (0);
-
-	d = gp->softc;
-	if (d == NULL)
-		return (0);
-
-	s = pp->private;
-	KASSERT(s != NULL, ("gv_drive_access: NULL s"));
-
-	LIST_FOREACH(s2, &d->subdisks, from_drive) {
-		if (s == s2)
-			continue;
-		if (s->drive_offset + s->size <= s2->drive_offset)
-			continue;
-		if (s2->drive_offset + s2->size <= s->drive_offset)
+	LIST_FOREACH(d, &sc->drives, drive) {
+		/*
+		 * We can't save the config on a drive that isn't up, but
+		 * drives that were just created aren't officially up yet, so
+		 * we check a special flag.
+		 */
+		if (d->state != GV_DRIVE_UP)
 			continue;
 
-		/* Overlap. */
-		pp2 = s2->provider;
-		KASSERT(s2 != NULL, ("gv_drive_access: NULL s2"));
-		if ((pp->acw + dw) > 0 && pp2->ace > 0)
-			return (EPERM);
-		if ((pp->ace + de) > 0 && pp2->acw > 0)
-			return (EPERM);
-	}
-
-	error = g_access(cp, dr, dw, de);
-	return (error);
-}
-
-static void
-gv_drive_done(struct bio *bp)
-{
-	struct gv_drive *d;
-
-	/* Put the BIO on the worker queue again. */
-	d = bp->bio_from->geom->softc;
-	bp->bio_cflags |= GV_BIO_DONE;
-	mtx_lock(&d->bqueue_mtx);
-	bioq_insert_tail(d->bqueue, bp);
-	wakeup(d);
-	mtx_unlock(&d->bqueue_mtx);
-}
-
-
-static void
-gv_drive_start(struct bio *bp)
-{
-	struct gv_drive *d;
-	struct gv_sd *s;
-
-	switch (bp->bio_cmd) {
-	case BIO_READ:
-	case BIO_WRITE:
-	case BIO_DELETE:
-		break;
-	case BIO_GETATTR:
-	default:
-		g_io_deliver(bp, EOPNOTSUPP);
-		return;
-	}
-
-	s = bp->bio_to->private;
-	if ((s->state == GV_SD_DOWN) || (s->state == GV_SD_STALE)) {
-		g_io_deliver(bp, ENXIO);
-		return;
-	}
-
-	d = bp->bio_to->geom->softc;
-
-	/*
-	 * Put the BIO on the worker queue, where the worker thread will pick
-	 * it up.
-	 */
-	mtx_lock(&d->bqueue_mtx);
-	bioq_disksort(d->bqueue, bp);
-	wakeup(d);
-	mtx_unlock(&d->bqueue_mtx);
-
-}
-
-static void
-gv_drive_worker(void *arg)
-{
-	struct bio *bp, *cbp;
-	struct g_geom *gp;
-	struct g_provider *pp;
-	struct gv_drive *d;
-	struct gv_sd *s;
-	int error;
-
-	d = arg;
-
-	mtx_lock(&d->bqueue_mtx);
-	for (;;) {
-		/* We were signaled to exit. */
-		if (d->flags & GV_DRIVE_THREAD_DIE)
-			break;
-
-		/* Take the first BIO from out queue. */
-		bp = bioq_takefirst(d->bqueue);
-		if (bp == NULL) {
-			msleep(d, &d->bqueue_mtx, PRIBIO, "-", hz/10);
+		cp = d->consumer;
+		if (cp == NULL) {
+			G_VINUM_DEBUG(0, "drive '%s' has no consumer!",
+			    d->name);
 			continue;
- 		}
-		mtx_unlock(&d->bqueue_mtx);
- 
-		pp = bp->bio_to;
-		gp = pp->geom;
-
-		/* Completed request. */
-		if (bp->bio_cflags & GV_BIO_DONE) {
-			error = bp->bio_error;
-
-			/* Deliver the original request. */
-			g_std_done(bp);
-
-			/* The request had an error, we need to clean up. */
-			if (error != 0) {
-				g_topology_lock();
-				gv_set_drive_state(d, GV_DRIVE_DOWN,
-				    GV_SETSTATE_FORCE | GV_SETSTATE_CONFIG);
-				g_topology_unlock();
-				g_post_event(gv_drive_dead, d, M_WAITOK, d,
-				    NULL);
-			}
-
-		/* New request, needs to be sent downwards. */
-		} else {
-			s = pp->private;
-
-			if ((s->state == GV_SD_DOWN) ||
-			    (s->state == GV_SD_STALE)) {
-				g_io_deliver(bp, ENXIO);
-				mtx_lock(&d->bqueue_mtx);
-				continue;
-			}
-			if (bp->bio_offset > s->size) {
-				g_io_deliver(bp, EINVAL);
-				mtx_lock(&d->bqueue_mtx);
-				continue;
-			}
-
-			cbp = g_clone_bio(bp);
-			if (cbp == NULL) {
-				g_io_deliver(bp, ENOMEM);
-				mtx_lock(&d->bqueue_mtx);
-				continue;
-			}
-			if (cbp->bio_offset + cbp->bio_length > s->size)
-				cbp->bio_length = s->size -
-				    cbp->bio_offset;
-			cbp->bio_done = gv_drive_done;
-			cbp->bio_offset += s->drive_offset;
-			g_io_request(cbp, LIST_FIRST(&gp->consumer));
-		}
-
-		mtx_lock(&d->bqueue_mtx);
-	}
-
-	while ((bp = bioq_takefirst(d->bqueue)) != NULL) {
-		mtx_unlock(&d->bqueue_mtx);
-		if (bp->bio_cflags & GV_BIO_DONE) 
-			g_std_done(bp);
-		else
-			g_io_deliver(bp, ENXIO);
-		mtx_lock(&d->bqueue_mtx);
-	}
-	mtx_unlock(&d->bqueue_mtx);
-	d->flags |= GV_DRIVE_THREAD_DEAD;
-
-	kproc_exit(ENXIO);
-}
-
-
-static void
-gv_drive_orphan(struct g_consumer *cp)
-{
-	struct g_geom *gp;
-	struct gv_drive *d;
-
-	g_topology_assert();
-	gp = cp->geom;
-	g_trace(G_T_TOPOLOGY, "gv_drive_orphan(%s)", gp->name);
-	d = gp->softc;
-	if (d != NULL) {
-		gv_set_drive_state(d, GV_DRIVE_DOWN,
-		    GV_SETSTATE_FORCE | GV_SETSTATE_CONFIG);
-		g_post_event(gv_drive_dead, d, M_WAITOK, d, NULL);
-	} else
-		g_wither_geom(gp, ENXIO);
-}
-
-static struct g_geom *
-gv_drive_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
-{
-	struct g_geom *gp, *gp2;
-	struct g_consumer *cp;
-	struct gv_drive *d;
-	struct gv_sd *s;
-	struct gv_softc *sc;
-	struct gv_freelist *fl;
-	struct gv_hdr *vhdr;
-	int error;
-	char *buf, errstr[ERRBUFSIZ];
-
-	vhdr = NULL;
-	d = NULL;
-
-	g_trace(G_T_TOPOLOGY, "gv_drive_taste(%s, %s)", mp->name, pp->name);
-	g_topology_assert();
-
-	/* Find the VINUM class and its associated geom. */
-	gp2 = find_vinum_geom();
-	if (gp2 == NULL)
-		return (NULL);
-	sc = gp2->softc;
-
-	gp = g_new_geomf(mp, "%s.vinumdrive", pp->name);
-	gp->start = gv_drive_start;
-	gp->orphan = gv_drive_orphan;
-	gp->access = gv_drive_access;
-	gp->start = gv_drive_start;
-
-	cp = g_new_consumer(gp);
-	g_attach(cp, pp);
-	error = g_access(cp, 1, 0, 0);
-	if (error) {
-		g_detach(cp);
-		g_destroy_consumer(cp);
-		g_destroy_geom(gp);
-		return (NULL);
-	}
-
-	g_topology_unlock();
-
-	/* Now check if the provided slice is a valid vinum drive. */
-	do {
-		vhdr = g_malloc(GV_HDR_LEN, M_WAITOK | M_ZERO);
-		error = gv_read_header(cp, vhdr);
-		if (error) {
-			g_free(vhdr);
-			break;
 		}
 
-		/* A valid vinum drive, let's parse the on-disk information. */
-		buf = g_read_data(cp, GV_CFG_OFFSET, GV_CFG_LEN, NULL);
-		if (buf == NULL) {
+		hdr = d->hdr;
+		if (hdr == NULL) {
+			G_VINUM_DEBUG(0, "drive '%s' has no header",
+			    d->name);
 			g_free(vhdr);
-			break;
+			continue;
 		}
+		bcopy(&last_update, &hdr->label.last_update,
+		    sizeof(struct timeval));
+		bcopy(&hdr->label, &vhdr->label, sizeof(struct gv_label));
 		g_topology_lock();
-		gv_parse_config(sc, buf, 1);
-		g_free(buf);
-
-		/*
-		 * Let's see if this drive is already known in the
-		 * configuration.
-		 */
-		d = gv_find_drive(sc, vhdr->label.name);
-
-		/* We already know about this drive. */
-		if (d != NULL) {
-			/* Check if this drive already has a geom. */
-			if (d->geom != NULL) {
-				g_topology_unlock();
-				g_free(vhdr);
-				break;
-			}
-			bcopy(vhdr, d->hdr, sizeof(*vhdr));
-			g_free(vhdr);
-
-		/* This is a new drive. */
-		} else {
-			d = g_malloc(sizeof(*d), M_WAITOK | M_ZERO);
-
-			/* Initialize all needed variables. */
-			d->size = pp->mediasize - GV_DATA_START;
-			d->avail = d->size;
-			d->hdr = vhdr;
-			strncpy(d->name, vhdr->label.name, GV_MAXDRIVENAME);
-			LIST_INIT(&d->subdisks);
-			LIST_INIT(&d->freelist);
-
-			/* We also need a freelist entry. */
-			fl = g_malloc(sizeof(*fl), M_WAITOK | M_ZERO);
-			fl->offset = GV_DATA_START;
-			fl->size = d->avail;
-			LIST_INSERT_HEAD(&d->freelist, fl, freelist);
-			d->freelist_entries = 1;
-
-			/* Save it into the main configuration. */
-			LIST_INSERT_HEAD(&sc->drives, d, drive);
-		}
-
-		/*
-		 * Create bio queue, queue mutex and a worker thread, if
-		 * necessary.
-		 */
-		if (d->bqueue == NULL) {
-			d->bqueue = g_malloc(sizeof(struct bio_queue_head),
-			    M_WAITOK | M_ZERO);
-			bioq_init(d->bqueue);
-		}
-		if (mtx_initialized(&d->bqueue_mtx) == 0)
-			mtx_init(&d->bqueue_mtx, "gv_drive", NULL, MTX_DEF);
-
-		if (!(d->flags & GV_DRIVE_THREAD_ACTIVE)) {
-			kproc_create(gv_drive_worker, d, NULL, 0, 0,
-			    "gv_d %s", d->name);
-			d->flags |= GV_DRIVE_THREAD_ACTIVE;
+		error = g_access(cp, 0, 1, 0);
+		if (error) {
+			G_VINUM_DEBUG(0, "g_access failed on "
+			    "drive %s, errno %d", d->name, error);
+			g_topology_unlock();
+			continue;
 		}
+		g_topology_unlock();
 
-		g_access(cp, -1, 0, 0);
-
-		gp->softc = d;
-		d->geom = gp;
-		d->vinumconf = sc;
-		strncpy(d->device, pp->name, GV_MAXDRIVENAME);
-
-		/*
-		 * Find out which subdisks belong to this drive and crosslink
-		 * them.
-		 */
-		LIST_FOREACH(s, &sc->subdisks, sd) {
-			if (!strncmp(s->drive, d->name, GV_MAXDRIVENAME))
-				/* XXX: errors ignored */
-				gv_sd_to_drive(sc, d, s, errstr,
-				    sizeof(errstr));
+		error = gv_write_header(cp, vhdr);
+		if (error) {
+			G_VINUM_DEBUG(0, "writing vhdr failed on drive %s, "
+			    "errno %d", d->name, error);
+			g_topology_lock();
+			g_access(cp, 0, -1, 0);
+			g_topology_unlock();
+			continue;
 		}
-
-		/* This drive is now up for sure. */
-		gv_set_drive_state(d, GV_DRIVE_UP, 0);
-
-		/*
-		 * If there are subdisks on this drive, we need to create
-		 * providers for them.
-		 */ 
-		if (d->sdcount)
-			gv_drive_modify(d);
-
-		return (gp);
-
-	} while (0);
-
-	g_topology_lock();
-	g_access(cp, -1, 0, 0);
-
-	g_detach(cp);
-	g_destroy_consumer(cp);
-	g_destroy_geom(gp);
-	return (NULL);
-}
-
-/*
- * Modify the providers for the given drive 'd'.  It is assumed that the
- * subdisk list of 'd' is already correctly set up.
- */
-void
-gv_drive_modify(struct gv_drive *d)
-{
-	struct g_geom *gp;
-	struct g_consumer *cp;
-	struct g_provider *pp, *pp2;
-	struct gv_sd *s;
-
-	KASSERT(d != NULL, ("gv_drive_modify: null d"));
-	gp = d->geom;
-	KASSERT(gp != NULL, ("gv_drive_modify: null gp"));
-	cp = LIST_FIRST(&gp->consumer);
-	KASSERT(cp != NULL, ("gv_drive_modify: null cp"));
-	pp = cp->provider;
-	KASSERT(pp != NULL, ("gv_drive_modify: null pp"));
-
-	g_topology_assert();
-
-	LIST_FOREACH(s, &d->subdisks, from_drive) {
-		/* This subdisk already has a provider. */
-		if (s->provider != NULL)
+		/* First config copy. */
+		error = g_write_data(cp, GV_CFG_OFFSET, sbuf_data(sb),
+		    GV_CFG_LEN);
+		if (error) {
+			G_VINUM_DEBUG(0, "writing first config copy failed on "
+			    "drive %s, errno %d", d->name, error);
+			g_topology_lock();
+			g_access(cp, 0, -1, 0);
+			g_topology_unlock();
 			continue;
-		pp2 = g_new_providerf(gp, "gvinum/sd/%s", s->name);
-		pp2->mediasize = s->size;
-		pp2->sectorsize = pp->sectorsize;
-		g_error_provider(pp2, 0);
-		s->provider = pp2;
-		pp2->private = s;
-	}
-}
-
-static void
-gv_drive_dead(void *arg, int flag)
-{
-	struct g_geom *gp;
-	struct g_consumer *cp;
-	struct gv_drive *d;
-	struct gv_sd *s;
-
-	g_topology_assert();
-	KASSERT(arg != NULL, ("gv_drive_dead: NULL arg"));
-
-	if (flag == EV_CANCEL)
-		return;
-
-	d = arg;
-	if (d->state != GV_DRIVE_DOWN)
-		return;
-
-	g_trace(G_T_TOPOLOGY, "gv_drive_dead(%s)", d->name);
-
-	gp = d->geom;
-	if (gp == NULL)
-		return;
-
-	LIST_FOREACH(cp, &gp->consumer, consumer) {
-		if (cp->nstart != cp->nend) {
-			G_VINUM_DEBUG(0, "dead drive '%s' still has "
-			    "active requests, cannot detach consumer",
-			    d->name);
-			g_post_event(gv_drive_dead, d, M_WAITOK, d,
-			    NULL);
-			return;
 		}
-		if (cp->acr != 0 || cp->acw != 0 || cp->ace != 0)
-			g_access(cp, -cp->acr, -cp->acw, -cp->ace);
-	}
+		/* Second config copy. */
+		error = g_write_data(cp, GV_CFG_OFFSET + GV_CFG_LEN,
+		    sbuf_data(sb), GV_CFG_LEN);
+		if (error)
+			G_VINUM_DEBUG(0, "writing second config copy failed on "
+			    "drive %s, errno %d", d->name, error);
 
-	G_VINUM_DEBUG(1, "lost drive '%s'", d->name);
-	d->geom = NULL;
-	LIST_FOREACH(s, &d->subdisks, from_drive) {
-		s->provider = NULL;
-		s->consumer = NULL;
+		g_topology_lock();
+		g_access(cp, 0, -1, 0);
+		g_topology_unlock();
 	}
-	gv_kill_drive_thread(d);
-	gp->softc = NULL;
-	g_wither_geom(gp, ENXIO);
-}
 
-static int
-gv_drive_destroy_geom(struct gctl_req *req, struct g_class *mp,
-    struct g_geom *gp)
-{
-	struct gv_drive *d;
-
-	g_trace(G_T_TOPOLOGY, "gv_drive_destroy_geom: %s", gp->name);
-	g_topology_assert();
-
-	d = gp->softc;
-	gv_kill_drive_thread(d);
-
-	g_wither_geom(gp, ENXIO);
-	return (0);
+	sbuf_delete(sb);
+	g_free(vhdr);
 }
-
-#define	VINUMDRIVE_CLASS_NAME "VINUMDRIVE"
-
-static struct g_class g_vinum_drive_class = {
-	.name = VINUMDRIVE_CLASS_NAME,
-	.version = G_VERSION,
-	.taste = gv_drive_taste,
-	.destroy_geom = gv_drive_destroy_geom
-};
-
-DECLARE_GEOM_CLASS(g_vinum_drive_class, g_vinum_drive);
diff --git a/sys/geom/vinum/geom_vinum_events.c b/sys/geom/vinum/geom_vinum_events.c
new file mode 100644
index 0000000..ce7fec0
--- /dev/null
+++ b/sys/geom/vinum/geom_vinum_events.c
@@ -0,0 +1,217 @@
+/*-
+ *  Copyright (c) 2007 Lukas Ertl
+ *  All rights reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/systm.h>
+
+#include <geom/geom.h>
+#include <geom/vinum/geom_vinum_var.h>
+#include <geom/vinum/geom_vinum.h>
+
+void
+gv_post_event(struct gv_softc *sc, int event, void *arg1, void *arg2,
+    intmax_t arg3, intmax_t arg4)
+{
+	struct gv_event *ev;
+
+	ev = g_malloc(sizeof(*ev), M_WAITOK | M_ZERO);
+	ev->type = event;
+	ev->arg1 = arg1;
+	ev->arg2 = arg2;
+	ev->arg3 = arg3;
+	ev->arg4 = arg4;
+
+	mtx_lock(&sc->queue_mtx);
+	TAILQ_INSERT_TAIL(&sc->equeue, ev, events);
+	wakeup(sc);
+	mtx_unlock(&sc->queue_mtx);
+}
+
+void
+gv_drive_tasted(struct gv_softc *sc, struct g_provider *pp)
+{
+	struct g_geom *gp;
+	struct g_consumer *cp;
+	struct gv_hdr *hdr;
+	struct gv_drive *d;
+	char *buf;
+	int error;
+
+	hdr = NULL;
+	buf = NULL;
+
+	G_VINUM_DEBUG(2, "tasted drive on '%s'", pp->name);
+
+	gp = sc->geom;
+	g_topology_lock();
+	cp = g_new_consumer(gp);
+	if (g_attach(cp, pp) != 0) {
+		g_destroy_consumer(cp);
+		g_topology_unlock();
+		G_VINUM_DEBUG(0, "failed to attach to provider on taste event");
+		return;
+	}
+	if (g_access(cp, 1, 0, 0) != 0) {
+		g_detach(cp);
+		g_destroy_consumer(cp);
+		g_topology_unlock();
+		G_VINUM_DEBUG(0, "failed to access consumer on taste event");
+		return;
+	}
+	g_topology_unlock();
+
+	hdr = g_malloc(GV_HDR_LEN, M_WAITOK | M_ZERO);
+	/* Read header and on-disk configuration. */
+	error = gv_read_header(cp, hdr);
+	if (error) {
+		G_VINUM_DEBUG(0, "failed to read header during taste");
+		goto failed;
+	}
+
+	/*
+	 * Setup the drive before we parse the on-disk configuration, so that
+	 * we already know about the drive then.
+	 */
+	d = gv_find_drive(sc, hdr->label.name);
+	if (d == NULL) {
+		d = g_malloc(sizeof(*d), M_WAITOK | M_ZERO);
+		strlcpy(d->name, hdr->label.name, sizeof(d->name));
+		strlcpy(d->device, pp->name, sizeof(d->device));
+	} else if (d->flags & GV_DRIVE_REFERENCED) {
+		strlcpy(d->device, pp->name, sizeof(d->device));
+		d->flags &= ~GV_DRIVE_REFERENCED;
+	} else {
+		G_VINUM_DEBUG(2, "drive '%s' is already known", d->name);
+		goto failed;
+	}
+
+	/* Add the consumer and header to the new drive. */
+	d->consumer = cp;
+	d->hdr = hdr;
+	gv_create_drive(sc, d);
+
+	buf = g_read_data(cp, GV_CFG_OFFSET, GV_CFG_LEN, NULL);
+	if (buf == NULL) {
+		G_VINUM_DEBUG(0, "failed to read config during taste");
+		goto failed;
+	}
+	gv_parse_config(sc, buf, d);
+	g_free(buf);
+
+	g_topology_lock();
+	g_access(cp, -1, 0, 0);
+	g_topology_unlock();
+
+	gv_setup_objects(sc);
+	gv_set_drive_state(d, GV_DRIVE_UP, 0);
+
+	return;
+
+failed:
+	if (hdr != NULL)
+		g_free(hdr);
+	g_topology_lock();
+	g_access(cp, -1, 0, 0);
+	g_detach(cp);
+	g_destroy_consumer(cp);
+	g_topology_unlock();
+}
+
+/*
+ * When losing a drive (e.g. hardware failure), we cut down the consumer
+ * attached to the underlying device and bring the drive itself to a
+ * "referenced" state so that normal tasting could bring it up cleanly if it
+ * possibly arrives again.
+ */
+void
+gv_drive_lost(struct gv_softc *sc, struct gv_drive *d)
+{
+	struct g_consumer *cp;
+	struct gv_drive *d2;
+	struct gv_sd *s, *s2;
+	struct gv_freelist *fl, *fl2;
+
+	gv_set_drive_state(d, GV_DRIVE_DOWN,
+	    GV_SETSTATE_FORCE | GV_SETSTATE_CONFIG);
+
+	cp = d->consumer;
+
+	if (cp != NULL) {
+		if (cp->nstart != cp->nend) {
+			G_VINUM_DEBUG(0, "dead drive '%s' has still active "
+			    "requests, can't detach consumer", d->name);
+			gv_post_event(sc, GV_EVENT_DRIVE_LOST, d, NULL, 0, 0);
+			return;
+		}
+		g_topology_lock();
+		if (cp->acr != 0 || cp->acw != 0 || cp->ace != 0)
+			g_access(cp, -cp->acr, -cp->acw, -cp->ace);
+		g_detach(cp);
+		g_destroy_consumer(cp);
+		g_topology_unlock();
+	}
+
+	LIST_FOREACH_SAFE(fl, &d->freelist, freelist, fl2) {
+		LIST_REMOVE(fl, freelist);
+		g_free(fl);
+	}
+
+	d->consumer = NULL;
+	g_free(d->hdr);
+	d->hdr = NULL;
+	d->flags |= GV_DRIVE_REFERENCED;
+	snprintf(d->device, sizeof(d->device), "???");
+	d->size = 0;
+	d->avail = 0;
+	d->freelist_entries = 0;
+	d->sdcount = 0;
+
+	/* Put the subdisk in tasted mode, and remove from drive list. */
+	LIST_FOREACH_SAFE(s, &d->subdisks, from_drive, s2) {
+		LIST_REMOVE(s, from_drive);
+		s->flags |= GV_SD_TASTED;
+	}
+
+	/*
+	 * Don't forget that gv_is_newer wants a "real" drive at the beginning
+	 * of the list, so, just to be safe, we shuffle around.
+	 */
+	LIST_REMOVE(d, drive);
+	d2 = LIST_FIRST(&sc->drives);
+	if (d2 == NULL)
+		LIST_INSERT_HEAD(&sc->drives, d, drive);
+	else
+		LIST_INSERT_AFTER(d2, d, drive);
+	gv_save_config(sc);
+}
diff --git a/sys/geom/vinum/geom_vinum_init.c b/sys/geom/vinum/geom_vinum_init.c
index fddc435..34f1156 100644
--- a/sys/geom/vinum/geom_vinum_init.c
+++ b/sys/geom/vinum/geom_vinum_init.c
@@ -1,5 +1,6 @@
 /*-
- * Copyright (c) 2004 Lukas Ertl
+ * Copyright (c) 2004, 2007 Lukas Ertl
+ * Copyright (c) 2007, 2009 Ulf Lilleengen
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -26,158 +27,21 @@
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
-
 #include <sys/param.h>
 #include <sys/bio.h>
-#include <sys/kernel.h>
-#include <sys/kthread.h>
 #include <sys/libkern.h>
 #include <sys/malloc.h>
-#include <sys/queue.h>
 
 #include <geom/geom.h>
 #include <geom/vinum/geom_vinum_var.h>
 #include <geom/vinum/geom_vinum.h>
-#include <geom/vinum/geom_vinum_share.h>
-
-static int	gv_init_plex(struct gv_plex *);
-void	gv_init_td(void *);
-static int	gv_rebuild_plex(struct gv_plex *);
-void	gv_rebuild_td(void *);
-static int	gv_start_plex(struct gv_plex *);
-static int	gv_start_vol(struct gv_volume *);
-static int	gv_sync(struct gv_volume *);
-void	gv_sync_td(void *);
-
-struct gv_sync_args {
-	struct gv_volume *v;
-	struct gv_plex *from;
-	struct gv_plex *to;
-	off_t syncsize;
-};
-
-void
-gv_parityop(struct g_geom *gp, struct gctl_req *req)
-{
-	struct gv_softc *sc;
-	struct gv_plex *p;
-	struct bio *bp;
-	struct g_consumer *cp;
-	int error, *flags, type, *rebuild, rv;
-	char *plex;
-
-	rv = -1;
-
-	plex = gctl_get_param(req, "plex", NULL);
-	if (plex == NULL) {
-		gctl_error(req, "no plex given");
-		goto out;
-	}
-
-	flags = gctl_get_paraml(req, "flags", sizeof(*flags));
-	if (flags == NULL) {
-		gctl_error(req, "no flags given");
-		goto out;
-	}
-
-	rebuild = gctl_get_paraml(req, "rebuild", sizeof(*rebuild));
-	if (rebuild == NULL) {
-		gctl_error(req, "no rebuild op given");
-		goto out;
-	}
-
-	sc = gp->softc;
-	type = gv_object_type(sc, plex);
-	switch (type) {
-	case GV_TYPE_PLEX:
-		break;
-	case GV_TYPE_VOL:
-	case GV_TYPE_SD:
-	case GV_TYPE_DRIVE:
-	default:
-		gctl_error(req, "'%s' is not a plex", plex);
-		goto out;
-	}
-
-	p = gv_find_plex(sc, plex);
-	if (p->state != GV_PLEX_UP) {
-		gctl_error(req, "plex %s is not completely accessible",
-		    p->name);
-		goto out;
-	}
-	if (p->org != GV_PLEX_RAID5) {
-		gctl_error(req, "plex %s is not a RAID5 plex", p->name);
-		goto out;
-	}
-
-	cp = p->consumer;
-	error = g_access(cp, 1, 1, 0);
-	if (error) {
-		gctl_error(req, "cannot access consumer");
-		goto out;
-	}
-	g_topology_unlock();
-
-	/* Reset the check pointer when using -f. */
-	if (*flags & GV_FLAG_F)
-		p->synced = 0;
-
-	bp = g_new_bio();
-	if (bp == NULL) {
-		gctl_error(req, "cannot create BIO - out of memory");
-		g_topology_lock();
-		error = g_access(cp, -1, -1, 0);
-		goto out;
-	}
-	bp->bio_cmd = BIO_WRITE;
-	bp->bio_done = NULL;
-	bp->bio_data = g_malloc(p->stripesize, M_WAITOK | M_ZERO);
-	bp->bio_cflags |= GV_BIO_CHECK;
-	if (*rebuild)
-		bp->bio_cflags |= GV_BIO_PARITY;
-	bp->bio_offset = p->synced;
-	bp->bio_length = p->stripesize;
-
-	/* Schedule it down ... */
-	g_io_request(bp, cp);
-
-	/* ... and wait for the result. */
-	error = biowait(bp, "gwrite");
-	g_free(bp->bio_data);
-	g_destroy_bio(bp);
-
-	if (error) {
-		/* Incorrect parity. */
-		if (error == EAGAIN)
-			rv = 1;
 
-		/* Some other error happened. */
-		else
-			gctl_error(req, "Parity check failed at offset 0x%jx, "
-			    "errno %d", (intmax_t)p->synced, error);
-
-	/* Correct parity. */
-	} else
-		rv = 0;
-
-	gctl_set_param(req, "offset", &p->synced, sizeof(p->synced));
-
-	/* Advance the checkpointer if there was no error. */
-	if (rv == 0)
-		p->synced += p->stripesize;
-
-	/* End of plex; reset the check pointer and signal it to the caller. */
-	if (p->synced >= p->size) {
-		p->synced = 0;
-		rv = -2;
-	}
-
-	g_topology_lock();
-	error = g_access(cp, -1, -1, 0);
-
-out:
-	gctl_set_param(req, "rv", &rv, sizeof(rv));
-}
+static int		 gv_sync(struct gv_volume *);
+static int		 gv_rebuild_plex(struct gv_plex *);
+static int		 gv_init_plex(struct gv_plex *);
+static int		 gv_grow_plex(struct gv_plex *);
+static int		 gv_sync_plex(struct gv_plex *, struct gv_plex *);
+static struct gv_plex	*gv_find_good_plex(struct gv_volume *);
 
 void
 gv_start_obj(struct g_geom *gp, struct gctl_req *req)
@@ -187,7 +51,7 @@ gv_start_obj(struct g_geom *gp, struct gctl_req *req)
 	struct gv_plex *p;
 	int *argc, *initsize;
 	char *argv, buf[20];
-	int err, i, type;
+	int i, type;
 
 	argc = gctl_get_paraml(req, "argc", sizeof(*argc));
 	initsize = gctl_get_paraml(req, "initsize", sizeof(*initsize));
@@ -208,37 +72,21 @@ gv_start_obj(struct g_geom *gp, struct gctl_req *req)
 		switch (type) {
 		case GV_TYPE_VOL:
 			v = gv_find_vol(sc, argv);
-			err = gv_start_vol(v);
-			if (err) {
-				if (err == EINPROGRESS) {
-					gctl_error(req, "cannot start volume "
-					    "'%s': already in progress", argv);
-				} else {
-					gctl_error(req, "cannot start volume "
-					    "'%s'; errno: %d", argv, err);
-				}
-				return;
-			}
+			if (v != NULL)
+				gv_post_event(sc, GV_EVENT_START_VOLUME, v,
+				    NULL, *initsize, 0);
 			break;
 
 		case GV_TYPE_PLEX:
 			p = gv_find_plex(sc, argv);
-			err = gv_start_plex(p);
-			if (err) {
-				if (err == EINPROGRESS) {
-					gctl_error(req, "cannot start plex "
-					    "'%s': already in progress", argv);
-				} else {
-					gctl_error(req, "cannot start plex "
-					    "'%s'; errno: %d", argv, err);
-				}
-				return;
-			}
+			if (p != NULL)
+				gv_post_event(sc, GV_EVENT_START_PLEX, p, NULL,
+				    *initsize, 0);
 			break;
 
 		case GV_TYPE_SD:
 		case GV_TYPE_DRIVE:
-			/* XXX not yet */
+			/* XXX Not implemented, but what is the use? */
 			gctl_error(req, "cannot start '%s' - not yet supported",
 			    argv);
 			return;
@@ -249,36 +97,73 @@ gv_start_obj(struct g_geom *gp, struct gctl_req *req)
 	}
 }
 
-static int
+int
 gv_start_plex(struct gv_plex *p)
 {
 	struct gv_volume *v;
+	struct gv_plex *up;
+	struct gv_sd *s;
 	int error;
 
 	KASSERT(p != NULL, ("gv_start_plex: NULL p"));
 
-	if (p->state == GV_PLEX_UP)
-		return (0);
-
 	error = 0;
 	v = p->vol_sc;
-	if ((v != NULL) && (v->plexcount > 1))
-		error = gv_sync(v);
-	else if (p->org == GV_PLEX_RAID5) {
-		if (p->state == GV_PLEX_DEGRADED)
+
+	/* RAID5 plexes can either be init, rebuilt or grown. */
+	if (p->org == GV_PLEX_RAID5) {
+		if (p->state > GV_PLEX_DEGRADED) {
+			LIST_FOREACH(s, &p->subdisks, in_plex) {
+				if (s->flags & GV_SD_GROW) {
+					error = gv_grow_plex(p);
+					return (error);
+				}
+			}
+		} else if (p->state == GV_PLEX_DEGRADED) {
 			error = gv_rebuild_plex(p);
-		else
+		} else
 			error = gv_init_plex(p);
+	} else {
+		/* We want to sync from the other plex if we're down. */
+		if (p->state == GV_PLEX_DOWN && v->plexcount > 1) {
+			up = gv_find_good_plex(v);
+			if (up == NULL) {
+				G_VINUM_DEBUG(1, "unable to find a good plex");
+				return (ENXIO);
+			}
+			g_topology_lock();
+			error = gv_access(v->provider, 1, 1, 0);
+			if (error) {
+				g_topology_unlock();
+				G_VINUM_DEBUG(0, "sync from '%s' failed to "
+				    "access volume: %d", up->name, error);
+				return (error);
+			}
+			g_topology_unlock();
+			error = gv_sync_plex(p, up);
+			if (error)
+				return (error);
+		/*
+		 * In case we have a stripe that is up, check whether it can be
+		 * grown.
+		 */
+		} else if (p->org == GV_PLEX_STRIPED &&
+		    p->state != GV_PLEX_DOWN) {
+			LIST_FOREACH(s, &p->subdisks, in_plex) {
+				if (s->flags & GV_SD_GROW) {
+					error = gv_grow_plex(p);
+					break;
+				}
+			}
+		}
 	}
-
 	return (error);
 }
 
-static int
+int
 gv_start_vol(struct gv_volume *v)
 {
 	struct gv_plex *p;
-	struct gv_sd *s;
 	int error;
 
 	KASSERT(v != NULL, ("gv_start_vol: NULL v"));
@@ -291,39 +176,46 @@ gv_start_vol(struct gv_volume *v)
 	else if (v->plexcount == 1) {
 		p = LIST_FIRST(&v->plexes);
 		KASSERT(p != NULL, ("gv_start_vol: NULL p on %s", v->name));
-		if (p->org == GV_PLEX_RAID5) {
-			switch (p->state) {
-			case GV_PLEX_DOWN:
-				error = gv_init_plex(p);
-				break;
-			case GV_PLEX_DEGRADED:
-				error = gv_rebuild_plex(p);
-				break;
-			default:
-				return (0);
-			}
-		} else {
-			LIST_FOREACH(s, &p->subdisks, in_plex) {
-				gv_set_sd_state(s, GV_SD_UP,
-				    GV_SETSTATE_CONFIG);
-			}
-		}
+		error = gv_start_plex(p);
 	} else
 		error = gv_sync(v);
 
 	return (error);
 }
 
+/* Sync a plex p from the plex up.  */
 static int
-gv_sync(struct gv_volume *v)
+gv_sync_plex(struct gv_plex *p, struct gv_plex *up)
 {
-	struct gv_softc *sc;
-	struct gv_plex *p, *up;
-	struct gv_sync_args *sync;
+	int error;
 
-	KASSERT(v != NULL, ("gv_sync: NULL v"));
-	sc = v->vinumconf;
-	KASSERT(sc != NULL, ("gv_sync: NULL sc on %s", v->name));
+	KASSERT(p != NULL, ("%s: NULL p", __func__));
+	KASSERT(up != NULL, ("%s: NULL up", __func__));
+	if ((p == up) || (p->state == GV_PLEX_UP))
+		return (0);
+	if (p->flags & GV_PLEX_SYNCING ||
+	    p->flags & GV_PLEX_REBUILDING ||
+	    p->flags & GV_PLEX_GROWING) {
+		return (EINPROGRESS);
+	}
+	p->synced = 0;
+	p->flags |= GV_PLEX_SYNCING;
+	G_VINUM_DEBUG(1, "starting sync of plex %s", p->name);
+	error = gv_sync_request(up, p, p->synced, 
+	    MIN(GV_DFLT_SYNCSIZE, up->size - p->synced), 
+	    BIO_READ, NULL);
+	if (error) {
+		G_VINUM_DEBUG(0, "error syncing plex %s", p->name);
+		return (error);
+	}
+	return (0);
+}
+
+/* Return a good plex from volume v. */
+static struct gv_plex *
+gv_find_good_plex(struct gv_volume *v)
+{
+	struct gv_plex *up;
 
 	/* Find the plex that's up. */
 	up = NULL;
@@ -331,341 +223,166 @@ gv_sync(struct gv_volume *v)
 		if (up->state == GV_PLEX_UP)
 			break;
 	}
-
 	/* Didn't find a good plex. */
-	if (up == NULL)
-		return (ENXIO);
-
-	LIST_FOREACH(p, &v->plexes, in_volume) {
-		if ((p == up) || (p->state == GV_PLEX_UP))
-			continue;
-		if (p->flags & GV_PLEX_SYNCING) {
-			return (EINPROGRESS);
-		}
-		p->flags |= GV_PLEX_SYNCING;
-		sync = g_malloc(sizeof(*sync), M_WAITOK | M_ZERO);
-		sync->v = v;
-		sync->from = up;
-		sync->to = p;
-		sync->syncsize = GV_DFLT_SYNCSIZE;
-		kproc_create(gv_sync_td, sync, NULL, 0, 0, "gv_sync '%s'",
-		    p->name);
-	}
-
-	return (0);
+	return (up);
 }
 
 static int
-gv_rebuild_plex(struct gv_plex *p)
-{
-	struct gv_sync_args *sync;
-
-	if (gv_is_open(p->geom))
-		return (EBUSY);
-
-	if (p->flags & GV_PLEX_SYNCING)
-		return (EINPROGRESS);
-	p->flags |= GV_PLEX_SYNCING;
-
-	sync = g_malloc(sizeof(*sync), M_WAITOK | M_ZERO);
-	sync->to = p;
-	sync->syncsize = GV_DFLT_SYNCSIZE;
-
-	kproc_create(gv_rebuild_td, sync, NULL, 0, 0, "gv_rebuild %s",
-	    p->name);
-
-	return (0);
-}
-
-static int
-gv_init_plex(struct gv_plex *p)
-{
-	struct gv_sd *s;
-
-	KASSERT(p != NULL, ("gv_init_plex: NULL p"));
-
-	LIST_FOREACH(s, &p->subdisks, in_plex) {
-		if (s->state == GV_SD_INITIALIZING)
-			return (EINPROGRESS);
-		gv_set_sd_state(s, GV_SD_INITIALIZING, GV_SETSTATE_FORCE);
-		s->init_size = GV_DFLT_SYNCSIZE;
-		kproc_create(gv_init_td, s, NULL, 0, 0, "gv_init %s",
-		    s->name);
-	}
-
-	return (0);
-}
-
-/* This thread is responsible for rebuilding a degraded RAID5 plex. */
-void
-gv_rebuild_td(void *arg)
+gv_sync(struct gv_volume *v)
 {
-	struct bio *bp;
-	struct gv_plex *p;
-	struct g_consumer *cp;
-	struct gv_sync_args *sync;
-	u_char *buf;
-	off_t i;
+	struct gv_softc *sc;
+	struct gv_plex *p, *up;
 	int error;
 
-	buf = NULL;
-	bp = NULL;
+	KASSERT(v != NULL, ("gv_sync: NULL v"));
+	sc = v->vinumconf;
+	KASSERT(sc != NULL, ("gv_sync: NULL sc on %s", v->name));
 
-	sync = arg;
-	p = sync->to;
-	p->synced = 0;
-	cp = p->consumer;
 
+	up = gv_find_good_plex(v);
+	if (up == NULL)
+		return (ENXIO);
 	g_topology_lock();
-	error = g_access(cp, 1, 1, 0);
+	error = gv_access(v->provider, 1, 1, 0);
 	if (error) {
 		g_topology_unlock();
-		G_VINUM_DEBUG(0, "rebuild of %s failed to access consumer: "
-		    "%d", p->name, error);
-		kproc_exit(error);
+		G_VINUM_DEBUG(0, "sync from '%s' failed to access volume: %d",
+		    up->name, error);
+		return (error);
 	}
 	g_topology_unlock();
 
-	buf = g_malloc(sync->syncsize, M_WAITOK);
-
-	G_VINUM_DEBUG(1, "rebuild of %s started", p->name);
-	i = 0;
-	for (i = 0; i < p->size; i += (p->stripesize * (p->sdcount - 1))) {
-/*
-		if (i + sync->syncsize > p->size)
-			sync->syncsize = p->size - i;
-*/
-		bp = g_new_bio();
-		if (bp == NULL) {
-			G_VINUM_DEBUG(0, "rebuild of %s failed creating bio: "
-			    "out of memory", p->name);
-			break;
-		}
-		bp->bio_cmd = BIO_WRITE;
-		bp->bio_done = NULL;
-		bp->bio_data = buf;
-		bp->bio_cflags |= GV_BIO_REBUILD;
-		bp->bio_offset = i;
-		bp->bio_length = p->stripesize;
-
-		/* Schedule it down ... */
-		g_io_request(bp, cp);
-
-		/* ... and wait for the result. */
-		error = biowait(bp, "gwrite");
-		if (error) {
-			G_VINUM_DEBUG(0, "rebuild of %s failed at offset %jd "
-			    "errno: %d", p->name, i, error);
+	/* Go through the good plex, and issue BIO's to all other plexes. */
+	LIST_FOREACH(p, &v->plexes, in_volume) {
+		error = gv_sync_plex(p, up);
+		if (error)
 			break;
-		}
-		g_destroy_bio(bp);
-		bp = NULL;
 	}
-
-	if (bp != NULL)
-		g_destroy_bio(bp);
-	if (buf != NULL)
-		g_free(buf);
-
-	g_topology_lock();
-	g_access(cp, -1, -1, 0);
-	gv_save_config_all(p->vinumconf);
-	g_topology_unlock();
-
-	p->flags &= ~GV_PLEX_SYNCING;
-	p->synced = 0;
-
-	/* Successful initialization. */
-	if (!error)
-		G_VINUM_DEBUG(1, "rebuild of %s finished", p->name);
-
-	g_free(sync);
-	kproc_exit(error);
+	return (0);
 }
 
-void
-gv_sync_td(void *arg)
+static int
+gv_rebuild_plex(struct gv_plex *p)
 {
-	struct bio *bp;
-	struct gv_plex *p;
-	struct g_consumer *from, *to;
-	struct gv_sync_args *sync;
-	u_char *buf;
-	off_t i;
+	struct gv_drive *d;
+	struct gv_sd *s;
 	int error;
 
-	sync = arg;
-
-	from = sync->from->consumer;
-	to = sync->to->consumer;
-
-	p = sync->to;
+	if (p->flags & GV_PLEX_SYNCING ||
+	    p->flags & GV_PLEX_REBUILDING ||
+	    p->flags & GV_PLEX_GROWING)
+		return (EINPROGRESS);
+	/*
+	 * Make sure that all subdisks have consumers. We won't allow a rebuild
+	 * unless every subdisk have one.
+	 */
+	LIST_FOREACH(s, &p->subdisks, in_plex) {
+		d = s->drive_sc;
+		if (d == NULL || (d->flags & GV_DRIVE_REFERENCED)) {
+			G_VINUM_DEBUG(0, "can't rebuild %s, subdisk(s) have no "
+			    "drives", p->name);
+			return (ENXIO);
+		}
+	}
+	p->flags |= GV_PLEX_REBUILDING;
 	p->synced = 0;
 
-	error = 0;
-
+	g_topology_assert_not();
 	g_topology_lock();
-	error = g_access(from, 1, 0, 0);
+	error = gv_access(p->vol_sc->provider, 1, 1, 0);
 	if (error) {
-		g_topology_unlock();
-		G_VINUM_DEBUG(0, "sync from '%s' failed to access "
-		    "consumer: %d", sync->from->name, error);
-		g_free(sync);
-		kproc_exit(error);
-	}
-	error = g_access(to, 0, 1, 0);
-	if (error) {
-		g_access(from, -1, 0, 0);
-		g_topology_unlock();
-		G_VINUM_DEBUG(0, "sync to '%s' failed to access "
-		    "consumer: %d", p->name, error);
-		g_free(sync);
-		kproc_exit(error);
+		G_VINUM_DEBUG(0, "unable to access provider");
+		return (0);
 	}
 	g_topology_unlock();
 
-	G_VINUM_DEBUG(1, "plex sync %s -> %s started", sync->from->name,
-	    sync->to->name);
-	for (i = 0; i < p->size; i+= sync->syncsize) {
-		/* Read some bits from the good plex. */
-		buf = g_read_data(from, i, sync->syncsize, &error);
-		if (buf == NULL) {
-			G_VINUM_DEBUG(0, "sync read from '%s' failed at "
-			    "offset %jd; errno: %d", sync->from->name, i,
-			    error);
-			break;
-		}
-
-		/*
-		 * Create a bio and schedule it down on the 'bad' plex.  We
-		 * cannot simply use g_write_data() because we have to let the
-		 * lower parts know that we are an initialization process and
-		 * not a 'normal' request.
-		 */
-		bp = g_new_bio();
-		if (bp == NULL) {
-			G_VINUM_DEBUG(0, "sync write to '%s' failed at "
-			    "offset %jd; out of memory", p->name, i);
-			g_free(buf);
-			break;
-		}
-		bp->bio_cmd = BIO_WRITE;
-		bp->bio_offset = i;
-		bp->bio_length = sync->syncsize;
-		bp->bio_data = buf;
-		bp->bio_done = NULL;
-
-		/*
-		 * This hack declare this bio as part of an initialization
-		 * process, so that the lower levels allow it to get through.
-		 */
-		bp->bio_cflags |= GV_BIO_SYNCREQ;
-
-		/* Schedule it down ... */
-		g_io_request(bp, to);
+	gv_parity_request(p, GV_BIO_REBUILD, 0);
+	return (0);
+}
 
-		/* ... and wait for the result. */
-		error = biowait(bp, "gwrite");
-		g_destroy_bio(bp);
-		g_free(buf);
-		if (error) {
-			G_VINUM_DEBUG(0, "sync write to '%s' failed at "
-			    "offset %jd; errno: %d\n", p->name, i, error);
-			break;
-		}
+static int
+gv_grow_plex(struct gv_plex *p)
+{
+	struct gv_volume *v;
+	struct gv_sd *s;
+	off_t origsize, origlength;
+	int error, sdcount;
 
-		/* Note that we have synced a little bit more. */
-		p->synced += sync->syncsize;
-	}
+	KASSERT(p != NULL, ("gv_grow_plex: NULL p"));
+	v = p->vol_sc;
+	KASSERT(v != NULL, ("gv_grow_plex: NULL v"));
 
+	if (p->flags & GV_PLEX_GROWING || 
+	    p->flags & GV_PLEX_SYNCING ||
+	    p->flags & GV_PLEX_REBUILDING)
+		return (EINPROGRESS);
 	g_topology_lock();
-	g_access(from, -1, 0, 0);
-	g_access(to, 0, -1, 0);
-	gv_save_config_all(p->vinumconf);
+	error = gv_access(v->provider, 1, 1, 0);
 	g_topology_unlock();
+	if (error) {
+		G_VINUM_DEBUG(0, "unable to access provider");
+		return (error);
+	}
 
-	/* Successful initialization. */
-	if (!error)
-		G_VINUM_DEBUG(1, "plex sync %s -> %s finished",
-		    sync->from->name, sync->to->name);
-
-	p->flags &= ~GV_PLEX_SYNCING;
+	/* XXX: This routine with finding origsize is used two other places as
+	 * well, so we should create a function for it. */
+	sdcount = p->sdcount;
+	LIST_FOREACH(s, &p->subdisks, in_plex) {
+		if (s->flags & GV_SD_GROW)
+			sdcount--;
+	}
+	s = LIST_FIRST(&p->subdisks);
+	if (s == NULL) {
+		G_VINUM_DEBUG(0, "error growing plex without subdisks");
+		return (GV_ERR_NOTFOUND);
+	}
+	p->flags |= GV_PLEX_GROWING;
+	origsize = (sdcount - 1) * s->size;
+	origlength = (sdcount - 1) * p->stripesize;
 	p->synced = 0;
+	G_VINUM_DEBUG(1, "starting growing of plex %s", p->name);
+	gv_grow_request(p, 0, MIN(origlength, origsize), BIO_READ, NULL);
 
-	g_free(sync);
-	kproc_exit(error);
+	return (0);
 }
 
-void
-gv_init_td(void *arg)
+static int
+gv_init_plex(struct gv_plex *p)
 {
-	struct gv_sd *s;
 	struct gv_drive *d;
-	struct g_geom *gp;
-	struct g_consumer *cp;
+	struct gv_sd *s;
 	int error;
-	off_t i, init_size, start, offset, length;
-	u_char *buf;
-
-	s = arg;
-	KASSERT(s != NULL, ("gv_init_td: NULL s"));
-	d = s->drive_sc;
-	KASSERT(d != NULL, ("gv_init_td: NULL d"));
-	gp = d->geom;
-	KASSERT(gp != NULL, ("gv_init_td: NULL gp"));
-
-	cp = LIST_FIRST(&gp->consumer);
-	KASSERT(cp != NULL, ("gv_init_td: NULL cp"));
+	off_t start;
+	caddr_t data;
 
-	s->init_error = 0;
-	init_size = s->init_size;
-	start = s->drive_offset + s->initialized;
-	offset = s->drive_offset;
-	length = s->size;
-
-	buf = g_malloc(s->init_size, M_WAITOK | M_ZERO);
-
-	g_topology_lock();
-	error = g_access(cp, 0, 1, 0);
-	if (error) {
-		s->init_error = error;
-		g_topology_unlock();
-		G_VINUM_DEBUG(0, "subdisk '%s' init: failed to access "
-		    "consumer; error: %d", s->name, error);
-		kproc_exit(error);
-	}
-	g_topology_unlock();
+	KASSERT(p != NULL, ("gv_init_plex: NULL p"));
 
-	for (i = start; i < offset + length; i += init_size) {
-		error = g_write_data(cp, i, buf, init_size);
-		if (error) {
-			G_VINUM_DEBUG(0, "subdisk '%s' init: write failed"
-			    " at offset %jd (drive offset %jd); error %d",
-			    s->name, (intmax_t)s->initialized, (intmax_t)i,
-			    error);
+	LIST_FOREACH(s, &p->subdisks, in_plex) {
+		if (s->state == GV_SD_INITIALIZING)
+			return (EINPROGRESS);
+		gv_set_sd_state(s, GV_SD_INITIALIZING, GV_SETSTATE_FORCE);
+		s->init_size = GV_DFLT_SYNCSIZE;
+		start = s->drive_offset + s->initialized;
+		d = s->drive_sc;
+		if (d == NULL) {
+			G_VINUM_DEBUG(0, "subdisk %s has no drive yet", s->name);
 			break;
 		}
-		s->initialized += init_size;
-	}
-
-	g_free(buf);
-
-	g_topology_lock();
-	g_access(cp, 0, -1, 0);
-	g_topology_unlock();
-	if (error) {
-		s->init_error = error;
-		g_topology_lock();
-		gv_set_sd_state(s, GV_SD_STALE,
-		    GV_SETSTATE_FORCE | GV_SETSTATE_CONFIG);
-		g_topology_unlock();
-	} else {
+		/*
+		 * Take the lock here since we need to avoid a race in
+		 * gv_init_request if the BIO is completed before the lock is
+		 * released.
+		 */
 		g_topology_lock();
-		gv_set_sd_state(s, GV_SD_UP, GV_SETSTATE_CONFIG);
+		error = g_access(d->consumer, 0, 1, 0);
 		g_topology_unlock();
-		s->initialized = 0;
-		G_VINUM_DEBUG(1, "subdisk '%s' init: finished successfully",
-		    s->name);
+		if (error) {
+			G_VINUM_DEBUG(0, "error accessing consumer when "
+			    "initializing %s", s->name);
+			break;
+		}
+		data = g_malloc(s->init_size, M_WAITOK | M_ZERO);
+		gv_init_request(s, start, data, s->init_size);
 	}
-	kproc_exit(error);
+	return (0);
 }
diff --git a/sys/geom/vinum/geom_vinum_list.c b/sys/geom/vinum/geom_vinum_list.c
index b1a668f..00b89495 100644
--- a/sys/geom/vinum/geom_vinum_list.c
+++ b/sys/geom/vinum/geom_vinum_list.c
@@ -1,5 +1,5 @@
 /*-
- *  Copyright (c) 2004 Lukas Ertl
+ *  Copyright (c) 2004, 2007 Lukas Ertl
  *  All rights reserved.
  * 
  * Redistribution and use in source and binary forms, with or without
@@ -28,7 +28,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <sys/param.h>
 #include <sys/libkern.h>
 #include <sys/malloc.h>
 
@@ -302,19 +301,37 @@ gv_lpi(struct gv_plex *p, struct sbuf *sb, int flags)
 		sbuf_printf(sb, "Plex %s:\tSize:\t%9jd bytes (%jd MB)\n",
 		    p->name, (intmax_t)p->size, (intmax_t)p->size / MEGABYTE);
 		sbuf_printf(sb, "\t\tSubdisks: %8d\n", p->sdcount);
-		sbuf_printf(sb, "\t\tState: %s\n\t\tOrganization: %s",
-		    gv_plexstate(p->state), gv_plexorg(p->org));
+		sbuf_printf(sb, "\t\tState: %s\n", gv_plexstate(p->state));
+		if ((p->flags & GV_PLEX_SYNCING) ||
+		    (p->flags & GV_PLEX_GROWING) ||
+		    (p->flags & GV_PLEX_REBUILDING)) {
+			sbuf_printf(sb, "\t\tSynced: ");
+			sbuf_printf(sb, "%16jd bytes (%d%%)\n",
+			    (intmax_t)p->synced,
+			    (p->size > 0) ? (int)((p->synced * 100) / p->size) :
+			    0);
+		}
+		sbuf_printf(sb, "\t\tOrganization: %s", gv_plexorg(p->org));
 		if (gv_is_striped(p)) {
 			sbuf_printf(sb, "\tStripe size: %s\n",
 			    gv_roughlength(p->stripesize, 1));
 		}
+		sbuf_printf(sb, "\t\tFlags: %d\n", p->flags);
 		if (p->vol_sc != NULL) {
 			sbuf_printf(sb, "\t\tPart of volume %s\n", p->volume);
 		}
 	} else {
-		sbuf_printf(sb, "P %-18s %2s State: %s\tSubdisks: %5d"
-		    "\tSize: %s\n", p->name, gv_plexorg_short(p->org),
-		    gv_plexstate(p->state), p->sdcount,
+		sbuf_printf(sb, "P %-18s %2s State: ", p->name,
+		gv_plexorg_short(p->org));
+		if ((p->flags & GV_PLEX_SYNCING) ||
+		    (p->flags & GV_PLEX_GROWING) ||
+		    (p->flags & GV_PLEX_REBUILDING)) {
+			sbuf_printf(sb, "S %d%%\t", (int)((p->synced * 100) /
+			    p->size));
+		} else {
+			sbuf_printf(sb, "%s\t", gv_plexstate(p->state));
+		}
+		sbuf_printf(sb, "Subdisks: %5d\tSize: %s\n", p->sdcount,
 		    gv_roughlength(p->size, 0));
 	}
 
@@ -396,6 +413,7 @@ gv_lsi(struct gv_sd *s, struct sbuf *sb, int flags)
 		    s->drive_sc == NULL ? "*missing*" : s->drive_sc->name,
 		    (intmax_t)s->drive_offset,
 		    gv_roughlength(s->drive_offset, 1));
+		sbuf_printf(sb, "\t\tFlags: %d\n", s->flags);
 	} else {
 		sbuf_printf(sb, "S %-21s State: ", s->name);
 		if (s->state == GV_SD_INITIALIZING ||
@@ -455,6 +473,7 @@ gv_ldi(struct gv_drive *d, struct sbuf *sb, int flags)
 		sbuf_printf(sb, "\t\tAvailable: %11jd bytes (%jd MB)\n",
 		    (intmax_t)d->avail, (intmax_t)d->avail / MEGABYTE);
 		sbuf_printf(sb, "\t\tState: %s\n", gv_drivestate(d->state));
+		sbuf_printf(sb, "\t\tFlags: %d\n", d->flags);
 
 		/* Be very verbose. */
 		if (flags & GV_FLAG_VV) {
@@ -469,7 +488,7 @@ gv_ldi(struct gv_drive *d, struct sbuf *sb, int flags)
 		sbuf_printf(sb, "D %-21s State: %s\t/dev/%s\tA: %jd/%jd MB "
 		    "(%d%%)\n", d->name, gv_drivestate(d->state), d->device,
 		    (intmax_t)d->avail / MEGABYTE, (intmax_t)d->size / MEGABYTE,
-		    (int)((d->avail * 100) / d->size));
+		    d->size > 0 ? (int)((d->avail * 100) / d->size) : 0);
 	}
 
 	/* Recursive listing. */
diff --git a/sys/geom/vinum/geom_vinum_move.c b/sys/geom/vinum/geom_vinum_move.c
index 04822bc..e55a6a2 100644
--- a/sys/geom/vinum/geom_vinum_move.c
+++ b/sys/geom/vinum/geom_vinum_move.c
@@ -32,26 +32,21 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <sys/param.h>
 #include <sys/libkern.h>
-#include <sys/kernel.h>
 #include <sys/malloc.h>
 
 #include <geom/geom.h>
 #include <geom/vinum/geom_vinum_var.h>
 #include <geom/vinum/geom_vinum.h>
-#include <geom/vinum/geom_vinum_share.h>
-
-static int      gv_move_sd(struct gv_softc *, struct gctl_req *,
-		    struct gv_sd *, char *, int);
 
 void
 gv_move(struct g_geom *gp, struct gctl_req *req)
 {
 	struct gv_softc *sc;
 	struct gv_sd *s;
+	struct gv_drive *d;
 	char buf[20], *destination, *object;
-	int *argc, err, *flags, i, type;
+	int *argc, *flags, i, type;
 
 	sc = gp->softc;
 
@@ -74,6 +69,7 @@ gv_move(struct g_geom *gp, struct gctl_req *req)
 		gctl_error(req, "destination '%s' is not a drive", destination);
 		return;
 	}
+	d = gv_find_drive(sc, destination);
 
 	/*
 	 * We start with 1 here, because argv[0] on the command line is the
@@ -97,67 +93,59 @@ gv_move(struct g_geom *gp, struct gctl_req *req)
 			gctl_error(req, "unknown subdisk '%s'", object);
 			return;
 		}
-		err = gv_move_sd(sc, req, s, destination, *flags);
-		if (err)
-			return;
+		gv_post_event(sc, GV_EVENT_MOVE_SD, s, d, *flags, 0);
 	}
-
-	gv_save_config_all(sc);
 }
 
 /* Move a subdisk. */
-static int
-gv_move_sd(struct gv_softc *sc, struct gctl_req *req, struct gv_sd *cursd, char *destination, int flags)
+int
+gv_move_sd(struct gv_softc *sc, struct gv_sd *cursd, 
+    struct gv_drive *destination, int flags)
 {
 	struct gv_drive *d;
 	struct gv_sd *newsd, *s, *s2;
 	struct gv_plex *p;
-	struct g_consumer *cp;
-	char errstr[ERRBUFSIZ];
 	int err;
 
 	g_topology_assert();
 	KASSERT(cursd != NULL, ("gv_move_sd: NULL cursd"));
+	KASSERT(destination != NULL, ("gv_move_sd: NULL destination"));
 
-	cp = cursd->consumer;
+	d = cursd->drive_sc;
 
-	if (cp != NULL && (cp->acr || cp->acw || cp->ace)) {
-		gctl_error(req, "subdisk '%s' is busy", cursd->name);
-		return (-1);
+	if ((gv_consumer_is_open(d->consumer) ||
+	    gv_consumer_is_open(destination->consumer)) &&
+	    !(flags && GV_FLAG_F)) {
+		G_VINUM_DEBUG(0, "consumers on current and destination drive "
+		    " still open");
+		return (GV_ERR_ISBUSY);
 	}
 
 	if (!(flags && GV_FLAG_F)) {
-		gctl_error(req, "-f flag not passed; move would be "
+		G_VINUM_DEBUG(1, "-f flag not passed; move would be "
 		    "destructive");
-		return (-1);
+		return (GV_ERR_INVFLAG);
 	}
 
-	d = gv_find_drive(sc, destination);
-	if (d == NULL) {
-		gctl_error(req, "destination drive '%s' not found",
-		    destination);
-		return (-1);
-	}
-
-	if (d == cursd->drive_sc) {
-		gctl_error(req, "subdisk '%s' already on drive '%s'",
-		    cursd->name, destination);
-		return (-1);
+	if (destination == cursd->drive_sc) {
+		G_VINUM_DEBUG(1, "subdisk '%s' already on drive '%s'",
+		    cursd->name, destination->name);
+		return (GV_ERR_ISATTACHED);
 	}
 
 	/* XXX: Does it have to be part of a plex? */
 	p = gv_find_plex(sc, cursd->plex);
 	if (p == NULL) {
-		gctl_error(req, "subdisk '%s' is not part of a plex",
+		G_VINUM_DEBUG(0, "subdisk '%s' is not part of a plex",
 		    cursd->name);
-		return (-1);
+		return (GV_ERR_NOTFOUND);
 	}
-	
+
 	/* Stale the old subdisk. */
 	err = gv_set_sd_state(cursd, GV_SD_STALE,
 	    GV_SETSTATE_FORCE | GV_SETSTATE_CONFIG);
 	if (err) {
-		gctl_error(req, "could not set the subdisk '%s' to state "
+		G_VINUM_DEBUG(0, "could not set the subdisk '%s' to state "
 		    "'stale'", cursd->name);
 		return (err);
 	}
@@ -171,54 +159,30 @@ gv_move_sd(struct gv_softc *sc, struct gctl_req *req, struct gv_sd *cursd, char
 	newsd->plex_offset = cursd->plex_offset;
 	newsd->size = cursd->size;
 	newsd->drive_offset = -1;
-	strncpy(newsd->name, cursd->name, GV_MAXSDNAME);
-	strncpy(newsd->drive, destination, GV_MAXDRIVENAME);
-	strncpy(newsd->plex, cursd->plex, GV_MAXPLEXNAME);
+	strlcpy(newsd->name, cursd->name, sizeof(newsd->name));
+	strlcpy(newsd->drive, destination->name, sizeof(newsd->drive));
+	strlcpy(newsd->plex, cursd->plex, sizeof(newsd->plex));
 	newsd->state = GV_SD_STALE;
 	newsd->vinumconf = cursd->vinumconf;
 
-	err = gv_sd_to_drive(sc, d, newsd, errstr, ERRBUFSIZ);
+	err = gv_sd_to_drive(newsd, destination);
 	if (err) {
 		/* XXX not enough free space? */
-		gctl_error(req, errstr);
 		g_free(newsd);
 		return (err);
 	}
 
 	/* Replace the old sd by the new one. */
-	if (cp != NULL)
-		g_detach(cp);
 	LIST_FOREACH_SAFE(s, &p->subdisks, in_plex, s2) {
 		if (s == cursd) {
-			p->sdcount--;
-			p->size -= s->size;
-			err = gv_rm_sd(sc, req, s, 0);
-			if (err)
-				return (err);
-			
+			gv_rm_sd(sc, s);
 		}
 	}
-
-	gv_sd_to_plex(p, newsd, 1);
-
-	/* Creates the new providers.... */
-	gv_drive_modify(d);
-
-	/* And reconnect the consumer ... */
-	if (cp != NULL) {
-		newsd->consumer = cp;
-		err = g_attach(cp, newsd->provider);
-		if (err) {
-			g_destroy_consumer(cp);
-			gctl_error(req, "proposed move would create a loop "
-			    "in GEOM config");
-			return (err);
-		}
-	}
-
+	gv_sd_to_plex(newsd, p);
 	LIST_INSERT_HEAD(&sc->subdisks, newsd, sd);
-
-	gv_save_config_all(sc);
-
+	/* Update volume size of plex. */
+	if (p->vol_sc != NULL)
+		gv_update_vol_size(p->vol_sc, gv_vol_size(p->vol_sc));
+	gv_save_config(p->vinumconf);
 	return (0);
 }
diff --git a/sys/geom/vinum/geom_vinum_plex.c b/sys/geom/vinum/geom_vinum_plex.c
index 8281cb2..f177068 100644
--- a/sys/geom/vinum/geom_vinum_plex.c
+++ b/sys/geom/vinum/geom_vinum_plex.c
@@ -1,5 +1,6 @@
 /*-
- * Copyright (c) 2004 Lukas Ertl
+ * Copyright (c) 2004, 2007 Lukas Ertl
+ * Copyright (c) 2007, 2009 Ulf Lilleengen
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -29,13 +30,8 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/bio.h>
-#include <sys/kernel.h>
-#include <sys/kthread.h>
-#include <sys/libkern.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
-#include <sys/module.h>
-#include <sys/mutex.h>
 #include <sys/systm.h>
 
 #include <geom/geom.h>
@@ -43,329 +39,422 @@ __FBSDID("$FreeBSD$");
 #include <geom/vinum/geom_vinum_raid5.h>
 #include <geom/vinum/geom_vinum.h>
 
-static void gv_plex_completed_request(struct gv_plex *, struct bio *);
-static void gv_plex_normal_request(struct gv_plex *, struct bio *);
-static void gv_plex_worker(void *);
-static int gv_check_parity(struct gv_plex *, struct bio *,
-    struct gv_raid5_packet *);
-static int gv_normal_parity(struct gv_plex *, struct bio *,
-    struct gv_raid5_packet *);
-
-/* XXX: is this the place to catch dying subdisks? */
-static void
-gv_plex_orphan(struct g_consumer *cp)
+static int	gv_check_parity(struct gv_plex *, struct bio *,
+		    struct gv_raid5_packet *);
+static int	gv_normal_parity(struct gv_plex *, struct bio *,
+		    struct gv_raid5_packet *);
+static void	gv_plex_flush(struct gv_plex *);
+static int	gv_plex_offset(struct gv_plex *, off_t, off_t, off_t *, off_t *,
+		    int *, int);
+static int 	gv_plex_normal_request(struct gv_plex *, struct bio *, off_t,
+		    off_t,  caddr_t);
+void
+gv_plex_start(struct gv_plex *p, struct bio *bp)
 {
-	struct g_geom *gp;
-	struct gv_plex *p;
-	int error;
+	struct bio *cbp;
+	struct gv_sd *s;
+	struct gv_raid5_packet *wp;
+	caddr_t addr;
+	off_t bcount, boff, len;
 
-	g_topology_assert();
-	gp = cp->geom;
-	g_trace(G_T_TOPOLOGY, "gv_plex_orphan(%s)", gp->name);
-
-	if (cp->acr != 0 || cp->acw != 0 || cp->ace != 0)
-		g_access(cp, -cp->acr, -cp->acw, -cp->ace);
-	error = cp->provider->error;
-	if (error == 0)
-		error = ENXIO;
-	g_detach(cp);
-	g_destroy_consumer(cp);	
-	if (!LIST_EMPTY(&gp->consumer))
-		return;
+	bcount = bp->bio_length;
+	addr = bp->bio_data;
+	boff = bp->bio_offset;
+
+	/* Walk over the whole length of the request, we might split it up. */
+	while (bcount > 0) {
+		wp = NULL;
+
+ 		/*
+		 * RAID5 plexes need special treatment, as a single request
+		 * might involve several read/write sub-requests.
+ 		 */
+		if (p->org == GV_PLEX_RAID5) {
+			wp = gv_raid5_start(p, bp, addr, boff, bcount);
+ 			if (wp == NULL)
+ 				return;
+ 
+			len = wp->length;
 
-	p = gp->softc;
-	if (p != NULL) {
-		gv_kill_plex_thread(p);
-		p->geom = NULL;
-		p->provider = NULL;
-		p->consumer = NULL;
+			if (TAILQ_EMPTY(&wp->bits))
+				g_free(wp);
+			else if (wp->lockbase != -1)
+				TAILQ_INSERT_TAIL(&p->packets, wp, list);
+
+		/*
+		 * Requests to concatenated and striped plexes go straight
+		 * through.
+		 */
+		} else {
+			len = gv_plex_normal_request(p, bp, boff, bcount, addr);
+		}
+		if (len < 0)
+			return;
+			
+		bcount -= len;
+		addr += len;
+		boff += len;
 	}
-	gp->softc = NULL;
-	g_wither_geom(gp, error);
-}
 
-void
-gv_plex_done(struct bio *bp)
-{
-	struct gv_plex *p;
-
-	p = bp->bio_from->geom->softc;
-	bp->bio_cflags |= GV_BIO_DONE;
-	mtx_lock(&p->bqueue_mtx);
-	bioq_insert_tail(p->bqueue, bp);
-	wakeup(p);
-	mtx_unlock(&p->bqueue_mtx);
+	/*
+	 * Fire off all sub-requests.  We get the correct consumer (== drive)
+	 * to send each request to via the subdisk that was stored in
+	 * cbp->bio_caller1.
+	 */
+	cbp = bioq_takefirst(p->bqueue);
+	while (cbp != NULL) {
+		/*
+		 * RAID5 sub-requests need to come in correct order, otherwise
+		 * we trip over the parity, as it might be overwritten by
+		 * another sub-request.  We abuse cbp->bio_caller2 to mark
+		 * potential overlap situations. 
+		 */
+		if (cbp->bio_caller2 != NULL && gv_stripe_active(p, cbp)) {
+			/* Park the bio on the waiting queue. */
+			cbp->bio_cflags |= GV_BIO_ONHOLD;
+			bioq_disksort(p->wqueue, cbp);
+		} else {
+			s = cbp->bio_caller1;
+			g_io_request(cbp, s->drive_sc->consumer);
+		}
+		cbp = bioq_takefirst(p->bqueue);
+	}
 }
 
-/* Find the correct subdisk to send the bio to and build a bio to send. */
 static int
-gv_plexbuffer(struct gv_plex *p, struct bio *bp, caddr_t addr, off_t boff, off_t bcount)
+gv_plex_offset(struct gv_plex *p, off_t boff, off_t bcount, off_t *real_off,
+    off_t *real_len, int *sdno, int growing)
 {
-	struct g_geom *gp;
 	struct gv_sd *s;
-	struct bio *cbp, *pbp;
-	int i, sdno;
-	off_t len_left, real_len, real_off;
-	off_t stripeend, stripeno, stripestart;
-
-	if (p == NULL || LIST_EMPTY(&p->subdisks))
-		return (ENXIO);
-
-	s = NULL;
-	gp = bp->bio_to->geom;
+	int i, sdcount;
+	off_t len_left, stripeend, stripeno, stripestart;
 
-	/*
-	 * We only handle concatenated and striped plexes here.  RAID5 plexes
-	 * are handled in build_raid5_request().
-	 */
 	switch (p->org) {
 	case GV_PLEX_CONCAT:
 		/*
 		 * Find the subdisk where this request starts.  The subdisks in
 		 * this list must be ordered by plex_offset.
 		 */
+		i = 0;
 		LIST_FOREACH(s, &p->subdisks, in_plex) {
 			if (s->plex_offset <= boff &&
-			    s->plex_offset + s->size > boff)
+			    s->plex_offset + s->size > boff) {
+				*sdno = i;
 				break;
+			}
+			i++;
 		}
-		/* Subdisk not found. */
-		if (s == NULL)
-			return (ENXIO);
+		if (s == NULL || s->drive_sc == NULL)
+			return (GV_ERR_NOTFOUND);
 
 		/* Calculate corresponding offsets on disk. */
-		real_off = boff - s->plex_offset;
-		len_left = s->size - real_off;
-		real_len = (bcount > len_left) ? len_left : bcount;
+		*real_off = boff - s->plex_offset;
+		len_left = s->size - (*real_off);
+		KASSERT(len_left >= 0, ("gv_plex_offset: len_left < 0"));
+		*real_len = (bcount > len_left) ? len_left : bcount;
 		break;
 
 	case GV_PLEX_STRIPED:
 		/* The number of the stripe where the request starts. */
 		stripeno = boff / p->stripesize;
+		KASSERT(stripeno >= 0, ("gv_plex_offset: stripeno < 0"));
 
-		/* The number of the subdisk where the stripe resides. */
-		sdno = stripeno % p->sdcount;
-
-		/* Find the right subdisk. */
-		i = 0;
-		LIST_FOREACH(s, &p->subdisks, in_plex) {
-			if (i == sdno)
-				break;
-			i++;
-		}
+		/* Take growing subdisks into account when calculating. */
+		sdcount = gv_sdcount(p, (boff >= p->synced));
 
-		/* Subdisk not found. */
-		if (s == NULL)
-			return (ENXIO);
+		if (!(boff + bcount <= p->synced) &&
+		    (p->flags & GV_PLEX_GROWING) &&
+		    !growing)
+			return (GV_ERR_ISBUSY);
+		*sdno = stripeno % sdcount;
 
-		/* The offset of the stripe from the start of the subdisk. */ 
-		stripestart = (stripeno / p->sdcount) *
+		KASSERT(sdno >= 0, ("gv_plex_offset: sdno < 0"));
+		stripestart = (stripeno / sdcount) *
 		    p->stripesize;
-
-		/* The offset at the end of the stripe. */
+		KASSERT(stripestart >= 0, ("gv_plex_offset: stripestart < 0"));
 		stripeend = stripestart + p->stripesize;
-
-		/* The offset of the request on this subdisk. */
-		real_off = boff - (stripeno * p->stripesize) +
+		*real_off = boff - (stripeno * p->stripesize) +
 		    stripestart;
+		len_left = stripeend - *real_off;
+		KASSERT(len_left >= 0, ("gv_plex_offset: len_left < 0"));
 
-		/* The length left in this stripe. */
-		len_left = stripeend - real_off;
-
-		real_len = (bcount <= len_left) ? bcount : len_left;
+		*real_len = (bcount <= len_left) ? bcount : len_left;
 		break;
 
 	default:
-		return (EINVAL);
+		return (GV_ERR_PLEXORG);
+	}
+	return (0);
+}
+
+/*
+ * Prepare a normal plex request.
+ */
+static int 
+gv_plex_normal_request(struct gv_plex *p, struct bio *bp, off_t boff,
+    off_t bcount,  caddr_t addr)
+{
+	struct gv_sd *s;
+	struct bio *cbp;
+	off_t real_len, real_off;
+	int i, err, sdno;
+
+	s = NULL;
+	sdno = -1;
+	real_len = real_off = 0;
+
+	err = ENXIO;
+
+	if (p == NULL || LIST_EMPTY(&p->subdisks)) 
+		goto bad;
+
+	err = gv_plex_offset(p, boff, bcount, &real_off,
+	    &real_len, &sdno, (bp->bio_pflags & GV_BIO_SYNCREQ));
+	/* If the request was blocked, put it into wait. */
+	if (err == GV_ERR_ISBUSY) {
+		bioq_disksort(p->rqueue, bp);
+		return (-1); /* "Fail", and delay request. */
+	}
+	if (err) {
+		err = ENXIO;
+		goto bad;
 	}
+	err = ENXIO;
+
+	/* Find the right subdisk. */
+	i = 0;
+	LIST_FOREACH(s, &p->subdisks, in_plex) {
+		if (i == sdno)
+			break;
+		i++;
+	}
+
+	/* Subdisk not found. */
+	if (s == NULL || s->drive_sc == NULL)
+		goto bad;
 
 	/* Now check if we can handle the request on this subdisk. */
 	switch (s->state) {
 	case GV_SD_UP:
 		/* If the subdisk is up, just continue. */
 		break;
-
+	case GV_SD_DOWN:
+		if (bp->bio_cflags & GV_BIO_INTERNAL)
+			G_VINUM_DEBUG(0, "subdisk must be in the stale state in"
+			    " order to perform administrative requests");
+		goto bad;
 	case GV_SD_STALE:
-		if (!(bp->bio_cflags & GV_BIO_SYNCREQ))
-			return (ENXIO);
+		if (!(bp->bio_cflags & GV_BIO_SYNCREQ)) {
+			G_VINUM_DEBUG(0, "subdisk stale, unable to perform "
+			    "regular requests");
+			goto bad;
+		}
 
 		G_VINUM_DEBUG(1, "sd %s is initializing", s->name);
 		gv_set_sd_state(s, GV_SD_INITIALIZING, GV_SETSTATE_FORCE);
 		break;
-
 	case GV_SD_INITIALIZING:
 		if (bp->bio_cmd == BIO_READ)
-			return (ENXIO);
+			goto bad;
 		break;
-
 	default:
 		/* All other subdisk states mean it's not accessible. */
-		return (ENXIO);
+		goto bad;
 	}
 
 	/* Clone the bio and adjust the offsets and sizes. */
 	cbp = g_clone_bio(bp);
-	if (cbp == NULL)
-		return (ENOMEM);
-	cbp->bio_offset = real_off;
+	if (cbp == NULL) {
+		err = ENOMEM;
+		goto bad;
+	}
+	cbp->bio_offset = real_off + s->drive_offset;
 	cbp->bio_length = real_len;
 	cbp->bio_data = addr;
-	cbp->bio_done = g_std_done;
-	cbp->bio_caller2 = s->consumer;
-	if ((bp->bio_cflags & GV_BIO_SYNCREQ)) {
+	cbp->bio_done = gv_done;
+	cbp->bio_caller1 = s;
+	if ((bp->bio_cflags & GV_BIO_SYNCREQ))
 		cbp->bio_cflags |= GV_BIO_SYNCREQ;
-		cbp->bio_done = gv_plex_done;
-	}
 
-	if (bp->bio_driver1 == NULL) {
-		bp->bio_driver1 = cbp;
-	} else {
-		pbp = bp->bio_driver1;
-		while (pbp->bio_caller1 != NULL)
-			pbp = pbp->bio_caller1;
-		pbp->bio_caller1 = cbp;
+	/* Store the sub-requests now and let others issue them. */
+	bioq_insert_tail(p->bqueue, cbp); 
+	return (real_len);
+bad:
+	G_VINUM_LOGREQ(0, bp, "plex request failed.");
+	/* Building the sub-request failed. If internal BIO, do not deliver. */
+	if (bp->bio_cflags & GV_BIO_INTERNAL) {
+		if (bp->bio_cflags & GV_BIO_MALLOC)
+			g_free(bp->bio_data);
+		g_destroy_bio(bp);
+		p->flags &= ~(GV_PLEX_SYNCING | GV_PLEX_REBUILDING |
+		    GV_PLEX_GROWING);
+		return (-1);
 	}
-
-	return (0);
+	g_io_deliver(bp, err);
+	return (-1);
 }
 
-static void
-gv_plex_start(struct bio *bp)
+/*
+ * Handle a completed request to a striped or concatenated plex.
+ */
+void
+gv_plex_normal_done(struct gv_plex *p, struct bio *bp)
 {
-	struct gv_plex *p;
-
-	switch(bp->bio_cmd) {
-	case BIO_READ:
-	case BIO_WRITE:
-	case BIO_DELETE:
-		break;
-	case BIO_GETATTR:
-	default:
-		g_io_deliver(bp, EOPNOTSUPP);
-		return;
-	}
+	struct bio *pbp;
 
-	/*
-	 * We cannot handle this request if too many of our subdisks are
-	 * inaccessible.
-	 */
-	p = bp->bio_to->geom->softc;
-	if ((p->state < GV_PLEX_DEGRADED) &&
-	    !(bp->bio_cflags & GV_BIO_SYNCREQ)) {
-		g_io_deliver(bp, ENXIO);
-		return;
+	pbp = bp->bio_parent;
+	if (pbp->bio_error == 0)
+		pbp->bio_error = bp->bio_error;
+	g_destroy_bio(bp);
+	pbp->bio_inbed++;
+	if (pbp->bio_children == pbp->bio_inbed) {
+		/* Just set it to length since multiple plexes will
+		 * screw things up. */
+		pbp->bio_completed = pbp->bio_length;
+		if (pbp->bio_cflags & GV_BIO_SYNCREQ)
+			gv_sync_complete(p, pbp);
+		else if (pbp->bio_pflags & GV_BIO_SYNCREQ)
+			gv_grow_complete(p, pbp);
+		else
+			g_io_deliver(pbp, pbp->bio_error);
 	}
-
-	mtx_lock(&p->bqueue_mtx);
-	bioq_disksort(p->bqueue, bp);
-	wakeup(p);
-	mtx_unlock(&p->bqueue_mtx);
 }
 
-static void
-gv_plex_worker(void *arg)
+/*
+ * Handle a completed request to a RAID-5 plex.
+ */
+void
+gv_plex_raid5_done(struct gv_plex *p, struct bio *bp)
 {
-	struct bio *bp;
-	struct gv_plex *p;
-	struct gv_sd *s;
+	struct gv_softc *sc;
+	struct bio *cbp, *pbp;
+	struct gv_bioq *bq, *bq2;
+	struct gv_raid5_packet *wp;
+	off_t completed;
+	int i;
 
-	p = arg;
-	KASSERT(p != NULL, ("NULL p"));
+	completed = 0;
+	sc = p->vinumconf;
+	wp = bp->bio_caller2;
 
-	mtx_lock(&p->bqueue_mtx);
-	for (;;) {
-		/* We were signaled to exit. */
-		if (p->flags & GV_PLEX_THREAD_DIE)
+	switch (bp->bio_parent->bio_cmd) {
+	case BIO_READ:
+		if (wp == NULL) {
+			completed = bp->bio_completed;
 			break;
+		}
 
-		/* Take the first BIO from our queue. */
-		bp = bioq_takefirst(p->bqueue);
-		if (bp == NULL) {
-			msleep(p, &p->bqueue_mtx, PRIBIO, "-", hz/10);
-			continue;
+		TAILQ_FOREACH_SAFE(bq, &wp->bits, queue, bq2) {
+			if (bq->bp != bp)
+				continue;
+			TAILQ_REMOVE(&wp->bits, bq, queue);
+			g_free(bq);
+			for (i = 0; i < wp->length; i++)
+				wp->data[i] ^= bp->bio_data[i];
+			break;
 		}
-		mtx_unlock(&p->bqueue_mtx);
-
-		/* A completed request. */
-		if (bp->bio_cflags & GV_BIO_DONE) {
-			if (bp->bio_cflags & GV_BIO_SYNCREQ ||
-			    bp->bio_cflags & GV_BIO_REBUILD) {
-				s = bp->bio_to->private;
-				if (bp->bio_error == 0)
-					s->initialized += bp->bio_length;
-				if (s->initialized >= s->size) {
-					g_topology_lock();
-					gv_set_sd_state(s, GV_SD_UP,
-					    GV_SETSTATE_CONFIG);
-					g_topology_unlock();
-					s->initialized = 0;
+		if (TAILQ_EMPTY(&wp->bits)) {
+			completed = wp->length;
+			if (wp->lockbase != -1) {
+				TAILQ_REMOVE(&p->packets, wp, list);
+				/* Bring the waiting bios back into the game. */
+				pbp = bioq_takefirst(p->wqueue);
+				while (pbp != NULL) {
+					mtx_lock(&sc->queue_mtx);
+					bioq_disksort(sc->bqueue, pbp);
+					mtx_unlock(&sc->queue_mtx);
+					pbp = bioq_takefirst(p->wqueue);
 				}
 			}
+			g_free(wp);
+		}
 
-			if (bp->bio_cflags & GV_BIO_SYNCREQ)
-				g_std_done(bp);
-			else
-				gv_plex_completed_request(p, bp);
-		/*
-		 * A sub-request that was hold back because it interfered with
-		 * another sub-request.
-		 */
-		} else if (bp->bio_cflags & GV_BIO_ONHOLD) {
-			/* Is it still locked out? */
-			if (gv_stripe_active(p, bp)) {
-				/* Park the bio on the waiting queue. */
-				mtx_lock(&p->bqueue_mtx);
-				bioq_disksort(p->wqueue, bp);
-				mtx_unlock(&p->bqueue_mtx);
-			} else {
-				bp->bio_cflags &= ~GV_BIO_ONHOLD;
-				g_io_request(bp, bp->bio_caller2);
-			}
+		break;
 
-		/* A normal request to this plex. */
-		} else
-			gv_plex_normal_request(p, bp);
+ 	case BIO_WRITE:
+		/* XXX can this ever happen? */
+		if (wp == NULL) {
+			completed = bp->bio_completed;
+			break;
+		}
 
-		mtx_lock(&p->bqueue_mtx);
-	}
-	mtx_unlock(&p->bqueue_mtx);
-	p->flags |= GV_PLEX_THREAD_DEAD;
-	wakeup(p);
+		/* Check if we need to handle parity data. */
+		TAILQ_FOREACH_SAFE(bq, &wp->bits, queue, bq2) {
+			if (bq->bp != bp)
+				continue;
+			TAILQ_REMOVE(&wp->bits, bq, queue);
+			g_free(bq);
+			cbp = wp->parity;
+			if (cbp != NULL) {
+				for (i = 0; i < wp->length; i++)
+					cbp->bio_data[i] ^= bp->bio_data[i];
+			}
+			break;
+		}
 
-	kproc_exit(ENXIO);
-}
+		/* Handle parity data. */
+		if (TAILQ_EMPTY(&wp->bits)) {
+			if (bp->bio_parent->bio_cflags & GV_BIO_CHECK)
+				i = gv_check_parity(p, bp, wp);
+			else
+				i = gv_normal_parity(p, bp, wp);
 
-static int
-gv_normal_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)
-{
-	struct bio *cbp, *pbp;
-	int finished, i;
+			/* All of our sub-requests have finished. */
+			if (i) {
+				completed = wp->length;
+				TAILQ_REMOVE(&p->packets, wp, list);
+				/* Bring the waiting bios back into the game. */
+				pbp = bioq_takefirst(p->wqueue);
+				while (pbp != NULL) {
+					mtx_lock(&sc->queue_mtx);
+					bioq_disksort(sc->bqueue, pbp);
+					mtx_unlock(&sc->queue_mtx);
+					pbp = bioq_takefirst(p->wqueue);
+				}
+				g_free(wp);
+			}
+		}
 
-	finished = 1;
+		break;
+	}
 
-	if (wp->waiting != NULL) {
-		pbp = wp->waiting;
-		wp->waiting = NULL;
-		cbp = wp->parity;
-		for (i = 0; i < wp->length; i++)
-			cbp->bio_data[i] ^= pbp->bio_data[i];
-		g_io_request(pbp, pbp->bio_caller2);
-		finished = 0;
+	pbp = bp->bio_parent;
+	if (pbp->bio_error == 0)
+		pbp->bio_error = bp->bio_error;
+	pbp->bio_completed += completed;
 
-	} else if (wp->parity != NULL) {
-		cbp = wp->parity;
-		wp->parity = NULL;
-		g_io_request(cbp, cbp->bio_caller2);
-		finished = 0;
+	/* When the original request is finished, we deliver it. */
+	pbp->bio_inbed++;
+	if (pbp->bio_inbed == pbp->bio_children) {
+		/* Hand it over for checking or delivery. */
+		if (pbp->bio_cmd == BIO_WRITE &&
+		    (pbp->bio_cflags & GV_BIO_CHECK)) {
+			gv_parity_complete(p, pbp);
+		} else if (pbp->bio_cmd == BIO_WRITE &&
+		    (pbp->bio_cflags & GV_BIO_REBUILD)) {
+			gv_rebuild_complete(p, pbp);
+		} else if (pbp->bio_cflags & GV_BIO_INIT) {
+			gv_init_complete(p, pbp);
+		} else if (pbp->bio_cflags & GV_BIO_SYNCREQ) {
+			gv_sync_complete(p, pbp);
+		} else if (pbp->bio_pflags & GV_BIO_SYNCREQ) {
+			gv_grow_complete(p, pbp);
+		} else {
+			g_io_deliver(pbp, pbp->bio_error);
+		}
 	}
 
-	return (finished);
+	/* Clean up what we allocated. */
+	if (bp->bio_cflags & GV_BIO_MALLOC)
+		g_free(bp->bio_data);
+	g_destroy_bio(bp);
 }
 
 static int
 gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)
 {
 	struct bio *pbp;
+	struct gv_sd *s;
 	int err, finished, i;
 
 	err = 0;
@@ -374,7 +463,8 @@ gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)
 	if (wp->waiting != NULL) {
 		pbp = wp->waiting;
 		wp->waiting = NULL;
-		g_io_request(pbp, pbp->bio_caller2);
+		s = pbp->bio_caller1;
+		g_io_request(pbp, s->drive_sc->consumer);
 		finished = 0;
 
 	} else if (wp->parity != NULL) {
@@ -395,7 +485,8 @@ gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)
 
 			/* ... but we rebuild it. */
 			if (bp->bio_parent->bio_cflags & GV_BIO_PARITY) {
-				g_io_request(pbp, pbp->bio_caller2);
+				s = pbp->bio_caller1;
+				g_io_request(pbp, s->drive_sc->consumer);
 				finished = 0;
 			}
 		}
@@ -414,454 +505,542 @@ gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)
 	return (finished);
 }
 
-void
-gv_plex_completed_request(struct gv_plex *p, struct bio *bp)
+static int
+gv_normal_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)
 {
 	struct bio *cbp, *pbp;
-	struct gv_bioq *bq, *bq2;
-	struct gv_raid5_packet *wp;
-	int i;
+	struct gv_sd *s;
+	int finished, i;
 
-	wp = bp->bio_driver1;
+	finished = 1;
 
-	switch (bp->bio_parent->bio_cmd) {
-	case BIO_READ:
-		if (wp == NULL)
-			break;
+	if (wp->waiting != NULL) {
+		pbp = wp->waiting;
+		wp->waiting = NULL;
+		cbp = wp->parity;
+		for (i = 0; i < wp->length; i++)
+			cbp->bio_data[i] ^= pbp->bio_data[i];
+		s = pbp->bio_caller1;
+		g_io_request(pbp, s->drive_sc->consumer);
+		finished = 0;
 
-		TAILQ_FOREACH_SAFE(bq, &wp->bits, queue, bq2) {
-			if (bq->bp == bp) {
-				TAILQ_REMOVE(&wp->bits, bq, queue);
-				g_free(bq);
-				for (i = 0; i < wp->length; i++)
-					wp->data[i] ^= bp->bio_data[i];
-				break;
-			}
-		}
-		if (TAILQ_EMPTY(&wp->bits)) {
-			bp->bio_parent->bio_completed += wp->length;
-			if (wp->lockbase != -1) {
-				TAILQ_REMOVE(&p->packets, wp, list);
-				/* Bring the waiting bios back into the game. */
-				mtx_lock(&p->bqueue_mtx);
-				pbp = bioq_takefirst(p->wqueue);
-				while (pbp != NULL) {
-					bioq_disksort(p->bqueue, pbp);
-					pbp = bioq_takefirst(p->wqueue);
-				}
-				mtx_unlock(&p->bqueue_mtx);
-			}
-			g_free(wp);
-		}
+	} else if (wp->parity != NULL) {
+		cbp = wp->parity;
+		wp->parity = NULL;
+		s = cbp->bio_caller1;
+		g_io_request(cbp, s->drive_sc->consumer);
+		finished = 0;
+	}
 
-		break;
+	return (finished);
+}
 
- 	case BIO_WRITE:
-		if (wp == NULL)
-			break;
+/* Flush the queue with delayed requests. */
+static void
+gv_plex_flush(struct gv_plex *p)
+{
+	struct gv_softc *sc;
+	struct bio *bp;
 
-		/* Check if we need to handle parity data. */
-		TAILQ_FOREACH_SAFE(bq, &wp->bits, queue, bq2) {
-			if (bq->bp == bp) {
-				TAILQ_REMOVE(&wp->bits, bq, queue);
-				g_free(bq);
-				cbp = wp->parity;
-				if (cbp != NULL) {
-					for (i = 0; i < wp->length; i++)
-						cbp->bio_data[i] ^=
-						    bp->bio_data[i];
-				}
-				break;
-			}
-		}
+	sc = p->vinumconf;
+	bp = bioq_takefirst(p->rqueue);
+	while (bp != NULL) {
+		gv_plex_start(p, bp);
+		bp = bioq_takefirst(p->rqueue);
+	}
+}
 
-		/* Handle parity data. */
-		if (TAILQ_EMPTY(&wp->bits)) {
-			if (bp->bio_parent->bio_cflags & GV_BIO_CHECK)
-				i = gv_check_parity(p, bp, wp);
-			else
-				i = gv_normal_parity(p, bp, wp);
+int
+gv_sync_request(struct gv_plex *from, struct gv_plex *to, off_t offset,
+    off_t length, int type, caddr_t data)
+{
+	struct gv_softc *sc;
+	struct bio *bp;
 
-			/* All of our sub-requests have finished. */
-			if (i) {
-				bp->bio_parent->bio_completed += wp->length;
-				TAILQ_REMOVE(&p->packets, wp, list);
-				/* Bring the waiting bios back into the game. */
-				mtx_lock(&p->bqueue_mtx);
-				pbp = bioq_takefirst(p->wqueue);
-				while (pbp != NULL) {
-					bioq_disksort(p->bqueue, pbp);
-					pbp = bioq_takefirst(p->wqueue);
-				}
-				mtx_unlock(&p->bqueue_mtx);
-				g_free(wp);
-			}
-		}
+	KASSERT(from != NULL, ("NULL from"));
+	KASSERT(to != NULL, ("NULL to"));
+	sc = from->vinumconf;
+	KASSERT(sc != NULL, ("NULL sc"));
 
-		break;
+	bp = g_new_bio();
+	if (bp == NULL) {
+		G_VINUM_DEBUG(0, "sync from '%s' failed at offset "
+		    " %jd; out of memory", from->name, offset);
+		return (ENOMEM);
 	}
+	bp->bio_length = length;
+	bp->bio_done = gv_done;
+	bp->bio_cflags |= GV_BIO_SYNCREQ;
+	bp->bio_offset = offset;
+	bp->bio_caller1 = from;		
+	bp->bio_caller2 = to;
+	bp->bio_cmd = type;
+	if (data == NULL)
+		data = g_malloc(length, M_WAITOK);
+	bp->bio_cflags |= GV_BIO_MALLOC; /* Free on the next run. */
+	bp->bio_data = data;
+
+	/* Send down next. */
+	mtx_lock(&sc->queue_mtx);
+	bioq_disksort(sc->bqueue, bp);
+	mtx_unlock(&sc->queue_mtx);
+	//gv_plex_start(from, bp);
+	return (0);
+}
 
-	pbp = bp->bio_parent;
-	if (pbp->bio_error == 0)
-		pbp->bio_error = bp->bio_error;
+/*
+ * Handle a finished plex sync bio.
+ */
+int
+gv_sync_complete(struct gv_plex *to, struct bio *bp)
+{
+	struct gv_plex *from, *p;
+	struct gv_sd *s;
+	struct gv_volume *v;
+	struct gv_softc *sc;
+	off_t offset;
+	int err;
 
-	/* When the original request is finished, we deliver it. */
-	pbp->bio_inbed++;
-	if (pbp->bio_inbed == pbp->bio_children)
-		g_io_deliver(pbp, pbp->bio_error);
+	g_topology_assert_not();
 
-	/* Clean up what we allocated. */
-	if (bp->bio_cflags & GV_BIO_MALLOC)
-		g_free(bp->bio_data);
+	err = 0;
+	KASSERT(to != NULL, ("NULL to"));
+	KASSERT(bp != NULL, ("NULL bp"));
+	from = bp->bio_caller2;
+	KASSERT(from != NULL, ("NULL from"));
+	v = to->vol_sc;
+	KASSERT(v != NULL, ("NULL v"));
+	sc = v->vinumconf;
+	KASSERT(sc != NULL, ("NULL sc"));
+
+	/* If it was a read, write it. */
+	if (bp->bio_cmd == BIO_READ) {
+		err = gv_sync_request(from, to, bp->bio_offset, bp->bio_length,
+	    	    BIO_WRITE, bp->bio_data);
+	/* If it was a write, read the next one. */
+	} else if (bp->bio_cmd == BIO_WRITE) {
+		if (bp->bio_cflags & GV_BIO_MALLOC)
+			g_free(bp->bio_data);
+		to->synced += bp->bio_length;
+		/* If we're finished, clean up. */
+		if (bp->bio_offset + bp->bio_length >= from->size) {
+			G_VINUM_DEBUG(1, "syncing of %s from %s completed",
+			    to->name, from->name);
+			/* Update our state. */
+			LIST_FOREACH(s, &to->subdisks, in_plex)
+				gv_set_sd_state(s, GV_SD_UP, 0);
+			gv_update_plex_state(to);
+			to->flags &= ~GV_PLEX_SYNCING;
+			to->synced = 0;
+			gv_post_event(sc, GV_EVENT_SAVE_CONFIG, sc, NULL, 0, 0);
+		} else {
+			offset = bp->bio_offset + bp->bio_length;
+			err = gv_sync_request(from, to, offset,
+			    MIN(bp->bio_length, from->size - offset),
+			    BIO_READ, NULL);
+		}
+	}
 	g_destroy_bio(bp);
+	/* Clean up if there was an error. */
+	if (err) {
+		to->flags &= ~GV_PLEX_SYNCING;
+		G_VINUM_DEBUG(0, "error syncing plexes: error code %d", err);
+	}
+
+	/* Check if all plexes are synced, and lower refcounts. */
+	g_topology_lock();
+	LIST_FOREACH(p, &v->plexes, in_volume) {
+		if (p->flags & GV_PLEX_SYNCING) {
+			g_topology_unlock();
+			return (-1);
+		}
+	}
+	/* If we came here, all plexes are synced, and we're free. */
+	gv_access(v->provider, -1, -1, 0);
+	g_topology_unlock();
+	G_VINUM_DEBUG(1, "plex sync completed");
+	gv_volume_flush(v);
+	return (0);
 }
 
-void
-gv_plex_normal_request(struct gv_plex *p, struct bio *bp)
+/*
+ * Create a new bio struct for the next grow request.
+ */
+int
+gv_grow_request(struct gv_plex *p, off_t offset, off_t length, int type,
+    caddr_t data)
 {
-	struct bio *cbp, *pbp;
-	struct gv_bioq *bq, *bq2;
-	struct gv_raid5_packet *wp, *wp2;
-	caddr_t addr;
-	off_t bcount, boff;
-	int err;
-
-	bcount = bp->bio_length;
-	addr = bp->bio_data;
-	boff = bp->bio_offset;
+	struct gv_softc *sc;
+	struct bio *bp;
 
-	/* Walk over the whole length of the request, we might split it up. */
-	while (bcount > 0) {
-		wp = NULL;
+	KASSERT(p != NULL, ("gv_grow_request: NULL p"));
+	sc = p->vinumconf;
+	KASSERT(sc != NULL, ("gv_grow_request: NULL sc"));
 
- 		/*
-		 * RAID5 plexes need special treatment, as a single write
-		 * request involves several read/write sub-requests.
- 		 */
-		if (p->org == GV_PLEX_RAID5) {
-			wp = g_malloc(sizeof(*wp), M_WAITOK | M_ZERO);
-			wp->bio = bp;
-			TAILQ_INIT(&wp->bits);
-
-			if (bp->bio_cflags & GV_BIO_REBUILD)
-				err = gv_rebuild_raid5(p, wp, bp, addr,
-				    boff, bcount);
-			else if (bp->bio_cflags & GV_BIO_CHECK)
-				err = gv_check_raid5(p, wp, bp, addr,
-				    boff, bcount);
-			else
-				err = gv_build_raid5_req(p, wp, bp, addr,
-				    boff, bcount);
-
- 			/*
-			 * Building the sub-request failed, we probably need to
-			 * clean up a lot.
- 			 */
- 			if (err) {
-				G_VINUM_LOGREQ(0, bp, "plex request failed.");
-				TAILQ_FOREACH_SAFE(bq, &wp->bits, queue, bq2) {
-					TAILQ_REMOVE(&wp->bits, bq, queue);
-					g_free(bq);
-				}
-				if (wp->waiting != NULL) {
-					if (wp->waiting->bio_cflags &
-					    GV_BIO_MALLOC)
-						g_free(wp->waiting->bio_data);
-					g_destroy_bio(wp->waiting);
-				}
-				if (wp->parity != NULL) {
-					if (wp->parity->bio_cflags &
-					    GV_BIO_MALLOC)
-						g_free(wp->parity->bio_data);
-					g_destroy_bio(wp->parity);
-				}
-				g_free(wp);
-
-				TAILQ_FOREACH_SAFE(wp, &p->packets, list, wp2) {
-					if (wp->bio == bp) {
-						TAILQ_REMOVE(&p->packets, wp,
-						    list);
-						TAILQ_FOREACH_SAFE(bq,
-						    &wp->bits, queue, bq2) {
-							TAILQ_REMOVE(&wp->bits,
-							    bq, queue);
-							g_free(bq);
-						}
-						g_free(wp);
-					}
-				}
+	bp = g_new_bio();
+	if (bp == NULL) {
+		G_VINUM_DEBUG(0, "grow of %s failed creating bio: "
+		    "out of memory", p->name);
+		return (ENOMEM);
+	}
 
-				cbp = bp->bio_driver1;
-				while (cbp != NULL) {
-					pbp = cbp->bio_caller1;
-					if (cbp->bio_cflags & GV_BIO_MALLOC)
-						g_free(cbp->bio_data);
-					g_destroy_bio(cbp);
-					cbp = pbp;
-				}
+	bp->bio_cmd = type;
+	bp->bio_done = gv_done;
+	bp->bio_error = 0;
+	bp->bio_caller1 = p;
+	bp->bio_offset = offset;
+	bp->bio_length = length;
+	bp->bio_pflags |= GV_BIO_SYNCREQ; /* XXX: misuse of pflags AND syncreq.*/
+	if (data == NULL)
+		data = g_malloc(length, M_WAITOK);
+	bp->bio_cflags |= GV_BIO_MALLOC;
+	bp->bio_data = data;
+
+	mtx_lock(&sc->queue_mtx);
+	bioq_disksort(sc->bqueue, bp);
+	mtx_unlock(&sc->queue_mtx);
+	//gv_plex_start(p, bp);
+	return (0);
+}
 
-				g_io_deliver(bp, err);
- 				return;
- 			}
- 
-			if (TAILQ_EMPTY(&wp->bits))
-				g_free(wp);
-			else if (wp->lockbase != -1)
-				TAILQ_INSERT_TAIL(&p->packets, wp, list);
+/*
+ * Finish handling of a bio to a growing plex.
+ */
+void
+gv_grow_complete(struct gv_plex *p, struct bio *bp)
+{
+	struct gv_softc *sc;
+	struct gv_sd *s;
+	struct gv_volume *v;
+	off_t origsize, offset;
+	int sdcount, err;
+
+	v = p->vol_sc;
+	KASSERT(v != NULL, ("gv_grow_complete: NULL v"));
+	sc = v->vinumconf;
+	KASSERT(sc != NULL, ("gv_grow_complete: NULL sc"));
+	err = 0;
 
-		/*
-		 * Requests to concatenated and striped plexes go straight
-		 * through.
-		 */
-		} else {
-			err = gv_plexbuffer(p, bp, addr, boff, bcount);
-
-			/* Building the sub-request failed. */
-			if (err) {
-				G_VINUM_LOGREQ(0, bp, "plex request failed.");
-				cbp = bp->bio_driver1;
-				while (cbp != NULL) {
-					pbp = cbp->bio_caller1;
-					g_destroy_bio(cbp);
-					cbp = pbp;
-				}
-				g_io_deliver(bp, err);
-				return;
+	/* If it was a read, write it. */
+	if (bp->bio_cmd == BIO_READ) {
+		p->synced += bp->bio_length;
+		err = gv_grow_request(p, bp->bio_offset, bp->bio_length,
+		    BIO_WRITE, bp->bio_data);
+	/* If it was a write, read next. */
+	} else if (bp->bio_cmd == BIO_WRITE) {
+		if (bp->bio_cflags & GV_BIO_MALLOC)
+			g_free(bp->bio_data);
+
+		/* Find the real size of the plex. */
+		sdcount = gv_sdcount(p, 1);
+		s = LIST_FIRST(&p->subdisks);
+		KASSERT(s != NULL, ("NULL s"));
+		origsize = (s->size * (sdcount - 1));
+		if (bp->bio_offset + bp->bio_length >= origsize) {
+			G_VINUM_DEBUG(1, "growing of %s completed", p->name);
+			p->flags &= ~GV_PLEX_GROWING;
+			LIST_FOREACH(s, &p->subdisks, in_plex) {
+				s->flags &= ~GV_SD_GROW;
+				gv_set_sd_state(s, GV_SD_UP, 0);
 			}
+			p->size = gv_plex_size(p);
+			gv_update_vol_size(v, gv_vol_size(v));
+			gv_set_plex_state(p, GV_PLEX_UP, 0);
+			g_topology_lock();
+			gv_access(v->provider, -1, -1, 0);
+			g_topology_unlock();
+			p->synced = 0;
+			gv_post_event(sc, GV_EVENT_SAVE_CONFIG, sc, NULL, 0, 0);
+			/* Issue delayed requests. */
+			gv_plex_flush(p);
+		} else {
+			offset = bp->bio_offset + bp->bio_length;
+			err = gv_grow_request(p, offset,
+			   MIN(bp->bio_length, origsize - offset),
+			   BIO_READ, NULL);
 		}
- 
-		/* Abuse bio_caller1 as linked list. */
-		pbp = bp->bio_driver1;
-		while (pbp->bio_caller1 != NULL)
-			pbp = pbp->bio_caller1;
-		bcount -= pbp->bio_length;
-		addr += pbp->bio_length;
-		boff += pbp->bio_length;
-	}
-
-	/* Fire off all sub-requests. */
-	pbp = bp->bio_driver1;
-	while (pbp != NULL) {
-		/*
-		 * RAID5 sub-requests need to come in correct order, otherwise
-		 * we trip over the parity, as it might be overwritten by
-		 * another sub-request.
-		 */
-		if (pbp->bio_driver1 != NULL &&
-		    gv_stripe_active(p, pbp)) {
-			/* Park the bio on the waiting queue. */
-			pbp->bio_cflags |= GV_BIO_ONHOLD;
-			mtx_lock(&p->bqueue_mtx);
-			bioq_disksort(p->wqueue, pbp);
-			mtx_unlock(&p->bqueue_mtx);
-		} else
-			g_io_request(pbp, pbp->bio_caller2);
-		pbp = pbp->bio_caller1;
+	}
+	g_destroy_bio(bp);
+
+	if (err) {
+		p->flags &= ~GV_PLEX_GROWING;
+		G_VINUM_DEBUG(0, "error growing plex: error code %d", err);
 	}
 }
 
-static int
-gv_plex_access(struct g_provider *pp, int dr, int dw, int de)
+
+/*
+ * Create an initialization BIO and send it off to the consumer. Assume that
+ * we're given initialization data as parameter.
+ */
+void
+gv_init_request(struct gv_sd *s, off_t start, caddr_t data, off_t length)
+{
+	struct gv_drive *d;
+	struct g_consumer *cp;
+	struct bio *bp, *cbp;
+
+	KASSERT(s != NULL, ("gv_init_request: NULL s"));
+	d = s->drive_sc;
+	KASSERT(d != NULL, ("gv_init_request: NULL d"));
+	cp = d->consumer;
+	KASSERT(cp != NULL, ("gv_init_request: NULL cp"));
+
+	bp = g_new_bio();
+	if (bp == NULL) {
+		G_VINUM_DEBUG(0, "subdisk '%s' init: write failed at offset %jd"
+		    " (drive offset %jd); out of memory", s->name,
+		    (intmax_t)s->initialized, (intmax_t)start);
+		return; /* XXX: Error codes. */
+	}
+	bp->bio_cmd = BIO_WRITE;
+	bp->bio_data = data;
+	bp->bio_done = gv_done;
+	bp->bio_error = 0;
+	bp->bio_length = length;
+	bp->bio_cflags |= GV_BIO_INIT;
+	bp->bio_offset = start;
+	bp->bio_caller1 = s;
+
+	/* Then ofcourse, we have to clone it. */
+	cbp = g_clone_bio(bp);
+	if (cbp == NULL) {
+		G_VINUM_DEBUG(0, "subdisk '%s' init: write failed at offset %jd"
+		    " (drive offset %jd); out of memory", s->name,
+		    (intmax_t)s->initialized, (intmax_t)start);
+		return; /* XXX: Error codes. */
+	}
+	cbp->bio_done = gv_done;
+	cbp->bio_caller1 = s;
+	/* Send it off to the consumer. */
+	g_io_request(cbp, cp);
+}
+
+/*
+ * Handle a finished initialization BIO.
+ */
+void
+gv_init_complete(struct gv_plex *p, struct bio *bp)
 {
-	struct gv_plex *p;
-	struct g_geom *gp;
-	struct g_consumer *cp, *cp2;
+	struct gv_softc *sc;
+	struct gv_drive *d;
+	struct g_consumer *cp;
+	struct gv_sd *s;
+	off_t start, length;
+	caddr_t data;
 	int error;
 
-	gp = pp->geom;
-	p = gp->softc;
-	KASSERT(p != NULL, ("NULL p"));
+	s = bp->bio_caller1;
+	start = bp->bio_offset;
+	length = bp->bio_length;
+	error = bp->bio_error;
+	data = bp->bio_data;
 
-	if (p->org == GV_PLEX_RAID5) {
-		if (dw > 0 && dr == 0)
-			dr = 1;
-		else if (dw < 0 && dr == 0)
-			dr = -1;
-	}
+	KASSERT(s != NULL, ("gv_init_complete: NULL s"));
+	d = s->drive_sc;
+	KASSERT(d != NULL, ("gv_init_complete: NULL d"));
+	cp = d->consumer;
+	KASSERT(cp != NULL, ("gv_init_complete: NULL cp"));
+	sc = p->vinumconf;
+	KASSERT(sc != NULL, ("gv_init_complete: NULL sc"));
+
+	g_destroy_bio(bp);
 
-	LIST_FOREACH(cp, &gp->consumer, consumer) {
-		error = g_access(cp, dr, dw, de);
+	/*
+	 * First we need to find out if it was okay, and abort if it's not.
+	 * Then we need to free previous buffers, find out the correct subdisk,
+	 * as well as getting the correct starting point and length of the BIO.
+	 */
+	if (start >= s->drive_offset + s->size) {
+		/* Free the data we initialized. */
+		if (data != NULL)
+			g_free(data);
+		g_topology_assert_not();
+		g_topology_lock();
+		g_access(cp, 0, -1, 0);
+		g_topology_unlock();
 		if (error) {
-			LIST_FOREACH(cp2, &gp->consumer, consumer) {
-				if (cp == cp2)
-					break;
-				g_access(cp2, -dr, -dw, -de);
-			}
-			return (error);
+			gv_set_sd_state(s, GV_SD_STALE, GV_SETSTATE_FORCE |
+			    GV_SETSTATE_CONFIG);
+		} else {
+			gv_set_sd_state(s, GV_SD_UP, GV_SETSTATE_CONFIG);
+			s->initialized = 0;
+			gv_post_event(sc, GV_EVENT_SAVE_CONFIG, sc, NULL, 0, 0);
+			G_VINUM_DEBUG(1, "subdisk '%s' init: finished "
+			    "successfully", s->name);
 		}
+		return;
 	}
-	return (0);
+	s->initialized += length;
+	start += length;
+	gv_init_request(s, start, data, length);
 }
 
-static struct g_geom *
-gv_plex_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
+/*
+ * Create a new bio struct for the next parity rebuild. Used both by internal
+ * rebuild of degraded plexes as well as user initiated rebuilds/checks.
+ */
+void
+gv_parity_request(struct gv_plex *p, int flags, off_t offset)
 {
-	struct g_geom *gp;
-	struct g_consumer *cp, *cp2;
-	struct g_provider *pp2;
-	struct gv_plex *p;
-	struct gv_sd *s;
 	struct gv_softc *sc;
-	int error;
-
-	g_trace(G_T_TOPOLOGY, "gv_plex_taste(%s, %s)", mp->name, pp->name);
-	g_topology_assert();
-
-	/* We only want to attach to subdisks. */
-	if (strcmp(pp->geom->class->name, "VINUMDRIVE"))
-		return (NULL);
-
-	/* Find the VINUM class and its associated geom. */
-	gp = find_vinum_geom();
-	if (gp == NULL)
-		return (NULL);
-	sc = gp->softc;
-	KASSERT(sc != NULL, ("gv_plex_taste: NULL sc"));
+	struct bio *bp;
 
-	/* Find out which subdisk the offered provider corresponds to. */
-	s = pp->private;
-	KASSERT(s != NULL, ("gv_plex_taste: NULL s"));
+	KASSERT(p != NULL, ("gv_parity_request: NULL p"));
+	sc = p->vinumconf;
+	KASSERT(sc != NULL, ("gv_parity_request: NULL sc"));
 
-	/* Now find the correct plex where this subdisk belongs to. */
-	p = gv_find_plex(sc, s->plex);
-	if (p == NULL) {
-		G_VINUM_DEBUG(0, "%s: NULL p for '%s'", __func__, s->name);
-		return (NULL);
+	bp = g_new_bio();
+	if (bp == NULL) {
+		G_VINUM_DEBUG(0, "rebuild of %s failed creating bio: "
+		    "out of memory", p->name);
+		return;
 	}
 
+	bp->bio_cmd = BIO_WRITE;
+	bp->bio_done = gv_done;
+	bp->bio_error = 0;
+	bp->bio_length = p->stripesize;
+	bp->bio_caller1 = p;
+
 	/*
-	 * Add this subdisk to this plex.  Since we trust the on-disk
-	 * configuration, we don't check the given value (should we?).
-	 * XXX: shouldn't be done here
+	 * Check if it's a rebuild of a degraded plex or a user request of
+	 * parity rebuild.
 	 */
-	gv_sd_to_plex(p, s, 0);
+	if (flags & GV_BIO_REBUILD)
+		bp->bio_data = g_malloc(GV_DFLT_SYNCSIZE, M_WAITOK);
+	else if (flags & GV_BIO_CHECK)
+		bp->bio_data = g_malloc(p->stripesize, M_WAITOK | M_ZERO);
+	else {
+		G_VINUM_DEBUG(0, "invalid flags given in rebuild");
+		return;
+	}
 
-	/* Now check if there's already a geom for this plex. */
-	gp = p->geom;
+	bp->bio_cflags = flags;
+	bp->bio_cflags |= GV_BIO_MALLOC;
 
-	/* Yes, there is already a geom, so we just add the consumer. */
-	if (gp != NULL) {
-		cp2 = LIST_FIRST(&gp->consumer);
-		/* Need to attach a new consumer to this subdisk. */
-		cp = g_new_consumer(gp);
-		error = g_attach(cp, pp);
-		if (error) {
-			G_VINUM_DEBUG(0, "unable to attach consumer to %s",
-			    pp->name);
-			g_destroy_consumer(cp);
-			return (NULL);
-		}
-		/* Adjust the access counts of the new consumer. */
-		if ((cp2 != NULL) && (cp2->acr || cp2->acw || cp2->ace)) {
-			error = g_access(cp, cp2->acr, cp2->acw, cp2->ace);
-			if (error) {
-				G_VINUM_DEBUG(0, "unable to set access counts"
-				    " for consumer on %s", pp->name);
-				g_detach(cp);
-				g_destroy_consumer(cp);
-				return (NULL);
-			}
-		}
-		s->consumer = cp;
+	/* We still have more parity to build. */
+	bp->bio_offset = offset;
+	mtx_lock(&sc->queue_mtx);
+	bioq_disksort(sc->bqueue, bp);
+	mtx_unlock(&sc->queue_mtx);
+	//gv_plex_start(p, bp); /* Send it down to the plex. */
+}
 
-		/* Adjust the size of the providers this plex has. */
-		LIST_FOREACH(pp2, &gp->provider, provider)
-			pp2->mediasize = p->size;
+/*
+ * Handle a finished parity write.
+ */
+void
+gv_parity_complete(struct gv_plex *p, struct bio *bp)
+{
+	struct gv_softc *sc;
+	int error, flags;
 
-		/* Update the size of the volume this plex is attached to. */
-		if (p->vol_sc != NULL)
-			gv_update_vol_size(p->vol_sc, p->size);
+	error = bp->bio_error;
+	flags = bp->bio_cflags;
+	flags &= ~GV_BIO_MALLOC;
 
-		/*
-		 * If necessary, create bio queues, queue mutex and a worker
-		 * thread.
-		 */
-		if (p->bqueue == NULL) {
-			p->bqueue = g_malloc(sizeof(struct bio_queue_head),
-			    M_WAITOK | M_ZERO);
-			bioq_init(p->bqueue);
-		}
-		if (p->wqueue == NULL) {
-			p->wqueue = g_malloc(sizeof(struct bio_queue_head),
-			    M_WAITOK | M_ZERO);
-			bioq_init(p->wqueue);
-		}
-		if (mtx_initialized(&p->bqueue_mtx) == 0)
-			mtx_init(&p->bqueue_mtx, "gv_plex", NULL, MTX_DEF);
-		if (!(p->flags & GV_PLEX_THREAD_ACTIVE)) {
-			kproc_create(gv_plex_worker, p, NULL, 0, 0, "gv_p %s",
-			    p->name);
-			p->flags |= GV_PLEX_THREAD_ACTIVE;
-		}
+	sc = p->vinumconf;
+	KASSERT(sc != NULL, ("gv_parity_complete: NULL sc"));
+
+	/* Clean up what we allocated. */
+	if (bp->bio_cflags & GV_BIO_MALLOC)
+		g_free(bp->bio_data);
+	g_destroy_bio(bp);
 
-		return (NULL);
+	if (error == EAGAIN) {
+		G_VINUM_DEBUG(0, "parity incorrect at offset 0x%jx",
+		    (intmax_t)p->synced);
+	}
 
-	/* We need to create a new geom. */
+	/* Any error is fatal, except EAGAIN when we're rebuilding. */
+	if (error && !(error == EAGAIN && (flags & GV_BIO_PARITY))) {
+		/* Make sure we don't have the lock. */
+		g_topology_assert_not();
+		g_topology_lock();
+		gv_access(p->vol_sc->provider, -1, -1, 0);
+		g_topology_unlock();
+		G_VINUM_DEBUG(0, "parity check on %s failed at 0x%jx "
+		    "errno %d", p->name, (intmax_t)p->synced, error);
+		return;
 	} else {
-		gp = g_new_geomf(mp, "%s", p->name);
-		gp->start = gv_plex_start;
-		gp->orphan = gv_plex_orphan;
-		gp->access = gv_plex_access;
-		gp->softc = p;
-		p->geom = gp;
-
-		TAILQ_INIT(&p->packets);
-		p->bqueue = g_malloc(sizeof(struct bio_queue_head),
-		    M_WAITOK | M_ZERO);
-		bioq_init(p->bqueue);
-		p->wqueue = g_malloc(sizeof(struct bio_queue_head),
-		    M_WAITOK | M_ZERO);
-		bioq_init(p->wqueue);
-		mtx_init(&p->bqueue_mtx, "gv_plex", NULL, MTX_DEF);
-		kproc_create(gv_plex_worker, p, NULL, 0, 0, "gv_p %s",
-		    p->name);
-		p->flags |= GV_PLEX_THREAD_ACTIVE;
-
-		/* Attach a consumer to this provider. */
-		cp = g_new_consumer(gp);
-		g_attach(cp, pp);
-		s->consumer = cp;
-
-		/* Create a provider for the outside world. */
-		pp2 = g_new_providerf(gp, "gvinum/plex/%s", p->name);
-		pp2->mediasize = p->size;
-		pp2->sectorsize = pp->sectorsize;
-		p->provider = pp2;
-		g_error_provider(pp2, 0);
-		return (gp);
+		p->synced += p->stripesize;
 	}
-}
 
-static int
-gv_plex_destroy_geom(struct gctl_req *req, struct g_class *mp,
-    struct g_geom *gp)
-{
-	struct gv_plex *p;
+	if (p->synced >= p->size) {
+		/* Make sure we don't have the lock. */
+		g_topology_assert_not();
+		g_topology_lock();
+		gv_access(p->vol_sc->provider, -1, -1, 0);
+		g_topology_unlock();
+		/* We're finished. */
+		G_VINUM_DEBUG(1, "parity operation on %s finished", p->name);
+		p->synced = 0;
+		gv_post_event(sc, GV_EVENT_SAVE_CONFIG, sc, NULL, 0, 0);
+		return;
+	}
 
-	g_trace(G_T_TOPOLOGY, "gv_plex_destroy_geom: %s", gp->name);
-	g_topology_assert();
+	/* Send down next. It will determine if we need to itself. */
+	gv_parity_request(p, flags, p->synced);
+}
 
-	p = gp->softc;
+/*
+ * Handle a finished plex rebuild bio.
+ */
+void
+gv_rebuild_complete(struct gv_plex *p, struct bio *bp)
+{
+	struct gv_softc *sc;
+	struct gv_sd *s;
+	int error, flags;
+	off_t offset;
 
-	KASSERT(p != NULL, ("gv_plex_destroy_geom: null p of '%s'", gp->name));
+	error = bp->bio_error;
+	flags = bp->bio_cflags;
+	offset = bp->bio_offset;
+	flags &= ~GV_BIO_MALLOC;
+	sc = p->vinumconf;
+	KASSERT(sc != NULL, ("gv_rebuild_complete: NULL sc"));
 
-	/*
-	 * If this is a RAID5 plex, check if its worker thread is still active
-	 * and signal it to self destruct.
-	 */
-	gv_kill_plex_thread(p);
-	/* g_free(sc); */
-	g_wither_geom(gp, ENXIO);
-	return (0);
-}
+	/* Clean up what we allocated. */
+	if (bp->bio_cflags & GV_BIO_MALLOC)
+		g_free(bp->bio_data);
+	g_destroy_bio(bp);
 
-#define	VINUMPLEX_CLASS_NAME "VINUMPLEX"
+	if (error) {
+		g_topology_assert_not();
+		g_topology_lock();
+		gv_access(p->vol_sc->provider, -1, -1, 0);
+		g_topology_unlock();
+	
+		G_VINUM_DEBUG(0, "rebuild of %s failed at offset %jd errno: %d",
+		    p->name, (intmax_t)offset, error);
+		p->flags &= ~GV_PLEX_REBUILDING;
+		p->synced = 0;
+		gv_plex_flush(p); /* Flush out remaining rebuild BIOs. */
+		return;
+	}
 
-static struct g_class g_vinum_plex_class = {
-	.name = VINUMPLEX_CLASS_NAME,
-	.version = G_VERSION,
-	.taste = gv_plex_taste,
-	.destroy_geom = gv_plex_destroy_geom,
-};
+	offset += (p->stripesize * (gv_sdcount(p, 1) - 1));
+	if (offset >= p->size) {
+		/* We're finished. */
+		g_topology_assert_not();
+		g_topology_lock();
+		gv_access(p->vol_sc->provider, -1, -1, 0);
+		g_topology_unlock();
+	
+		G_VINUM_DEBUG(1, "rebuild of %s finished", p->name);
+		gv_save_config(p->vinumconf);
+		p->flags &= ~GV_PLEX_REBUILDING;
+		p->synced = 0;
+		/* Try to up all subdisks. */
+		LIST_FOREACH(s, &p->subdisks, in_plex)
+			gv_update_sd_state(s);
+		gv_post_event(sc, GV_EVENT_SAVE_CONFIG, sc, NULL, 0, 0);
+		gv_plex_flush(p); /* Flush out remaining rebuild BIOs. */
+		return;
+	}
 
-DECLARE_GEOM_CLASS(g_vinum_plex_class, g_vinum_plex);
+	/* Send down next. It will determine if we need to itself. */
+	gv_parity_request(p, flags, offset);
+}
diff --git a/sys/geom/vinum/geom_vinum_raid5.c b/sys/geom/vinum/geom_vinum_raid5.c
index abfed51..088162e 100644
--- a/sys/geom/vinum/geom_vinum_raid5.c
+++ b/sys/geom/vinum/geom_vinum_raid5.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2004 Lukas Ertl
+ * Copyright (c) 2004, 2007 Lukas Ertl
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -29,14 +29,8 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/bio.h>
-#include <sys/conf.h>
-#include <sys/errno.h>
-#include <sys/kernel.h>
-#include <sys/kthread.h>
-#include <sys/libkern.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
-#include <sys/mutex.h>
 #include <sys/systm.h>
 
 #include <geom/geom.h>
@@ -44,8 +38,103 @@ __FBSDID("$FreeBSD$");
 #include <geom/vinum/geom_vinum_raid5.h>
 #include <geom/vinum/geom_vinum.h>
 
-int	gv_raid5_offset(struct gv_plex *, off_t, off_t, off_t *, off_t *,
-	    int *, int *);
+static int		gv_raid5_offset(struct gv_plex *, off_t, off_t,
+			    off_t *, off_t *, int *, int *, int);
+static struct bio *	gv_raid5_clone_bio(struct bio *, struct gv_sd *,
+			    struct gv_raid5_packet *, caddr_t, int);
+static int	gv_raid5_request(struct gv_plex *, struct gv_raid5_packet *,
+		    struct bio *, caddr_t, off_t, off_t, int *);
+static int	gv_raid5_check(struct gv_plex *, struct gv_raid5_packet *,
+		    struct bio *, caddr_t, off_t, off_t);
+static int	gv_raid5_rebuild(struct gv_plex *, struct gv_raid5_packet *,
+		    struct bio *, caddr_t, off_t, off_t);
+
+struct gv_raid5_packet *
+gv_raid5_start(struct gv_plex *p, struct bio *bp, caddr_t addr, off_t boff,
+    off_t bcount)
+{
+	struct bio *cbp;
+	struct gv_raid5_packet *wp, *wp2;
+	struct gv_bioq *bq, *bq2;
+	int err, delay;
+
+	delay = 0;
+	wp = g_malloc(sizeof(*wp), M_WAITOK | M_ZERO);
+	wp->bio = bp;
+	wp->waiting = NULL;
+	wp->parity = NULL;
+	TAILQ_INIT(&wp->bits);
+
+	if (bp->bio_cflags & GV_BIO_REBUILD)
+		err = gv_raid5_rebuild(p, wp, bp, addr, boff, bcount);
+	else if (bp->bio_cflags & GV_BIO_CHECK)
+		err = gv_raid5_check(p, wp, bp, addr, boff, bcount);
+	else
+		err = gv_raid5_request(p, wp, bp, addr, boff, bcount, &delay);
+
+	/* Means we have a delayed request. */
+	if (delay) {
+		g_free(wp);
+		return (NULL);
+	}
+	
+	/*
+	 * Building the sub-request failed, we probably need to clean up a lot.
+	 */
+	if (err) {
+		G_VINUM_LOGREQ(0, bp, "raid5 plex request failed.");
+		TAILQ_FOREACH_SAFE(bq, &wp->bits, queue, bq2) {
+			TAILQ_REMOVE(&wp->bits, bq, queue);
+			g_free(bq);
+		}
+		if (wp->waiting != NULL) {
+			if (wp->waiting->bio_cflags & GV_BIO_MALLOC)
+				g_free(wp->waiting->bio_data);
+			g_destroy_bio(wp->waiting);
+		}
+		if (wp->parity != NULL) {
+			if (wp->parity->bio_cflags & GV_BIO_MALLOC)
+				g_free(wp->parity->bio_data);
+			g_destroy_bio(wp->parity);
+		}
+		g_free(wp);
+
+		TAILQ_FOREACH_SAFE(wp, &p->packets, list, wp2) {
+			if (wp->bio != bp)
+				continue;
+
+			TAILQ_REMOVE(&p->packets, wp, list);
+			TAILQ_FOREACH_SAFE(bq, &wp->bits, queue, bq2) {
+				TAILQ_REMOVE(&wp->bits, bq, queue);
+				g_free(bq);
+			}
+			g_free(wp);
+		}
+
+		cbp = bioq_takefirst(p->bqueue);
+		while (cbp != NULL) {
+			if (cbp->bio_cflags & GV_BIO_MALLOC)
+				g_free(cbp->bio_data);
+			g_destroy_bio(cbp);
+			cbp = bioq_takefirst(p->bqueue);
+		}
+
+		/* If internal, stop and reset state. */
+		if (bp->bio_cflags & GV_BIO_INTERNAL) {
+			if (bp->bio_cflags & GV_BIO_MALLOC)
+				g_free(cbp->bio_data);
+			g_destroy_bio(bp);
+			/* Reset flags. */
+			p->flags &= ~(GV_PLEX_SYNCING | GV_PLEX_REBUILDING |
+			    GV_PLEX_GROWING);
+			return (NULL);
+		}
+		g_io_deliver(bp, err);
+		return (NULL);
+	}
+
+	return (wp);
+}
 
 /*
  * Check if the stripe that the work packet wants is already being used by
@@ -57,7 +146,7 @@ gv_stripe_active(struct gv_plex *p, struct bio *bp)
 	struct gv_raid5_packet *wp, *owp;
 	int overlap;
 
-	wp = bp->bio_driver1;
+	wp = bp->bio_caller2;
 	if (wp->lockbase == -1)
 		return (0);
 
@@ -80,20 +169,20 @@ gv_stripe_active(struct gv_plex *p, struct bio *bp)
 	return (overlap);
 }
 
-int
-gv_check_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp,
+static int
+gv_raid5_check(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp,
     caddr_t addr, off_t boff, off_t bcount)
 {
 	struct gv_sd *parity, *s;
 	struct gv_bioq *bq;
-	struct bio *cbp, *pbp;
+	struct bio *cbp;
 	int i, psdno;
 	off_t real_len, real_off;
 
 	if (p == NULL || LIST_EMPTY(&p->subdisks))
 		return (ENXIO);
 
-	gv_raid5_offset(p, boff, bcount, &real_off, &real_len, NULL, &psdno);
+	gv_raid5_offset(p, boff, bcount, &real_off, &real_len, NULL, &psdno, 1);
 
 	/* Find the right subdisk. */
 	parity = NULL;
@@ -122,20 +211,16 @@ gv_check_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp,
 		/* Skip the parity subdisk. */
 		if (s == parity)
 			continue;
+		/* Skip growing subdisks. */
+		if (s->flags & GV_SD_GROW)
+			continue;
 
-		cbp = g_clone_bio(bp);
+		cbp = gv_raid5_clone_bio(bp, s, wp, NULL, 1);
 		if (cbp == NULL)
 			return (ENOMEM);
 		cbp->bio_cmd = BIO_READ;
-		cbp->bio_data = g_malloc(real_len, M_WAITOK);
-		cbp->bio_cflags |= GV_BIO_MALLOC;
-		cbp->bio_offset = real_off;
-		cbp->bio_length = real_len;
-		cbp->bio_done = gv_plex_done;
-		cbp->bio_caller2 = s->consumer;
-		cbp->bio_driver1 = wp;
 
-		GV_ENQUEUE(bp, cbp, pbp);
+		bioq_insert_tail(p->bqueue, cbp);
 
 		bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO);
 		bq->bp = cbp;
@@ -143,51 +228,38 @@ gv_check_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp,
 	}
 
 	/* Read the parity data. */
-	cbp = g_clone_bio(bp);
+	cbp = gv_raid5_clone_bio(bp, parity, wp, NULL, 1);
 	if (cbp == NULL)
 		return (ENOMEM);
 	cbp->bio_cmd = BIO_READ;
-	cbp->bio_data = g_malloc(real_len, M_WAITOK | M_ZERO);
-	cbp->bio_cflags |= GV_BIO_MALLOC;
-	cbp->bio_offset = real_off;
-	cbp->bio_length = real_len;
-	cbp->bio_done = gv_plex_done;
-	cbp->bio_caller2 = parity->consumer;
-	cbp->bio_driver1 = wp;
 	wp->waiting = cbp;
 
 	/*
 	 * In case we want to rebuild the parity, create an extra BIO to write
 	 * it out.  It also acts as buffer for the XOR operations.
 	 */
-	cbp = g_clone_bio(bp);
+	cbp = gv_raid5_clone_bio(bp, parity, wp, addr, 1);
 	if (cbp == NULL)
 		return (ENOMEM);
-	cbp->bio_data = addr;
-	cbp->bio_offset = real_off;
-	cbp->bio_length = real_len;
-	cbp->bio_done = gv_plex_done;
-	cbp->bio_caller2 = parity->consumer;
-	cbp->bio_driver1 = wp;
 	wp->parity = cbp;
 
 	return (0);
 }
 
 /* Rebuild a degraded RAID5 plex. */
-int
-gv_rebuild_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp,
+static int
+gv_raid5_rebuild(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp,
     caddr_t addr, off_t boff, off_t bcount)
 {
 	struct gv_sd *broken, *s;
 	struct gv_bioq *bq;
-	struct bio *cbp, *pbp;
+	struct bio *cbp;
 	off_t real_len, real_off;
 
 	if (p == NULL || LIST_EMPTY(&p->subdisks))
 		return (ENXIO);
 
-	gv_raid5_offset(p, boff, bcount, &real_off, &real_len, NULL, NULL);
+	gv_raid5_offset(p, boff, bcount, &real_off, &real_len, NULL, NULL, 1);
 
 	/* Find the right subdisk. */
 	broken = NULL;
@@ -210,6 +282,8 @@ gv_rebuild_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp,
 
 		G_VINUM_DEBUG(1, "sd %s is reviving", broken->name);
 		gv_set_sd_state(broken, GV_SD_REVIVING, GV_SETSTATE_FORCE);
+		/* Set this bit now, but should be set at end. */
+		broken->flags |= GV_SD_CANGOUP;
 		break;
 
 	case GV_SD_REVIVING:
@@ -232,19 +306,16 @@ gv_rebuild_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp,
 		if (s == broken)
 			continue;
 
-		cbp = g_clone_bio(bp);
+		/* Skip growing subdisks. */
+		if (s->flags & GV_SD_GROW)
+			continue;
+
+		cbp = gv_raid5_clone_bio(bp, s, wp, NULL, 1);
 		if (cbp == NULL)
 			return (ENOMEM);
 		cbp->bio_cmd = BIO_READ;
-		cbp->bio_data = g_malloc(real_len, M_WAITOK);
-		cbp->bio_cflags |= GV_BIO_MALLOC;
-		cbp->bio_offset = real_off;
-		cbp->bio_length = real_len;
-		cbp->bio_done = gv_plex_done;
-		cbp->bio_caller2 = s->consumer;
-		cbp->bio_driver1 = wp;
 
-		GV_ENQUEUE(bp, cbp, pbp);
+		bioq_insert_tail(p->bqueue, cbp);
 
 		bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO);
 		bq->bp = cbp;
@@ -252,34 +323,28 @@ gv_rebuild_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp,
 	}
 
 	/* Write the parity data. */
-	cbp = g_clone_bio(bp);
+	cbp = gv_raid5_clone_bio(bp, broken, wp, NULL, 1);
 	if (cbp == NULL)
 		return (ENOMEM);
-	cbp->bio_data = g_malloc(real_len, M_WAITOK | M_ZERO);
-	cbp->bio_cflags |= GV_BIO_MALLOC;
-	cbp->bio_offset = real_off;
-	cbp->bio_length = real_len;
-	cbp->bio_done = gv_plex_done;
-	cbp->bio_caller2 = broken->consumer;
-	cbp->bio_driver1 = wp;
 	cbp->bio_cflags |= GV_BIO_REBUILD;
 	wp->parity = cbp;
 
 	p->synced = boff;
 
+	/* Post notification that we're finished. */
 	return (0);
 }
 
 /* Build a request group to perform (part of) a RAID5 request. */
-int
-gv_build_raid5_req(struct gv_plex *p, struct gv_raid5_packet *wp,
-    struct bio *bp, caddr_t addr, off_t boff, off_t bcount)
+static int
+gv_raid5_request(struct gv_plex *p, struct gv_raid5_packet *wp,
+    struct bio *bp, caddr_t addr, off_t boff, off_t bcount, int *delay)
 {
 	struct g_geom *gp;
 	struct gv_sd *broken, *original, *parity, *s;
 	struct gv_bioq *bq;
-	struct bio *cbp, *pbp;
-	int i, psdno, sdno, type;
+	struct bio *cbp;
+	int i, psdno, sdno, type, grow;
 	off_t real_len, real_off;
 
 	gp = bp->bio_to->geom;
@@ -295,7 +360,24 @@ gv_build_raid5_req(struct gv_plex *p, struct gv_raid5_packet *wp,
 	type = REQ_TYPE_NORMAL;
 	original = parity = broken = NULL;
 
-	gv_raid5_offset(p, boff, bcount, &real_off, &real_len, &sdno, &psdno);
+	/* XXX: The resize won't crash with rebuild or sync, but we should still
+	 * be aware of it. Also this should perhaps be done on rebuild/check as
+	 * well?
+	 */
+	/* If we're over, we must use the old. */ 
+	if (boff >= p->synced) {
+		grow = 1;
+	/* Or if over the resized offset, we use all drives. */
+	} else if (boff + bcount <= p->synced) {
+		grow = 0;
+	/* Else, we're in the middle, and must wait a bit. */
+	} else {
+		bioq_disksort(p->rqueue, bp);
+		*delay = 1;
+		return (0);
+	}
+	gv_raid5_offset(p, boff, bcount, &real_off, &real_len,
+	    &sdno, &psdno, grow);
 
 	/* Find the right subdisks. */
 	i = 0;
@@ -315,8 +397,13 @@ gv_build_raid5_req(struct gv_plex *p, struct gv_raid5_packet *wp,
 	/* Our data stripe is missing. */
 	if (original->state != GV_SD_UP)
 		type = REQ_TYPE_DEGRADED;
+
+	/* If synchronizing request, just write it if disks are stale. */
+	if (original->state == GV_SD_STALE && parity->state == GV_SD_STALE &&
+	    bp->bio_cflags & GV_BIO_SYNCREQ && bp->bio_cmd == BIO_WRITE) {
+		type = REQ_TYPE_NORMAL;
 	/* Our parity stripe is missing. */
-	if (parity->state != GV_SD_UP) {
+	} else if (parity->state != GV_SD_UP) {
 		/* We cannot take another failure if we're already degraded. */
 		if (type != REQ_TYPE_NORMAL)
 			return (ENXIO);
@@ -330,9 +417,15 @@ gv_build_raid5_req(struct gv_plex *p, struct gv_raid5_packet *wp,
 
 	KASSERT(wp->length >= 0, ("gv_build_raid5_request: wp->length < 0"));
 
-	if ((p->flags & GV_PLEX_SYNCING) && (boff + real_len < p->synced))
+	if ((p->flags & GV_PLEX_REBUILDING) && (boff + real_len < p->synced))
 		type = REQ_TYPE_NORMAL;
 
+	if ((p->flags & GV_PLEX_REBUILDING) && (boff + real_len >= p->synced)) {
+		bioq_disksort(p->rqueue, bp);
+		*delay = 1;
+		return (0);
+	}
+
 	switch (bp->bio_cmd) {
 	case BIO_READ:
 		/*
@@ -346,18 +439,14 @@ gv_build_raid5_req(struct gv_plex *p, struct gv_raid5_packet *wp,
 				/* Skip the broken subdisk. */
 				if (s == broken)
 					continue;
-				cbp = g_clone_bio(bp);
+				/* Skip growing if within offset. */
+				if (grow && s->flags & GV_SD_GROW)
+					continue;
+				cbp = gv_raid5_clone_bio(bp, s, wp, NULL, 1);
 				if (cbp == NULL)
 					return (ENOMEM);
-				cbp->bio_data = g_malloc(real_len, M_WAITOK);
-				cbp->bio_cflags |= GV_BIO_MALLOC;
-				cbp->bio_offset = real_off;
-				cbp->bio_length = real_len;
-				cbp->bio_done = gv_plex_done;
-				cbp->bio_caller2 = s->consumer;
-				cbp->bio_driver1 = wp;
 
-				GV_ENQUEUE(bp, cbp, pbp);
+				bioq_insert_tail(p->bqueue, cbp);
 
 				bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO);
 				bq->bp = cbp;
@@ -366,16 +455,11 @@ gv_build_raid5_req(struct gv_plex *p, struct gv_raid5_packet *wp,
 
 		/* A normal read can be fulfilled with the original subdisk. */
 		} else {
-			cbp = g_clone_bio(bp);
+			cbp = gv_raid5_clone_bio(bp, original, wp, addr, 0);
 			if (cbp == NULL)
 				return (ENOMEM);
-			cbp->bio_offset = real_off;
-			cbp->bio_length = real_len;
-			cbp->bio_data = addr;
-			cbp->bio_done = g_std_done;
-			cbp->bio_caller2 = original->consumer;
 
-			GV_ENQUEUE(bp, cbp, pbp);
+			bioq_insert_tail(p->bqueue, cbp);
 		}
 		wp->lockbase = -1;
 
@@ -394,20 +478,16 @@ gv_build_raid5_req(struct gv_plex *p, struct gv_raid5_packet *wp,
 				/* Skip the broken and the parity subdisk. */
 				if ((s == broken) || (s == parity))
 					continue;
+				/* Skip growing if within offset. */
+				if (grow && s->flags & GV_SD_GROW)
+					continue;
 
-				cbp = g_clone_bio(bp);
+				cbp = gv_raid5_clone_bio(bp, s, wp, NULL, 1);
 				if (cbp == NULL)
 					return (ENOMEM);
 				cbp->bio_cmd = BIO_READ;
-				cbp->bio_data = g_malloc(real_len, M_WAITOK);
-				cbp->bio_cflags |= GV_BIO_MALLOC;
-				cbp->bio_offset = real_off;
-				cbp->bio_length = real_len;
-				cbp->bio_done = gv_plex_done;
-				cbp->bio_caller2 = s->consumer;
-				cbp->bio_driver1 = wp;
 
-				GV_ENQUEUE(bp, cbp, pbp);
+				bioq_insert_tail(p->bqueue, cbp);
 
 				bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO);
 				bq->bp = cbp;
@@ -415,34 +495,21 @@ gv_build_raid5_req(struct gv_plex *p, struct gv_raid5_packet *wp,
 			}
 
 			/* Write the parity data. */
-			cbp = g_clone_bio(bp);
+			cbp = gv_raid5_clone_bio(bp, parity, wp, NULL, 1);
 			if (cbp == NULL)
 				return (ENOMEM);
-			cbp->bio_data = g_malloc(real_len, M_WAITOK);
-			cbp->bio_cflags |= GV_BIO_MALLOC;
-			bcopy(addr, cbp->bio_data, real_len);
-			cbp->bio_offset = real_off;
-			cbp->bio_length = real_len;
-			cbp->bio_done = gv_plex_done;
-			cbp->bio_caller2 = parity->consumer;
-			cbp->bio_driver1 = wp;
+			bcopy(addr, cbp->bio_data, wp->length);
 			wp->parity = cbp;
 
 		/*
 		 * When the parity stripe is missing we just write out the data.
 		 */
 		} else if (type == REQ_TYPE_NOPARITY) {
-			cbp = g_clone_bio(bp);
+			cbp = gv_raid5_clone_bio(bp, original, wp, addr, 1);
 			if (cbp == NULL)
 				return (ENOMEM);
-			cbp->bio_offset = real_off;
-			cbp->bio_length = real_len;
-			cbp->bio_data = addr;
-			cbp->bio_done = gv_plex_done;
-			cbp->bio_caller2 = original->consumer;
-			cbp->bio_driver1 = wp;
 
-			GV_ENQUEUE(bp, cbp, pbp);
+			bioq_insert_tail(p->bqueue, cbp);
 
 			bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO);
 			bq->bp = cbp;
@@ -455,54 +522,33 @@ gv_build_raid5_req(struct gv_plex *p, struct gv_raid5_packet *wp,
 		 */
 		} else {
 			/* Read old parity. */
-			cbp = g_clone_bio(bp);
+			cbp = gv_raid5_clone_bio(bp, parity, wp, NULL, 1);
 			if (cbp == NULL)
 				return (ENOMEM);
 			cbp->bio_cmd = BIO_READ;
-			cbp->bio_data = g_malloc(real_len, M_WAITOK);
-			cbp->bio_cflags |= GV_BIO_MALLOC;
-			cbp->bio_offset = real_off;
-			cbp->bio_length = real_len;
-			cbp->bio_done = gv_plex_done;
-			cbp->bio_caller2 = parity->consumer;
-			cbp->bio_driver1 = wp;
 
-			GV_ENQUEUE(bp, cbp, pbp);
+			bioq_insert_tail(p->bqueue, cbp);
 
 			bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO);
 			bq->bp = cbp;
 			TAILQ_INSERT_TAIL(&wp->bits, bq, queue);
 
 			/* Read old data. */
-			cbp = g_clone_bio(bp);
+			cbp = gv_raid5_clone_bio(bp, original, wp, NULL, 1);
 			if (cbp == NULL)
 				return (ENOMEM);
 			cbp->bio_cmd = BIO_READ;
-			cbp->bio_data = g_malloc(real_len, M_WAITOK);
-			cbp->bio_cflags |= GV_BIO_MALLOC;
-			cbp->bio_offset = real_off;
-			cbp->bio_length = real_len;
-			cbp->bio_done = gv_plex_done;
-			cbp->bio_caller2 = original->consumer;
-			cbp->bio_driver1 = wp;
 
-			GV_ENQUEUE(bp, cbp, pbp);
+			bioq_insert_tail(p->bqueue, cbp);
 
 			bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO);
 			bq->bp = cbp;
 			TAILQ_INSERT_TAIL(&wp->bits, bq, queue);
 
 			/* Write new data. */
-			cbp = g_clone_bio(bp);
+			cbp = gv_raid5_clone_bio(bp, original, wp, addr, 1);
 			if (cbp == NULL)
 				return (ENOMEM);
-			cbp->bio_data = addr;
-			cbp->bio_offset = real_off;
-			cbp->bio_length = real_len;
-			cbp->bio_done = gv_plex_done;
-			cbp->bio_caller2 = original->consumer;
-
-			cbp->bio_driver1 = wp;
 
 			/*
 			 * We must not write the new data until the old data
@@ -512,16 +558,9 @@ gv_build_raid5_req(struct gv_plex *p, struct gv_raid5_packet *wp,
 			wp->waiting = cbp;
 
 			/* The final bio for the parity. */
-			cbp = g_clone_bio(bp);
+			cbp = gv_raid5_clone_bio(bp, parity, wp, NULL, 1);
 			if (cbp == NULL)
 				return (ENOMEM);
-			cbp->bio_data = g_malloc(real_len, M_WAITOK | M_ZERO);
-			cbp->bio_cflags |= GV_BIO_MALLOC;
-			cbp->bio_offset = real_off;
-			cbp->bio_length = real_len;
-			cbp->bio_done = gv_plex_done;
-			cbp->bio_caller2 = parity->consumer;
-			cbp->bio_driver1 = wp;
 
 			/* Remember that this is the BIO for the parity data. */
 			wp->parity = cbp;
@@ -535,21 +574,36 @@ gv_build_raid5_req(struct gv_plex *p, struct gv_raid5_packet *wp,
 	return (0);
 }
 
-/* Calculate the offsets in the various subdisks for a RAID5 request. */
-int
+/*
+ * Calculate the offsets in the various subdisks for a RAID5 request. Also take
+ * care of new subdisks in an expanded RAID5 array. 
+ * XXX: This assumes that the new subdisks are inserted after the others (which
+ * is okay as long as plex_offset is larger). If subdisks are inserted into the
+ * plexlist before, we get problems.
+ */
+static int
 gv_raid5_offset(struct gv_plex *p, off_t boff, off_t bcount, off_t *real_off,
-    off_t *real_len, int *sdno, int *psdno)
+    off_t *real_len, int *sdno, int *psdno, int growing)
 {
-	int sd, psd;
+	struct gv_sd *s;
+	int sd, psd, sdcount;
 	off_t len_left, stripeend, stripeoff, stripestart;
 
+	sdcount = p->sdcount;
+	if (growing) {
+		LIST_FOREACH(s, &p->subdisks, in_plex) {
+			if (s->flags & GV_SD_GROW)
+				sdcount--;
+		}
+	}
+
 	/* The number of the subdisk containing the parity stripe. */
-	psd = p->sdcount - 1 - ( boff / (p->stripesize * (p->sdcount - 1))) %
-	    p->sdcount;
+	psd = sdcount - 1 - ( boff / (p->stripesize * (sdcount - 1))) %
+	    sdcount;
 	KASSERT(psdno >= 0, ("gv_raid5_offset: psdno < 0"));
 
 	/* Offset of the start address from the start of the stripe. */
-	stripeoff = boff % (p->stripesize * (p->sdcount - 1));
+	stripeoff = boff % (p->stripesize * (sdcount - 1));
 	KASSERT(stripeoff >= 0, ("gv_raid5_offset: stripeoff < 0"));
 
 	/* The number of the subdisk where the stripe resides. */
@@ -561,7 +615,7 @@ gv_raid5_offset(struct gv_plex *p, off_t boff, off_t bcount, off_t *real_off,
 		sd++;
 
 	/* The offset of the stripe on this subdisk. */
-	stripestart = (boff - stripeoff) / (p->sdcount - 1);
+	stripestart = (boff - stripeoff) / (sdcount - 1);
 	KASSERT(stripestart >= 0, ("gv_raid5_offset: stripestart < 0"));
 
 	stripeoff %= p->stripesize;
@@ -582,3 +636,27 @@ gv_raid5_offset(struct gv_plex *p, off_t boff, off_t bcount, off_t *real_off,
 
 	return (0);
 }
+
+static struct bio *
+gv_raid5_clone_bio(struct bio *bp, struct gv_sd *s, struct gv_raid5_packet *wp,
+    caddr_t addr, int use_wp)
+{
+	struct bio *cbp;
+
+	cbp = g_clone_bio(bp);
+	if (cbp == NULL)
+		return (NULL);
+	if (addr == NULL) {
+		cbp->bio_data = g_malloc(wp->length, M_WAITOK | M_ZERO);
+		cbp->bio_cflags |= GV_BIO_MALLOC;
+	} else
+		cbp->bio_data = addr;
+	cbp->bio_offset = wp->lockbase + s->drive_offset;
+	cbp->bio_length = wp->length;
+	cbp->bio_done = gv_done;
+	cbp->bio_caller1 = s;
+	if (use_wp)
+		cbp->bio_caller2 = wp;
+
+	return (cbp);
+}
diff --git a/sys/geom/vinum/geom_vinum_raid5.h b/sys/geom/vinum/geom_vinum_raid5.h
index 804920e..d7d55b2 100644
--- a/sys/geom/vinum/geom_vinum_raid5.h
+++ b/sys/geom/vinum/geom_vinum_raid5.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2004 Lukas Ertl
+ * Copyright (c) 2004, 2007 Lukas Ertl
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -35,26 +35,10 @@
  * transaction (read or write).
  */
 
-#define	GV_ENQUEUE(bp, cbp, pbp)				\
-	do { 							\
-		if (bp->bio_driver1 == NULL) {			\
-			bp->bio_driver1 = cbp;			\
-		} else {					\
-			pbp = bp->bio_driver1;			\
-			while (pbp->bio_caller1 != NULL)	\
-				pbp = pbp->bio_caller1;		\
-			pbp->bio_caller1 = cbp;			\
-		}						\
-	} while (0)
-
 struct gv_raid5_packet {
 	caddr_t	data;		/* Data buffer of this sub-request- */
 	off_t	length;		/* Size of data buffer. */
 	off_t	lockbase;	/* Deny access to our plex offset. */
-	off_t	offset;		/* The drive offset of the subdisk. */
-	int	bufmalloc;	/* Flag if data buffer was malloced. */
-	int	active;		/* Count of active subrequests. */
-	int	rqcount;	/* Count of subrequests. */
 
 	struct bio	*bio;	/* Pointer to the original bio. */
 	struct bio	*parity;  /* The bio containing the parity data. */
@@ -64,14 +48,8 @@ struct gv_raid5_packet {
 	TAILQ_ENTRY(gv_raid5_packet)	list; /* Entry in plex's packet list. */
 };
 
+struct gv_raid5_packet * gv_raid5_start(struct gv_plex *, struct bio *,
+		caddr_t, off_t, off_t);
 int	gv_stripe_active(struct gv_plex *, struct bio *);
-int	gv_build_raid5_req(struct gv_plex *, struct gv_raid5_packet *,
-	    struct bio *, caddr_t, off_t, off_t);
-int	gv_check_raid5(struct gv_plex *, struct gv_raid5_packet *,
-	    struct bio *, caddr_t, off_t, off_t);
-int	gv_rebuild_raid5(struct gv_plex *, struct gv_raid5_packet *,
-	    struct bio *, caddr_t, off_t, off_t);
-void	gv_raid5_worker(void *);
-void	gv_plex_done(struct bio *);
 
 #endif /* !_GEOM_VINUM_RAID5_H_ */
diff --git a/sys/geom/vinum/geom_vinum_rename.c b/sys/geom/vinum/geom_vinum_rename.c
index ee5fc9c..53c173f 100644
--- a/sys/geom/vinum/geom_vinum_rename.c
+++ b/sys/geom/vinum/geom_vinum_rename.c
@@ -34,22 +34,11 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/libkern.h>
-#include <sys/kernel.h>
 #include <sys/malloc.h>
 
 #include <geom/geom.h>
 #include <geom/vinum/geom_vinum_var.h>
 #include <geom/vinum/geom_vinum.h>
-#include <geom/vinum/geom_vinum_share.h>
-
-static int	gv_rename_drive(struct gv_softc *, struct gctl_req *,
-		    struct gv_drive *, char *, int);
-static int	gv_rename_plex(struct gv_softc *, struct gctl_req *,
-		    struct gv_plex *, char *, int);
-static int	gv_rename_sd(struct gv_softc *, struct gctl_req *,
-		    struct gv_sd *, char *, int);
-static int	gv_rename_vol(struct gv_softc *, struct gctl_req *,
-		    struct gv_volume *, char *, int);
 
 void
 gv_rename(struct g_geom *gp, struct gctl_req *req)
@@ -59,8 +48,8 @@ gv_rename(struct g_geom *gp, struct gctl_req *req)
 	struct gv_plex *p;
 	struct gv_sd *s;
 	struct gv_drive *d;
-	char *newname, *object;
-	int err, *flags, type;
+	char *newname, *object, *name;
+	int *flags, type;
 
 	sc = gp->softc;
 
@@ -90,9 +79,9 @@ gv_rename(struct g_geom *gp, struct gctl_req *req)
 			gctl_error(req, "unknown volume '%s'", object);
 			return;
 		}
-		err = gv_rename_vol(sc, req, v, newname, *flags);
-		if (err)
-			return;
+		name = g_malloc(GV_MAXVOLNAME, M_WAITOK | M_ZERO);
+		strlcpy(name, newname, GV_MAXVOLNAME);
+		gv_post_event(sc, GV_EVENT_RENAME_VOL, v, name, *flags, 0);
 		break;
 	case GV_TYPE_PLEX:
 		p = gv_find_plex(sc, object);
@@ -100,9 +89,9 @@ gv_rename(struct g_geom *gp, struct gctl_req *req)
 			gctl_error(req, "unknown plex '%s'", object);
 			return;
 		}
-		err = gv_rename_plex(sc, req, p, newname, *flags);
-		if (err)
-			return;
+		name = g_malloc(GV_MAXPLEXNAME, M_WAITOK | M_ZERO);
+		strlcpy(name, newname, GV_MAXPLEXNAME);
+		gv_post_event(sc, GV_EVENT_RENAME_PLEX, p, name, *flags, 0);
 		break;
 	case GV_TYPE_SD:
 		s = gv_find_sd(sc, object);
@@ -110,9 +99,9 @@ gv_rename(struct g_geom *gp, struct gctl_req *req)
 			gctl_error(req, "unknown subdisk '%s'", object);
 			return;
 		}
-		err = gv_rename_sd(sc, req, s, newname, *flags);
-		if (err)
-			return;
+		name = g_malloc(GV_MAXSDNAME, M_WAITOK | M_ZERO);
+		strlcpy(name, newname, GV_MAXSDNAME);
+		gv_post_event(sc, GV_EVENT_RENAME_SD, s, name, *flags, 0);
 		break;
 	case GV_TYPE_DRIVE:
 		d = gv_find_drive(sc, object);
@@ -120,122 +109,88 @@ gv_rename(struct g_geom *gp, struct gctl_req *req)
 			gctl_error(req, "unknown drive '%s'", object);
 			return;
 		}
-		err = gv_rename_drive(sc, req, d, newname, *flags);
-		if (err)
-			return;
+		name = g_malloc(GV_MAXDRIVENAME, M_WAITOK | M_ZERO);
+		strlcpy(name, newname, GV_MAXDRIVENAME);
+		gv_post_event(sc, GV_EVENT_RENAME_DRIVE, d, name, *flags, 0);
 		break;
 	default:
 		gctl_error(req, "unknown object '%s'", object);
 		return;
 	}
-
-	gv_save_config_all(sc);
 }
 
-static int
-gv_rename_drive(struct gv_softc *sc, struct gctl_req *req, struct gv_drive *d, char *newname, int flags)
+int
+gv_rename_drive(struct gv_softc *sc, struct gv_drive *d, char *newname,
+    int flags)
 {
 	struct gv_sd *s;
 
 	g_topology_assert();
 	KASSERT(d != NULL, ("gv_rename_drive: NULL d"));
 
-	if (gv_object_type(sc, newname) != -1) {
-		gctl_error(req, "drive name '%s' already in use", newname);
-		return (-1);
+	if (gv_object_type(sc, newname) != GV_ERR_NOTFOUND) {
+		G_VINUM_DEBUG(1, "drive name '%s' already in use", newname);
+		return (GV_ERR_NAMETAKEN);
 	}
 
-	strncpy(d->name, newname, GV_MAXDRIVENAME);
-	strncpy(d->hdr->label.name, newname, GV_MAXDRIVENAME);
-
-	/* XXX can we rename providers here? */
+	strlcpy(d->name, newname, sizeof(d->name));
+	if (d->hdr != NULL)
+		strlcpy(d->hdr->label.name, newname, sizeof(d->hdr->label.name));
 
 	LIST_FOREACH(s, &d->subdisks, from_drive)
-		strncpy(s->drive, d->name, GV_MAXDRIVENAME);
+		strlcpy(s->drive, d->name, sizeof(s->drive));
 
 	return (0);
 }
 
-static int	
-gv_rename_plex(struct gv_softc *sc, struct gctl_req *req, struct gv_plex *p, char *newname, int flags)
+int
+gv_rename_plex(struct gv_softc *sc, struct gv_plex *p, char *newname, int flags)
 {
+	char newsd[GV_MAXSDNAME];
 	struct gv_sd *s;
-	char *plexnum, *plexnump, *oldplex, *oldplexp;
-	char *newsd, *oldsd, *oldsdp;
+	char *ptr;
 	int err;
 
 	g_topology_assert();
 	KASSERT(p != NULL, ("gv_rename_plex: NULL p"));
 
-	err = 0;
-
-	if (gv_object_type(sc, newname) != -1) {
-		gctl_error(req, "plex name '%s' already in use", newname);
-		return (-1);
+	if (gv_object_type(sc, newname) != GV_ERR_NOTFOUND) {
+		G_VINUM_DEBUG(1, "plex name '%s' already in use", newname);
+		return (GV_ERR_NAMETAKEN);
 	}
 
-	/* Needed for sanity checking. */
-	plexnum = g_malloc(GV_MAXPLEXNAME, M_WAITOK | M_ZERO);
-	strncpy(plexnum, newname, GV_MAXPLEXNAME);
-	plexnump = plexnum;
-
-	oldplex = g_malloc(GV_MAXPLEXNAME, M_WAITOK | M_ZERO);
-	strncpy(oldplex, p->name, GV_MAXPLEXNAME);
-	oldplexp = oldplex;
-
 	/*
 	 * Locate the plex number part of the plex names.
-	 *
-	 * XXX: can we be sure that the current plex name has the format
-	 * 'foo.pX'?
+	 * XXX: might be a good idea to sanitize input a bit more
 	 */
-	strsep(&oldplexp, ".");
-	strsep(&plexnump, ".");
-	if (plexnump == NULL || *plexnump == '\0') {
-		gctl_error(req, "proposed plex name '%s' is not a valid plex "
+	ptr = strrchr(newname, '.');
+	if (ptr == NULL) {
+		G_VINUM_DEBUG(0, "proposed plex name '%s' is not a valid plex "
 		    "name", newname);
-		err = -1;
-		goto failure;
+		return (GV_ERR_INVNAME);
 	}
-	if (strcmp(oldplexp, plexnump)) {
-		gctl_error(req, "current and proposed plex numbers (%s, %s) "
-		    "do not match", plexnump, oldplexp);
-		err = -1;
-		goto failure;
-	}
-
-	strncpy(p->name, newname, GV_MAXPLEXNAME);
 
-	/* XXX can we rename providers here? */
+	strlcpy(p->name, newname, sizeof(p->name));
 
 	/* Fix up references and potentially rename subdisks. */
 	LIST_FOREACH(s, &p->subdisks, in_plex) {
-		strncpy(s->plex, p->name, GV_MAXPLEXNAME);
+		strlcpy(s->plex, p->name, sizeof(s->plex));
 		if (flags && GV_FLAG_R) {
-			newsd = g_malloc(GV_MAXSDNAME, M_WAITOK | M_ZERO);
-			oldsd = g_malloc(GV_MAXSDNAME, M_WAITOK | M_ZERO);
-			oldsdp = oldsd;
-			strncpy(oldsd, s->name, GV_MAXSDNAME);
 			/*
-			 * XXX: can we be sure that the current sd name has the
-			 * format 'foo.pX.sY'?
+			 * Look for the two last dots in the string, and assume
+			 * that the old value was ok.
 			 */
-			strsep(&oldsdp, ".");
-			strsep(&oldsdp, ".");
-			snprintf(newsd, GV_MAXSDNAME, "%s.%s", p->name, oldsdp);
-			err = gv_rename_sd(sc, req, s, newsd, flags);
-			g_free(newsd);
-			g_free(oldsd);
+			ptr = strrchr(s->name, '.');
+			if (ptr == NULL)
+				return (GV_ERR_INVNAME);
+			ptr++;
+			snprintf(newsd, sizeof(newsd), "%s.%s", p->name, ptr);
+			err = gv_rename_sd(sc, s, newsd, flags);
 			if (err)
-				goto failure;
+				return (err);
 		}
 	}
-
-failure:
-	g_free(plexnum);
-	g_free(oldplex);
-
-	return (err);
+	return (0);
 }
 
 /*
@@ -243,106 +198,64 @@ failure:
  * since there are no structures below a subdisk.  Similarly, we don't have to
  * clean up any references elsewhere to the subdisk's name.
  */
-static int
-gv_rename_sd(struct gv_softc *sc, struct gctl_req *req, struct gv_sd *s, char * newname, int flags)
+int
+gv_rename_sd(struct gv_softc *sc, struct gv_sd *s, char *newname, int flags)
 {
-	char *new, *newp, *old, *oldp;
-	int err;
+	char *dot1, *dot2;
 
 	g_topology_assert();
 	KASSERT(s != NULL, ("gv_rename_sd: NULL s"));
 
-	err = 0;
-
-	if (gv_object_type(sc, newname) != -1) {
-		gctl_error(req, "subdisk name %s already in use", newname);
-		return (-1);
+	if (gv_object_type(sc, newname) != GV_ERR_NOTFOUND) {
+		G_VINUM_DEBUG(1, "subdisk name %s already in use", newname);
+		return (GV_ERR_NAMETAKEN);
 	}
 
-	/* Needed for sanity checking. */
-	new = g_malloc(GV_MAXSDNAME, M_WAITOK | M_ZERO);
-	strncpy(new, newname, GV_MAXSDNAME);
-	newp = new;
-
-	old = g_malloc(GV_MAXSDNAME, M_WAITOK | M_ZERO);
-	strncpy(old, s->name, GV_MAXSDNAME);
-	oldp = old;
-
-	/*
-	 * Locate the sd number part of the sd names.
-	 *
-	 * XXX: can we be sure that the current sd name has the format
-	 * 'foo.pX.sY'?
-	 */
-	strsep(&oldp, ".");
-	strsep(&oldp, ".");
-	strsep(&newp, ".");
-	if (newp == NULL || *newp == '\0') {
-		gctl_error(req, "proposed sd name '%s' is not a valid sd name",
+	/* Locate the sd number part of the sd names. */
+	dot1 = strchr(newname, '.');
+	if (dot1 == NULL || (dot2 = strchr(dot1 +  1, '.')) == NULL) {
+		G_VINUM_DEBUG(0, "proposed sd name '%s' is not a valid sd name",
 		    newname);
-		err = -1;
-		goto fail;
+		return (GV_ERR_INVNAME);
 	}
-	strsep(&newp, ".");
-	if (newp == NULL || *newp == '\0') {
-		gctl_error(req, "proposed sd name '%s' is not a valid sd name",
-		    newname);
-		err = -1;
-		goto fail;
-	}
-	if (strcmp(newp, oldp)) {
-		gctl_error(req, "current and proposed sd numbers (%s, %s) do "
-		    "not match", oldp, newp);
-		err = -1;
-		goto fail;
-	}
-
-	strncpy(s->name, newname, GV_MAXSDNAME);
-
-	/* XXX: can we rename providers here? */
-
-fail:
-	g_free(new);
-	g_free(old);
-
-	return (err);
+	strlcpy(s->name, newname, sizeof(s->name));
+	return (0);
 }
 
-static int
-gv_rename_vol(struct gv_softc *sc, struct gctl_req *req, struct gv_volume *v, char *newname, int flags)
+int
+gv_rename_vol(struct gv_softc *sc, struct gv_volume *v, char *newname,
+    int flags)
 {
+	struct g_provider *pp;
 	struct gv_plex *p;
-	char *new, *old, *oldp;
+	char newplex[GV_MAXPLEXNAME], *ptr;
 	int err;
 
 	g_topology_assert();
 	KASSERT(v != NULL, ("gv_rename_vol: NULL v"));
+	pp = v->provider;
+	KASSERT(pp != NULL, ("gv_rename_vol: NULL pp"));
 
-	if (gv_object_type(sc, newname) != -1) {
-		gctl_error(req, "volume name %s already in use", newname);
-		return (-1);
+	if (gv_object_type(sc, newname) != GV_ERR_NOTFOUND) {
+		G_VINUM_DEBUG(1, "volume name %s already in use", newname);
+		return (GV_ERR_NAMETAKEN);
 	}
 
 	/* Rename the volume. */
-	strncpy(v->name, newname, GV_MAXVOLNAME);
+	strlcpy(v->name, newname, sizeof(v->name));
 
 	/* Fix up references and potentially rename plexes. */
 	LIST_FOREACH(p, &v->plexes, in_volume) {
-		strncpy(p->volume, v->name, GV_MAXVOLNAME);
+		strlcpy(p->volume, v->name, sizeof(p->volume));
 		if (flags && GV_FLAG_R) {
-			new = g_malloc(GV_MAXPLEXNAME, M_WAITOK | M_ZERO);
-			old = g_malloc(GV_MAXPLEXNAME, M_WAITOK | M_ZERO);
-			oldp = old;
-			strncpy(old, p->name, GV_MAXPLEXNAME);
 			/*
-			 * XXX: can we be sure that the current plex name has
-			 * the format 'foo.pX'?
+			 * Look for the last dot in the string, and assume that
+			 * the old value was ok.
 			 */
-			strsep(&oldp, ".");
-			snprintf(new, GV_MAXPLEXNAME, "%s.%s", v->name, oldp);
-			err = gv_rename_plex(sc, req, p, new, flags);
-			g_free(new);
-			g_free(old);
+			ptr = strrchr(p->name, '.');
+			ptr++;
+			snprintf(newplex, sizeof(newplex), "%s.%s", v->name, ptr);
+			err = gv_rename_plex(sc, p, newplex, flags);
 			if (err)
 				return (err);
 		}
diff --git a/sys/geom/vinum/geom_vinum_rm.c b/sys/geom/vinum/geom_vinum_rm.c
index d7748da..2b1d36b 100644
--- a/sys/geom/vinum/geom_vinum_rm.c
+++ b/sys/geom/vinum/geom_vinum_rm.c
@@ -1,5 +1,5 @@
 /*-
- *  Copyright (c) 2004 Lukas Ertl
+ *  Copyright (c) 2004, 2007 Lukas Ertl
  *  All rights reserved.
  * 
  * Redistribution and use in source and binary forms, with or without
@@ -30,20 +30,11 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/libkern.h>
-#include <sys/kernel.h>
 #include <sys/malloc.h>
 
 #include <geom/geom.h>
 #include <geom/vinum/geom_vinum_var.h>
 #include <geom/vinum/geom_vinum.h>
-#include <geom/vinum/geom_vinum_share.h>
-
-static int	gv_rm_drive(struct gv_softc *, struct gctl_req *,
-		    struct gv_drive *, int);
-static int	gv_rm_plex(struct gv_softc *, struct gctl_req *,
-		    struct gv_plex *, int);
-static int	gv_rm_vol(struct gv_softc *, struct gctl_req *,
-		    struct gv_volume *, int);
 
 /* General 'remove' routine. */
 void
@@ -56,7 +47,7 @@ gv_remove(struct g_geom *gp, struct gctl_req *req)
 	struct gv_drive *d;
 	int *argc, *flags;
 	char *argv, buf[20];
-	int i, type, err;
+	int i, type;
 
 	argc = gctl_get_paraml(req, "argc", sizeof(*argc));
 
@@ -73,6 +64,8 @@ gv_remove(struct g_geom *gp, struct gctl_req *req)
 
 	sc = gp->softc;
 
+	/* XXX config locking */
+
 	for (i = 0; i < *argc; i++) {
 		snprintf(buf, sizeof(buf), "argv%d", i);
 		argv = gctl_get_param(req, buf, NULL);
@@ -82,184 +75,173 @@ gv_remove(struct g_geom *gp, struct gctl_req *req)
 		switch (type) {
 		case GV_TYPE_VOL:
 			v = gv_find_vol(sc, argv);
-			if (v == NULL) {
-				gctl_error(req, "unknown volume '%s'", argv);
+
+			/*
+			 * If this volume has plexes, we want a recursive
+			 * removal.
+			 */
+			if (!LIST_EMPTY(&v->plexes) && !(*flags & GV_FLAG_R)) {
+				gctl_error(req, "volume '%s' has attached "
+				    "plexes - need recursive removal", v->name);
 				return;
 			}
-			err = gv_rm_vol(sc, req, v, *flags);
-			if (err)
-				return;
+
+			gv_post_event(sc, GV_EVENT_RM_VOLUME, v, NULL, 0, 0);
 			break;
+
 		case GV_TYPE_PLEX:
 			p = gv_find_plex(sc, argv);
-			if (p == NULL) {
-				gctl_error(req, "unknown plex '%s'", argv);
+
+			/*
+			 * If this plex has subdisks, we want a recursive
+			 * removal.
+			 */
+			if (!LIST_EMPTY(&p->subdisks) &&
+			    !(*flags & GV_FLAG_R)) {
+				gctl_error(req, "plex '%s' has attached "
+				    "subdisks - need recursive removal",
+				    p->name);
 				return;
 			}
-			err = gv_rm_plex(sc, req, p, *flags);
-			if (err)
+
+			/* Don't allow removal of the only plex of a volume. */
+			if (p->vol_sc != NULL && p->vol_sc->plexcount == 1) {
+				gctl_error(req, "plex '%s' is still attached "
+				    "to volume '%s'", p->name, p->volume);
 				return;
+			}
+
+			gv_post_event(sc, GV_EVENT_RM_PLEX, p, NULL, 0, 0);
 			break;
+
 		case GV_TYPE_SD:
 			s = gv_find_sd(sc, argv);
-			if (s == NULL) {
-				gctl_error(req, "unknown subdisk '%s'", argv);
+
+			/* Don't allow removal if attached to a plex. */
+			if (s->plex_sc != NULL) {
+				gctl_error(req, "subdisk '%s' is still attached"
+				    " to plex '%s'", s->name, s->plex_sc->name);
 				return;
 			}
-			err = gv_rm_sd(sc, req, s, *flags);
-			if (err)
-				return;
+
+			gv_post_event(sc, GV_EVENT_RM_SD, s, NULL, 0, 0);
 			break;
+
 		case GV_TYPE_DRIVE:
 			d = gv_find_drive(sc, argv);
-			if (d == NULL) {
-				gctl_error(req, "unknown drive '%s'", argv);
+			/* We don't allow to remove open drives. */
+			if (gv_consumer_is_open(d->consumer) &&
+			    !(*flags & GV_FLAG_F)) {
+				gctl_error(req, "drive '%s' is open", d->name);
 				return;
 			}
-			err = gv_rm_drive(sc, req, d, *flags);
-			if (err)
+
+			/* A drive with subdisks needs a recursive removal. */
+/*			if (!LIST_EMPTY(&d->subdisks) &&
+			    !(*flags & GV_FLAG_R)) {
+				gctl_error(req, "drive '%s' still has subdisks"
+				    " - need recursive removal", d->name);
 				return;
+			}*/
+
+			gv_post_event(sc, GV_EVENT_RM_DRIVE, d, NULL, *flags,
+			    0);
 			break;
+
 		default:
 			gctl_error(req, "unknown object '%s'", argv);
 			return;
 		}
 	}
 
-	gv_save_config_all(sc);
+	gv_post_event(sc, GV_EVENT_SAVE_CONFIG, sc, NULL, 0, 0);
 }
 
 /* Resets configuration */
 int
-gv_resetconfig(struct g_geom *gp, struct gctl_req *req)
+gv_resetconfig(struct gv_softc *sc)
 {
-	struct gv_softc *sc;
 	struct gv_drive *d, *d2;
 	struct gv_volume *v, *v2;
 	struct gv_plex *p, *p2;
 	struct gv_sd *s, *s2;
-	int flags;
-
-	d = NULL;
-	d2 = NULL;
-	p = NULL;
-	p2 = NULL;
-	s = NULL;
-	s2 = NULL;
-	flags = GV_FLAG_R;
-	sc = gp->softc;
-	/* First loop through to make sure no volumes are up */
-        LIST_FOREACH_SAFE(v, &sc->volumes, volume, v2) {
-		if (gv_is_open(v->geom)) {
-			gctl_error(req, "volume '%s' is busy", v->name);
-			return (-1);
+
+	/* First make sure nothing is open. */
+        LIST_FOREACH_SAFE(d, &sc->drives, drive, d2) {
+		if (gv_consumer_is_open(d->consumer)) {
+			return (GV_ERR_ISBUSY);
 		}
 	}
 	/* Then if not, we remove everything. */
-	LIST_FOREACH_SAFE(v, &sc->volumes, volume, v2)
-		gv_rm_vol(sc, req, v, flags);
-	LIST_FOREACH_SAFE(p, &sc->plexes, plex, p2)
-		gv_rm_plex(sc, req, p, flags);
 	LIST_FOREACH_SAFE(s, &sc->subdisks, sd, s2)
-		gv_rm_sd(sc, req, s, flags);
+		gv_rm_sd(sc, s);
 	LIST_FOREACH_SAFE(d, &sc->drives, drive, d2)
-		gv_rm_drive(sc, req, d, flags);
-	gv_save_config_all(sc);
+		gv_rm_drive(sc, d, 0);
+	LIST_FOREACH_SAFE(p, &sc->plexes, plex, p2)
+		gv_rm_plex(sc, p);
+	LIST_FOREACH_SAFE(v, &sc->volumes, volume, v2)
+		gv_rm_vol(sc, v);
+
+	gv_post_event(sc, GV_EVENT_SAVE_CONFIG, sc, NULL, 0, 0);
+
 	return (0);
 }
 
 /* Remove a volume. */
-static int
-gv_rm_vol(struct gv_softc *sc, struct gctl_req *req, struct gv_volume *v, int flags)
+void
+gv_rm_vol(struct gv_softc *sc, struct gv_volume *v)
 {
-	struct g_geom *gp;
+	struct g_provider *pp;
 	struct gv_plex *p, *p2;
-	int err;
 
-	g_topology_assert();
 	KASSERT(v != NULL, ("gv_rm_vol: NULL v"));
-
-	/* If this volume has plexes, we want a recursive removal. */
-	if (!LIST_EMPTY(&v->plexes) && !(flags & GV_FLAG_R)) {
-		gctl_error(req, "volume '%s' has attached plexes", v->name);
-		return (-1);
-	}
-
-	gp = v->geom;
+	pp = v->provider;
+	KASSERT(pp != NULL, ("gv_rm_vol: NULL pp"));
 
 	/* Check if any of our consumers is open. */
-	if (gp != NULL && gv_is_open(gp)) {
-		gctl_error(req, "volume '%s' is busy", v->name);
-		return (-1);
+	if (gv_provider_is_open(pp)) {
+		G_VINUM_DEBUG(0, "Unable to remove %s: volume still in use",
+		    v->name);
+		return;
 	}
 
 	/* Remove the plexes our volume has. */
-	LIST_FOREACH_SAFE(p, &v->plexes, in_volume, p2) {
-		v->plexcount--;
-		LIST_REMOVE(p, in_volume);
-		p->vol_sc = NULL;
-
-		err = gv_rm_plex(sc, req, p, flags);
-		if (err)
-			return (err);
-	}
+	LIST_FOREACH_SAFE(p, &v->plexes, in_volume, p2)
+		gv_rm_plex(sc, p);
 
-	/* Clean up and let our geom fade away. */
+	/* Clean up. */
 	LIST_REMOVE(v, volume);
-	gv_kill_vol_thread(v);
 	g_free(v);
-	if (gp != NULL) {
-		gp->softc = NULL;
-		g_wither_geom(gp, ENXIO);
-	}
 
-	return (0);
+	/* Get rid of the volume's provider. */
+	if (pp != NULL) {
+		g_topology_lock();
+		pp->flags |= G_PF_WITHER;
+		g_orphan_provider(pp, ENXIO);
+		g_topology_unlock();
+	}
 }
 
 /* Remove a plex. */
-static int
-gv_rm_plex(struct gv_softc *sc, struct gctl_req *req, struct gv_plex *p, int flags)
+void
+gv_rm_plex(struct gv_softc *sc, struct gv_plex *p)
 {
-	struct g_geom *gp;
 	struct gv_volume *v;
 	struct gv_sd *s, *s2;
-	int err;
-
-	g_topology_assert();
 
 	KASSERT(p != NULL, ("gv_rm_plex: NULL p"));
-
-	/* If this plex has subdisks, we want a recursive removal. */
-	if (!LIST_EMPTY(&p->subdisks) && !(flags & GV_FLAG_R)) {
-		gctl_error(req, "plex '%s' has attached subdisks", p->name);
-		return (-1);
-	}
-
-	if (p->vol_sc != NULL && p->vol_sc->plexcount == 1) {
-		gctl_error(req, "plex '%s' is still attached to volume '%s'",
-		    p->name, p->volume);
-		return (-1);
-	}
-
-	gp = p->geom;
+	v = p->vol_sc;
 
 	/* Check if any of our consumers is open. */
-	if (gp != NULL && gv_is_open(gp)) {
-		gctl_error(req, "plex '%s' is busy", p->name);
-		return (-1);
+	if (v != NULL && gv_provider_is_open(v->provider) && v->plexcount < 2) {
+		G_VINUM_DEBUG(0, "Unable to remove %s: volume still in use",
+		    p->name);
+		return;
 	}
 
 	/* Remove the subdisks our plex has. */
-	LIST_FOREACH_SAFE(s, &p->subdisks, in_plex, s2) {
-#if 0
-		LIST_REMOVE(s, in_plex);
-		s->plex_sc = NULL;
-#endif
-
-		err = gv_rm_sd(sc, req, s, flags);
-		if (err)
-			return (err);
-	}
+	LIST_FOREACH_SAFE(s, &p->subdisks, in_plex, s2)
+		gv_rm_sd(sc, s);
 
 	v = p->vol_sc;
 	/* Clean up and let our geom fade away. */
@@ -272,35 +254,25 @@ gv_rm_plex(struct gv_softc *sc, struct gctl_req *req, struct gv_plex *p, int fla
 		gv_update_vol_size(v, gv_vol_size(v));
 	}
 
-	gv_kill_plex_thread(p);
 	g_free(p);
-
-	if (gp != NULL) {
-		gp->softc = NULL;
-		g_wither_geom(gp, ENXIO);
-	}
-
-	return (0);
 }
 
 /* Remove a subdisk. */
-int
-gv_rm_sd(struct gv_softc *sc, struct gctl_req *req, struct gv_sd *s, int flags)
+void
+gv_rm_sd(struct gv_softc *sc, struct gv_sd *s)
 {
-	struct g_provider *pp;
 	struct gv_plex *p;
 	struct gv_volume *v;
 
 	KASSERT(s != NULL, ("gv_rm_sd: NULL s"));
 
-	pp = s->provider;
 	p = s->plex_sc;
 	v = NULL;
 
 	/* Clean up. */
 	if (p != NULL) {
 		LIST_REMOVE(s, in_plex);
-
+		s->plex_sc = NULL;
 		p->sdcount--;
 		/* Update the plexsize. */
 		p->size = gv_plex_size(p);
@@ -310,77 +282,64 @@ gv_rm_sd(struct gv_softc *sc, struct gctl_req *req, struct gv_sd *s, int flags)
 			gv_update_vol_size(v, gv_vol_size(v));
 		}
 	}
-	if (s->drive_sc)
+	if (s->drive_sc && !(s->drive_sc->flags & GV_DRIVE_REFERENCED))
 		LIST_REMOVE(s, from_drive);
 	LIST_REMOVE(s, sd);
 	gv_free_sd(s);
 	g_free(s);
-
-	/* If the subdisk has a provider we need to clean up this one too. */
-	if (pp != NULL) {
-		pp->flags |= G_PF_WITHER;
-		g_orphan_provider(pp, ENXIO);
-	}
-
-	return (0);
 }
 
 /* Remove a drive. */
-static int
-gv_rm_drive(struct gv_softc *sc, struct gctl_req *req, struct gv_drive *d, int flags)
+void
+gv_rm_drive(struct gv_softc *sc, struct gv_drive *d, int flags)
 {
-	struct g_geom *gp;
 	struct g_consumer *cp;
 	struct gv_freelist *fl, *fl2;
 	struct gv_plex *p;
 	struct gv_sd *s, *s2;
 	struct gv_volume *v;
+	struct gv_drive *d2;
 	int err;
 
 	KASSERT(d != NULL, ("gv_rm_drive: NULL d"));
-	gp = d->geom;
-	KASSERT(gp != NULL, ("gv_rm_drive: NULL gp"));
 
-	/* We don't allow to remove open drives. */
-	if (gv_is_open(gp)) {
-		gctl_error(req, "drive '%s' is open", d->name);
-		return (-1);
-	}
+	cp = d->consumer;
 
-	/* A drive with subdisks needs a recursive removal. */
-	if (!LIST_EMPTY(&d->subdisks) && !(flags & GV_FLAG_R)) {
-		gctl_error(req, "drive '%s' still has subdisks", d->name);
-		return (-1);
-	}
+	if (cp != NULL) {
+		g_topology_lock();
+		err = g_access(cp, 0, 1, 0);
+		g_topology_unlock();
 
-	cp = LIST_FIRST(&gp->consumer);
-	err = g_access(cp, 0, 1, 0);
-	if (err) {
-		G_VINUM_DEBUG(0, "%s: unable to access '%s', errno: "
-		    "%d", __func__, cp->provider->name, err);
-		return (err);
-	}
+		if (err) {
+			G_VINUM_DEBUG(0, "%s: couldn't access '%s', "
+			    "errno: %d", __func__, cp->provider->name, err);
+			return;
+		}
 
-	/* Clear the Vinum Magic. */
-	d->hdr->magic = GV_NOMAGIC;
-	g_topology_unlock();
-	err = gv_write_header(cp, d->hdr);
-	if (err) {
-		G_VINUM_DEBUG(0, "%s: unable to write header to '%s'"
-		    ", errno: %d", __func__, cp->provider->name, err);
-		d->hdr->magic = GV_MAGIC;
+		/* Clear the Vinum Magic. */
+		d->hdr->magic = GV_NOMAGIC;
+		err = gv_write_header(cp, d->hdr);
+		if (err)
+			G_VINUM_DEBUG(0, "gv_rm_drive: couldn't write header to"
+			    " '%s', errno: %d", cp->provider->name, err);
+
+		g_topology_lock();
+		g_access(cp, -cp->acr, -cp->acw, -cp->ace);
+		g_detach(cp);
+		g_destroy_consumer(cp);
+		g_topology_unlock();
 	}
-	g_topology_lock();
-	g_access(cp, 0, -1, 0);
 
 	/* Remove all associated subdisks, plexes, volumes. */
-	if (!LIST_EMPTY(&d->subdisks)) {
-		LIST_FOREACH_SAFE(s, &d->subdisks, from_drive, s2) {
-			p = s->plex_sc;
-			if (p != NULL) {
-				v = p->vol_sc;
-				if (v != NULL)
-					gv_rm_vol(sc, req, v, flags);
+	if (flags & GV_FLAG_R) {
+		if (!LIST_EMPTY(&d->subdisks)) {
+			LIST_FOREACH_SAFE(s, &d->subdisks, from_drive, s2) {
+				p = s->plex_sc;
+				if (p != NULL) {
+					v = p->vol_sc;
+					if (v != NULL)
+						gv_rm_vol(sc, v);
+				}
 			}
 		}
 	}
@@ -390,15 +349,33 @@ gv_rm_drive(struct gv_softc *sc, struct gctl_req *req, struct gv_drive *d, int f
 		LIST_REMOVE(fl, freelist);
 		g_free(fl);
 	}
-	LIST_REMOVE(d, drive);
 
-	gv_kill_drive_thread(d);
-	gp = d->geom;
-	d->geom = NULL;
+	LIST_REMOVE(d, drive);
 	g_free(d->hdr);
+
+	/* Put ourself into referenced state if we have subdisks. */
+	if (d->sdcount > 0) {
+		d->consumer = NULL;
+		d->hdr = NULL;
+		d->flags |= GV_DRIVE_REFERENCED;
+		snprintf(d->device, sizeof(d->device), "???");
+		d->size = 0;
+		d->avail = 0;
+		d->freelist_entries = 0;
+		LIST_FOREACH(s, &d->subdisks, from_drive) {
+			s->flags |= GV_SD_TASTED;
+			gv_set_sd_state(s, GV_SD_DOWN, GV_SETSTATE_FORCE);
+		}
+		/* Shuffle around so we keep gv_is_newer happy. */
+		LIST_REMOVE(d, drive);
+		d2 = LIST_FIRST(&sc->drives);
+		if (d2 == NULL)
+			LIST_INSERT_HEAD(&sc->drives, d, drive);
+		else
+			LIST_INSERT_AFTER(d2, d, drive);
+		return;
+	}
 	g_free(d);
-	gv_save_config_all(sc);
-	g_wither_geom(gp, ENXIO);
 
-	return (err);
+	gv_save_config(sc);
 }
diff --git a/sys/geom/vinum/geom_vinum_share.c b/sys/geom/vinum/geom_vinum_share.c
index ec97c5e..eb70c28 100644
--- a/sys/geom/vinum/geom_vinum_share.c
+++ b/sys/geom/vinum/geom_vinum_share.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2004 Lukas Ertl
+ * Copyright (c) 2004, 2007 Lukas Ertl
  * Copyright (c) 1997, 1998, 1999
  *      Nan Yang Computer Services Limited.  All rights reserved.
  *
@@ -45,10 +45,6 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #ifdef _KERNEL
-#include <sys/bio.h>
-#include <sys/conf.h>
-#include <sys/kernel.h>
-#include <sys/kthread.h>
 #include <sys/malloc.h>
 #include <sys/systm.h>
 
@@ -63,7 +59,6 @@ __FBSDID("$FreeBSD$");
 #define	g_free	free
 #endif /* _KERNEL */
 
-#include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/queue.h>
 
@@ -237,6 +232,8 @@ gv_sdstatei(char *buf)
 		return (GV_SD_UP);
 	else if (!strcmp(buf, "reviving"))
 		return (GV_SD_REVIVING);
+	else if (!strcmp(buf, "initializing"))
+		return (GV_SD_INITIALIZING);
 	else if (!strcmp(buf, "stale"))
 		return (GV_SD_STALE);
 	else
@@ -273,6 +270,8 @@ gv_plexstatei(char *buf)
 		return (GV_PLEX_INITIALIZING);
 	else if (!strcmp(buf, "degraded"))
 		return (GV_PLEX_DEGRADED);
+	else if (!strcmp(buf, "growable"))
+		return (GV_PLEX_GROWABLE);
 	else
 		return (GV_PLEX_DOWN);
 }
@@ -288,6 +287,8 @@ gv_plexstate(int state)
 		return "initializing";
 	case GV_PLEX_DEGRADED:
 		return "degraded";
+	case GV_PLEX_GROWABLE:
+		return "growable";
 	case GV_PLEX_UP:
 		return "up";
 	default:
@@ -366,27 +367,86 @@ gv_plexorg_short(int org)
 	}
 }
 
-/* Get a new drive object. */
-struct gv_drive *
-gv_new_drive(int max, char *token[])
+struct gv_sd *
+gv_alloc_sd(void)
 {
-	struct gv_drive *d;
-	int j, errors;
-	char *ptr;
+	struct gv_sd *s;
 
-	if (token[1] == NULL || *token[1] == '\0')
+#ifdef _KERNEL
+	s = g_malloc(sizeof(struct gv_sd), M_NOWAIT);
+#else
+	s = malloc(sizeof(struct gv_sd));
+#endif
+	if (s == NULL)
 		return (NULL);
+	bzero(s, sizeof(struct gv_sd));
+	s->plex_offset = -1;
+	s->size = -1;
+	s->drive_offset = -1;
+	return (s);
+}
 
-#ifdef _KERNEL
-	d = g_malloc(sizeof(struct gv_drive), M_WAITOK | M_ZERO);
+struct gv_drive *
+gv_alloc_drive(void)
+{
+	struct gv_drive *d;
 
+#ifdef _KERNEL
+	d = g_malloc(sizeof(struct gv_drive), M_NOWAIT);
 #else
 	d = malloc(sizeof(struct gv_drive));
+#endif
 	if (d == NULL)
 		return (NULL);
 	bzero(d, sizeof(struct gv_drive));
+	return (d);
+}
+
+struct gv_volume *
+gv_alloc_volume(void)
+{
+	struct gv_volume *v;
+
+#ifdef _KERNEL
+	v = g_malloc(sizeof(struct gv_volume), M_NOWAIT);
+#else
+	v = malloc(sizeof(struct gv_volume));
 #endif
+	if (v == NULL)
+		return (NULL);
+	bzero(v, sizeof(struct gv_volume));
+	return (v);
+}
+
+struct gv_plex *
+gv_alloc_plex(void)
+{
+	struct gv_plex *p;
 
+#ifdef _KERNEL
+	p = g_malloc(sizeof(struct gv_plex), M_NOWAIT);
+#else
+	p = malloc(sizeof(struct gv_plex));
+#endif
+	if (p == NULL)
+		return (NULL);
+	bzero(p, sizeof(struct gv_plex));
+	return (p);
+}
+
+/* Get a new drive object. */
+struct gv_drive *
+gv_new_drive(int max, char *token[])
+{
+	struct gv_drive *d;
+	int j, errors;
+	char *ptr;
+
+	if (token[1] == NULL || *token[1] == '\0')
+		return (NULL);
+	d = gv_alloc_drive();
+	if (d == NULL)
+		return (NULL);
 	errors = 0;
 	for (j = 1; j < max; j++) {
 		if (!strcmp(token[j], "state")) {
@@ -406,10 +466,10 @@ gv_new_drive(int max, char *token[])
 
 			if (strncmp(ptr, "/dev/", 5) == 0)
 				ptr += 5;
-			strncpy(d->device, ptr, GV_MAXDRIVENAME);
+			strlcpy(d->device, ptr, sizeof(d->device));
 		} else {
 			/* We assume this is the drive name. */
-			strncpy(d->name, token[j], GV_MAXDRIVENAME);
+			strlcpy(d->name, token[j], sizeof(d->name));
 		}
 	}
 
@@ -434,15 +494,9 @@ gv_new_volume(int max, char *token[])
 	if (token[1] == NULL || *token[1] == '\0')
 		return (NULL);
 
-#ifdef _KERNEL
-	v = g_malloc(sizeof(struct gv_volume), M_WAITOK | M_ZERO);
-
-#else
-	v = malloc(sizeof(struct gv_volume));
+	v = gv_alloc_volume();
 	if (v == NULL)
 		return (NULL);
-	bzero(v, sizeof(struct gv_volume));
-#endif
 
 	errors = 0;
 	for (j = 1; j < max; j++) {
@@ -455,7 +509,7 @@ gv_new_volume(int max, char *token[])
 			v->state = gv_volstatei(token[j]);
 		} else {
 			/* We assume this is the volume name. */
-			strncpy(v->name, token[j], GV_MAXVOLNAME);
+			strlcpy(v->name, token[j], sizeof(v->name));
 		}
 	}
 
@@ -480,14 +534,9 @@ gv_new_plex(int max, char *token[])
 	if (token[1] == NULL || *token[1] == '\0')
 		return (NULL);
 
-#ifdef _KERNEL
-	p = g_malloc(sizeof(struct gv_plex), M_WAITOK | M_ZERO);
-#else
-	p = malloc(sizeof(struct gv_plex));
+	p = gv_alloc_plex();
 	if (p == NULL)
 		return (NULL);
-	bzero(p, sizeof(struct gv_plex));
-#endif
 
 	errors = 0;
 	for (j = 1; j < max; j++) {
@@ -497,7 +546,7 @@ gv_new_plex(int max, char *token[])
 				errors++;
 				break;
 			}
-			strncpy(p->name, token[j], GV_MAXPLEXNAME);
+			strlcpy(p->name, token[j], sizeof(p->name));
 		} else if (!strcmp(token[j], "org")) {
 			j++;
 			if (j >= max) {
@@ -532,7 +581,7 @@ gv_new_plex(int max, char *token[])
 				errors++;
 				break;
 			}
-			strncpy(p->volume, token[j], GV_MAXVOLNAME);
+			strlcpy(p->volume, token[j], sizeof(p->volume));
 		} else {
 			errors++;
 			break;
@@ -547,6 +596,8 @@ gv_new_plex(int max, char *token[])
 	return (p);
 }
 
+
+
 /* Get a new subdisk object. */
 struct gv_sd *
 gv_new_sd(int max, char *token[])
@@ -555,20 +606,12 @@ gv_new_sd(int max, char *token[])
 	int j, errors;
 
 	if (token[1] == NULL || *token[1] == '\0')
-		return NULL;
+		return (NULL);
 
-#ifdef _KERNEL
-	s = g_malloc(sizeof(struct gv_sd), M_WAITOK | M_ZERO);
-#else
-	s = malloc(sizeof(struct gv_sd));
+	s = gv_alloc_sd();
 	if (s == NULL)
-		return NULL;
-	bzero(s, sizeof(struct gv_sd));
-#endif
+		return (NULL);
 
-	s->plex_offset = -1;
-	s->size = -1;
-	s->drive_offset = -1;
 	errors = 0;
 	for (j = 1; j < max; j++) {
 		if (!strcmp(token[j], "name")) {
@@ -577,21 +620,21 @@ gv_new_sd(int max, char *token[])
 				errors++;
 				break;
 			}
-			strncpy(s->name, token[j], GV_MAXSDNAME);
+			strlcpy(s->name, token[j], sizeof(s->name));
 		} else if (!strcmp(token[j], "drive")) {
 			j++;
 			if (j >= max) {
 				errors++;
 				break;
 			}
-			strncpy(s->drive, token[j], GV_MAXDRIVENAME);
+			strlcpy(s->drive, token[j], sizeof(s->drive));
 		} else if (!strcmp(token[j], "plex")) {
 			j++;
 			if (j >= max) {
 				errors++;
 				break;
 			}
-			strncpy(s->plex, token[j], GV_MAXPLEXNAME);
+			strlcpy(s->plex, token[j], sizeof(s->plex));
 		} else if (!strcmp(token[j], "state")) {
 			j++;
 			if (j >= max) {
diff --git a/sys/geom/vinum/geom_vinum_share.h b/sys/geom/vinum/geom_vinum_share.h
index f15f45d..5646d6b 100644
--- a/sys/geom/vinum/geom_vinum_share.h
+++ b/sys/geom/vinum/geom_vinum_share.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2004 Lukas Ertl
+ * Copyright (c) 2004, 2007 Lukas Ertl
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -41,6 +41,10 @@ enum {
 off_t	gv_sizespec(char *);
 int	gv_tokenize(char *, char **, int);
 
+struct gv_sd	 *gv_alloc_sd(void);
+struct gv_volume *gv_alloc_volume(void);
+struct gv_plex	 *gv_alloc_plex(void);
+struct gv_drive	 *gv_alloc_drive(void);
 struct gv_drive	 *gv_new_drive(int, char **);
 struct gv_plex	 *gv_new_plex(int, char **);
 struct gv_sd	 *gv_new_sd(int, char **);
diff --git a/sys/geom/vinum/geom_vinum_state.c b/sys/geom/vinum/geom_vinum_state.c
index db40390..568c784 100644
--- a/sys/geom/vinum/geom_vinum_state.c
+++ b/sys/geom/vinum/geom_vinum_state.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2004 Lukas Ertl
+ * Copyright (c) 2004, 2007 Lukas Ertl
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -27,8 +27,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <sys/param.h>
-#include <sys/kernel.h>
 #include <sys/libkern.h>
 #include <sys/malloc.h>
 
@@ -43,8 +41,10 @@ gv_setstate(struct g_geom *gp, struct gctl_req *req)
 	struct gv_softc *sc;
 	struct gv_sd *s;
 	struct gv_drive *d;
+	struct gv_volume *v;
+	struct gv_plex *p;
 	char *obj, *state;
-	int err, f, *flags, newstate, type;
+	int f, *flags, type;
 
 	f = 0;
 	obj = gctl_get_param(req, "object", NULL);
@@ -72,43 +72,52 @@ gv_setstate(struct g_geom *gp, struct gctl_req *req)
 	type = gv_object_type(sc, obj);
 	switch (type) {
 	case GV_TYPE_VOL:
+		if (gv_volstatei(state) < 0) {
+			gctl_error(req, "invalid volume state '%s'", state);
+			break;
+		}
+		v = gv_find_vol(sc, obj);
+		gv_post_event(sc, GV_EVENT_SET_VOL_STATE, v, NULL,
+		    gv_volstatei(state), f);
+		break;
+
 	case GV_TYPE_PLEX:
-		gctl_error(req, "volume or plex state cannot be set currently");
+		if (gv_plexstatei(state) < 0) {
+			gctl_error(req, "invalid plex state '%s'", state);
+			break;
+		}
+		p = gv_find_plex(sc, obj);
+		gv_post_event(sc, GV_EVENT_SET_PLEX_STATE, p, NULL,
+		    gv_plexstatei(state), f);
 		break;
 
 	case GV_TYPE_SD:
-		newstate = gv_sdstatei(state);
-		if (newstate < 0) {
+		if (gv_sdstatei(state) < 0) {
 			gctl_error(req, "invalid subdisk state '%s'", state);
 			break;
 		}
 		s = gv_find_sd(sc, obj);
-		err = gv_set_sd_state(s, newstate, f);
-		if (err)
-			gctl_error(req, "cannot set subdisk state");
+		gv_post_event(sc, GV_EVENT_SET_SD_STATE, s, NULL,
+		    gv_sdstatei(state), f);
 		break;
 
 	case GV_TYPE_DRIVE:
-		newstate = gv_drivestatei(state);
-		if (newstate < 0) {
+		if (gv_drivestatei(state) < 0) {
 			gctl_error(req, "invalid drive state '%s'", state);
 			break;
 		}
 		d = gv_find_drive(sc, obj);
-		err = gv_set_drive_state(d, newstate, f);
-		if (err)
-			gctl_error(req, "cannot set drive state");
+		gv_post_event(sc, GV_EVENT_SET_DRIVE_STATE, d, NULL,
+		    gv_drivestatei(state), f);
 		break;
 
 	default:
 		gctl_error(req, "unknown object '%s'", obj);
 		break;
 	}
-
-	return;
 }
 
-/* Update drive state; return 0 if the state changes, otherwise -1. */
+/* Update drive state; return 0 if the state changes, otherwise error. */
 int
 gv_set_drive_state(struct gv_drive *d, int newstate, int flags)
 {
@@ -123,9 +132,9 @@ gv_set_drive_state(struct gv_drive *d, int newstate, int flags)
 		return (0);
 
 	/* We allow to take down an open drive only with force. */
-	if ((newstate == GV_DRIVE_DOWN) && gv_is_open(d->geom) &&
+	if ((newstate == GV_DRIVE_DOWN) && gv_consumer_is_open(d->consumer) &&
 	    (!(flags & GV_SETSTATE_FORCE)))
-		return (-1);
+		return (GV_ERR_ISBUSY);
 
 	d->state = newstate;
 
@@ -136,7 +145,7 @@ gv_set_drive_state(struct gv_drive *d, int newstate, int flags)
 
 	/* Save the config back to disk. */
 	if (flags & GV_SETSTATE_CONFIG)
-		gv_save_config_all(d->vinumconf);
+		gv_save_config(d->vinumconf);
 
 	return (0);
 }
@@ -165,14 +174,24 @@ gv_set_sd_state(struct gv_sd *s, int newstate, int flags)
 		 * force.
 		 */
 		if ((s->plex_sc != NULL) && !(flags & GV_SETSTATE_FORCE))
-			return (-1);
+			return (GV_ERR_ISATTACHED);
+		break;
+
+	case GV_SD_REVIVING:
+	case GV_SD_INITIALIZING:
+		/*
+		 * Only do this if we're forced, since it usually is done
+		 * internally, and then we do use the force flag. 
+		 */
+		if (!flags & GV_SETSTATE_FORCE)
+			return (GV_ERR_SETSTATE);
 		break;
 
 	case GV_SD_UP:
 		/* We can't bring the subdisk up if our drive is dead. */
 		d = s->drive_sc;
 		if ((d == NULL) || (d->state != GV_DRIVE_UP))
-			return (-1);
+			return (GV_ERR_SETSTATE);
 
 		/* Check from where we want to be brought up. */
 		switch (s->state) {
@@ -201,12 +220,15 @@ gv_set_sd_state(struct gv_sd *s, int newstate, int flags)
 
 			if (p->org != GV_PLEX_RAID5)
 				break;
-			else if (flags & GV_SETSTATE_FORCE)
+			else if (s->flags & GV_SD_CANGOUP) {
+				s->flags &= ~GV_SD_CANGOUP;
+				break;
+			} else if (flags & GV_SETSTATE_FORCE)
 				break;
 			else
 				s->state = GV_SD_STALE;
 
-			status = -1;
+			status = GV_ERR_SETSTATE;
 			break;
 
 		case GV_SD_STALE:
@@ -221,21 +243,24 @@ gv_set_sd_state(struct gv_sd *s, int newstate, int flags)
 			if (p == NULL || flags & GV_SETSTATE_FORCE)
 				break;
 
-			if ((p->org != GV_PLEX_RAID5) &&
-			    (p->vol_sc->plexcount == 1))
+			if ((p->org != GV_PLEX_RAID5 &&
+			    p->vol_sc->plexcount == 1) ||
+			    (p->flags & GV_PLEX_SYNCING &&
+			    p->synced > 0 &&
+			    p->org == GV_PLEX_RAID5))
 				break;
 			else
-				return (-1);
+				return (GV_ERR_SETSTATE);
 
 		default:
-			return (-1);
+			return (GV_ERR_INVSTATE);
 		}
 		break;
 
 	/* Other state transitions are only possible with force. */
 	default:
 		if (!(flags & GV_SETSTATE_FORCE))
-			return (-1);
+			return (GV_ERR_SETSTATE);
 	}
 
 	/* We can change the state and do it. */
@@ -248,11 +273,102 @@ gv_set_sd_state(struct gv_sd *s, int newstate, int flags)
 
 	/* Save the config back to disk. */
 	if (flags & GV_SETSTATE_CONFIG)
-		gv_save_config_all(s->vinumconf);
+		gv_save_config(s->vinumconf);
 
 	return (status);
 }
 
+int
+gv_set_plex_state(struct gv_plex *p, int newstate, int flags)
+{
+	struct gv_volume *v;
+	int oldstate, plexdown;
+
+	KASSERT(p != NULL, ("gv_set_plex_state: NULL p"));
+
+	oldstate = p->state;
+	v = p->vol_sc;
+	plexdown = 0;
+
+	if (newstate == oldstate)
+		return (0);
+
+	switch (newstate) {
+	case GV_PLEX_UP:
+		/* Let update_plex handle if the plex can come up */
+		gv_update_plex_state(p);
+		if (p->state != GV_PLEX_UP && !(flags & GV_SETSTATE_FORCE))
+			return (GV_ERR_SETSTATE);
+		p->state = newstate;
+		break;
+	case GV_PLEX_DOWN:
+		/*
+		 * Set state to GV_PLEX_DOWN only if no-one is using the plex,
+		 * or if the state is forced.
+		 */
+		if (v != NULL) {
+			/* If the only one up, force is needed. */
+			plexdown = gv_plexdown(v);
+			if ((v->plexcount == 1 ||
+			    (v->plexcount - plexdown == 1)) &&
+			    ((flags & GV_SETSTATE_FORCE) == 0))
+				return (GV_ERR_SETSTATE);
+		}
+		p->state = newstate;
+		break;
+	case GV_PLEX_DEGRADED:
+		/* Only used internally, so we have to be forced. */
+		if (flags & GV_SETSTATE_FORCE)
+			p->state = newstate;
+		break;
+	}
+
+	/* Update our volume if we have one. */
+	if (v != NULL)
+		gv_update_vol_state(v);
+
+	/* Save config. */
+	if (flags & GV_SETSTATE_CONFIG)
+		gv_save_config(p->vinumconf);
+	return (0);
+}
+
+int
+gv_set_vol_state(struct gv_volume *v, int newstate, int flags)
+{
+	int oldstate;
+
+	KASSERT(v != NULL, ("gv_set_vol_state: NULL v"));
+
+	oldstate = v->state;
+
+	if (newstate == oldstate)
+		return (0);
+
+	switch (newstate) {
+	case GV_VOL_UP:
+		/* Let update handle if the volume can come up. */
+		gv_update_vol_state(v);
+		if (v->state != GV_VOL_UP && !(flags & GV_SETSTATE_FORCE))
+			return (GV_ERR_SETSTATE);
+		v->state = newstate;
+		break;
+	case GV_VOL_DOWN:
+		/*
+		 * Set state to GV_VOL_DOWN only if no-one is using the volume,
+		 * or if the state should be forced.
+		 */
+		if (!gv_provider_is_open(v->provider) &&
+		    !(flags & GV_SETSTATE_FORCE))
+			return (GV_ERR_ISBUSY);
+		v->state = newstate;
+		break;
+	}
+	/* Save config */
+	if (flags & GV_SETSTATE_CONFIG)
+		gv_save_config(v->vinumconf);
+	return (0);
+}
 
 /* Update the state of a subdisk based on its environment. */
 void
@@ -268,15 +384,19 @@ gv_update_sd_state(struct gv_sd *s)
 	oldstate = s->state;
 	
 	/* If our drive isn't up we cannot be up either. */
-	if (d->state != GV_DRIVE_UP)
+	if (d->state != GV_DRIVE_UP) {
 		s->state = GV_SD_DOWN;
 	/* If this subdisk was just created, we assume it is good.*/
-	else if (s->flags & GV_SD_NEWBORN) {
+	} else if (s->flags & GV_SD_NEWBORN) {
 		s->state = GV_SD_UP;
 		s->flags &= ~GV_SD_NEWBORN;
-	} else if (s->state != GV_SD_UP)
-		s->state = GV_SD_STALE;
-	else
+	} else if (s->state != GV_SD_UP) {
+		if (s->flags & GV_SD_CANGOUP) {
+			s->state = GV_SD_UP;
+			s->flags &= ~GV_SD_CANGOUP;
+		} else
+			s->state = GV_SD_STALE;
+	} else
 		s->state = GV_SD_UP;
 	
 	if (s->state != oldstate)
@@ -292,6 +412,7 @@ gv_update_sd_state(struct gv_sd *s)
 void
 gv_update_plex_state(struct gv_plex *p)
 {
+	struct gv_sd *s;
 	int sdstates;
 	int oldstate;
 
@@ -316,13 +437,24 @@ gv_update_plex_state(struct gv_plex *p)
 
 	/* Some of our subdisks are initializing. */
 	} else if (sdstates & GV_SD_INITSTATE) {
-		if (p->flags & GV_PLEX_SYNCING)
+
+		if (p->flags & GV_PLEX_SYNCING ||
+		    p->flags & GV_PLEX_REBUILDING)
 			p->state = GV_PLEX_DEGRADED;
 		else
 			p->state = GV_PLEX_DOWN;
 	} else
 		p->state = GV_PLEX_DOWN;
 
+	if (p->state == GV_PLEX_UP) {
+		LIST_FOREACH(s, &p->subdisks, in_plex) {
+			if (s->flags & GV_SD_GROW) {
+				p->state = GV_PLEX_GROWABLE;
+				break;
+			}
+		}
+	}
+
 	if (p->state != oldstate)
 		G_VINUM_DEBUG(1, "plex %s state change: %s -> %s", p->name,
 		    gv_plexstate(oldstate), gv_plexstate(p->state));
diff --git a/sys/geom/vinum/geom_vinum_subr.c b/sys/geom/vinum/geom_vinum_subr.c
index 848fbc7..70c7f3f 100644
--- a/sys/geom/vinum/geom_vinum_subr.c
+++ b/sys/geom/vinum/geom_vinum_subr.c
@@ -1,5 +1,6 @@
 /*-
- * Copyright (c) 2004 Lukas Ertl
+ * Copyright (c) 2004, 2007 Lukas Ertl
+ * Copyright (c) 2007, 2009 Ulf Lilleengen
  * Copyright (c) 1997, 1998, 1999
  *      Nan Yang Computer Services Limited.  All rights reserved.
  *
@@ -42,59 +43,28 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
-#include <sys/conf.h>
-#include <sys/kernel.h>
-#include <sys/libkern.h>
 #include <sys/malloc.h>
 #include <sys/systm.h>
 
 #include <geom/geom.h>
-#include <geom/geom_int.h>
 #include <geom/vinum/geom_vinum_var.h>
 #include <geom/vinum/geom_vinum.h>
 #include <geom/vinum/geom_vinum_share.h>
 
-static off_t gv_plex_smallest_sd(struct gv_plex *, off_t);
+int	gv_drive_is_newer(struct gv_softc *, struct gv_drive *);
+static off_t gv_plex_smallest_sd(struct gv_plex *);
 
-/* Find the VINUM class and it's associated geom. */
-struct g_geom *
-find_vinum_geom(void)
-{
-	struct g_class *mp;
-	struct g_geom *gp;
-
-	g_topology_assert();
-
-	gp = NULL;
-
-	LIST_FOREACH(mp, &g_classes, class) {
-		if (!strcmp(mp->name, "VINUM")) {
-			gp = LIST_FIRST(&mp->geom);
-			break;
-		}
-	}
-
-	return (gp);
-}
-
-/*
- * Parse the vinum config provided in *buf and store it in *gp's softc.
- * If parameter 'merge' is non-zero, then the given config is merged into
- * *gp.
- */
 void
-gv_parse_config(struct gv_softc *sc, u_char *buf, int merge)
+gv_parse_config(struct gv_softc *sc, char *buf, struct gv_drive *d)
 {
 	char *aptr, *bptr, *cptr;
 	struct gv_volume *v, *v2;
 	struct gv_plex *p, *p2;
 	struct gv_sd *s, *s2;
-	int tokens;
+	int error, is_newer, tokens;
 	char *token[GV_MAXARGS];
 
-	g_topology_assert();
-
-	KASSERT(sc != NULL, ("gv_parse_config: NULL softc"));
+	is_newer = gv_drive_is_newer(sc, d);
 
 	/* Until the end of the string *buf. */
 	for (aptr = buf; *aptr != '\0'; aptr = bptr) {
@@ -109,64 +79,95 @@ gv_parse_config(struct gv_softc *sc, u_char *buf, int merge)
 
 		tokens = gv_tokenize(cptr, token, GV_MAXARGS);
 
-		if (tokens > 0) {
-			if (!strcmp(token[0], "volume")) {
-				v = gv_new_volume(tokens, token);
-				if (v == NULL) {
-					G_VINUM_DEBUG(0, "failed volume");
-					break;
-				}
+		if (tokens <= 0)
+			continue;
 
-				if (merge) {
-					v2 = gv_find_vol(sc, v->name);
-					if (v2 != NULL) {
-						g_free(v);
-						continue;
-					}
+		if (!strcmp(token[0], "volume")) {
+			v = gv_new_volume(tokens, token);
+			if (v == NULL) {
+				G_VINUM_DEBUG(0, "config parse failed volume");
+				break;
+			}
+
+			v2 = gv_find_vol(sc, v->name);
+			if (v2 != NULL) {
+				if (is_newer) {
+					v2->state = v->state;
+					G_VINUM_DEBUG(2, "newer volume found!");
 				}
+				g_free(v);
+				continue;
+			}
 
-				v->vinumconf = sc;
-				LIST_INIT(&v->plexes);
-				LIST_INSERT_HEAD(&sc->volumes, v, volume);
+			gv_create_volume(sc, v);
 
-			} else if (!strcmp(token[0], "plex")) {
-				p = gv_new_plex(tokens, token);
-				if (p == NULL) {
-					G_VINUM_DEBUG(0, "failed plex");
-					break;
-				}
+		} else if (!strcmp(token[0], "plex")) {
+			p = gv_new_plex(tokens, token);
+			if (p == NULL) {
+				G_VINUM_DEBUG(0, "config parse failed plex");
+				break;
+			}
 
-				if (merge) {
-					p2 = gv_find_plex(sc, p->name);
-					if (p2 != NULL) {
-						g_free(p);
-						continue;
-					}
+			p2 = gv_find_plex(sc, p->name);
+			if (p2 != NULL) {
+				/* XXX */
+				if (is_newer) {
+					p2->state = p->state;
+					G_VINUM_DEBUG(2, "newer plex found!");
 				}
+				g_free(p);
+				continue;
+			}
 
-				p->vinumconf = sc;
-				LIST_INIT(&p->subdisks);
-				LIST_INSERT_HEAD(&sc->plexes, p, plex);
+			error = gv_create_plex(sc, p);
+			if (error)
+				continue;
+			/*
+			 * These flags were set in gv_create_plex() and are not
+			 * needed here (on-disk config parsing).
+			 */
+			p->flags &= ~GV_PLEX_ADDED;
+			p->flags &= ~GV_PLEX_NEWBORN;
 
-			} else if (!strcmp(token[0], "sd")) {
-				s = gv_new_sd(tokens, token);
+		} else if (!strcmp(token[0], "sd")) {
+			s = gv_new_sd(tokens, token);
 
-				if (s == NULL) {
-					G_VINUM_DEBUG(0, "failed subdisk");
-					break;
-				}
+			if (s == NULL) {
+				G_VINUM_DEBUG(0, "config parse failed subdisk");
+				break;
+			}
 
-				if (merge) {
-					s2 = gv_find_sd(sc, s->name);
-					if (s2 != NULL) {
-						g_free(s);
-						continue;
-					}
+			s2 = gv_find_sd(sc, s->name);
+			if (s2 != NULL) {
+				/* XXX */
+				if (is_newer) {
+					s2->state = s->state;
+					G_VINUM_DEBUG(2, "newer subdisk found!");
 				}
-
-				s->vinumconf = sc;
-				LIST_INSERT_HEAD(&sc->subdisks, s, sd);
+				g_free(s);
+				continue;
 			}
+
+			/*
+			 * Signal that this subdisk was tasted, and could
+			 * possibly reference a drive that isn't in our config
+			 * yet.
+			 */
+			s->flags |= GV_SD_TASTED;
+
+			if (s->state == GV_SD_UP)
+				s->flags |= GV_SD_CANGOUP;
+
+			error = gv_create_sd(sc, s);
+			if (error)
+				continue;
+
+			/*
+			 * This flag was set in gv_create_sd() and is not
+			 * needed here (on-disk config parsing).
+			 */
+			s->flags &= ~GV_SD_NEWBORN;
+			s->flags &= ~GV_SD_GROW;
 		}
 	}
 }
@@ -183,8 +184,6 @@ gv_format_config(struct gv_softc *sc, struct sbuf *sb, int ondisk, char *prefix)
 	struct gv_plex *p;
 	struct gv_volume *v;
 
-	g_topology_assert();
-
 	/*
 	 * We don't need the drive configuration if we're not writing the
 	 * config to disk.
@@ -233,17 +232,20 @@ gv_format_config(struct gv_softc *sc, struct sbuf *sb, int ondisk, char *prefix)
 			sbuf_printf(sb, " state %s", gv_sdstate(s->state));
 		sbuf_printf(sb, "\n");
 	}
-
-	return;
 }
 
 static off_t
-gv_plex_smallest_sd(struct gv_plex *p, off_t smallest)
+gv_plex_smallest_sd(struct gv_plex *p)
 {
 	struct gv_sd *s;
+	off_t smallest;
 
 	KASSERT(p != NULL, ("gv_plex_smallest_sd: NULL p"));
 
+	s = LIST_FIRST(&p->subdisks);
+	if (s == NULL)
+		return (-1);
+	smallest = s->size;
 	LIST_FOREACH(s, &p->subdisks, in_plex) {
 		if (s->size < smallest)
 			smallest = s->size;
@@ -251,12 +253,29 @@ gv_plex_smallest_sd(struct gv_plex *p, off_t smallest)
 	return (smallest);
 }
 
+/* Walk over plexes in a volume and count how many are down. */
 int
-gv_sd_to_plex(struct gv_plex *p, struct gv_sd *s, int check)
+gv_plexdown(struct gv_volume *v)
 {
-	struct gv_sd *s2;
+	int plexdown;
+	struct gv_plex *p;
 
-	g_topology_assert();
+	KASSERT(v != NULL, ("gv_plexdown: NULL v"));
+
+	plexdown = 0;
+
+	LIST_FOREACH(p, &v->plexes, plex) {
+		if (p->state == GV_PLEX_DOWN)
+			plexdown++;
+	}
+	return (plexdown);
+}
+
+int
+gv_sd_to_plex(struct gv_sd *s, struct gv_plex *p)
+{
+	struct gv_sd *s2;
+	off_t psizeorig, remainder, smallest;
 
 	/* If this subdisk was already given to this plex, do nothing. */
 	if (s->plex_sc == p)
@@ -264,15 +283,56 @@ gv_sd_to_plex(struct gv_plex *p, struct gv_sd *s, int check)
 
 	/* Check correct size of this subdisk. */
 	s2 = LIST_FIRST(&p->subdisks);
-	if (s2 != NULL && gv_is_striped(p) && (s2->size != s->size)) {
-		G_VINUM_DEBUG(0, "need equal sized subdisks for "
-		    "this plex organisation - %s (%jd) <-> %s (%jd)",
-		    s2->name, s2->size, s->name, s->size);
-		return (-1);
+	/* Adjust the subdisk-size if necessary. */
+	if (s2 != NULL && gv_is_striped(p)) {
+		/* First adjust to the stripesize. */
+		remainder = s->size % p->stripesize;
+
+		if (remainder) {
+			G_VINUM_DEBUG(1, "size of sd %s is not a "
+			    "multiple of plex stripesize, taking off "
+			    "%jd bytes", s->name,
+			    (intmax_t)remainder);
+			gv_adjust_freespace(s, remainder);
+		}
+
+		smallest = gv_plex_smallest_sd(p);
+		/* Then take off extra if other subdisks are smaller. */
+		remainder = s->size - smallest;
+
+		/*
+		 * Don't allow a remainder below zero for running plexes, it's too
+		 * painful, and if someone were to accidentally do this, the
+		 * resulting array might be smaller than the original... not god 
+		 */
+		if (remainder < 0) {
+			if (!(p->flags & GV_PLEX_NEWBORN)) {
+				G_VINUM_DEBUG(0, "sd %s too small for plex %s!",
+				    s->name, p->name);
+				return (GV_ERR_BADSIZE);
+			}
+			/* Adjust other subdisks. */
+			LIST_FOREACH(s2, &p->subdisks, in_plex) {
+				G_VINUM_DEBUG(1, "size of sd %s is to big, "
+				    "taking off %jd bytes", s->name,
+				    (intmax_t)remainder);
+				gv_adjust_freespace(s2, (remainder * -1));
+			}
+		} else if (remainder > 0) {
+			G_VINUM_DEBUG(1, "size of sd %s is to big, "
+			    "taking off %jd bytes", s->name,
+			    (intmax_t)remainder);
+			gv_adjust_freespace(s, remainder);
+		}
 	}
 
 	/* Find the correct plex offset for this subdisk, if needed. */
 	if (s->plex_offset == -1) {
+		/* 
+		 * First set it to 0 to catch the case where we had a detached
+		 * subdisk that didn't get any good offset.
+		 */
+		s->plex_offset = 0;
 		if (p->sdcount) {
 			LIST_FOREACH(s2, &p->subdisks, in_plex) {
 				if (gv_is_striped(p))
@@ -282,25 +342,7 @@ gv_sd_to_plex(struct gv_plex *p, struct gv_sd *s, int check)
 					s->plex_offset = s2->plex_offset +
 					    s2->size;
 			}
-		} else
-			s->plex_offset = 0;
-	}
-
-	p->sdcount++;
-
-	/* Adjust the size of our plex. */
-	switch (p->org) {
-	case GV_PLEX_CONCAT:
-	case GV_PLEX_STRIPED:
-		p->size += s->size;
-		break;
-
-	case GV_PLEX_RAID5:
-		p->size = (p->sdcount - 1) * gv_plex_smallest_sd(p, s->size);
-		break;
-
-	default:
-		break;
+		}
 	}
 
 	/* There are no subdisks for this plex yet, just insert it. */
@@ -321,6 +363,29 @@ gv_sd_to_plex(struct gv_plex *p, struct gv_sd *s, int check)
 	}
 
 	s->plex_sc = p;
+        /* Adjust the size of our plex. We check if the plex misses a subdisk,
+	 * so we don't make the plex smaller than it actually should be.
+	 */
+	psizeorig = p->size;
+	p->size = gv_plex_size(p);
+	/* Make sure the size is not changed. */
+	if (p->sddetached > 0) {
+		if (p->size < psizeorig) {
+			p->size = psizeorig;
+			/* We make sure wee need another subdisk. */
+			if (p->sddetached == 1)
+				p->sddetached++;
+		}
+		p->sddetached--;
+	} else {
+		if ((p->org == GV_PLEX_RAID5 ||
+		    p->org == GV_PLEX_STRIPED) &&
+		    !(p->flags & GV_PLEX_NEWBORN) && 
+		    p->state >= GV_PLEX_DEGRADED) {
+			s->flags |= GV_SD_GROW;
+		}
+		p->sdcount++;
+	}
 
 	return (0);
 }
@@ -328,21 +393,32 @@ gv_sd_to_plex(struct gv_plex *p, struct gv_sd *s, int check)
 void
 gv_update_vol_size(struct gv_volume *v, off_t size)
 {
-	struct g_geom *gp;
-	struct g_provider *pp;
-
 	if (v == NULL)
 		return;
+	if (v->provider != NULL) {
+		g_topology_lock();
+		v->provider->mediasize = size;
+		g_topology_unlock();
+	}
+	v->size = size;
+}
 
-	gp = v->geom;
-	if (gp == NULL)
-		return;
+/* Return how many subdisks that constitute the original plex. */
+int
+gv_sdcount(struct gv_plex *p, int growing)
+{
+	struct gv_sd *s;
+	int sdcount;
 
-	LIST_FOREACH(pp, &gp->provider, provider) {
-		pp->mediasize = size;
+	sdcount = p->sdcount;
+	if (growing) {
+		LIST_FOREACH(s, &p->subdisks, in_plex) {
+			if (s->flags & GV_SD_GROW)
+				sdcount--;
+		}
 	}
 
-	v->size = size;
+	return (sdcount);
 }
 
 /* Calculates the plex size. */
@@ -351,14 +427,13 @@ gv_plex_size(struct gv_plex *p)
 {
 	struct gv_sd *s;
 	off_t size;
+	int sdcount;
 
 	KASSERT(p != NULL, ("gv_plex_size: NULL p"));
 
-	if (p->sdcount == 0)
-		return (0);
-
 	/* Adjust the size of our plex. */
 	size = 0;
+	sdcount = gv_sdcount(p, 1);
 	switch (p->org) {
 	case GV_PLEX_CONCAT:
 		LIST_FOREACH(s, &p->subdisks, in_plex)
@@ -366,11 +441,11 @@ gv_plex_size(struct gv_plex *p)
 		break;
 	case GV_PLEX_STRIPED:
 		s = LIST_FIRST(&p->subdisks);
-		size = p->sdcount * s->size;
+		size = ((s != NULL) ? (sdcount * s->size) : 0);
 		break;
 	case GV_PLEX_RAID5:
 		s = LIST_FIRST(&p->subdisks);
-		size = (p->sdcount - 1) * s->size;
+		size = ((s != NULL) ? ((sdcount - 1) * s->size) : 0);
 		break;
 	}
 
@@ -391,7 +466,7 @@ gv_vol_size(struct gv_volume *v)
 		return (0);
 
 	minplexsize = p->size;
-	LIST_FOREACH(p, &v->plexes, plex) {
+	LIST_FOREACH(p, &v->plexes, in_volume) {
 		if (p->size < minplexsize) {
 			minplexsize = p->size;
 		}
@@ -408,12 +483,9 @@ gv_update_plex_config(struct gv_plex *p)
 
 	KASSERT(p != NULL, ("gv_update_plex_config: NULL p"));
 
-	/* This is what we want the plex to be. */
-	state = GV_PLEX_UP;
-
 	/* The plex was added to an already running volume. */
 	if (p->flags & GV_PLEX_ADDED)
-		state = GV_PLEX_DOWN;
+		gv_set_plex_state(p, GV_PLEX_DOWN, GV_SETSTATE_FORCE);
 
 	switch (p->org) {
 	case GV_PLEX_STRIPED:
@@ -430,7 +502,7 @@ gv_update_plex_config(struct gv_plex *p)
 
 	if (required_sds) {
 		if (p->sdcount < required_sds) {
-			state = GV_PLEX_DOWN;
+			gv_set_plex_state(p, GV_PLEX_DOWN, GV_SETSTATE_FORCE);
 		}
 
 		/*
@@ -442,12 +514,13 @@ gv_update_plex_config(struct gv_plex *p)
 				G_VINUM_DEBUG(0, "subdisk size mismatch %s"
 				    "(%jd) <> %s (%jd)", s->name, s->size,
 				    s2->name, s2->size);
-				state = GV_PLEX_DOWN;
+				gv_set_plex_state(p, GV_PLEX_DOWN,
+				    GV_SETSTATE_FORCE);
 			}
 		}
 
-		/* Trim subdisk sizes so that they match the stripe size. */
 		LIST_FOREACH(s, &p->subdisks, in_plex) {
+			/* Trim subdisk sizes to match the stripe size. */
 			remainder = s->size % p->stripesize;
 			if (remainder) {
 				G_VINUM_DEBUG(1, "size of sd %s is not a "
@@ -458,41 +531,32 @@ gv_update_plex_config(struct gv_plex *p)
 		}
 	}
 
-	/* Adjust the size of our plex. */
-	if (p->sdcount > 0) {
-		p->size = 0;
-		switch (p->org) {
-		case GV_PLEX_CONCAT:
-			LIST_FOREACH(s, &p->subdisks, in_plex)
-				p->size += s->size;
-			break;
-
-		case GV_PLEX_STRIPED:
-			s = LIST_FIRST(&p->subdisks);
-			p->size = p->sdcount * s->size;
-			break;
-		
-		case GV_PLEX_RAID5:
-			s = LIST_FIRST(&p->subdisks);
-			p->size = (p->sdcount - 1) * s->size;
-			break;
-		
-		default:
-			break;
-		}
-	}
-
+	p->size = gv_plex_size(p);
 	if (p->sdcount == 0)
-		state = GV_PLEX_DOWN;
-	else if ((p->flags & GV_PLEX_ADDED) ||
-	    ((p->org == GV_PLEX_RAID5) && (p->flags & GV_PLEX_NEWBORN))) {
+		gv_set_plex_state(p, GV_PLEX_DOWN, GV_SETSTATE_FORCE);
+	else if (p->org == GV_PLEX_RAID5 && p->flags & GV_PLEX_NEWBORN) {
+		LIST_FOREACH(s, &p->subdisks, in_plex)
+			gv_set_sd_state(s, GV_SD_UP, GV_SETSTATE_FORCE);
+		/* If added to a volume, we want the plex to be down. */
+		state = (p->flags & GV_PLEX_ADDED) ? GV_PLEX_DOWN : GV_PLEX_UP;
+		gv_set_plex_state(p, state, GV_SETSTATE_FORCE);
+		p->flags &= ~GV_PLEX_ADDED;
+	} else if (p->flags & GV_PLEX_ADDED) {
 		LIST_FOREACH(s, &p->subdisks, in_plex)
-			s->state = GV_SD_STALE;
+			gv_set_sd_state(s, GV_SD_STALE, GV_SETSTATE_FORCE);
+		gv_set_plex_state(p, GV_PLEX_DOWN, GV_SETSTATE_FORCE);
 		p->flags &= ~GV_PLEX_ADDED;
-		p->flags &= ~GV_PLEX_NEWBORN;
-		state = GV_PLEX_DOWN;
+	} else if (p->state == GV_PLEX_UP) {
+		LIST_FOREACH(s, &p->subdisks, in_plex) {
+			if (s->flags & GV_SD_GROW) {
+				gv_set_plex_state(p, GV_PLEX_GROWABLE,
+				    GV_SETSTATE_FORCE);
+				break;
+			}
+		}
 	}
-	p->state = state;
+	/* Our plex is grown up now. */
+	p->flags &= ~GV_PLEX_NEWBORN;
 }
 
 /*
@@ -500,76 +564,82 @@ gv_update_plex_config(struct gv_plex *p)
  * freelist.
  */
 int
-gv_sd_to_drive(struct gv_softc *sc, struct gv_drive *d, struct gv_sd *s,
-    char *errstr, int errlen)
+gv_sd_to_drive(struct gv_sd *s, struct gv_drive *d)
 {
 	struct gv_sd *s2;
 	struct gv_freelist *fl, *fl2;
 	off_t tmp;
 	int i;
 
-	g_topology_assert();
-
 	fl2 = NULL;
 
-	KASSERT(sc != NULL, ("gv_sd_to_drive: NULL softc"));
-	KASSERT(d != NULL, ("gv_sd_to_drive: NULL drive"));
-	KASSERT(s != NULL, ("gv_sd_to_drive: NULL subdisk"));
-	KASSERT(errstr != NULL, ("gv_sd_to_drive: NULL errstr"));
-	KASSERT(errlen >= ERRBUFSIZ, ("gv_sd_to_drive: short errlen (%d)",
-	    errlen));
+	/* Shortcut for "referenced" drives. */
+	if (d->flags & GV_DRIVE_REFERENCED) {
+		s->drive_sc = d;
+		return (0);
+	}
 
 	/* Check if this subdisk was already given to this drive. */
-	if (s->drive_sc == d)
-		return (0);
+	if (s->drive_sc != NULL) {
+		if (s->drive_sc == d) {
+			if (!(s->flags & GV_SD_TASTED)) {
+				return (0);
+			}
+		} else {
+			G_VINUM_DEBUG(0, "can't give sd '%s' to '%s' "
+			    "(already on '%s')", s->name, d->name,
+			    s->drive_sc->name);
+			return (GV_ERR_ISATTACHED);
+		}
+	}
 
 	/* Preliminary checks. */
-	if (s->size > d->avail || d->freelist_entries == 0) {
-		snprintf(errstr, errlen, "not enough space on '%s' for '%s'",
-		    d->name, s->name);
-		return (-1);
+	if ((s->size > d->avail) || (d->freelist_entries == 0)) {
+		G_VINUM_DEBUG(0, "not enough space on '%s' for '%s'", d->name,
+		    s->name);
+		return (GV_ERR_NOSPACE);
 	}
 
-	/* No size given, autosize it. */
+	/* If no size was given for this subdisk, try to auto-size it... */
 	if (s->size == -1) {
 		/* Find the largest available slot. */
 		LIST_FOREACH(fl, &d->freelist, freelist) {
-			if (fl->size >= s->size) {
-				s->size = fl->size;
-				s->drive_offset = fl->offset;
-				fl2 = fl;
-			}
+			if (fl->size < s->size)
+				continue;
+			s->size = fl->size;
+			s->drive_offset = fl->offset;
+			fl2 = fl;
 		}
 
 		/* No good slot found? */
 		if (s->size == -1) {
-			snprintf(errstr, errlen, "could not autosize '%s' on "
-			    "'%s'", s->name, d->name);
-			return (-1);
+			G_VINUM_DEBUG(0, "couldn't autosize '%s' on '%s'",
+			    s->name, d->name);
+			return (GV_ERR_BADSIZE);
 		}
 
 	/*
-	 * Check if we have a free slot that's large enough for the given size.
+	 * ... or check if we have a free slot that's large enough for the
+	 * given size.
 	 */
 	} else {
 		i = 0;
 		LIST_FOREACH(fl, &d->freelist, freelist) {
-			/* Yes, this subdisk fits. */
-			if (fl->size >= s->size) {
-				i++;
-				/* Assign drive offset, if not given. */
-				if (s->drive_offset == -1)
-					s->drive_offset = fl->offset;
-				fl2 = fl;
-				break;
-			}
+			if (fl->size < s->size)
+				continue;
+			/* Assign drive offset, if not given. */
+			if (s->drive_offset == -1)
+				s->drive_offset = fl->offset;
+			fl2 = fl;
+			i++;
+			break;
 		}
 
 		/* Couldn't find a good free slot. */
 		if (i == 0) {
-			snprintf(errstr, errlen, "free slots to small for '%s' "
-			    "on '%s'", s->name, d->name);
-			return (-1);
+			G_VINUM_DEBUG(0, "free slots to small for '%s' on '%s'",
+			    s->name, d->name);
+			return (GV_ERR_NOSPACE);
 		}
 	}
 
@@ -604,9 +674,9 @@ gv_sd_to_drive(struct gv_softc *sc, struct gv_drive *d, struct gv_sd *s,
 
 		/* Couldn't find a good free slot. */
 		if (i == 0) {
-			snprintf(errstr, errlen, "given drive_offset for '%s' "
-			    "won't fit on '%s'", s->name, d->name);
-			return (-1);
+			G_VINUM_DEBUG(0, "given drive_offset for '%s' won't fit "
+			    "on '%s'", s->name, d->name);
+			return (GV_ERR_NOSPACE);
 		}
 	}
 
@@ -617,49 +687,41 @@ gv_sd_to_drive(struct gv_softc *sc, struct gv_drive *d, struct gv_sd *s,
 
 	/* First, adjust the freelist. */
 	LIST_FOREACH(fl, &d->freelist, freelist) {
+		/* Look for the free slot that we have found before. */
+		if (fl != fl2)
+			continue;
+
+		/* The subdisk starts at the beginning of the free slot. */
+		if (fl->offset == s->drive_offset) {
+			fl->offset += s->size;
+			fl->size -= s->size;
+
+			/* The subdisk uses the whole slot, so remove it. */
+			if (fl->size == 0) {
+				d->freelist_entries--;
+				LIST_REMOVE(fl, freelist);
+			}
+		/*
+		 * The subdisk does not start at the beginning of the free
+		 * slot.
+		 */
+		} else {
+			tmp = fl->offset + fl->size;
+			fl->size = s->drive_offset - fl->offset;
 
-		/* This is the free slot that we have found before. */
-		if (fl == fl2) {
-	
-			/*
-			 * The subdisk starts at the beginning of the free
-			 * slot.
-			 */
-			if (fl->offset == s->drive_offset) {
-				fl->offset += s->size;
-				fl->size -= s->size;
-
-				/*
-				 * The subdisk uses the whole slot, so remove
-				 * it.
-				 */
-				if (fl->size == 0) {
-					d->freelist_entries--;
-					LIST_REMOVE(fl, freelist);
-				}
 			/*
-			 * The subdisk does not start at the beginning of the
-			 * free slot.
+			 * The subdisk didn't use the complete rest of the free
+			 * slot, so we need to split it.
 			 */
-			} else {
-				tmp = fl->offset + fl->size;
-				fl->size = s->drive_offset - fl->offset;
-
-				/*
-				 * The subdisk didn't use the complete rest of
-				 * the free slot, so we need to split it.
-				 */
-				if (s->drive_offset + s->size != tmp) {
-					fl2 = g_malloc(sizeof(*fl2),
-					    M_WAITOK | M_ZERO);
-					fl2->offset = s->drive_offset + s->size;
-					fl2->size = tmp - fl2->offset;
-					LIST_INSERT_AFTER(fl, fl2, freelist);
-					d->freelist_entries++;
-				}
+			if (s->drive_offset + s->size != tmp) {
+				fl2 = g_malloc(sizeof(*fl2), M_WAITOK | M_ZERO);
+				fl2->offset = s->drive_offset + s->size;
+				fl2->size = tmp - fl2->offset;
+				LIST_INSERT_AFTER(fl, fl2, freelist);
+				d->freelist_entries++;
 			}
-			break;
 		}
+		break;
 	}
 
 	/*
@@ -685,6 +747,8 @@ gv_sd_to_drive(struct gv_softc *sc, struct gv_drive *d, struct gv_sd *s,
 	d->sdcount++;
 	d->avail -= s->size;
 
+	s->flags &= ~GV_SD_TASTED;
+
 	/* Link back from the subdisk to this drive. */
 	s->drive_sc = d;
 
@@ -869,17 +933,65 @@ gv_find_drive(struct gv_softc *sc, char *name)
 	return (NULL);
 }
 
+/* Find a drive given a device. */
+struct gv_drive *
+gv_find_drive_device(struct gv_softc *sc, char *device)
+{
+	struct gv_drive *d;
+
+	LIST_FOREACH(d, &sc->drives, drive) {
+		if(!strcmp(d->device, device))
+			return (d);
+	}
+
+	return (NULL);
+}
+
 /* Check if any consumer of the given geom is open. */
 int
-gv_is_open(struct g_geom *gp)
+gv_consumer_is_open(struct g_consumer *cp)
 {
-	struct g_consumer *cp;
+	if (cp == NULL)
+		return (0);
 
-	if (gp == NULL)
+	if (cp->acr || cp->acw || cp->ace)
+		return (1);
+
+	return (0);
+}
+
+int
+gv_provider_is_open(struct g_provider *pp)
+{
+	if (pp == NULL)
 		return (0);
 
-	LIST_FOREACH(cp, &gp->consumer, consumer) {
-		if (cp->acr || cp->acw || cp->ace)
+	if (pp->acr || pp->acw || pp->ace)
+		return (1);
+
+	return (0);
+}
+
+/*
+ * Compare the modification dates of the drives.
+ * Return 1 if a > b, 0 otherwise.
+ */
+int
+gv_drive_is_newer(struct gv_softc *sc, struct gv_drive *d)
+{
+	struct gv_drive *d2;
+	struct timeval *a, *b;
+
+	KASSERT(!LIST_EMPTY(&sc->drives),
+	    ("gv_is_drive_newer: empty drive list"));
+
+	a = &d->hdr->label.last_update;
+	LIST_FOREACH(d2, &sc->drives, drive) {
+		if ((d == d2) || (d2->state != GV_DRIVE_UP) ||
+		    (d2->hdr == NULL))
+			continue;
+		b = &d2->hdr->label.last_update;
+		if (timevalcmp(a, b, >))
 			return (1);
 	}
 
@@ -915,58 +1027,255 @@ gv_object_type(struct gv_softc *sc, char *name)
 			return (GV_TYPE_DRIVE);
 	}
 
-	return (-1);
+	return (GV_ERR_NOTFOUND);
 }
 
 void
-gv_kill_drive_thread(struct gv_drive *d)
+gv_setup_objects(struct gv_softc *sc)
 {
-	if (d->flags & GV_DRIVE_THREAD_ACTIVE) {
-		d->flags |= GV_DRIVE_THREAD_DIE;
-		wakeup(d);
-		while (!(d->flags & GV_DRIVE_THREAD_DEAD))
-			tsleep(d, PRIBIO, "gv_die", hz);
-		d->flags &= ~GV_DRIVE_THREAD_ACTIVE;
-		d->flags &= ~GV_DRIVE_THREAD_DIE;
-		d->flags &= ~GV_DRIVE_THREAD_DEAD;
-		g_free(d->bqueue);
-		d->bqueue = NULL;
-		mtx_destroy(&d->bqueue_mtx);
+	struct g_provider *pp;
+	struct gv_volume *v;
+	struct gv_plex *p;
+	struct gv_sd *s;
+	struct gv_drive *d;
+
+	LIST_FOREACH(s, &sc->subdisks, sd) {
+		d = gv_find_drive(sc, s->drive);
+		if (d != NULL)
+			gv_sd_to_drive(s, d);
+		p = gv_find_plex(sc, s->plex);
+		if (p != NULL)
+			gv_sd_to_plex(s, p);
+		gv_update_sd_state(s);
+	}
+
+	LIST_FOREACH(p, &sc->plexes, plex) {
+		gv_update_plex_config(p);
+		v = gv_find_vol(sc, p->volume);
+		if (v != NULL && p->vol_sc != v) {
+			p->vol_sc = v;
+			v->plexcount++;
+			LIST_INSERT_HEAD(&v->plexes, p, in_volume);
+		}
+		gv_update_plex_config(p);
+	}
+
+	LIST_FOREACH(v, &sc->volumes, volume) {
+		v->size = gv_vol_size(v);
+		if (v->provider == NULL) {
+			g_topology_lock();
+			pp = g_new_providerf(sc->geom, "gvinum/%s", v->name);
+			pp->mediasize = v->size;
+			pp->sectorsize = 512;    /* XXX */
+			g_error_provider(pp, 0);
+			v->provider = pp;
+			pp->private = v;
+			g_topology_unlock();
+		} else if (v->provider->mediasize != v->size) {
+			g_topology_lock();
+			v->provider->mediasize = v->size;
+			g_topology_unlock();
+		}
+		v->flags &= ~GV_VOL_NEWBORN;
+		gv_update_vol_state(v);
 	}
 }
 
 void
-gv_kill_plex_thread(struct gv_plex *p)
+gv_cleanup(struct gv_softc *sc)
 {
-	if (p->flags & GV_PLEX_THREAD_ACTIVE) {
-		p->flags |= GV_PLEX_THREAD_DIE;
-		wakeup(p);
-		while (!(p->flags & GV_PLEX_THREAD_DEAD))
-			tsleep(p, PRIBIO, "gv_die", hz);
-		p->flags &= ~GV_PLEX_THREAD_ACTIVE;
-		p->flags &= ~GV_PLEX_THREAD_DIE;
-		p->flags &= ~GV_PLEX_THREAD_DEAD;
+	struct gv_volume *v, *v2;
+	struct gv_plex *p, *p2;
+	struct gv_sd *s, *s2;
+	struct gv_drive *d, *d2;
+	struct gv_freelist *fl, *fl2;
+
+	mtx_lock(&sc->config_mtx);
+	LIST_FOREACH_SAFE(v, &sc->volumes, volume, v2) {
+		LIST_REMOVE(v, volume);
+		g_free(v->wqueue);
+		g_free(v);
+	}
+	LIST_FOREACH_SAFE(p, &sc->plexes, plex, p2) {
+		LIST_REMOVE(p, plex);
 		g_free(p->bqueue);
+		g_free(p->rqueue);
 		g_free(p->wqueue);
-		p->bqueue = NULL;
-		p->wqueue = NULL;
-		mtx_destroy(&p->bqueue_mtx);
+		g_free(p);
+	}
+	LIST_FOREACH_SAFE(s, &sc->subdisks, sd, s2) {
+		LIST_REMOVE(s, sd);
+		g_free(s);
 	}
+	LIST_FOREACH_SAFE(d, &sc->drives, drive, d2) {
+		LIST_FOREACH_SAFE(fl, &d->freelist, freelist, fl2) {
+			LIST_REMOVE(fl, freelist);
+			g_free(fl);
+		}
+		LIST_REMOVE(d, drive);
+		g_free(d->hdr);
+		g_free(d);
+	}
+	mtx_destroy(&sc->config_mtx);
 }
 
-void
-gv_kill_vol_thread(struct gv_volume *v)
+/* General 'attach' routine. */
+int
+gv_attach_plex(struct gv_plex *p, struct gv_volume *v, int rename)
+{
+	struct gv_sd *s;
+	struct gv_softc *sc;
+
+	g_topology_assert();
+
+	sc = p->vinumconf;
+	KASSERT(sc != NULL, ("NULL sc"));
+
+	if (p->vol_sc != NULL) {
+		G_VINUM_DEBUG(1, "unable to attach %s: already attached to %s",
+		    p->name, p->volume);
+		return (GV_ERR_ISATTACHED);
+	}
+
+	/* Stale all subdisks of this plex. */
+	LIST_FOREACH(s, &p->subdisks, in_plex) {
+		if (s->state != GV_SD_STALE)
+			gv_set_sd_state(s, GV_SD_STALE, GV_SETSTATE_FORCE);
+	}
+	/* Attach to volume. Make sure volume is not up and running. */
+	if (gv_provider_is_open(v->provider)) {
+		G_VINUM_DEBUG(1, "unable to attach %s: volume %s is busy",
+		    p->name, v->name);
+		return (GV_ERR_ISBUSY);
+	}
+	p->vol_sc = v;
+	strlcpy(p->volume, v->name, sizeof(p->volume));
+	v->plexcount++;
+	if (rename) {
+		snprintf(p->name, sizeof(p->name), "%s.p%d", v->name,
+		    v->plexcount);
+	}
+	LIST_INSERT_HEAD(&v->plexes, p, in_volume);
+
+	/* Get plex up again. */
+	gv_update_vol_size(v, gv_vol_size(v));
+	gv_set_plex_state(p, GV_PLEX_UP, 0);
+	gv_save_config(p->vinumconf);
+	return (0);
+}
+
+int
+gv_attach_sd(struct gv_sd *s, struct gv_plex *p, off_t offset, int rename)
+{
+	struct gv_sd *s2;
+	int error, sdcount;
+
+	g_topology_assert();
+
+	/* If subdisk is attached, don't do it. */
+	if (s->plex_sc != NULL) {
+		G_VINUM_DEBUG(1, "unable to attach %s: already attached to %s",
+		    s->name, s->plex);
+		return (GV_ERR_ISATTACHED);
+	}
+
+	gv_set_sd_state(s, GV_SD_STALE, GV_SETSTATE_FORCE);
+	/* First check that this subdisk has a correct offset. If none other
+	 * starts at the same, and it's correct module stripesize, it is */
+	if (offset != -1 && offset % p->stripesize != 0)
+		return (GV_ERR_BADOFFSET);
+	LIST_FOREACH(s2, &p->subdisks, in_plex) {
+		if (s2->plex_offset == offset)
+			return (GV_ERR_BADOFFSET);
+	}
+
+	/* Attach the subdisk to the plex at given offset. */
+	s->plex_offset = offset;
+	strlcpy(s->plex, p->name, sizeof(s->plex));
+
+	sdcount = p->sdcount;
+	error = gv_sd_to_plex(s, p);
+	if (error)
+		return (error);
+	gv_update_plex_config(p);
+
+	if (rename) {
+		snprintf(s->name, sizeof(s->name), "%s.s%d", s->plex,
+		    p->sdcount);
+	}
+	if (p->vol_sc != NULL)
+		gv_update_vol_size(p->vol_sc, gv_vol_size(p->vol_sc));
+	gv_save_config(p->vinumconf);
+	/* We don't update the subdisk state since the user might have to
+	 * initiate a rebuild/sync first. */
+	return (0);
+}
+
+/* Detach a plex from a volume. */
+int
+gv_detach_plex(struct gv_plex *p, int flags)
 {
-	if (v->flags & GV_VOL_THREAD_ACTIVE) {
-		v->flags |= GV_VOL_THREAD_DIE;
-		wakeup(v);
-		while (!(v->flags & GV_VOL_THREAD_DEAD))
-			tsleep(v, PRIBIO, "gv_die", hz);
-		v->flags &= ~GV_VOL_THREAD_ACTIVE;
-		v->flags &= ~GV_VOL_THREAD_DIE;
-		v->flags &= ~GV_VOL_THREAD_DEAD;
-		g_free(v->bqueue);
-		v->bqueue = NULL;
-		mtx_destroy(&v->bqueue_mtx);
+	struct gv_volume *v;
+
+	g_topology_assert();
+	v = p->vol_sc;
+
+	if (v == NULL) {
+		G_VINUM_DEBUG(1, "unable to detach %s: already detached",
+		    p->name);
+		return (0); /* Not an error. */
+	}
+
+	/*
+	 * Only proceed if forced or volume inactive.
+	 */
+	if (!(flags & GV_FLAG_F) && (gv_provider_is_open(v->provider) ||
+	    p->state == GV_PLEX_UP)) {
+		G_VINUM_DEBUG(1, "unable to detach %s: volume %s is busy",
+		    p->name, p->volume);
+		return (GV_ERR_ISBUSY);
 	}
+	v->plexcount--;
+	/* Make sure someone don't read us when gone. */
+	v->last_read_plex = NULL; 
+	LIST_REMOVE(p, in_volume);
+	p->vol_sc = NULL;
+	memset(p->volume, 0, GV_MAXVOLNAME);
+	gv_update_vol_size(v, gv_vol_size(v));
+	gv_save_config(p->vinumconf);
+	return (0);
+}
+
+/* Detach a subdisk from a plex. */
+int
+gv_detach_sd(struct gv_sd *s, int flags)
+{
+	struct gv_plex *p;
+
+	g_topology_assert();
+	p = s->plex_sc;
+
+	if (p == NULL) {
+		G_VINUM_DEBUG(1, "unable to detach %s: already detached",
+		    s->name);
+		return (0); /* Not an error. */
+	}
+
+	/*
+	 * Don't proceed if we're not forcing, and the plex is up, or degraded
+	 * with this subdisk up.
+	 */
+	if (!(flags & GV_FLAG_F) && ((p->state > GV_PLEX_DEGRADED) ||
+	    ((p->state == GV_PLEX_DEGRADED) && (s->state == GV_SD_UP)))) {
+	    	G_VINUM_DEBUG(1, "unable to detach %s: plex %s is busy",
+		    s->name, s->plex);
+		return (GV_ERR_ISBUSY);
+	}
+
+	LIST_REMOVE(s, in_plex);
+	s->plex_sc = NULL;
+	memset(s->plex, 0, GV_MAXPLEXNAME);
+	p->sddetached++;
+	gv_save_config(s->vinumconf);
+	return (0);
 }
diff --git a/sys/geom/vinum/geom_vinum_var.h b/sys/geom/vinum/geom_vinum_var.h
index 5a86568..124944f 100644
--- a/sys/geom/vinum/geom_vinum_var.h
+++ b/sys/geom/vinum/geom_vinum_var.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2004 Lukas Ertl
+ * Copyright (c) 2004, 2007 Lukas Ertl
  * Copyright (c) 1997, 1998, 1999
  *      Nan Yang Computer Services Limited.  All rights reserved.
  *
@@ -112,11 +112,29 @@
 #define	GV_BIO_MALLOC	0x02
 #define	GV_BIO_ONHOLD	0x04
 #define	GV_BIO_SYNCREQ	0x08
-#define	GV_BIO_SUCCEED	0x10
+#define	GV_BIO_INIT	0x10
 #define	GV_BIO_REBUILD	0x20
 #define	GV_BIO_CHECK	0x40
 #define	GV_BIO_PARITY	0x80
 #define	GV_BIO_RETRY	0x100
+#define GV_BIO_INTERNAL \
+    (GV_BIO_SYNCREQ | GV_BIO_INIT | GV_BIO_REBUILD |GV_BIO_CHECK)
+
+/* Error codes to be used within gvinum. */
+#define	GV_ERR_SETSTATE		(-1)	/* Error setting state. */
+#define	GV_ERR_BADSIZE		(-2)	/* Object has wrong size. */
+#define	GV_ERR_INVTYPE		(-3)    /* Invalid object type. */
+#define	GV_ERR_CREATE		(-4)	/* Error creating gvinum object. */
+#define	GV_ERR_ISBUSY		(-5)	/* Object is busy. */
+#define	GV_ERR_ISATTACHED	(-6)	/* Object is attached to another. */
+#define	GV_ERR_INVFLAG		(-7)	/* Invalid flag passed. */
+#define	GV_ERR_INVSTATE		(-8)	/* Invalid state. */
+#define	GV_ERR_NOTFOUND		(-9)	/* Object not found. */
+#define	GV_ERR_NAMETAKEN	(-10)	/* Object name is taken. */
+#define	GV_ERR_NOSPACE		(-11)	/* No space left on drive/subdisk. */
+#define	GV_ERR_BADOFFSET	(-12)	/* Invalid offset specified. */
+#define	GV_ERR_INVNAME		(-13)	/* Invalid object name. */
+#define	GV_ERR_PLEXORG		(-14)	/* Invalid plex organization. */
 
 /*
  * hostname is 256 bytes long, but we don't need to shlep multiple copies in
@@ -162,16 +180,65 @@ struct gv_bioq {
 	TAILQ_ENTRY(gv_bioq)	queue;
 };
 
+#define	GV_EVENT_DRIVE_TASTED		1
+#define	GV_EVENT_DRIVE_LOST		2
+#define	GV_EVENT_THREAD_EXIT		3
+#define	GV_EVENT_CREATE_DRIVE		4
+#define	GV_EVENT_CREATE_VOLUME		5
+#define	GV_EVENT_CREATE_PLEX		6
+#define	GV_EVENT_CREATE_SD		7
+#define	GV_EVENT_SAVE_CONFIG		8
+#define	GV_EVENT_RM_VOLUME		9
+#define	GV_EVENT_RM_PLEX		10
+#define	GV_EVENT_RM_SD			11
+#define	GV_EVENT_RM_DRIVE		12
+#define	GV_EVENT_SET_SD_STATE		13
+#define	GV_EVENT_SET_DRIVE_STATE	14
+#define	GV_EVENT_SET_VOL_STATE		15
+#define	GV_EVENT_SET_PLEX_STATE		16
+#define	GV_EVENT_RESET_CONFIG		17
+#define	GV_EVENT_PARITY_REBUILD		18
+#define	GV_EVENT_PARITY_CHECK		19
+#define	GV_EVENT_START_PLEX		20
+#define	GV_EVENT_START_VOLUME		21
+#define	GV_EVENT_ATTACH_PLEX		22
+#define	GV_EVENT_ATTACH_SD		23
+#define	GV_EVENT_DETACH_PLEX		24
+#define	GV_EVENT_DETACH_SD		25
+#define	GV_EVENT_RENAME_VOL		26
+#define	GV_EVENT_RENAME_PLEX		27
+#define	GV_EVENT_RENAME_SD		28
+#define	GV_EVENT_RENAME_DRIVE		29
+#define	GV_EVENT_MOVE_SD		30
+#define	GV_EVENT_SETUP_OBJECTS		31
+
+#ifdef _KERNEL
+struct gv_event {
+	int	type;
+	void	*arg1;
+	void	*arg2;
+	intmax_t arg3;
+	intmax_t arg4;
+	TAILQ_ENTRY(gv_event)	events;
+};
+#endif
+
 /* This struct contains the main vinum config. */
 struct gv_softc {
-	/*struct mtx config_mtx; XXX not yet */
-
 	/* Linked lists of all objects in our setup. */
 	LIST_HEAD(,gv_drive)	drives;		/* All drives. */
 	LIST_HEAD(,gv_plex)	plexes;		/* All plexes. */
 	LIST_HEAD(,gv_sd)	subdisks;	/* All subdisks. */
 	LIST_HEAD(,gv_volume)	volumes;	/* All volumes. */
 
+	TAILQ_HEAD(,gv_event)	equeue;		/* Event queue. */
+	struct mtx		queue_mtx;	/* Queue lock. */
+	struct mtx		config_mtx;	/* Configuration lock. */
+#ifdef	_KERNEL
+	struct bio_queue_head	*bqueue;	/* BIO queue. */
+#else
+	char			*padding;
+#endif
 	struct g_geom		*geom;		/* Pointer to our VINUM geom. */
 };
 
@@ -188,26 +255,19 @@ struct gv_drive {
 	int	sdcount;			/* Number of subdisks. */
 
 	int	flags;
-#define	GV_DRIVE_THREAD_ACTIVE	0x01	/* Drive has an active worker thread. */
-#define	GV_DRIVE_THREAD_DIE	0x02	/* Signal the worker thread to die. */
-#define	GV_DRIVE_THREAD_DEAD	0x04	/* The worker thread has died. */
-#define	GV_DRIVE_NEWBORN	0x08	/* The drive was just created. */
+#define	GV_DRIVE_REFERENCED	0x01	/* The drive isn't really existing,
+					   but was referenced by a subdisk
+					   during taste. */
+
+	struct gv_hdr	*hdr;		/* The drive header. */
 
-	struct gv_hdr	*hdr;			/* The drive header. */
+	struct g_consumer *consumer;	/* Consumer attached to this drive. */
 
 	int freelist_entries;			/* Count of freelist entries. */
 	LIST_HEAD(,gv_freelist)	freelist;	/* List of freelist entries. */
 	LIST_HEAD(,gv_sd)	subdisks;	/* Subdisks on this drive. */
 	LIST_ENTRY(gv_drive)	drive;		/* Entry in the vinum config. */
 
-#ifdef _KERNEL
-	struct bio_queue_head	*bqueue;	/* BIO queue of this drive. */
-#else
-	char			*padding;
-#endif
-	struct mtx		bqueue_mtx;	/* Mtx. to protect the queue. */
-
-	struct g_geom	*geom;			/* The geom of this drive. */
 	struct gv_softc	*vinumconf;		/* Pointer to the vinum conf. */
 };
 
@@ -230,8 +290,10 @@ struct gv_sd {
 	int	init_error;		/* Flag error on initialization. */
 
 	int	flags;
-#define	GV_SD_NEWBORN		0x01	/* Subdisk was just created. */
-#define	GV_SD_INITCANCEL	0x02	/* Cancel initialization process. */
+#define	GV_SD_NEWBORN		0x01	/* Subdisk is created by user. */
+#define	GV_SD_TASTED		0x02	/* Subdisk is created during taste. */
+#define	GV_SD_CANGOUP		0x04	/* Subdisk can go up immediately. */
+#define GV_SD_GROW		0x08	/* Subdisk is added to striped plex. */
 
 	char drive[GV_MAXDRIVENAME];	/* Name of underlying drive. */
 	char plex[GV_MAXPLEXNAME];	/* Name of associated plex. */
@@ -239,9 +301,6 @@ struct gv_sd {
 	struct gv_drive	*drive_sc;	/* Pointer to underlying drive. */
 	struct gv_plex	*plex_sc;	/* Pointer to associated plex. */
 
-	struct g_provider *provider;	/* The provider this sd represents. */
-	struct g_consumer *consumer;	/* Consumer attached to our provider. */
-
 	LIST_ENTRY(gv_sd) from_drive;	/* Subdisk list of underlying drive. */
 	LIST_ENTRY(gv_sd) in_plex;	/* Subdisk list of associated plex. */
 	LIST_ENTRY(gv_sd) sd;		/* Entry in the vinum config. */
@@ -257,7 +316,8 @@ struct gv_plex {
 #define	GV_PLEX_DOWN		0
 #define	GV_PLEX_INITIALIZING	1
 #define	GV_PLEX_DEGRADED	2
-#define	GV_PLEX_UP		3
+#define GV_PLEX_GROWABLE	3
+#define	GV_PLEX_UP		4
 
 	int	org;			/* The plex organisation. */
 #define	GV_PLEX_DISORG	0
@@ -270,6 +330,7 @@ struct gv_plex {
 	char	volume[GV_MAXVOLNAME];	/* Name of associated volume. */
 	struct gv_volume *vol_sc;	/* Pointer to associated volume. */
 
+	int	sddetached;		/* Number of detached subdisks. */
 	int	sdcount;		/* Number of subdisks in this plex. */
 	int	sddown;			/* Number of subdisks that are down. */
 	int	flags;
@@ -279,26 +340,25 @@ struct gv_plex {
 #define	GV_PLEX_THREAD_DIE	0x08	/* Signal the RAID5 thread to die. */
 #define	GV_PLEX_THREAD_DEAD	0x10	/* The RAID5 thread has died. */
 #define	GV_PLEX_NEWBORN		0x20	/* The plex was just created. */
+#define GV_PLEX_REBUILDING	0x40	/* The plex is rebuilding. */
+#define GV_PLEX_GROWING		0x80	/* The plex is growing. */
 
 	off_t	synced;			/* Count of synced bytes. */
 
-	struct mtx		bqueue_mtx; /* Lock for the BIO queue. */
-#ifdef _KERNEL
-	struct bio_queue_head	*bqueue; /* BIO queue. */
-	struct bio_queue_head	*wqueue; /* Waiting BIO queue. */
-#else
-	char			*bpad, *wpad;
-#endif
 	TAILQ_HEAD(,gv_raid5_packet)	packets; /* RAID5 sub-requests. */
 
 	LIST_HEAD(,gv_sd)   subdisks;	/* List of attached subdisks. */
 	LIST_ENTRY(gv_plex) in_volume;	/* Plex list of associated volume. */
 	LIST_ENTRY(gv_plex) plex;	/* Entry in the vinum config. */
 
-	struct g_provider *provider;	/* The provider this plex represents. */
-	struct g_consumer *consumer;	/* Consumer attached to our provider. */
+#ifdef	_KERNEL
+	struct bio_queue_head	*bqueue;	/* BIO queue. */
+	struct bio_queue_head	*wqueue;	/* Waiting BIO queue. */
+	struct bio_queue_head	*rqueue;	/* Rebuild waiting BIO queue. */
+#else
+	char			*bpad, *wpad, *rpad; /* Padding for userland. */
+#endif
 
-	struct g_geom	*geom;		/* The geom of this plex. */
 	struct gv_softc	*vinumconf;	/* Pointer to the vinum config. */
 };
 
@@ -315,19 +375,20 @@ struct gv_volume {
 #define	GV_VOL_THREAD_ACTIVE	0x01	/* Volume has an active thread. */
 #define	GV_VOL_THREAD_DIE	0x02	/* Signal the thread to die. */
 #define	GV_VOL_THREAD_DEAD	0x04	/* The thread has died. */
+#define GV_VOL_NEWBORN		0x08	/* The volume was just created. */
 
-	struct mtx		bqueue_mtx; /* Lock for the BIO queue. */
-#ifdef _KERNEL
-	struct bio_queue_head	*bqueue; /* BIO queue. */
+	LIST_HEAD(,gv_plex)	plexes;		/* List of attached plexes. */
+	LIST_ENTRY(gv_volume)	volume;		/* Entry in vinum config. */
+
+	struct g_provider	*provider;	/* Provider of this volume. */
+
+#ifdef	_KERNEL
+	struct bio_queue_head	*wqueue;	/* BIO delayed request queue. */
 #else
-	char			*padding;
+	char			*wpad; /* Padding for userland. */
 #endif
 
-	LIST_HEAD(,gv_plex)   plexes;	/* List of attached plexes. */
-	LIST_ENTRY(gv_volume) volume;	/* Entry in vinum config. */
-
 	struct gv_plex	*last_read_plex;
-	struct g_geom	*geom;		/* The geom of this volume. */
 	struct gv_softc	*vinumconf;	/* Pointer to the vinum config. */
 };
 
diff --git a/sys/geom/vinum/geom_vinum_volume.c b/sys/geom/vinum/geom_vinum_volume.c
index 88face2..32b353b 100644
--- a/sys/geom/vinum/geom_vinum_volume.c
+++ b/sys/geom/vinum/geom_vinum_volume.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2004 Lukas Ertl
+ * Copyright (c) 2007 Lukas Ertl
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -29,416 +29,136 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/bio.h>
-#include <sys/conf.h>
-#include <sys/kernel.h>
-#include <sys/kthread.h>
-#include <sys/libkern.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
-#include <sys/module.h>
-#include <sys/mutex.h>
 #include <sys/systm.h>
 
 #include <geom/geom.h>
 #include <geom/vinum/geom_vinum_var.h>
 #include <geom/vinum/geom_vinum.h>
 
-static void gv_vol_completed_request(struct gv_volume *, struct bio *);
-static void gv_vol_normal_request(struct gv_volume *, struct bio *);
-
-static void
-gv_volume_orphan(struct g_consumer *cp)
-{
-	struct g_geom *gp;
-	struct gv_volume *v;
-	int error;
-
-	g_topology_assert();
-	gp = cp->geom;
-	g_trace(G_T_TOPOLOGY, "gv_volume_orphan(%s)", gp->name);
-	if (cp->acr != 0 || cp->acw != 0 || cp->ace != 0)
-		g_access(cp, -cp->acr, -cp->acw, -cp->ace);
-	error = cp->provider->error;
-	if (error == 0)
-		error = ENXIO;
-	g_detach(cp);
-	g_destroy_consumer(cp);	
-	if (!LIST_EMPTY(&gp->consumer))
-		return;
-	v = gp->softc;
-	if (v != NULL) {
-		gv_kill_vol_thread(v);
-		v->geom = NULL;
-	}
-	gp->softc = NULL;
-	g_wither_geom(gp, error);
-}
-
-/* We end up here after the requests to our plexes are done. */
-static void
-gv_volume_done(struct bio *bp)
-{
-	struct gv_volume *v;
-
-	v = bp->bio_from->geom->softc;
-	bp->bio_cflags |= GV_BIO_DONE;
-	mtx_lock(&v->bqueue_mtx);
-	bioq_insert_tail(v->bqueue, bp);
-	wakeup(v);
-	mtx_unlock(&v->bqueue_mtx);
-}
-
-static void
-gv_volume_start(struct bio *bp)
-{
-	struct gv_volume *v;
-
-	switch(bp->bio_cmd) {
-	case BIO_READ:
-	case BIO_WRITE:
-	case BIO_DELETE:
-		break;
-	case BIO_GETATTR:
-	default:
-		g_io_deliver(bp, EOPNOTSUPP);
-		return;
-	}
-
-	v = bp->bio_to->geom->softc;
-	if (v->state != GV_VOL_UP) {
-		g_io_deliver(bp, ENXIO);
-		return;
-	}
-
-	mtx_lock(&v->bqueue_mtx);
-	bioq_disksort(v->bqueue, bp);
-	wakeup(v);
-	mtx_unlock(&v->bqueue_mtx);
-}
-
-static void
-gv_vol_worker(void *arg)
+void
+gv_volume_flush(struct gv_volume *v)
 {
+	struct gv_softc *sc;
 	struct bio *bp;
-	struct gv_volume *v;
 
-	v = arg;
 	KASSERT(v != NULL, ("NULL v"));
-	mtx_lock(&v->bqueue_mtx);
-	for (;;) {
-		/* We were signaled to exit. */
-		if (v->flags & GV_VOL_THREAD_DIE)
-			break;
-
-		/* Take the first BIO from our queue. */
-		bp = bioq_takefirst(v->bqueue);
-		if (bp == NULL) {
-			msleep(v, &v->bqueue_mtx, PRIBIO, "-", hz/10);
-			continue;
-		}
-		mtx_unlock(&v->bqueue_mtx);
+	sc = v->vinumconf;
+	KASSERT(sc != NULL, ("NULL sc"));
 
-		if (bp->bio_cflags & GV_BIO_DONE)
-			gv_vol_completed_request(v, bp);
-		else
-			gv_vol_normal_request(v, bp);
-
-		mtx_lock(&v->bqueue_mtx);
+	bp = bioq_takefirst(v->wqueue);
+	while (bp != NULL) {
+		gv_volume_start(sc, bp);
+		bp = bioq_takefirst(v->wqueue);
 	}
-	mtx_unlock(&v->bqueue_mtx);
-	v->flags |= GV_VOL_THREAD_DEAD;
-	wakeup(v);
-
-	kproc_exit(ENXIO);
 }
 
-static void
-gv_vol_completed_request(struct gv_volume *v, struct bio *bp)
+void
+gv_volume_start(struct gv_softc *sc, struct bio *bp)
 {
-	struct bio *pbp;
 	struct g_geom *gp;
-	struct g_consumer *cp, *cp2;
-
-	pbp = bp->bio_parent;
-
-	if (pbp->bio_error == 0)
-		pbp->bio_error = bp->bio_error;
-
-	switch (pbp->bio_cmd) {
-	case BIO_READ:
-		if (bp->bio_error == 0)
-			break;
-
-		if (pbp->bio_cflags & GV_BIO_RETRY)
-			break;
-
-		/* Check if we have another plex left. */
-		cp = bp->bio_from;
-		gp = cp->geom;
-		cp2 = LIST_NEXT(cp, consumer);
-		if (cp2 == NULL)
-			break;
-
-		if (LIST_NEXT(cp2, consumer) == NULL)
-			pbp->bio_cflags |= GV_BIO_RETRY;
+	struct gv_volume *v;
+	struct gv_plex *p, *lp;
+	int numwrites;
 
-		g_destroy_bio(bp);
-		pbp->bio_children--;
-		mtx_lock(&v->bqueue_mtx);
-		bioq_disksort(v->bqueue, pbp);
-		mtx_unlock(&v->bqueue_mtx);
+	gp = sc->geom;
+	v = bp->bio_to->private;
+	if (v == NULL || v->state != GV_VOL_UP) {
+		g_io_deliver(bp, ENXIO);
 		return;
-
-	case BIO_WRITE:
-	case BIO_DELETE:
-		/* Remember if this write request succeeded. */
-		if (bp->bio_error == 0)
-			pbp->bio_cflags |= GV_BIO_SUCCEED;
-		break;
 	}
 
-	/* When the original request is finished, we deliver it. */
-	pbp->bio_inbed++;
-	if (pbp->bio_inbed == pbp->bio_children) {
-		if (pbp->bio_cflags & GV_BIO_SUCCEED)
-			pbp->bio_error = 0;
-		pbp->bio_completed = bp->bio_length;
-		g_io_deliver(pbp, pbp->bio_error);
-	}
-
-	g_destroy_bio(bp);
-}
-
-static void
-gv_vol_normal_request(struct gv_volume *v, struct bio *bp)
-{
-	struct bio_queue_head queue;
-	struct g_geom *gp;
-	struct gv_plex *p, *lp;
-	struct bio *cbp;
-
-	gp = v->geom;
-
 	switch (bp->bio_cmd) {
 	case BIO_READ:
-		cbp = g_clone_bio(bp);
-		if (cbp == NULL) {
-			g_io_deliver(bp, ENOMEM);
-			return;
-		}
-		cbp->bio_done = gv_volume_done;
 		/*
-		 * Try to find a good plex where we can send the request to.
-		 * The plex either has to be up, or it's a degraded RAID5 plex.
+		 * Try to find a good plex where we can send the request to,
+		 * round-robin-style.  The plex either has to be up, or it's a
+		 * degraded RAID5 plex. Check if we have delayed requests. Put
+		 * this request on the delayed queue if so. This makes sure that
+		 * we don't read old values.
 		 */
+		if (bioq_first(v->wqueue) != NULL) {
+			bioq_insert_tail(v->wqueue, bp);
+			break;
+		}
 		lp = v->last_read_plex;
 		if (lp == NULL)
 			lp = LIST_FIRST(&v->plexes);
 		p = LIST_NEXT(lp, in_volume);
+		if (p == NULL)
+			p = LIST_FIRST(&v->plexes);
 		do {
-			if (p == NULL)
-				p = LIST_FIRST(&v->plexes);
+			if (p == NULL) {
+				p = lp;
+				break;
+			}
 			if ((p->state > GV_PLEX_DEGRADED) ||
 			    (p->state >= GV_PLEX_DEGRADED &&
 			    p->org == GV_PLEX_RAID5))
 				break;
 			p = LIST_NEXT(p, in_volume);
+			if (p == NULL)
+				p = LIST_FIRST(&v->plexes);
 		} while (p != lp);
 
-		if (p == NULL ||
+		if ((p == NULL) ||
 		    (p->org == GV_PLEX_RAID5 && p->state < GV_PLEX_DEGRADED) ||
 		    (p->org != GV_PLEX_RAID5 && p->state <= GV_PLEX_DEGRADED)) {
-			g_destroy_bio(cbp);
-			bp->bio_children--;
 			g_io_deliver(bp, ENXIO);
 			return;
 		}
-		g_io_request(cbp, p->consumer);
 		v->last_read_plex = p;
 
+		/* Hand it down to the plex logic. */
+		gv_plex_start(p, bp);
 		break;
 
 	case BIO_WRITE:
 	case BIO_DELETE:
-		bioq_init(&queue);
+		/* Delay write-requests if any plex is synchronizing. */
 		LIST_FOREACH(p, &v->plexes, in_volume) {
-			if (p->state < GV_PLEX_DEGRADED)
-				continue;
-			cbp = g_clone_bio(bp);
-			if (cbp == NULL) {
-				for (cbp = bioq_first(&queue); cbp != NULL;
-				    cbp = bioq_first(&queue)) {
-					bioq_remove(&queue, cbp);
-					g_destroy_bio(cbp);
-				}
-				if (bp->bio_error == 0)
-					bp->bio_error = ENOMEM;
-				g_io_deliver(bp, bp->bio_error);
+			if (p->flags & GV_PLEX_SYNCING) {
+				bioq_insert_tail(v->wqueue, bp);
 				return;
 			}
-			bioq_insert_tail(&queue, cbp);
-			cbp->bio_done = gv_volume_done;
-			cbp->bio_caller1 = p->consumer;
-		}
-		/* Fire off all sub-requests. */
-		for (cbp = bioq_first(&queue); cbp != NULL;
-		     cbp = bioq_first(&queue)) {
-			bioq_remove(&queue, cbp);
-			g_io_request(cbp, cbp->bio_caller1);
 		}
-		break;
-	}
-}
-
-static int
-gv_volume_access(struct g_provider *pp, int dr, int dw, int de)
-{
-	struct g_geom *gp;
-	struct g_consumer *cp, *cp2;
-	int error;
 
-	gp = pp->geom;
-
-	error = ENXIO;
-	LIST_FOREACH(cp, &gp->consumer, consumer) {
-		error = g_access(cp, dr, dw, de);
-		if (error) {
-			LIST_FOREACH(cp2, &gp->consumer, consumer) {
-				if (cp == cp2)
-					break;
-				g_access(cp2, -dr, -dw, -de);
-			}
-			return (error);
+		numwrites = 0;
+		/* Give the BIO to each plex of this volume. */
+		LIST_FOREACH(p, &v->plexes, in_volume) {
+			if (p->state < GV_PLEX_DEGRADED)
+				continue;
+			gv_plex_start(p, bp);
+			numwrites++;
 		}
+		if (numwrites == 0)
+			g_io_deliver(bp, ENXIO);
+		break;
 	}
-	return (error);
 }
 
-static struct g_geom *
-gv_volume_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
+void
+gv_bio_done(struct gv_softc *sc, struct bio *bp)
 {
-	struct g_geom *gp;
-	struct g_provider *pp2;
-	struct g_consumer *cp, *ocp;
-	struct gv_softc *sc;
 	struct gv_volume *v;
 	struct gv_plex *p;
-	int error, first;
-
-	g_trace(G_T_TOPOLOGY, "gv_volume_taste(%s, %s)", mp->name, pp->name);
-	g_topology_assert();
-
-	/* First, find the VINUM class and its associated geom. */
-	gp = find_vinum_geom();
-	if (gp == NULL)
-		return (NULL);
-
-	sc = gp->softc;
-	KASSERT(sc != NULL, ("gv_volume_taste: NULL sc"));
-
-	gp = pp->geom;
-
-	/* We only want to attach to plexes. */
-	if (strcmp(gp->class->name, "VINUMPLEX"))
-		return (NULL);
-
-	first = 0;
-	p = gp->softc;
-
-	/* Let's see if the volume this plex wants is already configured. */
-	v = gv_find_vol(sc, p->volume);
-	if (v == NULL)
-		return (NULL);
-	if (v->geom == NULL) {
-		gp = g_new_geomf(mp, "%s", p->volume);
-		gp->start = gv_volume_start;
-		gp->orphan = gv_volume_orphan;
-		gp->access = gv_volume_access;
-		gp->softc = v;
-		first++;
-	} else
-		gp = v->geom;
-
-	/* Create bio queue, queue mutex, and worker thread, if necessary. */
-	if (v->bqueue == NULL) {
-		v->bqueue = g_malloc(sizeof(struct bio_queue_head),
-		    M_WAITOK | M_ZERO);
-		bioq_init(v->bqueue);
-	}
-	if (mtx_initialized(&v->bqueue_mtx) == 0)
-		mtx_init(&v->bqueue_mtx, "gv_plex", NULL, MTX_DEF);
-
-	if (!(v->flags & GV_VOL_THREAD_ACTIVE)) {
-		kproc_create(gv_vol_worker, v, NULL, 0, 0, "gv_v %s",
-		    v->name);
-		v->flags |= GV_VOL_THREAD_ACTIVE;
-	}
-
-	/*
-	 * Create a new consumer and attach it to the plex geom.  Since this
-	 * volume might already have a plex attached, we need to adjust the
-	 * access counts of the new consumer.
-	 */
-	ocp = LIST_FIRST(&gp->consumer);
-	cp = g_new_consumer(gp);
-	g_attach(cp, pp);
-	if ((ocp != NULL) && (ocp->acr > 0 || ocp->acw > 0 || ocp->ace > 0)) {
-		error = g_access(cp, ocp->acr, ocp->acw, ocp->ace);
-		if (error) {
-			G_VINUM_DEBUG(0, "failed g_access %s -> %s; "
-			    "errno %d", v->name, p->name, error);
-			g_detach(cp);
-			g_destroy_consumer(cp);
-			if (first)
-				g_destroy_geom(gp);
-			return (NULL);
-		}
-	}
-
-	p->consumer = cp;
-
-	if (p->vol_sc != v) {
-		p->vol_sc = v;
-		v->plexcount++;
-		LIST_INSERT_HEAD(&v->plexes, p, in_volume);
-	}
-
-	/* We need to setup a new VINUMVOLUME geom. */
-	if (first) {
-		pp2 = g_new_providerf(gp, "gvinum/%s", v->name);
-		pp2->mediasize = pp->mediasize;
-		pp2->sectorsize = pp->sectorsize;
-		g_error_provider(pp2, 0);
-		v->size = pp2->mediasize;
-		v->geom = gp;
-		return (gp);
+	struct gv_sd *s;
+
+	s = bp->bio_caller1;
+	KASSERT(s != NULL, ("gv_bio_done: NULL s"));
+	p = s->plex_sc;
+	KASSERT(p != NULL, ("gv_bio_done: NULL p"));
+	v = p->vol_sc;
+	KASSERT(v != NULL, ("gv_bio_done: NULL v"));
+
+	switch (p->org) {
+	case GV_PLEX_CONCAT:
+	case GV_PLEX_STRIPED:
+		gv_plex_normal_done(p, bp);
+		break;
+	case GV_PLEX_RAID5:
+		gv_plex_raid5_done(p, bp);
+		break;
 	}
-
-	return (NULL);
 }
-
-static int
-gv_volume_destroy_geom(struct gctl_req *req, struct g_class *mp,
-    struct g_geom *gp)
-{
-	struct gv_volume *v;
-
-	g_trace(G_T_TOPOLOGY, "gv_volume_destroy_geom: %s", gp->name);
-	g_topology_assert();
-
-	v = gp->softc;
-	gv_kill_vol_thread(v);
-	g_wither_geom(gp, ENXIO);
-	return (0);
-}
-
-#define	VINUMVOLUME_CLASS_NAME "VINUMVOLUME"
-
-static struct g_class g_vinum_volume_class = {
-	.name = VINUMVOLUME_CLASS_NAME,
-	.version = G_VERSION,
-	.taste = gv_volume_taste,
-	.destroy_geom = gv_volume_destroy_geom,
-};
-
-DECLARE_GEOM_CLASS(g_vinum_volume_class, g_vinum_volume);
diff --git a/sys/gnu/fs/xfs/FreeBSD/xfs_buf.c b/sys/gnu/fs/xfs/FreeBSD/xfs_buf.c
index 9759d5a..ff06865 100644
--- a/sys/gnu/fs/xfs/FreeBSD/xfs_buf.c
+++ b/sys/gnu/fs/xfs/FreeBSD/xfs_buf.c
@@ -81,7 +81,7 @@ xfs_buf_get_empty(size_t size,  xfs_buftarg_t *target)
 {
 	struct buf *bp;
 
-	bp = geteblk(0);
+	bp = geteblk(0, 0);
 	if (bp != NULL) {
 		bp->b_bufsize = size;
 		bp->b_bcount = size;
@@ -100,7 +100,7 @@ xfs_buf_get_noaddr(size_t len, xfs_buftarg_t *target)
 	if (len >= MAXPHYS)
 		return (NULL);
 
-	bp = geteblk(len);
+	bp = geteblk(len, 0);
 	if (bp != NULL) {
 		BUF_ASSERT_HELD(bp);
 
diff --git a/sys/i386/conf/GENERIC b/sys/i386/conf/GENERIC
index 2915cc6..70aac72 100644
--- a/sys/i386/conf/GENERIC
+++ b/sys/i386/conf/GENERIC
@@ -309,7 +309,6 @@ device		ural		# Ralink Technology RT2500USB wireless NICs
 device		rum		# Ralink Technology RT2501USB wireless NICs
 device		zyd		# ZyDAS zb1211/zb1211b wireless NICs
 device		urio		# Diamond Rio 500 MP3 player
-device		uscanner	# Scanners
 # USB Serial devices
 device		u3g		# USB-based 3G modems (Option, Huawei, Sierra)
 device		uark		# Technologies ARK3116 based serial adapters
diff --git a/sys/i386/conf/NOTES b/sys/i386/conf/NOTES
index 1f69960..b9c3fd1 100644
--- a/sys/i386/conf/NOTES
+++ b/sys/i386/conf/NOTES
@@ -528,8 +528,6 @@ hint.mse.0.irq="5"
 # Network interfaces:
 #
 
-# ar:   Arnet SYNC/570i hdlc sync 2/4 port V.35/X.21 serial driver
-#       (requires sppp)
 # ath:  Atheros a/b/g WiFi adapters (requires ath_hal and wlan)
 # ce:   Cronyx Tau-PCI/32 sync single/dual port G.703/E1 serial adaptor
 #       with 32 HDLC subchannels (requires sppp (default), or NETGRAPH if
@@ -554,17 +552,11 @@ hint.mse.0.irq="5"
 # nve:	nVidia nForce MCP on-board Ethernet Networking
 # ral:	Ralink Technology IEEE 802.11 wireless adapter
 # sbni: Granch SBNI12-xx ISA and PCI adapters
-# sr:   RISCom/N2 hdlc sync 1/2 port V.35/X.21 serial driver (requires sppp)
 # wl:   Lucent Wavelan (ISA card only).
 # wpi:	Intel 3945ABG Wireless LAN controller
 
 # Order for ISA/EISA devices is important here
 
-device		ar
-hint.ar.0.at="isa"
-hint.ar.0.port="0x300"
-hint.ar.0.irq="10"
-hint.ar.0.maddr="0xd0000"
 device		ath		# Atheros pci/cardbus NIC's
 device		ath_hal		# pci/cardbus chip support
 #device		ath_ar5210	# AR5210 chips
@@ -619,11 +611,6 @@ hint.sbni.0.at="isa"
 hint.sbni.0.port="0x210"
 hint.sbni.0.irq="0xefdead"
 hint.sbni.0.flags="0"
-device		sr
-hint.sr.0.at="isa"
-hint.sr.0.port="0x300"
-hint.sr.0.irq="5"
-hint.sr.0.maddr="0xd0000"
 device		wl
 hint.wl.0.at="isa"
 hint.wl.0.port="0x300"
diff --git a/sys/i386/conf/XBOX b/sys/i386/conf/XBOX
index 3867165..0a2e8c0 100644
--- a/sys/i386/conf/XBOX
+++ b/sys/i386/conf/XBOX
@@ -91,7 +91,6 @@ device		ulpt		# Printer
 device		umass		# Disks/Mass storage - Requires scbus and da
 device		ums		# Mouse
 device		urio		# Diamond Rio 500 MP3 player
-device		uscanner	# Scanners
 
 device		miibus
 device		aue		# ADMtek USB Ethernet
diff --git a/sys/i386/cpufreq/hwpstate.c b/sys/i386/cpufreq/hwpstate.c
new file mode 100644
index 0000000..cc5d03e
--- /dev/null
+++ b/sys/i386/cpufreq/hwpstate.c
@@ -0,0 +1,515 @@
+/*-
+ * Copyright (c) 2005 Nate Lawson
+ * Copyright (c) 2004 Colin Percival
+ * Copyright (c) 2004-2005 Bruno Durcot
+ * Copyright (c) 2004 FUKUDA Nobuhiko
+ * Copyright (c) 2009 Michael Reifenberger
+ * Copyright (c) 2009 Norikatsu Shigemura
+ * Copyright (c) 2008-2009 Gen Otsuji
+ *
+ * This code is depending on kern_cpu.c, est.c, powernow.c, p4tcc.c, smist.c
+ * in various parts. The authors of these files are Nate Lawson,
+ * Colin Percival, Bruno Durcot, and FUKUDA Nobuhiko.
+ * This code contains patches by Michael Reifenberger and Norikatsu Shigemura.
+ * Thank you.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted providing that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * For more info:
+ * BIOS and Kernel Developer's Guide(BKDG) for AMD Family 10h Processors
+ * 31116 Rev 3.20  February 04, 2009
+ * BIOS and Kernel Developer's Guide(BKDG) for AMD Family 11h Processors
+ * 41256 Rev 3.00 - July 07, 2008
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/cpu.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/malloc.h>
+#include <sys/proc.h>
+#include <sys/pcpu.h>
+#include <sys/smp.h>
+#include <sys/sched.h>
+
+#include <machine/md_var.h>
+#include <machine/cputypes.h>
+#include <machine/specialreg.h>
+
+#include <contrib/dev/acpica/acpi.h>
+#include <dev/acpica/acpivar.h>
+
+#include "acpi_if.h"
+#include "cpufreq_if.h"
+
+#define	MSR_AMD_10H_11H_LIMIT	0xc0010061
+#define	MSR_AMD_10H_11H_CONTROL	0xc0010062
+#define	MSR_AMD_10H_11H_STATUS	0xc0010063
+#define	MSR_AMD_10H_11H_CONFIG	0xc0010064
+
+#define	AMD_10H_11H_MAX_STATES	16
+
+/* for MSR_AMD_10H_11H_LIMIT C001_0061 */
+#define	AMD_10H_11H_GET_PSTATE_MAX_VAL(msr)	(((msr) >> 4) & 0x7)
+#define	AMD_10H_11H_GET_PSTATE_LIMIT(msr)	(((msr)) & 0x7)
+/* for MSR_AMD_10H_11H_CONFIG 10h:C001_0064:68 / 11h:C001_0064:6B */
+#define	AMD_10H_11H_CUR_VID(msr)		(((msr) >> 9) & 0x7F)
+#define	AMD_10H_11H_CUR_DID(msr)		(((msr) >> 6) & 0x07)
+#define	AMD_10H_11H_CUR_FID(msr)		((msr) & 0x3F)
+
+#if defined(__amd64__)
+#define CPU_FAMILY(id)	AMD64_CPU_FAMILY(id)
+#elif defined(__i386__)
+#define CPU_FAMILY(id)	I386_CPU_FAMILY(id)
+#endif
+
+#define	HWPSTATE_DEBUG(dev, msg...)			\
+	do{						\
+		if(hwpstate_verbose)			\
+			device_printf(dev, msg);	\
+	}while(0)
+
+struct hwpstate_setting {
+	int	freq;		/* CPU clock in Mhz or 100ths of a percent. */
+	int	volts;		/* Voltage in mV. */
+	int	power;		/* Power consumed in mW. */
+	int	lat;		/* Transition latency in us. */
+	int	pstate_id;	/* P-State id */
+};
+
+struct hwpstate_softc {
+	device_t		dev;
+	struct hwpstate_setting	hwpstate_settings[AMD_10H_11H_MAX_STATES];
+	int			cfnum;
+};
+
+static void	hwpstate_identify(driver_t *driver, device_t parent);
+static int	hwpstate_probe(device_t dev);
+static int	hwpstate_attach(device_t dev);
+static int	hwpstate_detach(device_t dev);
+static int	hwpstate_set(device_t dev, const struct cf_setting *cf);
+static int	hwpstate_get(device_t dev, struct cf_setting *cf);
+static int	hwpstate_settings(device_t dev, struct cf_setting *sets, int *count);
+static int	hwpstate_type(device_t dev, int *type);
+static int	hwpstate_shutdown(device_t dev);
+static int	hwpstate_features(driver_t *driver, u_int *features);
+static int	hwpstate_get_info_from_acpi_perf(device_t dev, device_t perf_dev);
+static int	hwpstate_get_info_from_msr(device_t dev);
+static int	hwpstate_goto_pstate(device_t dev, int pstate_id);
+
+static int	hwpstate_verbose = 0;
+SYSCTL_INT(_debug, OID_AUTO, hwpstate_verbose, CTLFLAG_RDTUN,
+       &hwpstate_verbose, 0, "Debug hwpstate");
+
+static device_method_t hwpstate_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_identify,	hwpstate_identify),
+	DEVMETHOD(device_probe,		hwpstate_probe),
+	DEVMETHOD(device_attach,	hwpstate_attach),
+	DEVMETHOD(device_detach,	hwpstate_detach),
+	DEVMETHOD(device_shutdown,	hwpstate_shutdown),
+
+	/* cpufreq interface */
+	DEVMETHOD(cpufreq_drv_set,	hwpstate_set),
+	DEVMETHOD(cpufreq_drv_get,	hwpstate_get),
+	DEVMETHOD(cpufreq_drv_settings,	hwpstate_settings),
+	DEVMETHOD(cpufreq_drv_type,	hwpstate_type),
+
+	/* ACPI interface */
+	DEVMETHOD(acpi_get_features,	hwpstate_features),
+
+	{0, 0}
+};
+
+static devclass_t hwpstate_devclass;
+static driver_t hwpstate_driver = {
+	"hwpstate",
+	hwpstate_methods,
+	sizeof(struct hwpstate_softc),
+};
+
+DRIVER_MODULE(hwpstate, cpu, hwpstate_driver, hwpstate_devclass, 0, 0);
+
+/*
+ * Go to Px-state on all cpus considering the limit.
+ */
+static int
+hwpstate_goto_pstate(device_t dev, int pstate)
+{
+	struct hwpstate_softc *sc;
+	struct pcpu *pc;
+	int i;
+	uint64_t msr;
+	int j;
+	int limit;
+	int id = pstate;
+	int error;
+	
+	sc = device_get_softc(dev);
+	/* get the current pstate limit */
+	msr = rdmsr(MSR_AMD_10H_11H_LIMIT);
+	limit = AMD_10H_11H_GET_PSTATE_LIMIT(msr);
+	if(limit > id)
+		id = limit;
+
+	error = 0;
+	/*
+	 * We are going to the same Px-state on all cpus.
+	 */
+	for (i = 0; i < mp_ncpus; i++) {
+		/* Find each cpu. */
+		pc = pcpu_find(i);
+		if (pc == NULL)
+			return (ENXIO);
+		thread_lock(curthread);
+		/* Bind to each cpu. */
+		sched_bind(curthread, pc->pc_cpuid);
+		thread_unlock(curthread);
+		HWPSTATE_DEBUG(dev, "setting P%d-state on cpu%d\n",
+			id, PCPU_GET(cpuid));
+		/* Go To Px-state */
+		wrmsr(MSR_AMD_10H_11H_CONTROL, id);
+		/* wait loop (100*100 usec is enough ?) */
+		for(j = 0; j < 100; j++){
+			msr = rdmsr(MSR_AMD_10H_11H_STATUS);
+			if(msr == id){
+				break;
+			}
+			DELAY(100);
+		}
+		/* get the result. not assure msr=id */
+		msr = rdmsr(MSR_AMD_10H_11H_STATUS);
+		HWPSTATE_DEBUG(dev, "result  P%d-state on cpu%d\n",
+		    (int)msr, PCPU_GET(cpuid));
+		if (msr != id) {
+			HWPSTATE_DEBUG(dev, "error: loop is not enough.\n");
+			error = ENXIO;
+		}
+		thread_lock(curthread);
+		sched_unbind(curthread);
+		thread_unlock(curthread);
+	}
+	return (error);
+}
+
+static int
+hwpstate_set(device_t dev, const struct cf_setting *cf)
+{
+	struct hwpstate_softc *sc;
+	struct hwpstate_setting *set;
+	int i;
+
+	if (cf == NULL)
+		return (EINVAL);
+	sc = device_get_softc(dev);
+	set = sc->hwpstate_settings;
+	for (i = 0; i < sc->cfnum; i++)
+		if (CPUFREQ_CMP(cf->freq, set[i].freq))
+			break;
+	if (i == sc->cfnum)
+		return (EINVAL);
+
+	return (hwpstate_goto_pstate(dev, set[i].pstate_id));
+}
+
+static int
+hwpstate_get(device_t dev, struct cf_setting *cf)
+{
+	struct hwpstate_softc *sc;
+	struct hwpstate_setting set;
+	uint64_t msr;
+
+	sc = device_get_softc(dev);
+	if (cf == NULL)
+		return (EINVAL);
+	msr = rdmsr(MSR_AMD_10H_11H_STATUS);
+	if(msr >= sc->cfnum)
+		return (EINVAL);
+	set = sc->hwpstate_settings[msr];
+
+	cf->freq = set.freq;
+	cf->volts = set.volts;
+	cf->power = set.power;
+	cf->lat = set.lat;
+	cf->dev = dev;
+	return (0);
+}
+
+static int
+hwpstate_settings(device_t dev, struct cf_setting *sets, int *count)
+{
+	struct hwpstate_softc *sc;
+	struct hwpstate_setting set;
+	int i;
+
+	if (sets == NULL || count == NULL)
+		return (EINVAL);
+	sc = device_get_softc(dev);
+	if (*count < sc->cfnum)
+		return (E2BIG);
+	for (i = 0; i < sc->cfnum; i++, sets++) {
+		set = sc->hwpstate_settings[i];
+		sets->freq = set.freq;
+		sets->volts = set.volts;
+		sets->power = set.power;
+		sets->lat = set.lat;
+		sets->dev = dev;
+	}
+	*count = sc->cfnum;
+
+	return (0);
+}
+
+static int
+hwpstate_type(device_t dev, int *type)
+{
+
+	if (type == NULL)
+		return (EINVAL);
+
+	*type = CPUFREQ_TYPE_ABSOLUTE;
+	return (0);
+}
+
+static void
+hwpstate_identify(driver_t *driver, device_t parent)
+{
+	device_t child;
+
+	if (device_find_child(parent, "hwpstate", -1) != NULL)
+		return;
+
+	if (cpu_vendor_id != CPU_VENDOR_AMD || CPU_FAMILY(cpu_id) < 0x10)
+		return;
+
+	/*
+	 * Check if hardware pstate enable bit is set.
+	 */
+	if ((amd_pminfo & AMDPM_HW_PSTATE) == 0) {
+		HWPSTATE_DEBUG(parent, "hwpstate enable bit is not set.\n");
+		return;
+	}
+
+	if (resource_disabled("hwpstate", 0))
+		return;
+
+	if ((child = BUS_ADD_CHILD(parent, 10, "hwpstate", -1)) == NULL)
+		device_printf(parent, "hwpstate: add child failed\n");
+}
+
+static int
+hwpstate_probe(device_t dev)
+{
+	struct hwpstate_softc *sc;
+	device_t perf_dev;
+	uint64_t msr;
+	int error, type;
+
+	/*
+	 * Only hwpstate0.
+	 * It goes well with acpi_throttle.
+	 */
+	if (device_get_unit(dev) != 0)
+		return (ENXIO);
+
+	sc = device_get_softc(dev);
+	sc->dev = dev;
+
+	/*
+	 * Check if acpi_perf has INFO only flag.
+	 */
+	perf_dev = device_find_child(device_get_parent(dev), "acpi_perf", -1);
+	error = TRUE;
+	if (perf_dev && device_is_attached(perf_dev)) {
+		error = CPUFREQ_DRV_TYPE(perf_dev, &type);
+		if (error == 0) {
+			if ((type & CPUFREQ_FLAG_INFO_ONLY) == 0) {
+				/*
+				 * If acpi_perf doesn't have INFO_ONLY flag,
+				 * it will take care of pstate transitions.
+				 */
+				HWPSTATE_DEBUG(dev, "acpi_perf will take care of pstate transitions.\n");
+				return (ENXIO);
+			} else {
+				/*
+				 * If acpi_perf has INFO_ONLY flag, (_PCT has FFixedHW)
+				 * we can get _PSS info from acpi_perf
+				 * without going into ACPI.
+				 */
+				HWPSTATE_DEBUG(dev, "going to fetch info from acpi_perf\n");
+				error = hwpstate_get_info_from_acpi_perf(dev, perf_dev);
+			}
+		}
+	}
+
+	if (error == 0) {
+		/*
+		 * Now we get _PSS info from acpi_perf without error.
+		 * Let's check it.
+		 */
+		msr = rdmsr(MSR_AMD_10H_11H_LIMIT);
+		if (sc->cfnum != 1 + AMD_10H_11H_GET_PSTATE_MAX_VAL(msr)) {
+			HWPSTATE_DEBUG(dev, "msr and acpi _PSS count mismatch.\n");
+			error = TRUE;
+		}
+	}
+
+	/*
+	 * If we cannot get info from acpi_perf,
+	 * Let's get info from MSRs.
+	 */
+	if (error)
+		error = hwpstate_get_info_from_msr(dev);
+	if (error)
+		return (error);
+
+	device_set_desc(dev, "Cool`n'Quiet 2.0");
+	return (0);
+}
+
+static int
+hwpstate_attach(device_t dev)
+{
+
+	return (cpufreq_register(dev));
+}
+
+static int
+hwpstate_get_info_from_msr(device_t dev)
+{
+	struct hwpstate_softc *sc;
+	struct hwpstate_setting *hwpstate_set;
+	uint64_t msr;
+	int family, i, fid, did;
+
+	family = CPU_FAMILY(cpu_id);
+	sc = device_get_softc(dev);
+	/* Get pstate count */
+	msr = rdmsr(MSR_AMD_10H_11H_LIMIT);
+	sc->cfnum = 1 + AMD_10H_11H_GET_PSTATE_MAX_VAL(msr);
+	hwpstate_set = sc->hwpstate_settings;
+	for (i = 0; i < sc->cfnum; i++) {
+		msr = rdmsr(MSR_AMD_10H_11H_CONFIG + i);
+		if ((msr & ((uint64_t)1 << 63)) != ((uint64_t)1 << 63)) {
+			HWPSTATE_DEBUG(dev, "msr is not valid.\n");
+			return (ENXIO);
+		}
+		did = AMD_10H_11H_CUR_DID(msr);
+		fid = AMD_10H_11H_CUR_FID(msr);
+		switch(family) {
+		case 0x11:
+			/* fid/did to frequency */
+			hwpstate_set[i].freq = 100 * (fid + 0x08) / (1 << did);
+			break;
+		case 0x10:
+			/* fid/did to frequency */
+			hwpstate_set[i].freq = 100 * (fid + 0x10) / (1 << did);
+			break;
+		default:
+			HWPSTATE_DEBUG(dev, "get_info_from_msr: AMD family %d CPU's are not implemented yet. sorry.\n", family);
+			return (ENXIO);
+			break;
+		}
+		hwpstate_set[i].pstate_id = i;
+		/* There was volts calculation, but deleted it. */
+		hwpstate_set[i].volts = CPUFREQ_VAL_UNKNOWN;
+		hwpstate_set[i].power = CPUFREQ_VAL_UNKNOWN;
+		hwpstate_set[i].lat = CPUFREQ_VAL_UNKNOWN;
+	}
+	return (0);
+}
+
+static int
+hwpstate_get_info_from_acpi_perf(device_t dev, device_t perf_dev)
+{
+	struct hwpstate_softc *sc;
+	struct cf_setting *perf_set;
+	struct hwpstate_setting *hwpstate_set;
+	int count, error, i;
+
+	perf_set = malloc(MAX_SETTINGS * sizeof(*perf_set), M_TEMP, M_NOWAIT);
+	if (perf_set == NULL) {
+		HWPSTATE_DEBUG(dev, "nomem\n");
+		return (ENOMEM);
+	}
+	/*
+	 * Fetch settings from acpi_perf.
+	 * Now it is attached, and has info only flag.
+	 */
+	count = MAX_SETTINGS;
+	error = CPUFREQ_DRV_SETTINGS(perf_dev, perf_set, &count);
+	if (error) {
+		HWPSTATE_DEBUG(dev, "error: CPUFREQ_DRV_SETTINGS.\n");
+		goto out;
+	}
+	sc = device_get_softc(dev);
+	sc->cfnum = count;
+	hwpstate_set = sc->hwpstate_settings;
+	for (i = 0; i < count; i++) {
+		if (i == perf_set[i].spec[0]) {
+			hwpstate_set[i].pstate_id = i;
+			hwpstate_set[i].freq = perf_set[i].freq;
+			hwpstate_set[i].volts = perf_set[i].volts;
+			hwpstate_set[i].power = perf_set[i].power;
+			hwpstate_set[i].lat = perf_set[i].lat;
+		} else {
+			HWPSTATE_DEBUG(dev, "ACPI _PSS object mismatch.\n");
+			error = ENXIO;
+			goto out;
+		}
+	}
+out:
+	if (perf_set)
+		free(perf_set, M_TEMP);
+	return (error);
+}
+
+static int
+hwpstate_detach(device_t dev)
+{
+
+	hwpstate_goto_pstate(dev, 0);
+	return (cpufreq_unregister(dev));
+}
+
+static int
+hwpstate_shutdown(device_t dev)
+{
+
+	/* hwpstate_goto_pstate(dev, 0); */
+	return (0);
+}
+
+static int
+hwpstate_features(driver_t *driver, u_int *features)
+{
+
+	/* Notify the ACPI CPU that we support direct access to MSRs */
+	*features = ACPI_CAP_PERF_MSRS;
+	return (0);
+}
diff --git a/sys/i386/i386/elf_machdep.c b/sys/i386/i386/elf_machdep.c
index 19eddd0..9f5e3a4 100644
--- a/sys/i386/i386/elf_machdep.c
+++ b/sys/i386/i386/elf_machdep.c
@@ -84,7 +84,8 @@ static Elf32_Brandinfo freebsd_brand_info = {
 	.interp_path	= "/libexec/ld-elf.so.1",
 	.sysvec		= &elf32_freebsd_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &elf32_freebsd_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 SYSINIT(elf32, SI_SUB_EXEC, SI_ORDER_ANY,
@@ -99,7 +100,8 @@ static Elf32_Brandinfo freebsd_brand_oinfo = {
 	.interp_path	= "/usr/libexec/ld-elf.so.1",
 	.sysvec		= &elf32_freebsd_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &elf32_freebsd_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 SYSINIT(oelf32, SI_SUB_EXEC, SI_ORDER_ANY,
diff --git a/sys/i386/i386/i686_mem.c b/sys/i386/i386/i686_mem.c
index fc88be1..fe229cc 100644
--- a/sys/i386/i386/i686_mem.c
+++ b/sys/i386/i386/i686_mem.c
@@ -73,11 +73,13 @@ static void	i686_mrinit(struct mem_range_softc *sc);
 static int	i686_mrset(struct mem_range_softc *sc,
 		    struct mem_range_desc *mrd, int *arg);
 static void	i686_mrAPinit(struct mem_range_softc *sc);
+static void	i686_mrreinit(struct mem_range_softc *sc);
 
 static struct mem_range_ops i686_mrops = {
 	i686_mrinit,
 	i686_mrset,
-	i686_mrAPinit
+	i686_mrAPinit,
+	i686_mrreinit
 };
 
 /* XXX for AP startup hook */
@@ -668,6 +670,30 @@ i686_mrAPinit(struct mem_range_softc *sc)
 	wrmsr(MSR_MTRRdefType, mtrrdef);
 }
 
+/*
+ * Re-initialise running CPU(s) MTRRs to match the ranges in the descriptor
+ * list.
+ *
+ * XXX Must be called with interrupts enabled.
+ */
+static void
+i686_mrreinit(struct mem_range_softc *sc)
+{
+#ifdef SMP
+	/*
+	 * We should use ipi_all_but_self() to call other CPUs into a
+	 * locking gate, then call a target function to do this work.
+	 * The "proper" solution involves a generalised locking gate
+	 * implementation, not ready yet.
+	 */
+	smp_rendezvous(NULL, (void *)i686_mrAPinit, NULL, sc);
+#else
+	disable_intr();				/* disable interrupts */
+	i686_mrAPinit(sc);
+	enable_intr();
+#endif
+}
+
 static void
 i686_mem_drvinit(void *unused)
 {
diff --git a/sys/i386/i386/k6_mem.c b/sys/i386/i386/k6_mem.c
index 9cbacfe..c99cf27 100644
--- a/sys/i386/i386/k6_mem.c
+++ b/sys/i386/i386/k6_mem.c
@@ -70,6 +70,7 @@ static struct mem_range_ops k6_mrops =
 {
 	k6_mrinit,
 	k6_mrset,
+	NULL,
 	NULL
 };
 
diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c
index e10217a..3987907 100644
--- a/sys/i386/i386/machdep.c
+++ b/sys/i386/i386/machdep.c
@@ -243,6 +243,7 @@ static void
 cpu_startup(dummy)
 	void *dummy;
 {
+	uintmax_t memsize;
 	char *sysenv;
 	
 	/*
@@ -273,8 +274,18 @@ cpu_startup(dummy)
 #ifdef PERFMON
 	perfmon_init();
 #endif
-	printf("real memory  = %ju (%ju MB)\n", ptoa((uintmax_t)Maxmem),
-	    ptoa((uintmax_t)Maxmem) / 1048576);
+	sysenv = getenv("smbios.memory.enabled");
+	if (sysenv != NULL) {
+		memsize = (uintmax_t)strtoul(sysenv, (char **)NULL, 10);
+		freeenv(sysenv);
+	} else
+		memsize = 0;
+	if (memsize > 0)
+		printf("real memory  = %ju (%ju MB)\n", memsize << 10,
+		    memsize >> 10);
+	else
+		printf("real memory  = %ju (%ju MB)\n", ptoa((uintmax_t)Maxmem),
+		    ptoa((uintmax_t)Maxmem) / 1048576);
 	realmem = Maxmem;
 	/*
 	 * Display any holes after the first chunk of extended memory.
@@ -578,6 +589,7 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	struct sigacts *psp;
 	char *sp;
 	struct trapframe *regs;
+	struct segment_descriptor *sdp;
 	int sig;
 	int oonstack;
 
@@ -614,6 +626,15 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	sf.sf_uc.uc_mcontext.mc_len = sizeof(sf.sf_uc.uc_mcontext); /* magic */
 	get_fpcontext(td, &sf.sf_uc.uc_mcontext);
 	fpstate_drop(td);
+	/*
+	 * Unconditionally fill the fsbase and gsbase into the mcontext.
+	 */
+	sdp = &td->td_pcb->pcb_gsd;
+	sf.sf_uc.uc_mcontext.mc_fsbase = sdp->sd_hibase << 24 |
+	    sdp->sd_lobase;
+	sdp = &td->td_pcb->pcb_fsd;
+	sf.sf_uc.uc_mcontext.mc_gsbase = sdp->sd_hibase << 24 |
+	    sdp->sd_lobase;
 
 	/* Allocate space for the signal handler context. */
 	if ((td->td_pflags & TDP_ALTSTACK) != 0 && !oonstack &&
@@ -1515,236 +1536,236 @@ extern  vm_offset_t	proc0kstack;
  */
 struct soft_segment_descriptor gdt_segs[] = {
 /* GNULL_SEL	0 Null Descriptor */
-{	0x0,			/* segment base address  */
-	0x0,			/* length */
-	0,			/* segment type */
-	SEL_KPL,		/* segment descriptor priority level */
-	0,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = SEL_KPL,
+	.ssd_p = 0,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 /* GPRIV_SEL	1 SMP Per-Processor Private Data Descriptor */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMRWA,		/* segment type */
-	SEL_KPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = SEL_KPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GUFS_SEL	2 %fs Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMRWA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GUGS_SEL	3 %gs Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMRWA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GCODE_SEL	4 Code Descriptor for kernel */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMERA,		/* segment type */
-	SEL_KPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMERA,
+	.ssd_dpl = SEL_KPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GDATA_SEL	5 Data Descriptor for kernel */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMRWA,		/* segment type */
-	SEL_KPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = SEL_KPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GUCODE_SEL	6 Code Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMERA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMERA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GUDATA_SEL	7 Data Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMRWA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GBIOSLOWMEM_SEL 8 BIOS access to realmode segment 0x40, must be #8 in GDT */
-{	0x400,			/* segment base address */
-	0xfffff,		/* length */
-	SDT_MEMRWA,		/* segment type */
-	SEL_KPL,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x400,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = SEL_KPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 #ifndef XEN
 /* GPROC0_SEL	9 Proc 0 Tss Descriptor */
 {
-	0x0,			/* segment base address */
-	sizeof(struct i386tss)-1,/* length  */
-	SDT_SYS386TSS,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* unused - default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+	.ssd_base = 0x0,
+	.ssd_limit = sizeof(struct i386tss)-1,
+	.ssd_type = SDT_SYS386TSS,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 /* GLDT_SEL	10 LDT Descriptor */
-{	(int) ldt,		/* segment base address  */
-	sizeof(ldt)-1,		/* length - all address space */
-	SDT_SYSLDT,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* unused - default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = (int) ldt,
+	.ssd_limit = sizeof(ldt)-1,
+	.ssd_type = SDT_SYSLDT,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 /* GUSERLDT_SEL	11 User LDT Descriptor per process */
-{	(int) ldt,		/* segment base address  */
-	(512 * sizeof(union descriptor)-1),		/* length */
-	SDT_SYSLDT,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* unused - default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = (int) ldt,
+	.ssd_limit = (512 * sizeof(union descriptor)-1),
+	.ssd_type = SDT_SYSLDT,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 /* GPANIC_SEL	12 Panic Tss Descriptor */
-{	(int) &dblfault_tss,	/* segment base address  */
-	sizeof(struct i386tss)-1,/* length - all address space */
-	SDT_SYS386TSS,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* unused - default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = (int) &dblfault_tss,
+	.ssd_limit = sizeof(struct i386tss)-1,
+	.ssd_type = SDT_SYS386TSS,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 /* GBIOSCODE32_SEL 13 BIOS 32-bit interface (32bit Code) */
-{	0,			/* segment base address (overwritten)  */
-	0xfffff,		/* length */
-	SDT_MEMERA,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMERA,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 1		},
 /* GBIOSCODE16_SEL 14 BIOS 32-bit interface (16bit Code) */
-{	0,			/* segment base address (overwritten)  */
-	0xfffff,		/* length */
-	SDT_MEMERA,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMERA,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 1		},
 /* GBIOSDATA_SEL 15 BIOS 32-bit interface (Data) */
-{	0,			/* segment base address (overwritten) */
-	0xfffff,		/* length */
-	SDT_MEMRWA,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GBIOSUTIL_SEL 16 BIOS 16-bit interface (Utility) */
-{	0,			/* segment base address (overwritten) */
-	0xfffff,		/* length */
-	SDT_MEMRWA,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 1		},
 /* GBIOSARGS_SEL 17 BIOS 16-bit interface (Arguments) */
-{	0,			/* segment base address (overwritten) */
-	0xfffff,		/* length */
-	SDT_MEMRWA,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 1		},
 /* GNDIS_SEL	18 NDIS Descriptor */
-{	0x0,			/* segment base address  */
-	0x0,			/* length */
-	0,			/* segment type */
-	0,			/* segment descriptor priority level */
-	0,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 #endif /* !XEN */
 };
 
 static struct soft_segment_descriptor ldt_segs[] = {
 	/* Null Descriptor - overwritten by call gate */
-{	0x0,			/* segment base address  */
-	0x0,			/* length - all address space */
-	0,			/* segment type */
-	0,			/* segment descriptor priority level */
-	0,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 	/* Null Descriptor - overwritten by call gate */
-{	0x0,			/* segment base address  */
-	0x0,			/* length - all address space */
-	0,			/* segment type */
-	0,			/* segment descriptor priority level */
-	0,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 	/* Null Descriptor - overwritten by call gate */
-{	0x0,			/* segment base address  */
-	0x0,			/* length - all address space */
-	0,			/* segment type */
-	0,			/* segment descriptor priority level */
-	0,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 	/* Code Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMERA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMERA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 	/* Null Descriptor - overwritten by call gate */
-{	0x0,			/* segment base address  */
-	0x0,			/* length - all address space */
-	0,			/* segment type */
-	0,			/* segment descriptor priority level */
-	0,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 	/* Data Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMRWA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 };
 
 void
@@ -3067,6 +3088,7 @@ int
 get_mcontext(struct thread *td, mcontext_t *mcp, int flags)
 {
 	struct trapframe *tp;
+	struct segment_descriptor *sdp;
 
 	tp = td->td_frame;
 
@@ -3098,6 +3120,11 @@ get_mcontext(struct thread *td, mcontext_t *mcp, int flags)
 	mcp->mc_ss = tp->tf_ss;
 	mcp->mc_len = sizeof(*mcp);
 	get_fpcontext(td, mcp);
+	sdp = &td->td_pcb->pcb_gsd;
+	mcp->mc_fsbase = sdp->sd_hibase << 24 | sdp->sd_lobase;
+	sdp = &td->td_pcb->pcb_fsd;
+	mcp->mc_gsbase = sdp->sd_hibase << 24 | sdp->sd_lobase;
+
 	return (0);
 }
 
@@ -3528,45 +3555,24 @@ lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, enum intr_trigger trigger)
 #ifdef KDB
 
 /*
- * Provide inb() and outb() as functions.  They are normally only
- * available as macros calling inlined functions, thus cannot be
- * called from the debugger.
- *
- * The actual code is stolen from <machine/cpufunc.h>, and de-inlined.
+ * Provide inb() and outb() as functions.  They are normally only available as
+ * inline functions, thus cannot be called from the debugger.
  */
 
-#undef inb
-#undef outb
-
 /* silence compiler warnings */
-u_char inb(u_int);
-void outb(u_int, u_char);
+u_char inb_(u_short);
+void outb_(u_short, u_char);
 
 u_char
-inb(u_int port)
+inb_(u_short port)
 {
-	u_char	data;
-	/*
-	 * We use %%dx and not %1 here because i/o is done at %dx and not at
-	 * %edx, while gcc generates inferior code (movw instead of movl)
-	 * if we tell it to load (u_short) port.
-	 */
-	__asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port));
-	return (data);
+	return inb(port);
 }
 
 void
-outb(u_int port, u_char data)
+outb_(u_short port, u_char data)
 {
-	u_char	al;
-	/*
-	 * Use an unnecessary assignment to help gcc's register allocator.
-	 * This make a large difference for gcc-1.40 and a tiny difference
-	 * for gcc-2.6.0.  For gcc-1.40, al had to be ``asm("ax")'' for
-	 * best results.  gcc-2.6.0 can't handle this.
-	 */
-	al = data;
-	__asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port));
+	outb(port, data);
 }
 
 #endif /* KDB */
diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c
index 83da708..7f6900d 100644
--- a/sys/i386/i386/pmap.c
+++ b/sys/i386/i386/pmap.c
@@ -2442,6 +2442,7 @@ pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva,
 		mpte = pmap_lookup_pt_page(pmap, sva);
 		if (mpte != NULL) {
 			pmap_remove_pt_page(pmap, mpte);
+			pmap->pm_stats.resident_count--;
 			KASSERT(mpte->wire_count == NPTEPG,
 			    ("pmap_remove_pde: pte page wire count error"));
 			mpte->wire_count = 0;
@@ -3990,6 +3991,7 @@ pmap_remove_pages(pmap_t pmap)
 					mpte = pmap_lookup_pt_page(pmap, pv->pv_va);
 					if (mpte != NULL) {
 						pmap_remove_pt_page(pmap, mpte);
+						pmap->pm_stats.resident_count--;
 						KASSERT(mpte->wire_count == NPTEPG,
 						    ("pmap_remove_pages: pte page wire count error"));
 						mpte->wire_count = 0;
diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c
index 2a6ca0f..e06a9a2 100644
--- a/sys/i386/i386/vm_machdep.c
+++ b/sys/i386/i386/vm_machdep.c
@@ -424,11 +424,6 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 	 * Set registers for trampoline to user mode.  Leave space for the
 	 * return address on stack.  These are the kernel mode register values.
 	 */
-#ifdef PAE
-	pcb2->pcb_cr3 = vtophys(vmspace_pmap(td->td_proc->p_vmspace)->pm_pdpt);
-#else
-	pcb2->pcb_cr3 = vtophys(vmspace_pmap(td->td_proc->p_vmspace)->pm_pdir);
-#endif
 	pcb2->pcb_edi = 0;
 	pcb2->pcb_esi = (int)fork_return;		    /* trampoline arg */
 	pcb2->pcb_ebp = 0;
@@ -439,6 +434,7 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 	pcb2->pcb_gs = rgs();
 	/*
 	 * If we didn't copy the pcb, we'd need to do the following registers:
+	 * pcb2->pcb_cr3:	cloned above.
 	 * pcb2->pcb_dr*:	cloned above.
 	 * pcb2->pcb_savefpu:	cloned above.
 	 * pcb2->pcb_flags:	cloned above.
diff --git a/sys/i386/include/cpufunc.h b/sys/i386/include/cpufunc.h
index ad5fc4c..98eb9ac 100644
--- a/sys/i386/include/cpufunc.h
+++ b/sys/i386/include/cpufunc.h
@@ -170,70 +170,12 @@ halt(void)
 	__asm __volatile("hlt");
 }
 
-#if !defined(__GNUCLIKE_BUILTIN_CONSTANT_P) || __GNUCLIKE_ASM < 3
-
-#define	inb(port)		inbv(port)
-#define	outb(port, data)	outbv(port, data)
-
-#else /* __GNUCLIKE_BUILTIN_CONSTANT_P && __GNUCLIKE_ASM >= 3 */
-
-/*
- * The following complications are to get around gcc not having a
- * constraint letter for the range 0..255.  We still put "d" in the
- * constraint because "i" isn't a valid constraint when the port
- * isn't constant.  This only matters for -O0 because otherwise
- * the non-working version gets optimized away.
- * 
- * Use an expression-statement instead of a conditional expression
- * because gcc-2.6.0 would promote the operands of the conditional
- * and produce poor code for "if ((inb(var) & const1) == const2)".
- *
- * The unnecessary test `(port) < 0x10000' is to generate a warning if
- * the `port' has type u_short or smaller.  Such types are pessimal.
- * This actually only works for signed types.  The range check is
- * careful to avoid generating warnings.
- */
-#define	inb(port) __extension__ ({					\
-	u_char	_data;							\
-	if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100	\
-	    && (port) < 0x10000)					\
-		_data = inbc(port);					\
-	else								\
-		_data = inbv(port);					\
-	_data; })
-
-#define	outb(port, data) (						\
-	__builtin_constant_p(port) && ((port) & 0xffff) < 0x100		\
-	&& (port) < 0x10000						\
-	? outbc(port, data) : outbv(port, data))
-
 static __inline u_char
-inbc(u_int port)
+inb(u_int port)
 {
 	u_char	data;
 
-	__asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port)));
-	return (data);
-}
-
-static __inline void
-outbc(u_int port, u_char data)
-{
-	__asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port)));
-}
-
-#endif /* __GNUCLIKE_BUILTIN_CONSTANT_P  && __GNUCLIKE_ASM >= 3*/
-
-static __inline u_char
-inbv(u_int port)
-{
-	u_char	data;
-	/*
-	 * We use %%dx and not %1 here because i/o is done at %dx and not at
-	 * %edx, while gcc generates inferior code (movw instead of movl)
-	 * if we tell it to load (u_short) port.
-	 */
-	__asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port));
+	__asm volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
 	return (data);
 }
 
@@ -242,7 +184,7 @@ inl(u_int port)
 {
 	u_int	data;
 
-	__asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port));
+	__asm volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
 	return (data);
 }
 
@@ -284,33 +226,20 @@ inw(u_int port)
 {
 	u_short	data;
 
-	__asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port));
+	__asm volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
 	return (data);
 }
 
 static __inline void
-outbv(u_int port, u_char data)
+outb(u_int port, u_char data)
 {
-	u_char	al;
-	/*
-	 * Use an unnecessary assignment to help gcc's register allocator.
-	 * This make a large difference for gcc-1.40 and a tiny difference
-	 * for gcc-2.6.0.  For gcc-1.40, al had to be ``asm("ax")'' for
-	 * best results.  gcc-2.6.0 can't handle this.
-	 */
-	al = data;
-	__asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port));
+	__asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
 }
 
 static __inline void
 outl(u_int port, u_int data)
 {
-	/*
-	 * outl() and outw() aren't used much so we haven't looked at
-	 * possible micro-optimizations such as the unnecessary
-	 * assignment for them.
-	 */
-	__asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port));
+	__asm volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
 }
 
 static __inline void
@@ -340,7 +269,7 @@ outsl(u_int port, const void *addr, size_t cnt)
 static __inline void
 outw(u_int port, u_short data)
 {
-	__asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port));
+	__asm volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
 }
 
 static __inline void
diff --git a/sys/i386/include/elf.h b/sys/i386/include/elf.h
index 1470173..af71ab8 100644
--- a/sys/i386/include/elf.h
+++ b/sys/i386/include/elf.h
@@ -84,16 +84,14 @@ __ElfType(Auxinfo);
 #define	AT_BASE		7	/* Interpreter's base address. */
 #define	AT_FLAGS	8	/* Flags (unused for i386). */
 #define	AT_ENTRY	9	/* Where interpreter should transfer control. */
-/*
- * The following non-standard values are used in Linux ELF binaries.
- */
 #define	AT_NOTELF	10	/* Program is not ELF ?? */
 #define	AT_UID		11	/* Real uid. */
 #define	AT_EUID		12	/* Effective uid. */
 #define	AT_GID		13	/* Real gid. */
 #define	AT_EGID		14	/* Effective gid. */
+#define	AT_EXECPATH	15	/* Path to the executable. */
 
-#define	AT_COUNT	15	/* Count of defined aux entry types. */
+#define	AT_COUNT	16	/* Count of defined aux entry types. */
 
 /*
  * Relocation types.
diff --git a/sys/i386/include/endian.h b/sys/i386/include/endian.h
index 59fa4a5..6522ec4 100644
--- a/sys/i386/include/endian.h
+++ b/sys/i386/include/endian.h
@@ -69,25 +69,6 @@ extern "C" {
 
 #if defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE_BUILTIN_CONSTANT_P)
 
-#define __word_swap_int_var(x) \
-__extension__ ({ register __uint32_t __X = (x); \
-   __asm ("rorl $16, %0" : "+r" (__X)); \
-   __X; })
-
-#ifdef __OPTIMIZE__
-
-#define	__word_swap_int_const(x) \
-	((((x) & 0xffff0000) >> 16) | \
-	 (((x) & 0x0000ffff) << 16))
-#define	__word_swap_int(x) (__builtin_constant_p(x) ? \
-	__word_swap_int_const(x) : __word_swap_int_var(x))
-
-#else	/* __OPTIMIZE__ */
-
-#define	__word_swap_int(x) __word_swap_int_var(x)
-
-#endif	/* __OPTIMIZE__ */
-
 #define __byte_swap_int_var(x) \
 __extension__ ({ register __uint32_t __X = (x); \
    __asm ("bswap %0" : "+r" (__X)); \
@@ -109,26 +90,6 @@ __extension__ ({ register __uint32_t __X = (x); \
 
 #endif	/* __OPTIMIZE__ */
 
-#define __byte_swap_word_var(x) \
-__extension__ ({ register __uint16_t __X = (x); \
-   __asm ("xchgb %h0, %b0" : "+q" (__X)); \
-   __X; })
-
-#ifdef __OPTIMIZE__
-
-#define	__byte_swap_word_const(x) \
-	((((x) & 0xff00) >> 8) | \
-	 (((x) & 0x00ff) << 8))
-
-#define	__byte_swap_word(x) (__builtin_constant_p(x) ? \
-	__byte_swap_word_const(x) : __byte_swap_word_var(x))
-
-#else	/* __OPTIMIZE__ */
-
-#define	__byte_swap_word(x) __byte_swap_word_var(x)
-
-#endif	/* __OPTIMIZE__ */
-
 static __inline __uint64_t
 __bswap64(__uint64_t _x)
 {
@@ -149,8 +110,7 @@ __bswap32(__uint32_t _x)
 static __inline __uint16_t
 __bswap16(__uint16_t _x)
 {
-
-	return (__byte_swap_word(_x));
+	return (_x << 8 | _x >> 8);
 }
 
 #define	__htonl(x)	__bswap32(x)
diff --git a/sys/i386/include/pmap.h b/sys/i386/include/pmap.h
index 00a3f7b..519ff6c 100644
--- a/sys/i386/include/pmap.h
+++ b/sys/i386/include/pmap.h
@@ -174,8 +174,7 @@ typedef uint32_t pt_entry_t;
 #endif
 
 /*
- * Address of current and alternate address space page table maps
- * and directories.
+ * Address of current address space page table maps and directories.
  */
 #ifdef _KERNEL
 extern pt_entry_t PTmap[];
diff --git a/sys/i386/include/signal.h b/sys/i386/include/signal.h
index 8d6b935..7a5f876 100644
--- a/sys/i386/include/signal.h
+++ b/sys/i386/include/signal.h
@@ -116,7 +116,11 @@ struct sigcontext {
 	int	sc_ownedfp;
 	int	sc_spare1[1];
 	int	sc_fpstate[128] __aligned(16);
-	int	sc_spare2[8];
+
+	int	sc_fsbase;
+	int	sc_gsbase;
+
+	int	sc_spare2[6];
 };
 
 #define	sc_sp		sc_esp
diff --git a/sys/i386/include/ucontext.h b/sys/i386/include/ucontext.h
index c992495..d8657d3 100644
--- a/sys/i386/include/ucontext.h
+++ b/sys/i386/include/ucontext.h
@@ -72,10 +72,15 @@ typedef struct __mcontext {
 	 * See <machine/npx.h> for the internals of mc_fpstate[].
 	 */
 	int	mc_fpstate[128] __aligned(16);
-	int	mc_spare2[8];
+
+	__register_t mc_fsbase;
+	__register_t mc_gsbase;
+
+	int	mc_spare2[6];
 } mcontext_t;
 
 #if defined(_KERNEL) && defined(COMPAT_FREEBSD4)
+
 struct mcontext4 {
 	__register_t	mc_onstack;	/* XXX - sigcontext compat. */
 	__register_t	mc_gs;		/* machine state (struct trapframe) */
diff --git a/sys/i386/include/vmparam.h b/sys/i386/include/vmparam.h
index f114232..e5f596c 100644
--- a/sys/i386/include/vmparam.h
+++ b/sys/i386/include/vmparam.h
@@ -43,10 +43,6 @@
  * Machine dependent constants for 386.
  */
 
-#ifndef PAE
-#define VM_PROT_READ_IS_EXEC	/* if you can read -- then you can exec */
-#endif
-
 /*
  * Virtual memory related constants, all in bytes
  */
diff --git a/sys/i386/include/xen/xenpmap.h b/sys/i386/include/xen/xenpmap.h
index 4bfd99e..b8a545e 100644
--- a/sys/i386/include/xen/xenpmap.h
+++ b/sys/i386/include/xen/xenpmap.h
@@ -45,6 +45,8 @@ void xen_pt_pin(vm_paddr_t);
 void xen_pt_unpin(vm_paddr_t);
 void xen_flush_queue(void);
 void pmap_ref(pt_entry_t *pte, vm_paddr_t ma);
+void pmap_suspend(void);
+void pmap_resume(void);
 void xen_check_queue(void);
 
 #ifdef INVARIANTS
diff --git a/sys/i386/include/xen/xenvar.h b/sys/i386/include/xen/xenvar.h
index 402bc8a..cefbb05 100644
--- a/sys/i386/include/xen/xenvar.h
+++ b/sys/i386/include/xen/xenvar.h
@@ -40,6 +40,8 @@ extern int xendebug_flags;
 #include <machine/xen/features.h>
 
 extern	xen_pfn_t *xen_phys_machine;
+extern	xen_pfn_t *xen_pfn_to_mfn_frame_list[16];
+extern	xen_pfn_t *xen_pfn_to_mfn_frame_list_list;
 
 #if 0
 #define TRACE_ENTER XENPRINTF("(file=%s, line=%d) entered %s\n", __FILE__, __LINE__, __FUNCTION__)
diff --git a/sys/i386/isa/npx.c b/sys/i386/isa/npx.c
index a4f5b34..6da4b4c 100644
--- a/sys/i386/isa/npx.c
+++ b/sys/i386/isa/npx.c
@@ -181,7 +181,7 @@ SYSCTL_INT(_hw, HW_FLOATINGPT, floatingpoint, CTLFLAG_RD,
 static	volatile u_int		npx_intrs_while_probing;
 static	volatile u_int		npx_traps_while_probing;
 
-static	union savefpu		npx_cleanstate;
+static	union savefpu		npx_initialstate;
 static	bool_t			npx_ex16;
 static	bool_t			npx_exists;
 static	bool_t			npx_irq13;
@@ -423,24 +423,24 @@ npx_attach(dev)
 
 	s = intr_disable();
 	stop_emulating();
-	fpusave(&npx_cleanstate);
+	fpusave(&npx_initialstate);
 	start_emulating();
 #ifdef CPU_ENABLE_SSE
 	if (cpu_fxsr) {
-		if (npx_cleanstate.sv_xmm.sv_env.en_mxcsr_mask)
+		if (npx_initialstate.sv_xmm.sv_env.en_mxcsr_mask)
 			cpu_mxcsr_mask = 
-			    npx_cleanstate.sv_xmm.sv_env.en_mxcsr_mask;
+			    npx_initialstate.sv_xmm.sv_env.en_mxcsr_mask;
 		else
 			cpu_mxcsr_mask = 0xFFBF;
-		bzero(npx_cleanstate.sv_xmm.sv_fp,
-		    sizeof(npx_cleanstate.sv_xmm.sv_fp));
-		bzero(npx_cleanstate.sv_xmm.sv_xmm,
-		    sizeof(npx_cleanstate.sv_xmm.sv_xmm));
+		bzero(npx_initialstate.sv_xmm.sv_fp,
+		    sizeof(npx_initialstate.sv_xmm.sv_fp));
+		bzero(npx_initialstate.sv_xmm.sv_xmm,
+		    sizeof(npx_initialstate.sv_xmm.sv_xmm));
 		/* XXX might need even more zeroing. */
 	} else
 #endif
-		bzero(npx_cleanstate.sv_87.sv_ac,
-		    sizeof(npx_cleanstate.sv_87.sv_ac));
+		bzero(npx_initialstate.sv_87.sv_ac,
+		    sizeof(npx_initialstate.sv_87.sv_ac));
 	intr_restore(s);
 #ifdef I586_CPU_XXX
 	if (cpu_class == CPUCLASS_586 && npx_ex16 &&
@@ -794,13 +794,18 @@ npxdna(void)
 	PCPU_SET(fpcurthread, curthread);
 	pcb = PCPU_GET(curpcb);
 
+#ifdef CPU_ENABLE_SSE
+	if (cpu_fxsr)
+		fpu_clean_state();
+#endif
+
 	if ((pcb->pcb_flags & PCB_NPXINITDONE) == 0) {
 		/*
 		 * This is the first time this thread has used the FPU or
 		 * the PCB doesn't contain a clean FPU state.  Explicitly
-		 * load sanitized registers.
+		 * load an initial state.
 		 */
-		fpurstor(&npx_cleanstate);
+		fpurstor(&npx_initialstate);
 		if (pcb->pcb_initial_npxcw != __INITIAL_NPXCW__)
 			fldcw(&pcb->pcb_initial_npxcw);
 		pcb->pcb_flags |= PCB_NPXINITDONE;
@@ -900,7 +905,7 @@ npxgetregs(td, addr)
 		return (_MC_FPOWNED_NONE);
 
 	if ((td->td_pcb->pcb_flags & PCB_NPXINITDONE) == 0) {
-		bcopy(&npx_cleanstate, addr, sizeof(npx_cleanstate));
+		bcopy(&npx_initialstate, addr, sizeof(npx_initialstate));
 		SET_FPU_CW(addr, td->td_pcb->pcb_initial_npxcw);
 		return (_MC_FPOWNED_NONE);
 	}
@@ -976,10 +981,10 @@ fpusave(addr)
  * In order to avoid leaking this information across processes, we clean
  * these values by performing a dummy load before executing fxrstor().
  */
-static	double	dummy_variable = 0.0;
 static void
 fpu_clean_state(void)
 {
+	static float dummy_variable = 0.0;
 	u_short status;
 
 	/*
@@ -1005,10 +1010,9 @@ fpurstor(addr)
 {
 
 #ifdef CPU_ENABLE_SSE
-	if (cpu_fxsr) {
-		fpu_clean_state();
+	if (cpu_fxsr)
 		fxrstor(addr);
-	} else
+	else
 #endif
 		frstor(addr);
 }
diff --git a/sys/i386/linux/linux_sysvec.c b/sys/i386/linux/linux_sysvec.c
index 4efd496..707ffb3 100644
--- a/sys/i386/linux/linux_sysvec.c
+++ b/sys/i386/linux/linux_sysvec.c
@@ -1019,6 +1019,16 @@ struct sysentvec elf_linux_sysvec = {
 	.sv_flags	= SV_ABI_LINUX | SV_IA32 | SV_ILP32
 };
 
+static char GNULINUX_ABI_VENDOR[] = "GNU";
+
+static Elf_Brandnote linux_brandnote = {
+	.hdr.n_namesz	= sizeof(GNULINUX_ABI_VENDOR),
+	.hdr.n_descsz	= 16,
+	.hdr.n_type	= 1,
+	.vendor		= GNULINUX_ABI_VENDOR,
+	.flags		= 0
+};
+
 static Elf32_Brandinfo linux_brand = {
 	.brand		= ELFOSABI_LINUX,
 	.machine	= EM_386,
@@ -1027,7 +1037,8 @@ static Elf32_Brandinfo linux_brand = {
 	.interp_path	= "/lib/ld-linux.so.1",
 	.sysvec		= &elf_linux_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &linux_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 static Elf32_Brandinfo linux_glibc2brand = {
@@ -1038,7 +1049,8 @@ static Elf32_Brandinfo linux_glibc2brand = {
 	.interp_path	= "/lib/ld-linux.so.2",
 	.sysvec		= &elf_linux_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &linux_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 Elf32_Brandinfo *linux_brandlist[] = {
diff --git a/sys/i386/pci/pci_cfgreg.c b/sys/i386/pci/pci_cfgreg.c
index 3c41ba7..ecefa17 100644
--- a/sys/i386/pci/pci_cfgreg.c
+++ b/sys/i386/pci/pci_cfgreg.c
@@ -206,6 +206,7 @@ pci_docfgregread(int bus, int slot, int func, int reg, int bytes)
 {
 
 	if (cfgmech == CFGMECH_PCIE &&
+	    (bus >= pcie_minbus && bus <= pcie_maxbus) &&
 	    (bus != 0 || !(1 << slot & pcie_badslots)))
 		return (pciereg_cfgread(bus, slot, func, reg, bytes));
 	else
@@ -240,6 +241,7 @@ pci_cfgregwrite(int bus, int slot, int func, int reg, u_int32_t data, int bytes)
 {
 
 	if (cfgmech == CFGMECH_PCIE &&
+	    (bus >= pcie_minbus && bus <= pcie_maxbus) &&
 	    (bus != 0 || !(1 << slot & pcie_badslots)))
 		pciereg_cfgwrite(bus, slot, func, reg, data, bytes);
 	else
diff --git a/sys/i386/xen/pmap.c b/sys/i386/xen/pmap.c
index 3725d0a..59947e8 100644
--- a/sys/i386/xen/pmap.c
+++ b/sys/i386/xen/pmap.c
@@ -4100,6 +4100,72 @@ pmap_align_superpage(vm_object_t object, vm_ooffset_t offset,
 		*addr = ((*addr + PDRMASK) & ~PDRMASK) + superpage_offset;
 }
 
+#ifdef XEN
+
+void
+pmap_suspend()
+{
+	pmap_t pmap;
+	int i, pdir, offset;
+	vm_paddr_t pdirma;
+	mmu_update_t mu[4];
+
+	/*
+	 * We need to remove the recursive mapping structure from all
+	 * our pmaps so that Xen doesn't get confused when it restores
+	 * the page tables. The recursive map lives at page directory
+	 * index PTDPTDI. We assume that the suspend code has stopped
+	 * the other vcpus (if any).
+	 */
+	LIST_FOREACH(pmap, &allpmaps, pm_list) {
+		for (i = 0; i < 4; i++) {
+			/*
+			 * Figure out which page directory (L2) page
+			 * contains this bit of the recursive map and
+			 * the offset within that page of the map
+			 * entry
+			 */
+			pdir = (PTDPTDI + i) / NPDEPG;
+			offset = (PTDPTDI + i) % NPDEPG;
+			pdirma = pmap->pm_pdpt[pdir] & PG_FRAME;
+			mu[i].ptr = pdirma + offset * sizeof(pd_entry_t);
+			mu[i].val = 0;
+		}
+		HYPERVISOR_mmu_update(mu, 4, NULL, DOMID_SELF);
+	}
+}
+
+void
+pmap_resume()
+{
+	pmap_t pmap;
+	int i, pdir, offset;
+	vm_paddr_t pdirma;
+	mmu_update_t mu[4];
+
+	/*
+	 * Restore the recursive map that we removed on suspend.
+	 */
+	LIST_FOREACH(pmap, &allpmaps, pm_list) {
+		for (i = 0; i < 4; i++) {
+			/*
+			 * Figure out which page directory (L2) page
+			 * contains this bit of the recursive map and
+			 * the offset within that page of the map
+			 * entry
+			 */
+			pdir = (PTDPTDI + i) / NPDEPG;
+			offset = (PTDPTDI + i) % NPDEPG;
+			pdirma = pmap->pm_pdpt[pdir] & PG_FRAME;
+			mu[i].ptr = pdirma + offset * sizeof(pd_entry_t);
+			mu[i].val = (pmap->pm_pdpt[i] & PG_FRAME) | PG_V;
+		}
+		HYPERVISOR_mmu_update(mu, 4, NULL, DOMID_SELF);
+	}
+}
+
+#endif
+
 #if defined(PMAP_DEBUG)
 pmap_pid_dump(int pid)
 {
diff --git a/sys/i386/xen/xen_machdep.c b/sys/i386/xen/xen_machdep.c
index c99d754..878f436 100644
--- a/sys/i386/xen/xen_machdep.c
+++ b/sys/i386/xen/xen_machdep.c
@@ -89,6 +89,8 @@ start_info_t *xen_start_info;
 shared_info_t *HYPERVISOR_shared_info;
 xen_pfn_t *xen_machine_phys = machine_to_phys_mapping;
 xen_pfn_t *xen_phys_machine;
+xen_pfn_t *xen_pfn_to_mfn_frame_list[16];
+xen_pfn_t *xen_pfn_to_mfn_frame_list_list;
 int preemptable, init_first;
 extern unsigned int avail_space;
 
@@ -810,6 +812,39 @@ shift_phys_machine(unsigned long *phys_machine, int nr_pages)
 }
 #endif /* ADD_ISA_HOLE */
 
+/*
+ * Build a directory of the pages that make up our Physical to Machine
+ * mapping table. The Xen suspend/restore code uses this to find our
+ * mapping table.
+ */
+static void
+init_frame_list_list(void *arg)
+{
+	unsigned long nr_pages = xen_start_info->nr_pages;
+#define FPP	(PAGE_SIZE/sizeof(xen_pfn_t))
+	int i, j, k;
+
+	xen_pfn_to_mfn_frame_list_list = malloc(PAGE_SIZE, M_DEVBUF, M_WAITOK);
+	for (i = 0, j = 0, k = -1; i < nr_pages;
+	     i += FPP, j++) {
+		if ((j & (FPP - 1)) == 0) {
+			k++;
+			xen_pfn_to_mfn_frame_list[k] =
+				malloc(PAGE_SIZE, M_DEVBUF, M_WAITOK);
+			xen_pfn_to_mfn_frame_list_list[k] =
+				VTOMFN(xen_pfn_to_mfn_frame_list[k]);
+			j = 0;
+		}
+		xen_pfn_to_mfn_frame_list[k][j] = 
+			VTOMFN(&xen_phys_machine[i]);
+	}
+
+	HYPERVISOR_shared_info->arch.max_pfn = nr_pages;
+	HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list
+		= VTOMFN(xen_pfn_to_mfn_frame_list_list);
+}	
+SYSINIT(init_fll, SI_SUB_DEVFS, SI_ORDER_ANY, init_frame_list_list, NULL);
+
 extern unsigned long physfree;
 
 int pdir, curoffset;
@@ -1081,7 +1116,6 @@ initvalues(start_info_t *startinfo)
 	PT_SET_MA(console_page, console_page_ma | PG_KERNEL);
 
 	printk("#5\n");
-	HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = (unsigned long)xen_phys_machine;
 
 	set_iopl.iopl = 1;
 	PANIC_IF(HYPERVISOR_physdev_op(PHYSDEVOP_SET_IOPL, &set_iopl));
diff --git a/sys/dev/usb/usb_sw_transfer.h b/sys/ia64/ia32/ia32_misc.c
index d2da0eb..c85e31d 100644
--- a/sys/dev/usb/usb_sw_transfer.h
+++ b/sys/ia64/ia32/ia32_misc.c
@@ -1,6 +1,6 @@
-/* $FreeBSD$ */
 /*-
- * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
+ * Copyright (c) 2009 Konstantin Belousov
+ * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -24,39 +24,27 @@
  * SUCH DAMAGE.
  */
 
-#ifndef _USB2_SW_TRANSFER_H_
-#define	_USB2_SW_TRANSFER_H_
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
 
-/* Software transfer function state argument values */
+#include "opt_compat.h"
 
-enum {
-	USB_SW_TR_SETUP,
-	USB_SW_TR_STATUS,
-	USB_SW_TR_PRE_DATA,
-	USB_SW_TR_POST_DATA,
-	USB_SW_TR_PRE_CALLBACK,
-};
+#include <sys/param.h>
+#include <sys/mount.h>
+#include <sys/proc.h>
+#include <sys/socket.h>
+#include <sys/sysent.h>
+#include <sys/sysproto.h>
+#include <sys/systm.h>
+#include <sys/uio.h>
 
-struct usb2_sw_transfer;
+#include <compat/freebsd32/freebsd32_util.h>
+#include <compat/freebsd32/freebsd32.h>
+#include <compat/freebsd32/freebsd32_proto.h>
 
-typedef void (usb2_sw_transfer_func_t)(struct usb2_xfer *, struct usb2_sw_transfer *);
+int
+freebsd32_sysarch(struct thread *td, struct freebsd32_sysarch_args *uap)
+{
 
-/*
- * The following structure is used to keep the state of a standard
- * root transfer.
- */
-struct usb2_sw_transfer {
-	struct usb2_device_request req;
-	struct usb2_xfer *xfer;
-	uint8_t *ptr;
-	uint16_t len;
-	uint8_t	state;
-	usb2_error_t err;
-};
-
-/* prototypes */
-
-void	usb2_sw_transfer(struct usb2_sw_transfer *std,
-	    usb2_sw_transfer_func_t *func);
-
-#endif					/* _USB2_SW_TRANSFER_H_ */
+	return (EOPNOTSUPP);
+}
diff --git a/sys/ia64/ia64/elf_machdep.c b/sys/ia64/ia64/elf_machdep.c
index a3a6e57..3d3d214 100644
--- a/sys/ia64/ia64/elf_machdep.c
+++ b/sys/ia64/ia64/elf_machdep.c
@@ -92,7 +92,8 @@ static Elf64_Brandinfo freebsd_brand_info = {
 	.interp_path	= "/libexec/ld-elf.so.1",
 	.sysvec		= &elf64_freebsd_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &elf64_freebsd_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 SYSINIT(elf64, SI_SUB_EXEC, SI_ORDER_ANY,
     (sysinit_cfunc_t)elf64_insert_brand_entry, &freebsd_brand_info);
@@ -105,7 +106,8 @@ static Elf64_Brandinfo freebsd_brand_oinfo = {
 	.interp_path	= "/usr/libexec/ld-elf.so.1",
 	.sysvec		= &elf64_freebsd_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &elf64_freebsd_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 SYSINIT(oelf64, SI_SUB_EXEC, SI_ORDER_ANY,
     (sysinit_cfunc_t)elf64_insert_brand_entry, &freebsd_brand_oinfo);
diff --git a/sys/ia64/include/elf.h b/sys/ia64/include/elf.h
index faab8d1..65802aa 100644
--- a/sys/ia64/include/elf.h
+++ b/sys/ia64/include/elf.h
@@ -82,16 +82,14 @@ __ElfType(Auxinfo);
 #define	AT_BASE		7	/* Interpreter's base address. */
 #define	AT_FLAGS	8	/* Flags (unused for i386). */
 #define	AT_ENTRY	9	/* Where interpreter should transfer control. */
-/*
- * The following non-standard values are used in Linux ELF binaries.
- */
 #define	AT_NOTELF	10	/* Program is not ELF ?? */
 #define	AT_UID		11	/* Real uid. */
 #define	AT_EUID		12	/* Effective uid. */
 #define	AT_GID		13	/* Real gid. */
 #define	AT_EGID		14	/* Effective gid. */
+#define	AT_EXECPATH	15	/* Path to the executable. */
 
-#define	AT_COUNT	15	/* Count of defined aux entry types. */
+#define	AT_COUNT	16	/* Count of defined aux entry types. */
 
 /*
  * Values for e_flags.
diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c
index ec96974..e2c0a12 100644
--- a/sys/kern/imgact_elf.c
+++ b/sys/kern/imgact_elf.c
@@ -78,14 +78,16 @@ __FBSDID("$FreeBSD$");
 #define OLD_EI_BRAND	8
 
 static int __elfN(check_header)(const Elf_Ehdr *hdr);
-static Elf_Brandinfo *__elfN(get_brandinfo)(const Elf_Ehdr *hdr,
-    const char *interp);
+static Elf_Brandinfo *__elfN(get_brandinfo)(struct image_params *imgp,
+    const char *interp, int32_t *osrel);
 static int __elfN(load_file)(struct proc *p, const char *file, u_long *addr,
     u_long *entry, size_t pagesize);
 static int __elfN(load_section)(struct vmspace *vmspace, vm_object_t object,
     vm_offset_t offset, caddr_t vmaddr, size_t memsz, size_t filsz,
     vm_prot_t prot, size_t pagesize);
 static int __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp);
+static boolean_t __elfN(check_note)(struct image_params *imgp,
+    Elf_Brandnote *checknote, int32_t *osrel);
 
 SYSCTL_NODE(_kern, OID_AUTO, __CONCAT(elf, __ELF_WORD_SIZE), CTLFLAG_RW, 0,
     "");
@@ -107,6 +109,16 @@ static Elf_Brandinfo *elf_brand_list[MAX_BRANDS];
 #define	round_page_ps(va, ps)	(((va) + (ps - 1)) & ~(ps - 1))
 #define	aligned(a, t)	(trunc_page_ps((u_long)(a), sizeof(t)) == (u_long)(a))
 
+static const char FREEBSD_ABI_VENDOR[] = "FreeBSD";
+
+Elf_Brandnote __elfN(freebsd_brandnote) = {
+	.hdr.n_namesz	= sizeof(FREEBSD_ABI_VENDOR),
+	.hdr.n_descsz	= sizeof(int32_t),
+	.hdr.n_type	= 1,
+	.vendor		= FREEBSD_ABI_VENDOR,
+	.flags		= BN_CAN_FETCH_OSREL
+};
+
 int
 __elfN(insert_brand_entry)(Elf_Brandinfo *entry)
 {
@@ -158,19 +170,32 @@ __elfN(brand_inuse)(Elf_Brandinfo *entry)
 }
 
 static Elf_Brandinfo *
-__elfN(get_brandinfo)(const Elf_Ehdr *hdr, const char *interp)
+__elfN(get_brandinfo)(struct image_params *imgp, const char *interp,
+    int32_t *osrel)
 {
+	const Elf_Ehdr *hdr = (const Elf_Ehdr *)imgp->image_header;
 	Elf_Brandinfo *bi;
+	boolean_t ret;
 	int i;
 
 	/*
-	 * We support three types of branding -- (1) the ELF EI_OSABI field
+	 * We support four types of branding -- (1) the ELF EI_OSABI field
 	 * that SCO added to the ELF spec, (2) FreeBSD 3.x's traditional string
-	 * branding w/in the ELF header, and (3) path of the `interp_path'
-	 * field.  We should also look for an ".note.ABI-tag" ELF section now
-	 * in all Linux ELF binaries, FreeBSD 4.1+, and some NetBSD ones.
+	 * branding w/in the ELF header, (3) path of the `interp_path'
+	 * field, and (4) the ".note.ABI-tag" ELF section.
 	 */
 
+	/* Look for an ".note.ABI-tag" ELF section */
+	for (i = 0; i < MAX_BRANDS; i++) {
+		bi = elf_brand_list[i];
+		if (bi != NULL && hdr->e_machine == bi->machine &&
+		    (bi->flags & BI_BRAND_NOTE) != 0) {
+			ret = __elfN(check_note)(imgp, bi->brand_note, osrel);
+			if (ret)
+				return (bi);
+		}
+	}
+
 	/* If the executable has a brand, search for it in the brand list. */
 	for (i = 0; i < MAX_BRANDS; i++) {
 		bi = elf_brand_list[i];
@@ -590,13 +615,11 @@ fail:
 	return (error);
 }
 
-static const char FREEBSD_ABI_VENDOR[] = "FreeBSD";
-
 static int
 __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 {
 	const Elf_Ehdr *hdr = (const Elf_Ehdr *)imgp->image_header;
-	const Elf_Phdr *phdr, *pnote = NULL;
+	const Elf_Phdr *phdr;
 	Elf_Auxargs *elf_auxargs;
 	struct vmspace *vmspace;
 	vm_prot_t prot;
@@ -604,12 +627,11 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 	u_long text_addr = 0, data_addr = 0;
 	u_long seg_size, seg_addr;
 	u_long addr, entry = 0, proghdr = 0;
+	int32_t osrel = 0;
 	int error = 0, i;
 	const char *interp = NULL, *newinterp = NULL;
 	Elf_Brandinfo *brand_info;
-	const Elf_Note *note, *note_end;
 	char *path;
-	const char *note_name;
 	struct sysentvec *sv;
 
 	/*
@@ -646,7 +668,7 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 		}
 	}
 
-	brand_info = __elfN(get_brandinfo)(hdr, interp);
+	brand_info = __elfN(get_brandinfo)(imgp, interp, &osrel);
 	if (brand_info == NULL) {
 		uprintf("ELF binary type \"%u\" not known.\n",
 		    hdr->e_ident[EI_OSABI]);
@@ -750,9 +772,6 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 		case PT_PHDR: 	/* Program header table info */
 			proghdr = phdr[i].p_vaddr;
 			break;
-		case PT_NOTE:
-			pnote = &phdr[i];
-			break;
 		default:
 			break;
 		}
@@ -840,41 +859,7 @@ __CONCAT(exec_, __elfN(imgact))(struct image_params *imgp)
 
 	imgp->auxargs = elf_auxargs;
 	imgp->interpreted = 0;
-
-	/*
-	 * Try to fetch the osreldate for FreeBSD binary from the ELF
-	 * OSABI-note. Only the first page of the image is searched,
-	 * the same as for headers.
-	 */
-	if (pnote != NULL && pnote->p_offset < PAGE_SIZE &&
-	    pnote->p_offset + pnote->p_filesz < PAGE_SIZE ) {
-		note = (const Elf_Note *)(imgp->image_header + pnote->p_offset);
-		if (!aligned(note, Elf32_Addr)) {
-			free(imgp->auxargs, M_TEMP);
-			imgp->auxargs = NULL;
-			return (ENOEXEC);
-		}
-		note_end = (const Elf_Note *)(imgp->image_header + pnote->p_offset +
-		    pnote->p_filesz);
-		while (note < note_end) {
-			if (note->n_namesz == sizeof(FREEBSD_ABI_VENDOR) &&
-			    note->n_descsz == sizeof(int32_t) &&
-			    note->n_type == 1 /* ABI_NOTETYPE */) {
-				note_name = (const char *)(note + 1);
-				if (strncmp(FREEBSD_ABI_VENDOR, note_name,
-				    sizeof(FREEBSD_ABI_VENDOR)) == 0) {
-					imgp->proc->p_osrel = *(const int32_t *)
-					    (note_name +
-					    round_page_ps(sizeof(FREEBSD_ABI_VENDOR),
-						sizeof(Elf32_Addr)));
-					break;
-				}
-			}
-			note = (const Elf_Note *)((const char *)(note + 1) +
-			    round_page_ps(note->n_namesz, sizeof(Elf32_Addr)) +
-			    round_page_ps(note->n_descsz, sizeof(Elf32_Addr)));
-		}
-	}
+	imgp->proc->p_osrel = osrel;
 
 	return (error);
 }
@@ -900,6 +885,8 @@ __elfN(freebsd_fixup)(register_t **stack_base, struct image_params *imgp)
 	AUXARGS_ENTRY(pos, AT_FLAGS, args->flags);
 	AUXARGS_ENTRY(pos, AT_ENTRY, args->entry);
 	AUXARGS_ENTRY(pos, AT_BASE, args->base);
+	if (imgp->execpathp != 0)
+		AUXARGS_ENTRY(pos, AT_EXECPATH, imgp->execpathp);
 	AUXARGS_ENTRY(pos, AT_NULL, 0);
 
 	free(imgp->auxargs, M_TEMP);
@@ -1336,6 +1323,71 @@ __elfN(putnote)(void *dst, size_t *off, const char *name, int type,
 }
 
 /*
+ * Try to find the appropriate ABI-note section for checknote,
+ * fetch the osreldate for binary from the ELF OSABI-note. Only the
+ * first page of the image is searched, the same as for headers.
+ */
+static boolean_t
+__elfN(check_note)(struct image_params *imgp, Elf_Brandnote *checknote,
+    int32_t *osrel)
+{
+	const Elf_Note *note, *note0, *note_end;
+	const Elf_Phdr *phdr, *pnote;
+	const Elf_Ehdr *hdr;
+	const char *note_name;
+	int i;
+
+	pnote = NULL;
+	hdr = (const Elf_Ehdr *)imgp->image_header;
+	phdr = (const Elf_Phdr *)(imgp->image_header + hdr->e_phoff);
+
+	for (i = 0; i < hdr->e_phnum; i++) {
+		if (phdr[i].p_type == PT_NOTE) {
+			pnote = &phdr[i];
+			break;
+		}
+	}
+
+	if (pnote == NULL || pnote->p_offset >= PAGE_SIZE ||
+	    pnote->p_offset + pnote->p_filesz >= PAGE_SIZE)
+		return (FALSE);
+
+	note = note0 = (const Elf_Note *)(imgp->image_header + pnote->p_offset);
+	note_end = (const Elf_Note *)(imgp->image_header +
+	    pnote->p_offset + pnote->p_filesz);
+	for (i = 0; i < 100 && note >= note0 && note < note_end; i++) {
+		if (!aligned(note, Elf32_Addr))
+			return (FALSE);
+		if (note->n_namesz != checknote->hdr.n_namesz ||
+		    note->n_descsz != checknote->hdr.n_descsz ||
+		    note->n_type != checknote->hdr.n_type)
+			goto nextnote;
+		note_name = (const char *)(note + 1);
+		if (strncmp(checknote->vendor, note_name,
+		    checknote->hdr.n_namesz) != 0)
+			goto nextnote;
+
+		/*
+		 * Fetch the osreldate for binary
+		 * from the ELF OSABI-note if necessary.
+		 */
+		if ((checknote->flags & BN_CAN_FETCH_OSREL) != 0 &&
+		    osrel != NULL)
+			*osrel = *(const int32_t *) (note_name +
+			    roundup2(checknote->hdr.n_namesz,
+			    sizeof(Elf32_Addr)));
+		return (TRUE);
+
+nextnote:
+		note = (const Elf_Note *)((const char *)(note + 1) +
+		    roundup2(note->n_namesz, sizeof(Elf32_Addr)) +
+		    roundup2(note->n_descsz, sizeof(Elf32_Addr)));
+	}
+
+	return (FALSE);
+}
+
+/*
  * Tell kern_execve.c about it, with a little help from the linker.
  */
 static struct execsw __elfN(execsw) = {
diff --git a/sys/kern/kern_acct.c b/sys/kern/kern_acct.c
index d45e3b7..8355ca5 100644
--- a/sys/kern/kern_acct.c
+++ b/sys/kern/kern_acct.c
@@ -436,7 +436,6 @@ acct_process(struct thread *td)
 	 * Write the accounting information to the file.
 	 */
 	vfslocked = VFS_LOCK_GIANT(acct_vp->v_mount);
-	VOP_LEASE(acct_vp, td, acct_cred, LEASE_WRITE);
 	ret = vn_rdwr(UIO_WRITE, acct_vp, (caddr_t)&acct, sizeof (acct),
 	    (off_t)0, UIO_SYSSPACE, IO_APPEND|IO_UNIT, acct_cred, NOCRED,
 	    (int *)0, td);
diff --git a/sys/kern/kern_alq.c b/sys/kern/kern_alq.c
index 512f358..9f37244 100644
--- a/sys/kern/kern_alq.c
+++ b/sys/kern/kern_alq.c
@@ -293,7 +293,6 @@ alq_doio(struct alq *alq)
 	vfslocked = VFS_LOCK_GIANT(vp->v_mount);
 	vn_start_write(vp, &mp, V_WAIT);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
-	VOP_LEASE(vp, td, alq->aq_cred, LEASE_WRITE);
 	/*
 	 * XXX: VOP_WRITE error checks are ignored.
 	 */
diff --git a/sys/kern/kern_environment.c b/sys/kern/kern_environment.c
index a5658ca..038515b 100644
--- a/sys/kern/kern_environment.c
+++ b/sys/kern/kern_environment.c
@@ -87,7 +87,7 @@ kenv(td, uap)
 	} */ *uap;
 {
 	char *name, *value, *buffer = NULL;
-	size_t len, done, needed;
+	size_t len, done, needed, buflen;
 	int error, i;
 
 	KASSERT(dynamic_kenv, ("kenv: dynamic_kenv = 0"));
@@ -100,13 +100,17 @@ kenv(td, uap)
 			return (error);
 #endif
 		done = needed = 0;
+		buflen = uap->len;
+		if (buflen > KENV_SIZE * (KENV_MNAMELEN + KENV_MVALLEN + 2))
+			buflen = KENV_SIZE * (KENV_MNAMELEN +
+			    KENV_MVALLEN + 2);
 		if (uap->len > 0 && uap->value != NULL)
-			buffer = malloc(uap->len, M_TEMP, M_WAITOK|M_ZERO);
+			buffer = malloc(buflen, M_TEMP, M_WAITOK|M_ZERO);
 		mtx_lock(&kenv_lock);
 		for (i = 0; kenvp[i] != NULL; i++) {
 			len = strlen(kenvp[i]) + 1;
 			needed += len;
-			len = min(len, uap->len - done);
+			len = min(len, buflen - done);
 			/*
 			 * If called with a NULL or insufficiently large
 			 * buffer, just keep computing the required size.
diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c
index af6c9e4..f36f803 100644
--- a/sys/kern/kern_exec.c
+++ b/sys/kern/kern_exec.c
@@ -379,6 +379,8 @@ do_execve(td, args, mac_p)
 	imgp->ps_strings = 0;
 	imgp->auxarg_size = 0;
 	imgp->args = args;
+	imgp->execpath = imgp->freepath = NULL;
+	imgp->execpathp = 0;
 
 #ifdef MAC
 	error = mac_execve_enter(imgp, mac_p);
@@ -519,6 +521,15 @@ interpret:
 	 * of the sv_copyout_strings/sv_fixup operations require the vnode.
 	 */
 	VOP_UNLOCK(imgp->vp, 0);
+
+	/*
+	 * Do the best to calculate the full path to the image file.
+	 */
+	if (imgp->auxargs != NULL &&
+	    ((args->fname != NULL && args->fname[0] == '/') ||
+	     vn_fullpath(td, imgp->vp, &imgp->execpath, &imgp->freepath) != 0))
+		imgp->execpath = args->fname;
+
 	/*
 	 * Copy out strings (args and env) and initialize stack base
 	 */
@@ -859,6 +870,8 @@ exec_fail_dealloc:
 	if (imgp->object != NULL)
 		vm_object_deallocate(imgp->object);
 
+	free(imgp->freepath, M_TEMP);
+
 	if (error == 0) {
 		/*
 		 * Stop the process here if its stop event mask has
@@ -1164,18 +1177,24 @@ exec_copyout_strings(imgp)
 	register_t *stack_base;
 	struct ps_strings *arginfo;
 	struct proc *p;
+	size_t execpath_len;
 	int szsigcode;
 
 	/*
 	 * Calculate string base and vector table pointers.
 	 * Also deal with signal trampoline code for this exec type.
 	 */
+	if (imgp->execpath != NULL && imgp->auxargs != NULL)
+		execpath_len = strlen(imgp->execpath) + 1;
+	else
+		execpath_len = 0;
 	p = imgp->proc;
 	szsigcode = 0;
 	arginfo = (struct ps_strings *)p->p_sysent->sv_psstrings;
 	if (p->p_sysent->sv_szsigcode != NULL)
 		szsigcode = *(p->p_sysent->sv_szsigcode);
 	destp =	(caddr_t)arginfo - szsigcode - SPARE_USRSPACE -
+	    roundup(execpath_len, sizeof(char *)) -
 	    roundup((ARG_MAX - imgp->args->stringspace), sizeof(char *));
 
 	/*
@@ -1186,6 +1205,15 @@ exec_copyout_strings(imgp)
 		    szsigcode), szsigcode);
 
 	/*
+	 * Copy the image path for the rtld.
+	 */
+	if (execpath_len != 0) {
+		imgp->execpathp = (uintptr_t)arginfo - szsigcode - execpath_len;
+		copyout(imgp->execpath, (void *)imgp->execpathp,
+		    execpath_len);
+	}
+
+	/*
 	 * If we have a valid auxargs ptr, prepare some room
 	 * on the stack.
 	 */
@@ -1202,9 +1230,8 @@ exec_copyout_strings(imgp)
 		 * for argument of Runtime loader.
 		 */
 		vectp = (char **)(destp - (imgp->args->argc +
-		    imgp->args->envc + 2 + imgp->auxarg_size) *
+		    imgp->args->envc + 2 + imgp->auxarg_size + execpath_len) *
 		    sizeof(char *));
-
 	} else {
 		/*
 		 * The '+ 2' is for the null pointers at the end of each of
diff --git a/sys/kern/kern_jail.c b/sys/kern/kern_jail.c
index e802780..00be7aa 100644
--- a/sys/kern/kern_jail.c
+++ b/sys/kern/kern_jail.c
@@ -2,7 +2,7 @@
  * Copyright (c) 1999 Poul-Henning Kamp.
  * Copyright (c) 2008 Bjoern A. Zeeb.
  * All rights reserved.
- * 
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -119,14 +119,14 @@ struct	sx allprison_lock;
 int	lastprid = 0;
 int	prisoncount = 0;
 
-static void		 init_prison(void *);
-static void		 prison_complete(void *context, int pending);
-static int		 sysctl_jail_list(SYSCTL_HANDLER_ARGS);
+static void init_prison(void *);
+static void prison_complete(void *context, int pending);
+static int sysctl_jail_list(SYSCTL_HANDLER_ARGS);
 #ifdef INET
-static int		_prison_check_ip4(struct prison *, struct in_addr *);
+static int _prison_check_ip4(struct prison *pr, struct in_addr *ia);
 #endif
 #ifdef INET6
-static int		_prison_check_ip6(struct prison *, struct in6_addr *);
+static int _prison_check_ip6(struct prison *pr, struct in6_addr *ia6);
 #endif
 
 static void
@@ -177,7 +177,7 @@ qcmp_v6(const void *ip1, const void *ip2)
 	ia6b = (const struct in6_addr *)ip2;
 
 	rc = 0;
-	for (i=0; rc == 0 && i < sizeof(struct in6_addr); i++) {
+	for (i = 0; rc == 0 && i < sizeof(struct in6_addr); i++) {
 		if (ia6a->s6_addr[i] > ia6b->s6_addr[i])
 			rc = 1;
 		else if (ia6a->s6_addr[i] < ia6b->s6_addr[i])
@@ -240,7 +240,7 @@ static int
 jail_copyin_ips(struct jail *j)
 {
 #ifdef INET
-	struct in_addr  *ip4;
+	struct in_addr *ip4;
 #endif
 #ifdef INET6
 	struct in6_addr *ip6;
@@ -348,7 +348,7 @@ jail_handle_ips(struct jail *j)
 	 * Finish conversion for older versions, copyin and setup IPs.
 	 */
 	switch (j->version) {
-	case 0:	
+	case 0:
 	{
 #ifdef INET
 		/* FreeBSD single IPv4 jails. */
@@ -594,6 +594,7 @@ e_killmtx:
 	return (error);
 }
 
+
 /*
  * struct jail_attach_args {
  *	int jid;
@@ -807,7 +808,7 @@ prison_proc_free(struct prison *pr)
  * Pass back primary IPv4 address of this jail.
  *
  * If not jailed return success but do not alter the address.  Caller has to
- * make sure to intialize it correctly (e.g. INADDR_ANY).
+ * make sure to initialize it correctly (e.g. INADDR_ANY).
  *
  * Returns 0 on success, EAFNOSUPPORT if the jail doesn't allow IPv4.
  * Address returned in NBO.
@@ -822,7 +823,6 @@ prison_get_ip4(struct ucred *cred, struct in_addr *ia)
 	if (!jailed(cred))
 		/* Do not change address passed in. */
 		return (0);
-
 	if (cred->cr_prison->pr_ip4 == NULL)
 		return (EAFNOSUPPORT);
 
@@ -956,7 +956,7 @@ prison_check_ip4(struct ucred *cred, struct in_addr *ia)
  * Pass back primary IPv6 address for this jail.
  *
  * If not jailed return success but do not alter the address.  Caller has to
- * make sure to intialize it correctly (e.g. IN6ADDR_ANY_INIT).
+ * make sure to initialize it correctly (e.g. IN6ADDR_ANY_INIT).
  *
  * Returns 0 on success, EAFNOSUPPORT if the jail doesn't allow IPv6.
  */
diff --git a/sys/kern/kern_ktrace.c b/sys/kern/kern_ktrace.c
index 6ccf2c0..6e680fb 100644
--- a/sys/kern/kern_ktrace.c
+++ b/sys/kern/kern_ktrace.c
@@ -992,7 +992,6 @@ ktr_writerequest(struct thread *td, struct ktr_request *req)
 	vfslocked = VFS_LOCK_GIANT(vp->v_mount);
 	vn_start_write(vp, &mp, V_WAIT);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
-	(void)VOP_LEASE(vp, td, cred, LEASE_WRITE);
 #ifdef MAC
 	error = mac_vnode_check_write(cred, NOCRED, vp);
 	if (error == 0)
diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c
index b2294f8..f19e249 100644
--- a/sys/kern/kern_lock.c
+++ b/sys/kern/kern_lock.c
@@ -333,16 +333,17 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
     const char *wmesg, int pri, int timo, const char *file, int line)
 {
 	GIANT_DECLARE;
-	uint64_t waittime;
 	struct lock_class *class;
 	const char *iwmesg;
 	uintptr_t tid, v, x;
 	u_int op;
-	int contested, error, ipri, itimo, queue, wakeup_swapper;
+	int error, ipri, itimo, queue, wakeup_swapper;
+#ifdef LOCK_PROFILING
+	uint64_t waittime = 0;
+	int contested = 0;
+#endif
 
-	contested = 0;
 	error = 0;
-	waittime = 0;
 	tid = (uintptr_t)curthread;
 	op = (flags & LK_TYPE_MASK);
 	iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg;
@@ -686,7 +687,8 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
 				lk->lk_recurse--;
 				break;
 			}
-			lock_profile_release_lock(&lk->lock_object);
+			if (tid != LK_KERNPROC)
+				lock_profile_release_lock(&lk->lock_object);
 
 			if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid,
 			    LK_UNLOCKED))
@@ -874,6 +876,7 @@ _lockmgr_disown(struct lock *lk, const char *file, int line)
 	 */
 	if (LK_HOLDER(lk->lk_lock) != tid)
 		return;
+	lock_profile_release_lock(&lk->lock_object);
 	LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file, line);
 	WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
 	TD_LOCKS_DEC(curthread);
diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c
index 523d911..c56e124 100644
--- a/sys/kern/kern_mutex.c
+++ b/sys/kern/kern_mutex.c
@@ -254,8 +254,11 @@ _mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line)
 int
 _mtx_trylock(struct mtx *m, int opts, const char *file, int line)
 {
-	int rval, contested = 0;
+#ifdef LOCK_PROFILING
 	uint64_t waittime = 0;
+	int contested = 0;
+#endif
+	int rval;
 
 	MPASS(curthread != NULL);
 	KASSERT(m->mtx_lock != MTX_DESTROYED,
@@ -296,15 +299,17 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file,
     int line)
 {
 	struct turnstile *ts;
+	uintptr_t v;
 #ifdef ADAPTIVE_MUTEXES
 	volatile struct thread *owner;
 #endif
 #ifdef KTR
 	int cont_logged = 0;
 #endif
+#ifdef LOCK_PROFILING
 	int contested = 0;
 	uint64_t waittime = 0;
-	uintptr_t v;
+#endif
 
 	if (mtx_owned(m)) {
 		KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0,
@@ -448,8 +453,11 @@ void
 _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file,
     int line)
 {
-	int i = 0, contested = 0;
+	int i = 0;
+#ifdef LOCK_PROFILING
+	int contested = 0;
 	uint64_t waittime = 0;
+#endif
 
 	if (LOCK_LOG_TEST(&m->lock_object, opts))
 		CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m);
@@ -486,11 +494,13 @@ _thread_lock_flags(struct thread *td, int opts, const char *file, int line)
 {
 	struct mtx *m;
 	uintptr_t tid;
-	int i, contested;
-	uint64_t waittime;
+	int i;
+#ifdef LOCK_PROFILING
+	int contested = 0;
+	uint64_t waittime = 0;
+#endif
 
-	contested = i = 0;
-	waittime = 0;
+	i = 0;
 	tid = (uintptr_t)curthread;
 	for (;;) {
 retry:
@@ -765,6 +775,7 @@ mtx_destroy(struct mtx *m)
 		else
 			curthread->td_locks--;
 
+		lock_profile_release_lock(&m->lock_object);
 		/* Tell witness this isn't locked to make it happy. */
 		WITNESS_UNLOCK(&m->lock_object, LOP_EXCLUSIVE, __FILE__,
 		    __LINE__);
diff --git a/sys/kern/kern_poll.c b/sys/kern/kern_poll.c
index ca11177..8ca6b93 100644
--- a/sys/kern/kern_poll.c
+++ b/sys/kern/kern_poll.c
@@ -551,9 +551,7 @@ poll_switch(SYSCTL_HANDLER_ARGS)
 			else
 				ifr.ifr_reqcap =
 				    ifp->if_capenable & ~IFCAP_POLLING;
-			IFF_LOCKGIANT(ifp);	/* LOR here */
 			(void) (*ifp->if_ioctl)(ifp, SIOCSIFCAP, (caddr_t)&ifr);
-			IFF_UNLOCKGIANT(ifp);
 		}
 	}
 	IFNET_RUNLOCK();
diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c
index 531e145..ccaa690 100644
--- a/sys/kern/kern_rwlock.c
+++ b/sys/kern/kern_rwlock.c
@@ -282,8 +282,10 @@ _rw_rlock(struct rwlock *rw, const char *file, int line)
 	int spintries = 0;
 	int i;
 #endif
+#ifdef LOCK_PROFILING
 	uint64_t waittime = 0;
 	int contested = 0;
+#endif
 	uintptr_t v;
 
 	KASSERT(rw->rw_lock != RW_DESTROYED,
@@ -584,9 +586,11 @@ _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
 	int spintries = 0;
 	int i;
 #endif
-	uint64_t waittime = 0;
 	uintptr_t v, x;
+#ifdef LOCK_PROFILING
+	uint64_t waittime = 0;
 	int contested = 0;
+#endif
 
 	if (rw_wlocked(rw)) {
 		KASSERT(rw->lock_object.lo_flags & RW_RECURSE,
diff --git a/sys/kern/kern_shutdown.c b/sys/kern/kern_shutdown.c
index 157da53..7702ad8 100644
--- a/sys/kern/kern_shutdown.c
+++ b/sys/kern/kern_shutdown.c
@@ -680,15 +680,6 @@ dump_write(struct dumperinfo *di, void *virtual, vm_offset_t physical,
 	return (di->dumper(di->priv, virtual, physical, offset, length));
 }
 
-#if defined(__powerpc__)
-void
-dumpsys(struct dumperinfo *di __unused)
-{
-
-	printf("Kernel dumps not implemented on this architecture\n");
-}
-#endif
-
 void
 mkdumpheader(struct kerneldumpheader *kdh, char *magic, uint32_t archver,
     uint64_t dumplen, uint32_t blksz)
diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c
index 7460c13..5c1d553 100644
--- a/sys/kern/kern_sig.c
+++ b/sys/kern/kern_sig.c
@@ -2984,7 +2984,6 @@ restart:
 	if (set_core_nodump_flag)
 		vattr.va_flags = UF_NODUMP;
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
-	VOP_LEASE(vp, td, cred, LEASE_WRITE);
 	VOP_SETATTR(vp, &vattr, cred);
 	VOP_UNLOCK(vp, 0);
 	vn_finished_write(mp);
diff --git a/sys/kern/kern_sx.c b/sys/kern/kern_sx.c
index e1e34ec..9a2c9a2 100644
--- a/sys/kern/kern_sx.c
+++ b/sys/kern/kern_sx.c
@@ -431,9 +431,12 @@ _sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, const char *file,
 #ifdef ADAPTIVE_SX
 	volatile struct thread *owner;
 #endif
-	uint64_t waittime = 0;
 	uintptr_t x;
-	int contested = 0, error = 0;
+#ifdef LOCK_PROFILING
+	uint64_t waittime = 0;
+	int contested = 0;
+#endif
+	int error = 0;
 
 	/* If we already hold an exclusive lock, then recurse. */
 	if (sx_xlocked(sx)) {
@@ -652,8 +655,10 @@ _sx_slock_hard(struct sx *sx, int opts, const char *file, int line)
 #ifdef ADAPTIVE_SX
 	volatile struct thread *owner;
 #endif
+#ifdef LOCK_PROFILING
 	uint64_t waittime = 0;
 	int contested = 0;
+#endif
 	uintptr_t x;
 	int error = 0;
 
diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c
index 6409842..48dc4f8 100644
--- a/sys/kern/kern_tc.c
+++ b/sys/kern/kern_tc.c
@@ -103,24 +103,6 @@ static int timestepwarnings;
 SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW,
     &timestepwarnings, 0, "");
 
-#ifdef TC_COUNTERS
-#define TC_STATS(foo) \
-	static u_int foo; \
-	SYSCTL_UINT(_kern_timecounter, OID_AUTO, foo, CTLFLAG_RD, &foo, 0, "");\
-	struct __hack
-
-TC_STATS(nbinuptime);    TC_STATS(nnanouptime);    TC_STATS(nmicrouptime);
-TC_STATS(nbintime);      TC_STATS(nnanotime);      TC_STATS(nmicrotime);
-TC_STATS(ngetbinuptime); TC_STATS(ngetnanouptime); TC_STATS(ngetmicrouptime);
-TC_STATS(ngetbintime);   TC_STATS(ngetnanotime);   TC_STATS(ngetmicrotime);
-TC_STATS(nsetclock);
-
-#define	TC_COUNT(var)	var++
-#undef TC_STATS
-#else
-#define	TC_COUNT(var)	/* nothing */
-#endif /* TC_COUNTERS */
-
 static void tc_windup(void);
 static void cpu_tick_calibrate(int);
 
@@ -185,7 +167,6 @@ binuptime(struct bintime *bt)
 	struct timehands *th;
 	u_int gen;
 
-	TC_COUNT(nbinuptime);
 	do {
 		th = timehands;
 		gen = th->th_generation;
@@ -199,7 +180,6 @@ nanouptime(struct timespec *tsp)
 {
 	struct bintime bt;
 
-	TC_COUNT(nnanouptime);
 	binuptime(&bt);
 	bintime2timespec(&bt, tsp);
 }
@@ -209,7 +189,6 @@ microuptime(struct timeval *tvp)
 {
 	struct bintime bt;
 
-	TC_COUNT(nmicrouptime);
 	binuptime(&bt);
 	bintime2timeval(&bt, tvp);
 }
@@ -218,7 +197,6 @@ void
 bintime(struct bintime *bt)
 {
 
-	TC_COUNT(nbintime);
 	binuptime(bt);
 	bintime_add(bt, &boottimebin);
 }
@@ -228,7 +206,6 @@ nanotime(struct timespec *tsp)
 {
 	struct bintime bt;
 
-	TC_COUNT(nnanotime);
 	bintime(&bt);
 	bintime2timespec(&bt, tsp);
 }
@@ -238,7 +215,6 @@ microtime(struct timeval *tvp)
 {
 	struct bintime bt;
 
-	TC_COUNT(nmicrotime);
 	bintime(&bt);
 	bintime2timeval(&bt, tvp);
 }
@@ -249,7 +225,6 @@ getbinuptime(struct bintime *bt)
 	struct timehands *th;
 	u_int gen;
 
-	TC_COUNT(ngetbinuptime);
 	do {
 		th = timehands;
 		gen = th->th_generation;
@@ -263,7 +238,6 @@ getnanouptime(struct timespec *tsp)
 	struct timehands *th;
 	u_int gen;
 
-	TC_COUNT(ngetnanouptime);
 	do {
 		th = timehands;
 		gen = th->th_generation;
@@ -277,7 +251,6 @@ getmicrouptime(struct timeval *tvp)
 	struct timehands *th;
 	u_int gen;
 
-	TC_COUNT(ngetmicrouptime);
 	do {
 		th = timehands;
 		gen = th->th_generation;
@@ -291,7 +264,6 @@ getbintime(struct bintime *bt)
 	struct timehands *th;
 	u_int gen;
 
-	TC_COUNT(ngetbintime);
 	do {
 		th = timehands;
 		gen = th->th_generation;
@@ -306,7 +278,6 @@ getnanotime(struct timespec *tsp)
 	struct timehands *th;
 	u_int gen;
 
-	TC_COUNT(ngetnanotime);
 	do {
 		th = timehands;
 		gen = th->th_generation;
@@ -320,7 +291,6 @@ getmicrotime(struct timeval *tvp)
 	struct timehands *th;
 	u_int gen;
 
-	TC_COUNT(ngetmicrotime);
 	do {
 		th = timehands;
 		gen = th->th_generation;
@@ -411,7 +381,6 @@ tc_setclock(struct timespec *ts)
 	struct bintime bt, bt2;
 
 	cpu_tick_calibrate(1);
-	TC_COUNT(nsetclock);
 	nanotime(&tbef);
 	timespec2bintime(ts, &bt);
 	binuptime(&bt2);
diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c
index 4606005..bb8779b 100644
--- a/sys/kern/kern_thread.c
+++ b/sys/kern/kern_thread.c
@@ -306,6 +306,8 @@ thread_alloc(void)
 void
 thread_free(struct thread *td)
 {
+
+	lock_profile_thread_exit(td);
 	if (td->td_cpuset)
 		cpuset_rel(td->td_cpuset);
 	td->td_cpuset = NULL;
@@ -439,6 +441,7 @@ thread_wait(struct proc *p)
 	/* Wait for any remaining threads to exit cpu_throw(). */
 	while (p->p_exitthreads)
 		sched_relinquish(curthread);
+	lock_profile_thread_exit(td);
 	cpuset_rel(td->td_cpuset);
 	td->td_cpuset = NULL;
 	cpu_thread_clean(td);
diff --git a/sys/kern/kern_time.c b/sys/kern/kern_time.c
index a8ecace..3aea2bd 100644
--- a/sys/kern/kern_time.c
+++ b/sys/kern/kern_time.c
@@ -74,7 +74,6 @@ static uma_zone_t	itimer_zone = NULL;
 
 static int	settime(struct thread *, struct timeval *);
 static void	timevalfix(struct timeval *);
-static void	no_lease_updatetime(int);
 
 static void	itimer_start(void);
 static int	itimer_init(void *, int, int);
@@ -106,14 +105,6 @@ int		itimespecfix(struct timespec *ts);
 SYSINIT(posix_timer, SI_SUB_P1003_1B, SI_ORDER_FIRST+4, itimer_start, NULL);
 
 
-static void 
-no_lease_updatetime(deltat)
-	int deltat;
-{
-}
-
-void (*lease_updatetime)(int)  = no_lease_updatetime;
-
 static int
 settime(struct thread *td, struct timeval *tv)
 {
@@ -168,9 +159,6 @@ settime(struct thread *td, struct timeval *tv)
 	ts.tv_nsec = tv->tv_usec * 1000;
 	mtx_lock(&Giant);
 	tc_setclock(&ts);
-	(void) splsoftclock();
-	lease_updatetime(delta.tv_sec);
-	splx(s);
 	resettodr();
 	mtx_unlock(&Giant);
 	return (0);
@@ -1085,7 +1073,8 @@ itimer_find(struct proc *p, int timerid)
 	struct itimer *it;
 
 	PROC_LOCK_ASSERT(p, MA_OWNED);
-	if ((p->p_itimers == NULL) || (timerid >= TIMER_MAX) ||
+	if ((p->p_itimers == NULL) ||
+	    (timerid < 0) || (timerid >= TIMER_MAX) ||
 	    (it = p->p_itimers->its_timers[timerid]) == NULL) {
 		return (NULL);
 	}
diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c
index 09573ab..46e06c0 100644
--- a/sys/kern/kern_umtx.c
+++ b/sys/kern/kern_umtx.c
@@ -166,6 +166,7 @@ struct umtxq_chain {
 };
 
 #define	UMTXQ_LOCKED_ASSERT(uc)		mtx_assert(&(uc)->uc_lock, MA_OWNED)
+#define	UMTXQ_BUSY_ASSERT(uc)	KASSERT(&(uc)->uc_busy, ("umtx chain is not busy"))
 
 /*
  * Don't propagate time-sharing priority, there is a security reason,
@@ -1392,7 +1393,8 @@ umtx_unpropagate_priority(struct umtx_pi *pi)
 		oldpri = pi->pi_owner->td_user_pri;
 		sched_unlend_user_prio(pi->pi_owner, pri);
 		thread_unlock(pi->pi_owner);
-		umtx_pi_adjust_locked(pi->pi_owner, oldpri);
+		if (uq_owner->uq_pi_blocked != NULL)
+			umtx_pi_adjust_locked(pi->pi_owner, oldpri);
 		pi = uq_owner->uq_pi_blocked;
 	}
 }
@@ -1513,7 +1515,9 @@ umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi,
 	KASSERT(td == curthread, ("inconsistent uq_thread"));
 	uc = umtxq_getchain(&uq->uq_key);
 	UMTXQ_LOCKED_ASSERT(uc);
+	UMTXQ_BUSY_ASSERT(uc);
 	umtxq_insert(uq);
+	mtx_lock_spin(&umtx_lock);
 	if (pi->pi_owner == NULL) {
 		/* XXX
 		 * Current, We only support process private PI-mutex,
@@ -1524,6 +1528,7 @@ umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi,
 		 * For process private PI-mutex, we can find owner
 		 * thread and boost its priority safely.
 		 */
+		mtx_unlock_spin(&umtx_lock);
 		PROC_LOCK(curproc);
 		td1 = thread_find(curproc, owner);
 		mtx_lock_spin(&umtx_lock);
@@ -1532,8 +1537,6 @@ umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi,
 			umtx_pi_setowner(pi, td1);
 		}
 		PROC_UNLOCK(curproc);
-	} else {
-		mtx_lock_spin(&umtx_lock);
 	}
 
 	TAILQ_FOREACH(uq1, &pi->pi_blocked, uq_lockq) {
@@ -1551,26 +1554,18 @@ umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi,
 	thread_lock(td);
 	td->td_flags |= TDF_UPIBLOCKED;
 	thread_unlock(td);
-	mtx_unlock_spin(&umtx_lock);
-	umtxq_unlock(&uq->uq_key);
-
-	mtx_lock_spin(&umtx_lock);
 	umtx_propagate_priority(td);
 	mtx_unlock_spin(&umtx_lock);
+	umtxq_unbusy(&uq->uq_key);
 
-	umtxq_lock(&uq->uq_key);
 	if (uq->uq_flags & UQF_UMTXQ) {
 		error = msleep(uq, &uc->uc_lock, PCATCH, wmesg, timo);
 		if (error == EWOULDBLOCK)
 			error = ETIMEDOUT;
 		if (uq->uq_flags & UQF_UMTXQ) {
-			umtxq_busy(&uq->uq_key);
 			umtxq_remove(uq);
-			umtxq_unbusy(&uq->uq_key);
 		}
 	}
-	umtxq_unlock(&uq->uq_key);
-
 	mtx_lock_spin(&umtx_lock);
 	uq->uq_pi_blocked = NULL;
 	thread_lock(td);
@@ -1579,8 +1574,7 @@ umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi,
 	TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
 	umtx_unpropagate_priority(pi);
 	mtx_unlock_spin(&umtx_lock);
-
-	umtxq_lock(&uq->uq_key);
+	umtxq_unlock(&uq->uq_key);
 
 	return (error);
 }
@@ -1606,7 +1600,6 @@ static void
 umtx_pi_unref(struct umtx_pi *pi)
 {
 	struct umtxq_chain *uc;
-	int free = 0;
 
 	uc = umtxq_getchain(&pi->pi_key);
 	UMTXQ_LOCKED_ASSERT(uc);
@@ -1622,10 +1615,8 @@ umtx_pi_unref(struct umtx_pi *pi)
 			("blocked queue not empty"));
 		mtx_unlock_spin(&umtx_lock);
 		TAILQ_REMOVE(&uc->uc_pi_list, pi, pi_hashlink);
-		free = 1;
-	}
-	if (free)
 		umtx_pi_free(pi);
+	}
 }
 
 /*
@@ -1686,7 +1677,6 @@ _do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, int timo,
 		if (new_pi == NULL) {
 			umtxq_unlock(&uq->uq_key);
 			new_pi = umtx_pi_alloc(M_WAITOK);
-			new_pi->pi_key = uq->uq_key;
 			umtxq_lock(&uq->uq_key);
 			pi = umtx_pi_lookup(&uq->uq_key);
 			if (pi != NULL) {
@@ -1732,7 +1722,9 @@ _do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, int timo,
 
 			if (owner == UMUTEX_CONTESTED) {
 				umtxq_lock(&uq->uq_key);
+				umtxq_busy(&uq->uq_key);
 				error = umtx_pi_claim(pi, td);
+				umtxq_unbusy(&uq->uq_key);
 				umtxq_unlock(&uq->uq_key);
 				break;
 			}
@@ -1787,7 +1779,6 @@ _do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, int timo,
 		}
 
 		umtxq_lock(&uq->uq_key);
-		umtxq_unbusy(&uq->uq_key);
 		/*
 		 * We set the contested bit, sleep. Otherwise the lock changed
 		 * and we need to retry or we lost a race to the thread
@@ -1796,7 +1787,10 @@ _do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, int timo,
 		if (old == owner)
 			error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED,
 				 "umtxpi", timo);
-		umtxq_unlock(&uq->uq_key);
+		else {
+			umtxq_unbusy(&uq->uq_key);
+			umtxq_unlock(&uq->uq_key);
+		}
 	}
 
 	umtxq_lock(&uq->uq_key);
@@ -1851,18 +1845,26 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags)
 	umtxq_busy(&key);
 	count = umtxq_count_pi(&key, &uq_first);
 	if (uq_first != NULL) {
+		mtx_lock_spin(&umtx_lock);
 		pi = uq_first->uq_pi_blocked;
+		KASSERT(pi != NULL, ("pi == NULL?"));
 		if (pi->pi_owner != curthread) {
+			mtx_unlock_spin(&umtx_lock);
 			umtxq_unbusy(&key);
 			umtxq_unlock(&key);
+			umtx_key_release(&key);
 			/* userland messed the mutex */
 			return (EPERM);
 		}
 		uq_me = curthread->td_umtxq;
-		mtx_lock_spin(&umtx_lock);
 		pi->pi_owner = NULL;
 		TAILQ_REMOVE(&uq_me->uq_pi_contested, pi, pi_link);
+		/* get highest priority thread which is still sleeping. */
 		uq_first = TAILQ_FIRST(&pi->pi_blocked);
+		while (uq_first != NULL && 
+		       (uq_first->uq_flags & UQF_UMTXQ) == 0) {
+			uq_first = TAILQ_NEXT(uq_first, uq_lockq);
+		}
 		pri = PRI_MAX;
 		TAILQ_FOREACH(pi2, &uq_me->uq_pi_contested, pi_link) {
 			uq_first2 = TAILQ_FIRST(&pi2->pi_blocked);
@@ -1875,6 +1877,8 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags)
 		sched_unlend_user_prio(curthread, pri);
 		thread_unlock(curthread);
 		mtx_unlock_spin(&umtx_lock);
+		if (uq_first)
+			umtxq_signal_thread(uq_first);
 	}
 	umtxq_unlock(&key);
 
@@ -1887,8 +1891,6 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags)
 		count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED);
 
 	umtxq_lock(&key);
-	if (uq_first != NULL)
-		umtxq_signal_thread(uq_first);
 	umtxq_unbusy(&key);
 	umtxq_unlock(&key);
 	umtx_key_release(&key);
diff --git a/sys/kern/kern_vimage.c b/sys/kern/kern_vimage.c
index 0cba35a..c426ee6 100644
--- a/sys/kern/kern_vimage.c
+++ b/sys/kern/kern_vimage.c
@@ -44,19 +44,136 @@ __FBSDID("$FreeBSD$");
 MALLOC_DEFINE(M_VIMAGE, "vimage", "vimage resource container");
 
 static TAILQ_HEAD(vnet_modlink_head, vnet_modlink) vnet_modlink_head;
+static TAILQ_HEAD(vnet_modpending_head, vnet_modlink) vnet_modpending_head;
+static void vnet_mod_complete_registration(struct vnet_modlink *);
+static int vnet_mod_constructor(struct vnet_modlink *);
 
 void
 vnet_mod_register(const struct vnet_modinfo *vmi)
 {
+
+	vnet_mod_register_multi(vmi, NULL, NULL);
+}
+
+void
+vnet_mod_register_multi(const struct vnet_modinfo *vmi, void *iarg,
+    char *iname)
+{
 	struct vnet_modlink *vml, *vml_iter;
 	
-	/* Do not register the same module instance more than once. */
+	/* Do not register the same {module, iarg} pair more than once. */
 	TAILQ_FOREACH(vml_iter, &vnet_modlink_head, vml_mod_le)
-		if (vml_iter->vml_modinfo == vmi)
-			panic("%s: %s", __func__, vmi->vmi_name);
+		if (vml_iter->vml_modinfo == vmi && vml_iter->vml_iarg == iarg)
+			break;
+	if (vml_iter != NULL)
+		panic("registering an already registered vnet module: %s",
+		    vml_iter->vml_modinfo->vmi_name);
 	vml = malloc(sizeof(struct vnet_modlink), M_VIMAGE, M_NOWAIT);
+
+	/*
+	 * XXX we support only statically assigned module IDs at the time.
+	 * In principle modules should be able to get a dynamically
+	 * assigned ID at registration time.
+	 *
+	 * If a module is registered in multiple instances, then each
+	 * instance must have both iarg and iname set.
+	 */
+	if (vmi->vmi_id >= VNET_MOD_MAX)
+		panic("invalid vnet module ID: %d", vmi->vmi_id);
+	if (vmi->vmi_name == NULL)
+		panic("vnet module with no name: %d", vmi->vmi_id);
+	if ((iarg == NULL) ^ (iname == NULL))
+		panic("invalid vnet module instance: %s", vmi->vmi_name);
+
 	vml->vml_modinfo = vmi;
+	vml->vml_iarg = iarg;
+	vml->vml_iname = iname;
+
+	/* Check whether the module we depend on is already registered. */
+	if (vmi->vmi_dependson != vmi->vmi_id) {
+		TAILQ_FOREACH(vml_iter, &vnet_modlink_head, vml_mod_le)
+			if (vml_iter->vml_modinfo->vmi_id ==
+			    vmi->vmi_dependson)
+				break;	/* Depencency found, we are done. */
+		if (vml_iter == NULL) {
+#ifdef DEBUG_ORDERING
+			printf("dependency %d missing for vnet mod %s,"
+			    "postponing registration\n",
+			    vmi->vmi_dependson, vmi->vmi_name);
+#endif /* DEBUG_ORDERING */
+			TAILQ_INSERT_TAIL(&vnet_modpending_head, vml,
+			    vml_mod_le);
+			return;
+		}
+	}
+
+	vnet_mod_complete_registration(vml);
+}
+	
+void
+vnet_mod_complete_registration(struct vnet_modlink *vml)
+{
+	VNET_ITERATOR_DECL(vnet_iter);
+	struct vnet_modlink *vml_iter;
+
 	TAILQ_INSERT_TAIL(&vnet_modlink_head, vml, vml_mod_le);
+
+	VNET_FOREACH(vnet_iter) {
+		CURVNET_SET_QUIET(vnet_iter);
+		vnet_mod_constructor(vml);
+		CURVNET_RESTORE();
+	}
+
+	/* Check for pending modules depending on us. */
+	do {
+		TAILQ_FOREACH(vml_iter, &vnet_modpending_head, vml_mod_le)
+			if (vml_iter->vml_modinfo->vmi_dependson ==
+			    vml->vml_modinfo->vmi_id)
+				break;
+		if (vml_iter != NULL) {
+#ifdef DEBUG_ORDERING
+			printf("vnet mod %s now registering,"
+			    "dependency %d loaded\n",
+			    vml_iter->vml_modinfo->vmi_name,
+			    vml->vml_modinfo->vmi_id);
+#endif /* DEBUG_ORDERING */
+			TAILQ_REMOVE(&vnet_modpending_head, vml_iter,
+			    vml_mod_le);
+			vnet_mod_complete_registration(vml_iter);
+		}
+	} while (vml_iter != NULL);
+}
+
+static int vnet_mod_constructor(struct vnet_modlink *vml)
+{
+	const struct vnet_modinfo *vmi = vml->vml_modinfo;
+
+#ifdef DEBUG_ORDERING
+	printf("instantiating vnet_%s", vmi->vmi_name);
+	if (vml->vml_iarg)
+		printf("/%s", vml->vml_iname);
+	printf(": ");
+	if (vmi->vmi_struct_size)
+		printf("malloc(%zu); ", vmi->vmi_struct_size);
+	if (vmi->vmi_iattach != NULL)
+		printf("iattach()");
+	printf("\n");
+#endif
+
+#ifdef VIMAGE
+	if (vmi->vmi_struct_size) {
+		void *mem = malloc(vmi->vmi_struct_size, M_VNET,
+		    M_NOWAIT | M_ZERO);
+		if (mem == NULL) /* XXX should return error, not panic. */
+			panic("vi_alloc: malloc for %s\n", vmi->vmi_name);
+		curvnet->mod_data[vmi->vmi_id] = mem;
+	}
+#endif
+
+	if (vmi->vmi_iattach != NULL)
+		vmi->vmi_iattach(vml->vml_iarg);
+
+	return (0);
 }
 
 /*
@@ -75,7 +192,7 @@ vi_symlookup(struct kld_sym_lookup *lookup, char *symstr)
 		if (vml->vml_modinfo->vmi_symmap == NULL)
 			continue;
 		for (mapentry = vml->vml_modinfo->vmi_symmap;
-		     mapentry->name != NULL; mapentry++) {
+		    mapentry->name != NULL; mapentry++) {
 			if (strcmp(symstr, mapentry->name) == 0) {
 				lookup->symvalue = (u_long) mapentry->base;
 				lookup->symsize = mapentry->size;
@@ -91,8 +208,27 @@ vi_init(void *unused)
 {
 
 	TAILQ_INIT(&vnet_modlink_head);
+	TAILQ_INIT(&vnet_modpending_head);
+}
+
+static void
+vi_init_done(void *unused)
+{
+	struct vnet_modlink *vml_iter;
+
+	if (TAILQ_EMPTY(&vnet_modpending_head))
+		return;
+
+	printf("vnet modules with unresolved dependencies:\n");
+	TAILQ_FOREACH(vml_iter, &vnet_modpending_head, vml_mod_le)
+		printf("    %d:%s depending on %d\n",
+		    vml_iter->vml_modinfo->vmi_id,
+		    vml_iter->vml_modinfo->vmi_name,
+		    vml_iter->vml_modinfo->vmi_dependson);
+	panic("going nowhere without my vnet modules!");
 }
 
 SYSINIT(vimage, SI_SUB_VIMAGE, SI_ORDER_FIRST, vi_init, NULL);
+SYSINIT(vimage_done, SI_SUB_VIMAGE_DONE, SI_ORDER_FIRST, vi_init_done, NULL);
 
 #endif /* !VIMAGE_GLOBALS */
diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
index 96f7280..a748b70 100644
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -1337,11 +1337,11 @@ sched_initticks(void *dummy)
 	 */
 	balance_interval = realstathz;
 	/*
-	 * Set steal thresh to log2(mp_ncpu) but no greater than 4.  This
-	 * prevents excess thrashing on large machines and excess idle on
-	 * smaller machines.
+	 * Set steal thresh to roughly log2(mp_ncpu) but no greater than 4. 
+	 * This prevents excess thrashing on large machines and excess idle 
+	 * on smaller machines.
 	 */
-	steal_thresh = min(ffs(mp_ncpus) - 1, 3);
+	steal_thresh = min(fls(mp_ncpus) - 1, 3);
 	affinity = SCHED_AFFINITY_DEFAULT;
 #endif
 }
@@ -2417,6 +2417,11 @@ sched_affinity(struct thread *td)
 	ts = td->td_sched;
 	if (THREAD_CAN_SCHED(td, ts->ts_cpu))
 		return;
+	if (TD_ON_RUNQ(td)) {
+		sched_rem(td);
+		sched_add(td, SRQ_BORING);
+		return;
+	}
 	if (!TD_IS_RUNNING(td))
 		return;
 	td->td_flags |= TDF_NEEDRESCHED;
diff --git a/sys/kern/subr_bus.c b/sys/kern/subr_bus.c
index 3c526a6..92a4038 100644
--- a/sys/kern/subr_bus.c
+++ b/sys/kern/subr_bus.c
@@ -532,6 +532,12 @@ devctl_queue_data(char *data)
 	struct dev_event_info *n1 = NULL;
 	struct proc *p;
 
+	/*
+	 * Do not allow empty strings to be queued, as they
+	 * cause devd to exit prematurely.
+	 */
+	if (strlen(data) == 0)
+		return;
 	n1 = malloc(sizeof(*n1), M_BUS, M_NOWAIT);
 	if (n1 == NULL)
 		return;
@@ -814,7 +820,7 @@ devclass_find_internal(const char *classname, const char *parentname,
 	 */
 	if (parentname && dc && !dc->parent &&
 	    strcmp(classname, parentname) != 0) {
-		dc->parent = devclass_find_internal(parentname, NULL, FALSE);
+		dc->parent = devclass_find_internal(parentname, NULL, TRUE);
 		dc->parent->flags |= DC_HAS_CHILDREN;
 	}
 
diff --git a/sys/kern/subr_lock.c b/sys/kern/subr_lock.c
index 3e7c85d..977f9e5 100644
--- a/sys/kern/subr_lock.c
+++ b/sys/kern/subr_lock.c
@@ -46,9 +46,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/lock_profile.h>
 #include <sys/malloc.h>
+#include <sys/mutex.h>
 #include <sys/pcpu.h>
 #include <sys/proc.h>
 #include <sys/sbuf.h>
+#include <sys/sched.h>
 #include <sys/smp.h>
 #include <sys/sysctl.h>
 
@@ -186,7 +188,8 @@ struct lock_prof_cpu {
 
 struct lock_prof_cpu *lp_cpu[MAXCPU];
 
-int lock_prof_enable = 0;
+volatile int lock_prof_enable = 0;
+static volatile int lock_prof_resetting;
 
 /* SWAG: sbuf size = avg stat. line size * number of locks */
 #define LPROF_SBUF_SIZE		256 * 400
@@ -239,25 +242,77 @@ lock_prof_init(void *arg)
 }
 SYSINIT(lockprof, SI_SUB_SMP, SI_ORDER_ANY, lock_prof_init, NULL);
 
+/*
+ * To be certain that lock profiling has idled on all cpus before we
+ * reset, we schedule the resetting thread on all active cpus.  Since
+ * all operations happen within critical sections we can be sure that
+ * it is safe to zero the profiling structures.
+ */
+static void
+lock_prof_idle(void)
+{
+	struct thread *td;
+	int cpu;
+
+	td = curthread;
+	thread_lock(td);
+	for (cpu = 0; cpu <= mp_maxid; cpu++) {
+		if (CPU_ABSENT(cpu))
+			continue;
+		sched_bind(td, cpu);
+	}
+	sched_unbind(td);
+	thread_unlock(td);
+}
+
+static void
+lock_prof_reset_wait(void)
+{
+
+	/*
+	 * Spin relinquishing our cpu so that lock_prof_idle may
+	 * run on it.
+	 */
+	while (lock_prof_resetting)
+		sched_relinquish(curthread);
+}
+
 static void
 lock_prof_reset(void)
 {
 	struct lock_prof_cpu *lpc;
 	int enabled, i, cpu;
 
+	/*
+	 * We not only race with acquiring and releasing locks but also
+	 * thread exit.  To be certain that threads exit without valid head
+	 * pointers they must see resetting set before enabled is cleared.
+	 * Otherwise a lock may not be removed from a per-thread list due
+	 * to disabled being set but not wait for reset() to remove it below.
+	 */
+	atomic_store_rel_int(&lock_prof_resetting, 1);
 	enabled = lock_prof_enable;
 	lock_prof_enable = 0;
-	pause("lpreset", hz / 10);
+	lock_prof_idle();
+	/*
+	 * Some objects may have migrated between CPUs.  Clear all links
+	 * before we zero the structures.  Some items may still be linked
+	 * into per-thread lists as well.
+	 */
 	for (cpu = 0; cpu <= mp_maxid; cpu++) {
 		lpc = lp_cpu[cpu];
 		for (i = 0; i < LPROF_CACHE_SIZE; i++) {
 			LIST_REMOVE(&lpc->lpc_types[0].lpt_objs[i], lpo_link);
 			LIST_REMOVE(&lpc->lpc_types[1].lpt_objs[i], lpo_link);
 		}
+	}
+	for (cpu = 0; cpu <= mp_maxid; cpu++) {
+		lpc = lp_cpu[cpu];
 		bzero(lpc, sizeof(*lpc));
 		lock_prof_init_type(&lpc->lpc_types[0]);
 		lock_prof_init_type(&lpc->lpc_types[1]);
 	}
+	atomic_store_rel_int(&lock_prof_resetting, 0);
 	lock_prof_enable = enabled;
 }
 
@@ -351,7 +406,7 @@ retry_sbufops:
 	    "max", "wait_max", "total", "wait_total", "count", "avg", "wait_avg", "cnt_hold", "cnt_lock", "name");
 	enabled = lock_prof_enable;
 	lock_prof_enable = 0;
-	pause("lpreset", hz / 10);
+	lock_prof_idle();
 	t = ticks;
 	for (cpu = 0; cpu <= mp_maxid; cpu++) {
 		if (lp_cpu[cpu] == NULL)
@@ -461,16 +516,13 @@ lock_profile_object_lookup(struct lock_object *lo, int spin, const char *file,
 		if (l->lpo_obj == lo && l->lpo_file == file &&
 		    l->lpo_line == line)
 			return (l);
-	critical_enter();
 	type = &lp_cpu[PCPU_GET(cpuid)]->lpc_types[spin];
 	l = LIST_FIRST(&type->lpt_lpoalloc);
 	if (l == NULL) {
 		lock_prof_rejected++;
-		critical_exit();
 		return (NULL);
 	}
 	LIST_REMOVE(l, lpo_link);
-	critical_exit();
 	l->lpo_obj = lo;
 	l->lpo_file = file;
 	l->lpo_line = line;
@@ -497,18 +549,49 @@ lock_profile_obtain_lock_success(struct lock_object *lo, int contested,
 	spin = (LOCK_CLASS(lo)->lc_flags & LC_SPINLOCK) ? 1 : 0;
 	if (spin && lock_prof_skipspin == 1)
 		return;
+	critical_enter();
+	/* Recheck enabled now that we're in a critical section. */
+	if (lock_prof_enable == 0)
+		goto out;
 	l = lock_profile_object_lookup(lo, spin, file, line);
 	if (l == NULL)
-		return;
+		goto out;
 	l->lpo_cnt++;
 	if (++l->lpo_ref > 1)
-		return;
+		goto out;
 	l->lpo_contest_locking = contested;
 	l->lpo_acqtime = nanoseconds(); 
 	if (waittime && (l->lpo_acqtime > waittime))
 		l->lpo_waittime = l->lpo_acqtime - waittime;
 	else
 		l->lpo_waittime = 0;
+out:
+	critical_exit();
+}
+
+void
+lock_profile_thread_exit(struct thread *td)
+{
+#ifdef INVARIANTS
+	struct lock_profile_object *l;
+
+	MPASS(curthread->td_critnest == 0);
+#endif
+	/*
+	 * If lock profiling was disabled we have to wait for reset to
+	 * clear our pointers before we can exit safely.
+	 */
+	lock_prof_reset_wait();
+#ifdef INVARIANTS
+	LIST_FOREACH(l, &td->td_lprof[0], lpo_link)
+		printf("thread still holds lock acquired at %s:%d\n",
+		    l->lpo_file, l->lpo_line);
+	LIST_FOREACH(l, &td->td_lprof[1], lpo_link)
+		printf("thread still holds lock acquired at %s:%d\n",
+		    l->lpo_file, l->lpo_line);
+#endif
+	MPASS(LIST_FIRST(&td->td_lprof[0]) == NULL);
+	MPASS(LIST_FIRST(&td->td_lprof[1]) == NULL);
 }
 
 void
@@ -521,11 +604,20 @@ lock_profile_release_lock(struct lock_object *lo)
 	struct lpohead *head;
 	int spin;
 
-	if (!lock_prof_enable || (lo->lo_flags & LO_NOPROFILE))
+	if (lo->lo_flags & LO_NOPROFILE)
 		return;
 	spin = (LOCK_CLASS(lo)->lc_flags & LC_SPINLOCK) ? 1 : 0;
 	head = &curthread->td_lprof[spin];
+	if (LIST_FIRST(head) == NULL)
+		return;
 	critical_enter();
+	/* Recheck enabled now that we're in a critical section. */
+	if (lock_prof_enable == 0 && lock_prof_resetting == 1)
+		goto out;
+	/*
+	 * If lock profiling is not enabled we still want to remove the
+	 * lpo from our queue.
+	 */
 	LIST_FOREACH(l, head, lpo_link)
 		if (l->lpo_obj == lo)
 			break;
diff --git a/sys/kern/subr_param.c b/sys/kern/subr_param.c
index f45a64d..ed0437f 100644
--- a/sys/kern/subr_param.c
+++ b/sys/kern/subr_param.c
@@ -100,23 +100,30 @@ u_long	dflssiz;			/* initial stack size limit */
 u_long	maxssiz;			/* max stack size */
 u_long	sgrowsiz;			/* amount to grow stack */
 
-SYSCTL_INT(_kern, OID_AUTO, hz, CTLFLAG_RDTUN, &hz, 0, "ticks/second");
-SYSCTL_INT(_kern, OID_AUTO, maxswzone, CTLFLAG_RDTUN, &maxswzone, 0,
-    "max swmeta KVA storage");
-SYSCTL_INT(_kern, OID_AUTO, maxbcache, CTLFLAG_RDTUN, &maxbcache, 0,
-    "max buffer cache KVA storage");
+SYSCTL_INT(_kern, OID_AUTO, hz, CTLFLAG_RDTUN, &hz, 0,
+    "Number of clock ticks per second");
+SYSCTL_INT(_kern, OID_AUTO, ncallout, CTLFLAG_RDTUN, &ncallout, 0,
+    "Number of pre-allocated timer events");
+SYSCTL_INT(_kern, OID_AUTO, nbuf, CTLFLAG_RDTUN, &nbuf, 0,
+    "Number of buffers in the buffer cache");
+SYSCTL_INT(_kern, OID_AUTO, nswbuf, CTLFLAG_RDTUN, &nswbuf, 0,
+    "Number of swap buffers");
+SYSCTL_LONG(_kern, OID_AUTO, maxswzone, CTLFLAG_RDTUN, &maxswzone, 0,
+    "Maximum memory for swap metadata");
+SYSCTL_LONG(_kern, OID_AUTO, maxbcache, CTLFLAG_RDTUN, &maxbcache, 0,
+    "Maximum value of vfs.maxbufspace");
 SYSCTL_ULONG(_kern, OID_AUTO, maxtsiz, CTLFLAG_RDTUN, &maxtsiz, 0,
-    "max text size");
+    "Maximum text size");
 SYSCTL_ULONG(_kern, OID_AUTO, dfldsiz, CTLFLAG_RDTUN, &dfldsiz, 0,
-    "initial data size limit");
+    "Initial data size limit");
 SYSCTL_ULONG(_kern, OID_AUTO, maxdsiz, CTLFLAG_RDTUN, &maxdsiz, 0,
-    "max data size");
+    "Maximum data size");
 SYSCTL_ULONG(_kern, OID_AUTO, dflssiz, CTLFLAG_RDTUN, &dflssiz, 0,
-    "initial stack size limit");
+    "Initial stack size limit");
 SYSCTL_ULONG(_kern, OID_AUTO, maxssiz, CTLFLAG_RDTUN, &maxssiz, 0,
-    "max stack size");
+    "Maximum stack size");
 SYSCTL_ULONG(_kern, OID_AUTO, sgrowsiz, CTLFLAG_RDTUN, &sgrowsiz, 0,
-    "amount to grow stack");
+    "Amount to grow stack on a stack fault");
 SYSCTL_PROC(_kern, OID_AUTO, vm_guest, CTLFLAG_RD | CTLTYPE_STRING,
     NULL, 0, sysctl_kern_vm_guest, "A",
     "Virtual machine detected? (none|generic|xen)");
diff --git a/sys/kern/subr_rtc.c b/sys/kern/subr_rtc.c
index 0f3ec17..e919bf0 100644
--- a/sys/kern/subr_rtc.c
+++ b/sys/kern/subr_rtc.c
@@ -109,44 +109,36 @@ clock_register(device_t dev, long res)	/* res has units of microseconds */
 void
 inittodr(time_t base)
 {
-	struct timespec diff, ref, ts;
+	struct timespec ref, ts;
 	int error;
 
-	if (base) {
-		ref.tv_sec = base;
-		ref.tv_nsec = 0;
-		tc_setclock(&ref);
-	}
-
 	if (clock_dev == NULL) {
 		printf("warning: no time-of-day clock registered, system time "
 		    "will not be set accurately\n");
-		return;
+		goto wrong_time;
 	}
 	/* XXX: We should poll all registered RTCs in case of failure */
 	error = CLOCK_GETTIME(clock_dev, &ts);
 	if (error != 0 && error != EINVAL) {
 		printf("warning: clock_gettime failed (%d), the system time "
 		    "will not be set accurately\n", error);
-		return;
+		goto wrong_time;
 	}
 	if (error == EINVAL || ts.tv_sec < 0) {
-		printf("Invalid time in real time clock.\n");
-		printf("Check and reset the date immediately!\n");
+		printf("Invalid time in real time clock.\n"
+		    "Check and reset the date immediately!\n");
+		goto wrong_time;
 	}
 
 	ts.tv_sec += utc_offset();
+	tc_setclock(&ts);
+	return;
 
-	if (timespeccmp(&ref, &ts, >)) {
-		diff = ref;
-		timespecsub(&ref, &ts);
-	} else {
-		diff = ts;
-		timespecsub(&diff, &ref);
-	}
-	if (ts.tv_sec >= 2) {
-		/* badly off, adjust it */
-		tc_setclock(&ts);
+wrong_time:
+	if (base > 0) {
+		ref.tv_sec = base;
+		ref.tv_nsec = 0;
+		tc_setclock(&ref);
 	}
 }
 
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index e807e48..1a3a6fa 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -262,6 +262,54 @@ stop_cpus(cpumask_t map)
 	return 1;
 }
 
+#if defined(__amd64__)
+/*
+ * When called the executing CPU will send an IPI to all other CPUs
+ *  requesting that they halt execution.
+ *
+ * Usually (but not necessarily) called with 'other_cpus' as its arg.
+ *
+ *  - Signals all CPUs in map to suspend.
+ *  - Waits for each to suspend.
+ *
+ * Returns:
+ *  -1: error
+ *   0: NA
+ *   1: ok
+ *
+ * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
+ *            from executing at same time.
+ */
+int
+suspend_cpus(cpumask_t map)
+{
+	int i;
+
+	if (!smp_started)
+		return (0);
+
+	CTR1(KTR_SMP, "suspend_cpus(%x)", map);
+
+	/* send the suspend IPI to all CPUs in map */
+	ipi_selected(map, IPI_SUSPEND);
+
+	i = 0;
+	while ((stopped_cpus & map) != map) {
+		/* spin */
+		cpu_spinwait();
+		i++;
+#ifdef DIAGNOSTIC
+		if (i == 100000) {
+			printf("timeout suspending cpus\n");
+			break;
+		}
+#endif
+	}
+
+	return (1);
+}
+#endif
+
 /*
  * Called by a CPU to restart stopped CPUs. 
  *
diff --git a/sys/kern/sysv_sem.c b/sys/kern/sysv_sem.c
index 0112a04..12a2c4e 100644
--- a/sys/kern/sysv_sem.c
+++ b/sys/kern/sysv_sem.c
@@ -285,6 +285,7 @@ semunload(void)
 	free(semu, M_SEM);
 	for (i = 0; i < seminfo.semmni; i++)
 		mtx_destroy(&sema_mtx[i]);
+	free(sema_mtx, M_SEM);
 	mtx_destroy(&sem_mtx);
 	mtx_destroy(&sem_undo_mtx);
 	return (0);
diff --git a/sys/kern/tty.c b/sys/kern/tty.c
index e12f270..43c7f68 100644
--- a/sys/kern/tty.c
+++ b/sys/kern/tty.c
@@ -371,23 +371,31 @@ tty_wait_background(struct tty *tp, struct thread *td, int sig)
 		 *   exit
 		 * - the signal to send to the process isn't masked
 		 */
-		if (!tty_is_ctty(tp, p) ||
-		    p->p_pgrp == tp->t_pgrp || p->p_flag & P_PPWAIT ||
-		    SIGISMEMBER(p->p_sigacts->ps_sigignore, sig) ||
-		    SIGISMEMBER(td->td_sigmask, sig)) {
+		if (!tty_is_ctty(tp, p) || p->p_pgrp == tp->t_pgrp) {
 			/* Allow the action to happen. */
 			PROC_UNLOCK(p);
 			return (0);
 		}
 
+		if (SIGISMEMBER(p->p_sigacts->ps_sigignore, sig) ||
+		    SIGISMEMBER(td->td_sigmask, sig)) {
+			/* Only allow them in write()/ioctl(). */
+			PROC_UNLOCK(p);
+			return (sig == SIGTTOU ? 0 : EIO);
+		}
+
+		pg = p->p_pgrp;
+		if (p->p_flag & P_PPWAIT || pg->pg_jobc == 0) {
+			/* Don't allow the action to happen. */
+			PROC_UNLOCK(p);
+			return (EIO);
+		}
+		PROC_UNLOCK(p);
+
 		/*
 		 * Send the signal and sleep until we're the new
 		 * foreground process group.
 		 */
-		pg = p->p_pgrp;
-		PROC_UNLOCK(p);
-		if (pg->pg_jobc == 0)
-			return (EIO);
 		PGRP_LOCK(pg);
 		pgsignal(pg, sig, 1);
 		PGRP_UNLOCK(pg);
diff --git a/sys/kern/uipc_domain.c b/sys/kern/uipc_domain.c
index 9c9b05f..df35bcb 100644
--- a/sys/kern/uipc_domain.c
+++ b/sys/kern/uipc_domain.c
@@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mutex.h>
 #include <sys/socketvar.h>
 #include <sys/systm.h>
+#include <sys/vimage.h>
 #include <vm/uma.h>
 
 /*
@@ -64,6 +65,8 @@ static void domainfinalize(void *);
 SYSINIT(domainfin, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_FIRST, domainfinalize,
     NULL);
 
+static vnet_attach_fn net_init_domain;
+
 static struct callout pffast_callout;
 static struct callout pfslow_callout;
 
@@ -100,6 +103,14 @@ struct pr_usrreqs nousrreqs = {
 	.pru_sopoll =		pru_sopoll_notsupp,
 };
 
+#ifndef VIMAGE_GLOBALS
+vnet_modinfo_t vnet_domain_modinfo = {
+	.vmi_id		= VNET_MOD_DOMAIN,
+	.vmi_name	= "domain",
+	.vmi_iattach	= net_init_domain
+};
+#endif
+
 static void
 protosw_init(struct protosw *pr)
 {
@@ -159,9 +170,10 @@ protosw_init(struct protosw *pr)
  * Note: you cant unload it again because a socket may be using it.
  * XXX can't fail at this time.
  */
-static void
-net_init_domain(struct domain *dp)
+static int
+net_init_domain(const void *arg)
 {
+	const struct domain *dp = arg;
 	struct protosw *pr;
 
 	if (dp->dom_init)
@@ -175,6 +187,7 @@ net_init_domain(struct domain *dp)
 	max_datalen = MHLEN - max_hdr;
 	if (max_datalen < 1)
 		panic("%s: max_datalen < 1", __func__);
+	return (0);
 }
 
 /*
@@ -210,7 +223,11 @@ net_add_domain(void *data)
 		    "domainfinalize()\n", dp->dom_name);
 #endif
 	mtx_unlock(&dom_mtx);
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register_multi(&vnet_domain_modinfo, dp, dp->dom_name);
+#else
 	net_init_domain(dp);
+#endif
 }
 
 static void
diff --git a/sys/kern/uipc_sem.c b/sys/kern/uipc_sem.c
index 47b711a..da233a6 100644
--- a/sys/kern/uipc_sem.c
+++ b/sys/kern/uipc_sem.c
@@ -548,6 +548,8 @@ int
 ksem_open(struct thread *td, struct ksem_open_args *uap)
 {
 
+	DP((">>> ksem_open start, pid=%d\n", (int)td->td_proc->p_pid));
+
 	if ((uap->oflag & ~(O_CREAT | O_EXCL)) != 0)
 		return (EINVAL);
 	return (ksem_create(td, uap->name, uap->idp, uap->mode, uap->value,
@@ -708,12 +710,14 @@ kern_sem_wait(struct thread *td, semid_t id, int tryflag,
 	struct ksem *ks;
 	int error;
 
-	DP((">>> kern_sem_wait entered!\n"));
+	DP((">>> kern_sem_wait entered! pid=%d\n", (int)td->td_proc->p_pid));
 	error = ksem_get(td, id, &fp);
 	if (error)
 		return (error);
 	ks = fp->f_data;
 	mtx_lock(&sem_lock);
+	DP((">>> kern_sem_wait critical section entered! pid=%d\n",
+	    (int)td->td_proc->p_pid));
 #ifdef MAC
 	error = mac_posixsem_check_wait(td->td_ucred, fp->f_cred, ks);
 	if (error) {
@@ -723,7 +727,7 @@ kern_sem_wait(struct thread *td, semid_t id, int tryflag,
 #endif
 	DP(("kern_sem_wait value = %d, tryflag %d\n", ks->ks_value, tryflag));
 	vfs_timestamp(&ks->ks_atime);
-	if (ks->ks_value == 0) {
+	while (ks->ks_value == 0) {
 		ks->ks_waiters++;
 		if (tryflag != 0)
 			error = EAGAIN;
@@ -750,11 +754,13 @@ kern_sem_wait(struct thread *td, semid_t id, int tryflag,
 			goto err;
 	}
 	ks->ks_value--;
+	DP(("kern_sem_wait value post-decrement = %d\n", ks->ks_value));
 	error = 0;
 err:
 	mtx_unlock(&sem_lock);
 	fdrop(fp, td);
-	DP(("<<< kern_sem_wait leaving, error = %d\n", error));
+	DP(("<<< kern_sem_wait leaving, pid=%d, error = %d\n",
+	    (int)td->td_proc->p_pid, error));
 	return (error);
 }
 
diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index 2233e65..662b3abd 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -1811,7 +1811,7 @@ kern_sendfile(struct thread *td, struct sendfile_args *uap,
 	if ((error = fgetvp_read(td, uap->fd, &vp)) != 0)
 		goto out;
 	vfslocked = VFS_LOCK_GIANT(vp->v_mount);
-	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
+	vn_lock(vp, LK_SHARED | LK_RETRY);
 	if (vp->v_type == VREG) {
 		obj = vp->v_object;
 		if (obj != NULL) {
diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
index 517784a..6ddc837 100644
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@@ -457,10 +457,8 @@ restart:
 	error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp, &nd.ni_cnd,
 	    &vattr);
 #endif
-	if (error == 0) {
-		VOP_LEASE(nd.ni_dvp, td, td->td_ucred, LEASE_WRITE);
+	if (error == 0)
 		error = VOP_CREATE(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr);
-	}
 	NDFREE(&nd, NDF_ONLY_PNBUF);
 	vput(nd.ni_dvp);
 	if (error) {
diff --git a/sys/kern/vfs_acl.c b/sys/kern/vfs_acl.c
index 760b0e7..6a7c461 100644
--- a/sys/kern/vfs_acl.c
+++ b/sys/kern/vfs_acl.c
@@ -91,7 +91,6 @@ vacl_set_acl(struct thread *td, struct vnode *vp, acl_type_t type,
 	error = vn_start_write(vp, &mp, V_WAIT | PCATCH);
 	if (error != 0)
 		return (error);
-	VOP_LEASE(vp, td, td->td_ucred, LEASE_WRITE);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 #ifdef MAC
 	error = mac_vnode_check_setacl(td->td_ucred, vp, type, &inkernacl);
@@ -117,7 +116,6 @@ vacl_get_acl(struct thread *td, struct vnode *vp, acl_type_t type,
 	struct acl inkernelacl;
 	int error;
 
-	VOP_LEASE(vp, td, td->td_ucred, LEASE_WRITE);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 #ifdef MAC
 	error = mac_vnode_check_getacl(td->td_ucred, vp, type);
@@ -146,7 +144,6 @@ vacl_delete(struct thread *td, struct vnode *vp, acl_type_t type)
 	error = vn_start_write(vp, &mp, V_WAIT | PCATCH);
 	if (error)
 		return (error);
-	VOP_LEASE(vp, td, td->td_ucred, LEASE_WRITE);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 #ifdef MAC
 	error = mac_vnode_check_deleteacl(td->td_ucred, vp, type);
diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c
index 14c3569..0715e91 100644
--- a/sys/kern/vfs_bio.c
+++ b/sys/kern/vfs_bio.c
@@ -106,7 +106,8 @@ static void vfs_setdirty_locked_object(struct buf *bp);
 static void vfs_vmio_release(struct buf *bp);
 static int vfs_bio_clcheck(struct vnode *vp, int size,
 		daddr_t lblkno, daddr_t blkno);
-static int flushbufqueues(int, int);
+static int buf_do_flush(struct vnode *vp);
+static int flushbufqueues(struct vnode *, int, int);
 static void buf_daemon(void);
 static void bremfreel(struct buf *bp);
 #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
@@ -124,10 +125,10 @@ static long bufspace;
 #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
     defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
 SYSCTL_PROC(_vfs, OID_AUTO, bufspace, CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RD,
-    &bufspace, 0, sysctl_bufspace, "L", "KVA memory used for bufs");
+    &bufspace, 0, sysctl_bufspace, "L", "Virtual memory used for buffers");
 #else
 SYSCTL_LONG(_vfs, OID_AUTO, bufspace, CTLFLAG_RD, &bufspace, 0,
-    "KVA memory used for bufs");
+    "Virtual memory used for buffers");
 #endif
 static long maxbufspace;
 SYSCTL_LONG(_vfs, OID_AUTO, maxbufspace, CTLFLAG_RD, &maxbufspace, 0,
@@ -198,6 +199,9 @@ SYSCTL_INT(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RW, &getnewbufcalls, 0,
 static int getnewbufrestarts;
 SYSCTL_INT(_vfs, OID_AUTO, getnewbufrestarts, CTLFLAG_RW, &getnewbufrestarts, 0,
     "Number of times getnewbuf has had to restart a buffer aquisition");
+static int flushbufqtarget = 100;
+SYSCTL_INT(_vfs, OID_AUTO, flushbufqtarget, CTLFLAG_RW, &flushbufqtarget, 0,
+    "Amount of work to do in flushbufqueues when helping bufdaemon");
 
 /*
  * Wakeup point for bufdaemon, as well as indicator of whether it is already
@@ -258,6 +262,7 @@ static struct mtx nblock;
 #define QUEUE_DIRTY_GIANT 3	/* B_DELWRI buffers that need giant */
 #define QUEUE_EMPTYKVA	4	/* empty buffer headers w/KVA assignment */
 #define QUEUE_EMPTY	5	/* empty buffer headers */
+#define QUEUE_SENTINEL	1024	/* not an queue index, but mark for sentinel */
 
 /* Queues for free buffers with various properties */
 static TAILQ_HEAD(bqueues, buf) bufqueues[BUFFER_QUEUES] = { { 0 } };
@@ -1364,9 +1369,23 @@ brelse(struct buf *bp)
 	if (bp->b_qindex != QUEUE_NONE)
 		panic("brelse: free buffer onto another queue???");
 
+	/*
+	 * If the buffer has junk contents signal it and eventually
+	 * clean up B_DELWRI and diassociate the vnode so that gbincore()
+	 * doesn't find it.
+	 */
+	if (bp->b_bufsize == 0 || (bp->b_ioflags & BIO_ERROR) != 0 ||
+	    (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF)) != 0)
+		bp->b_flags |= B_INVAL;
+	if (bp->b_flags & B_INVAL) {
+		if (bp->b_flags & B_DELWRI)
+			bundirty(bp);
+		if (bp->b_vp)
+			brelvp(bp);
+	}
+
 	/* buffers with no memory */
 	if (bp->b_bufsize == 0) {
-		bp->b_flags |= B_INVAL;
 		bp->b_xflags &= ~(BX_BKGRDWRITE | BX_ALTDATA);
 		if (bp->b_vflags & BV_BKGRDINPROG)
 			panic("losing buffer 1");
@@ -1379,7 +1398,6 @@ brelse(struct buf *bp)
 	/* buffers with junk contents */
 	} else if (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF) ||
 	    (bp->b_ioflags & BIO_ERROR)) {
-		bp->b_flags |= B_INVAL;
 		bp->b_xflags &= ~(BX_BKGRDWRITE | BX_ALTDATA);
 		if (bp->b_vflags & BV_BKGRDINPROG)
 			panic("losing buffer 2");
@@ -1402,19 +1420,6 @@ brelse(struct buf *bp)
 	mtx_unlock(&bqlock);
 
 	/*
-	 * If B_INVAL and B_DELWRI is set, clear B_DELWRI.  We have already
-	 * placed the buffer on the correct queue.  We must also disassociate
-	 * the device and vnode for a B_INVAL buffer so gbincore() doesn't
-	 * find it.
-	 */
-	if (bp->b_flags & B_INVAL) {
-		if (bp->b_flags & B_DELWRI)
-			bundirty(bp);
-		if (bp->b_vp)
-			brelvp(bp);
-	}
-
-	/*
 	 * Fixup numfreebuffers count.  The bp is on an appropriate queue
 	 * unless locked.  We then bump numfreebuffers if it is not B_DELWRI.
 	 * We've already handled the B_INVAL case ( B_DELWRI will be clear
@@ -1710,21 +1715,23 @@ vfs_bio_awrite(struct buf *bp)
  */
 
 static struct buf *
-getnewbuf(int slpflag, int slptimeo, int size, int maxsize)
+getnewbuf(struct vnode *vp, int slpflag, int slptimeo, int size, int maxsize,
+    int gbflags)
 {
+	struct thread *td;
 	struct buf *bp;
 	struct buf *nbp;
 	int defrag = 0;
 	int nqindex;
 	static int flushingbufs;
 
+	td = curthread;
 	/*
 	 * We can't afford to block since we might be holding a vnode lock,
 	 * which may prevent system daemons from running.  We deal with
 	 * low-memory situations by proactively returning memory and running
 	 * async I/O rather then sync I/O.
 	 */
-
 	atomic_add_int(&getnewbufcalls, 1);
 	atomic_subtract_int(&getnewbufrestarts, 1);
 restart:
@@ -1956,8 +1963,9 @@ restart:
 	 */
 
 	if (bp == NULL) {
-		int flags;
+		int flags, norunbuf;
 		char *waitmsg;
+		int fl;
 
 		if (defrag) {
 			flags = VFS_BIO_NEED_BUFSPACE;
@@ -1975,9 +1983,35 @@ restart:
 		mtx_unlock(&bqlock);
 
 		bd_speedup();	/* heeeelp */
+		if (gbflags & GB_NOWAIT_BD)
+			return (NULL);
 
 		mtx_lock(&nblock);
 		while (needsbuffer & flags) {
+			if (vp != NULL && (td->td_pflags & TDP_BUFNEED) == 0) {
+				mtx_unlock(&nblock);
+				/*
+				 * getblk() is called with a vnode
+				 * locked, and some majority of the
+				 * dirty buffers may as well belong to
+				 * the vnode. Flushing the buffers
+				 * there would make a progress that
+				 * cannot be achieved by the
+				 * buf_daemon, that cannot lock the
+				 * vnode.
+				 */
+				norunbuf = ~(TDP_BUFNEED | TDP_NORUNNINGBUF) |
+				    (td->td_pflags & TDP_NORUNNINGBUF);
+				/* play bufdaemon */
+				td->td_pflags |= TDP_BUFNEED | TDP_NORUNNINGBUF;
+				fl = buf_do_flush(vp);
+				td->td_pflags &= norunbuf;
+				mtx_lock(&nblock);
+				if (fl != 0)
+					continue;
+				if ((needsbuffer & flags) == 0)
+					break;
+			}
 			if (msleep(&needsbuffer, &nblock,
 			    (PRIBIO + 4) | slpflag, waitmsg, slptimeo)) {
 				mtx_unlock(&nblock);
@@ -2046,6 +2080,35 @@ static struct kproc_desc buf_kp = {
 };
 SYSINIT(bufdaemon, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start, &buf_kp);
 
+static int
+buf_do_flush(struct vnode *vp)
+{
+	int flushed;
+
+	flushed = flushbufqueues(vp, QUEUE_DIRTY, 0);
+	/* The list empty check here is slightly racy */
+	if (!TAILQ_EMPTY(&bufqueues[QUEUE_DIRTY_GIANT])) {
+		mtx_lock(&Giant);
+		flushed += flushbufqueues(vp, QUEUE_DIRTY_GIANT, 0);
+		mtx_unlock(&Giant);
+	}
+	if (flushed == 0) {
+		/*
+		 * Could not find any buffers without rollback
+		 * dependencies, so just write the first one
+		 * in the hopes of eventually making progress.
+		 */
+		flushbufqueues(vp, QUEUE_DIRTY, 1);
+		if (!TAILQ_EMPTY(
+			    &bufqueues[QUEUE_DIRTY_GIANT])) {
+			mtx_lock(&Giant);
+			flushbufqueues(vp, QUEUE_DIRTY_GIANT, 1);
+			mtx_unlock(&Giant);
+		}
+	}
+	return (flushed);
+}
+
 static void
 buf_daemon()
 {
@@ -2059,7 +2122,7 @@ buf_daemon()
 	/*
 	 * This process is allowed to take the buffer cache to the limit
 	 */
-	curthread->td_pflags |= TDP_NORUNNINGBUF;
+	curthread->td_pflags |= TDP_NORUNNINGBUF | TDP_BUFNEED;
 	mtx_lock(&bdlock);
 	for (;;) {
 		bd_request = 0;
@@ -2074,30 +2137,8 @@ buf_daemon()
 		 * normally would so they can run in parallel with our drain.
 		 */
 		while (numdirtybuffers > lodirtybuffers) {
-			int flushed;
-
-			flushed = flushbufqueues(QUEUE_DIRTY, 0);
-			/* The list empty check here is slightly racy */
-			if (!TAILQ_EMPTY(&bufqueues[QUEUE_DIRTY_GIANT])) {
-				mtx_lock(&Giant);
-				flushed += flushbufqueues(QUEUE_DIRTY_GIANT, 0);
-				mtx_unlock(&Giant);
-			}
-			if (flushed == 0) {
-				/*
-				 * Could not find any buffers without rollback
-				 * dependencies, so just write the first one
-				 * in the hopes of eventually making progress.
-				 */
-				flushbufqueues(QUEUE_DIRTY, 1);
-				if (!TAILQ_EMPTY(
-				    &bufqueues[QUEUE_DIRTY_GIANT])) {
-					mtx_lock(&Giant);
-					flushbufqueues(QUEUE_DIRTY_GIANT, 1);
-					mtx_unlock(&Giant);
-				}
+			if (buf_do_flush(NULL) == 0)
 				break;
-			}
 			uio_yield();
 		}
 
@@ -2143,7 +2184,7 @@ SYSCTL_INT(_vfs, OID_AUTO, flushwithdeps, CTLFLAG_RW, &flushwithdeps,
     0, "Number of buffers flushed with dependecies that require rollbacks");
 
 static int
-flushbufqueues(int queue, int flushdeps)
+flushbufqueues(struct vnode *lvp, int queue, int flushdeps)
 {
 	struct buf sentinel;
 	struct vnode *vp;
@@ -2153,20 +2194,37 @@ flushbufqueues(int queue, int flushdeps)
 	int flushed;
 	int target;
 
-	target = numdirtybuffers - lodirtybuffers;
-	if (flushdeps && target > 2)
-		target /= 2;
+	if (lvp == NULL) {
+		target = numdirtybuffers - lodirtybuffers;
+		if (flushdeps && target > 2)
+			target /= 2;
+	} else
+		target = flushbufqtarget;
 	flushed = 0;
 	bp = NULL;
+	sentinel.b_qindex = QUEUE_SENTINEL;
 	mtx_lock(&bqlock);
-	TAILQ_INSERT_TAIL(&bufqueues[queue], &sentinel, b_freelist);
+	TAILQ_INSERT_HEAD(&bufqueues[queue], &sentinel, b_freelist);
 	while (flushed != target) {
-		bp = TAILQ_FIRST(&bufqueues[queue]);
-		if (bp == &sentinel)
+		bp = TAILQ_NEXT(&sentinel, b_freelist);
+		if (bp != NULL) {
+			TAILQ_REMOVE(&bufqueues[queue], &sentinel, b_freelist);
+			TAILQ_INSERT_AFTER(&bufqueues[queue], bp, &sentinel,
+			    b_freelist);
+		} else
 			break;
-		TAILQ_REMOVE(&bufqueues[queue], bp, b_freelist);
-		TAILQ_INSERT_TAIL(&bufqueues[queue], bp, b_freelist);
-
+		/*
+		 * Skip sentinels inserted by other invocations of the
+		 * flushbufqueues(), taking care to not reorder them.
+		 */
+		if (bp->b_qindex == QUEUE_SENTINEL)
+			continue;
+		/*
+		 * Only flush the buffers that belong to the
+		 * vnode locked by the curthread.
+		 */
+		if (lvp != NULL && bp->b_vp != lvp)
+			continue;
 		if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
 			continue;
 		if (bp->b_pin_count > 0) {
@@ -2214,16 +2272,27 @@ flushbufqueues(int queue, int flushdeps)
 			BUF_UNLOCK(bp);
 			continue;
 		}
-		if (vn_lock(vp, LK_EXCLUSIVE | LK_NOWAIT) == 0) {
+		if (vn_lock(vp, LK_EXCLUSIVE | LK_NOWAIT | LK_CANRECURSE) == 0) {
 			mtx_unlock(&bqlock);
 			CTR3(KTR_BUF, "flushbufqueue(%p) vp %p flags %X",
 			    bp, bp->b_vp, bp->b_flags);
-			vfs_bio_awrite(bp);
+			if (curproc == bufdaemonproc)
+				vfs_bio_awrite(bp);
+			else {
+				bremfree(bp);
+				bwrite(bp);
+			}
 			vn_finished_write(mp);
 			VOP_UNLOCK(vp, 0);
 			flushwithdeps += hasdeps;
 			flushed++;
-			waitrunningbufspace();
+
+			/*
+			 * Sleeping on runningbufspace while holding
+			 * vnode lock leads to deadlock.
+			 */
+			if (curproc == bufdaemonproc)
+				waitrunningbufspace();
 			numdirtywakeup((lodirtybuffers + hidirtybuffers) / 2);
 			mtx_lock(&bqlock);
 			continue;
@@ -2605,7 +2674,7 @@ loop:
 		maxsize = vmio ? size + (offset & PAGE_MASK) : size;
 		maxsize = imax(maxsize, bsize);
 
-		bp = getnewbuf(slpflag, slptimeo, size, maxsize);
+		bp = getnewbuf(vp, slpflag, slptimeo, size, maxsize, flags);
 		if (bp == NULL) {
 			if (slpflag || slptimeo)
 				return NULL;
@@ -2680,14 +2749,17 @@ loop:
  * set to B_INVAL.
  */
 struct buf *
-geteblk(int size)
+geteblk(int size, int flags)
 {
 	struct buf *bp;
 	int maxsize;
 
 	maxsize = (size + BKVAMASK) & ~BKVAMASK;
-	while ((bp = getnewbuf(0, 0, size, maxsize)) == 0)
-		continue;
+	while ((bp = getnewbuf(NULL, 0, 0, size, maxsize, flags)) == NULL) {
+		if ((flags & GB_NOWAIT_BD) &&
+		    (curthread->td_pflags & TDP_BUFNEED) != 0)
+			return (NULL);
+	}
 	allocbuf(bp, size);
 	bp->b_flags |= B_INVAL;	/* b_dep cleared by getnewbuf() */
 	BUF_ASSERT_HELD(bp);
diff --git a/sys/kern/vfs_cache.c b/sys/kern/vfs_cache.c
index cec1e33..3c81b01 100644
--- a/sys/kern/vfs_cache.c
+++ b/sys/kern/vfs_cache.c
@@ -35,6 +35,9 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_kdtrace.h"
+#include "opt_ktrace.h"
+
 #include <sys/param.h>
 #include <sys/filedesc.h>
 #include <sys/fnv_hash.h>
@@ -45,14 +48,43 @@ __FBSDID("$FreeBSD$");
 #include <sys/namei.h>
 #include <sys/proc.h>
 #include <sys/rwlock.h>
+#include <sys/sdt.h>
 #include <sys/syscallsubr.h>
 #include <sys/sysctl.h>
 #include <sys/sysproto.h>
 #include <sys/systm.h>
 #include <sys/vnode.h>
+#ifdef KTRACE
+#include <sys/ktrace.h>
+#endif
 
 #include <vm/uma.h>
 
+SDT_PROVIDER_DECLARE(vfs);
+SDT_PROBE_DEFINE3(vfs, namecache, enter, done, "struct vnode *", "char *",
+    "struct vnode *");
+SDT_PROBE_DEFINE2(vfs, namecache, enter_negative, done, "struct vnode *",
+    "char *");
+SDT_PROBE_DEFINE1(vfs, namecache, fullpath, entry, "struct vnode *");
+SDT_PROBE_DEFINE3(vfs, namecache, fullpath, hit, "struct vnode *",
+    "struct char *", "struct vnode *");
+SDT_PROBE_DEFINE1(vfs, namecache, fullpath, miss, "struct vnode *");
+SDT_PROBE_DEFINE3(vfs, namecache, fullpath, return, "int", "struct vnode *",
+    "struct char *");
+SDT_PROBE_DEFINE3(vfs, namecache, lookup, hit, "struct vnode *", "char *",
+    "struct vnode *");
+SDT_PROBE_DEFINE2(vfs, namecache, lookup, hit_negative, "struct vnode *",
+    "char *");
+SDT_PROBE_DEFINE2(vfs, namecache, lookup, miss, "struct vnode *",
+    "char *");
+SDT_PROBE_DEFINE1(vfs, namecache, purge, done, "struct vnode *");
+SDT_PROBE_DEFINE1(vfs, namecache, purge_negative, done, "struct vnode *");
+SDT_PROBE_DEFINE1(vfs, namecache, purgevfs, done, "struct mount *");
+SDT_PROBE_DEFINE3(vfs, namecache, zap, done, "struct vnode *", "char *",
+    "struct vnode *");
+SDT_PROBE_DEFINE2(vfs, namecache, zap_negative, done, "struct vnode *",
+    "char *");
+
 /*
  * This structure describes the elements in the cache of recent
  * names looked up by namei.
@@ -66,7 +98,7 @@ struct	namecache {
 	struct	vnode *nc_vp;		/* vnode the name refers to */
 	u_char	nc_flag;		/* flag bits */
 	u_char	nc_nlen;		/* length of name */
-	char	nc_name[0];		/* segment name */
+	char	nc_name[0];		/* segment name + nul */
 };
 
 /*
@@ -128,9 +160,10 @@ RW_SYSINIT(vfscache, &cache_lock, "Name Cache");
 static uma_zone_t cache_zone_small;
 static uma_zone_t cache_zone_large;
 
-#define	CACHE_PATH_CUTOFF	32
-#define	CACHE_ZONE_SMALL	(sizeof(struct namecache) + CACHE_PATH_CUTOFF)
-#define	CACHE_ZONE_LARGE	(sizeof(struct namecache) + NAME_MAX)
+#define	CACHE_PATH_CUTOFF	35
+#define	CACHE_ZONE_SMALL	(sizeof(struct namecache) + CACHE_PATH_CUTOFF \
+				    + 1)
+#define	CACHE_ZONE_LARGE	(sizeof(struct namecache) + NAME_MAX + 1)
 
 #define cache_alloc(len)	uma_zalloc(((len) <= CACHE_PATH_CUTOFF) ? \
 	cache_zone_small : cache_zone_large, M_WAITOK)
@@ -182,7 +215,8 @@ static MALLOC_DEFINE(M_VFSCACHE, "vfscache", "VFS name cache entries");
 /*
  * Flags in namecache.nc_flag
  */
-#define NCF_WHITE	1
+#define NCF_WHITE	0x01
+#define NCF_ISDOTDOT	0x02
 
 #ifdef DIAGNOSTIC
 /*
@@ -285,16 +319,31 @@ cache_zap(ncp)
 
 	rw_assert(&cache_lock, RA_WLOCKED);
 	CTR2(KTR_VFS, "cache_zap(%p) vp %p", ncp, ncp->nc_vp);
+#ifdef KDTRACE_HOOKS
+	if (ncp->nc_vp != NULL) {
+		SDT_PROBE(vfs, namecache, zap, done, ncp->nc_dvp,
+		    ncp->nc_name, ncp->nc_vp, 0, 0);
+	} else {
+		SDT_PROBE(vfs, namecache, zap_negative, done, ncp->nc_dvp,
+		    ncp->nc_name, 0, 0, 0);
+	}
+#endif
 	vp = NULL;
 	LIST_REMOVE(ncp, nc_hash);
-	LIST_REMOVE(ncp, nc_src);
-	if (LIST_EMPTY(&ncp->nc_dvp->v_cache_src)) {
-		vp = ncp->nc_dvp;
-		numcachehv--;
+	if (ncp->nc_flag & NCF_ISDOTDOT) {
+		if (ncp == ncp->nc_dvp->v_cache_dd)
+			ncp->nc_dvp->v_cache_dd = NULL;
+	} else {
+		LIST_REMOVE(ncp, nc_src);
+		if (LIST_EMPTY(&ncp->nc_dvp->v_cache_src)) {
+			vp = ncp->nc_dvp;
+			numcachehv--;
+		}
 	}
 	if (ncp->nc_vp) {
 		TAILQ_REMOVE(&ncp->nc_vp->v_cache_dst, ncp, nc_dst);
-		ncp->nc_vp->v_dd = NULL;
+		if (ncp == ncp->nc_vp->v_cache_dd)
+			ncp->nc_vp->v_cache_dd = NULL;
 	} else {
 		TAILQ_REMOVE(&ncneg, ncp, nc_dst);
 		numneg--;
@@ -315,7 +364,7 @@ cache_zap(ncp)
  * (negative cacheing), a status of ENOENT is returned. If the lookup
  * fails, a status of zero is returned.  If the directory vnode is
  * recycled out from under us due to a forced unmount, a status of
- * EBADF is returned.
+ * ENOENT is returned.
  *
  * vpp is locked and ref'd on return.  If we're looking up DOTDOT, dvp is
  * unlocked.  If we're looking up . an extra ref is taken, but the lock is
@@ -349,17 +398,33 @@ retry_wlocked:
 			CTR2(KTR_VFS, "cache_lookup(%p, %s) found via .",
 			    dvp, cnp->cn_nameptr);
 			dothits++;
+			SDT_PROBE(vfs, namecache, lookup, hit, dvp, ".",
+			    *vpp, 0, 0);
 			goto success;
 		}
 		if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') {
 			dotdothits++;
-			if (dvp->v_dd == NULL ||
-			    (cnp->cn_flags & MAKEENTRY) == 0) {
+			if (dvp->v_cache_dd == NULL) {
+				SDT_PROBE(vfs, namecache, lookup, miss, dvp,
+				    "..", NULL, 0, 0);
+				goto unlock;
+			}
+			if ((cnp->cn_flags & MAKEENTRY) == 0) {
+				if (!wlocked && !CACHE_UPGRADE_LOCK())
+					goto wlock;
+				if (dvp->v_cache_dd->nc_flag & NCF_ISDOTDOT)
+					cache_zap(dvp->v_cache_dd);
+				dvp->v_cache_dd = NULL;
 				goto unlock;
 			}
-			*vpp = dvp->v_dd;
+			if (dvp->v_cache_dd->nc_flag & NCF_ISDOTDOT)
+				*vpp = dvp->v_cache_dd->nc_vp;
+			else
+				*vpp = dvp->v_cache_dd->nc_dvp;
 			CTR3(KTR_VFS, "cache_lookup(%p, %s) found %p via ..",
 			    dvp, cnp->cn_nameptr, *vpp);
+			SDT_PROBE(vfs, namecache, lookup, hit, dvp, "..",
+			    *vpp, 0, 0);
 			goto success;
 		}
 	}
@@ -375,6 +440,8 @@ retry_wlocked:
 
 	/* We failed to find an entry */
 	if (ncp == NULL) {
+		SDT_PROBE(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr,
+		    NULL, 0, 0);
 		if ((cnp->cn_flags & MAKEENTRY) == 0) {
 			nummisszap++;
 		} else {
@@ -402,6 +469,8 @@ retry_wlocked:
 		*vpp = ncp->nc_vp;
 		CTR4(KTR_VFS, "cache_lookup(%p, %s) found %p via ncp %p",
 		    dvp, cnp->cn_nameptr, *vpp, ncp);
+		SDT_PROBE(vfs, namecache, lookup, hit, dvp, ncp->nc_name,
+		    *vpp, 0, 0);
 		goto success;
 	}
 
@@ -430,6 +499,8 @@ retry_wlocked:
 	nchstats.ncs_neghits++;
 	if (ncp->nc_flag & NCF_WHITE)
 		cnp->cn_flags |= ISWHITEOUT;
+	SDT_PROBE(vfs, namecache, lookup, hit_negative, dvp, ncp->nc_name,
+	    0, 0, 0);
 	CACHE_WUNLOCK();
 	return (ENOENT);
 
@@ -467,7 +538,7 @@ success:
 					/* forced unmount */
 					vrele(*vpp);
 					*vpp = NULL;
-					return (EBADF);
+					return (ENOENT);
 				}
 			} else
 				vn_lock(*vpp, LK_DOWNGRADE | LK_RETRY);
@@ -485,8 +556,15 @@ success:
 	else
 		CACHE_RUNLOCK();
 	error = vget(*vpp, cnp->cn_lkflags | LK_INTERLOCK, cnp->cn_thread);
-	if (cnp->cn_flags & ISDOTDOT)
+	if (cnp->cn_flags & ISDOTDOT) {
 		vn_lock(dvp, ltype | LK_RETRY);
+		if (dvp->v_iflag & VI_DOOMED) {
+			if (error == 0)
+				vput(*vpp);
+			*vpp = NULL;
+			return (ENOENT);
+		}
+	}
 	if (error) {
 		*vpp = NULL;
 		goto retry;
@@ -517,6 +595,7 @@ cache_enter(dvp, vp, cnp)
 	struct namecache *ncp, *n2;
 	struct nchashhead *ncpp;
 	u_int32_t hash;
+	int flag;
 	int hold;
 	int zap;
 	int len;
@@ -534,23 +613,35 @@ cache_enter(dvp, vp, cnp)
 	if (numcache >= desiredvnodes * 2)
 		return;
 
+	flag = 0;
 	if (cnp->cn_nameptr[0] == '.') {
-		if (cnp->cn_namelen == 1) {
+		if (cnp->cn_namelen == 1)
 			return;
-		}
-		/*
-		 * For dotdot lookups only cache the v_dd pointer if the
-		 * directory has a link back to its parent via v_cache_dst.
-		 * Without this an unlinked directory would keep a soft
-		 * reference to its parent which could not be NULLd at
-		 * cache_purge() time.
-		 */
 		if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') {
 			CACHE_WLOCK();
-			if (!TAILQ_EMPTY(&dvp->v_cache_dst))
-				dvp->v_dd = vp;
+			/*
+			 * If dotdot entry already exists, just retarget it
+			 * to new parent vnode, otherwise continue with new
+			 * namecache entry allocation.
+			 */
+			if ((ncp = dvp->v_cache_dd) != NULL) {
+				if (ncp->nc_flag & NCF_ISDOTDOT) {
+					KASSERT(ncp->nc_dvp == dvp,
+					    ("wrong isdotdot parent"));
+					TAILQ_REMOVE(&ncp->nc_vp->v_cache_dst,
+					    ncp, nc_dst);
+					TAILQ_INSERT_HEAD(&vp->v_cache_dst,
+					    ncp, nc_dst);
+					ncp->nc_vp = vp;
+					CACHE_WUNLOCK();
+					return;
+				}
+			}
+			dvp->v_cache_dd = NULL;
+			SDT_PROBE(vfs, namecache, enter, done, dvp, "..", vp,
+			    0, 0);
 			CACHE_WUNLOCK();
-			return;
+			flag = NCF_ISDOTDOT;
 		}
 	}
 
@@ -564,9 +655,10 @@ cache_enter(dvp, vp, cnp)
 	ncp = cache_alloc(cnp->cn_namelen);
 	ncp->nc_vp = vp;
 	ncp->nc_dvp = dvp;
+	ncp->nc_flag = flag;
 	len = ncp->nc_nlen = cnp->cn_namelen;
 	hash = fnv_32_buf(cnp->cn_nameptr, len, FNV1_32_INIT);
-	bcopy(cnp->cn_nameptr, ncp->nc_name, len);
+	strlcpy(ncp->nc_name, cnp->cn_nameptr, len + 1);
 	hash = fnv_32_buf(&dvp, sizeof(dvp), hash);
 	CACHE_WLOCK();
 
@@ -586,14 +678,35 @@ cache_enter(dvp, vp, cnp)
 		}
 	}
 
+	if (flag == NCF_ISDOTDOT) {
+		/*
+		 * See if we are trying to add .. entry, but some other lookup
+		 * has populated v_cache_dd pointer already.
+		 */
+		if (dvp->v_cache_dd != NULL) {
+		    CACHE_WUNLOCK();
+		    cache_free(ncp);
+		    return;
+		}
+		KASSERT(vp == NULL || vp->v_type == VDIR,
+		    ("wrong vnode type %p", vp));
+		dvp->v_cache_dd = ncp;
+	}
+
 	numcache++;
 	if (!vp) {
 		numneg++;
-		ncp->nc_flag = cnp->cn_flags & ISWHITEOUT ? NCF_WHITE : 0;
+		if (cnp->cn_flags & ISWHITEOUT)
+			ncp->nc_flag |= NCF_WHITE;
 	} else if (vp->v_type == VDIR) {
-		vp->v_dd = dvp;
+		if (flag != NCF_ISDOTDOT) {
+			if ((n2 = vp->v_cache_dd) != NULL &&
+			    (n2->nc_flag & NCF_ISDOTDOT) != 0)
+				cache_zap(n2);
+			vp->v_cache_dd = ncp;
+		}
 	} else {
-		vp->v_dd = NULL;
+		vp->v_cache_dd = NULL;
 	}
 
 	/*
@@ -601,11 +714,14 @@ cache_enter(dvp, vp, cnp)
 	 * within the cache entries table.
 	 */
 	LIST_INSERT_HEAD(ncpp, ncp, nc_hash);
-	if (LIST_EMPTY(&dvp->v_cache_src)) {
-		hold = 1;
-		numcachehv++;
+	if (flag != NCF_ISDOTDOT) {
+		if (LIST_EMPTY(&dvp->v_cache_src)) {
+			hold = 1;
+			numcachehv++;
+		}
+		LIST_INSERT_HEAD(&dvp->v_cache_src, ncp, nc_src);
 	}
-	LIST_INSERT_HEAD(&dvp->v_cache_src, ncp, nc_src);
+
 	/*
 	 * If the entry is "negative", we place it into the
 	 * "negative" cache queue, otherwise, we place it into the
@@ -613,8 +729,12 @@ cache_enter(dvp, vp, cnp)
 	 */
 	if (vp) {
 		TAILQ_INSERT_HEAD(&vp->v_cache_dst, ncp, nc_dst);
+		SDT_PROBE(vfs, namecache, enter, done, dvp, ncp->nc_name, vp,
+		    0, 0);
 	} else {
 		TAILQ_INSERT_TAIL(&ncneg, ncp, nc_dst);
+		SDT_PROBE(vfs, namecache, enter_negative, done, dvp,
+		    ncp->nc_name, 0, 0, 0);
 	}
 	if (numneg * ncnegfactor > numcache) {
 		ncp = TAILQ_FIRST(&ncneg);
@@ -655,12 +775,18 @@ cache_purge(vp)
 {
 
 	CTR1(KTR_VFS, "cache_purge(%p)", vp);
+	SDT_PROBE(vfs, namecache, purge, done, vp, 0, 0, 0, 0);
 	CACHE_WLOCK();
 	while (!LIST_EMPTY(&vp->v_cache_src))
 		cache_zap(LIST_FIRST(&vp->v_cache_src));
 	while (!TAILQ_EMPTY(&vp->v_cache_dst))
 		cache_zap(TAILQ_FIRST(&vp->v_cache_dst));
-	vp->v_dd = NULL;
+	if (vp->v_cache_dd != NULL) {
+		KASSERT(vp->v_cache_dd->nc_flag & NCF_ISDOTDOT,
+		   ("lost dotdot link"));
+		cache_zap(vp->v_cache_dd);
+	}
+	KASSERT(vp->v_cache_dd == NULL, ("incomplete purge"));
 	CACHE_WUNLOCK();
 }
 
@@ -674,6 +800,7 @@ cache_purge_negative(vp)
 	struct namecache *cp, *ncp;
 
 	CTR1(KTR_VFS, "cache_purge_negative(%p)", vp);
+	SDT_PROBE(vfs, namecache, purge_negative, done, vp, 0, 0, 0, 0);
 	CACHE_WLOCK();
 	LIST_FOREACH_SAFE(cp, &vp->v_cache_src, nc_src, ncp) {
 		if (cp->nc_vp == NULL)
@@ -693,6 +820,7 @@ cache_purgevfs(mp)
 	struct namecache *ncp, *nnp;
 
 	/* Scan hash tables for applicable entries */
+	SDT_PROBE(vfs, namecache, purgevfs, done, mp, 0, 0, 0, 0);
 	CACHE_WLOCK();
 	for (ncpp = &nchashtbl[nchash]; ncpp >= nchashtbl; ncpp--) {
 		LIST_FOREACH_SAFE(ncp, ncpp, nc_hash, nnp) {
@@ -807,6 +935,10 @@ kern___getcwd(struct thread *td, u_char *buf, enum uio_seg bufseg, u_int buflen)
 			bcopy(bp, buf, strlen(bp) + 1);
 		else
 			error = copyout(bp, buf, strlen(bp) + 1);
+#ifdef KTRACE
+	if (KTRPOINT(curthread, KTR_NAMEI))
+		ktrnamei(bp);
+#endif
 	}
 	free(tmpbuf, M_TEMP);
 	return (error);
@@ -935,6 +1067,9 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 	char *bp;
 	int error, i, slash_prefixed;
 	struct namecache *ncp;
+#ifdef KDTRACE_HOOKS
+	struct vnode *startvp = vp;
+#endif
 
 	buflen--;
 	bp = buf + buflen;
@@ -942,39 +1077,53 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 	error = 0;
 	slash_prefixed = 0;
 
+	SDT_PROBE(vfs, namecache, fullpath, entry, vp, 0, 0, 0, 0);
 	CACHE_RLOCK();
 	numfullpathcalls++;
 	if (vp->v_type != VDIR) {
 		ncp = TAILQ_FIRST(&vp->v_cache_dst);
 		if (ncp != NULL) {
-			for (i = ncp->nc_nlen - 1; i >= 0 && bp > buf; i--)
+			buflen -= ncp->nc_nlen;
+			for (i = ncp->nc_nlen - 1; i >= 0 && bp != buf; i--)
 				*--bp = ncp->nc_name[i];
 			if (bp == buf) {
 				numfullpathfail4++;
 				CACHE_RUNLOCK();
-				return (ENOMEM);
+				error = ENOMEM;
+				SDT_PROBE(vfs, namecache, fullpath, return,
+				    error, startvp, NULL, 0, 0);
+				return (error);
 			}
+			SDT_PROBE(vfs, namecache, fullpath, hit, ncp->nc_dvp,
+			    ncp->nc_name, vp, 0, 0);
 			vp = ncp->nc_dvp;
 		} else {
+			SDT_PROBE(vfs, namecache, fullpath, miss, vp, 0, 0,
+			    0, 0);
 			error = vn_vptocnp(&vp, &bp, buf, &buflen);
 			if (error) {
+				SDT_PROBE(vfs, namecache, fullpath, return,
+				    error, startvp, NULL, 0, 0);
 				return (error);
 			}
 		}
-		*--bp = '/';
-		buflen--;
-		if (buflen < 0) {
+		if (buflen <= 0) {
 			numfullpathfail4++;
 			CACHE_RUNLOCK();
-			return (ENOMEM);
+			error = ENOMEM;
+			SDT_PROBE(vfs, namecache, fullpath, return, error,
+			    startvp, NULL, 0, 0);
+			return (error);
 		}
+		*--bp = '/';
+		buflen--;
 		slash_prefixed = 1;
 	}
 	while (vp != rdir && vp != rootvnode) {
 		if (vp->v_vflag & VV_ROOT) {
 			if (vp->v_iflag & VI_DOOMED) {	/* forced unmount */
 				CACHE_RUNLOCK();
-				error = EBADF;
+				error = ENOENT;
 				break;
 			}
 			vp = vp->v_mount->mnt_vnodecovered;
@@ -986,10 +1135,11 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 			error = ENOTDIR;
 			break;
 		}
-		ncp = TAILQ_FIRST(&vp->v_cache_dst);
+ 		TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst)
+ 			if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
+ 				break;
 		if (ncp != NULL) {
-			MPASS(vp->v_dd == NULL || ncp->nc_dvp == vp->v_dd);
-			buflen -= ncp->nc_nlen - 1;
+			buflen -= ncp->nc_nlen;
 			for (i = ncp->nc_nlen - 1; i >= 0 && bp != buf; i--)
 				*--bp = ncp->nc_name[i];
 			if (bp == buf) {
@@ -998,37 +1148,45 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 				error = ENOMEM;
 				break;
 			}
+			SDT_PROBE(vfs, namecache, fullpath, hit, ncp->nc_dvp,
+			    ncp->nc_name, vp, 0, 0);
 			vp = ncp->nc_dvp;
 		} else {
+			SDT_PROBE(vfs, namecache, fullpath, miss, vp, 0, 0,
+			    0, 0);
 			error = vn_vptocnp(&vp, &bp, buf, &buflen);
-			if (error) {
+			if (error)
 				break;
-			}
 		}
-		*--bp = '/';
-		buflen--;
-		if (buflen < 0) {
+		if (buflen <= 0) {
 			numfullpathfail4++;
 			CACHE_RUNLOCK();
 			error = ENOMEM;
 			break;
 		}
+		*--bp = '/';
+		buflen--;
 		slash_prefixed = 1;
 	}
-	if (error)
+	if (error) {
+		SDT_PROBE(vfs, namecache, fullpath, return, error, startvp,
+		    NULL, 0, 0);
 		return (error);
+	}
 	if (!slash_prefixed) {
 		if (bp == buf) {
 			numfullpathfail4++;
 			CACHE_RUNLOCK();
+			SDT_PROBE(vfs, namecache, fullpath, return, 0,
+			    startvp, bp, 0, 0);
 			return (ENOMEM);
-		} else {
+		} else
 			*--bp = '/';
-		}
 	}
 	numfullpathfound++;
 	CACHE_RUNLOCK();
 
+	SDT_PROBE(vfs, namecache, fullpath, return, 0, startvp, bp, 0, 0);
 	*retbuf = bp;
 	return (0);
 }
@@ -1040,8 +1198,10 @@ vn_commname(struct vnode *vp, char *buf, u_int buflen)
 	int l;
 
 	CACHE_RLOCK();
-	ncp = TAILQ_FIRST(&vp->v_cache_dst);
-	if (!ncp) {
+	TAILQ_FOREACH(ncp, &vp->v_cache_dst, nc_dst)
+		if ((ncp->nc_flag & NCF_ISDOTDOT) == 0)
+			break;
+	if (ncp == NULL) {
 		CACHE_RUNLOCK();
 		return (ENOENT);
 	}
diff --git a/sys/kern/vfs_default.c b/sys/kern/vfs_default.c
index 1001497..2fe64b3 100644
--- a/sys/kern/vfs_default.c
+++ b/sys/kern/vfs_default.c
@@ -98,7 +98,6 @@ struct vop_vector default_vnodeops = {
 	.vop_ioctl =		VOP_ENOTTY,
 	.vop_kqfilter =		vop_stdkqfilter,
 	.vop_islocked =		vop_stdislocked,
-	.vop_lease =		VOP_NULL,
 	.vop_lock1 =		vop_stdlock,
 	.vop_lookup =		vop_nolookup,
 	.vop_open =		VOP_NULL,
diff --git a/sys/kern/vfs_extattr.c b/sys/kern/vfs_extattr.c
index c8b59d9..18890b7 100644
--- a/sys/kern/vfs_extattr.c
+++ b/sys/kern/vfs_extattr.c
@@ -175,7 +175,6 @@ extattr_set_vp(struct vnode *vp, int attrnamespace, const char *attrname,
 	error = vn_start_write(vp, &mp, V_WAIT | PCATCH);
 	if (error)
 		return (error);
-	VOP_LEASE(vp, td, td->td_ucred, LEASE_WRITE);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 
 	aiov.iov_base = data;
@@ -341,7 +340,6 @@ extattr_get_vp(struct vnode *vp, int attrnamespace, const char *attrname,
 	int error;
 
 	VFS_ASSERT_GIANT(vp->v_mount);
-	VOP_LEASE(vp, td, td->td_ucred, LEASE_READ);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 
 	/*
@@ -522,7 +520,6 @@ extattr_delete_vp(struct vnode *vp, int attrnamespace, const char *attrname,
 	error = vn_start_write(vp, &mp, V_WAIT | PCATCH);
 	if (error)
 		return (error);
-	VOP_LEASE(vp, td, td->td_ucred, LEASE_WRITE);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 
 #ifdef MAC
@@ -664,7 +661,6 @@ extattr_list_vp(struct vnode *vp, int attrnamespace, void *data,
 	int error;
 
 	VFS_ASSERT_GIANT(vp->v_mount);
-	VOP_LEASE(vp, td, td->td_ucred, LEASE_READ);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 
 	auiop = NULL;
diff --git a/sys/kern/vfs_lookup.c b/sys/kern/vfs_lookup.c
index daecd6f..757e56e 100644
--- a/sys/kern/vfs_lookup.c
+++ b/sys/kern/vfs_lookup.c
@@ -37,6 +37,7 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_kdtrace.h"
 #include "opt_ktrace.h"
 #include "opt_mac.h"
 
@@ -51,6 +52,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mount.h>
 #include <sys/filedesc.h>
 #include <sys/proc.h>
+#include <sys/sdt.h>
 #include <sys/syscallsubr.h>
 #include <sys/sysctl.h>
 #ifdef KTRACE
@@ -65,6 +67,11 @@ __FBSDID("$FreeBSD$");
 #define	NAMEI_DIAGNOSTIC 1
 #undef NAMEI_DIAGNOSTIC
 
+SDT_PROVIDER_DECLARE(vfs);
+SDT_PROBE_DEFINE3(vfs, namei, lookup, entry, "struct vnode *", "char *",
+    "unsigned long");
+SDT_PROBE_DEFINE2(vfs, namei, lookup, return, "int", "struct vnode *");
+
 /*
  * Allocation zone for namei
  */
@@ -181,7 +188,6 @@ namei(struct nameidata *ndp)
 		ktrnamei(cnp->cn_pnbuf);
 	}
 #endif
-
 	/*
 	 * Get starting point for the translation.
 	 */
@@ -224,6 +230,8 @@ namei(struct nameidata *ndp)
 			VFS_UNLOCK_GIANT(vfslocked);
 		}
 	}
+	SDT_PROBE(vfs, namei, lookup, entry, dp, cnp->cn_pnbuf,
+	    cnp->cn_flags, 0, 0);
 	vfslocked = VFS_LOCK_GIANT(dp->v_mount);
 	for (;;) {
 		/*
@@ -252,6 +260,8 @@ namei(struct nameidata *ndp)
 			cnp->cn_pnbuf = NULL;
 			cnp->cn_nameptr = NULL;
 #endif
+			SDT_PROBE(vfs, namei, lookup, return, error, NULL, 0,
+			    0, 0);
 			return (error);
 		}
 		vfslocked = (ndp->ni_cnd.cn_flags & GIANTHELD) != 0;
@@ -273,6 +283,8 @@ namei(struct nameidata *ndp)
 				VFS_UNLOCK_GIANT(vfslocked);
 			} else if (vfslocked)
 				ndp->ni_cnd.cn_flags |= GIANTHELD;
+			SDT_PROBE(vfs, namei, lookup, return, 0, ndp->ni_vp,
+			    0, 0, 0);
 			return (0);
 		}
 		if (ndp->ni_loopcnt++ >= MAXSYMLINKS) {
@@ -338,6 +350,7 @@ namei(struct nameidata *ndp)
 	ndp->ni_vp = NULL;
 	vrele(ndp->ni_dvp);
 	VFS_UNLOCK_GIANT(vfslocked);
+	SDT_PROBE(vfs, namei, lookup, return, error, NULL, 0, 0, 0);
 	return (error);
 }
 
@@ -602,7 +615,7 @@ dirloop:
 			if ((dp->v_vflag & VV_ROOT) == 0)
 				break;
 			if (dp->v_iflag & VI_DOOMED) {	/* forced unmount */
-				error = EBADF;
+				error = ENOENT;
 				goto bad;
 			}
 			tdp = dp;
@@ -764,9 +777,11 @@ unionlookup:
 	     *ndp->ni_next == '/')) {
 		cnp->cn_flags |= ISSYMLINK;
 		if (dp->v_iflag & VI_DOOMED) {
-			/* We can't know whether the directory was mounted with
-			 * NOSYMFOLLOW, so we can't follow safely. */
-			error = EBADF;
+			/*
+			 * We can't know whether the directory was mounted with
+			 * NOSYMFOLLOW, so we can't follow safely.
+			 */
+			error = ENOENT;
 			goto bad2;
 		}
 		if (dp->v_mount->mnt_flag & MNT_NOSYMFOLLOW) {
diff --git a/sys/kern/vfs_mount.c b/sys/kern/vfs_mount.c
index e489d80..e4256c7 100644
--- a/sys/kern/vfs_mount.c
+++ b/sys/kern/vfs_mount.c
@@ -1357,6 +1357,9 @@ root_mount_hold(const char *identifier)
 {
 	struct root_hold_token *h;
 
+	if (root_mounted())
+		return (NULL);
+
 	h = malloc(sizeof *h, M_DEVBUF, M_ZERO | M_WAITOK);
 	h->who = identifier;
 	mtx_lock(&mountlist_mtx);
@@ -1372,6 +1375,8 @@ void
 root_mount_rel(struct root_hold_token *h)
 {
 
+	if (h == NULL)
+		return;
 	mtx_lock(&mountlist_mtx);
 	LIST_REMOVE(h, list);
 	wakeup(&root_holds);
@@ -1386,6 +1391,8 @@ static void
 root_mount_prepare(void)
 {
 	struct root_hold_token *h;
+	struct timeval lastfail;
+	int curfail = 0;
 
 	for (;;) {
 		DROP_GIANT();
@@ -1396,10 +1403,12 @@ root_mount_prepare(void)
 			mtx_unlock(&mountlist_mtx);
 			break;
 		}
-		printf("Root mount waiting for:");
-		LIST_FOREACH(h, &root_holds, list)
-			printf(" %s", h->who);
-		printf("\n");
+		if (ppsratecheck(&lastfail, &curfail, 1)) {
+			printf("Root mount waiting for:");
+			LIST_FOREACH(h, &root_holds, list)
+				printf(" %s", h->who);
+			printf("\n");
+		}
 		msleep(&root_holds, &mountlist_mtx, PZERO | PDROP, "roothold",
 		    hz);
 	}
diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index 9411e81..fcaf514 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -860,6 +860,7 @@ vdestroy(struct vnode *vp)
 	VNASSERT(bo->bo_dirty.bv_root == NULL, vp, ("dirtyblkroot not NULL"));
 	VNASSERT(TAILQ_EMPTY(&vp->v_cache_dst), vp, ("vp has namecache dst"));
 	VNASSERT(LIST_EMPTY(&vp->v_cache_src), vp, ("vp has namecache src"));
+	VNASSERT(vp->v_cache_dd == NULL, vp, ("vp has namecache for .."));
 	VI_UNLOCK(vp);
 #ifdef MAC
 	mac_vnode_destroy(vp);
diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c
index ad69698..202f2d1 100644
--- a/sys/kern/vfs_syscalls.c
+++ b/sys/kern/vfs_syscalls.c
@@ -1161,7 +1161,6 @@ kern_openat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
 	if (flags & O_TRUNC) {
 		if ((error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0)
 			goto bad;
-		VOP_LEASE(vp, td, td->td_ucred, LEASE_WRITE);
 		VATTR_NULL(&vat);
 		vat.va_size = 0;
 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
@@ -1353,7 +1352,6 @@ restart:
 		    &nd.ni_cnd, &vattr);
 #endif
 	if (!error) {
-		VOP_LEASE(nd.ni_dvp, td, td->td_ucred, LEASE_WRITE);
 		if (whiteout)
 			error = VOP_WHITEOUT(nd.ni_dvp, &nd.ni_cnd, CREATE);
 		else {
@@ -1460,7 +1458,6 @@ restart:
 	if (error)
 		goto out;
 #endif
-	VOP_LEASE(nd.ni_dvp, td, td->td_ucred, LEASE_WRITE);
 	error = VOP_MKNOD(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr);
 	if (error == 0)
 		vput(nd.ni_vp);
@@ -1606,8 +1603,6 @@ kern_linkat(struct thread *td, int fd1, int fd2, char *path1, char *path2,
 			error = EEXIST;
 		} else if ((error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY))
 		    == 0) {
-			VOP_LEASE(nd.ni_dvp, td, td->td_ucred, LEASE_WRITE);
-			VOP_LEASE(vp, td, td->td_ucred, LEASE_WRITE);
 			error = can_hardlink(vp, td->td_ucred);
 			if (error == 0)
 #ifdef MAC
@@ -1727,7 +1722,6 @@ restart:
 	if (error)
 		goto out2;
 #endif
-	VOP_LEASE(nd.ni_dvp, td, td->td_ucred, LEASE_WRITE);
 	error = VOP_SYMLINK(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr, syspath);
 	if (error == 0)
 		vput(nd.ni_vp);
@@ -1787,7 +1781,6 @@ restart:
 			return (error);
 		goto restart;
 	}
-	VOP_LEASE(nd.ni_dvp, td, td->td_ucred, LEASE_WRITE);
 	error = VOP_WHITEOUT(nd.ni_dvp, &nd.ni_cnd, DELETE);
 	NDFREE(&nd, NDF_ONLY_PNBUF);
 	vput(nd.ni_dvp);
@@ -1893,7 +1886,6 @@ restart:
 		if (error)
 			goto out;
 #endif
-		VOP_LEASE(nd.ni_dvp, td, td->td_ucred, LEASE_WRITE);
 		error = VOP_REMOVE(nd.ni_dvp, vp, &nd.ni_cnd);
 #ifdef MAC
 out:
@@ -2667,7 +2659,6 @@ setfflags(td, vp, flags)
 
 	if ((error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0)
 		return (error);
-	VOP_LEASE(vp, td, td->td_ucred, LEASE_WRITE);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 	VATTR_NULL(&vattr);
 	vattr.va_flags = flags;
@@ -2795,7 +2786,6 @@ setfmode(td, vp, mode)
 
 	if ((error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0)
 		return (error);
-	VOP_LEASE(vp, td, td->td_ucred, LEASE_WRITE);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 	VATTR_NULL(&vattr);
 	vattr.va_mode = mode & ALLPERMS;
@@ -2958,7 +2948,6 @@ setfown(td, vp, uid, gid)
 
 	if ((error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0)
 		return (error);
-	VOP_LEASE(vp, td, td->td_ucred, LEASE_WRITE);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 	VATTR_NULL(&vattr);
 	vattr.va_uid = uid;
@@ -3172,7 +3161,6 @@ setutimes(td, vp, ts, numtimes, nullflag)
 
 	if ((error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0)
 		return (error);
-	VOP_LEASE(vp, td, td->td_ucred, LEASE_WRITE);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 	setbirthtime = 0;
 	if (numtimes < 3 && !VOP_GETATTR(vp, &vattr, td->td_ucred) &&
@@ -3403,7 +3391,6 @@ kern_truncate(struct thread *td, char *path, enum uio_seg pathseg, off_t length)
 		return (error);
 	}
 	NDFREE(&nd, NDF_ONLY_PNBUF);
-	VOP_LEASE(vp, td, td->td_ucred, LEASE_WRITE);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 	if (vp->v_type == VDIR)
 		error = EISDIR;
@@ -3647,13 +3634,6 @@ kern_renameat(struct thread *td, int oldfd, char *old, int newfd, char *new,
 #endif
 out:
 	if (!error) {
-		VOP_LEASE(tdvp, td, td->td_ucred, LEASE_WRITE);
-		if (fromnd.ni_dvp != tdvp) {
-			VOP_LEASE(fromnd.ni_dvp, td, td->td_ucred, LEASE_WRITE);
-		}
-		if (tvp) {
-			VOP_LEASE(tvp, td, td->td_ucred, LEASE_WRITE);
-		}
 		error = VOP_RENAME(fromnd.ni_dvp, fromnd.ni_vp, &fromnd.ni_cnd,
 				   tond.ni_dvp, tond.ni_vp, &tond.ni_cnd);
 		NDFREE(&fromnd, NDF_ONLY_PNBUF);
@@ -3779,7 +3759,6 @@ restart:
 	if (error)
 		goto out;
 #endif
-	VOP_LEASE(nd.ni_dvp, td, td->td_ucred, LEASE_WRITE);
 	error = VOP_MKDIR(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr);
 #ifdef MAC
 out:
@@ -3872,8 +3851,6 @@ restart:
 			return (error);
 		goto restart;
 	}
-	VOP_LEASE(nd.ni_dvp, td, td->td_ucred, LEASE_WRITE);
-	VOP_LEASE(vp, td, td->td_ucred, LEASE_WRITE);
 	error = VOP_RMDIR(nd.ni_dvp, nd.ni_vp, &nd.ni_cnd);
 	vn_finished_write(mp);
 out:
@@ -4462,7 +4439,6 @@ fhopen(td, uap)
 			vrele(vp);
 			goto out;
 		}
-		VOP_LEASE(vp, td, td->td_ucred, LEASE_WRITE);
 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);	/* XXX */
 #ifdef MAC
 		/*
diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c
index 681c0eb..48a74ac 100644
--- a/sys/kern/vfs_vnops.c
+++ b/sys/kern/vfs_vnops.c
@@ -153,14 +153,10 @@ restart:
 #ifdef MAC
 			error = mac_vnode_check_create(cred, ndp->ni_dvp,
 			    &ndp->ni_cnd, vap);
-			if (error == 0) {
+			if (error == 0)
 #endif
-				VOP_LEASE(ndp->ni_dvp, td, cred, LEASE_WRITE);
 				error = VOP_CREATE(ndp->ni_dvp, &ndp->ni_vp,
 						   &ndp->ni_cnd, vap);
-#ifdef MAC
-			}
-#endif
 			vput(ndp->ni_dvp);
 			vn_finished_write(mp);
 			if (error) {
@@ -521,7 +517,6 @@ vn_read(fp, uio, active_cred, flags, td)
 	if (fp->f_flag & O_DIRECT)
 		ioflag |= IO_DIRECT;
 	vfslocked = VFS_LOCK_GIANT(vp->v_mount);
-	VOP_LEASE(vp, td, fp->f_cred, LEASE_READ);
 	/*
 	 * According to McKusick the vn lock was protecting f_offset here.
 	 * It is now protected by the FOFFSET_LOCKED flag.
@@ -598,7 +593,6 @@ vn_write(fp, uio, active_cred, flags, td)
 	if (vp->v_type != VCHR &&
 	    (error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0)
 		goto unlock;
-	VOP_LEASE(vp, td, fp->f_cred, LEASE_WRITE);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 	if ((flags & FOF_OFFSET) == 0)
 		uio->uio_offset = fp->f_offset;
@@ -642,7 +636,6 @@ vn_truncate(fp, length, active_cred, td)
 		VFS_UNLOCK_GIANT(vfslocked);
 		return (error);
 	}
-	VOP_LEASE(vp, td, active_cred, LEASE_WRITE);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 	if (vp->v_type == VDIR) {
 		error = EISDIR;
diff --git a/sys/kern/vnode_if.src b/sys/kern/vnode_if.src
index 0e198bf..b7d9cb1 100644
--- a/sys/kern/vnode_if.src
+++ b/sys/kern/vnode_if.src
@@ -199,16 +199,6 @@ vop_write {
 };
 
 
-%% lease	vp	= = =
-
-vop_lease {
-	IN struct vnode *vp;
-	IN struct thread *td;
-	IN struct ucred *cred;
-	IN int flag;
-};
-
-
 %% ioctl	vp	U U U
 
 vop_ioctl {
diff --git a/sys/mips/alchemy/alchemy_machdep.c b/sys/mips/alchemy/alchemy_machdep.c
deleted file mode 100644
index eca9b18..0000000
--- a/sys/mips/alchemy/alchemy_machdep.c
+++ /dev/null
@@ -1,157 +0,0 @@
-/*-
- * Copyright (C) 2007 by Oleksandr Tymoshenko. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR OR HIS RELATIVES BE LIABLE FOR ANY DIRECT,
- * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- * SERVICES; LOSS OF MIND, USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
- * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include "opt_ddb.h"
-
-#include <sys/param.h>
-#include <sys/conf.h>
-#include <sys/kernel.h>
-#include <sys/systm.h>
-#include <sys/imgact.h>
-#include <sys/bio.h>
-#include <sys/buf.h>
-#include <sys/bus.h>
-#include <sys/cpu.h>
-#include <sys/cons.h>
-#include <sys/exec.h>
-#include <sys/ucontext.h>
-#include <sys/proc.h>
-#include <sys/kdb.h>
-#include <sys/ptrace.h>
-#include <sys/reboot.h>
-#include <sys/signalvar.h>
-#include <sys/sysent.h>
-#include <sys/sysproto.h>
-#include <sys/user.h>
-
-#include <vm/vm.h>
-#include <vm/vm_object.h>
-#include <vm/vm_page.h>
-#include <vm/vm_pager.h>
-
-#include <machine/cache.h>
-#include <machine/clock.h>
-#include <machine/cpu.h>
-#include <machine/cpuinfo.h>
-#include <machine/cpufunc.h>
-#include <machine/cpuregs.h>
-#include <machine/hwfunc.h>
-#include <machine/intr_machdep.h>
-#include <machine/locore.h>
-#include <machine/md_var.h>
-#include <machine/pte.h>
-#include <machine/sigframe.h>
-#include <machine/trap.h>
-#include <machine/vmparam.h>
-
-extern int	*edata;
-extern int	*end;
-
-static void
-mips_init(void)
-{
-	int i;
-
-	printf("entry: mips_init()\n");
-
-	bootverbose = 1;
-	realmem = btoc(16 << 20);
-
-	for (i = 0; i < 10; i++) {
-		phys_avail[i] = 0;
-	}
-
-	/* phys_avail regions are in bytes */
-	phys_avail[0] = MIPS_KSEG0_TO_PHYS((vm_offset_t)&end);
-	phys_avail[1] = ctob(realmem);
-
-	physmem = realmem;
-
-	init_param1();
-	init_param2(physmem);
-	mips_cpu_init();
-	pmap_bootstrap();
-	mips_proc0_init();
-	mutex_init();
-#ifdef DDB
-	kdb_init();
-#endif
-}
-
-void
-platform_halt(void)
-{
-
-}
-
-
-void
-platform_identify(void)
-{
-
-}
-
-void
-platform_reset(void)
-{
-
-	__asm __volatile("li	$25, 0xbfc00000");
-	__asm __volatile("j	$25");
-}
-
-void
-platform_trap_enter(void)
-{
-
-}
-
-void
-platform_trap_exit(void)
-{
-
-}
-
-void
-platform_start(__register_t a0 __unused, __register_t a1 __unused, 
-    __register_t a2 __unused, __register_t a3 __unused)
-{
-	vm_offset_t kernend;
-	uint64_t platform_counter_freq = 175 * 1000 * 1000;
-
-	/* clear the BSS and SBSS segments */
-	kernend = round_page((vm_offset_t)&end);
-	memset(&edata, 0, kernend - (vm_offset_t)(&edata));
-
-	cninit();
-	mips_init();
-	/* Set counter_freq for tick_init_params() */
-	platform_counter_freq = 175 * 1000 * 1000;
-
-	mips_timer_init_params(platform_counter_freq, 0);
-}
diff --git a/sys/mips/alchemy/aureg.h b/sys/mips/alchemy/aureg.h
deleted file mode 100644
index dfa2103..0000000
--- a/sys/mips/alchemy/aureg.h
+++ /dev/null
@@ -1,373 +0,0 @@
-/* $NetBSD: aureg.h,v 1.18 2006/10/02 06:44:00 gdamore Exp $ */
-
-/*
- * Copyright 2002 Wasabi Systems, Inc.
- * All rights reserved.
- *
- * Written by Simon Burge for Wasabi Systems, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *      This product includes software developed for the NetBSD Project by
- *      Wasabi Systems, Inc.
- * 4. The name of Wasabi Systems, Inc. may not be used to endorse
- *    or promote products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
- * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef	_MIPS_ALCHEMY_AUREG_H
-#define	_MIPS_ALCHEMY_AUREG_H
-
-/************************************************************************/
-/********************   AC97 Controller registers   *********************/
-/************************************************************************/
-#define	AC97_BASE		0x10000000
-
-/************************************************************************/
-/***********************   USB Host registers   *************************/
-/************************************************************************/
-#define	USBH_BASE		0x10100000
-#define	AU1550_USBH_BASE	0x14020000
-
-#define	USBH_ENABLE		0x7fffc
-#define	USBH_SIZE		0x100000
-
-#define	AU1550_USBH_ENABLE	0x7ffc
-#define AU1550_USBH_SIZE	0x60000
-
-/************************************************************************/
-/**********************   USB Device registers   ************************/
-/************************************************************************/
-#define	USBD_BASE		0x10200000
-
-/************************************************************************/
-/*************************   IRDA registers   ***************************/
-/************************************************************************/
-#define	IRDA_BASE		0x10300000
-
-/************************************************************************/
-/******************   Interrupt Controller registers   ******************/
-/************************************************************************/
-
-#define	IC0_BASE		0x10400000
-#define	IC1_BASE		0x11800000
-
-/*
- * The *_READ registers read the current value of the register
- * The *_SET registers set to 1 all bits that are written 1
- * The *_CLEAR registers clear to zero all bits that are written as 1
- */
-#define	IC_CONFIG0_READ			0x40	/* See table below */
-#define	IC_CONFIG0_SET			0x40
-#define	IC_CONFIG0_CLEAR		0x44
-
-#define	IC_CONFIG1_READ			0x48	/* See table below */
-#define	IC_CONFIG1_SET			0x48
-#define	IC_CONFIG1_CLEAR		0x4c
-
-#define	IC_CONFIG2_READ			0x50	/* See table below */
-#define	IC_CONFIG2_SET			0x50
-#define	IC_CONFIG2_CLEAR		0x54
-
-#define	IC_REQUEST0_INT			0x54	/* Show active interrupts on request 0 */
-
-#define	IC_SOURCE_READ			0x58	/* Interrupt source */
-#define	IC_SOURCE_SET			0x58	/*  0 - test bit used as source */
-#define	IC_SOURCE_CLEAR			0x5c	/*  1 - peripheral/GPIO used as source */
-
-#define	IC_REQUEST1_INT			0x5c	/* Show active interrupts on request 1 */
-
-#define	IC_ASSIGN_REQUEST_READ		0x60	/* Assigns the interrupt to one of the */
-#define	IC_ASSIGN_REQUEST_SET		0x60	/* CPU requests (0 - assign to request 1, */
-#define	IC_ASSIGN_REQUEST_CLEAR		0x64	/* 1 - assign to request 0) */
-
-#define	IC_WAKEUP_READ			0x68	/* Controls whether the interrupt can */
-#define	IC_WAKEUP_SET			0x68	/* cause a wakeup from IDLE */
-#define	IC_WAKEUP_CLEAR			0x6c
-
-#define	IC_MASK_READ			0x70	/* Enables/Disables the interrupt */
-#define	IC_MASK_SET			0x70
-#define	IC_MASK_CLEAR			0x74
-
-#define	IC_RISING_EDGE			0x78	/* Check/clear rising edge */
-
-#define	IC_FALLING_EDGE			0x7c	/* Check/clear falling edge */
-
-#define	IC_TEST_BIT			0x80	/* single bit source select */
-
-/*
- *	Interrupt Configuration Register Functions
- *
- *	Cfg2[n]	Cfg1[n]	Cfg0[n]		Function
- *	   0	   0	   0		Interrupts Disabled
- *	   0	   0	   1		Rising Edge Enabled
- *	   0	   1	   0		Falling Edge Enabled
- *	   0	   1	   1		Rising and Falling Edge Enabled
- *	   1	   0	   0		Interrupts Disabled
- *	   1	   0	   1		High Level Enabled
- *	   1	   1	   0		Low Level Enabled
- *	   1	   1	   1		Both Levels and Both Edges Enabled
- */
-
-/************************************************************************/
-/*************   Programable Serial Controller registers   **************/
-/************************************************************************/
-
-#define	PSC0_BASE		0x11A00000
-#define	PSC1_BASE		0x11B00000
-#define	PSC2_BASE		0x10A00000
-#define	PSC3_BASE		0x10B00000
-
-
-/************************************************************************/
-/**********************   Ethernet MAC registers   **********************/
-/************************************************************************/
-
-#define	MAC0_BASE		0x10500000
-#define	MAC1_BASE		0x10510000
-#define	MACx_SIZE		0x28
-
-#define	AU1500_MAC0_BASE	0x11500000	/* Grr, different on Au1500 */
-#define	AU1500_MAC1_BASE	0x11510000	/* Grr, different on Au1500 */
-
-#define	MAC0_ENABLE		0x10520000
-#define	MAC1_ENABLE		0x10520004
-#define	MACENx_SIZE		0x04
-
-#define	AU1500_MAC0_ENABLE	0x11520000	/* Grr, different on Au1500 */
-#define	AU1500_MAC1_ENABLE	0x11520004	/* Grr, different on Au1500 */
-
-#define	MAC0_DMA_BASE		0x14004000
-#define	MAC1_DMA_BASE		0x14004200
-#define	MACx_DMA_SIZE		0x140
-
-/************************************************************************/
-/**********************   Static Bus registers   ************************/
-/************************************************************************/
-#define	STATIC_BUS_BASE		0x14001000
-
-/************************************************************************/
-/********************   Secure Digital registers   **********************/
-/************************************************************************/
-#define	SD0_BASE		0x10600000
-#define	SD1_BASE		0x10680000
-
-/************************************************************************/
-/*************************   I^2S registers   ***************************/
-/************************************************************************/
-#define	I2S_BASE		0x11000000
-
-/************************************************************************/
-/**************************   UART registers   **************************/
-/************************************************************************/
-
-#define	UART0_BASE		0x11100000
-#define	UART1_BASE		0x11200000
-#define	UART2_BASE		0x11300000
-#define	UART3_BASE		0x11400000
-
-/************************************************************************/
-/*************************   SSI registers   ****************************/
-/************************************************************************/
-#define	SSI0_BASE		0x11600000
-#define	SSI1_BASE		0x11680000
-
-/************************************************************************/
-/************************   GPIO2 registers   ***************************/
-/************************************************************************/
-#define	GPIO_BASE		0x11900100
-
-/************************************************************************/
-/************************   GPIO2 registers   ***************************/
-/************************************************************************/
-#define	GPIO2_BASE		0x11700000
-
-/************************************************************************/
-/*************************   PCI registers   ****************************/
-/************************************************************************/
-#define	PCI_BASE		0x14005000
-#define	PCI_HEADER		0x14005100
-#define	PCI_MEM_BASE		0x400000000ULL
-#define	PCI_IO_BASE		0x500000000ULL
-#define	PCI_CONFIG_BASE		0x600000000ULL
-
-/************************************************************************/
-/***********************   PCMCIA registers   ***************************/
-/************************************************************************/
-#define	PCMCIA_BASE		0xF00000000ULL
-
-/************************************************************************/
-/******************   Programmable Counter registers   ******************/
-/************************************************************************/
-
-#define	SYS_BASE		0x11900000
-
-#define	PC_BASE			SYS_BASE
-
-#define	PC_TRIM0		0x00		/* PC0 Divide (16 bits) */
-#define	PC_COUNTER_WRITE0	0x04		/* set PC0 */
-#define	PC_MATCH0_0		0x08		/* match counter & interrupt */
-#define	PC_MATCH1_0		0x0c		/* match counter & interrupt */
-#define	PC_MATCH2_0		0x10		/* match counter & interrupt */
-#define	PC_COUNTER_CONTROL	0x14		/* Programmable Counter Control */
-#define	  CC_E1S		  0x00800000	/* Enable PC1 write status */
-#define	  CC_T1S		  0x00100000	/* Trim PC1 write status */
-#define	  CC_M21		  0x00080000	/* Match 2 of PC1 write status */
-#define	  CC_M11		  0x00040000	/* Match 1 of PC1 write status */
-#define	  CC_M01		  0x00020000	/* Match 0 of PC1 write status */
-#define	  CC_C1S		  0x00010000	/* PC1 write status */
-#define	  CC_BP			  0x00004000	/* Bypass OSC (use GPIO1) */
-#define	  CC_EN1		  0x00002000	/* Enable PC1 */
-#define	  CC_BT1		  0x00001000	/* Bypass Trim on PC1 */
-#define	  CC_EN0		  0x00000800	/* Enable PC0 */
-#define	  CC_BT0		  0x00000400	/* Bypass Trim on PC0 */
-#define	  CC_EO			  0x00000100	/* Enable Oscillator */
-#define	  CC_E0S		  0x00000080	/* Enable PC0 write status */
-#define	  CC_32S		  0x00000020	/* 32.768kHz OSC status */
-#define	  CC_T0S		  0x00000010	/* Trim PC0 write status */
-#define	  CC_M20		  0x00000008	/* Match 2 of PC0 write status */
-#define	  CC_M10		  0x00000004	/* Match 1 of PC0 write status */
-#define	  CC_M00		  0x00000002	/* Match 0 of PC0 write status */
-#define	  CC_C0S		  0x00000001	/* PC0 write status */
-#define	PC_COUNTER_READ_0	0x40		/* get PC0 */
-#define	PC_TRIM1		0x44		/* PC1 Divide (16 bits) */
-#define	PC_COUNTER_WRITE1	0x48		/* set PC1 */
-#define	PC_MATCH0_1		0x4c		/* match counter & interrupt */
-#define	PC_MATCH1_1		0x50		/* match counter & interrupt */
-#define	PC_MATCH2_1		0x54		/* match counter & interrupt */
-#define	PC_COUNTER_READ_1	0x58		/* get PC1 */
-
-#define	PC_SIZE			0x5c		/* size of register set */
-#define	PC_RATE			32768		/* counter rate is 32.768kHz */
-
-/************************************************************************/
-/*******************   Frequency Generator Registers   ******************/
-/************************************************************************/
-
-#define SYS_FREQCTRL0		(SYS_BASE + 0x20)
-#define SFC_FRDIV2(f)		(f<<22)		/* 29:22. Freq Divider 2 */
-#define SFC_FE2			(1<<21)		/* Freq generator output enable 2 */
-#define SFC_FS2			(1<<20)		/* Freq generator source 2 */
-#define SFC_FRDIV1(f)		(f<<12)		/* 19:12. Freq Divider 1 */
-#define SFC_FE1			(1<<11)		/* Freq generator output enable 1 */
-#define SFC_FS1			(1<<10)		/* Freq generator source 1 */
-#define SFC_FRDIV0(f)		(f<<2)		/* 9:2. Freq Divider 0 */
-#define SFC_FE0			2		/* Freq generator output enable 0 */
-#define SFC_FS0			1		/* Freq generator source 0 */
-
-#define SYS_FREQCTRL1		(SYS_BASE + 0x24)
-#define SFC_FRDIV5(f)		(f<<22)		/* 29:22. Freq Divider 5 */
-#define SFC_FE5			(1<<21)		/* Freq generator output enable 5 */
-#define SFC_FS5			(1<<20)		/* Freq generator source 5 */
-#define SFC_FRDIV4(f)		(f<<12)		/* 19:12. Freq Divider 4 */
-#define SFC_FE4			(1<<11)		/* Freq generator output enable 4 */
-#define SFC_FS4			(1<<10)		/* Freq generator source 4 */
-#define SFC_FRDIV3(f)		(f<<2)		/* 9:2. Freq Divider 3 */
-#define SFC_FE3			2		/* Freq generator output enable 3 */
-#define SFC_FS3			1		/* Freq generator source 3 */
-
-/************************************************************************/
-/******************   Clock Source Control Registers   ******************/
-/************************************************************************/
-
-#define SYS_CLKSRC		(SYS_BASE + 0x28)
-#define  SCS_ME1(n)		(n<<27)		/* EXTCLK1 Clock Mux input select */
-#define  SCS_ME0(n)		(n<<22)		/* EXTCLK0 Clock Mux input select */
-#define  SCS_MPC(n)		(n<<17)		/* PCI clock mux input select */
-#define  SCS_MUH(n)		(n<<12)		/* USB Host clock mux input select */
-#define  SCS_MUD(n)		(n<<7)		/* USB Device clock mux input select */
-#define   SCS_MEx_AUX		0x1		/* Aux clock */
-#define   SCS_MEx_FREQ0		0x2		/* FREQ0 */
-#define   SCS_MEx_FREQ1		0x3		/* FREQ1 */
-#define   SCS_MEx_FREQ2		0x4		/* FREQ2 */
-#define   SCS_MEx_FREQ3		0x5		/* FREQ3 */
-#define   SCS_MEx_FREQ4		0x6		/* FREQ4 */
-#define   SCS_MEx_FREQ5		0x7		/* FREQ5 */
-#define  SCS_DE1		(1<<26)		/* EXTCLK1 clock divider select */
-#define  SCS_CE1		(1<<25)		/* EXTCLK1 clock select */
-#define  SCS_DE0		(1<<21)		/* EXTCLK0 clock divider select */
-#define  SCS_CE0		(1<<20)		/* EXTCLK0 clock select */
-#define  SCS_DPC		(1<<16)		/* PCI clock divider select */
-#define  SCS_CPC		(1<<15)		/* PCI clock select */
-#define  SCS_DUH		(1<<11)		/* USB Host clock divider select */
-#define  SCS_CUH		(1<<10)		/* USB Host clock select */
-#define  SCS_DUD		(1<<6)		/* USB Device clock divider select */
-#define  SCS_CUD		(1<<5)		/* USB Device clock select */
-/*
- * Au1550 bits, needed for PSCs. Note that some bits collide with
- * earlier parts.  On Au1550, USB clocks (both device and host) are
- * shared with PSC2, and must be configured for 48MHz.  DBAU1550 YAMON
- * does this by default.  Also, EXTCLK0 is shared with PSC3.  DBAU1550
- * YAMON does not configure any clocks besides PSC2.
- */
-#define  SCS_MP3(n)		(n<<22)		/* psc3_intclock mux */
-#define	 SCS_DP3		(1<<21)		/* psc3_intclock divider */
-#define	 SCS_CP3		(1<<20)		/* psc3_intclock select */
-#define  SCS_MP1(n)		(n<<12)		/* psc1_intclock mux */
-#define	 SCS_DP1		(1<<11)		/* psc1_intclock divider */
-#define	 SCS_CP1		(1<<10)		/* psc1_intclock select */
-#define	 SCS_MP0(n)		(n<<7)		/* psc0_intclock mux */
-#define  SCS_DP0		(1<<6)		/* psc0_intclock divider */
-#define	 SCS_CP0		(1<<5)		/* psc0_intclock seelct */
-#define	 SCS_MP2(n)		(n<<2)		/* psc2_intclock mux */
-#define	 SCS_DP2		(1<<1)		/* psc2_intclock divider */
-#define	 SCS_CP2		(1<<0)		/* psc2_intclock select */
-
-/************************************************************************/
-/***************************  PIN Function  *****************************/
-/************************************************************************/
-
-#define	SYS_PINFUNC		(SYS_BASE + 0x2c)
-#define	 SPF_PSC3_MASK		(7<<20)
-#define	 SPF_PSC3_AC97		(0<<17)		/* select AC97/SPI */
-#define	 SPF_PSC3_I2S		(1<<17)		/* select I2S */
-#define	 SPF_PSC3_SMBUS		(3<<17)		/* select SMbus */
-#define	 SPF_PSC3_GPIO		(7<<17)		/* select gpio215:211 */
-#define  SPF_PSC2_MASK		(7<<17)
-#define	 SPF_PSC2_AC97		(0<<17)		/* select AC97/SPI */
-#define	 SPF_PSC2_I2S		(1<<17)		/* select I2S */
-#define	 SPF_PSC2_SMBUS		(3<<17)		/* select SMbus */
-#define	 SPF_PSC2_GPIO		(7<<17)		/* select gpio210:206*/
-#define	 SPF_CS			(1<<16)		/* extclk0 or 32kHz osc */
-#define	 SPF_USB		(1<<15)		/* host or device usb otg */
-#define	 SPF_U3T		(1<<14)		/* uart3 tx or gpio23 */
-#define	 SPF_U1R		(1<<13)		/* uart1 rx or gpio22 */
-#define	 SPF_U1T		(1<<12)		/* uart1 tx or gpio21 */
-#define	 SPF_EX1		(1<<10)		/* gpio3 or extclk1 */
-#define	 SPF_EX0		(1<<9)		/* gpio2 or extclk0/32kHz osc*/
-#define	 SPF_U3			(1<<7)		/* gpio14:9 or uart3 */
-#define	 SPF_MBSa		(1<<5)		/* must be set */
-#define	 SPF_NI2		(1<<4)		/* enet1 or gpio28:24 */
-#define	 SPF_U0			(1<<3)		/* uart0 or gpio20 */
-#define	 SPF_MBSb		(1<<2)		/* must be set */
-#define	 SPF_S1			(1<<1)		/* gpio17 or psc1_sync1 */
-#define	 SPF_S0			(1<<0)		/* gpio16 or psc0_sync1 */
-
-/************************************************************************/
-/***************************   PLL Control  *****************************/
-/************************************************************************/
-
-#define SYS_CPUPLL		(SYS_BASE + 0x60)
-#define SYS_AUXPLL              (SYS_BASE + 0x64)
-
-#endif	/* _MIPS_ALCHEMY_AUREG_H */
diff --git a/sys/mips/alchemy/files.alchemy b/sys/mips/alchemy/files.alchemy
deleted file mode 100644
index 8534431..0000000
--- a/sys/mips/alchemy/files.alchemy
+++ /dev/null
@@ -1,7 +0,0 @@
-# $FreeBSD$
-# Alchmy on-board devices
-# mips/alchemy/console.c		standard
-mips/alchemy/alchemy_machdep.c	standard
-mips/alchemy/obio.c		standard
-mips/alchemy/uart_bus_alchemy.c	optional uart
-mips/alchemy/uart_cpu_alchemy.c	optional uart
diff --git a/sys/mips/alchemy/obio.c b/sys/mips/alchemy/obio.c
deleted file mode 100644
index 03e098d..0000000
--- a/sys/mips/alchemy/obio.c
+++ /dev/null
@@ -1,501 +0,0 @@
-/*	$NetBSD: obio.c,v 1.11 2003/07/15 00:25:05 lukem Exp $	*/
-
-/*-
- * Copyright (c) 2001, 2002, 2003 Wasabi Systems, Inc.
- * All rights reserved.
- *
- * Written by Jason R. Thorpe for Wasabi Systems, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed for the NetBSD Project by
- *	Wasabi Systems, Inc.
- * 4. The name of Wasabi Systems, Inc. may not be used to endorse
- *    or promote products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
- * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/bus.h>
-#include <sys/interrupt.h>
-#include <sys/kernel.h>
-#include <sys/module.h>
-#include <sys/rman.h>
-#include <sys/malloc.h>
-
-#include <machine/bus.h>
-
-#include <mips/adm5120/adm5120reg.h>
-#include <mips/adm5120/obiovar.h>
-
-/* MIPS HW interrupts of IRQ/FIQ respectively */
-#define ADM5120_INTR		0
-#define ADM5120_FAST_INTR	1
-
-/* Interrupt levels */
-#define INTR_IRQ 0
-#define INTR_FIQ 1
-
-int irq_priorities[NIRQS] = {
-	INTR_IRQ,	/* flash */
-	INTR_FIQ,	/* uart0 */
-	INTR_FIQ,	/* uart1 */
-	INTR_IRQ,	/* ahci  */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* admsw */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-	INTR_IRQ,	/* unknown */
-};
-
-
-#define REG_READ(o) *((volatile uint32_t *)MIPS_PHYS_TO_KSEG1(ADM5120_BASE_ICU + (o)))
-#define REG_WRITE(o,v) (REG_READ(o)) = (v)
-
-static int	obio_activate_resource(device_t, device_t, int, int,
-		    struct resource *);
-static device_t	obio_add_child(device_t, int, const char *, int);
-static struct resource *
-		obio_alloc_resource(device_t, device_t, int, int *, u_long,
-		    u_long, u_long, u_int);
-static int	obio_attach(device_t);
-static int	obio_deactivate_resource(device_t, device_t, int, int,
-		    struct resource *);
-static struct resource_list *
-		obio_get_resource_list(device_t, device_t);
-static void	obio_hinted_child(device_t, const char *, int);
-static int	obio_intr(void *);
-static int	obio_probe(device_t);
-static int	obio_release_resource(device_t, device_t, int, int,
-		    struct resource *);
-static int	obio_setup_intr(device_t, device_t, struct resource *, int,
-		    driver_filter_t *, driver_intr_t *, void *, void **);
-static int	obio_teardown_intr(device_t, device_t, struct resource *,
-		    void *);
-
-static int
-obio_probe(device_t dev)
-{
-
-	return (0);
-}
-
-static int
-obio_attach(device_t dev)
-{
-	struct obio_softc *sc = device_get_softc(dev);
-	int rid;
-
-	sc->oba_mem_rman.rm_type = RMAN_ARRAY;
-	sc->oba_mem_rman.rm_descr = "OBIO memeory";
-	if (rman_init(&sc->oba_mem_rman) != 0 ||
-	    rman_manage_region(&sc->oba_mem_rman, OBIO_MEM_START,
-	        OBIO_MEM_START + OBIO_MEM_SIZE) != 0)
-		panic("obio_attach: failed to set up I/O rman");
-
-	sc->oba_irq_rman.rm_type = RMAN_ARRAY;
-	sc->oba_irq_rman.rm_descr = "OBIO IRQ";
-
-	if (rman_init(&sc->oba_irq_rman) != 0 ||
-	    rman_manage_region(&sc->oba_irq_rman, 0, NIRQS-1) != 0)
-		panic("obio_attach: failed to set up IRQ rman");
-
-	/* Hook up our interrupt handler. */
-	if ((sc->sc_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid,
-	    ADM5120_INTR, ADM5120_INTR, 1,
-	    RF_SHAREABLE | RF_ACTIVE)) == NULL) {
-		device_printf(dev, "unable to allocate IRQ resource\n");
-		return (ENXIO);
-	}
-
-	if ((bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_MISC, obio_intr, NULL,
-	    sc, &sc->sc_ih))) {
-		device_printf(dev,
-		    "WARNING: unable to register interrupt handler\n");
-		return (ENXIO);
-	}
-
-	/* Hook up our FAST interrupt handler. */
-	if ((sc->sc_fast_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid,
-	    ADM5120_FAST_INTR, ADM5120_FAST_INTR, 1,
-	    RF_SHAREABLE | RF_ACTIVE)) == NULL) {
-		device_printf(dev, "unable to allocate IRQ resource\n");
-		return (ENXIO);
-	}
-
-	if ((bus_setup_intr(dev, sc->sc_fast_irq, INTR_TYPE_MISC, obio_intr,
-	    NULL, sc, &sc->sc_fast_ih))) {
-		device_printf(dev,
-		    "WARNING: unable to register interrupt handler\n");
-		return (ENXIO);
-	}
-
-	/* disable all interrupts */
-	REG_WRITE(ICU_ENABLE_REG, ICU_INT_MASK);
-
-	bus_generic_probe(dev);
-	bus_enumerate_hinted_children(dev);
-	bus_generic_attach(dev);
-
-	return (0);
-}
-
-static struct resource *
-obio_alloc_resource(device_t bus, device_t child, int type, int *rid,
-    u_long start, u_long end, u_long count, u_int flags)
-{
-	struct obio_softc		*sc = device_get_softc(bus);
-	struct obio_ivar		*ivar = device_get_ivars(child);
-	struct resource			*rv;
-	struct resource_list_entry	*rle;
-	struct rman			*rm;
-	int				 isdefault, needactivate, passthrough;
-
-	isdefault = (start == 0UL && end == ~0UL && count == 1);
-	needactivate = flags & RF_ACTIVE;
-	passthrough = (device_get_parent(child) != bus);
-	rle = NULL;
-
-	if (passthrough)
-		return (BUS_ALLOC_RESOURCE(device_get_parent(bus), child, type,
-		    rid, start, end, count, flags));
-
-	/*
-	 * If this is an allocation of the "default" range for a given RID,
-	 * and we know what the resources for this device are (ie. they aren't
-	 * maintained by a child bus), then work out the start/end values.
-	 */
-	if (isdefault) {
-		rle = resource_list_find(&ivar->resources, type, *rid);
-		if (rle == NULL)
-			return (NULL);
-		if (rle->res != NULL) {
-			panic("%s: resource entry is busy", __func__);
-		}
-		start = rle->start;
-		end = rle->end;
-		count = rle->count;
-	}
-
-	switch (type) {
-	case SYS_RES_IRQ:
-		rm = &sc->oba_irq_rman;
-		break;
-	case SYS_RES_MEMORY:
-		rm = &sc->oba_mem_rman;
-		break;
-	default:
-		printf("%s: unknown resource type %d\n", __func__, type);
-		return (0);
-	}
-
-	rv = rman_reserve_resource(rm, start, end, count, flags, child);
-	if (rv == 0) {
-		printf("%s: could not reserve resource\n", __func__);
-		return (0);
-	}
-
-	rman_set_rid(rv, *rid);
-
-	if (needactivate) {
-		if (bus_activate_resource(child, type, *rid, rv)) {
-			printf("%s: could not activate resource\n", __func__);
-			rman_release_resource(rv);
-			return (0);
-		}
-	}
-
-	return (rv);
-}
-
-static int
-obio_activate_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
-{
-
-	/*
-	 * If this is a memory resource, track the direct mapping
-	 * in the uncached MIPS KSEG1 segment.
-	 */
-	if (type == SYS_RES_MEMORY) {
-		void *vaddr;
-
-		vaddr = (void *)MIPS_PHYS_TO_KSEG1((intptr_t)rman_get_start(r));
-		rman_set_virtual(r, vaddr);
-		rman_set_bustag(r, MIPS_BUS_SPACE_MEM);
-		rman_set_bushandle(r, (bus_space_handle_t)vaddr);
-	}
-
-	return (rman_activate_resource(r));
-}
-
-static int
-obio_deactivate_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
-{
-
-	return (rman_deactivate_resource(r));
-}
-
-static int
-obio_release_resource(device_t dev, device_t child, int type,
-    int rid, struct resource *r)
-{
-	struct resource_list *rl;
-	struct resource_list_entry *rle;
-
-	rl = obio_get_resource_list(dev, child);
-	if (rl == NULL)
-		return (EINVAL);
-	rle = resource_list_find(rl, type, rid);
-	if (rle == NULL)
-		return (EINVAL);
-	rman_release_resource(r);
-	rle->res = NULL;
-
-	return (0);
-}
-
-static int
-obio_setup_intr(device_t dev, device_t child, struct resource *ires,
-		int flags, driver_filter_t *filt, driver_intr_t *handler,
-		void *arg, void **cookiep)
-{
-	struct obio_softc *sc = device_get_softc(dev);
-	struct intr_event *event;
-	int irq, error, priority;
-	uint32_t irqmask;
-
-	irq = rman_get_start(ires);
-
-	if (irq >= NIRQS)
-		panic("%s: bad irq %d", __func__, irq);
-
-	event = sc->sc_eventstab[irq];
-	if (event == NULL) {
-		error = intr_event_create(&event, (void *)irq, 0, irq,
-		    (mask_fn)mips_mask_irq, (mask_fn)mips_unmask_irq,
-		    NULL, NULL, "obio intr%d:", irq);
-
-		sc->sc_eventstab[irq] = event;
-	}
-	else
-		panic("obio: Can't share IRQs");
-
-	intr_event_add_handler(event, device_get_nameunit(child), filt,
-	    handler, arg, intr_priority(flags), flags, cookiep);
-
-	irqmask = 1 << irq;
-	priority = irq_priorities[irq];
-
-	if (priority == INTR_FIQ)
-		REG_WRITE(ICU_MODE_REG, REG_READ(ICU_MODE_REG) | irqmask);
-	else
-		REG_WRITE(ICU_MODE_REG, REG_READ(ICU_MODE_REG) & ~irqmask);
-
-	/* enable */
-	REG_WRITE(ICU_ENABLE_REG, irqmask);
-
-	return (0);
-}
-
-static int
-obio_teardown_intr(device_t dev, device_t child, struct resource *ires,
-    void *cookie)
-{
-	struct obio_softc *sc = device_get_softc(dev);
-	int irq, result;
-	uint32_t irqmask;
-
-	irq = rman_get_start(ires);
-	if (irq >= NIRQS)
-		panic("%s: bad irq %d", __func__, irq);
-
-	if (sc->sc_eventstab[irq] == NULL)
-		panic("Trying to teardown unoccupied IRQ");
-
-	irqmask = 1 << irq;     /* only used as a mask from here on */
-
-	/* disable this irq in HW */
-	REG_WRITE(ICU_DISABLE_REG, irqmask);
-
-	result = intr_event_remove_handler(cookie);
-	if (!result) {
-		sc->sc_eventstab[irq] = NULL;
-	}
-
-	return (result);
-}
-
-static int
-obio_intr(void *arg)
-{
-	struct obio_softc *sc = arg;
-	struct intr_event *event;
-	uint32_t irqstat;
-	int irq;
-
-	irqstat = REG_READ(ICU_FIQ_STATUS_REG);
-	irqstat |= REG_READ(ICU_STATUS_REG);
-
-	irq = 0;
-	while (irqstat != 0) {
-		if ((irqstat & 1) == 1) {
-			event = sc->sc_eventstab[irq];
-			if (!event || TAILQ_EMPTY(&event->ie_handlers))
-				continue;
-
-			/* TODO: pass frame as an argument*/
-			/* TODO: log stray interrupt */
-			intr_event_handle(event, NULL);
-		}
-
-		irq++;
-		irqstat >>= 1;
-	}
-
-	return (FILTER_HANDLED);
-}
-
-static void
-obio_hinted_child(device_t bus, const char *dname, int dunit)
-{
-	device_t		child;
-	long			maddr;
-	int			msize;
-	int			irq;
-	int			result;
-
-	child = BUS_ADD_CHILD(bus, 0, dname, dunit);
-
-	/*
-	 * Set hard-wired resources for hinted child using
-	 * specific RIDs.
-	 */
-	resource_long_value(dname, dunit, "maddr", &maddr);
-	resource_int_value(dname, dunit, "msize", &msize);
-
-
-	result = bus_set_resource(child, SYS_RES_MEMORY, 0,
-	    maddr, msize);
-	if (result != 0)
-		device_printf(bus, "warning: bus_set_resource() failed\n");
-
-	if (resource_int_value(dname, dunit, "irq", &irq) == 0) {
-		result = bus_set_resource(child, SYS_RES_IRQ, 0, irq, 1);
-		if (result != 0)
-			device_printf(bus,
-			    "warning: bus_set_resource() failed\n");
-	}
-}
-
-static device_t
-obio_add_child(device_t bus, int order, const char *name, int unit)
-{
-	device_t		child;
-	struct obio_ivar	*ivar;
-
-	ivar = malloc(sizeof(struct obio_ivar), M_DEVBUF, M_WAITOK | M_ZERO);
-	if (ivar == NULL) {
-		printf("Failed to allocate ivar\n");
-		return (0);
-	}
-	resource_list_init(&ivar->resources);
-
-	child = device_add_child_ordered(bus, order, name, unit);
-	if (child == NULL) {
-		printf("Can't add child %s%d ordered\n", name, unit);
-		return (0);
-	}
-
-	device_set_ivars(child, ivar);
-
-	return (child);
-}
-
-/*
- * Helper routine for bus_generic_rl_get_resource/bus_generic_rl_set_resource
- * Provides pointer to resource_list for these routines
- */
-static struct resource_list *
-obio_get_resource_list(device_t dev, device_t child)
-{
-	struct obio_ivar *ivar;
-
-	ivar = device_get_ivars(child);
-	return (&(ivar->resources));
-}
-
-static device_method_t obio_methods[] = {
-	DEVMETHOD(bus_activate_resource,	obio_activate_resource),
-	DEVMETHOD(bus_add_child,		obio_add_child),
-	DEVMETHOD(bus_alloc_resource,		obio_alloc_resource),
-	DEVMETHOD(bus_deactivate_resource,	obio_deactivate_resource),
-	DEVMETHOD(bus_get_resource_list,	obio_get_resource_list),
-	DEVMETHOD(bus_hinted_child,		obio_hinted_child),
-	DEVMETHOD(bus_release_resource,		obio_release_resource),
-	DEVMETHOD(bus_setup_intr,		obio_setup_intr),
-	DEVMETHOD(bus_teardown_intr,		obio_teardown_intr),
-	DEVMETHOD(device_attach,		obio_attach),
-	DEVMETHOD(device_probe,			obio_probe),
-        DEVMETHOD(bus_get_resource,		bus_generic_rl_get_resource),
-        DEVMETHOD(bus_set_resource,		bus_generic_rl_set_resource),
-
-	{0, 0},
-};
-
-static driver_t obio_driver = {
-	"obio",
-	obio_methods,
-	sizeof(struct obio_softc),
-};
-static devclass_t obio_devclass;
-
-DRIVER_MODULE(obio, nexus, obio_driver, obio_devclass, 0, 0);
diff --git a/sys/mips/alchemy/std.alchemy b/sys/mips/alchemy/std.alchemy
deleted file mode 100644
index a955b67..0000000
--- a/sys/mips/alchemy/std.alchemy
+++ /dev/null
@@ -1,8 +0,0 @@
-# $FreeBSD$
-# Standard include file for Alchemy Au1xxx CPUs: 
-# Au1000, Au1200, Au1250, Au1500 and Au1550
-
-files	"../alchemy/files.alchemy"
-
-cpu		CPU_MIPS4KC
-options		ISA_MIPS32
diff --git a/sys/mips/alchemy/uart_bus_alchemy.c b/sys/mips/alchemy/uart_bus_alchemy.c
deleted file mode 100644
index 5c2315b..0000000
--- a/sys/mips/alchemy/uart_bus_alchemy.c
+++ /dev/null
@@ -1,87 +0,0 @@
-/*-
- * Copyright (c) 2007 Bruce M. Simpson.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * $Id$
- */
-/*
- * Skeleton of this file was based on respective code for ARM
- * code written by Olivier Houchard.
- */
-
-#include "opt_uart.h"
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/bus.h>
-#include <sys/conf.h>
-#include <sys/kernel.h>
-#include <sys/module.h>
-#include <machine/bus.h>
-#include <sys/rman.h>
-#include <machine/resource.h>
-
-#include <dev/pci/pcivar.h>
-
-#include <dev/uart/uart.h>
-#include <dev/uart/uart_bus.h>
-#include <dev/uart/uart_cpu.h>
-
-#include <mips/alchemy/aureg.h>
-
-#include "uart_if.h"
-
-static int uart_alchemy_probe(device_t dev);
-
-static device_method_t uart_alchemy_methods[] = {
-	/* Device interface */
-	DEVMETHOD(device_probe,		uart_alchemy_probe),
-	DEVMETHOD(device_attach,	uart_bus_attach),
-	DEVMETHOD(device_detach,	uart_bus_detach),
-	{ 0, 0 }
-};
-
-static driver_t uart_alchemy_driver = {
-	uart_driver_name,
-	uart_alchemy_methods,
-	sizeof(struct uart_softc),
-};
-
-extern SLIST_HEAD(uart_devinfo_list, uart_devinfo) uart_sysdevs;
-
-static int
-uart_alchemy_probe(device_t dev)
-{
-	struct uart_softc *sc;
-
-	sc = device_get_softc(dev);
-	sc->sc_sysdev = SLIST_FIRST(&uart_sysdevs);
-	sc->sc_class = &uart_ns8250_class;
-	bcopy(&sc->sc_sysdev->bas, &sc->sc_bas, sizeof(sc->sc_bas));
-
-	return (uart_bus_probe(dev, 0, 0, 0, 0));
-}
-
-DRIVER_MODULE(uart, obio, uart_alchemy_driver, uart_devclass, 0, 0);
diff --git a/sys/mips/atheros/apb.c b/sys/mips/atheros/apb.c
deleted file mode 100644
index d53408c..0000000
--- a/sys/mips/atheros/apb.c
+++ /dev/null
@@ -1,433 +0,0 @@
-/*-
- * Copyright (c) 2009, Oleksandr Tymoshenko <gonzo@FreeBSD.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice unmodified, this list of conditions, and the following
- *    disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/bus.h>
-#include <sys/interrupt.h>
-#include <sys/kernel.h>
-#include <sys/module.h>
-#include <sys/rman.h>
-#include <sys/malloc.h>
-
-#include <machine/bus.h>
-
-#include <mips/atheros/apbvar.h>
-#include <mips/atheros/ar71xxreg.h>
-
-#undef APB_DEBUG
-#ifdef APB_DEBUG
-#define dprintf printf
-#else 
-#define dprintf(x, arg...)
-#endif  /* APB_DEBUG */
-
-static int	apb_activate_resource(device_t, device_t, int, int,
-		    struct resource *);
-static device_t	apb_add_child(device_t, int, const char *, int);
-static struct resource *
-		apb_alloc_resource(device_t, device_t, int, int *, u_long,
-		    u_long, u_long, u_int);
-static int	apb_attach(device_t);
-static int	apb_deactivate_resource(device_t, device_t, int, int,
-		    struct resource *);
-static struct resource_list *
-		apb_get_resource_list(device_t, device_t);
-static void	apb_hinted_child(device_t, const char *, int);
-static int	apb_intr(void *);
-static int	apb_probe(device_t);
-static int	apb_release_resource(device_t, device_t, int, int,
-		    struct resource *);
-static int	apb_setup_intr(device_t, device_t, struct resource *, int,
-		    driver_filter_t *, driver_intr_t *, void *, void **);
-static int	apb_teardown_intr(device_t, device_t, struct resource *,
-		    void *);
-
-static void apb_mask_irq(unsigned int irq)
-{
-	uint32_t reg;
-
-	reg = ATH_READ_REG(AR71XX_MISC_INTR_MASK);
-	ATH_WRITE_REG(AR71XX_MISC_INTR_MASK, reg & ~(1 << irq));
-
-}
-
-static void apb_unmask_irq(unsigned int irq)
-{
-	uint32_t reg;
-
-	reg = ATH_READ_REG(AR71XX_MISC_INTR_MASK);
-	ATH_WRITE_REG(AR71XX_MISC_INTR_MASK, reg | (1 << irq));
-}
-
-static int
-apb_probe(device_t dev)
-{
-
-	return (0);
-}
-
-static int
-apb_attach(device_t dev)
-{
-	struct apb_softc *sc = device_get_softc(dev);
-	int rid = 0;
-
-	device_set_desc(dev, "APB Bus bridge");
-	sc->apb_irq_rman.rm_type = RMAN_ARRAY;
-	sc->apb_irq_rman.rm_descr = "APB IRQ";
-
-	if (rman_init(&sc->apb_irq_rman) != 0 ||
-	    rman_manage_region(&sc->apb_irq_rman, 
-			APB_IRQ_BASE, APB_IRQ_END) != 0)
-		panic("apb_attach: failed to set up IRQ rman");
-
-	if ((sc->sc_misc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 
-	    RF_SHAREABLE | RF_ACTIVE)) == NULL) {
-		device_printf(dev, "unable to allocate IRQ resource\n");
-		return (ENXIO);
-	}
-
-	if ((bus_setup_intr(dev, sc->sc_misc_irq, INTR_TYPE_MISC, 
-	    apb_intr, NULL, sc, &sc->sc_misc_ih))) {
-		device_printf(dev,
-		    "WARNING: unable to register interrupt handler\n");
-		return (ENXIO);
-	}
-
-	bus_generic_probe(dev);
-	bus_enumerate_hinted_children(dev);
-	bus_generic_attach(dev);
-
-	return (0);
-}
-
-static struct resource *
-apb_alloc_resource(device_t bus, device_t child, int type, int *rid,
-    u_long start, u_long end, u_long count, u_int flags)
-{
-	struct apb_softc		*sc = device_get_softc(bus);
-	struct apb_ivar			*ivar = device_get_ivars(child);
-	struct resource			*rv;
-	struct resource_list_entry	*rle;
-	struct rman			*rm;
-	int				 isdefault, needactivate, passthrough;
-
-	isdefault = (start == 0UL && end == ~0UL);
-	needactivate = flags & RF_ACTIVE;
-	/*
-	 * Pass memory requests to nexus device
-	 */
-	passthrough = (device_get_parent(child) != bus) ||
-		(type == SYS_RES_MEMORY);
-	rle = NULL;
-
-	dprintf("%s: entry (%p, %p, %d, %p, %p, %p, %ld, %d)\n",
-	    __func__, bus, child, type, rid, (void *)(intptr_t)start,
-	    (void *)(intptr_t)end, count, flags);
-
-	if (passthrough)
-		return (BUS_ALLOC_RESOURCE(device_get_parent(bus), child, type,
-		    rid, start, end, count, flags));
-
-	/*
-	 * If this is an allocation of the "default" range for a given RID,
-	 * and we know what the resources for this device are (ie. they aren't
-	 * maintained by a child bus), then work out the start/end values.
-	 */
-
-	if (isdefault) {
-		rle = resource_list_find(&ivar->resources, type, *rid);
-		if (rle == NULL) {
-			return (NULL);
-		}
-
-		if (rle->res != NULL) {
-			panic("%s: resource entry is busy", __func__);
-		}
-		start = rle->start;
-		end = rle->end;
-		count = rle->count;
-
-		dprintf("%s: default resource (%p, %p, %ld)\n",
-		    __func__, (void *)(intptr_t)start,
-		    (void *)(intptr_t)end, count);
-	}
-
-	switch (type) {
-	case SYS_RES_IRQ:
-		rm = &sc->apb_irq_rman;
-		break;
-	default:
-		printf("%s: unknown resource type %d\n", __func__, type);
-		return (0);
-	}
-
-	rv = rman_reserve_resource(rm, start, end, count, flags, child);
-	if (rv == 0) {
-		printf("%s: could not reserve resource\n", __func__);
-		return (0);
-	}
-
-	rman_set_rid(rv, *rid);
-
-	if (needactivate) {
-		if (bus_activate_resource(child, type, *rid, rv)) {
-			printf("%s: could not activate resource\n", __func__);
-			rman_release_resource(rv);
-			return (0);
-		}
-	}
-
-	return (rv);
-}
-
-static int
-apb_activate_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
-{
-
-	/* XXX: should we mask/unmask IRQ here? */
-	return (BUS_ACTIVATE_RESOURCE(device_get_parent(bus), child,
-		type, rid, r));
-}
-
-static int
-apb_deactivate_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
-{
-
-	/* XXX: should we mask/unmask IRQ here? */
-	return (BUS_DEACTIVATE_RESOURCE(device_get_parent(bus), child,
-		type, rid, r));
-}
-
-static int
-apb_release_resource(device_t dev, device_t child, int type,
-    int rid, struct resource *r)
-{
-	struct resource_list *rl;
-	struct resource_list_entry *rle;
-
-	rl = apb_get_resource_list(dev, child);
-	if (rl == NULL)
-		return (EINVAL);
-	rle = resource_list_find(rl, type, rid);
-	if (rle == NULL)
-		return (EINVAL);
-	rman_release_resource(r);
-	rle->res = NULL;
-
-	return (0);
-}
-
-static int
-apb_setup_intr(device_t bus, device_t child, struct resource *ires,
-		int flags, driver_filter_t *filt, driver_intr_t *handler,
-		void *arg, void **cookiep)
-{
-	struct apb_softc *sc = device_get_softc(bus);
-	struct intr_event *event;
-	int irq, error;
-
-	irq = rman_get_start(ires);
-
-	if (irq > APB_IRQ_END)
-		panic("%s: bad irq %d", __func__, irq);
-
-	event = sc->sc_eventstab[irq];
-	if (event == NULL) {
-		error = intr_event_create(&event, (void *)irq, 0, irq, 
-		    (mask_fn)apb_mask_irq, (mask_fn)apb_unmask_irq,
-		    NULL, NULL,
-		    "apb intr%d:", irq);
-
-		sc->sc_eventstab[irq] = event;
-	}
-
-	intr_event_add_handler(event, device_get_nameunit(child), filt,
-	    handler, arg, intr_priority(flags), flags, cookiep);
-
-	return (0);
-}
-
-static int
-apb_teardown_intr(device_t dev, device_t child, struct resource *ires,
-    void *cookie)
-{
-	struct apb_softc *sc = device_get_softc(dev);
-	int irq, result;
-
-	irq = rman_get_start(ires);
-	if (irq > APB_IRQ_END)
-		panic("%s: bad irq %d", __func__, irq);
-
-	if (sc->sc_eventstab[irq] == NULL)
-		panic("Trying to teardown unoccupied IRQ");
-
-	apb_mask_irq(irq);
-
-	result = intr_event_remove_handler(cookie);
-	if (!result)
-		sc->sc_eventstab[irq] = NULL;
-
-	return (result);
-}
-
-static int
-apb_intr(void *arg)
-{
-	struct apb_softc *sc = arg;
-	struct intr_event *event;
-	uint32_t reg, irq;
-
-	reg = ATH_READ_REG(AR71XX_MISC_INTR_STATUS);
-	for (irq = 0; irq < APB_NIRQS; irq++) {
-		if (reg & (1 << irq)) {
-			event = sc->sc_eventstab[irq];
-			if (!event || TAILQ_EMPTY(&event->ie_handlers)) {
-				printf("Stray IRQ %d\n", irq);
-				continue;
-			}
-
-			/* TODO: frame instead of NULL? */
-			intr_event_handle(event, NULL);
-		}
-	}
-
-	return (FILTER_HANDLED);
-}
-
-static void
-apb_hinted_child(device_t bus, const char *dname, int dunit)
-{
-	device_t		child;
-	long			maddr;
-	int			msize;
-	int			irq;
-	int			result;
-	int			mem_hints_count;
-
-	child = BUS_ADD_CHILD(bus, 0, dname, dunit);
-
-	/*
-	 * Set hard-wired resources for hinted child using
-	 * specific RIDs.
-	 */
-	mem_hints_count = 0;
-	if (resource_long_value(dname, dunit, "maddr", &maddr) == 0)
-		mem_hints_count++;
-	if (resource_int_value(dname, dunit, "msize", &msize) == 0)
-		mem_hints_count++;
-
-	/* check if all info for mem resource has been provided */
-	if ((mem_hints_count > 0) && (mem_hints_count < 2)) {
-		printf("Either maddr or msize hint is missing for %s%d\n",
-		    dname, dunit);
-	} else if (mem_hints_count) {
-		result = bus_set_resource(child, SYS_RES_MEMORY, 0,
-		    maddr, msize);
-		if (result != 0)
-			device_printf(bus, 
-			    "warning: bus_set_resource() failed\n");
-	}
-
-	if (resource_int_value(dname, dunit, "irq", &irq) == 0) {
-		result = bus_set_resource(child, SYS_RES_IRQ, 0, irq, 1);
-		if (result != 0)
-			device_printf(bus,
-			    "warning: bus_set_resource() failed\n");
-	}
-}
-
-static device_t
-apb_add_child(device_t bus, int order, const char *name, int unit)
-{
-	device_t		child;
-	struct apb_ivar	*ivar;
-
-	ivar = malloc(sizeof(struct apb_ivar), M_DEVBUF, M_WAITOK | M_ZERO);
-	if (ivar == NULL) {
-		printf("Failed to allocate ivar\n");
-		return (0);
-	}
-	resource_list_init(&ivar->resources);
-
-	child = device_add_child_ordered(bus, order, name, unit);
-	if (child == NULL) {
-		printf("Can't add child %s%d ordered\n", name, unit);
-		return (0);
-	}
-
-	device_set_ivars(child, ivar);
-
-	return (child);
-}
-
-/*
- * Helper routine for bus_generic_rl_get_resource/bus_generic_rl_set_resource
- * Provides pointer to resource_list for these routines
- */
-static struct resource_list *
-apb_get_resource_list(device_t dev, device_t child)
-{
-	struct apb_ivar *ivar;
-
-	ivar = device_get_ivars(child);
-	return (&(ivar->resources));
-}
-
-static device_method_t apb_methods[] = {
-	DEVMETHOD(bus_activate_resource,	apb_activate_resource),
-	DEVMETHOD(bus_add_child,		apb_add_child),
-	DEVMETHOD(bus_alloc_resource,		apb_alloc_resource),
-	DEVMETHOD(bus_deactivate_resource,	apb_deactivate_resource),
-	DEVMETHOD(bus_get_resource_list,	apb_get_resource_list),
-	DEVMETHOD(bus_hinted_child,		apb_hinted_child),
-	DEVMETHOD(bus_print_child,		bus_generic_print_child),
-	DEVMETHOD(bus_release_resource,		apb_release_resource),
-	DEVMETHOD(bus_setup_intr,		apb_setup_intr),
-	DEVMETHOD(bus_teardown_intr,		apb_teardown_intr),
-	DEVMETHOD(device_attach,		apb_attach),
-	DEVMETHOD(device_probe,			apb_probe),
-	DEVMETHOD(bus_get_resource,		bus_generic_rl_get_resource),
-	DEVMETHOD(bus_set_resource,		bus_generic_rl_set_resource),
-
-	{0, 0},
-};
-
-static driver_t apb_driver = {
-	"apb",
-	apb_methods,
-	sizeof(struct apb_softc),
-};
-static devclass_t apb_devclass;
-
-DRIVER_MODULE(apb, nexus, apb_driver, apb_devclass, 0, 0);
diff --git a/sys/mips/atheros/ar71xx_machdep.c b/sys/mips/atheros/ar71xx_machdep.c
deleted file mode 100644
index a5699ed..0000000
--- a/sys/mips/atheros/ar71xx_machdep.c
+++ /dev/null
@@ -1,161 +0,0 @@
-/*-
- * Copyright (c) 2009 Oleksandr Tymoshenko
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <machine/cpuregs.h>
-
-#include <mips/sentry5/s5reg.h>
-
-#include "opt_ddb.h"
-
-#include <sys/param.h>
-#include <sys/conf.h>
-#include <sys/kernel.h>
-#include <sys/systm.h>
-#include <sys/bus.h>
-#include <sys/cons.h>
-#include <sys/kdb.h>
-
-#include <vm/vm.h>
-#include <vm/vm_page.h>
-
-#include <machine/clock.h>
-#include <machine/cpu.h>
-#include <machine/hwfunc.h>
-#include <machine/md_var.h>
-#include <machine/trap.h>
-#include <machine/vmparam.h>
-
-#include <mips/atheros/ar71xxreg.h>
-
-extern int *edata;
-extern int *end;
-
-void
-platform_halt(void)
-{
-
-}
-
-void
-platform_identify(void)
-{
-
-}
-
-void
-platform_reset(void)
-{
-	uint32_t reg = ATH_READ_REG(AR71XX_RST_RESET);
-
-	ATH_WRITE_REG(AR71XX_RST_RESET, reg | RST_RESET_FULL_CHIP);
-	/* Wait for reset */
-	while(1)
-		;
-}
-
-void
-platform_trap_enter(void)
-{
-
-}
-
-void
-platform_trap_exit(void)
-{
-
-}
-
-void
-platform_start(__register_t a0 __unused, __register_t a1 __unused, 
-    __register_t a2 __unused, __register_t a3 __unused)
-{
-	vm_offset_t kernend;
-	uint64_t platform_counter_freq;
-	uint32_t reg;
-
-	/* clear the BSS and SBSS segments */
-	kernend = round_page((vm_offset_t)&end);
-	memset(&edata, 0, kernend - (vm_offset_t)(&edata));
-
-	/* TODO: Get available memory from RedBoot. Is it possible? */
-	realmem = btoc(64*1024*1024);
-	/* phys_avail regions are in bytes */
-	phys_avail[0] = MIPS_KSEG0_TO_PHYS((vm_offset_t)&end);
-	phys_avail[1] = ctob(realmem);
-
-	physmem = realmem;
-
-	/*
-	 * ns8250 uart code uses DELAY so ticker should be inititalized 
-	 * before cninit. And tick_init_params refers to hz, so * init_param1 
-	 * should be called first.
-	 */
-	init_param1();
-	/* TODO: Get CPU freq from RedBoot. Is it possible? */
-	platform_counter_freq = 680000000UL;
-	mips_timer_init_params(platform_counter_freq, 0);
-	cninit();
-
-	printf("arguments: \n");
-	printf("  a0 = %08x\n", a0);
-	printf("  a1 = %08x\n", a1);
-	printf("  a2 = %08x\n", a2);
-	printf("  a3 = %08x\n", a3);
-
-	init_param2(physmem);
-	mips_cpu_init();
-	pmap_bootstrap();
-	mips_proc0_init();
-	mutex_init();
-
-	/*
-	 * Reset USB devices 
-	 */
-	reg = ATH_READ_REG(AR71XX_RST_RESET);
-	reg |= 
-	    RST_RESET_USB_OHCI_DLL | RST_RESET_USB_HOST | RST_RESET_USB_PHY;
-	ATH_WRITE_REG(AR71XX_RST_RESET, reg);
-	DELAY(1000);
-	reg &= 
-	    ~(RST_RESET_USB_OHCI_DLL | RST_RESET_USB_HOST | RST_RESET_USB_PHY);
-	ATH_WRITE_REG(AR71XX_RST_RESET, reg);
-	
-	ATH_WRITE_REG(AR71XX_USB_CTRL_CONFIG,
-	    USB_CTRL_CONFIG_OHCI_DES_SWAP | USB_CTRL_CONFIG_OHCI_BUF_SWAP |
-	    USB_CTRL_CONFIG_EHCI_DES_SWAP | USB_CTRL_CONFIG_EHCI_BUF_SWAP);
-
-	ATH_WRITE_REG(AR71XX_USB_CTRL_FLADJ, 
-	    (32 << USB_CTRL_FLADJ_HOST_SHIFT) | (3 << USB_CTRL_FLADJ_A5_SHIFT));
-	DELAY(1000);
-
-#ifdef DDB
-	kdb_init();
-#endif
-}
diff --git a/sys/mips/atheros/ar71xx_ohci.c b/sys/mips/atheros/ar71xx_ohci.c
deleted file mode 100644
index b7adc61..0000000
--- a/sys/mips/atheros/ar71xx_ohci.c
+++ /dev/null
@@ -1,205 +0,0 @@
-/*-
- * Copyright (c) 2009, Oleksandr Tymoshenko <gonzo@FreeBSD.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice unmodified, this list of conditions, and the following
- *    disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/kernel.h>
-#include <sys/lock.h>
-#include <sys/module.h>
-#include <sys/mutex.h>
-#include <sys/bus.h>
-#include <sys/queue.h>
-#include <machine/bus.h>
-#include <sys/rman.h>
-#include <machine/resource.h>
-
-#include <dev/usb/usb.h>
-#include <dev/usb/usbdi.h>
-#include <dev/usb/usbdivar.h>
-#include <dev/usb/usb_mem.h>
-
-#include <dev/usb/ohcireg.h>
-#include <dev/usb/ohcivar.h>
-
-static int ar71xx_ohci_attach(device_t dev);
-static int ar71xx_ohci_detach(device_t dev);
-static int ar71xx_ohci_probe(device_t dev);
-
-struct ar71xx_ohci_softc
-{
-	struct ohci_softc sc_ohci;
-};
-
-static int
-ar71xx_ohci_probe(device_t dev)
-{
-	device_set_desc(dev, "AR71XX integrated OHCI controller");
-	return (BUS_PROBE_DEFAULT);
-}
-
-static int
-ar71xx_ohci_attach(device_t dev)
-{
-	struct ar71xx_ohci_softc *sc = device_get_softc(dev);
-	int err;
-	int rid;
-
-	rid = 0;
-	sc->sc_ohci.io_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
-	    RF_ACTIVE);
-	if (sc->sc_ohci.io_res == NULL) {
-		err = ENOMEM;
-		goto error;
-	}
-	sc->sc_ohci.iot = rman_get_bustag(sc->sc_ohci.io_res);
-	sc->sc_ohci.ioh = rman_get_bushandle(sc->sc_ohci.io_res);
-
-	rid = 0;
-	sc->sc_ohci.irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
-	    RF_ACTIVE);
-	if (sc->sc_ohci.irq_res == NULL) {
-		err = ENOMEM;
-		goto error;
-	}
-	sc->sc_ohci.sc_bus.bdev = device_add_child(dev, "usb", -1);
-	if (sc->sc_ohci.sc_bus.bdev == NULL) {
-		err = ENOMEM;
-		goto error;
-	}
-	device_set_ivars(sc->sc_ohci.sc_bus.bdev, &sc->sc_ohci.sc_bus);
-
-	/* Allocate a parent dma tag for DMA maps */
-	err = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0,
-	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
-	    BUS_SPACE_MAXSIZE_32BIT, USB_DMA_NSEG, BUS_SPACE_MAXSIZE_32BIT, 0,
-	    NULL, NULL, &sc->sc_ohci.sc_bus.parent_dmatag);
-	if (err) {
-		device_printf(dev, "Could not allocate parent DMA tag (%d)\n",
-		    err);
-		err = ENXIO;
-		goto error;
-	}
-
-	/* Allocate a dma tag for transfer buffers */
-	err = bus_dma_tag_create(sc->sc_ohci.sc_bus.parent_dmatag, 1, 0,
-	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
-	    BUS_SPACE_MAXSIZE_32BIT, USB_DMA_NSEG, BUS_SPACE_MAXSIZE_32BIT, 0,
-	    busdma_lock_mutex, &Giant, &sc->sc_ohci.sc_bus.buffer_dmatag);
-	if (err) {
-		device_printf(dev, "Could not allocate transfer tag (%d)\n",
-		    err);
-		err = ENXIO;
-		goto error;
-	}
-
-	err = bus_setup_intr(dev, sc->sc_ohci.irq_res, INTR_TYPE_BIO, NULL, 
-	    ohci_intr, sc, &sc->sc_ohci.ih);
-	if (err) {
-		err = ENXIO;
-		goto error;
-	}
-	strlcpy(sc->sc_ohci.sc_vendor, "Atheros", 
-	    sizeof(sc->sc_ohci.sc_vendor));
-
-	bus_space_write_4(sc->sc_ohci.iot, sc->sc_ohci.ioh, OHCI_CONTROL, 0);
-
-	err = ohci_init(&sc->sc_ohci);
-	if (!err) {
-		sc->sc_ohci.sc_flags |= OHCI_SCFLG_DONEINIT;
-		err = device_probe_and_attach(sc->sc_ohci.sc_bus.bdev);
-	}
-
-error:
-	if (err) {
-		ar71xx_ohci_detach(dev);
-		return (err);
-	}
-	return (err);
-}
-
-static int
-ar71xx_ohci_detach(device_t dev)
-{
-	struct ar71xx_ohci_softc *sc = device_get_softc(dev);
-
-	if (sc->sc_ohci.sc_flags & OHCI_SCFLG_DONEINIT) {
-		ohci_detach(&sc->sc_ohci, 0);
-		sc->sc_ohci.sc_flags &= ~OHCI_SCFLG_DONEINIT;
-	}
-
-	if (sc->sc_ohci.ih) {
-		bus_teardown_intr(dev, sc->sc_ohci.irq_res, sc->sc_ohci.ih);
-		sc->sc_ohci.ih = NULL;
-	}
-
-	if (sc->sc_ohci.sc_bus.parent_dmatag != NULL)
-		bus_dma_tag_destroy(sc->sc_ohci.sc_bus.parent_dmatag);
-	if (sc->sc_ohci.sc_bus.buffer_dmatag != NULL)
-		bus_dma_tag_destroy(sc->sc_ohci.sc_bus.buffer_dmatag);
-
-	if (sc->sc_ohci.sc_bus.bdev) {
-		device_delete_child(dev, sc->sc_ohci.sc_bus.bdev);
-		sc->sc_ohci.sc_bus.bdev = NULL;
-	}
-	if (sc->sc_ohci.irq_res) {
-		bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_ohci.irq_res);
-		sc->sc_ohci.irq_res = NULL;
-	}
-	if (sc->sc_ohci.io_res) {
-		bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_ohci.io_res);
-		sc->sc_ohci.io_res = NULL;
-		sc->sc_ohci.iot = 0;
-		sc->sc_ohci.ioh = 0;
-	}
-	return (0);
-}
-
-static device_method_t ohci_methods[] = {
-	/* Device interface */
-	DEVMETHOD(device_probe, ar71xx_ohci_probe),
-	DEVMETHOD(device_attach, ar71xx_ohci_attach),
-	DEVMETHOD(device_detach, ar71xx_ohci_detach),
-	DEVMETHOD(device_shutdown, bus_generic_shutdown),
-
-	/* Bus interface */
-	DEVMETHOD(bus_print_child, bus_generic_print_child),
-
-	{0, 0}
-};
-
-static driver_t ohci_driver = {
-	"ohci",
-	ohci_methods,
-	sizeof(struct ar71xx_ohci_softc),
-};
-
-static devclass_t ohci_devclass;
-
-DRIVER_MODULE(ohci, apb, ohci_driver, ohci_devclass, 0, 0);
diff --git a/sys/mips/atheros/ar71xx_pci.c b/sys/mips/atheros/ar71xx_pci.c
deleted file mode 100644
index d3a9295..0000000
--- a/sys/mips/atheros/ar71xx_pci.c
+++ /dev/null
@@ -1,429 +0,0 @@
-/*-
- * Copyright (c) 2009, Oleksandr Tymoshenko <gonzo@FreeBSD.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice unmodified, this list of conditions, and the following
- *    disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-
-#include <sys/bus.h>
-#include <sys/interrupt.h>
-#include <sys/malloc.h>
-#include <sys/kernel.h>
-#include <sys/module.h>
-#include <sys/rman.h>
-
-#include <vm/vm.h>
-#include <vm/pmap.h>
-#include <vm/vm_extern.h>
-
-#include <machine/bus.h>
-#include <machine/cpu.h>
-#include <machine/pmap.h>
-
-#include <dev/pci/pcivar.h>
-#include <dev/pci/pcireg.h>
-
-#include <dev/pci/pcib_private.h>
-#include "pcib_if.h"
-
-#include "mips/atheros/ar71xxreg.h"
-
-#undef AR71XX_PCI_DEBUG
-#ifdef AR71XX_PCI_DEBUG
-#define dprintf printf
-#else
-#define dprintf(x, arg...)
-#endif
-
-struct ar71xx_pci_softc {
-	device_t		sc_dev;
-
-	int			sc_busno;
-	struct rman		sc_mem_rman;
-	struct rman		sc_irq_rman;
-
-	struct resource		*sc_irq;
-	void			*sc_ih;
-};
-
-/* 
- * get bitmask for bytes of interest: 
- *   0 - we want this byte, 1 - ignore it. e.g: we read 1 byte 
- *   from register 7. Bitmask would be: 0111
- */
-static uint32_t
-ar71xx_get_bytes_to_read(int reg, int bytes)
-{
-	uint32_t bytes_to_read = 0;
-	if ((bytes % 4) == 0)
-		bytes_to_read = 0;
-	else if ((bytes % 4) == 1)
-		bytes_to_read = (~(1 << (reg % 4))) & 0xf;
-	else if ((bytes % 4) == 2)
-		bytes_to_read = (~(3 << (reg % 4))) & 0xf;
-	else
-		panic("%s: wrong combination", __func__);
-
-	return (bytes_to_read);
-}
-
-static int 
-ar71xx_pci_check_bus_error(void)
-{
-	uint32_t error, addr, has_errors = 0;
-	error = ATH_READ_REG(AR71XX_PCI_ERROR) & 0x3;
-	dprintf("%s: PCI error = %02x\n", __func__, error);
-	if (error) {
-		addr = ATH_READ_REG(AR71XX_PCI_ERROR_ADDR);
-
-		/* Do not report it yet */
-#if 0
-		printf("PCI bus error %d at addr 0x%08x\n", error, addr);
-#endif
-		ATH_WRITE_REG(AR71XX_PCI_ERROR, error);
-		has_errors = 1;
-	}
-
-	error = ATH_READ_REG(AR71XX_PCI_AHB_ERROR) & 0x1;
-	dprintf("%s: AHB error = %02x\n", __func__, error);
-	if (error) {
-		addr = ATH_READ_REG(AR71XX_PCI_AHB_ERROR_ADDR);
-		/* Do not report it yet */
-#if 0
-		printf("AHB bus error %d at addr 0x%08x\n", error, addr);
-#endif
-		ATH_WRITE_REG(AR71XX_PCI_AHB_ERROR, error);
-		has_errors = 1;
-	}
-
-	return (has_errors);
-}
-
-static uint32_t
-ar71xx_pci_make_addr(int bus, int slot, int func, int reg)
-{
-	if (bus == 0) {
-		return ((1 << slot) | (func << 8) | (reg & ~3));
-	} else {
-		return ((bus << 16) | (slot << 11) | (func << 8) 
-		    | (reg  & ~3) | 1);
-	}
-}
-
-static int
-ar71xx_pci_conf_setup(int bus, int slot, int func, int reg, int bytes, 
-    uint32_t cmd)
-{
-	uint32_t addr = ar71xx_pci_make_addr(bus, slot, func, (reg & ~3));
-	cmd |= (ar71xx_get_bytes_to_read(reg, bytes) << 4);
-	
-	ATH_WRITE_REG(AR71XX_PCI_CONF_ADDR, addr);
-	ATH_WRITE_REG(AR71XX_PCI_CONF_CMD, cmd);
-
-	dprintf("%s: tag (%x, %x, %x) %d/%d addr=%08x, cmd=%08x\n", __func__, 
-	    bus, slot, func, reg, bytes, addr, cmd);
-
-	return ar71xx_pci_check_bus_error();
-}
-
-static uint32_t
-ar71xx_pci_read_config(device_t dev, int bus, int slot, int func, int reg,
-    int bytes)
-{
-	uint32_t data;
-	uint32_t cmd, shift, mask;
-
-	/* register access is 32-bit aligned */
-	shift = (reg & 3) * 8;
-	if (shift)
-		mask = (1 << shift) - 1;
-	else
-		mask = 0xffffffff;
-
-	dprintf("%s: tag (%x, %x, %x) reg %d(%d)\n", __func__, bus, slot, 
-	    func, reg, bytes);
-
-	if ((bus == 0) && (slot == 0) && (func == 0)) {
-		cmd = PCI_LCONF_CMD_READ | (reg & ~3);
-		ATH_WRITE_REG(AR71XX_PCI_LCONF_CMD, cmd);
-		data = ATH_READ_REG(AR71XX_PCI_LCONF_READ_DATA);
-	} else {
-		 if (ar71xx_pci_conf_setup(bus, slot, func, reg, bytes, 
-		     PCI_CONF_CMD_READ) == 0)
-			 data = ATH_READ_REG(AR71XX_PCI_CONF_READ_DATA);
-		 else
-			 data = -1;
-	}
-
-	/* get request bytes from 32-bit word */
-	data = (data >> shift) & mask;
-
- 	dprintf("%s: read 0x%x\n", __func__, data);
-
-	return (data);
-}
-
-static void
-ar71xx_pci_write_config(device_t dev, int bus, int slot, int func, int reg,
-    uint32_t data, int bytes)
-{
-	uint32_t cmd;
-
-	dprintf("%s: tag (%x, %x, %x) reg %d(%d)\n", __func__, bus, slot, 
-	    func, reg, bytes);
-
-	data = data << (8*(reg % 4));
-
-	if ((bus == 0) && (slot == 0) && (func == 0)) {
-		cmd = PCI_LCONF_CMD_WRITE | (reg & ~3);
-		cmd |= ar71xx_get_bytes_to_read(reg, bytes) << 20;
-		ATH_WRITE_REG(AR71XX_PCI_LCONF_CMD, cmd);
-		ATH_WRITE_REG(AR71XX_PCI_LCONF_WRITE_DATA, data);
-	} else {
-		 if (ar71xx_pci_conf_setup(bus, slot, func, reg, bytes, 
-		     PCI_CONF_CMD_WRITE) == 0)
-			 ATH_WRITE_REG(AR71XX_PCI_CONF_WRITE_DATA, data);
-	}
-}
-
-static int
-at71xx_pci_intr(void *v)
-{
-	panic("Implement me: %s\n", __func__);
-	return FILTER_HANDLED;
-}
-
-static int
-ar71xx_pci_probe(device_t dev)
-{
-
-	return (0);
-}
-
-static int
-ar71xx_pci_attach(device_t dev)
-{
-	int busno = 0;
-	int rid = 0;
-	uint32_t reset;
-	struct ar71xx_pci_softc *sc = device_get_softc(dev);
-
-	sc->sc_mem_rman.rm_type = RMAN_ARRAY;
-	sc->sc_mem_rman.rm_descr = "ar71xx PCI memory window";
-	if (rman_init(&sc->sc_mem_rman) != 0 || 
-	    rman_manage_region(&sc->sc_mem_rman, AR71XX_PCI_MEM_BASE, 
-		AR71XX_PCI_MEM_BASE + AR71XX_PCI_MEM_SIZE - 1) != 0) {
-		panic("ar71xx_pci_attach: failed to set up I/O rman");
-	}
-
-	sc->sc_irq_rman.rm_type = RMAN_ARRAY;
-	sc->sc_irq_rman.rm_descr = "ar71xx PCI IRQs";
-	if (rman_init(&sc->sc_irq_rman) != 0 ||
-	    rman_manage_region(&sc->sc_irq_rman, AR71XX_PCI_IRQ_START, 
-	        AR71XX_PCI_IRQ_END) != 0)
-		panic("ar71xx_pci_attach: failed to set up IRQ rman");
-
-
-	ATH_WRITE_REG(AR71XX_PCI_INTR_STATUS, 0);
-	ATH_WRITE_REG(AR71XX_PCI_INTR_MASK, 0);
-
-	/* Hook up our interrupt handler. */
-	if ((sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
-	    RF_SHAREABLE | RF_ACTIVE)) == NULL) {
-		device_printf(dev, "unable to allocate IRQ resource\n");
-		return ENXIO;
-	}
-
-	if ((bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_MISC,
-			    at71xx_pci_intr, NULL, sc, &sc->sc_ih))) {
-		device_printf(dev, 
-		    "WARNING: unable to register interrupt handler\n");
-		return ENXIO;
-	}
-
-	/* reset PCI core and PCI bus */
-	reset = ATH_READ_REG(AR71XX_RST_RESET);
-	reset |= (RST_RESET_PCI_CORE | RST_RESET_PCI_BUS);
-	ATH_WRITE_REG(AR71XX_RST_RESET, reset);
-	DELAY(1000);
-
-	reset &= ~(RST_RESET_PCI_CORE | RST_RESET_PCI_BUS);
-	ATH_WRITE_REG(AR71XX_RST_RESET, reset);
-	DELAY(1000);
-
-	/* Init PCI windows */
-	ATH_WRITE_REG(AR71XX_PCI_WINDOW0, PCI_WINDOW0_ADDR);
-	ATH_WRITE_REG(AR71XX_PCI_WINDOW1, PCI_WINDOW1_ADDR);
-	ATH_WRITE_REG(AR71XX_PCI_WINDOW2, PCI_WINDOW2_ADDR);
-	ATH_WRITE_REG(AR71XX_PCI_WINDOW3, PCI_WINDOW3_ADDR);
-	ATH_WRITE_REG(AR71XX_PCI_WINDOW4, PCI_WINDOW4_ADDR);
-	ATH_WRITE_REG(AR71XX_PCI_WINDOW5, PCI_WINDOW5_ADDR);
-	ATH_WRITE_REG(AR71XX_PCI_WINDOW6, PCI_WINDOW6_ADDR);
-	ATH_WRITE_REG(AR71XX_PCI_WINDOW7, PCI_WINDOW7_CONF_ADDR);
-	DELAY(1000);
-
-	ar71xx_pci_check_bus_error();
-
-	/* Fixup internal PCI bridge */
-	ar71xx_pci_write_config(dev, 0, 0, 0, PCIR_COMMAND, 
-            PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN 
-	    | PCIM_CMD_SERRESPEN | PCIM_CMD_BACKTOBACK
-	    | PCIM_CMD_PERRESPEN | PCIM_CMD_MWRICEN, 2);
-
-	device_add_child(dev, "pci", busno);
-	return (bus_generic_attach(dev));
-}
-
-static int
-ar71xx_pci_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
-{
-	struct ar71xx_pci_softc *sc = device_get_softc(dev);
-
-	switch (which) {
-	case PCIB_IVAR_DOMAIN:
-		*result = 0;
-		return (0);
-	case PCIB_IVAR_BUS:
-		*result = sc->sc_busno;
-		return (0);
-	}
-
-	return (ENOENT);
-}
-
-static int
-ar71xx_pci_write_ivar(device_t dev, device_t child, int which, uintptr_t result)
-{
-	struct ar71xx_pci_softc * sc = device_get_softc(dev);
-
-	switch (which) {
-	case PCIB_IVAR_BUS:
-		sc->sc_busno = result;
-		return (0);
-	}
-
-	return (ENOENT);
-}
-
-static struct resource *
-ar71xx_pci_alloc_resource(device_t bus, device_t child, int type, int *rid,
-    u_long start, u_long end, u_long count, u_int flags)
-{
-
-	struct ar71xx_pci_softc *sc = device_get_softc(bus);	
-	struct resource *rv = NULL;
-	struct rman *rm;
-
-	switch (type) {
-	case SYS_RES_IRQ:
-		rm = &sc->sc_irq_rman;
-		break;
-	case SYS_RES_MEMORY:
-		rm = &sc->sc_mem_rman;
-		break;
-	default:
-		return (NULL);
-	}
-
-	rv = rman_reserve_resource(rm, start, end, count, flags, child);
-
-	if (rv == NULL)
-		return (NULL);
-
-	rman_set_rid(rv, *rid);
-
-	if (flags & RF_ACTIVE) {
-		if (bus_activate_resource(child, type, *rid, rv)) {
-			rman_release_resource(rv);
-			return (NULL);
-		}
-	} 
-
-	return (rv);
-}
-
-static int
-ar71xx_pci_teardown_intr(device_t dev, device_t child, struct resource *res,
-    void *cookie)
-{
-
-	return (intr_event_remove_handler(cookie));
-}
-
-static int
-ar71xx_pci_maxslots(device_t dev)
-{
-
-	return (PCI_SLOTMAX);
-}
-
-static int
-ar71xx_pci_route_interrupt(device_t pcib, device_t device, int pin)
-{
-
-	return (pin);
-}
-
-static device_method_t ar71xx_pci_methods[] = {
-	/* Device interface */
-	DEVMETHOD(device_probe,		ar71xx_pci_probe),
-	DEVMETHOD(device_attach,	ar71xx_pci_attach),
-	DEVMETHOD(device_shutdown,	bus_generic_shutdown),
-	DEVMETHOD(device_suspend,	bus_generic_suspend),
-	DEVMETHOD(device_resume,	bus_generic_resume),
-
-	/* Bus interface */
-	DEVMETHOD(bus_print_child,	bus_generic_print_child),
-	DEVMETHOD(bus_read_ivar,	ar71xx_pci_read_ivar),
-	DEVMETHOD(bus_write_ivar,	ar71xx_pci_write_ivar),
-	DEVMETHOD(bus_alloc_resource,	ar71xx_pci_alloc_resource),
-	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
-	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
-	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
-	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
-	DEVMETHOD(bus_teardown_intr,	ar71xx_pci_teardown_intr),
-
-	/* pcib interface */
-	DEVMETHOD(pcib_maxslots,	ar71xx_pci_maxslots),
-	DEVMETHOD(pcib_read_config,	ar71xx_pci_read_config),
-	DEVMETHOD(pcib_write_config,	ar71xx_pci_write_config),
-	DEVMETHOD(pcib_route_interrupt,	ar71xx_pci_route_interrupt),
-
-	{0, 0}
-};
-
-static driver_t ar71xx_pci_driver = {
-	"pcib",
-	ar71xx_pci_methods,
-	sizeof(struct ar71xx_pci_softc),
-};
-
-static devclass_t ar71xx_pci_devclass;
-
-DRIVER_MODULE(ar71xx_pci, nexus, ar71xx_pci_driver, ar71xx_pci_devclass, 0, 0);
diff --git a/sys/mips/atheros/ar71xxreg.h b/sys/mips/atheros/ar71xxreg.h
deleted file mode 100644
index 9367792..0000000
--- a/sys/mips/atheros/ar71xxreg.h
+++ /dev/null
@@ -1,317 +0,0 @@
-/*-
- * Copyright (c) 2009 Oleksandr Tymoshenko
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-#ifndef _AR71XX_REG_H_
-#define _AR71XX_REG_H_
-
-#define ATH_READ_REG(reg) \
-    *((volatile uint32_t *)MIPS_PHYS_TO_KSEG1((reg)))
-
-#define ATH_WRITE_REG(reg, val) \
-    *((volatile uint32_t *)MIPS_PHYS_TO_KSEG1((reg))) = (val)
-
-/* PCI region */
-#define AR71XX_PCI_MEM_BASE		0x10000000
-/* 
- * PCI mem windows is 0x08000000 bytes long but we exclude control 
- * region from the resource manager
- */
-#define AR71XX_PCI_MEM_SIZE		0x07000000
-#define AR71XX_PCI_IRQ_START		0
-#define AR71XX_PCI_IRQ_END		2
-
-/* PCI config registers */
-#define	AR71XX_PCI_LCONF_CMD		0x17010000
-#define			PCI_LCONF_CMD_READ	0x00000000
-#define			PCI_LCONF_CMD_WRITE	0x00010000
-#define	AR71XX_PCI_LCONF_WRITE_DATA	0x17010004
-#define	AR71XX_PCI_LCONF_READ_DATA	0x17010008
-#define	AR71XX_PCI_CONF_ADDR		0x1701000C
-#define	AR71XX_PCI_CONF_CMD		0x17010010
-#define			PCI_CONF_CMD_READ	0x0000000A
-#define			PCI_CONF_CMD_WRITE	0x0000000B
-#define	AR71XX_PCI_CONF_WRITE_DATA	0x17010014
-#define	AR71XX_PCI_CONF_READ_DATA	0x17010018
-#define	AR71XX_PCI_ERROR		0x1701001C
-#define	AR71XX_PCI_ERROR_ADDR		0x17010020
-#define	AR71XX_PCI_AHB_ERROR		0x17010024
-#define	AR71XX_PCI_AHB_ERROR_ADDR	0x17010028
-
-/* APB region */
-/* DDR registers */
-#define AR71XX_DDR_CONFIG		0x18000000
-#define AR71XX_DDR_CONFIG2		0x18000004
-#define AR71XX_DDR_MODE_REGISTER	0x18000008
-#define AR71XX_DDR_EXT_MODE_REGISTER	0x1800000C
-#define AR71XX_DDR_CONTROL		0x18000010
-#define AR71XX_DDR_REFRESH		0x18000014
-#define AR71XX_DDR_RD_DATA_THIS_CYCLE	0x18000018
-#define AR71XX_TAP_CONTROL0		0x1800001C
-#define AR71XX_TAP_CONTROL1		0x18000020
-#define AR71XX_TAP_CONTROL2		0x18000024
-#define AR71XX_TAP_CONTROL3		0x18000028
-#define AR71XX_PCI_WINDOW0		0x1800007C
-#define AR71XX_PCI_WINDOW1		0x18000080
-#define AR71XX_PCI_WINDOW2		0x18000084
-#define AR71XX_PCI_WINDOW3		0x18000088
-#define AR71XX_PCI_WINDOW4		0x1800008C
-#define AR71XX_PCI_WINDOW5		0x18000090
-#define AR71XX_PCI_WINDOW6		0x18000094
-#define AR71XX_PCI_WINDOW7		0x18000098
-#define AR71XX_WB_FLUSH_GE0		0x1800009C
-#define AR71XX_WB_FLUSH_GE1		0x180000A0
-#define AR71XX_WB_FLUSH_USB		0x180000A4
-#define AR71XX_WB_FLUSH_PCI		0x180000A8
-
-/*
- * Values for PCI_WINDOW_X registers 
- */
-#define PCI_WINDOW0_ADDR		0x10000000
-#define PCI_WINDOW1_ADDR		0x11000000
-#define PCI_WINDOW2_ADDR		0x12000000
-#define PCI_WINDOW3_ADDR		0x13000000
-#define PCI_WINDOW4_ADDR		0x14000000
-#define PCI_WINDOW5_ADDR		0x15000000
-#define PCI_WINDOW6_ADDR		0x16000000
-#define PCI_WINDOW7_ADDR		0x17000000
-/* This value enables acces to PCI config registers */
-#define PCI_WINDOW7_CONF_ADDR		0x07000000
-
-#define	AR71XX_UART_ADDR		0x18020000
-
-#define	AR71XX_USB_CTRL_FLADJ		0x18030000
-#define		USB_CTRL_FLADJ_HOST_SHIFT	12
-#define		USB_CTRL_FLADJ_A5_SHIFT		10
-#define		USB_CTRL_FLADJ_A4_SHIFT		8
-#define		USB_CTRL_FLADJ_A3_SHIFT		6
-#define		USB_CTRL_FLADJ_A2_SHIFT		4
-#define		USB_CTRL_FLADJ_A1_SHIFT		2
-#define		USB_CTRL_FLADJ_A0_SHIFT		0
-#define	AR71XX_USB_CTRL_CONFIG		0x18030004
-#define		USB_CTRL_CONFIG_OHCI_DES_SWAP	(1 << 19)
-#define		USB_CTRL_CONFIG_OHCI_BUF_SWAP	(1 << 18)
-#define		USB_CTRL_CONFIG_EHCI_DES_SWAP	(1 << 17)
-#define		USB_CTRL_CONFIG_EHCI_BUF_SWAP	(1 << 16)
-#define		USB_CTRL_CONFIG_DISABLE_XTL	(1 << 13)
-#define		USB_CTRL_CONFIG_OVERRIDE_XTL	(1 << 12)
-#define		USB_CTRL_CONFIG_CLK_SEL_SHIFT	4
-#define		USB_CTRL_CONFIG_CLK_SEL_MASK	3
-#define		USB_CTRL_CONFIG_CLK_SEL_12	0
-#define		USB_CTRL_CONFIG_CLK_SEL_24	1
-#define		USB_CTRL_CONFIG_CLK_SEL_48	2
-#define		USB_CTRL_CONFIG_OVER_CURRENT_AS_GPIO	(1 << 8)
-#define		USB_CTRL_CONFIG_SS_SIMULATION_MODE	(1 << 2)
-#define		USB_CTRL_CONFIG_RESUME_UTMI_PLS_DIS	(1 << 1)
-#define		USB_CTRL_CONFIG_UTMI_BACKWARD_ENB	(1 << 0)
-
-#define	AR71XX_PLL_CPU_CONFIG		0x18050000
-#define	AR71XX_PLL_SEC_CONFIG		0x18050004
-#define	AR71XX_PLL_CPU_CLK_CTRL		0x18050008
-#define	AR71XX_PLL_ETH_INT0_CLK		0x18050010
-#define	AR71XX_PLL_ETH_INT1_CLK		0x18050014
-#define		XPLL_ETH_INT_CLK_10		0x00991099
-#define		XPLL_ETH_INT_CLK_100		0x00441011
-#define		XPLL_ETH_INT_CLK_1000		0x13110000
-#define		XPLL_ETH_INT_CLK_1000_GMII	0x14110000
-#define		PLL_ETH_INT_CLK_10		0x00991099
-#define		PLL_ETH_INT_CLK_100		0x00001099
-#define		PLL_ETH_INT_CLK_1000		0x00110000
-#define	AR71XX_PLL_ETH_EXT_CLK		0x18050018
-#define	AR71XX_PLL_PCI_CLK		0x1805001C
-
-/* 
- * APB interrupt status and mask register and interrupt bit numbers for 
- */
-#define AR71XX_MISC_INTR_STATUS	0x18060010
-#define AR71XX_MISC_INTR_MASK	0x18060014
-#define		MISC_INTR_TIMER		0
-#define		MISC_INTR_ERROR		1
-#define		MISC_INTR_GPIO		2
-#define		MISC_INTR_UART		3
-#define		MISC_INTR_WATCHDOG	4
-#define		MISC_INTR_PERF		5
-#define		MISC_INTR_OHCI		6
-#define		MISC_INTR_DMA		7
-
-#define AR71XX_PCI_INTR_STATUS	0x18060018
-#define AR71XX_PCI_INTR_MASK	0x1806001C
-#define		PCI_INTR_CORE		(1 << 4)
-
-#define AR71XX_RST_RESET	0x18060024
-#define		RST_RESET_FULL_CHIP	(1 << 24) /* Same as pulling
-							     the reset pin */
-#define		RST_RESET_CPU_COLD	(1 << 20) /* Cold reset */
-#define		RST_RESET_GE1_MAC	(1 << 13)
-#define		RST_RESET_GE1_PHY	(1 << 12)
-#define		RST_RESET_GE0_MAC	(1 <<  9)
-#define		RST_RESET_GE0_PHY	(1 <<  8)
-#define		RST_RESET_USB_OHCI_DLL	(1 <<  6)
-#define		RST_RESET_USB_HOST	(1 <<  5)
-#define		RST_RESET_USB_PHY	(1 <<  4)
-#define		RST_RESET_PCI_BUS	(1 <<  1)
-#define		RST_RESET_PCI_CORE	(1 <<  0)
-
-/*
- * GigE adapters region
- */
-#define AR71XX_MAC0_BASE	0x19000000
-#define AR71XX_MAC1_BASE	0x1A000000
-
-#define		AR71XX_MAC_CFG1			0x00
-#define			MAC_CFG1_SOFT_RESET		(1 << 31)
-#define			MAC_CFG1_SIMUL_RESET		(1 << 30)
-#define			MAC_CFG1_MAC_RX_BLOCK_RESET	(1 << 19)
-#define			MAC_CFG1_MAC_TX_BLOCK_RESET	(1 << 18)
-#define			MAC_CFG1_RX_FUNC_RESET		(1 << 17)
-#define			MAC_CFG1_TX_FUNC_RESET		(1 << 16)
-#define			MAC_CFG1_LOOPBACK		(1 <<  8)
-#define			MAC_CFG1_RXFLOW_CTRL		(1 <<  5)
-#define			MAC_CFG1_TXFLOW_CTRL		(1 <<  4)
-#define			MAC_CFG1_SYNC_RX		(1 <<  3)
-#define			MAC_CFG1_RX_ENABLE		(1 <<  2)
-#define			MAC_CFG1_SYNC_TX		(1 <<  1)
-#define			MAC_CFG1_TX_ENABLE		(1 <<  0)
-#define		AR71XX_MAC_CFG2			0x04
-#define			MAC_CFG2_PREAMBLE_LEN_MASK	0xf
-#define			MAC_CFG2_PREAMBLE_LEN_SHIFT	12
-#define			MAC_CFG2_IFACE_MODE_1000	(2 << 8)
-#define			MAC_CFG2_IFACE_MODE_10_100	(1 << 8)
-#define			MAC_CFG2_IFACE_MODE_SHIFT	8
-#define			MAC_CFG2_IFACE_MODE_MASK	3
-#define			MAC_CFG2_HUGE_FRAME		(1 << 5)
-#define			MAC_CFG2_LENGTH_FIELD		(1 << 4)
-#define			MAC_CFG2_ENABLE_PADCRC		(1 << 2)
-#define			MAC_CFG2_ENABLE_CRC		(1 << 1)
-#define			MAC_CFG2_FULL_DUPLEX		(1 << 0)
-#define		AR71XX_MAC_IFG			0x08
-#define		AR71XX_MAC_HDUPLEX		0x0C
-#define		AR71XX_MAC_MAX_FRAME_LEN	0x10
-#define		AR71XX_MAC_MII_CFG		0x20
-#define			MAC_MII_CFG_RESET		(1 << 31)
-#define			MAC_MII_CFG_SCAN_AUTO_INC	(1 <<  5)
-#define			MAC_MII_CFG_PREAMBLE_SUP	(1 <<  4)
-#define			MAC_MII_CFG_CLOCK_SELECT_MASK	0x7
-#define			MAC_MII_CFG_CLOCK_DIV_4		0
-#define			MAC_MII_CFG_CLOCK_DIV_6		2
-#define			MAC_MII_CFG_CLOCK_DIV_8		3
-#define			MAC_MII_CFG_CLOCK_DIV_10	4
-#define			MAC_MII_CFG_CLOCK_DIV_14	5
-#define			MAC_MII_CFG_CLOCK_DIV_20	6
-#define			MAC_MII_CFG_CLOCK_DIV_28	7
-#define		AR71XX_MAC_MII_CMD		0x24
-#define			MAC_MII_CMD_SCAN_CYCLE		(1 << 1)
-#define			MAC_MII_CMD_READ		1
-#define			MAC_MII_CMD_WRITE		0
-#define		AR71XX_MAC_MII_ADDR		0x28
-#define			MAC_MII_PHY_ADDR_SHIFT		8
-#define			MAC_MII_PHY_ADDR_MASK		0xff
-#define			MAC_MII_REG_MASK		0x1f
-#define		AR71XX_MAC_MII_CONTROL		0x2C
-#define			MAC_MII_CONTROL_MASK		0xffff
-#define		AR71XX_MAC_MII_STATUS		0x30
-#define			MAC_MII_STATUS_MASK		0xffff
-#define		AR71XX_MAC_MII_INDICATOR	0x34
-#define			MAC_MII_INDICATOR_NOT_VALID	(1 << 2)
-#define			MAC_MII_INDICATOR_SCANNING	(1 << 1)
-#define			MAC_MII_INDICATOR_BUSY		(1 << 0)
-#define		AR71XX_MAC_IFCONTROL		0x38
-#define			MAC_IFCONTROL_SPEED	(1 << 16)
-#define		AR71XX_MAC_STA_ADDR1		0x40
-#define		AR71XX_MAC_STA_ADDR2		0x44
-#define		AR71XX_MAC_FIFO_CFG0		0x48
-#define			FIFO_CFG0_TX_FABRIC		(1 << 4)
-#define			FIFO_CFG0_TX_SYSTEM		(1 << 3)
-#define			FIFO_CFG0_RX_FABRIC		(1 << 2)
-#define			FIFO_CFG0_RX_SYSTEM		(1 << 1)
-#define			FIFO_CFG0_WATERMARK		(1 << 0)
-#define			FIFO_CFG0_ALL			((1 << 5) - 1)
-#define			FIFO_CFG0_ENABLE_SHIFT		8
-#define		AR71XX_MAC_FIFO_CFG1		0x4C
-#define		AR71XX_MAC_FIFO_CFG2		0x50
-#define		AR71XX_MAC_FIFO_TX_THRESHOLD	0x54
-#define		AR71XX_MAC_FIFO_RX_FILTMATCH	0x58
-#define			FIFO_RX_FILTMATCH_ALL		((1 << 18) - 1)
-#define		AR71XX_MAC_FIFO_RX_FILTMASK	0x5C
-#define			FIFO_RX_FILTMASK_BYTE_MODE	(1 << 19)
-#define			FIFO_RX_FILTMASK_NO_SHORT_FRAME	(1 << 18)
-#define			FIFO_RX_FILTMASK_ALL		((1 << 20) - 1)
-/* 
- * These flags applicable both to AR71XX_MAC_FIFO_RX_FILTMASK and
- * to AR71XX_MAC_FIFO_RX_FILTMATCH
- */
-#define			FIFO_RX_FILT_UNICAST		(1 << 17)
-#define			FIFO_RX_FILT_TRUNC_FRAME	(1 << 16)
-#define			FIFO_RX_FILT_VLAN_TAG		(1 << 15)
-#define			FIFO_RX_FILT_UNSUP_OPCODE	(1 << 14)
-#define			FIFO_RX_FILT_PAUSE_FRAME	(1 << 13)
-#define			FIFO_RX_FILT_CTRL_FRAME		(1 << 12)
-#define			FIFO_RX_FILT_LONG_EVENT		(1 << 11)
-#define			FIFO_RX_FILT_DRIBBLE_NIBBLE	(1 << 10)
-#define			FIFO_RX_FILT_BCAST		(1 <<  9)
-#define			FIFO_RX_FILT_MCAST		(1 <<  8)
-#define			FIFO_RX_FILT_OK			(1 <<  7)
-#define			FIFO_RX_FILT_OORANGE		(1 <<  6)
-#define			FIFO_RX_FILT_LEN_MSMTCH		(1 <<  5)
-#define			FIFO_RX_FILT_CRC_ERROR		(1 <<  4)
-#define			FIFO_RX_FILT_CODE_ERROR		(1 <<  3)
-#define			FIFO_RX_FILT_FALSE_CARRIER	(1 <<  2)
-#define			FIFO_RX_FILT_RX_DV_EVENT	(1 <<  1)
-#define			FIFO_RX_FILT_DROP_EVENT		(1 <<  0)
-#define		AR71XX_MAC_FIFO_RAM0		0x60
-#define		AR71XX_MAC_FIFO_RAM1		0x64
-#define		AR71XX_MAC_FIFO_RAM2		0x68
-#define		AR71XX_MAC_FIFO_RAM3		0x6C
-#define		AR71XX_MAC_FIFO_RAM4		0x70
-#define		AR71XX_MAC_FIFO_RAM5		0x74
-#define		AR71XX_MAC_FIFO_RAM6		0x78
-#define		AR71XX_DMA_TX_CONTROL		0x180
-#define			DMA_TX_CONTROL_EN		(1 << 0)
-#define		AR71XX_DMA_TX_DESC		0x184
-#define		AR71XX_DMA_TX_STATUS		0x188
-#define			DMA_TX_STATUS_PCOUNT_MASK	0xff
-#define			DMA_TX_STATUS_PCOUNT_SHIFT	16
-#define			DMA_TX_STATUS_BUS_ERROR		(1 << 3) 
-#define			DMA_TX_STATUS_UNDERRUN		(1 << 1) 
-#define			DMA_TX_STATUS_PKT_SENT		(1 << 0) 
-#define		AR71XX_DMA_RX_CONTROL		0x18C
-#define			DMA_RX_CONTROL_EN		(1 << 0)
-#define		AR71XX_DMA_RX_DESC		0x190
-#define		AR71XX_DMA_RX_STATUS		0x194
-#define			DMA_RX_STATUS_PCOUNT_MASK	0xff
-#define			DMA_RX_STATUS_PCOUNT_SHIFT	16
-#define			DMA_RX_STATUS_BUS_ERROR		(1 << 3) 
-#define			DMA_RX_STATUS_OVERFLOW		(1 << 1) 
-#define			DMA_RX_STATUS_PKT_RECVD		(1 << 0) 
-#define		AR71XX_DMA_INTR				0x198
-#define		AR71XX_DMA_INTR_STATUS			0x19C
-#define			DMA_INTR_ALL			((1 << 8) - 1)
-#define			DMA_INTR_RX_BUS_ERROR		(1 << 7)
-#define			DMA_INTR_RX_OVERFLOW		(1 << 6)
-#define			DMA_INTR_RX_PKT_RCVD		(1 << 4)
-#define			DMA_INTR_TX_BUS_ERROR		(1 << 3)
-#define			DMA_INTR_TX_UNDERRUN		(1 << 1)
-#define			DMA_INTR_TX_PKT_SENT		(1 << 0)
-
-#endif /* _AR71XX_REG_H_ */
diff --git a/sys/mips/atheros/files.ar71xx b/sys/mips/atheros/files.ar71xx
deleted file mode 100644
index 78e1d9c..0000000
--- a/sys/mips/atheros/files.ar71xx
+++ /dev/null
@@ -1,9 +0,0 @@
-# $FreeBSD$
-
-mips/atheros/apb.c		standard
-mips/atheros/ar71xx_machdep.c	standard
-mips/atheros/ar71xx_ohci.c	optional ohci
-mips/atheros/ar71xx_pci.c	optional pci
-mips/atheros/if_arge.c		optional arge
-mips/atheros/uart_bus_ar71xx.c	optional uart
-mips/atheros/uart_cpu_ar71xx.c	optional uart
diff --git a/sys/mips/atheros/if_arge.c b/sys/mips/atheros/if_arge.c
deleted file mode 100644
index 7dbfe70..0000000
--- a/sys/mips/atheros/if_arge.c
+++ /dev/null
@@ -1,1657 +0,0 @@
-/*-
- * Copyright (c) 2009, Oleksandr Tymoshenko
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice unmodified, this list of conditions, and the following
- *    disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-/*
- * AR71XX gigabit ethernet driver
- */
-#include <sys/param.h>
-#include <sys/endian.h>
-#include <sys/systm.h>
-#include <sys/sockio.h>
-#include <sys/mbuf.h>
-#include <sys/malloc.h>
-#include <sys/kernel.h>
-#include <sys/module.h>
-#include <sys/socket.h>
-#include <sys/taskqueue.h>
-
-#include <net/if.h>
-#include <net/if_arp.h>
-#include <net/ethernet.h>
-#include <net/if_dl.h>
-#include <net/if_media.h>
-#include <net/if_types.h>
-
-#include <net/bpf.h>
-
-#include <machine/bus.h>
-#include <machine/cache.h>
-#include <machine/resource.h>
-#include <vm/vm_param.h>
-#include <vm/vm.h>
-#include <vm/pmap.h>
-#include <machine/pmap.h>
-#include <sys/bus.h>
-#include <sys/rman.h>
-
-#include <dev/mii/mii.h>
-#include <dev/mii/miivar.h>
-
-#include <dev/pci/pcireg.h>
-#include <dev/pci/pcivar.h>
-
-MODULE_DEPEND(arge, ether, 1, 1, 1);
-MODULE_DEPEND(arge, miibus, 1, 1, 1);
-
-#include "miibus_if.h"
-
-#include <mips/atheros/ar71xxreg.h>
-#include <mips/atheros/if_argevar.h>
-
-#undef ARGE_DEBUG
-#ifdef ARGE_DEBUG
-#define dprintf printf
-#else
-#define dprintf(x, arg...)
-#endif
-
-static int arge_attach(device_t);
-static int arge_detach(device_t);
-static int arge_fix_chain(struct mbuf **mp);
-static void arge_flush_ddr(struct arge_softc *);
-static int arge_ifmedia_upd(struct ifnet *);
-static void arge_ifmedia_sts(struct ifnet *, struct ifmediareq *);
-static int arge_ioctl(struct ifnet *, u_long, caddr_t);
-static void arge_init(void *);
-static void arge_init_locked(struct arge_softc *);
-static void arge_link_task(void *, int);
-static int arge_miibus_readreg(device_t, int, int);
-static void arge_miibus_statchg(device_t);
-static int arge_miibus_writereg(device_t, int, int, int);
-static int arge_probe(device_t);
-static void arge_reset_dma(struct arge_softc *);
-static int arge_resume(device_t);
-static int arge_rx_ring_init(struct arge_softc *);
-static int arge_tx_ring_init(struct arge_softc *);
-static void arge_shutdown(device_t);
-static void arge_start(struct ifnet *);
-static void arge_start_locked(struct ifnet *);
-static void arge_stop(struct arge_softc *);
-static int arge_suspend(device_t);
-
-static void arge_rx_locked(struct arge_softc *);
-static void arge_tx_locked(struct arge_softc *);
-static void arge_intr(void *);
-static int arge_intr_filter(void *);
-static void arge_tx_intr(struct arge_softc *, uint32_t);
-static void arge_rx_intr(struct arge_softc *, uint32_t);
-static void arge_tick(void *);
-
-static void arge_dmamap_cb(void *, bus_dma_segment_t *, int, int);
-static int arge_dma_alloc(struct arge_softc *);
-static void arge_dma_free(struct arge_softc *);
-static int arge_newbuf(struct arge_softc *, int);
-static __inline void arge_fixup_rx(struct mbuf *);
-
-static device_method_t arge_methods[] = {
-	/* Device interface */
-	DEVMETHOD(device_probe,		arge_probe),
-	DEVMETHOD(device_attach,	arge_attach),
-	DEVMETHOD(device_detach,	arge_detach),
-	DEVMETHOD(device_suspend,	arge_suspend),
-	DEVMETHOD(device_resume,	arge_resume),
-	DEVMETHOD(device_shutdown,	arge_shutdown),
-
-	/* bus interface */
-	DEVMETHOD(bus_print_child,	bus_generic_print_child),
-	DEVMETHOD(bus_driver_added,	bus_generic_driver_added),
-
-	/* MII interface */
-	DEVMETHOD(miibus_readreg,	arge_miibus_readreg),
-	DEVMETHOD(miibus_writereg,	arge_miibus_writereg),
-	DEVMETHOD(miibus_statchg,	arge_miibus_statchg),
-
-	{ 0, 0 }
-};
-
-static driver_t arge_driver = {
-	"arge",
-	arge_methods,
-	sizeof(struct arge_softc)
-};
-
-static devclass_t arge_devclass;
-
-DRIVER_MODULE(arge, nexus, arge_driver, arge_devclass, 0, 0);
-DRIVER_MODULE(miibus, arge, miibus_driver, miibus_devclass, 0, 0);
-
-/*
- * Flushes all 
- */
-static void
-arge_flush_ddr(struct arge_softc *sc)
-{
-
-	ATH_WRITE_REG(sc->arge_ddr_flush_reg, 1);
-	while (ATH_READ_REG(sc->arge_ddr_flush_reg) & 1)
-		;
-
-	ATH_WRITE_REG(sc->arge_ddr_flush_reg, 1);
-	while (ATH_READ_REG(sc->arge_ddr_flush_reg) & 1)
-		;
-}
-
-static int 
-arge_probe(device_t dev)
-{
-
-	device_set_desc(dev, "Atheros AR71xx built-in ethernet interface");
-	return (0);
-}
-
-static int
-arge_attach(device_t dev)
-{
-	uint8_t			eaddr[ETHER_ADDR_LEN];
-	struct ifnet		*ifp;
-	struct arge_softc	*sc;
-	int			error = 0, rid, phynum;
-	uint32_t		reg;
-
-	sc = device_get_softc(dev);
-	sc->arge_dev = dev;
-	sc->arge_mac_unit = device_get_unit(dev);
-
-	KASSERT(((sc->arge_mac_unit == 0) || (sc->arge_mac_unit == 1)), 
-	    ("if_arge: Only MAC0 and MAC1 supported"));
-	if (sc->arge_mac_unit == 0) {
-		sc->arge_ddr_flush_reg = AR71XX_WB_FLUSH_GE0;
-		sc->arge_pll_reg = AR71XX_PLL_ETH_INT0_CLK;
-	} else {
-		sc->arge_ddr_flush_reg = AR71XX_WB_FLUSH_GE1;
-		sc->arge_pll_reg = AR71XX_PLL_ETH_INT1_CLK;
-	}
-
-	/*
-	 *  Get which PHY of 5 available we should use for this unit
-	 */
-	if (resource_int_value(device_get_name(dev), device_get_unit(dev), 
-	    "phy", &phynum) != 0) {
-		/*
-		 * Use port 4 (WAN) for GE0. For any other port use 
-		 * its PHY the same as its unit number 
-		 */
-		if (sc->arge_mac_unit == 0)
-			phynum = 4;
-		else
-			phynum = sc->arge_mac_unit;
-
-		device_printf(dev, "No PHY specified, using %d\n", phynum);
-	}
-
-	sc->arge_phy_num = phynum;
-
-
-	mtx_init(&sc->arge_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK,
-	    MTX_DEF);
-	callout_init_mtx(&sc->arge_stat_callout, &sc->arge_mtx, 0);
-	TASK_INIT(&sc->arge_link_task, 0, arge_link_task, sc);
-
-	/* Map control/status registers. */
-	sc->arge_rid = 0;
-	sc->arge_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 
-	    &sc->arge_rid, RF_ACTIVE);
-
-	if (sc->arge_res == NULL) {
-		device_printf(dev, "couldn't map memory\n");
-		error = ENXIO;
-		goto fail;
-	}
-
-	/* Allocate interrupts */
-	rid = 0;
-	sc->arge_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 
-	    RF_SHAREABLE | RF_ACTIVE);
-
-	if (sc->arge_irq == NULL) {
-		device_printf(dev, "couldn't map interrupt\n");
-		error = ENXIO;
-		goto fail;
-	}
-
-	/* Allocate ifnet structure. */
-	ifp = sc->arge_ifp = if_alloc(IFT_ETHER);
-
-	if (ifp == NULL) {
-		device_printf(dev, "couldn't allocate ifnet structure\n");
-		error = ENOSPC;
-		goto fail;
-	}
-
-	ifp->if_softc = sc;
-	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
-	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
-	ifp->if_ioctl = arge_ioctl;
-	ifp->if_start = arge_start;
-	ifp->if_init = arge_init;
-
-	/* XXX: add real size */
-	IFQ_SET_MAXLEN(&ifp->if_snd, 9);
-	ifp->if_snd.ifq_maxlen = 9;
-	IFQ_SET_READY(&ifp->if_snd);
-
-	ifp->if_capenable = ifp->if_capabilities;
-
-	eaddr[0] = 0x00;
-	eaddr[1] = 0x15;
-	eaddr[2] = 0x6d;
-	eaddr[3] = 0xc1;
-	eaddr[4] = 0x28;
-	eaddr[5] = 0x2e;
-
-	if (arge_dma_alloc(sc) != 0) {
-		error = ENXIO;
-		goto fail;
-	}
-
-	ARGE_WRITE(sc, AR71XX_MAC_CFG1, 
-		MAC_CFG1_SYNC_RX | MAC_CFG1_RX_ENABLE |
-		MAC_CFG1_SYNC_TX | MAC_CFG1_TX_ENABLE);
-
-	reg = ARGE_READ(sc, AR71XX_MAC_CFG2);
-	reg |= MAC_CFG2_ENABLE_PADCRC | MAC_CFG2_LENGTH_FIELD ;
-	ARGE_WRITE(sc, AR71XX_MAC_CFG2, reg);
-
-	ARGE_WRITE(sc, AR71XX_MAC_MAX_FRAME_LEN, 1536);
-
-	/* Reset MII bus */
-	ARGE_WRITE(sc, AR71XX_MAC_MII_CFG, MAC_MII_CFG_RESET);
-	DELAY(100);
-	ARGE_WRITE(sc, AR71XX_MAC_MII_CFG, MAC_MII_CFG_CLOCK_DIV_28);
-	DELAY(100);
-
-	/* 
-	 * Set all Ethernet address registers to the same initial values
-	 * set all four addresses to 66-88-aa-cc-dd-ee 
-	 */
-	ARGE_WRITE(sc, AR71XX_MAC_STA_ADDR1, 0x6dc1282e);
-	ARGE_WRITE(sc, AR71XX_MAC_STA_ADDR2, 0x00000015);
-
-	ARGE_WRITE(sc, AR71XX_MAC_FIFO_CFG0, 
-	    FIFO_CFG0_ALL << FIFO_CFG0_ENABLE_SHIFT);
-	ARGE_WRITE(sc, AR71XX_MAC_FIFO_CFG1, 0x0fff0000);
-	ARGE_WRITE(sc, AR71XX_MAC_FIFO_CFG2, 0x00001fff);
-
-	reg = FIFO_RX_FILTMATCH_ALL;
-	ARGE_WRITE(sc, AR71XX_MAC_FIFO_RX_FILTMATCH, reg);
-
-	reg = FIFO_RX_FILTMASK_ALL;
-	reg &= ~FIFO_RX_FILTMASK_BYTE_MODE;
-	ARGE_WRITE(sc, AR71XX_MAC_FIFO_RX_FILTMASK, reg);
-
-	/* Do MII setup. */
-	if (mii_phy_probe(dev, &sc->arge_miibus,
-	    arge_ifmedia_upd, arge_ifmedia_sts)) {
-		device_printf(dev, "MII without any phy!\n");
-		error = ENXIO;
-		goto fail;
-	}
-
-	/* Call MI attach routine. */
-	ether_ifattach(ifp, eaddr);
-
-	/* Hook interrupt last to avoid having to lock softc */
-	error = bus_setup_intr(dev, sc->arge_irq, INTR_TYPE_NET | INTR_MPSAFE,
-	    arge_intr_filter, arge_intr, sc, &sc->arge_intrhand);
-
-	if (error) {
-		device_printf(dev, "couldn't set up irq\n");
-		ether_ifdetach(ifp);
-		goto fail;
-	}
-
-fail:
-	if (error) 
-		arge_detach(dev);
-
-	return (error);
-}
-
-static int
-arge_detach(device_t dev)
-{
-	struct arge_softc		*sc = device_get_softc(dev);
-	struct ifnet		*ifp = sc->arge_ifp;
-
-	KASSERT(mtx_initialized(&sc->arge_mtx), ("arge mutex not initialized"));
-
-	/* These should only be active if attach succeeded */
-	if (device_is_attached(dev)) {
-		ARGE_LOCK(sc);
-		sc->arge_detach = 1;
-		arge_stop(sc);
-		ARGE_UNLOCK(sc);
-		taskqueue_drain(taskqueue_swi, &sc->arge_link_task);
-		ether_ifdetach(ifp);
-	}
-
-	if (sc->arge_miibus)
-		device_delete_child(dev, sc->arge_miibus);
-	bus_generic_detach(dev);
-
-	if (sc->arge_intrhand)
-		bus_teardown_intr(dev, sc->arge_irq, sc->arge_intrhand);
-
-	if (sc->arge_res)
-		bus_release_resource(dev, SYS_RES_MEMORY, sc->arge_rid, 
-		    sc->arge_res);
-
-	if (ifp)
-		if_free(ifp);
-
-	arge_dma_free(sc);
-
-	mtx_destroy(&sc->arge_mtx);
-
-	return (0);
-
-}
-
-static int
-arge_suspend(device_t dev)
-{
-
-	panic("%s", __func__);
-	return 0;
-}
-
-static int
-arge_resume(device_t dev)
-{
-
-	panic("%s", __func__);
-	return 0;
-}
-
-static void
-arge_shutdown(device_t dev)
-{
-	struct arge_softc	*sc;
-
-	sc = device_get_softc(dev);
-
-	ARGE_LOCK(sc);
-	arge_stop(sc);
-	ARGE_UNLOCK(sc);
-}
-
-static int
-arge_miibus_readreg(device_t dev, int phy, int reg)
-{
-	struct arge_softc * sc = device_get_softc(dev);
-	int i, result;
-	uint32_t addr = 0x1000 | (phy << MAC_MII_PHY_ADDR_SHIFT) 
-	    | (reg & MAC_MII_REG_MASK);
-
-	if (phy != sc->arge_phy_num)
-		return (0);
-
-	ARGE_WRITE(sc, AR71XX_MAC_MII_CMD, MAC_MII_CMD_WRITE);
-	ARGE_WRITE(sc, AR71XX_MAC_MII_ADDR, addr);
-	ARGE_WRITE(sc, AR71XX_MAC_MII_CMD, MAC_MII_CMD_READ);
-
-	i = ARGE_MII_TIMEOUT;
-	while ((ARGE_READ(sc, AR71XX_MAC_MII_INDICATOR) & 
-	    MAC_MII_INDICATOR_BUSY) && (i--))
-		DELAY(5);
-
-	if (i < 0) {
-		dprintf("%s timedout\n", __func__);
-		/* XXX: return ERRNO istead? */
-		return (-1);
-	}
-
-	result = ARGE_READ(sc, AR71XX_MAC_MII_STATUS) & MAC_MII_STATUS_MASK;
-	ARGE_WRITE(sc, AR71XX_MAC_MII_CMD, MAC_MII_CMD_WRITE);
-	dprintf("%s: phy=%d, reg=%02x, value[%08x]=%04x\n", __func__, 
-		 phy, reg, addr, result);
-
-	return (result);
-}
-
-static int
-arge_miibus_writereg(device_t dev, int phy, int reg, int data)
-{
-	struct arge_softc * sc = device_get_softc(dev);
-	int i;
-	uint32_t addr = 0x1000 
-	    | (phy << MAC_MII_PHY_ADDR_SHIFT) | (reg & MAC_MII_REG_MASK);
-
-	dprintf("%s: phy=%d, reg=%02x, value=%04x\n", __func__, 
-	    phy, reg, data);
-
-	ARGE_WRITE(sc, AR71XX_MAC_MII_ADDR, addr);
-	ARGE_WRITE(sc, AR71XX_MAC_MII_CONTROL, data);
-
-	i = ARGE_MII_TIMEOUT;
-	while ((ARGE_READ(sc, AR71XX_MAC_MII_INDICATOR) & 
-	    MAC_MII_INDICATOR_BUSY) && (i--))
-		DELAY(5);
-
-	if (i < 0) {
-		dprintf("%s timedout\n", __func__);
-		/* XXX: return ERRNO istead? */
-		return (-1);
-	}
-
-	return (0);
-}
-
-static void
-arge_miibus_statchg(device_t dev)
-{
-	struct arge_softc		*sc;
-
-	sc = device_get_softc(dev);
-	taskqueue_enqueue(taskqueue_swi, &sc->arge_link_task);
-}
-
-static void
-arge_link_task(void *arg, int pending)
-{
-	struct arge_softc	*sc;
-	struct mii_data		*mii;
-	struct ifnet		*ifp;
-	uint32_t		media;
-	uint32_t		cfg, ifcontrol, rx_filtmask, pll, sec_cfg;
-
-	sc = (struct arge_softc *)arg;
-
-	ARGE_LOCK(sc);
-	mii = device_get_softc(sc->arge_miibus);
-	ifp = sc->arge_ifp;
-	if (mii == NULL || ifp == NULL ||
-	    (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
-		ARGE_UNLOCK(sc);
-		return;
-	}
-
-	if (mii->mii_media_status & IFM_ACTIVE) {
-
-		media = IFM_SUBTYPE(mii->mii_media_active);
-
-		if (media != IFM_NONE) {
-			sc->arge_link_status = 1;
-
-			cfg = ARGE_READ(sc, AR71XX_MAC_CFG2);
-			ifcontrol = ARGE_READ(sc, AR71XX_MAC_IFCONTROL);
-			rx_filtmask = 
-			    ARGE_READ(sc, AR71XX_MAC_FIFO_RX_FILTMASK);
-			
-			cfg &= ~(MAC_CFG2_IFACE_MODE_1000 
-			    | MAC_CFG2_IFACE_MODE_10_100 
-			    | MAC_CFG2_FULL_DUPLEX);
-			ifcontrol &= ~MAC_IFCONTROL_SPEED;
-			rx_filtmask &= ~FIFO_RX_FILTMASK_BYTE_MODE;
-
-			switch(media) {
-			case IFM_10_T:
-				cfg |= MAC_CFG2_IFACE_MODE_10_100;
-				pll = PLL_ETH_INT_CLK_10;
-				break;
-			case IFM_100_TX:
-				cfg |= MAC_CFG2_IFACE_MODE_10_100;
-				ifcontrol |= MAC_IFCONTROL_SPEED;
-				pll = PLL_ETH_INT_CLK_100;
-				break;
-			case IFM_1000_T:
-			case IFM_1000_SX:
-				cfg |= MAC_CFG2_IFACE_MODE_1000;
-				rx_filtmask |= FIFO_RX_FILTMASK_BYTE_MODE;
-				pll = PLL_ETH_INT_CLK_1000;
-				break;
-			default:
-				pll = PLL_ETH_INT_CLK_100;
-				device_printf(sc->arge_dev, 
-				    "Unknown media %d\n", media);
-			}
-
-			ARGE_WRITE(sc, AR71XX_MAC_FIFO_TX_THRESHOLD,
-			    0x008001ff);
-
-			ARGE_WRITE(sc, AR71XX_MAC_CFG2, cfg);
-			ARGE_WRITE(sc, AR71XX_MAC_IFCONTROL, ifcontrol);
-			ARGE_WRITE(sc, AR71XX_MAC_FIFO_RX_FILTMASK, 
-			    rx_filtmask);
-
-			/* set PLL registers */
-			sec_cfg = ATH_READ_REG(AR71XX_PLL_CPU_CONFIG);
-			sec_cfg &= ~(3 << 17);
-			sec_cfg |= (2 << 17);
-
-			ATH_WRITE_REG(AR71XX_PLL_CPU_CONFIG, sec_cfg);
-			DELAY(100);
-
-			ATH_WRITE_REG(sc->arge_pll_reg, pll);
-
-			sec_cfg |= (3 << 17);
-			ATH_WRITE_REG(AR71XX_PLL_CPU_CONFIG, sec_cfg);
-			DELAY(100);
-
-			sec_cfg &= ~(3 << 17);
-			ATH_WRITE_REG(AR71XX_PLL_CPU_CONFIG, sec_cfg);
-			DELAY(100);
-		}
-	} else
-		sc->arge_link_status = 0;
-
-	ARGE_UNLOCK(sc);
-}
-
-static void
-arge_reset_dma(struct arge_softc *sc)
-{
-	unsigned int i;
-
-	ARGE_WRITE(sc, AR71XX_DMA_RX_CONTROL, 0);
-	ARGE_WRITE(sc, AR71XX_DMA_TX_CONTROL, 0);
-
-	ARGE_WRITE(sc, AR71XX_DMA_RX_DESC, 0);
-	ARGE_WRITE(sc, AR71XX_DMA_TX_DESC, 0);
-
-	/* Clear all possible RX interrupts */
-	for (i = 0; i < ARGE_RX_RING_COUNT; i++)
-		ARGE_WRITE(sc, AR71XX_DMA_RX_STATUS, DMA_RX_STATUS_PKT_RECVD);
-
-	/* 
-	 * Clear all possible TX interrupts
-	 */
-	for (i = 0; i < ARGE_TX_RING_COUNT; i++)
-		ARGE_WRITE(sc, AR71XX_DMA_TX_STATUS, DMA_TX_STATUS_PKT_SENT);
-
-	/* 
-	 * Now Rx/Tx errors
-	 */
-	ARGE_WRITE(sc, AR71XX_DMA_RX_STATUS, 
-	    DMA_RX_STATUS_BUS_ERROR | DMA_RX_STATUS_OVERFLOW);
-	ARGE_WRITE(sc, AR71XX_DMA_TX_STATUS, 
-	    DMA_TX_STATUS_BUS_ERROR | DMA_TX_STATUS_UNDERRUN);
-}
-
-
-
-static void
-arge_init(void *xsc)
-{
-	struct arge_softc	 *sc = xsc;
-
-	ARGE_LOCK(sc);
-	arge_init_locked(sc);
-	ARGE_UNLOCK(sc);
-}
-
-static void
-arge_init_locked(struct arge_softc *sc)
-{
-	struct ifnet		*ifp = sc->arge_ifp;
-	struct mii_data		*mii;
-
-	ARGE_LOCK_ASSERT(sc);
-
-	mii = device_get_softc(sc->arge_miibus);
-
-	arge_stop(sc);
-
-	/* Init circular RX list. */
-	if (arge_rx_ring_init(sc) != 0) {
-		device_printf(sc->arge_dev,
-		    "initialization failed: no memory for rx buffers\n");
-		arge_stop(sc);
-		return;
-	}
-
-	/* Init tx descriptors. */
-	arge_tx_ring_init(sc);
-
-	arge_reset_dma(sc);
-
-	sc->arge_link_status = 0;
-	mii_mediachg(mii);
-
-	ifp->if_drv_flags |= IFF_DRV_RUNNING;
-	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
-
-	callout_reset(&sc->arge_stat_callout, hz, arge_tick, sc);
-	ARGE_WRITE(sc, AR71XX_DMA_TX_DESC, ARGE_TX_RING_ADDR(sc, 0));
-	ARGE_WRITE(sc, AR71XX_DMA_RX_DESC, ARGE_RX_RING_ADDR(sc, 0));
-
-	/* Start listening */
-	ARGE_WRITE(sc, AR71XX_DMA_RX_CONTROL, DMA_RX_CONTROL_EN);
-
-	/* Enable interrupts */
-	ARGE_WRITE(sc, AR71XX_DMA_INTR, DMA_INTR_ALL);
-}
-
-/*
- * Encapsulate an mbuf chain in a descriptor by coupling the mbuf data
- * pointers to the fragment pointers.
- */
-static int
-arge_encap(struct arge_softc *sc, struct mbuf **m_head)
-{
-	struct arge_txdesc	*txd;
-	struct arge_desc	*desc, *prev_desc;
-	bus_dma_segment_t	txsegs[ARGE_MAXFRAGS];
-	int			error, i, nsegs, prod, si, prev_prod;
-
-	ARGE_LOCK_ASSERT(sc);
-
-	prod = sc->arge_cdata.arge_tx_prod;
-	txd = &sc->arge_cdata.arge_txdesc[prod];
-	error = bus_dmamap_load_mbuf_sg(sc->arge_cdata.arge_tx_tag, 
-	    txd->tx_dmamap, *m_head, txsegs, &nsegs, BUS_DMA_NOWAIT);
-
-	if (error == EFBIG) {
-		panic("EFBIG");
-	} else if (error != 0)
-		return (error);
-
-	if (nsegs == 0) {
-		m_freem(*m_head);
-		*m_head = NULL;
-		return (EIO);
-	}
-
-	/* Check number of available descriptors. */
-	if (sc->arge_cdata.arge_tx_cnt + nsegs >= (ARGE_TX_RING_COUNT - 1)) {
-		bus_dmamap_unload(sc->arge_cdata.arge_tx_tag, txd->tx_dmamap);
-		return (ENOBUFS);
-	}
-
-	txd->tx_m = *m_head;
-	bus_dmamap_sync(sc->arge_cdata.arge_tx_tag, txd->tx_dmamap,
-	    BUS_DMASYNC_PREWRITE);
-
-	si = prod;
-
-	/* 
-	 * Make a list of descriptors for this packet. DMA controller will
-	 * walk through it while arge_link is not zero.
-	 */
-	prev_prod = prod;
-	desc = prev_desc = NULL;
-	for (i = 0; i < nsegs; i++) {
-		desc = &sc->arge_rdata.arge_tx_ring[prod];
-		desc->packet_ctrl = ARGE_DMASIZE(txsegs[i].ds_len);
-
-		desc->packet_addr = txsegs[i].ds_addr;
-		/* link with previous descriptor */
-		if (prev_desc)
-			prev_desc->packet_ctrl |= ARGE_DESC_MORE;
-
-		sc->arge_cdata.arge_tx_cnt++;
-		prev_desc = desc;
-		ARGE_INC(prod, ARGE_TX_RING_COUNT);
-	}
-
-	/* Update producer index. */
-	sc->arge_cdata.arge_tx_prod = prod;
-
-	/* Sync descriptors. */
-	bus_dmamap_sync(sc->arge_cdata.arge_tx_ring_tag,
-	    sc->arge_cdata.arge_tx_ring_map,
-	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-
-	/* Start transmitting */
-	ARGE_WRITE(sc, AR71XX_DMA_TX_CONTROL, DMA_TX_CONTROL_EN);
-	return (0);
-}
-
-static void
-arge_start(struct ifnet *ifp)
-{
-	struct arge_softc	 *sc;
-
-	sc = ifp->if_softc;
-
-	ARGE_LOCK(sc);
-	arge_start_locked(ifp);
-	ARGE_UNLOCK(sc);
-}
-
-static void
-arge_start_locked(struct ifnet *ifp)
-{
-	struct arge_softc	*sc;
-	struct mbuf		*m_head;
-	int			enq;
-
-	sc = ifp->if_softc;
-
-	ARGE_LOCK_ASSERT(sc);
-
-	if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
-	    IFF_DRV_RUNNING || sc->arge_link_status == 0 )
-		return;
-
-	arge_flush_ddr(sc);
-
-	for (enq = 0; !IFQ_DRV_IS_EMPTY(&ifp->if_snd) &&
-	    sc->arge_cdata.arge_tx_cnt < ARGE_TX_RING_COUNT - 2; ) {
-		IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
-		if (m_head == NULL)
-			break;
-
-		/*
-		 * Fix mbuf chain, all fragments should be 4 bytes aligned and
-		 * even 4 bytes
-		 */
-		arge_fix_chain(&m_head);
-
-		if (m_head == NULL) {
-			dprintf("failed to adjust mbuf chain\n");
-		}
-
-		/*
-		 * Pack the data into the transmit ring.
-		 */
-		if (arge_encap(sc, &m_head)) {
-			if (m_head == NULL)
-				break;
-			IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
-			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
-			break;
-		}
-
-		enq++;
-		/*
-		 * If there's a BPF listener, bounce a copy of this frame
-		 * to him.
-		 */
-		ETHER_BPF_MTAP(ifp, m_head);
-	}
-}
-
-static void
-arge_stop(struct arge_softc *sc)
-{
-	struct ifnet	    *ifp;
-
-	ARGE_LOCK_ASSERT(sc);
-
-	ifp = sc->arge_ifp;
-	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
-	callout_stop(&sc->arge_stat_callout);
-
-	/* mask out interrupts */
-	ARGE_WRITE(sc, AR71XX_DMA_INTR, 0);
-
-	arge_reset_dma(sc);
-}
-
-
-static int
-arge_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
-{
-	struct arge_softc		*sc = ifp->if_softc;
-	struct ifreq		*ifr = (struct ifreq *) data;
-	struct mii_data		*mii;
-	int			error;
-
-	switch (command) {
-	case SIOCSIFFLAGS:
-		printf("Implement me: SIOCSIFFLAGS\n");
-		error = 0;
-		break;
-	case SIOCADDMULTI:
-	case SIOCDELMULTI:
-		printf("Implement me: SIOCDELMULTI\n");
-		error = 0;
-		break;
-	case SIOCGIFMEDIA:
-	case SIOCSIFMEDIA:
-		printf("Implement me: SIOCSIFMEDIA\n");
-		mii = device_get_softc(sc->arge_miibus);
-		error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, command);
-		break;
-	case SIOCSIFCAP:
-		error = 0;
-		ifp->if_hwassist = 0;
-		printf("Implement me: SIOCSIFCAP\n");
-		break;
-	default:
-		error = ether_ioctl(ifp, command, data);
-		break;
-	}
-
-	return (error);
-}
-
-/*
- * Set media options.
- */
-static int
-arge_ifmedia_upd(struct ifnet *ifp)
-{
-	struct arge_softc		*sc;
-	struct mii_data		*mii;
-	struct mii_softc	*miisc;
-	int			error;
-
-	sc = ifp->if_softc;
-	ARGE_LOCK(sc);
-	mii = device_get_softc(sc->arge_miibus);
-	if (mii->mii_instance) {
-		LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
-			mii_phy_reset(miisc);
-	}
-	error = mii_mediachg(mii);
-	ARGE_UNLOCK(sc);
-
-	return (error);
-}
-
-/*
- * Report current media status.
- */
-static void
-arge_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
-{
-	struct arge_softc		*sc = ifp->if_softc;
-	struct mii_data		*mii;
-
-	mii = device_get_softc(sc->arge_miibus);
-	ARGE_LOCK(sc);
-	mii_pollstat(mii);
-	ARGE_UNLOCK(sc);
-	ifmr->ifm_active = mii->mii_media_active;
-	ifmr->ifm_status = mii->mii_media_status;
-}
-
-struct arge_dmamap_arg {
-	bus_addr_t	arge_busaddr;
-};
-
-static void
-arge_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
-{
-	struct arge_dmamap_arg	*ctx;
-
-	if (error != 0)
-		return;
-	ctx = arg;
-	ctx->arge_busaddr = segs[0].ds_addr;
-}
-
-static int
-arge_dma_alloc(struct arge_softc *sc)
-{
-	struct arge_dmamap_arg	ctx;
-	struct arge_txdesc	*txd;
-	struct arge_rxdesc	*rxd;
-	int			error, i;
-
-	/* Create parent DMA tag. */
-	error = bus_dma_tag_create(
-	    bus_get_dma_tag(sc->arge_dev),	/* parent */
-	    1, 0,			/* alignment, boundary */
-	    BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
-	    BUS_SPACE_MAXADDR,		/* highaddr */
-	    NULL, NULL,			/* filter, filterarg */
-	    BUS_SPACE_MAXSIZE_32BIT,	/* maxsize */
-	    0,				/* nsegments */
-	    BUS_SPACE_MAXSIZE_32BIT,	/* maxsegsize */
-	    0,				/* flags */
-	    NULL, NULL,			/* lockfunc, lockarg */
-	    &sc->arge_cdata.arge_parent_tag);
-	if (error != 0) {
-		device_printf(sc->arge_dev, "failed to create parent DMA tag\n");
-		goto fail;
-	}
-	/* Create tag for Tx ring. */
-	error = bus_dma_tag_create(
-	    sc->arge_cdata.arge_parent_tag,	/* parent */
-	    ARGE_RING_ALIGN, 0,		/* alignment, boundary */
-	    BUS_SPACE_MAXADDR,		/* lowaddr */
-	    BUS_SPACE_MAXADDR,		/* highaddr */
-	    NULL, NULL,			/* filter, filterarg */
-	    ARGE_TX_DMA_SIZE,		/* maxsize */
-	    1,				/* nsegments */
-	    ARGE_TX_DMA_SIZE,		/* maxsegsize */
-	    0,				/* flags */
-	    NULL, NULL,			/* lockfunc, lockarg */
-	    &sc->arge_cdata.arge_tx_ring_tag);
-	if (error != 0) {
-		device_printf(sc->arge_dev, "failed to create Tx ring DMA tag\n");
-		goto fail;
-	}
-
-	/* Create tag for Rx ring. */
-	error = bus_dma_tag_create(
-	    sc->arge_cdata.arge_parent_tag,	/* parent */
-	    ARGE_RING_ALIGN, 0,		/* alignment, boundary */
-	    BUS_SPACE_MAXADDR,		/* lowaddr */
-	    BUS_SPACE_MAXADDR,		/* highaddr */
-	    NULL, NULL,			/* filter, filterarg */
-	    ARGE_RX_DMA_SIZE,		/* maxsize */
-	    1,				/* nsegments */
-	    ARGE_RX_DMA_SIZE,		/* maxsegsize */
-	    0,				/* flags */
-	    NULL, NULL,			/* lockfunc, lockarg */
-	    &sc->arge_cdata.arge_rx_ring_tag);
-	if (error != 0) {
-		device_printf(sc->arge_dev, "failed to create Rx ring DMA tag\n");
-		goto fail;
-	}
-
-	/* Create tag for Tx buffers. */
-	error = bus_dma_tag_create(
-	    sc->arge_cdata.arge_parent_tag,	/* parent */
-	    sizeof(uint32_t), 0,	/* alignment, boundary */
-	    BUS_SPACE_MAXADDR,		/* lowaddr */
-	    BUS_SPACE_MAXADDR,		/* highaddr */
-	    NULL, NULL,			/* filter, filterarg */
-	    MCLBYTES * ARGE_MAXFRAGS,	/* maxsize */
-	    ARGE_MAXFRAGS,		/* nsegments */
-	    MCLBYTES,			/* maxsegsize */
-	    0,				/* flags */
-	    NULL, NULL,			/* lockfunc, lockarg */
-	    &sc->arge_cdata.arge_tx_tag);
-	if (error != 0) {
-		device_printf(sc->arge_dev, "failed to create Tx DMA tag\n");
-		goto fail;
-	}
-
-	/* Create tag for Rx buffers. */
-	error = bus_dma_tag_create(
-	    sc->arge_cdata.arge_parent_tag,	/* parent */
-	    ARGE_RX_ALIGN, 0,		/* alignment, boundary */
-	    BUS_SPACE_MAXADDR,		/* lowaddr */
-	    BUS_SPACE_MAXADDR,		/* highaddr */
-	    NULL, NULL,			/* filter, filterarg */
-	    MCLBYTES,			/* maxsize */
-	    1,				/* nsegments */
-	    MCLBYTES,			/* maxsegsize */
-	    0,				/* flags */
-	    NULL, NULL,			/* lockfunc, lockarg */
-	    &sc->arge_cdata.arge_rx_tag);
-	if (error != 0) {
-		device_printf(sc->arge_dev, "failed to create Rx DMA tag\n");
-		goto fail;
-	}
-
-	/* Allocate DMA'able memory and load the DMA map for Tx ring. */
-	error = bus_dmamem_alloc(sc->arge_cdata.arge_tx_ring_tag,
-	    (void **)&sc->arge_rdata.arge_tx_ring, BUS_DMA_WAITOK |
-	    BUS_DMA_COHERENT | BUS_DMA_ZERO, &sc->arge_cdata.arge_tx_ring_map);
-	if (error != 0) {
-		device_printf(sc->arge_dev,
-		    "failed to allocate DMA'able memory for Tx ring\n");
-		goto fail;
-	}
-
-	ctx.arge_busaddr = 0;
-	error = bus_dmamap_load(sc->arge_cdata.arge_tx_ring_tag,
-	    sc->arge_cdata.arge_tx_ring_map, sc->arge_rdata.arge_tx_ring,
-	    ARGE_TX_DMA_SIZE, arge_dmamap_cb, &ctx, 0);
-	if (error != 0 || ctx.arge_busaddr == 0) {
-		device_printf(sc->arge_dev,
-		    "failed to load DMA'able memory for Tx ring\n");
-		goto fail;
-	}
-	sc->arge_rdata.arge_tx_ring_paddr = ctx.arge_busaddr;
-
-	/* Allocate DMA'able memory and load the DMA map for Rx ring. */
-	error = bus_dmamem_alloc(sc->arge_cdata.arge_rx_ring_tag,
-	    (void **)&sc->arge_rdata.arge_rx_ring, BUS_DMA_WAITOK |
-	    BUS_DMA_COHERENT | BUS_DMA_ZERO, &sc->arge_cdata.arge_rx_ring_map);
-	if (error != 0) {
-		device_printf(sc->arge_dev,
-		    "failed to allocate DMA'able memory for Rx ring\n");
-		goto fail;
-	}
-
-	ctx.arge_busaddr = 0;
-	error = bus_dmamap_load(sc->arge_cdata.arge_rx_ring_tag,
-	    sc->arge_cdata.arge_rx_ring_map, sc->arge_rdata.arge_rx_ring,
-	    ARGE_RX_DMA_SIZE, arge_dmamap_cb, &ctx, 0);
-	if (error != 0 || ctx.arge_busaddr == 0) {
-		device_printf(sc->arge_dev,
-		    "failed to load DMA'able memory for Rx ring\n");
-		goto fail;
-	}
-	sc->arge_rdata.arge_rx_ring_paddr = ctx.arge_busaddr;
-
-	/* Create DMA maps for Tx buffers. */
-	for (i = 0; i < ARGE_TX_RING_COUNT; i++) {
-		txd = &sc->arge_cdata.arge_txdesc[i];
-		txd->tx_m = NULL;
-		txd->tx_dmamap = NULL;
-		error = bus_dmamap_create(sc->arge_cdata.arge_tx_tag, 0,
-		    &txd->tx_dmamap);
-		if (error != 0) {
-			device_printf(sc->arge_dev,
-			    "failed to create Tx dmamap\n");
-			goto fail;
-		}
-	}
-	/* Create DMA maps for Rx buffers. */
-	if ((error = bus_dmamap_create(sc->arge_cdata.arge_rx_tag, 0,
-	    &sc->arge_cdata.arge_rx_sparemap)) != 0) {
-		device_printf(sc->arge_dev,
-		    "failed to create spare Rx dmamap\n");
-		goto fail;
-	}
-	for (i = 0; i < ARGE_RX_RING_COUNT; i++) {
-		rxd = &sc->arge_cdata.arge_rxdesc[i];
-		rxd->rx_m = NULL;
-		rxd->rx_dmamap = NULL;
-		error = bus_dmamap_create(sc->arge_cdata.arge_rx_tag, 0,
-		    &rxd->rx_dmamap);
-		if (error != 0) {
-			device_printf(sc->arge_dev,
-			    "failed to create Rx dmamap\n");
-			goto fail;
-		}
-	}
-
-fail:
-	return (error);
-}
-
-static void
-arge_dma_free(struct arge_softc *sc)
-{
-	struct arge_txdesc	*txd;
-	struct arge_rxdesc	*rxd;
-	int			i;
-
-	/* Tx ring. */
-	if (sc->arge_cdata.arge_tx_ring_tag) {
-		if (sc->arge_cdata.arge_tx_ring_map)
-			bus_dmamap_unload(sc->arge_cdata.arge_tx_ring_tag,
-			    sc->arge_cdata.arge_tx_ring_map);
-		if (sc->arge_cdata.arge_tx_ring_map &&
-		    sc->arge_rdata.arge_tx_ring)
-			bus_dmamem_free(sc->arge_cdata.arge_tx_ring_tag,
-			    sc->arge_rdata.arge_tx_ring,
-			    sc->arge_cdata.arge_tx_ring_map);
-		sc->arge_rdata.arge_tx_ring = NULL;
-		sc->arge_cdata.arge_tx_ring_map = NULL;
-		bus_dma_tag_destroy(sc->arge_cdata.arge_tx_ring_tag);
-		sc->arge_cdata.arge_tx_ring_tag = NULL;
-	}
-	/* Rx ring. */
-	if (sc->arge_cdata.arge_rx_ring_tag) {
-		if (sc->arge_cdata.arge_rx_ring_map)
-			bus_dmamap_unload(sc->arge_cdata.arge_rx_ring_tag,
-			    sc->arge_cdata.arge_rx_ring_map);
-		if (sc->arge_cdata.arge_rx_ring_map &&
-		    sc->arge_rdata.arge_rx_ring)
-			bus_dmamem_free(sc->arge_cdata.arge_rx_ring_tag,
-			    sc->arge_rdata.arge_rx_ring,
-			    sc->arge_cdata.arge_rx_ring_map);
-		sc->arge_rdata.arge_rx_ring = NULL;
-		sc->arge_cdata.arge_rx_ring_map = NULL;
-		bus_dma_tag_destroy(sc->arge_cdata.arge_rx_ring_tag);
-		sc->arge_cdata.arge_rx_ring_tag = NULL;
-	}
-	/* Tx buffers. */
-	if (sc->arge_cdata.arge_tx_tag) {
-		for (i = 0; i < ARGE_TX_RING_COUNT; i++) {
-			txd = &sc->arge_cdata.arge_txdesc[i];
-			if (txd->tx_dmamap) {
-				bus_dmamap_destroy(sc->arge_cdata.arge_tx_tag,
-				    txd->tx_dmamap);
-				txd->tx_dmamap = NULL;
-			}
-		}
-		bus_dma_tag_destroy(sc->arge_cdata.arge_tx_tag);
-		sc->arge_cdata.arge_tx_tag = NULL;
-	}
-	/* Rx buffers. */
-	if (sc->arge_cdata.arge_rx_tag) {
-		for (i = 0; i < ARGE_RX_RING_COUNT; i++) {
-			rxd = &sc->arge_cdata.arge_rxdesc[i];
-			if (rxd->rx_dmamap) {
-				bus_dmamap_destroy(sc->arge_cdata.arge_rx_tag,
-				    rxd->rx_dmamap);
-				rxd->rx_dmamap = NULL;
-			}
-		}
-		if (sc->arge_cdata.arge_rx_sparemap) {
-			bus_dmamap_destroy(sc->arge_cdata.arge_rx_tag,
-			    sc->arge_cdata.arge_rx_sparemap);
-			sc->arge_cdata.arge_rx_sparemap = 0;
-		}
-		bus_dma_tag_destroy(sc->arge_cdata.arge_rx_tag);
-		sc->arge_cdata.arge_rx_tag = NULL;
-	}
-
-	if (sc->arge_cdata.arge_parent_tag) {
-		bus_dma_tag_destroy(sc->arge_cdata.arge_parent_tag);
-		sc->arge_cdata.arge_parent_tag = NULL;
-	}
-}
-
-/*
- * Initialize the transmit descriptors.
- */
-static int
-arge_tx_ring_init(struct arge_softc *sc)
-{
-	struct arge_ring_data	*rd;
-	struct arge_txdesc	*txd;
-	bus_addr_t		addr;
-	int			i;
-
-	sc->arge_cdata.arge_tx_prod = 0;
-	sc->arge_cdata.arge_tx_cons = 0;
-	sc->arge_cdata.arge_tx_cnt = 0;
-	sc->arge_cdata.arge_tx_pkts = 0;
-
-	rd = &sc->arge_rdata;
-	bzero(rd->arge_tx_ring, sizeof(rd->arge_tx_ring));
-	for (i = 0; i < ARGE_TX_RING_COUNT; i++) {
-		if (i == ARGE_TX_RING_COUNT - 1)
-			addr = ARGE_TX_RING_ADDR(sc, 0);
-		else
-			addr = ARGE_TX_RING_ADDR(sc, i + 1);
-		rd->arge_tx_ring[i].packet_ctrl = ARGE_DESC_EMPTY;
-		rd->arge_tx_ring[i].next_desc = addr;
-		txd = &sc->arge_cdata.arge_txdesc[i];
-		txd->tx_m = NULL;
-	}
-
-	bus_dmamap_sync(sc->arge_cdata.arge_tx_ring_tag,
-	    sc->arge_cdata.arge_tx_ring_map,
-	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-
-	return (0);
-}
-
-/*
- * Initialize the RX descriptors and allocate mbufs for them. Note that
- * we arrange the descriptors in a closed ring, so that the last descriptor
- * points back to the first.
- */
-static int
-arge_rx_ring_init(struct arge_softc *sc)
-{
-	struct arge_ring_data	*rd;
-	struct arge_rxdesc	*rxd;
-	bus_addr_t		addr;
-	int			i;
-
-	sc->arge_cdata.arge_rx_cons = 0;
-
-	rd = &sc->arge_rdata;
-	bzero(rd->arge_rx_ring, sizeof(rd->arge_rx_ring));
-	for (i = 0; i < ARGE_RX_RING_COUNT; i++) {
-		rxd = &sc->arge_cdata.arge_rxdesc[i];
-		rxd->rx_m = NULL;
-		rxd->desc = &rd->arge_rx_ring[i];
-		if (i == ARGE_RX_RING_COUNT - 1)
-			addr = ARGE_RX_RING_ADDR(sc, 0);
-		else
-			addr = ARGE_RX_RING_ADDR(sc, i + 1);
-		rd->arge_rx_ring[i].packet_ctrl = ARGE_DESC_EMPTY;
-		rd->arge_rx_ring[i].next_desc = addr;
-		if (arge_newbuf(sc, i) != 0)
-			return (ENOBUFS);
-	}
-
-	bus_dmamap_sync(sc->arge_cdata.arge_rx_ring_tag,
-	    sc->arge_cdata.arge_rx_ring_map,
-	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-
-	return (0);
-}
-
-/*
- * Initialize an RX descriptor and attach an MBUF cluster.
- */
-static int
-arge_newbuf(struct arge_softc *sc, int idx)
-{
-	struct arge_desc		*desc;
-	struct arge_rxdesc	*rxd;
-	struct mbuf		*m;
-	bus_dma_segment_t	segs[1];
-	bus_dmamap_t		map;
-	int			nsegs;
-
-	m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
-	if (m == NULL)
-		return (ENOBUFS);
-	m->m_len = m->m_pkthdr.len = MCLBYTES;
-	m_adj(m, sizeof(uint64_t));
-
-	if (bus_dmamap_load_mbuf_sg(sc->arge_cdata.arge_rx_tag,
-	    sc->arge_cdata.arge_rx_sparemap, m, segs, &nsegs, 0) != 0) {
-		m_freem(m);
-		return (ENOBUFS);
-	}
-	KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs));
-
-	rxd = &sc->arge_cdata.arge_rxdesc[idx];
-	if (rxd->rx_m != NULL) {
-		bus_dmamap_sync(sc->arge_cdata.arge_rx_tag, rxd->rx_dmamap,
-		    BUS_DMASYNC_POSTREAD);
-		bus_dmamap_unload(sc->arge_cdata.arge_rx_tag, rxd->rx_dmamap);
-	}
-	map = rxd->rx_dmamap;
-	rxd->rx_dmamap = sc->arge_cdata.arge_rx_sparemap;
-	sc->arge_cdata.arge_rx_sparemap = map;
-	bus_dmamap_sync(sc->arge_cdata.arge_rx_tag, rxd->rx_dmamap,
-	    BUS_DMASYNC_PREREAD);
-	rxd->rx_m = m;
-	desc = rxd->desc;
-	desc->packet_addr = segs[0].ds_addr;
-	desc->packet_ctrl =  (desc->packet_ctrl & ~ARGE_DESC_SIZE_MASK)
-	    | ARGE_DMASIZE(segs[0].ds_len);
-
-	return (0);
-}
-
-static __inline void
-arge_fixup_rx(struct mbuf *m)
-{
-        int		i;
-        uint16_t	*src, *dst;
-
-	src = mtod(m, uint16_t *);
-	dst = src - 1;
-
-	for (i = 0; i < (m->m_len / sizeof(uint16_t) + 1); i++)
-		*dst++ = *src++;
-
-	m->m_data -= ETHER_ALIGN;
-}
-
-
-static void
-arge_tx_locked(struct arge_softc *sc)
-{
-	struct arge_txdesc	*txd;
-	struct arge_desc	*cur_tx;
-	struct ifnet		*ifp;
-	uint32_t		ctrl;
-	int			cons, prod;
-
-	ARGE_LOCK_ASSERT(sc);
-
-	cons = sc->arge_cdata.arge_tx_cons;
-	prod = sc->arge_cdata.arge_tx_prod;
-	if (cons == prod)
-		return;
-
-	bus_dmamap_sync(sc->arge_cdata.arge_tx_ring_tag,
-	    sc->arge_cdata.arge_tx_ring_map,
-	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
-
-	ifp = sc->arge_ifp;
-	/*
-	 * Go through our tx list and free mbufs for those
-	 * frames that have been transmitted.
-	 */
-	for (; cons != prod; ARGE_INC(cons, ARGE_TX_RING_COUNT)) {
-		cur_tx = &sc->arge_rdata.arge_tx_ring[cons];
-		ctrl = cur_tx->packet_ctrl;
-		/* Check if descriptor has "finished" flag */
-		if ((ctrl & ARGE_DESC_EMPTY) == 0)
-			break;
-
-		ARGE_WRITE(sc, AR71XX_DMA_TX_STATUS, DMA_TX_STATUS_PKT_SENT);
-
-		sc->arge_cdata.arge_tx_cnt--;
-		ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
-
-		txd = &sc->arge_cdata.arge_txdesc[cons];
-
-		cur_tx->packet_ctrl = ARGE_DESC_EMPTY;
-
-		ifp->if_opackets++;
-
-		bus_dmamap_sync(sc->arge_cdata.arge_tx_tag, txd->tx_dmamap,
-		    BUS_DMASYNC_POSTWRITE);
-		bus_dmamap_unload(sc->arge_cdata.arge_tx_tag, txd->tx_dmamap);
-
-		/* Free only if it's first descriptor in list */
-		if (txd->tx_m)
-			m_freem(txd->tx_m);
-		txd->tx_m = NULL;
-
-		/* reset descriptor */
-		cur_tx->packet_ctrl = ARGE_DESC_EMPTY;
-		cur_tx->packet_addr = 0;
-	}
-
-	sc->arge_cdata.arge_tx_cons = cons;
-
-	bus_dmamap_sync(sc->arge_cdata.arge_tx_ring_tag,
-	    sc->arge_cdata.arge_tx_ring_map, BUS_DMASYNC_PREWRITE);
-}
-
-
-static void
-arge_rx_locked(struct arge_softc *sc)
-{
-	struct arge_rxdesc	*rxd;
-	struct ifnet		*ifp = sc->arge_ifp;
-	int			cons, prog, packet_len;
-	struct arge_desc	*cur_rx;
-	struct mbuf		*m;
-
-	ARGE_LOCK_ASSERT(sc);
-
-	cons = sc->arge_cdata.arge_rx_cons;
-
-	bus_dmamap_sync(sc->arge_cdata.arge_rx_ring_tag,
-	    sc->arge_cdata.arge_rx_ring_map,
-	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
-
-	for (prog = 0; prog < ARGE_RX_RING_COUNT; 
-	    ARGE_INC(cons, ARGE_RX_RING_COUNT)) {
-		cur_rx = &sc->arge_rdata.arge_rx_ring[cons];
-		rxd = &sc->arge_cdata.arge_rxdesc[cons];
-		m = rxd->rx_m;
-
-		if ((cur_rx->packet_ctrl & ARGE_DESC_EMPTY) != 0)
-		       break;	
-
-		ARGE_WRITE(sc, AR71XX_DMA_RX_STATUS, DMA_RX_STATUS_PKT_RECVD);
-
-		prog++;
-
-		packet_len = ARGE_DMASIZE(cur_rx->packet_ctrl);
-		bus_dmamap_sync(sc->arge_cdata.arge_rx_tag, rxd->rx_dmamap,
-		    BUS_DMASYNC_PREREAD);
-		m = rxd->rx_m;
-
-		arge_fixup_rx(m);
-		m->m_pkthdr.rcvif = ifp;
-		/* Skip 4 bytes of CRC */
-		m->m_pkthdr.len = m->m_len = packet_len - ETHER_CRC_LEN;
-		ifp->if_ipackets++;
-
-		ARGE_UNLOCK(sc);
-		(*ifp->if_input)(ifp, m);
-		ARGE_LOCK(sc);
-
-		/* Reinit descriptor */
-		cur_rx->packet_ctrl = ARGE_DESC_EMPTY;
-		cur_rx->packet_addr = 0;
-		if (arge_newbuf(sc, cons) != 0) {
-			device_printf(sc->arge_dev, 
-			    "Failed to allocate buffer\n");
-			break;
-		}
-
-		bus_dmamap_sync(sc->arge_cdata.arge_rx_ring_tag,
-		    sc->arge_cdata.arge_rx_ring_map,
-		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
-
-	}
-
-	if (prog > 0) {
-		sc->arge_cdata.arge_rx_cons = cons;
-
-		bus_dmamap_sync(sc->arge_cdata.arge_rx_ring_tag,
-		    sc->arge_cdata.arge_rx_ring_map,
-		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
-	}
-}
-
-static void
-arge_rx_intr(struct arge_softc *sc, uint32_t status)
-{
-
-	ARGE_LOCK(sc);
-	/* interrupts are masked by filter */
-	arge_rx_locked(sc);
-
-	/* unmask interrupts */
-	ARGE_SET_BITS(sc, 
-	    AR71XX_DMA_INTR, DMA_INTR_RX_OVERFLOW | DMA_INTR_RX_PKT_RCVD);
-	ARGE_UNLOCK(sc);
-}
-
-static int
-arge_intr_filter(void *arg)
-{
-	struct arge_softc	*sc = arg;
-	uint32_t		status, ints;
-
-	status = ARGE_READ(sc, AR71XX_DMA_INTR_STATUS);
-	ints = ARGE_READ(sc, AR71XX_DMA_INTR);
-
-#if 0
-	dprintf("int mask(filter) = %b\n", ints,
-	    "\20\10RX_BUS_ERROR\7RX_OVERFLOW\5RX_PKT_RCVD"
-	    "\4TX_BUS_ERROR\2TX_UNDERRUN\1TX_PKT_SENT");
-	dprintf("status(filter) = %b\n", status, 
-	    "\20\10RX_BUS_ERROR\7RX_OVERFLOW\5RX_PKT_RCVD"
-	    "\4TX_BUS_ERROR\2TX_UNDERRUN\1TX_PKT_SENT");
-#endif
-
-	if (status & DMA_INTR_ALL) {
-		if (status & (DMA_INTR_RX_PKT_RCVD | DMA_INTR_RX_OVERFLOW))
-			ARGE_CLEAR_BITS(sc, AR71XX_DMA_INTR, 
-			    DMA_INTR_RX_OVERFLOW | DMA_INTR_RX_PKT_RCVD);
-
-		if (status & (DMA_INTR_TX_PKT_SENT | DMA_INTR_TX_UNDERRUN))
-			ARGE_CLEAR_BITS(sc, AR71XX_DMA_INTR, 
-			    DMA_INTR_TX_UNDERRUN | DMA_INTR_TX_PKT_SENT);
-
-		sc->arge_intr_status = status;
-		return (FILTER_SCHEDULE_THREAD);
-	}
-
-	sc->arge_intr_status = 0;
-	return (FILTER_STRAY);
-}
-
-static void
-arge_intr(void *arg)
-{
-	struct arge_softc	*sc = arg;
-	uint32_t		status;
-
-	status = sc->arge_intr_status;
-
-#if 0
-	dprintf("int status(intr) = %b\n", status, 
-	    "\20\10\7RX_OVERFLOW\5RX_PKT_RCVD"
-	    "\4TX_BUS_ERROR\2TX_UNDERRUN\1TX_PKT_SENT");
-#endif
-
-	/* 
-	 * Is it our interrupt at all? 
-	 */
-	if (status == 0)
-		return;
-
-	if (status & DMA_INTR_RX_BUS_ERROR) {
-		ARGE_WRITE(sc, AR71XX_DMA_RX_STATUS, DMA_RX_STATUS_BUS_ERROR);
-		device_printf(sc->arge_dev, "RX bus error");
-		return;
-	}
-
-	if (status & DMA_INTR_TX_BUS_ERROR) {
-		ARGE_WRITE(sc, AR71XX_DMA_TX_STATUS, DMA_TX_STATUS_BUS_ERROR);
-		device_printf(sc->arge_dev, "TX bus error");
-		return;
-	}
-
-	if (status & (DMA_INTR_RX_PKT_RCVD | DMA_INTR_RX_OVERFLOW))
-		arge_rx_intr(sc, status);
-
-	if (status & (DMA_INTR_TX_PKT_SENT | DMA_INTR_TX_UNDERRUN))
-		arge_tx_intr(sc, status);
-}
-
-static void
-arge_tx_intr(struct arge_softc *sc, uint32_t status)
-{
-	ARGE_LOCK(sc);
-
-	/* Interrupts are masked by filter */
-	arge_tx_locked(sc);
-
-	ARGE_UNLOCK(sc);
-}
-
-static void
-arge_tick(void *xsc)
-{
-	struct arge_softc	*sc = xsc;
-	struct mii_data		*mii;
-
-	ARGE_LOCK_ASSERT(sc);
-
-	mii = device_get_softc(sc->arge_miibus);
-	mii_tick(mii);
-	callout_reset(&sc->arge_stat_callout, hz, arge_tick, sc);
-}
-
-/*
- * Create a copy of a single mbuf. It can have either internal or
- * external data, it may have a packet header. External data is really
- * copied, so the new buffer is writeable.
- */
-static struct mbuf *
-copy_mbuf(struct mbuf *m)
-{
-	struct mbuf *new;
-
-	MGET(new, M_DONTWAIT, MT_DATA);
-	if (new == NULL)
-		return (NULL);
-
-	if (m->m_flags & M_PKTHDR) {
-		M_MOVE_PKTHDR(new, m);
-		if (m->m_len > MHLEN)
-			MCLGET(new, M_WAIT);
-	} else {
-		if (m->m_len > MLEN)
-			MCLGET(new, M_WAIT);
-	}
-
-	bcopy(m->m_data, new->m_data, m->m_len);
-	new->m_len = m->m_len;
-	new->m_flags &= ~M_RDONLY;
-
-	return (new);
-}
-
-
-
-static int
-arge_fix_chain(struct mbuf **mp)
-{
-	struct mbuf *m = *mp, *prev = NULL, *next, *new;
-	u_int mlen = 0, fill = 0;	
-	int first, off;
-	u_char *d, *cp;
-
-	do {
-		next = m->m_next;
-
-		if ((uintptr_t)mtod(m, void *) % 4 != 0 ||
-		   (m->m_len % 4 != 0 && next)) {
-			/*
-			 * Needs fixing
-			 */
-			first = (m == *mp);
-
-			d = mtod(m, u_char *);
-			if ((off = (uintptr_t)(void *)d % 4) != 0) {
-				if (M_WRITABLE(m)) {
-					bcopy(d, d - off, m->m_len);
-					m->m_data = (caddr_t)(d - off);
-				} else {
-					if ((new = copy_mbuf(m)) == NULL) {
-						goto fail;
-					}
-					if (prev)
-						prev->m_next = new;
-					new->m_next = next;
-					m_free(m);
-					m = new;
-				}
-			}
-
-			if ((off = m->m_len % 4) != 0) {
-				if (!M_WRITABLE(m)) {
-					if ((new = copy_mbuf(m)) == NULL) {
-						goto fail;
-					}
-					if (prev)
-						prev->m_next = new;
-					new->m_next = next;
-					m_free(m);
-					m = new;
-				}
-				d = mtod(m, u_char *) + m->m_len;
-				off = 4 - off;
-				while (off) {
-					if (next == NULL) {
-						*d++ = 0;
-						fill++;
-					} else if (next->m_len == 0) {
-						next = m_free(next);
-						continue;
-					} else {
-						cp = mtod(next, u_char *);
-						*d++ = *cp++;
-						next->m_len--;
-						next->m_data = (caddr_t)cp;
-					}
-					off--;
-					m->m_len++;
-				}
-			}
-
-			if (first)
-				*mp = m;
-		}
-
-		mlen += m->m_len;
-		prev = m;
-	} while ((m = next) != NULL);
-
-	return (mlen - fill);
-
-  fail:
-	m_freem(*mp);
-	*mp = NULL;
-	return (0);
-}
diff --git a/sys/mips/atheros/if_argevar.h b/sys/mips/atheros/if_argevar.h
deleted file mode 100644
index 1966e61..0000000
--- a/sys/mips/atheros/if_argevar.h
+++ /dev/null
@@ -1,138 +0,0 @@
-/*-
- * Copyright (c) 2009, Oleksandr Tymoshenko
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice unmodified, this list of conditions, and the following
- *    disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#ifndef __IF_ARGEVAR_H__
-#define __IF_ARGEVAR_H__
-
-#define	ARGE_TX_RING_COUNT	128
-#define	ARGE_RX_RING_COUNT	128
-#define	ARGE_RX_DMA_SIZE	ARGE_RX_RING_COUNT * sizeof(struct arge_desc)
-#define	ARGE_TX_DMA_SIZE	ARGE_TX_RING_COUNT * sizeof(struct arge_desc)
-#define	ARGE_MAXFRAGS		8
-#define ARGE_RING_ALIGN		sizeof(struct arge_desc)
-#define ARGE_RX_ALIGN		sizeof(uint32_t)
-#define ARGE_MAXFRAGS		8
-#define	ARGE_TX_RING_ADDR(sc, i)	\
-    ((sc)->arge_rdata.arge_tx_ring_paddr + sizeof(struct arge_desc) * (i))
-#define	ARGE_RX_RING_ADDR(sc, i)	\
-    ((sc)->arge_rdata.arge_rx_ring_paddr + sizeof(struct arge_desc) * (i))
-#define	ARGE_INC(x,y)		(x) = (((x) + 1) % y)
-
-
-#define	ARGE_MII_TIMEOUT	1000
-
-#define	ARGE_LOCK(_sc)		mtx_lock(&(_sc)->arge_mtx)
-#define	ARGE_UNLOCK(_sc)	mtx_unlock(&(_sc)->arge_mtx)
-#define	ARGE_LOCK_ASSERT(_sc)	mtx_assert(&(_sc)->arge_mtx, MA_OWNED)
-
-/*
- * register space access macros
- */
-#define ARGE_WRITE(sc, reg, val)	do {	\
-		bus_write_4(sc->arge_res, (reg), (val)); \
-	} while (0)
-
-#define ARGE_READ(sc, reg)	 bus_read_4(sc->arge_res, (reg))
-
-#define ARGE_SET_BITS(sc, reg, bits)	\
-	ARGE_WRITE(sc, reg, ARGE_READ(sc, (reg)) | (bits))
-
-#define ARGE_CLEAR_BITS(sc, reg, bits)	\
-	ARGE_WRITE(sc, reg, ARGE_READ(sc, (reg)) & ~(bits))
-
-#define ARGE_DESC_EMPTY		(1 << 31)
-#define ARGE_DESC_MORE		(1 << 24)
-#define ARGE_DESC_SIZE_MASK	((1 << 12) - 1)
-#define	ARGE_DMASIZE(len)	((len) & ARGE_DESC_SIZE_MASK)
-struct arge_desc {
-	uint32_t	packet_addr;
-	uint32_t	packet_ctrl;
-	uint32_t	next_desc;
-	uint32_t	padding;
-};
-
-struct arge_txdesc {
-	struct mbuf	*tx_m;
-	bus_dmamap_t	tx_dmamap;
-};
-
-struct arge_rxdesc {
-	struct mbuf		*rx_m;
-	bus_dmamap_t		rx_dmamap;
-	struct arge_desc	*desc;
-};
-
-struct arge_chain_data {
-	bus_dma_tag_t		arge_parent_tag;
-	bus_dma_tag_t		arge_tx_tag;
-	struct arge_txdesc	arge_txdesc[ARGE_TX_RING_COUNT];
-	bus_dma_tag_t		arge_rx_tag;
-	struct arge_rxdesc	arge_rxdesc[ARGE_RX_RING_COUNT];
-	bus_dma_tag_t		arge_tx_ring_tag;
-	bus_dma_tag_t		arge_rx_ring_tag;
-	bus_dmamap_t		arge_tx_ring_map;
-	bus_dmamap_t		arge_rx_ring_map;
-	bus_dmamap_t		arge_rx_sparemap;
-	int			arge_tx_pkts;
-	int			arge_tx_prod;
-	int			arge_tx_cons;
-	int			arge_tx_cnt;
-	int			arge_rx_cons;
-};
-
-struct arge_ring_data {
-	struct arge_desc	*arge_rx_ring;
-	struct arge_desc	*arge_tx_ring;
-	bus_addr_t		arge_rx_ring_paddr;
-	bus_addr_t		arge_tx_ring_paddr;
-};
-
-struct arge_softc {
-	struct ifnet		*arge_ifp;	/* interface info */
-	device_t		arge_dev;
-	struct resource		*arge_res;
-	int			arge_rid;
-	struct resource		*arge_irq;
-	void			*arge_intrhand;
-	device_t		arge_miibus;
-	bus_dma_tag_t		arge_parent_tag;
-	bus_dma_tag_t		arge_tag;
-	struct mtx		arge_mtx;
-	struct callout		arge_stat_callout;
-	struct task		arge_link_task;
-	struct arge_chain_data	arge_cdata;
-	struct arge_ring_data	arge_rdata;
-	int			arge_link_status;
-	int			arge_detach;
-	uint32_t		arge_intr_status;
-	int			arge_mac_unit;
-	int			arge_phy_num;
-	uint32_t		arge_ddr_flush_reg;
-	uint32_t		arge_pll_reg;
-};
-
-#endif /* __IF_ARGEVAR_H__ */
diff --git a/sys/mips/atheros/uart_bus_ar71xx.c b/sys/mips/atheros/uart_bus_ar71xx.c
deleted file mode 100644
index 1cb611c..0000000
--- a/sys/mips/atheros/uart_bus_ar71xx.c
+++ /dev/null
@@ -1,79 +0,0 @@
-/*-
- * Copyright (c) 2009, Oleksandr Tymoshenko <gonzo@FreeBSD.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- */
-#include "opt_uart.h"
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/bus.h>
-#include <sys/conf.h>
-#include <sys/kernel.h>
-#include <sys/module.h>
-
-#include <machine/bus.h>
-
-#include <dev/uart/uart.h>
-#include <dev/uart/uart_cpu.h>
-#include <dev/uart/uart_bus.h>
-
-#include <mips/atheros/ar71xxreg.h>
-
-#include "uart_if.h"
-
-static int uart_ar71xx_probe(device_t dev);
-extern struct uart_class uart_ar71xx_uart_class;
-
-static device_method_t uart_ar71xx_methods[] = {
-	/* Device interface */
-	DEVMETHOD(device_probe,		uart_ar71xx_probe),
-	DEVMETHOD(device_attach,	uart_bus_attach),
-	DEVMETHOD(device_detach,	uart_bus_detach),
-	{ 0, 0 }
-};
-
-static driver_t uart_ar71xx_driver = {
-	uart_driver_name,
-	uart_ar71xx_methods,
-	sizeof(struct uart_softc),
-};
-
-extern SLIST_HEAD(uart_devinfo_list, uart_devinfo) uart_sysdevs;
-
-static int
-uart_ar71xx_probe(device_t dev)
-{
-	struct uart_softc *sc;
-
-	sc = device_get_softc(dev);
-	sc->sc_sysdev = SLIST_FIRST(&uart_sysdevs);
-	sc->sc_class = &uart_ns8250_class;
-	bcopy(&sc->sc_sysdev->bas, &sc->sc_bas, sizeof(sc->sc_bas));
-
-	return (uart_bus_probe(dev, 2, 85000000, 0, 0));
-}
-
-DRIVER_MODULE(uart, apb, uart_ar71xx_driver, uart_devclass, 0, 0);
diff --git a/sys/mips/atheros/uart_cpu_ar71xx.c b/sys/mips/atheros/uart_cpu_ar71xx.c
deleted file mode 100644
index 4d6b7e7..0000000
--- a/sys/mips/atheros/uart_cpu_ar71xx.c
+++ /dev/null
@@ -1,78 +0,0 @@
-/*-
- * Copyright (c) 2009 Oleksandr Tymoshenko
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-#include "opt_uart.h"
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/bus.h>
-
-#include <machine/bus.h>
-
-#include <dev/uart/uart.h>
-#include <dev/uart/uart_cpu.h>
-
-#include <mips/atheros/ar71xxreg.h>
-
-bus_space_tag_t uart_bus_space_io;
-bus_space_tag_t uart_bus_space_mem;
-
-int
-uart_cpu_eqres(struct uart_bas *b1, struct uart_bas *b2)
-{
-	return ((b1->bsh == b2->bsh && b1->bst == b2->bst) ? 1 : 0);
-}
-
-int
-uart_cpu_getdev(int devtype, struct uart_devinfo *di)
-{
-	di->ops = uart_getops(&uart_ns8250_class);
-	di->bas.chan = 0;
-	di->bas.bst = MIPS_BUS_SPACE_MEM;
-	di->bas.regshft = 2;
-	/* TODO: calculate proper AHB freq using PLL registers */
-	di->bas.rclk = 85000000;
-	di->baudrate = 115200;
-	di->databits = 8;
-	di->stopbits = 1;
-	di->parity = UART_PARITY_NONE;
-
-	/* TODO: check if uart_bus_space_io mandatory to set */
-	uart_bus_space_io = MIPS_BUS_SPACE_IO;
-	uart_bus_space_mem = MIPS_BUS_SPACE_MEM;
-	/* 
-	 * FIXME:
-	 * 3 is to compensate big endian, uart operates 
-	 * with bus_space_read_1/bus_space_write_1 and hence gets 
-	 * highest byte instead of lowest one. Actual fix will involve
-	 * MIPS bus_space fixing.
-	 */
-	di->bas.bsh = MIPS_PHYS_TO_KSEG1(AR71XX_UART_ADDR) + 3;
-	return (0);
-}
diff --git a/sys/mips/conf/ADM5120 b/sys/mips/conf/ADM5120
index 7a47abe..a03d102 100644
--- a/sys/mips/conf/ADM5120
+++ b/sys/mips/conf/ADM5120
@@ -25,6 +25,7 @@ makeoptions	MIPS_LITTLE_ENDIAN=defined
 # Don't build any modules yet.
 makeoptions	MODULES_OVERRIDE=""
 
+options		KERNVIRTADDR=0x80100000
 include		"../adm5120/std.adm5120"
 
 hints		"ADM5120.hints"		#Default places to look for devices.
diff --git a/sys/mips/conf/ALCHEMY b/sys/mips/conf/ALCHEMY
deleted file mode 100644
index c1fb59a..0000000
--- a/sys/mips/conf/ALCHEMY
+++ /dev/null
@@ -1,66 +0,0 @@
-# ALCHEMY -- Generic kernel for Alchemy Au1xxx CPUs.
-#
-# For more information on this file, please read the handbook section on
-# Kernel Configuration Files:
-#
-#    http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html
-#
-# The handbook is also available locally in /usr/share/doc/handbook
-# if you've installed the doc distribution, otherwise always see the
-# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the
-# latest information.
-#
-# An exhaustive list of options and more detailed explanations of the
-# device lines is also present in the ../../conf/NOTES and NOTES files. 
-# If you are in doubt as to the purpose or necessity of a line, check first 
-# in NOTES.
-#
-# $FreeBSD$
-
-ident		ALCHEMY
-
-makeoptions	ARCH_FLAGS=-march=mips32
-makeoptions	MIPS_LITTLE_ENDIAN=defined
-
-# Don't build any modules yet.
-makeoptions	MODULES_OVERRIDE=""
-
-include		"../alchemy/std.alchemy"
-
-hints		"ALCHEMY.hints"		#Default places to look for devices.
-
-makeoptions	DEBUG=-g		#Build kernel with gdb(1) debug symbols
-
-options		DDB
-options		KDB
-
-options		SCHED_4BSD		#4BSD scheduler
-options		INET			#InterNETworking
-options		NFSCLIENT		#Network Filesystem Client
-options		NFS_ROOT		#NFS usable as /, requires NFSCLIENT
-options		PSEUDOFS		#Pseudo-filesystem framework
-# options		_KPOSIX_PRIORITY_SCHEDULING #Posix P1003_1B real-time extensions
-
-options BOOTP
-options BOOTP_NFSROOT
-options BOOTP_NFSV3
-options BOOTP_WIRED_TO=admsw0
-options BOOTP_COMPAT
-
-# options         FFS                     #Berkeley Fast Filesystem
-# options         SOFTUPDATES             #Enable FFS soft updates support
-# options         UFS_ACL                 #Support for access control lists
-# options         UFS_DIRHASH             #Improve performance on big directories
-options		ROOTDEVNAME=\"nfs:10.0.0.1:/mnt/bsd\"
-
-
-# Debugging for use in -current
-options		INVARIANTS		#Enable calls of extra sanity checking
-options		INVARIANT_SUPPORT	#Extra sanity checks of internal structures, required by INVARIANTS
-#options		WITNESS			#Enable checks to detect deadlocks and cycles
-#options		WITNESS_SKIPSPIN	#Don't run witness on spinlocks for speed
-
-device		loop
-device		ether
-device		uart
-# device		md
diff --git a/sys/mips/conf/AR71XX b/sys/mips/conf/AR71XX
deleted file mode 100644
index 936d0f1..0000000
--- a/sys/mips/conf/AR71XX
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# $FreeBSD$
-#
-
-ident		AR71XX
-cpu		CPU_MIPS4KC
-options		CPU_NOFPU
-options 	ISA_MIPS32
-makeoptions	TARGET_BIG_ENDIAN
-makeoptions	KERNLOADADDR=0x80050000
-
-files		"../atheros/files.ar71xx"
-hints		"AR71XX.hints"
-
-makeoptions	DEBUG=-g		#Build kernel with gdb(1) debug symbols
-makeoptions	MODULES_OVERRIDE=""
-
-options		DDB
-options		KDB
-
-options		SCHED_4BSD		#4BSD scheduler
-options		INET			#InterNETworking
-options		NFSCLIENT		#Network Filesystem Client
-options		NFS_ROOT		#NFS usable as /, requires NFSCLIENT
-options		PSEUDOFS		#Pseudo-filesystem framework
-options		_KPOSIX_PRIORITY_SCHEDULING #Posix P1003_1B real-time extensions
-
-# Debugging for use in -current
-options		INVARIANTS
-options		INVARIANT_SUPPORT
-
-device		pci
-device		uart
-
-device		loop
-device		ether
-device		md
diff --git a/sys/mips/conf/AR71XX.hints b/sys/mips/conf/AR71XX.hints
deleted file mode 100644
index 58fbdde..0000000
--- a/sys/mips/conf/AR71XX.hints
+++ /dev/null
@@ -1,25 +0,0 @@
-# $FreeBSD$
-hint.apb.0.at="nexus0"
-hint.apb.0.maddr=0x18000000
-hint.apb.0.msize=0x01000000
-hint.apb.0.irq=4
-
-# uart0
-hint.uart.0.at="apb0"
-# see atheros/uart_cpu_ar71xx.c why +3
-hint.uart.0.maddr=0x18020003
-hint.uart.0.msize=0x18
-hint.uart.0.irq=3
-
-# pci
-hint.pcib.0.at="nexus0"
-hint.pcib.0.irq=0
-
-hint.arge.0.at="nexus0"
-hint.arge.0.maddr=0x19000000
-hint.arge.0.msize=0x1000
-hint.arge.0.irq=2
-# hint.arge.1.at="nexus0"
-# hint.arge.1.maddr=0x1A000000
-# hint.arge.1.msize=0x1000
-# hint.arge.1.irq=3
diff --git a/sys/mips/conf/MALTA b/sys/mips/conf/MALTA
index 022cc52..9771c55 100644
--- a/sys/mips/conf/MALTA
+++ b/sys/mips/conf/MALTA
@@ -27,6 +27,7 @@ options		YAMON
 # Don't build any modules yet.
 makeoptions	MODULES_OVERRIDE=""
 
+options		KERNVIRTADDR=0x80100000
 options		TICK_USE_YAMON_FREQ=defined
 #options		TICK_USE_MALTA_RTC=defined
 
diff --git a/sys/mips/conf/QEMU b/sys/mips/conf/QEMU
index bc7d649..0a2b122 100644
--- a/sys/mips/conf/QEMU
+++ b/sys/mips/conf/QEMU
@@ -27,6 +27,7 @@ makeoptions	ARCH_FLAGS=-march=mips32
 # Don't build any modules yet.
 makeoptions	MODULES_OVERRIDE=""
 
+options		KERNVIRTADDR=0x80100000
 include		"../adm5120/std.adm5120"
 
 #hints		"GENERIC.hints"		#Default places to look for devices.
diff --git a/sys/mips/conf/SENTRY5 b/sys/mips/conf/SENTRY5
index c3918e2..7611e44 100644
--- a/sys/mips/conf/SENTRY5
+++ b/sys/mips/conf/SENTRY5
@@ -41,6 +41,13 @@ options		CFE
 options		CFE_CONSOLE
 options		ALT_BREAK_TO_DEBUGGER
 
+# cfe loader expects kernel at 0x80001000 for mips32 w/o backwards
+# offsets in the linked elf image (see ldscript hack)
+# XXX can we conditionalize the linker stuff on options CFE?
+options		KERNVIRTADDR=0x80001000
+
+makeoptions	LDSCRIPT_NAME=	ldscript.mips.cfe
+
 #makeoptions	ARCH_FLAGS=-march=mips32
 makeoptions	MIPS_LITTLE_ENDIAN=defined
 makeoptions	DEBUG=-g		#Build kernel with gdb(1) debug symbols
@@ -66,8 +73,8 @@ options		INVARIANT_SUPPORT
 device		siba			# Sonics SiliconBackplane
 device		pci			# siba_pcib
 
-# device		bfe			# XXX will build both pci and siba
-# device		miibus			# attachments
+device		bfe			# XXX will build both pci and siba
+device		miibus			# attachments
 
 # pci devices
 # notyet:
diff --git a/sys/mips/include/bus.h b/sys/mips/include/bus.h
index 42ac1df..92557d7 100644
--- a/sys/mips/include/bus.h
+++ b/sys/mips/include/bus.h
@@ -101,8 +101,9 @@
  * Map a region of device bus space into CPU virtual address space.
  */
 
-__inline int	bus_space_map(bus_space_tag_t t, bus_addr_t addr,
-		    bus_size_t size, int flags, bus_space_handle_t *bshp);
+static __inline int bus_space_map(bus_space_tag_t t, bus_addr_t addr,
+				  bus_size_t size, int flags,
+				  bus_space_handle_t *bshp);
 
 static __inline int
 bus_space_map(bus_space_tag_t t __unused, bus_addr_t addr,
diff --git a/sys/mips/include/elf.h b/sys/mips/include/elf.h
index 6e48ec1..3a31daa 100644
--- a/sys/mips/include/elf.h
+++ b/sys/mips/include/elf.h
@@ -241,15 +241,13 @@ __ElfType(Auxinfo);
 #define	AT_BASE		7	/* Interpreter's base address. */
 #define	AT_FLAGS	8	/* Flags (unused for i386). */
 #define	AT_ENTRY	9	/* Where interpreter should transfer control. */
-/*
- * The following non-standard values are used in Linux ELF binaries.
- */
 #define	AT_NOTELF	10	/* Program is not ELF ?? */
 #define	AT_UID		11	/* Real uid. */
 #define	AT_EUID		12	/* Effective uid. */
 #define	AT_GID		13	/* Real gid. */
 #define	AT_EGID		14	/* Effective gid. */
+#define	AT_EXECPATH	15	/* Path to the executable. */
 
-#define	AT_COUNT	15	/* Count of defined aux entry types. */
+#define	AT_COUNT	16	/* Count of defined aux entry types. */
 
 #endif /* !_MACHINE_ELF_H_ */
diff --git a/sys/mips/mips/elf64_machdep.c b/sys/mips/mips/elf64_machdep.c
index e5a2f5d..c10fb5f 100644
--- a/sys/mips/mips/elf64_machdep.c
+++ b/sys/mips/mips/elf64_machdep.c
@@ -88,7 +88,8 @@ static Elf64_Brandinfo freebsd_brand_gnutools_info64 = {
 	.interp_path	= "/libexec/ld-elf.so.1",
 	.sysvec		= &elf64_freebsd_sysvec,
 	.interp_path	= "/libexec/ld-elf.so.1",
-	.flags		= BI_CAN_EXEC_DYN
+	.brand_note	= &elf64_freebsd_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 SYSINIT(gnu_mips_elf64, SI_SUB_EXEC, SI_ORDER_ANY,
@@ -103,7 +104,8 @@ static Elf64_Brandinfo freebsd_brand_info64 = {
 	.interp_path	= "/libexec/ld-elf.so.1",
 	.sysvec		= &elf64_freebsd_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= 0
+	.brand_note	= &elf64_freebsd_brandnote,
+	.flags		= BI_BRAND_NOTE
 };
 
 SYSINIT(elf64, SI_SUB_EXEC, SI_ORDER_ANY,
diff --git a/sys/mips/mips/elf_machdep.c b/sys/mips/mips/elf_machdep.c
index 163d0ee..19cb1a1 100644
--- a/sys/mips/mips/elf_machdep.c
+++ b/sys/mips/mips/elf_machdep.c
@@ -86,7 +86,8 @@ static Elf32_Brandinfo freebsd_brand_info = {
 	.interp_path	= "/libexec/ld-elf.so.1",
 	.sysvec		= &elf32_freebsd_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= 0
+	.brand_note	= &elf32_freebsd_brandnote,
+	.flags		= BI_BRAND_NOTE
 };
 
 SYSINIT(elf32, SI_SUB_EXEC, SI_ORDER_ANY,
diff --git a/sys/mips/mips/elf_trampoline.c b/sys/mips/mips/elf_trampoline.c
deleted file mode 100644
index 0f2ccc9..0000000
--- a/sys/mips/mips/elf_trampoline.c
+++ /dev/null
@@ -1,133 +0,0 @@
-/*-
- * Copyright (c) 2005 Olivier Houchard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-#include <machine/asm.h>
-#include <sys/param.h>
-#include <sys/elf32.h>
-#include <sys/inflate.h>
-#include <machine/elf.h>
-#include <machine/cpufunc.h>
-#include <machine/stdarg.h>
-
-/*
- * Since we are compiled outside of the normal kernel build process, we
- * need to include opt_global.h manually.
- */
-#include "opt_global.h"
-#include "opt_kernname.h"
-
-extern char kernel_start[];
-extern char kernel_end[];
-
-static __inline void *
-memcpy(void *dst, const void *src, int len)
-{
-	const char *s = src;
-    	char *d = dst;
-
-	while (len) {
-		if (0 && len >= 4 && !((vm_offset_t)d & 3) &&
-		    !((vm_offset_t)s & 3)) {
-			*(uint32_t *)d = *(uint32_t *)s;
-			s += 4;
-			d += 4;
-			len -= 4;
-		} else {
-			*d++ = *s++;
-			len--;
-		}
-	}
-	return (dst);
-}
-
-static __inline void
-bzero(void *addr, int count)
-{
-	char *tmp = (char *)addr;
-
-	while (count > 0) {
-		if (count >= 4 && !((vm_offset_t)tmp & 3)) {
-			*(uint32_t *)tmp = 0;
-			tmp += 4;
-			count -= 4;
-		} else {
-			*tmp = 0;
-			tmp++;
-			count--;
-		}
-	}
-}
-
-/*
- * Relocate PT_LOAD segements of kernel ELF image to their respective
- * virtual addresses and return entry point
- */
-void *
-load_kernel(void * kstart)
-{
-	Elf32_Ehdr *eh;
-	Elf32_Phdr phdr[64] /* XXX */;
-	int i;
-	void *entry_point;
-	
-	eh = (Elf32_Ehdr *)kstart;
-	entry_point = (void*)eh->e_entry;
-	memcpy(phdr, (void *)(kstart + eh->e_phoff ),
-	    eh->e_phnum * sizeof(phdr[0]));
-
-	for (i = 0; i < eh->e_phnum; i++) {
-		volatile char c;
-
-		if (phdr[i].p_type != PT_LOAD)
-			continue;
-		
-		memcpy((void *)(phdr[i].p_vaddr),
-		    (void*)(kstart + phdr[i].p_offset), phdr[i].p_filesz);
-		/* Clean space from oversized segments, eg: bss. */
-		if (phdr[i].p_filesz < phdr[i].p_memsz)
-			bzero((void *)(phdr[i].p_vaddr + phdr[i].p_filesz), 
-			    phdr[i].p_memsz - phdr[i].p_filesz);
-	}
-
-	return entry_point;
-}
-
-void
-_startC(register_t a0, register_t a1, register_t a2, register_t a3)
-{
-	unsigned int * code;
-	int i;
-	void (*entry_point)(register_t, register_t, register_t, register_t);
-
-	/* 
-	 * Relocate segment to the predefined memory location
-	 * Most likely it will be KSEG0/KSEG1 address
-	 */
-	entry_point = load_kernel(kernel_start);
-
-	/* Pass saved registers to original _start */
-	entry_point(a0, a1, a2, a3);
-}
diff --git a/sys/mips/mips/inckern.S b/sys/mips/mips/inckern.S
deleted file mode 100644
index 8610196..0000000
--- a/sys/mips/mips/inckern.S
+++ /dev/null
@@ -1,34 +0,0 @@
-/*-
- * Copyright (c) 2005 Olivier Houchard.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "opt_kernname.h"
-
-#include <machine/asm.h>
-__FBSDID("$FreeBSD$")
-.section ".real_kernel","aw"
-.globl kernel_start;
-kernel_start:
-.incbin KERNNAME
-.globl kernel_end;
-kernel_end:
diff --git a/sys/mips/mips/nexus.c b/sys/mips/mips/nexus.c
index e96eeab..64cba66 100644
--- a/sys/mips/mips/nexus.c
+++ b/sys/mips/mips/nexus.c
@@ -249,8 +249,6 @@ nexus_hinted_child(device_t bus, const char *dname, int dunit)
 	long	maddr;
 	int	msize;
 	int	result;
-	int	irq;
-	int	mem_hints_count;
 
 	child = BUS_ADD_CHILD(bus, 0, dname, dunit);
 
@@ -258,34 +256,17 @@ nexus_hinted_child(device_t bus, const char *dname, int dunit)
 	 * Set hard-wired resources for hinted child using
 	 * specific RIDs.
 	 */
-	mem_hints_count = 0;
-	if (resource_long_value(dname, dunit, "maddr", &maddr) == 0)
-		mem_hints_count++;
-	if (resource_int_value(dname, dunit, "msize", &msize) == 0)
-		mem_hints_count++;
-
-	/* check if all info for mem resource has been provided */
-	if ((mem_hints_count > 0) && (mem_hints_count < 2)) {
-		printf("Either maddr or msize hint is missing for %s%d\n",
-		    dname, dunit);
-	} else if (mem_hints_count) {
-		dprintf("%s: discovered hinted child %s at maddr %p(%d)\n",
-		    __func__, device_get_nameunit(child),
-		    (void *)(intptr_t)maddr, msize);
-
-		result = bus_set_resource(child, SYS_RES_MEMORY, MIPS_MEM_RID,
-		    maddr, msize);
-		if (result != 0) {
-			device_printf(bus, 
-			    "warning: bus_set_resource() failed\n");
-		}
-	}
+	resource_long_value(dname, dunit, "maddr", &maddr);
+	resource_int_value(dname, dunit, "msize", &msize);
+
+	dprintf("%s: discovered hinted child %s at maddr %p(%d)\n",
+	    __func__, device_get_nameunit(child),
+	    (void *)(intptr_t)maddr, msize);
 
-	if (resource_int_value(dname, dunit, "irq", &irq) == 0) {
-		result = bus_set_resource(child, SYS_RES_IRQ, 0, irq, 1);
-		if (result != 0)
-			device_printf(bus,
-			    "warning: bus_set_resource() failed\n");
+	result = bus_set_resource(child, SYS_RES_MEMORY, MIPS_MEM_RID,
+	    maddr, msize);
+	if (result != 0) {
+		device_printf(bus, "warning: bus_set_resource() failed\n");
 	}
 }
 
diff --git a/sys/mips/sentry5/files.sentry5 b/sys/mips/sentry5/files.sentry5
index 7992509..61038e0 100644
--- a/sys/mips/sentry5/files.sentry5
+++ b/sys/mips/sentry5/files.sentry5
@@ -4,4 +4,11 @@
 # for USB 1.1 OHCI, Ethernet and IPSEC cores
 # which are believed to be devices we have drivers for
 # which just need to be tweaked for attachment to an SSB system bus.
+
 mips/sentry5/s5_machdep.c		standard
+dev/siba/siba.c					optional siba
+dev/siba/siba_pcib.c				optional siba pci
+mips/sentry5/siba_cc.c			optional siba
+
+# notyet
+#mips/sentry5/siba_mips.c			optional siba
diff --git a/sys/dev/siba/siba_cc.c b/sys/mips/sentry5/siba_cc.c
index cd78f0b..cd78f0b 100644
--- a/sys/dev/siba/siba_cc.c
+++ b/sys/mips/sentry5/siba_cc.c
diff --git a/sys/dev/siba/siba_mips.c b/sys/mips/sentry5/siba_mips.c
index 676da83..676da83 100644
--- a/sys/dev/siba/siba_mips.c
+++ b/sys/mips/sentry5/siba_mips.c
diff --git a/sys/dev/siba/siba_sdram.c b/sys/mips/sentry5/siba_sdram.c
index 8e74e53..8e74e53 100644
--- a/sys/dev/siba/siba_sdram.c
+++ b/sys/mips/sentry5/siba_sdram.c
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
index bb46b9c..969f704 100644
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -18,12 +18,12 @@ SUBDIR=	${_3dfx} \
 	aic7xxx \
 	aio \
 	${_amd} \
+	${_amdtemp} \
 	ale \
 	amr \
 	${_an} \
 	${_aout} \
 	${_apm} \
-	${_ar} \
 	${_arcmsr} \
 	${_arcnet} \
 	${_asmc} \
@@ -137,7 +137,6 @@ SUBDIR=	${_3dfx} \
 	${_ixgb} \
 	jme \
 	joy \
-	${_k8temp} \
 	kbdmux \
 	krpc \
 	le \
@@ -188,6 +187,7 @@ SUBDIR=	${_3dfx} \
 	nfsclient \
 	nfslockd \
 	nfsserver \
+	nfssvc \
 	nge \
 	nmdm \
 	${_nsp} \
@@ -219,7 +219,6 @@ SUBDIR=	${_3dfx} \
 	puc \
 	ral \
 	${_random} \
-	${_ray} \
 	rc4 \
 	${_rdma} \
 	re \
@@ -244,7 +243,6 @@ SUBDIR=	${_3dfx} \
 	${_speaker} \
 	${_splash} \
 	${_sppp} \
-	${_sr} \
 	ste \
 	${_stg} \
 	stge \
@@ -332,7 +330,6 @@ _amd=		amd
 _an=		an
 _aout=		aout
 _apm=		apm
-_ar=		ar
 _arcnet=	arcnet
 _bktr=		bktr
 _cardbus=	cardbus
@@ -385,7 +382,6 @@ _opensolaris=	opensolaris
 _pccard=	pccard
 _pcfclock=	pcfclock
 _pst=		pst
-_ray=		ray
 _rdma=		rdma
 _safe=		safe
 _sbni=		sbni
@@ -395,7 +391,6 @@ _sound=		sound
 _speaker=	speaker
 _splash=	splash
 _sppp=		sppp
-_sr=		sr
 _stg=		stg
 _streams=	streams
 _svr4=		svr4
@@ -409,6 +404,7 @@ _zfs=		zfs
 _aac=		aac
 _acpi=		acpi
 _ahb=		ahb
+_amdtemp=	amdtemp
 _arcmsr=	arcmsr
 _asmc=		asmc
 _asr=		asr
@@ -435,7 +431,6 @@ _iwifw=		iwifw
 _iwn=		iwn
 _iwnfw=		iwnfw
 _ixgb=		ixgb
-_k8temp=	k8temp
 _mly=		mly
 _nfe=		nfe
 _nve=		nve
@@ -464,6 +459,7 @@ _aac=		aac
 _acpi=		acpi
 _agp=		agp
 _an=		an
+_amdtemp=	amdtemp
 _arcmsr=	arcmsr
 _asmc=		asmc
 _cardbus=	cardbus
@@ -502,7 +498,6 @@ _ipwfw=		ipwfw
 _iwn=		iwn
 _iwnfw=		iwnfw
 _ixgb=		ixgb
-_k8temp=	k8temp
 _linprocfs=	linprocfs
 _linsysfs=	linsysfs
 _linux=		linux
@@ -587,6 +582,9 @@ _auxio=		auxio
 _em=		em
 _i2c=		i2c
 _igb=		igb
+.if ${MK_CDDL} != "no" || defined(ALL_MODULES)
+_opensolaris=	opensolaris
+.endif
 _sound=		sound
 .if ${MK_ZFS} != "no" || defined(ALL_MODULES)
 _zfs=		zfs
diff --git a/sys/modules/amdtemp/Makefile b/sys/modules/amdtemp/Makefile
new file mode 100644
index 0000000..1f7cff5
--- /dev/null
+++ b/sys/modules/amdtemp/Makefile
@@ -0,0 +1,8 @@
+# $FreeBSD$
+
+.PATH:	${.CURDIR}/../../dev/amdtemp
+
+KMOD=	amdtemp
+SRCS=	amdtemp.c bus_if.h device_if.h pci_if.h
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/ath/Makefile b/sys/modules/ath/Makefile
index 068d337..d494c45 100644
--- a/sys/modules/ath/Makefile
+++ b/sys/modules/ath/Makefile
@@ -38,7 +38,7 @@ KMOD=	if_ath
 SRCS=	if_ath.c if_ath_pci.c
 # NB: v3 eeprom support used by both AR5211 and AR5212; just include it
 SRCS+=	ah_osdep.c ah.c ah_regdomain.c ah_eeprom_v3.c
-SRCS+=	device_if.h bus_if.h pci_if.h opt_inet.h opt_ath.h opt_ah.h
+SRCS+=	device_if.h bus_if.h pci_if.h opt_inet.h opt_ath.h opt_ah.h opt_wlan.h
 
 #
 # AR5210 support; these are first generation 11a-only devices.
@@ -95,7 +95,7 @@ SRCS+=	ar2133.c
 # NB: rate control is bound to the driver by symbol names so only pick one
 .if ${ATH_RATE} == "sample"
 .PATH:	${.CURDIR}/../../dev/ath/ath_rate/sample
-SRCS+=	sample.c opt_wlan.h
+SRCS+=	sample.c
 .elif ${ATH_RATE} == "onoe"
 .PATH:	${.CURDIR}/../../dev/ath/ath_rate/onoe
 SRCS+=	onoe.c
diff --git a/sys/modules/cpufreq/Makefile b/sys/modules/cpufreq/Makefile
index 4e82008..108c2b1 100644
--- a/sys/modules/cpufreq/Makefile
+++ b/sys/modules/cpufreq/Makefile
@@ -12,7 +12,7 @@ SRCS+=	bus_if.h cpufreq_if.h device_if.h pci_if.h
 CFLAGS+= -I${.CURDIR}/../../contrib/dev/acpica
 
 SRCS+=	acpi_if.h opt_acpi.h
-SRCS+=	est.c p4tcc.c powernow.c
+SRCS+=	est.c hwpstate.c p4tcc.c powernow.c
 .endif
 
 .if ${MACHINE} == "i386"
diff --git a/sys/modules/dtrace/Makefile b/sys/modules/dtrace/Makefile
index 45c2b2f..671ccc9 100644
--- a/sys/modules/dtrace/Makefile
+++ b/sys/modules/dtrace/Makefile
@@ -4,6 +4,7 @@
 .include "Makefile.inc"
 
 SUBDIR=		dtmalloc	\
+		dtnfsclient	\
 		dtrace		\
 		dtraceall	\
 		dtrace_test	\
diff --git a/sys/modules/dtrace/dtnfsclient/Makefile b/sys/modules/dtrace/dtnfsclient/Makefile
new file mode 100644
index 0000000..5e26bba
--- /dev/null
+++ b/sys/modules/dtrace/dtnfsclient/Makefile
@@ -0,0 +1,13 @@
+# $FreeBSD$
+
+.PATH: ${.CURDIR}/../../../nfsclient
+
+KMOD=		dtnfsclient
+SRCS=		nfs_kdtrace.c
+SRCS+=		vnode_if.h
+
+CFLAGS+=	-I${.CURDIR}/../../../cddl/compat/opensolaris \
+		-I${.CURDIR}/../../../cddl/contrib/opensolaris/uts/common \
+		-I${.CURDIR}/../../..
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/dtrace/dtraceall/dtraceall.c b/sys/modules/dtrace/dtraceall/dtraceall.c
index 4f7bdfe..6a71797 100644
--- a/sys/modules/dtrace/dtraceall/dtraceall.c
+++ b/sys/modules/dtrace/dtraceall/dtraceall.c
@@ -65,6 +65,7 @@ MODULE_DEPEND(dtraceall, cyclic, 1, 1, 1);
 MODULE_DEPEND(dtraceall, opensolaris, 1, 1, 1);
 MODULE_DEPEND(dtraceall, dtrace, 1, 1, 1);
 MODULE_DEPEND(dtraceall, dtmalloc, 1, 1, 1);
+MODULE_DEPEND(dtraceall, dtnfsclient, 1, 1, 1);
 #if defined(__amd64__) || defined(__i386__)
 MODULE_DEPEND(dtraceall, fbt, 1, 1, 1);
 #endif
diff --git a/sys/modules/geom/geom_vinum/Makefile b/sys/modules/geom/geom_vinum/Makefile
index fe14515..48ac962 100644
--- a/sys/modules/geom/geom_vinum/Makefile
+++ b/sys/modules/geom/geom_vinum/Makefile
@@ -3,10 +3,10 @@
 .PATH: ${.CURDIR}/../../../geom/vinum
 
 KMOD=	geom_vinum
-SRCS=	geom_vinum.c geom_vinum_drive.c geom_vinum_plex.c \
+SRCS=	geom_vinum.c geom_vinum_create.c geom_vinum_drive.c geom_vinum_plex.c \
 	geom_vinum_volume.c geom_vinum_subr.c geom_vinum_raid5.c \
 	geom_vinum_share.c geom_vinum_list.c geom_vinum_rm.c \
 	geom_vinum_init.c geom_vinum_state.c geom_vinum_rename.c \
-	geom_vinum_move.c
+	geom_vinum_move.c geom_vinum_events.c
 
 .include <bsd.kmod.mk>
diff --git a/sys/modules/ip6_mroute_mod/Makefile b/sys/modules/ip6_mroute_mod/Makefile
new file mode 100644
index 0000000..f361c9d
--- /dev/null
+++ b/sys/modules/ip6_mroute_mod/Makefile
@@ -0,0 +1,19 @@
+# $FreeBSD$
+
+.include <bsd.own.mk>
+
+.PATH: ${.CURDIR}/../../netinet6
+
+KMOD=	ip6_mroute
+
+SRCS=	ip6_mroute.c
+SRCS+=	opt_inet6.h opt_mac.h opt_mrouting.h
+
+.if !defined(KERNBUILDDIR)
+opt_inet6.h:
+	echo "#define INET6 1" > ${.TARGET}
+opt_mrouting.h:
+	echo "#define MROUTING 1" > ${.TARGET}
+.endif
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/ip_mroute_mod/Makefile b/sys/modules/ip_mroute_mod/Makefile
index 3ca0a3dc..141eceb 100644
--- a/sys/modules/ip_mroute_mod/Makefile
+++ b/sys/modules/ip_mroute_mod/Makefile
@@ -8,21 +8,12 @@ KMOD=	ip_mroute
 
 SRCS=	ip_mroute.c
 SRCS+=	opt_inet.h opt_mac.h opt_mrouting.h opt_route.h
-SRCS+=	opt_inet6.h
-
-.if ${MK_INET6_SUPPORT} != "no"
-SRCS+=	ip6_mroute.c
-.endif
 
 .if !defined(KERNBUILDDIR)
 opt_inet.h:
 	echo "#define INET 1" > ${.TARGET}
 opt_mrouting.h:
 	echo "#define MROUTING 1" > ${.TARGET}
-.if ${MK_INET6_SUPPORT} != "no"
-opt_inet6.h:
-	echo "#define INET6 1" > ${.TARGET}
-.endif
 .endif
 
 .include <bsd.kmod.mk>
diff --git a/sys/modules/ipmi/Makefile b/sys/modules/ipmi/Makefile
index 3f45c73..7493b23 100644
--- a/sys/modules/ipmi/Makefile
+++ b/sys/modules/ipmi/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+SUBDIR+= ipmi_linux
+
 .PATH: ${.CURDIR}/../../dev/ipmi
 
 # XXX - ipmi_smbus and ipmi_ssif depend on smbus
diff --git a/sys/modules/ipmi/ipmi_linux/Makefile b/sys/modules/ipmi/ipmi_linux/Makefile
new file mode 100644
index 0000000..bbdef1b
--- /dev/null
+++ b/sys/modules/ipmi/ipmi_linux/Makefile
@@ -0,0 +1,8 @@
+# $FreeBSD$
+
+.PATH: ${.CURDIR}/../../../dev/ipmi
+
+KMOD=	ipmi_linux
+SRCS=	ipmi_linux.c
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/ixgbe/Makefile b/sys/modules/ixgbe/Makefile
index a40a909..e159f46 100644
--- a/sys/modules/ixgbe/Makefile
+++ b/sys/modules/ixgbe/Makefile
@@ -4,7 +4,8 @@ KMOD    = ixgbe
 SRCS    = device_if.h bus_if.h pci_if.h
 SRCS    += ixgbe.c
 # Shared source
-SRCS    += ixgbe_common.c ixgbe_api.c ixgbe_phy.c ixgbe_82598.c
+SRCS    += ixgbe_common.c ixgbe_api.c ixgbe_phy.c
+SRCS    += ixgbe_82599.c ixgbe_82598.c
 CFLAGS+= -I${.CURDIR}/../../dev/ixgbe
 
 clean:
diff --git a/sys/modules/k8temp/Makefile b/sys/modules/k8temp/Makefile
deleted file mode 100644
index 32909ad..0000000
--- a/sys/modules/k8temp/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-# $FreeBSD$
-
-.PATH:	${.CURDIR}/../../dev/k8temp
-
-KMOD=	k8temp
-SRCS=	k8temp.c bus_if.h device_if.h pci_if.h
-
-.include <bsd.kmod.mk>
diff --git a/sys/modules/linprocfs/Makefile b/sys/modules/linprocfs/Makefile
index a7cb129..62af13e 100644
--- a/sys/modules/linprocfs/Makefile
+++ b/sys/modules/linprocfs/Makefile
@@ -4,6 +4,7 @@
 
 KMOD=	linprocfs
 SRCS=	vnode_if.h \
+	device_if.h bus_if.h \
 	linprocfs.c \
 	opt_compat.h \
 	opt_route.h
diff --git a/sys/modules/mii/Makefile b/sys/modules/mii/Makefile
index 57647ea..6232b5e 100644
--- a/sys/modules/mii/Makefile
+++ b/sys/modules/mii/Makefile
@@ -3,7 +3,8 @@
 .PATH: ${.CURDIR}/../../dev/mii
 
 KMOD=	miibus
-SRCS=	acphy.c amphy.c atphy.c bmtphy.c brgphy.c bus_if.h ciphy.c device_if.h
+SRCS=	acphy.c amphy.c atphy.c axphy.c bmtphy.c brgphy.c bus_if.h
+SRCS+=	ciphy.c device_if.h
 SRCS+=	e1000phy.c exphy.c gentbi.c icsphy.c inphy.c ip1000phy.c jmphy.c
 SRCS+=	lxtphy.c miibus_if.c miibus_if.h mii.c miidevs.h mii_physubr.c
 SRCS+=	mlphy.c nsgphy.c nsphy.c nsphyter.c pci_if.h pnaphy.c qsphy.c
diff --git a/sys/modules/netgraph/Makefile b/sys/modules/netgraph/Makefile
index 760604d..2bae5ef 100644
--- a/sys/modules/netgraph/Makefile
+++ b/sys/modules/netgraph/Makefile
@@ -44,8 +44,6 @@ SUBDIR=	async \
 	source \
 	split \
 	sppp \
-	${_sync_ar} \
-	${_sync_sr} \
 	tag \
 	tcpmss \
 	tee \
@@ -54,11 +52,6 @@ SUBDIR=	async \
 	vjc \
 	vlan
 
-.if ${MACHINE_ARCH} == "i386"
-_sync_ar=	sync_ar
-_sync_sr=	sync_sr
-.endif
-
 .if ${MK_BLUETOOTH} != "no" || defined(ALL_MODULES)
 _bluetooth=	bluetooth
 .endif
diff --git a/sys/modules/nfsclient/Makefile b/sys/modules/nfsclient/Makefile
index 11742f5..a420502 100644
--- a/sys/modules/nfsclient/Makefile
+++ b/sys/modules/nfsclient/Makefile
@@ -10,7 +10,7 @@ SRCS=	vnode_if.h \
 	opt_inet.h opt_nfs.h opt_bootp.h opt_nfsroot.h
 SRCS+=	nfs4_dev.c nfs4_idmap.c nfs4_socket.c nfs4_subs.c \
 	nfs4_vfs_subs.c  nfs4_vfsops.c nfs4_vn_subs.c nfs4_vnops.c
-SRCS+=	opt_inet6.h opt_kgssapi.h opt_route.h
+SRCS+=	opt_inet6.h opt_kdtrace.h opt_kgssapi.h opt_route.h
 
 # USE THE RPCCLNT:
 CFLAGS+= -DRPCCLNT_DEBUG
diff --git a/sys/modules/nfssvc/Makefile b/sys/modules/nfssvc/Makefile
new file mode 100644
index 0000000..0073d13
--- /dev/null
+++ b/sys/modules/nfssvc/Makefile
@@ -0,0 +1,9 @@
+
+# $FreeBSD$
+
+.PATH: ${.CURDIR}/../../nfs
+KMOD=	nfssvc
+SRCS=	nfs_nfssvc.c \
+	opt_nfs.h
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/opensolaris/Makefile b/sys/modules/opensolaris/Makefile
index 5e2195a..a1df2a8 100644
--- a/sys/modules/opensolaris/Makefile
+++ b/sys/modules/opensolaris/Makefile
@@ -7,7 +7,7 @@ SRCS=		opensolaris.c		\
 		opensolaris_cmn_err.c	\
 		opensolaris_kmem.c
 
-.if ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "amd64" || ${MACHINE_ARCH} == "ia64"
+.if ${MACHINE_ARCH} == "i386" || ${MACHINE_ARCH} == "amd64" || ${MACHINE_ARCH} == "ia64" || ${MACHINE_ARCH} == "sparc64"
 .PATH:	${.CURDIR}/../../cddl/contrib/opensolaris/common/atomic/${MACHINE_ARCH}
 SRCS+=		atomic.S
 .else
diff --git a/sys/modules/usb/Makefile b/sys/modules/usb/Makefile
index a002111..44f55c6 100644
--- a/sys/modules/usb/Makefile
+++ b/sys/modules/usb/Makefile
@@ -28,8 +28,8 @@
 SUBDIR = usb
 #SUBDIR += ubt bluetooth_ng ubtfw
 SUBDIR += ehci musb ohci uhci uss820dci ${_at91dci} ${_atmegadci}
-SUBDIR += rum ural zyd
-SUBDIR += uhid ukbd ums udbp ufm uscanner
+SUBDIR += rum uath ural zyd
+SUBDIR += uhid ukbd ums udbp ufm
 SUBDIR += ucom u3g uark ubsa ubser uchcom ucycom ufoma uftdi ugensa uipaq ulpt \
 	  umct umodem umoscom uplcom uslcom uvisor uvscom
 SUBDIR += uether aue axe cdce cue kue rue udav
diff --git a/sys/modules/usb/uath/Makefile b/sys/modules/usb/uath/Makefile
new file mode 100644
index 0000000..a734dfc
--- /dev/null
+++ b/sys/modules/usb/uath/Makefile
@@ -0,0 +1,10 @@
+# $FreeBSD$
+
+.PATH: ${.CURDIR}/../../../dev/usb/wlan
+
+KMOD    = if_uath
+SRCS    = if_uath.c if_uathvar.h if_uathreg.h	\
+	  bus_if.h device_if.h \
+	  opt_bus.h opt_usb.h usb_if.h usbdevs.h
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/usb/usb/Makefile b/sys/modules/usb/usb/Makefile
index 39b8a99..5b0104e 100644
--- a/sys/modules/usb/usb/Makefile
+++ b/sys/modules/usb/usb/Makefile
@@ -36,7 +36,7 @@ SRCS=	bus_if.h device_if.h usb_if.h usb_if.c vnode_if.h \
 	usb_busdma.c usb_controller.c usb_compat_linux.c usb_core.c usb_debug.c \
 	usb_dev.c usb_device.c usb_dynamic.c usb_error.c usb_generic.c \
 	usb_handle_request.c usb_hid.c usb_hub.c usb_lookup.c usb_mbuf.c \
-	usb_msctest.c usb_parse.c usb_process.c usb_request.c usb_sw_transfer.c \
+	usb_msctest.c usb_parse.c usb_process.c usb_request.c \
 	usb_transfer.c usb_util.c 
 
 .include <bsd.kmod.mk>
diff --git a/sys/modules/usb/uscanner/Makefile b/sys/modules/usb/uscanner/Makefile
deleted file mode 100644
index 4534a3e..0000000
--- a/sys/modules/usb/uscanner/Makefile
+++ /dev/null
@@ -1,36 +0,0 @@
-#
-# $FreeBSD$
-#
-# Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
-#
-
-S=     ${.CURDIR}/../../..
-
-.PATH: $S/dev/usb/image
-
-KMOD=	uscanner
-SRCS=	opt_bus.h opt_usb.h device_if.h bus_if.h usb_if.h vnode_if.h usbdevs.h \
-	uscanner.c
-
-.include <bsd.kmod.mk>
diff --git a/sys/net/bpf.h b/sys/net/bpf.h
index 3d3d079..726483a 100644
--- a/sys/net/bpf.h
+++ b/sys/net/bpf.h
@@ -726,6 +726,94 @@ struct bpf_zbuf_header {
 #define DLT_BLUETOOTH_HCI_H4_WITH_PHDR	201
 
 /*
+ * AX.25 packet with a 1-byte KISS header; see
+ *
+ *      http://www.ax25.net/kiss.htm
+ *
+ * as per Richard Stearn <richard@rns-stearn.demon.co.uk>.
+ */
+#define DLT_AX25_KISS           202
+
+/*
+ * LAPD packets from an ISDN channel, starting with the address field,
+ * with no pseudo-header.
+ * Requested by Varuna De Silva <varunax@gmail.com>.
+ */
+#define DLT_LAPD                203
+
+/*
+ * Variants of various link-layer headers, with a one-byte direction
+ * pseudo-header prepended - zero means "received by this host",
+ * non-zero (any non-zero value) means "sent by this host" - as per
+ * Will Barker <w.barker@zen.co.uk>.
+ */
+#define DLT_PPP_WITH_DIR        204     /* PPP - don't confuse with DLT_PPP_WITH_DIRECTION */
+#define DLT_C_HDLC_WITH_DIR     205     /* Cisco HDLC */
+#define DLT_FRELAY_WITH_DIR     206     /* Frame Relay */
+#define DLT_LAPB_WITH_DIR       207     /* LAPB */
+
+/*
+ * 208 is reserved for an as-yet-unspecified proprietary link-layer
+ * type, as requested by Will Barker.
+ */
+
+/*
+ * IPMB with a Linux-specific pseudo-header; as requested by Alexey Neyman
+ * <avn@pigeonpoint.com>.
+ */
+#define DLT_IPMB_LINUX          209
+
+/*
+ * FlexRay automotive bus - http://www.flexray.com/ - as requested
+ * by Hannes Kaelber <hannes.kaelber@x2e.de>.
+ */
+#define DLT_FLEXRAY             210
+
+/*
+ * Media Oriented Systems Transport (MOST) bus for multimedia
+ * transport - http://www.mostcooperation.com/ - as requested
+ * by Hannes Kaelber <hannes.kaelber@x2e.de>.
+ */
+#define DLT_MOST                211
+
+/*
+ * Local Interconnect Network (LIN) bus for vehicle networks -
+ * http://www.lin-subbus.org/ - as requested by Hannes Kaelber
+ * <hannes.kaelber@x2e.de>.
+ */
+#define DLT_LIN                 212
+
+/*
+ * X2E-private data link type used for serial line capture,
+ * as requested by Hannes Kaelber <hannes.kaelber@x2e.de>.
+ */
+#define DLT_X2E_SERIAL          213
+
+/*
+ * X2E-private data link type used for the Xoraya data logger
+ * family, as requested by Hannes Kaelber <hannes.kaelber@x2e.de>.
+ */
+#define DLT_X2E_XORAYA          214
+
+/*
+ * IEEE 802.15.4, exactly as it appears in the spec (no padding, no
+ * nothing), but with the PHY-level data for non-ASK PHYs (4 octets
+ * of 0 as preamble, one octet of SFD, one octet of frame length+
+ * reserved bit, and then the MAC-layer data, starting with the
+ * frame control field).
+ *
+ * Requested by Max Filippov <jcmvbkbc@gmail.com>.
+ */
+#define DLT_IEEE802_15_4_NONASK_PHY     215
+
+/*
+ * DLT and savefile link type values are split into a class and
+ * a member of that class.  A class value of 0 indicates a regular
+ * DLT_/LINKTYPE_ value.
+ */
+#define DLT_CLASS(x)            ((x) & 0x03ff0000)
+
+/*
  * The instruction encodings.
  */
 /* instruction classes */
diff --git a/sys/net/bsd_comp.c b/sys/net/bsd_comp.c
deleted file mode 100644
index 462567b..0000000
--- a/sys/net/bsd_comp.c
+++ /dev/null
@@ -1,1117 +0,0 @@
-/*-
- * Because this code is derived from the 4.3BSD compress source:
- *
- *
- * Copyright (c) 1985, 1986 The Regents of the University of California.
- * All rights reserved.
- *
- * This code is derived from software contributed to Berkeley by
- * James A. Woods, derived from original work by Spencer Thomas
- * and Joseph Orost.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-/*
- * This version is for use with mbufs on BSD-derived systems.
- *
- * $FreeBSD$
- */
-
-#include "opt_mac.h"
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/lock.h>
-#include <sys/malloc.h>
-#include <sys/mbuf.h>
-#include <sys/mutex.h>
-
-#include <net/ppp_defs.h>
-
-#define PACKETPTR	struct mbuf *
-#include <net/ppp_comp.h>
-
-#include <security/mac/mac_framework.h>
-
-/*
- * PPP "BSD compress" compression
- *  The differences between this compression and the classic BSD LZW
- *  source are obvious from the requirement that the classic code worked
- *  with files while this handles arbitrarily long streams that
- *  are broken into packets.  They are:
- *
- *	When the code size expands, a block of junk is not emitted by
- *	    the compressor and not expected by the decompressor.
- *
- *	New codes are not necessarily assigned every time an old
- *	    code is output by the compressor.  This is because a packet
- *	    end forces a code to be emitted, but does not imply that a
- *	    new sequence has been seen.
- *
- *	The compression ratio is checked at the first end of a packet
- *	    after the appropriate gap.	Besides simplifying and speeding
- *	    things up, this makes it more likely that the transmitter
- *	    and receiver will agree when the dictionary is cleared when
- *	    compression is not going well.
- */
-
-/*
- * A dictionary for doing BSD compress.
- */
-struct bsd_db {
-    int	    totlen;			/* length of this structure */
-    u_int   hsize;			/* size of the hash table */
-    u_char  hshift;			/* used in hash function */
-    u_char  n_bits;			/* current bits/code */
-    u_char  maxbits;
-    u_char  debug;
-    u_char  unit;
-    u_int16_t seqno;			/* sequence # of next packet */
-    u_int   hdrlen;			/* header length to preallocate */
-    u_int   mru;
-    u_int   maxmaxcode;			/* largest valid code */
-    u_int   max_ent;			/* largest code in use */
-    u_int   in_count;			/* uncompressed bytes, aged */
-    u_int   bytes_out;			/* compressed bytes, aged */
-    u_int   ratio;			/* recent compression ratio */
-    u_int   checkpoint;			/* when to next check the ratio */
-    u_int   clear_count;		/* times dictionary cleared */
-    u_int   incomp_count;		/* incompressible packets */
-    u_int   incomp_bytes;		/* incompressible bytes */
-    u_int   uncomp_count;		/* uncompressed packets */
-    u_int   uncomp_bytes;		/* uncompressed bytes */
-    u_int   comp_count;			/* compressed packets */
-    u_int   comp_bytes;			/* compressed bytes */
-    u_int16_t *lens;			/* array of lengths of codes */
-    struct bsd_dict {
-	union {				/* hash value */
-	    u_int32_t	fcode;
-	    struct {
-#if BYTE_ORDER == LITTLE_ENDIAN
-		u_int16_t prefix;	/* preceding code */
-		u_char	suffix;		/* last character of new code */
-		u_char	pad;
-#else
-		u_char	pad;
-		u_char	suffix;		/* last character of new code */
-		u_int16_t prefix;	/* preceding code */
-#endif
-	    } hs;
-	} f;
-	u_int16_t codem1;		/* output of hash table -1 */
-	u_int16_t cptr;			/* map code to hash table entry */
-    } dict[1];
-};
-
-#define BSD_OVHD	2		/* BSD compress overhead/packet */
-#define BSD_INIT_BITS	BSD_MIN_BITS
-
-static void	bsd_clear(struct bsd_db *db);
-static int	bsd_check(struct bsd_db *db);
-static void	*bsd_alloc(u_char *options, int opt_len, int decomp);
-static int	bsd_init(struct bsd_db *db, u_char *options, int opt_len,
-		    int unit, int hdrlen, int mru, int debug, int decomp);
-static void	*bsd_comp_alloc(u_char *options, int opt_len);
-static void	*bsd_decomp_alloc(u_char *options, int opt_len);
-static void	bsd_free(void *state);
-static int	bsd_comp_init(void *state, u_char *options, int opt_len,
-		    int unit, int hdrlen, int debug);
-static int	bsd_decomp_init(void *state, u_char *options, int opt_len,
-		    int unit, int hdrlen, int mru, int debug);
-static int	bsd_compress(void *state, struct mbuf **mret, struct mbuf *mp,
-		    int slen, int maxolen);
-static void	bsd_incomp(void *state, struct mbuf *dmsg);
-static int	bsd_decompress(void *state, struct mbuf *cmp,
-		    struct mbuf **dmpp);
-static void	bsd_reset(void *state);
-static void	bsd_comp_stats(void *state, struct compstat *stats);
-
-/*
- * Procedures exported to if_ppp.c.
- */
-struct compressor ppp_bsd_compress = {
-    CI_BSD_COMPRESS,		/* compress_proto */
-    bsd_comp_alloc,		/* comp_alloc */
-    bsd_free,			/* comp_free */
-    bsd_comp_init,		/* comp_init */
-    bsd_reset,			/* comp_reset */
-    bsd_compress,		/* compress */
-    bsd_comp_stats,		/* comp_stat */
-    bsd_decomp_alloc,		/* decomp_alloc */
-    bsd_free,			/* decomp_free */
-    bsd_decomp_init,		/* decomp_init */
-    bsd_reset,			/* decomp_reset */
-    bsd_decompress,		/* decompress */
-    bsd_incomp,			/* incomp */
-    bsd_comp_stats,		/* decomp_stat */
-};
-
-/*
- * the next two codes should not be changed lightly, as they must not
- * lie within the contiguous general code space.
- */
-#define CLEAR	256			/* table clear output code */
-#define FIRST	257			/* first free entry */
-#define LAST	255
-
-#define MAXCODE(b)	((1 << (b)) - 1)
-#define BADCODEM1	MAXCODE(BSD_MAX_BITS)
-
-#define BSD_HASH(prefix,suffix,hshift)	((((u_int32_t)(suffix)) << (hshift)) \
-					 ^ (u_int32_t)(prefix))
-#define BSD_KEY(prefix,suffix)		((((u_int32_t)(suffix)) << 16) \
-					 + (u_int32_t)(prefix))
-
-#define CHECK_GAP	10000		/* Ratio check interval */
-
-#define RATIO_SCALE_LOG	8
-#define RATIO_SCALE	(1<<RATIO_SCALE_LOG)
-#define RATIO_MAX	(0x7fffffff>>RATIO_SCALE_LOG)
-
-/*
- * clear the dictionary
- */
-static void
-bsd_clear(db)
-    struct bsd_db *db;
-{
-    db->clear_count++;
-    db->max_ent = FIRST-1;
-    db->n_bits = BSD_INIT_BITS;
-    db->ratio = 0;
-    db->bytes_out = 0;
-    db->in_count = 0;
-    db->checkpoint = CHECK_GAP;
-}
-
-/*
- * If the dictionary is full, then see if it is time to reset it.
- *
- * Compute the compression ratio using fixed-point arithmetic
- * with 8 fractional bits.
- *
- * Since we have an infinite stream instead of a single file,
- * watch only the local compression ratio.
- *
- * Since both peers must reset the dictionary at the same time even in
- * the absence of CLEAR codes (while packets are incompressible), they
- * must compute the same ratio.
- */
-static int				/* 1=output CLEAR */
-bsd_check(db)
-    struct bsd_db *db;
-{
-    u_int new_ratio;
-
-    if (db->in_count >= db->checkpoint) {
-	/* age the ratio by limiting the size of the counts */
-	if (db->in_count >= RATIO_MAX
-	    || db->bytes_out >= RATIO_MAX) {
-	    db->in_count -= db->in_count/4;
-	    db->bytes_out -= db->bytes_out/4;
-	}
-
-	db->checkpoint = db->in_count + CHECK_GAP;
-
-	if (db->max_ent >= db->maxmaxcode) {
-	    /* Reset the dictionary only if the ratio is worse,
-	     * or if it looks as if it has been poisoned
-	     * by incompressible data.
-	     *
-	     * This does not overflow, because
-	     *	db->in_count <= RATIO_MAX.
-	     */
-	    new_ratio = db->in_count << RATIO_SCALE_LOG;
-	    if (db->bytes_out != 0)
-		new_ratio /= db->bytes_out;
-
-	    if (new_ratio < db->ratio || new_ratio < 1 * RATIO_SCALE) {
-		bsd_clear(db);
-		return 1;
-	    }
-	    db->ratio = new_ratio;
-	}
-    }
-    return 0;
-}
-
-/*
- * Return statistics.
- */
-static void
-bsd_comp_stats(state, stats)
-    void *state;
-    struct compstat *stats;
-{
-    struct bsd_db *db = (struct bsd_db *) state;
-    u_int out;
-
-    stats->unc_bytes = db->uncomp_bytes;
-    stats->unc_packets = db->uncomp_count;
-    stats->comp_bytes = db->comp_bytes;
-    stats->comp_packets = db->comp_count;
-    stats->inc_bytes = db->incomp_bytes;
-    stats->inc_packets = db->incomp_count;
-    stats->ratio = db->in_count;
-    out = db->bytes_out;
-    if (stats->ratio <= 0x7fffff)
-	stats->ratio <<= 8;
-    else
-	out >>= 8;
-    if (out != 0)
-	stats->ratio /= out;
-}
-
-/*
- * Reset state, as on a CCP ResetReq.
- */
-static void
-bsd_reset(state)
-    void *state;
-{
-    struct bsd_db *db = (struct bsd_db *) state;
-
-    db->seqno = 0;
-    bsd_clear(db);
-    db->clear_count = 0;
-}
-
-/*
- * Allocate space for a (de) compressor.
- */
-static void *
-bsd_alloc(options, opt_len, decomp)
-    u_char *options;
-    int opt_len, decomp;
-{
-    int bits;
-    u_int newlen, hsize, hshift, maxmaxcode;
-    struct bsd_db *db;
-
-    if (opt_len < CILEN_BSD_COMPRESS || options[0] != CI_BSD_COMPRESS
-	|| options[1] != CILEN_BSD_COMPRESS
-	|| BSD_VERSION(options[2]) != BSD_CURRENT_VERSION)
-	return NULL;
-    bits = BSD_NBITS(options[2]);
-    switch (bits) {
-    case 9:			/* needs 82152 for both directions */
-    case 10:			/* needs 84144 */
-    case 11:			/* needs 88240 */
-    case 12:			/* needs 96432 */
-	hsize = 5003;
-	hshift = 4;
-	break;
-    case 13:			/* needs 176784 */
-	hsize = 9001;
-	hshift = 5;
-	break;
-    case 14:			/* needs 353744 */
-	hsize = 18013;
-	hshift = 6;
-	break;
-    case 15:			/* needs 691440 */
-	hsize = 35023;
-	hshift = 7;
-	break;
-    case 16:			/* needs 1366160--far too much, */
-	/* hsize = 69001; */	/* and 69001 is too big for cptr */
-	/* hshift = 8; */	/* in struct bsd_db */
-	/* break; */
-    default:
-	return NULL;
-    }
-
-    maxmaxcode = MAXCODE(bits);
-    newlen = sizeof(*db) + (hsize-1) * (sizeof(db->dict[0]));
-    db = malloc(newlen, M_DEVBUF, M_NOWAIT);
-    if (!db)
-	return NULL;
-    bzero(db, sizeof(*db) - sizeof(db->dict));
-
-    if (!decomp) {
-	db->lens = NULL;
-    } else {
-	db->lens = malloc((maxmaxcode+1) * sizeof(db->lens[0]),
-	       M_DEVBUF, M_NOWAIT);
-	if (!db->lens) {
-	    free(db, M_DEVBUF);
-	    return NULL;
-	}
-    }
-
-    db->totlen = newlen;
-    db->hsize = hsize;
-    db->hshift = hshift;
-    db->maxmaxcode = maxmaxcode;
-    db->maxbits = bits;
-
-    return (void *) db;
-}
-
-static void
-bsd_free(state)
-    void *state;
-{
-    struct bsd_db *db = (struct bsd_db *) state;
-
-    if (db->lens)
-	free(db->lens, M_DEVBUF);
-    free(db, M_DEVBUF);
-}
-
-static void *
-bsd_comp_alloc(options, opt_len)
-    u_char *options;
-    int opt_len;
-{
-    return bsd_alloc(options, opt_len, 0);
-}
-
-static void *
-bsd_decomp_alloc(options, opt_len)
-    u_char *options;
-    int opt_len;
-{
-    return bsd_alloc(options, opt_len, 1);
-}
-
-/*
- * Initialize the database.
- */
-static int
-bsd_init(db, options, opt_len, unit, hdrlen, mru, debug, decomp)
-    struct bsd_db *db;
-    u_char *options;
-    int opt_len, unit, hdrlen, mru, debug, decomp;
-{
-    int i;
-
-    if (opt_len < CILEN_BSD_COMPRESS || options[0] != CI_BSD_COMPRESS
-	|| options[1] != CILEN_BSD_COMPRESS
-	|| BSD_VERSION(options[2]) != BSD_CURRENT_VERSION
-	|| BSD_NBITS(options[2]) != db->maxbits
-	|| (decomp && db->lens == NULL))
-	return 0;
-
-    if (decomp) {
-	i = LAST+1;
-	while (i != 0)
-	    db->lens[--i] = 1;
-    }
-    i = db->hsize;
-    while (i != 0) {
-	db->dict[--i].codem1 = BADCODEM1;
-	db->dict[i].cptr = 0;
-    }
-
-    db->unit = unit;
-    db->hdrlen = hdrlen;
-    db->mru = mru;
-#ifndef DEBUG
-    if (debug)
-#endif
-	db->debug = 1;
-
-    bsd_reset(db);
-
-    return 1;
-}
-
-static int
-bsd_comp_init(state, options, opt_len, unit, hdrlen, debug)
-    void *state;
-    u_char *options;
-    int opt_len, unit, hdrlen, debug;
-{
-    return bsd_init((struct bsd_db *) state, options, opt_len,
-		    unit, hdrlen, 0, debug, 0);
-}
-
-static int
-bsd_decomp_init(state, options, opt_len, unit, hdrlen, mru, debug)
-    void *state;
-    u_char *options;
-    int opt_len, unit, hdrlen, mru, debug;
-{
-    return bsd_init((struct bsd_db *) state, options, opt_len,
-		    unit, hdrlen, mru, debug, 1);
-}
-
-
-/*
- * compress a packet
- *	One change from the BSD compress command is that when the
- *	code size expands, we do not output a bunch of padding.
- */
-static int					/* new slen */
-bsd_compress(state, mret, mp, slen, maxolen)
-    void *state;
-    struct mbuf **mret;		/* return compressed mbuf chain here */
-    struct mbuf *mp;		/* from here */
-    int slen;			/* uncompressed length */
-    int maxolen;		/* max compressed length */
-{
-    struct bsd_db *db = (struct bsd_db *) state;
-    int hshift = db->hshift;
-    u_int max_ent = db->max_ent;
-    u_int n_bits = db->n_bits;
-    u_int bitno = 32;
-    u_int32_t accm = 0, fcode;
-    struct bsd_dict *dictp;
-    u_char c;
-    int hval, disp, ent, ilen;
-    u_char *rptr, *wptr;
-    u_char *cp_end;
-    int olen;
-    struct mbuf *m;
-
-#define PUTBYTE(v) {					\
-    ++olen;						\
-    if (wptr) {						\
-	*wptr++ = (v);					\
-	if (wptr >= cp_end) {				\
-	    m->m_len = wptr - mtod(m, u_char *);	\
-	    MGET(m->m_next, M_DONTWAIT, MT_DATA);	\
-	    m = m->m_next;				\
-	    if (m) {					\
-		m->m_len = 0;				\
-		if (maxolen - olen > MLEN)		\
-		    MCLGET(m, M_DONTWAIT);		\
-		wptr = mtod(m, u_char *);		\
-		cp_end = wptr + M_TRAILINGSPACE(m);	\
-	    } else					\
-		wptr = NULL;				\
-	}						\
-    }							\
-}
-
-#define OUTPUT(ent) {					\
-    bitno -= n_bits;					\
-    accm |= ((ent) << bitno);				\
-    do {						\
-	PUTBYTE(accm >> 24);				\
-	accm <<= 8;					\
-	bitno += 8;					\
-    } while (bitno <= 24);				\
-}
-
-    /*
-     * If the protocol is not in the range we're interested in,
-     * just return without compressing the packet.  If it is,
-     * the protocol becomes the first byte to compress.
-     */
-    rptr = mtod(mp, u_char *);
-    ent = PPP_PROTOCOL(rptr);
-    if (ent < 0x21 || ent > 0xf9) {
-	*mret = NULL;
-	return slen;
-    }
-
-    /* Don't generate compressed packets which are larger than
-       the uncompressed packet. */
-    if (maxolen > slen)
-	maxolen = slen;
-
-    /* Allocate one mbuf to start with. */
-    MGET(m, M_DONTWAIT, MT_DATA);
-    *mret = m;
-    if (m != NULL) {
-	m->m_len = 0;
-	if (maxolen + db->hdrlen > MLEN)
-	    MCLGET(m, M_DONTWAIT);
-	m->m_data += db->hdrlen;
-	wptr = mtod(m, u_char *);
-	cp_end = wptr + M_TRAILINGSPACE(m);
-    } else
-	wptr = cp_end = NULL;
-
-    /*
-     * Copy the PPP header over, changing the protocol,
-     * and install the 2-byte packet sequence number.
-     */
-    if (wptr) {
-	*wptr++ = PPP_ADDRESS(rptr);	/* assumes the ppp header is */
-	*wptr++ = PPP_CONTROL(rptr);	/* all in one mbuf */
-	*wptr++ = 0;			/* change the protocol */
-	*wptr++ = PPP_COMP;
-	*wptr++ = db->seqno >> 8;
-	*wptr++ = db->seqno;
-    }
-    ++db->seqno;
-
-    olen = 0;
-    rptr += PPP_HDRLEN;
-    slen = mp->m_len - PPP_HDRLEN;
-    ilen = slen + 1;
-    for (;;) {
-	if (slen <= 0) {
-	    mp = mp->m_next;
-	    if (!mp)
-		break;
-	    rptr = mtod(mp, u_char *);
-	    slen = mp->m_len;
-	    if (!slen)
-		continue;   /* handle 0-length buffers */
-	    ilen += slen;
-	}
-
-	slen--;
-	c = *rptr++;
-	fcode = BSD_KEY(ent, c);
-	hval = BSD_HASH(ent, c, hshift);
-	dictp = &db->dict[hval];
-
-	/* Validate and then check the entry. */
-	if (dictp->codem1 >= max_ent)
-	    goto nomatch;
-	if (dictp->f.fcode == fcode) {
-	    ent = dictp->codem1+1;
-	    continue;	/* found (prefix,suffix) */
-	}
-
-	/* continue probing until a match or invalid entry */
-	disp = (hval == 0) ? 1 : hval;
-	do {
-	    hval += disp;
-	    if (hval >= db->hsize)
-		hval -= db->hsize;
-	    dictp = &db->dict[hval];
-	    if (dictp->codem1 >= max_ent)
-		goto nomatch;
-	} while (dictp->f.fcode != fcode);
-	ent = dictp->codem1 + 1;	/* finally found (prefix,suffix) */
-	continue;
-
-    nomatch:
-	OUTPUT(ent);		/* output the prefix */
-
-	/* code -> hashtable */
-	if (max_ent < db->maxmaxcode) {
-	    struct bsd_dict *dictp2;
-	    /* expand code size if needed */
-	    if (max_ent >= MAXCODE(n_bits))
-		db->n_bits = ++n_bits;
-
-	    /* Invalidate old hash table entry using
-	     * this code, and then take it over.
-	     */
-	    dictp2 = &db->dict[max_ent+1];
-	    if (db->dict[dictp2->cptr].codem1 == max_ent)
-		db->dict[dictp2->cptr].codem1 = BADCODEM1;
-	    dictp2->cptr = hval;
-	    dictp->codem1 = max_ent;
-	    dictp->f.fcode = fcode;
-
-	    db->max_ent = ++max_ent;
-	}
-	ent = c;
-    }
-
-    OUTPUT(ent);		/* output the last code */
-    db->bytes_out += olen;
-    db->in_count += ilen;
-    if (bitno < 32)
-	++db->bytes_out;	/* count complete bytes */
-
-    if (bsd_check(db))
-	OUTPUT(CLEAR);		/* do not count the CLEAR */
-
-    /*
-     * Pad dribble bits of last code with ones.
-     * Do not emit a completely useless byte of ones.
-     */
-    if (bitno != 32)
-	PUTBYTE((accm | (0xff << (bitno-8))) >> 24);
-
-    if (m != NULL) {
-	m->m_len = wptr - mtod(m, u_char *);
-	m->m_next = NULL;
-    }
-
-    /*
-     * Increase code size if we would have without the packet
-     * boundary and as the decompressor will.
-     */
-    if (max_ent >= MAXCODE(n_bits) && max_ent < db->maxmaxcode)
-	db->n_bits++;
-
-    db->uncomp_bytes += ilen;
-    ++db->uncomp_count;
-    if (olen + PPP_HDRLEN + BSD_OVHD > maxolen) {
-	/* throw away the compressed stuff if it is longer than uncompressed */
-	if (*mret != NULL) {
-	    m_freem(*mret);
-	    *mret = NULL;
-	}
-	++db->incomp_count;
-	db->incomp_bytes += ilen;
-    } else {
-	++db->comp_count;
-	db->comp_bytes += olen + BSD_OVHD;
-    }
-
-    return olen + PPP_HDRLEN + BSD_OVHD;
-#undef OUTPUT
-#undef PUTBYTE
-}
-
-
-/*
- * Update the "BSD Compress" dictionary on the receiver for
- * incompressible data by pretending to compress the incoming data.
- */
-static void
-bsd_incomp(state, dmsg)
-    void *state;
-    struct mbuf *dmsg;
-{
-    struct bsd_db *db = (struct bsd_db *) state;
-    u_int hshift = db->hshift;
-    u_int max_ent = db->max_ent;
-    u_int n_bits = db->n_bits;
-    struct bsd_dict *dictp;
-    u_int32_t fcode;
-    u_char c;
-    u_int32_t hval, disp;
-    int slen, ilen;
-    u_int bitno = 7;
-    u_char *rptr;
-    u_int ent;
-
-    /*
-     * If the protocol is not in the range we're interested in,
-     * just return without looking at the packet.  If it is,
-     * the protocol becomes the first byte to "compress".
-     */
-    rptr = mtod(dmsg, u_char *);
-    ent = PPP_PROTOCOL(rptr);
-    if (ent < 0x21 || ent > 0xf9)
-	return;
-
-    db->seqno++;
-    ilen = 1;		/* count the protocol as 1 byte */
-    rptr += PPP_HDRLEN;
-    slen = dmsg->m_len - PPP_HDRLEN;
-    for (;;) {
-	if (slen <= 0) {
-	    dmsg = dmsg->m_next;
-	    if (!dmsg)
-		break;
-	    rptr = mtod(dmsg, u_char *);
-	    slen = dmsg->m_len;
-	    continue;
-	}
-	ilen += slen;
-
-	do {
-	    c = *rptr++;
-	    fcode = BSD_KEY(ent, c);
-	    hval = BSD_HASH(ent, c, hshift);
-	    dictp = &db->dict[hval];
-
-	    /* validate and then check the entry */
-	    if (dictp->codem1 >= max_ent)
-		goto nomatch;
-	    if (dictp->f.fcode == fcode) {
-		ent = dictp->codem1+1;
-		continue;   /* found (prefix,suffix) */
-	    }
-
-	    /* continue probing until a match or invalid entry */
-	    disp = (hval == 0) ? 1 : hval;
-	    do {
-		hval += disp;
-		if (hval >= db->hsize)
-		    hval -= db->hsize;
-		dictp = &db->dict[hval];
-		if (dictp->codem1 >= max_ent)
-		    goto nomatch;
-	    } while (dictp->f.fcode != fcode);
-	    ent = dictp->codem1+1;
-	    continue;	/* finally found (prefix,suffix) */
-
-	nomatch:		/* output (count) the prefix */
-	    bitno += n_bits;
-
-	    /* code -> hashtable */
-	    if (max_ent < db->maxmaxcode) {
-		struct bsd_dict *dictp2;
-		/* expand code size if needed */
-		if (max_ent >= MAXCODE(n_bits))
-		    db->n_bits = ++n_bits;
-
-		/* Invalidate previous hash table entry
-		 * assigned this code, and then take it over.
-		 */
-		dictp2 = &db->dict[max_ent+1];
-		if (db->dict[dictp2->cptr].codem1 == max_ent)
-		    db->dict[dictp2->cptr].codem1 = BADCODEM1;
-		dictp2->cptr = hval;
-		dictp->codem1 = max_ent;
-		dictp->f.fcode = fcode;
-
-		db->max_ent = ++max_ent;
-		db->lens[max_ent] = db->lens[ent]+1;
-	    }
-	    ent = c;
-	} while (--slen != 0);
-    }
-    bitno += n_bits;		/* output (count) the last code */
-    db->bytes_out += bitno/8;
-    db->in_count += ilen;
-    (void)bsd_check(db);
-
-    ++db->incomp_count;
-    db->incomp_bytes += ilen;
-    ++db->uncomp_count;
-    db->uncomp_bytes += ilen;
-
-    /* Increase code size if we would have without the packet
-     * boundary and as the decompressor will.
-     */
-    if (max_ent >= MAXCODE(n_bits) && max_ent < db->maxmaxcode)
-	db->n_bits++;
-}
-
-
-/*
- * Decompress "BSD Compress".
- *
- * Because of patent problems, we return DECOMP_ERROR for errors
- * found by inspecting the input data and for system problems, but
- * DECOMP_FATALERROR for any errors which could possibly be said to
- * be being detected "after" decompression.  For DECOMP_ERROR,
- * we can issue a CCP reset-request; for DECOMP_FATALERROR, we may be
- * infringing a patent of Motorola's if we do, so we take CCP down
- * instead.
- *
- * Given that the frame has the correct sequence number and a good FCS,
- * errors such as invalid codes in the input most likely indicate a
- * bug, so we return DECOMP_FATALERROR for them in order to turn off
- * compression, even though they are detected by inspecting the input.
- */
-static int
-bsd_decompress(state, cmp, dmpp)
-    void *state;
-    struct mbuf *cmp, **dmpp;
-{
-    struct bsd_db *db = (struct bsd_db *) state;
-    u_int max_ent = db->max_ent;
-    u_int32_t accm = 0;
-    u_int bitno = 32;		/* 1st valid bit in accm */
-    u_int n_bits = db->n_bits;
-    u_int tgtbitno = 32-n_bits;	/* bitno when we have a code */
-    struct bsd_dict *dictp;
-    int explen, i, seq, len;
-    u_int incode, oldcode, finchar;
-    u_char *p, *rptr, *wptr;
-    struct mbuf *m, *dmp, *mret;
-    int adrs, ctrl, ilen;
-    int space, codelen, extra;
-
-    /*
-     * Save the address/control from the PPP header
-     * and then get the sequence number.
-     */
-    *dmpp = NULL;
-    rptr = mtod(cmp, u_char *);
-    adrs = PPP_ADDRESS(rptr);
-    ctrl = PPP_CONTROL(rptr);
-    rptr += PPP_HDRLEN;
-    len = cmp->m_len - PPP_HDRLEN;
-    seq = 0;
-    for (i = 0; i < 2; ++i) {
-	while (len <= 0) {
-	    cmp = cmp->m_next;
-	    if (cmp == NULL)
-		return DECOMP_ERROR;
-	    rptr = mtod(cmp, u_char *);
-	    len = cmp->m_len;
-	}
-	seq = (seq << 8) + *rptr++;
-	--len;
-    }
-
-    /*
-     * Check the sequence number and give up if it differs from
-     * the value we're expecting.
-     */
-    if (seq != db->seqno) {
-	if (db->debug)
-	    printf("bsd_decomp%d: bad sequence # %d, expected %d\n",
-		   db->unit, seq, db->seqno - 1);
-	return DECOMP_ERROR;
-    }
-    ++db->seqno;
-
-    /*
-     * Allocate one mbuf to start with.
-     */
-    MGETHDR(dmp, M_DONTWAIT, MT_DATA);
-    if (dmp == NULL)
-	return DECOMP_ERROR;
-    mret = dmp;
-    dmp->m_len = 0;
-    dmp->m_next = NULL;
-    MCLGET(dmp, M_DONTWAIT);
-    dmp->m_data += db->hdrlen;
-    wptr = mtod(dmp, u_char *);
-    space = M_TRAILINGSPACE(dmp) - PPP_HDRLEN + 1;
-#ifdef MAC
-    mac_mbuf_copy(cmp, dmp);
-#endif
-
-    /*
-     * Fill in the ppp header, but not the last byte of the protocol
-     * (that comes from the decompressed data).
-     */
-    wptr[0] = adrs;
-    wptr[1] = ctrl;
-    wptr[2] = 0;
-    wptr += PPP_HDRLEN - 1;
-
-    ilen = len;
-    oldcode = CLEAR;
-    explen = 0;
-    for (;;) {
-	if (len == 0) {
-	    cmp = cmp->m_next;
-	    if (!cmp)		/* quit at end of message */
-		break;
-	    rptr = mtod(cmp, u_char *);
-	    len = cmp->m_len;
-	    ilen += len;
-	    continue;		/* handle 0-length buffers */
-	}
-
-	/*
-	 * Accumulate bytes until we have a complete code.
-	 * Then get the next code, relying on the 32-bit,
-	 * unsigned accm to mask the result.
-	 */
-	bitno -= 8;
-	accm |= *rptr++ << bitno;
-	--len;
-	if (tgtbitno < bitno)
-	    continue;
-	incode = accm >> tgtbitno;
-	accm <<= n_bits;
-	bitno += n_bits;
-
-	if (incode == CLEAR) {
-	    /*
-	     * The dictionary must only be cleared at
-	     * the end of a packet.  But there could be an
-	     * empty mbuf at the end.
-	     */
-	    if (len > 0 || cmp->m_next != NULL) {
-		while ((cmp = cmp->m_next) != NULL)
-		    len += cmp->m_len;
-		if (len > 0) {
-		    m_freem(mret);
-		    if (db->debug)
-			printf("bsd_decomp%d: bad CLEAR\n", db->unit);
-		    return DECOMP_FATALERROR;	/* probably a bug */
-		}
-	    }
-	    bsd_clear(db);
-	    explen = ilen = 0;
-	    break;
-	}
-
-	if (incode > max_ent + 2 || incode > db->maxmaxcode
-	    || (incode > max_ent && oldcode == CLEAR)) {
-	    m_freem(mret);
-	    if (db->debug) {
-		printf("bsd_decomp%d: bad code 0x%x oldcode=0x%x ",
-		       db->unit, incode, oldcode);
-		printf("max_ent=0x%x explen=%d seqno=%d\n",
-		       max_ent, explen, db->seqno);
-	    }
-	    return DECOMP_FATALERROR;	/* probably a bug */
-	}
-
-	/* Special case for KwKwK string. */
-	if (incode > max_ent) {
-	    finchar = oldcode;
-	    extra = 1;
-	} else {
-	    finchar = incode;
-	    extra = 0;
-	}
-
-	codelen = db->lens[finchar];
-	explen += codelen + extra;
-	if (explen > db->mru + 1) {
-	    m_freem(mret);
-	    if (db->debug) {
-		printf("bsd_decomp%d: ran out of mru\n", db->unit);
-#ifdef DEBUG
-		while ((cmp = cmp->m_next) != NULL)
-		    len += cmp->m_len;
-		printf("  len=%d, finchar=0x%x, codelen=%d, explen=%d\n",
-		       len, finchar, codelen, explen);
-#endif
-	    }
-	    return DECOMP_FATALERROR;
-	}
-
-	/*
-	 * For simplicity, the decoded characters go in a single mbuf,
-	 * so we allocate a single extra cluster mbuf if necessary.
-	 */
-	if ((space -= codelen + extra) < 0) {
-	    dmp->m_len = wptr - mtod(dmp, u_char *);
-	    MGET(m, M_DONTWAIT, MT_DATA);
-	    if (m == NULL) {
-		m_freem(mret);
-		return DECOMP_ERROR;
-	    }
-	    m->m_len = 0;
-	    m->m_next = NULL;
-	    dmp->m_next = m;
-	    MCLGET(m, M_DONTWAIT);
-	    space = M_TRAILINGSPACE(m) - (codelen + extra);
-	    if (space < 0) {
-		/* now that's what I call *compression*. */
-		m_freem(mret);
-		return DECOMP_ERROR;
-	    }
-	    dmp = m;
-	    wptr = mtod(dmp, u_char *);
-	}
-
-	/*
-	 * Decode this code and install it in the decompressed buffer.
-	 */
-	p = (wptr += codelen);
-	while (finchar > LAST) {
-	    dictp = &db->dict[db->dict[finchar].cptr];
-#ifdef DEBUG
-	    if (--codelen <= 0 || dictp->codem1 != finchar-1)
-		goto bad;
-#endif
-	    *--p = dictp->f.hs.suffix;
-	    finchar = dictp->f.hs.prefix;
-	}
-	*--p = finchar;
-
-#ifdef DEBUG
-	if (--codelen != 0)
-	    printf("bsd_decomp%d: short by %d after code 0x%x, max_ent=0x%x\n",
-		   db->unit, codelen, incode, max_ent);
-#endif
-
-	if (extra)		/* the KwKwK case again */
-	    *wptr++ = finchar;
-
-	/*
-	 * If not first code in a packet, and
-	 * if not out of code space, then allocate a new code.
-	 *
-	 * Keep the hash table correct so it can be used
-	 * with uncompressed packets.
-	 */
-	if (oldcode != CLEAR && max_ent < db->maxmaxcode) {
-	    struct bsd_dict *dictp2;
-	    u_int32_t fcode;
-	    u_int32_t hval, disp;
-
-	    fcode = BSD_KEY(oldcode,finchar);
-	    hval = BSD_HASH(oldcode,finchar,db->hshift);
-	    dictp = &db->dict[hval];
-
-	    /* look for a free hash table entry */
-	    if (dictp->codem1 < max_ent) {
-		disp = (hval == 0) ? 1 : hval;
-		do {
-		    hval += disp;
-		    if (hval >= db->hsize)
-			hval -= db->hsize;
-		    dictp = &db->dict[hval];
-		} while (dictp->codem1 < max_ent);
-	    }
-
-	    /*
-	     * Invalidate previous hash table entry
-	     * assigned this code, and then take it over
-	     */
-	    dictp2 = &db->dict[max_ent+1];
-	    if (db->dict[dictp2->cptr].codem1 == max_ent) {
-		db->dict[dictp2->cptr].codem1 = BADCODEM1;
-	    }
-	    dictp2->cptr = hval;
-	    dictp->codem1 = max_ent;
-	    dictp->f.fcode = fcode;
-
-	    db->max_ent = ++max_ent;
-	    db->lens[max_ent] = db->lens[oldcode]+1;
-
-	    /* Expand code size if needed. */
-	    if (max_ent >= MAXCODE(n_bits) && max_ent < db->maxmaxcode) {
-		db->n_bits = ++n_bits;
-		tgtbitno = 32-n_bits;
-	    }
-	}
-	oldcode = incode;
-    }
-    dmp->m_len = wptr - mtod(dmp, u_char *);
-
-    /*
-     * Keep the checkpoint right so that incompressible packets
-     * clear the dictionary at the right times.
-     */
-    db->bytes_out += ilen;
-    db->in_count += explen;
-    if (bsd_check(db) && db->debug) {
-	printf("bsd_decomp%d: peer should have cleared dictionary\n",
-	       db->unit);
-    }
-
-    ++db->comp_count;
-    db->comp_bytes += ilen + BSD_OVHD;
-    ++db->uncomp_count;
-    db->uncomp_bytes += explen;
-
-    *dmpp = mret;
-    return DECOMP_OK;
-
-#ifdef DEBUG
- bad:
-    if (codelen <= 0) {
-	printf("bsd_decomp%d: fell off end of chain ", db->unit);
-	printf("0x%x at 0x%x by 0x%x, max_ent=0x%x\n",
-	       incode, finchar, db->dict[finchar].cptr, max_ent);
-    } else if (dictp->codem1 != finchar-1) {
-	printf("bsd_decomp%d: bad code chain 0x%x finchar=0x%x ",
-	       db->unit, incode, finchar);
-	printf("oldcode=0x%x cptr=0x%x codem1=0x%x\n", oldcode,
-	       db->dict[finchar].cptr, dictp->codem1);
-    }
-    m_freem(mret);
-    return DECOMP_FATALERROR;
-#endif /* DEBUG */
-}
diff --git a/sys/net/if.c b/sys/net/if.c
index 7e8d90b..da3c041 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -138,10 +138,10 @@ static void	link_rtrequest(int, struct rtentry *, struct rt_addrinfo *);
 static int	if_rtdel(struct radix_node *, void *);
 static int	ifhwioctl(u_long, struct ifnet *, caddr_t, struct thread *);
 static int	if_delmulti_locked(struct ifnet *, struct ifmultiaddr *, int);
-static void	if_start_deferred(void *context, int pending);
 static void	do_link_state_change(void *, int);
 static int	if_getgroup(struct ifgroupreq *, struct ifnet *);
 static int	if_getgroupmembers(struct ifgroupreq *);
+static void	if_delgroups(struct ifnet *);
 
 #ifdef INET6
 /*
@@ -151,6 +151,8 @@ static int	if_getgroupmembers(struct ifgroupreq *);
 extern void	nd6_setmtu(struct ifnet *);
 #endif
 
+static int	vnet_net_iattach(const void *);
+
 #ifdef VIMAGE_GLOBALS
 struct	ifnethead ifnet;	/* depend on static init XXX */
 struct	ifgrouphead ifg_head;
@@ -181,9 +183,13 @@ static struct vnet_symmap vnet_net_symmap[] = {
 	VNET_SYMMAP_END
 };
 
-VNET_MOD_DECLARE(NET, net, vnet_net_iattach, vnet_net_idetach,
-    NONE, vnet_net_symmap)
-#endif
+static const vnet_modinfo_t vnet_net_modinfo = {
+	.vmi_id		= VNET_MOD_NET,
+	.vmi_name	= "net",
+	.vmi_symmap	= vnet_net_symmap,
+	.vmi_iattach	= vnet_net_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
 
 /*
  * System initialization
@@ -392,24 +398,33 @@ filt_netdev(struct knote *kn, long hint)
 static void
 if_init(void *dummy __unused)
 {
-	INIT_VNET_NET(curvnet);
 
 #ifndef VIMAGE_GLOBALS
 	vnet_mod_register(&vnet_net_modinfo);
 #endif
+	vnet_net_iattach(NULL);
+
+	IFNET_LOCK_INIT();
+	ifdev_setbyindex(0, make_dev(&net_cdevsw, 0, UID_ROOT, GID_WHEEL,
+	    0600, "network"));
+	if_clone_init();
+}
+
+static int
+vnet_net_iattach(const void *unused __unused)
+{
+	INIT_VNET_NET(curvnet);
 
 	V_if_index = 0;
 	V_ifindex_table = NULL;
 	V_if_indexlim = 8;
 
-	IFNET_LOCK_INIT();
 	TAILQ_INIT(&V_ifnet);
 	TAILQ_INIT(&V_ifg_head);
 	knlist_init(&V_ifklist, NULL, NULL, NULL, NULL);
 	if_grow();				/* create initial table */
-	ifdev_setbyindex(0, make_dev(&net_cdevsw, 0, UID_ROOT, GID_WHEEL,
-	    0600, "network"));
-	if_clone_init();
+
+	return (0);
 }
 
 static void
@@ -533,7 +548,7 @@ if_free_type(struct ifnet *ifp, u_char type)
 
 	IF_ADDR_LOCK_DESTROY(ifp);
 	free(ifp, M_IFNET);
-};
+}
 
 void
 ifq_attach(struct ifaltq *ifq, struct ifnet *ifp)
@@ -582,7 +597,6 @@ if_attach(struct ifnet *ifp)
 		panic ("%s: BUG: if_attach called without if_alloc'd input()\n",
 		    ifp->if_xname);
 
-	TASK_INIT(&ifp->if_starttask, 0, if_start_deferred, ifp);
 	TASK_INIT(&ifp->if_linktask, 0, do_link_state_change, ifp);
 	IF_AFDATA_LOCK_INIT(ifp);
 	ifp->if_afdata_initialized = 0;
@@ -674,9 +688,6 @@ if_attach(struct ifnet *ifp)
 		if (atomic_cmpset_int(&slowtimo_started, 0, 1) && !cold)
 			if_slowtimo(0);
 	}
-	if (ifp->if_flags & IFF_NEEDSGIANT)
-		if_printf(ifp,
-		    "WARNING: using obsoleted IFF_NEEDSGIANT flag\n");
 }
 
 static void
@@ -881,6 +892,7 @@ if_detach(struct ifnet *ifp)
 	rt_ifannouncemsg(ifp, IFAN_DEPARTURE);
 	EVENTHANDLER_INVOKE(ifnet_departure_event, ifp);
 	devctl_notify("IFNET", ifp->if_xname, "DETACH", NULL);
+	if_delgroups(ifp);
 
 	IF_AFDATA_LOCK(ifp);
 	for (dp = domains; dp; dp = dp->dom_next) {
@@ -1018,6 +1030,54 @@ if_delgroup(struct ifnet *ifp, const char *groupname)
 }
 
 /*
+ * Remove an interface from all groups
+ */
+static void
+if_delgroups(struct ifnet *ifp)
+{
+	INIT_VNET_NET(ifp->if_vnet);
+	struct ifg_list		*ifgl;
+	struct ifg_member	*ifgm;
+	char groupname[IFNAMSIZ];
+
+	IFNET_WLOCK();
+	while (!TAILQ_EMPTY(&ifp->if_groups)) {
+		ifgl = TAILQ_FIRST(&ifp->if_groups);
+
+		strlcpy(groupname, ifgl->ifgl_group->ifg_group, IFNAMSIZ);
+
+		IF_ADDR_LOCK(ifp);
+		TAILQ_REMOVE(&ifp->if_groups, ifgl, ifgl_next);
+		IF_ADDR_UNLOCK(ifp);
+
+		TAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next)
+			if (ifgm->ifgm_ifp == ifp)
+				break;
+
+		if (ifgm != NULL) {
+			TAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm,
+			    ifgm_next);
+			free(ifgm, M_TEMP);
+		}
+
+		if (--ifgl->ifgl_group->ifg_refcnt == 0) {
+			TAILQ_REMOVE(&V_ifg_head, ifgl->ifgl_group, ifg_next);
+			EVENTHANDLER_INVOKE(group_detach_event,
+			    ifgl->ifgl_group);
+			free(ifgl->ifgl_group, M_TEMP);
+		}
+		IFNET_WUNLOCK();
+
+		free(ifgl, M_TEMP);
+
+		EVENTHANDLER_INVOKE(group_change_event, groupname);
+
+		IFNET_WLOCK();
+	}
+	IFNET_WUNLOCK();
+}
+
+/*
  * Stores all groups from an interface in memory pointed
  * to by data
  */
@@ -1607,8 +1667,7 @@ if_qflush(struct ifnet *ifp)
  * call the appropriate interface routine on expiration.
  *
  * XXXRW: Note that because timeouts run with Giant, if_watchdog() is called
- * holding Giant.  If we switch to an MPSAFE callout, we likely need to grab
- * Giant before entering if_watchdog() on an IFF_NEEDSGIANT interface.
+ * holding Giant.
  */
 static void
 if_slowtimo(void *arg)
@@ -1741,9 +1800,7 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td)
 		ifp->if_flags = (ifp->if_flags & IFF_CANTCHANGE) |
 			(new_flags &~ IFF_CANTCHANGE);
 		if (ifp->if_ioctl) {
-			IFF_LOCKGIANT(ifp);
 			(void) (*ifp->if_ioctl)(ifp, cmd, data);
-			IFF_UNLOCKGIANT(ifp);
 		}
 		getmicrotime(&ifp->if_lastchange);
 		break;
@@ -1756,9 +1813,7 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td)
 			return (EOPNOTSUPP);
 		if (ifr->ifr_reqcap & ~ifp->if_capabilities)
 			return (EINVAL);
-		IFF_LOCKGIANT(ifp);
 		error = (*ifp->if_ioctl)(ifp, cmd, data);
-		IFF_UNLOCKGIANT(ifp);
 		if (error == 0)
 			getmicrotime(&ifp->if_lastchange);
 		break;
@@ -1830,9 +1885,7 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td)
 			return (error);
 		if (ifp->if_ioctl == NULL)
 			return (EOPNOTSUPP);
-		IFF_LOCKGIANT(ifp);
 		error = (*ifp->if_ioctl)(ifp, cmd, data);
-		IFF_UNLOCKGIANT(ifp);
 		if (error == 0)
 			getmicrotime(&ifp->if_lastchange);
 		break;
@@ -1848,9 +1901,7 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td)
 			return (EINVAL);
 		if (ifp->if_ioctl == NULL)
 			return (EOPNOTSUPP);
-		IFF_LOCKGIANT(ifp);
 		error = (*ifp->if_ioctl)(ifp, cmd, data);
-		IFF_UNLOCKGIANT(ifp);
 		if (error == 0) {
 			getmicrotime(&ifp->if_lastchange);
 			rt_ifmsg(ifp);
@@ -1920,9 +1971,7 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td)
 			return (error);
 		if (ifp->if_ioctl == NULL)
 			return (EOPNOTSUPP);
-		IFF_LOCKGIANT(ifp);
 		error = (*ifp->if_ioctl)(ifp, cmd, data);
-		IFF_UNLOCKGIANT(ifp);
 		if (error == 0)
 			getmicrotime(&ifp->if_lastchange);
 		break;
@@ -1938,9 +1987,7 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data, struct thread *td)
 	case SIOCGIFGENERIC:
 		if (ifp->if_ioctl == NULL)
 			return (EOPNOTSUPP);
-		IFF_LOCKGIANT(ifp);
 		error = (*ifp->if_ioctl)(ifp, cmd, data);
-		IFF_UNLOCKGIANT(ifp);
 		break;
 
 	case SIOCSIFLLADDR:
@@ -2043,6 +2090,8 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td)
 	error = ((*so->so_proto->pr_usrreqs->pru_control)(so, cmd,
 								 data,
 								 ifp, td));
+	if (error == EOPNOTSUPP && ifp != NULL && ifp->if_ioctl != NULL)
+		error = (*ifp->if_ioctl)(ifp, cmd, data);
 #else
 	{
 		int ocmd = cmd;
@@ -2084,6 +2133,9 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct thread *td)
 								   cmd,
 								   data,
 								   ifp, td));
+		if (error == EOPNOTSUPP && ifp != NULL &&
+		    ifp->if_ioctl != NULL)
+			error = (*ifp->if_ioctl)(ifp, cmd, data);
 		switch (ocmd) {
 
 		case OSIOCGIFADDR:
@@ -2168,9 +2220,7 @@ if_setflag(struct ifnet *ifp, int flag, int pflag, int *refcount, int onswitch)
 	}
 	ifr.ifr_flags = ifp->if_flags & 0xffff;
 	ifr.ifr_flagshigh = ifp->if_flags >> 16;
-	IFF_LOCKGIANT(ifp);
 	error = (*ifp->if_ioctl)(ifp, SIOCSIFFLAGS, (caddr_t)&ifr);
-	IFF_UNLOCKGIANT(ifp);
 	if (error)
 		goto recover;
 	/* Notify userland that interface flags have changed */
@@ -2540,9 +2590,7 @@ if_addmulti(struct ifnet *ifp, struct sockaddr *sa,
 	 * interface to let them know about it.
 	 */
 	if (ifp->if_ioctl != NULL) {
-		IFF_LOCKGIANT(ifp);
 		(void) (*ifp->if_ioctl)(ifp, SIOCADDMULTI, 0);
-		IFF_UNLOCKGIANT(ifp);
 	}
 
 	if (llsa != NULL)
@@ -2601,9 +2649,7 @@ if_delmulti(struct ifnet *ifp, struct sockaddr *sa)
 		return (ENOENT);
 
 	if (lastref && ifp->if_ioctl != NULL) {
-		IFF_LOCKGIANT(ifp);
 		(void)(*ifp->if_ioctl)(ifp, SIOCDELMULTI, 0);
-		IFF_UNLOCKGIANT(ifp);
 	}
 
 	return (0);
@@ -2613,9 +2659,7 @@ if_delmulti(struct ifnet *ifp, struct sockaddr *sa)
  * Delete a multicast group membership by group membership pointer.
  * Network-layer protocol domains must use this routine.
  *
- * It is safe to call this routine if the ifp disappeared. Callers should
- * hold IFF_LOCKGIANT() to avoid a LOR in case the hardware needs to be
- * reconfigured.
+ * It is safe to call this routine if the ifp disappeared.
  */
 void
 if_delmulti_ifma(struct ifmultiaddr *ifma)
@@ -2660,9 +2704,7 @@ if_delmulti_ifma(struct ifmultiaddr *ifma)
 		 */
 		IF_ADDR_UNLOCK(ifp);
 		if (lastref && ifp->if_ioctl != NULL) {
-			IFF_LOCKGIANT(ifp);
 			(void)(*ifp->if_ioctl)(ifp, SIOCDELMULTI, 0);
-			IFF_UNLOCKGIANT(ifp);
 		}
 	}
 }
@@ -2772,6 +2814,7 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len)
 	case IFT_BRIDGE:
 	case IFT_ARCNET:
 	case IFT_IEEE8023ADLAG:
+	case IFT_IEEE80211:
 		bcopy(lladdr, LLADDR(sdl), len);
 		break;
 	default:
@@ -2784,7 +2827,6 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len)
 	 */
 	if ((ifp->if_flags & IFF_UP) != 0) {
 		if (ifp->if_ioctl) {
-			IFF_LOCKGIANT(ifp);
 			ifp->if_flags &= ~IFF_UP;
 			ifr.ifr_flags = ifp->if_flags & 0xffff;
 			ifr.ifr_flagshigh = ifp->if_flags >> 16;
@@ -2793,7 +2835,6 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr, int len)
 			ifr.ifr_flags = ifp->if_flags & 0xffff;
 			ifr.ifr_flagshigh = ifp->if_flags >> 16;
 			(*ifp->if_ioctl)(ifp, SIOCSIFFLAGS, (caddr_t)&ifr);
-			IFF_UNLOCKGIANT(ifp);
 		}
 #ifdef INET
 		/*
@@ -2839,39 +2880,11 @@ if_printf(struct ifnet *ifp, const char * fmt, ...)
 	return (retval);
 }
 
-/*
- * When an interface is marked IFF_NEEDSGIANT, its if_start() routine cannot
- * be called without Giant.  However, we often can't acquire the Giant lock
- * at those points; instead, we run it via a task queue that holds Giant via
- * if_start_deferred.
- *
- * XXXRW: We need to make sure that the ifnet isn't fully detached until any
- * outstanding if_start_deferred() tasks that will run after the free.  This
- * probably means waiting in if_detach().
- */
 void
 if_start(struct ifnet *ifp)
 {
 
-	if (ifp->if_flags & IFF_NEEDSGIANT) {
-		if (mtx_owned(&Giant))
-			(*(ifp)->if_start)(ifp);
-		else
-			taskqueue_enqueue(taskqueue_swi_giant,
-			    &ifp->if_starttask);
-	} else
-		(*(ifp)->if_start)(ifp);
-}
-
-static void
-if_start_deferred(void *context, int pending)
-{
-	struct ifnet *ifp;
-
-	GIANT_REQUIRED;
-
-	ifp = context;
-	(ifp->if_start)(ifp);
+	(*(ifp)->if_start)(ifp);
 }
 
 /*
@@ -2908,7 +2921,7 @@ if_handoff(struct ifqueue *ifq, struct mbuf *m, struct ifnet *ifp, int adjust)
 	_IF_ENQUEUE(ifq, m);
 	IF_UNLOCK(ifq);
 	if (ifp != NULL && !active)
-		if_start(ifp);
+		(*(ifp)->if_start)(ifp);
 	return (1);
 }
 
diff --git a/sys/net/if_bridge.c b/sys/net/if_bridge.c
index da73543..ef8cf5b 100644
--- a/sys/net/if_bridge.c
+++ b/sys/net/if_bridge.c
@@ -829,9 +829,7 @@ bridge_set_ifcap(struct bridge_softc *sc, struct bridge_iflist *bif, int set)
 	ifr.ifr_reqcap = set;
 
 	if (ifp->if_capenable != set) {
-		IFF_LOCKGIANT(ifp);
 		error = (*ifp->if_ioctl)(ifp, SIOCSIFCAP, (caddr_t)&ifr);
-		IFF_UNLOCKGIANT(ifp);
 		if (error)
 			if_printf(sc->sc_ifp,
 			    "error setting interface capabilities on %s\n",
@@ -3245,12 +3243,12 @@ bridge_ip_checkbasic(struct mbuf **mp)
 		if ((m = m_copyup(m, sizeof(struct ip),
 			(max_linkhdr + 3) & ~3)) == NULL) {
 			/* XXXJRT new stat, please */
-			V_ipstat.ips_toosmall++;
+			IPSTAT_INC(ips_toosmall);
 			goto bad;
 		}
 	} else if (__predict_false(m->m_len < sizeof (struct ip))) {
 		if ((m = m_pullup(m, sizeof (struct ip))) == NULL) {
-			V_ipstat.ips_toosmall++;
+			IPSTAT_INC(ips_toosmall);
 			goto bad;
 		}
 	}
@@ -3258,17 +3256,17 @@ bridge_ip_checkbasic(struct mbuf **mp)
 	if (ip == NULL) goto bad;
 
 	if (ip->ip_v != IPVERSION) {
-		V_ipstat.ips_badvers++;
+		IPSTAT_INC(ips_badvers);
 		goto bad;
 	}
 	hlen = ip->ip_hl << 2;
 	if (hlen < sizeof(struct ip)) { /* minimum header length */
-		V_ipstat.ips_badhlen++;
+		IPSTAT_INC(ips_badhlen);
 		goto bad;
 	}
 	if (hlen > m->m_len) {
 		if ((m = m_pullup(m, hlen)) == 0) {
-			V_ipstat.ips_badhlen++;
+			IPSTAT_INC(ips_badhlen);
 			goto bad;
 		}
 		ip = mtod(m, struct ip *);
@@ -3285,7 +3283,7 @@ bridge_ip_checkbasic(struct mbuf **mp)
 		}
 	}
 	if (sum) {
-		V_ipstat.ips_badsum++;
+		IPSTAT_INC(ips_badsum);
 		goto bad;
 	}
 
@@ -3296,7 +3294,7 @@ bridge_ip_checkbasic(struct mbuf **mp)
 	 * Check for additional length bogosity
 	 */
 	if (len < hlen) {
-		V_ipstat.ips_badlen++;
+		IPSTAT_INC(ips_badlen);
 		goto bad;
 	}
 
@@ -3306,7 +3304,7 @@ bridge_ip_checkbasic(struct mbuf **mp)
 	 * Drop packet if shorter than we expect.
 	 */
 	if (m->m_pkthdr.len < len) {
-		V_ipstat.ips_tooshort++;
+		IPSTAT_INC(ips_tooshort);
 		goto bad;
 	}
 
@@ -3421,7 +3419,7 @@ bridge_fragment(struct ifnet *ifp, struct mbuf *m, struct ether_header *eh,
 	}
 
 	if (error == 0)
-		V_ipstat.ips_fragmented++;
+		IPSTAT_INC(ips_fragmented);
 
 	return (error);
 
diff --git a/sys/net/if_gif.c b/sys/net/if_gif.c
index be7fa9f..1c10f17 100644
--- a/sys/net/if_gif.c
+++ b/sys/net/if_gif.c
@@ -121,6 +121,16 @@ void	(*ng_gif_detach_p)(struct ifnet *ifp);
 static void	gif_start(struct ifnet *);
 static int	gif_clone_create(struct if_clone *, int, caddr_t);
 static void	gif_clone_destroy(struct ifnet *);
+static int	vnet_gif_iattach(const void *);
+
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_gif_modinfo = {
+	.vmi_id		= VNET_MOD_GIF,
+	.vmi_name	= "gif",
+	.vmi_dependson	= VNET_MOD_NET,
+	.vmi_iattach	= vnet_gif_iattach
+};
+#endif
 
 IFC_SIMPLE_DECLARE(gif, 0);
 
@@ -251,6 +261,26 @@ gif_clone_destroy(ifp)
 }
 
 static int
+vnet_gif_iattach(const void *unused __unused)
+{
+	INIT_VNET_GIF(curvnet);
+
+	LIST_INIT(&V_gif_softc_list);
+	V_max_gif_nesting = MAX_GIF_NEST;
+#ifdef XBONEHACK
+	V_parallel_tunnels = 1;
+#else
+	V_parallel_tunnels = 0;
+#endif
+	V_ip_gif_ttl = GIF_TTL;
+#ifdef INET6
+	V_ip6_gif_hlim = GIF_HLIM;
+#endif
+  
+	return (0);
+}
+
+static int
 gifmodevent(mod, type, data)
 	module_t mod;
 	int type;
@@ -261,18 +291,10 @@ gifmodevent(mod, type, data)
 	case MOD_LOAD:
 		mtx_init(&gif_mtx, "gif_mtx", NULL, MTX_DEF);
 
-		LIST_INIT(&V_gif_softc_list);
-		V_max_gif_nesting = MAX_GIF_NEST;
-#ifdef XBONEHACK
-		V_parallel_tunnels = 1;
+#ifndef VIMAGE_GLOBALS
+		vnet_mod_register(&vnet_gif_modinfo);
 #else
-		V_parallel_tunnels = 0;
-#endif
-#ifdef INET
-		V_ip_gif_ttl = GIF_TTL;
-#endif
-#ifdef INET6
-		V_ip6_gif_hlim = GIF_HLIM;
+		vnet_gif_iattach(NULL);
 #endif
 		if_clone_attach(&gif_cloner);
 
@@ -281,7 +303,7 @@ gifmodevent(mod, type, data)
 		if_clone_detach(&gif_cloner);
 		mtx_destroy(&gif_mtx);
 #ifdef INET6
-		V_ip6_gif_hlim = 0;
+		V_ip6_gif_hlim = 0;	/* XXX -> vnet_gif_idetach() */
 #endif
 		break;
 	default:
diff --git a/sys/net/if_loop.c b/sys/net/if_loop.c
index d5548fc..c31acef 100644
--- a/sys/net/if_loop.c
+++ b/sys/net/if_loop.c
@@ -94,17 +94,31 @@
 #define LOMTU	16384
 #endif
 
+#define	LO_CSUM_FEATURES	(CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_SCTP)
+#define	LO_CSUM_SET		(CSUM_DATA_VALID | CSUM_PSEUDO_HDR | \
+				    CSUM_IP_CHECKED | CSUM_IP_VALID | \
+				    CSUM_SCTP_VALID)
+
 int		loioctl(struct ifnet *, u_long, caddr_t);
 static void	lortrequest(int, struct rtentry *, struct rt_addrinfo *);
 int		looutput(struct ifnet *ifp, struct mbuf *m,
 		    struct sockaddr *dst, struct rtentry *rt);
 static int	lo_clone_create(struct if_clone *, int, caddr_t);
 static void	lo_clone_destroy(struct ifnet *);
+static int	vnet_loif_iattach(const void *);
 
 #ifdef VIMAGE_GLOBALS
 struct ifnet *loif;			/* Used externally */
 #endif
 
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_loif_modinfo = {
+	.vmi_id		= VNET_MOD_LOIF,
+	.vmi_name	= "loif",
+	.vmi_iattach	= vnet_loif_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
+
 IFC_SIMPLE_DECLARE(lo, 1);
 
 static void
@@ -138,6 +152,8 @@ lo_clone_create(struct if_clone *ifc, int unit, caddr_t params)
 	ifp->if_ioctl = loioctl;
 	ifp->if_output = looutput;
 	ifp->if_snd.ifq_maxlen = ifqmaxlen;
+	ifp->if_capabilities = ifp->if_capenable = IFCAP_HWCSUM;
+	ifp->if_hwassist = LO_CSUM_FEATURES;
 	if_attach(ifp);
 	bpfattach(ifp, DLT_NULL, sizeof(u_int32_t));
 	if (V_loif == NULL)
@@ -146,6 +162,15 @@ lo_clone_create(struct if_clone *ifc, int unit, caddr_t params)
 	return (0);
 }
 
+static int vnet_loif_iattach(const void *unused __unused)
+{
+	INIT_VNET_NET(curvnet);
+
+	V_loif = NULL;
+	if_clone_attach(&lo_cloner);
+	return (0);
+}
+
 static int
 loop_modevent(module_t mod, int type, void *data)
 {
@@ -153,8 +178,11 @@ loop_modevent(module_t mod, int type, void *data)
 
 	switch (type) {
 	case MOD_LOAD:
-		V_loif = NULL;
-		if_clone_attach(&lo_cloner);
+#ifndef VIMAGE_GLOBALS
+		vnet_mod_register(&vnet_loif_modinfo);
+#else
+		vnet_loif_iattach(NULL);
+#endif
 		break;
 
 	case MOD_UNLOAD:
@@ -212,6 +240,11 @@ looutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
 #if 1	/* XXX */
 	switch (dst->sa_family) {
 	case AF_INET:
+		if (ifp->if_capenable & IFCAP_RXCSUM) {
+			m->m_pkthdr.csum_data = 0xffff;
+			m->m_pkthdr.csum_flags = LO_CSUM_SET;
+		}
+		m->m_pkthdr.csum_flags &= ~LO_CSUM_FEATURES;
 	case AF_INET6:
 	case AF_IPX:
 	case AF_APPLETALK:
@@ -348,7 +381,7 @@ loioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 {
 	struct ifaddr *ifa;
 	struct ifreq *ifr = (struct ifreq *)data;
-	int error = 0;
+	int error = 0, mask;
 
 	switch (cmd) {
 	case SIOCSIFADDR:
@@ -391,6 +424,18 @@ loioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 	case SIOCSIFFLAGS:
 		break;
 
+	case SIOCSIFCAP:
+		mask = ifp->if_capenable ^ ifr->ifr_reqcap;
+		if ((mask & IFCAP_RXCSUM) != 0)
+			ifp->if_capenable ^= IFCAP_RXCSUM;
+		if ((mask & IFCAP_TXCSUM) != 0)
+			ifp->if_capenable ^= IFCAP_TXCSUM;
+		if (ifp->if_capenable & IFCAP_TXCSUM)
+			ifp->if_hwassist = LO_CSUM_FEATURES;
+		else
+			ifp->if_hwassist = 0;
+		break;
+
 	default:
 		error = EINVAL;
 	}
diff --git a/sys/net/if_ppp.c b/sys/net/if_ppp.c
deleted file mode 100644
index 753adf9..0000000
--- a/sys/net/if_ppp.c
+++ /dev/null
@@ -1,1733 +0,0 @@
-/*
- * if_ppp.c - Point-to-Point Protocol (PPP) Asynchronous driver.
- */
-
-/*-
- * Copyright (c) 1989 Carnegie Mellon University.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms are permitted
- * provided that the above copyright notice and this paragraph are
- * duplicated in all such forms and that any documentation,
- * advertising materials, and other materials related to such
- * distribution and use acknowledge that the software was developed
- * by Carnegie Mellon University.  The name of the
- * University may not be used to endorse or promote products derived
- * from this software without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
- * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
- *
- * Drew D. Perkins
- * Carnegie Mellon University
- * 4910 Forbes Ave.
- * Pittsburgh, PA 15213
- * (412) 268-8576
- * ddp@andrew.cmu.edu
- *
- * Based on:
- *	@(#)if_sl.c	7.6.1.2 (Berkeley) 2/15/89
- *
- * Copyright (c) 1987 Regents of the University of California.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms are permitted
- * provided that the above copyright notice and this paragraph are
- * duplicated in all such forms and that any documentation,
- * advertising materials, and other materials related to such
- * distribution and use acknowledge that the software was developed
- * by the University of California, Berkeley.  The name of the
- * University may not be used to endorse or promote products derived
- * from this software without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
- * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
- *
- * Serial Line interface
- *
- * Rick Adams
- * Center for Seismic Studies
- * 1300 N 17th Street, Suite 1450
- * Arlington, Virginia 22209
- * (703)276-7900
- * rick@seismo.ARPA
- * seismo!rick
- *
- * Pounded on heavily by Chris Torek (chris@mimsy.umd.edu, umcp-cs!chris).
- * Converted to 4.3BSD Beta by Chris Torek.
- * Other changes made at Berkeley, based in part on code by Kirk Smith.
- *
- * Converted to 4.3BSD+ 386BSD by Brad Parker (brad@cayman.com)
- * Added VJ tcp header compression; more unified ioctls
- *
- * Extensively modified by Paul Mackerras (paulus@cs.anu.edu.au).
- * Cleaned up a lot of the mbuf-related code to fix bugs that
- * caused system crashes and packet corruption.  Changed pppstart
- * so that it doesn't just give up with a collision if the whole
- * packet doesn't fit in the output ring buffer.
- *
- * Added priority queueing for interactive IP packets, following
- * the model of if_sl.c, plus hooks for bpf.
- * Paul Mackerras (paulus@cs.anu.edu.au).
- */
-
-/* $FreeBSD$ */
-/* from if_sl.c,v 1.11 84/10/04 12:54:47 rick Exp */
-/* from NetBSD: if_ppp.c,v 1.15.2.2 1994/07/28 05:17:58 cgd Exp */
-
-#include "opt_inet.h"
-#include "opt_inet6.h"
-#include "opt_ipx.h"
-#include "opt_mac.h"
-#include "opt_ppp.h"
-
-#ifdef INET
-#define VJC
-#endif
-#define PPP_COMPRESS
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/priv.h>
-#include <sys/proc.h>
-#include <sys/mbuf.h>
-#include <sys/socket.h>
-#include <sys/filio.h>
-#include <sys/sockio.h>
-#include <sys/kernel.h>
-#include <sys/time.h>
-#include <sys/malloc.h>
-#include <sys/module.h>
-
-#include <net/if.h>
-#include <net/if_clone.h>
-#include <net/if_types.h>
-#include <net/netisr.h>
-#include <net/bpf.h>
-
-#ifdef INET
-#include <netinet/in.h>
-#include <netinet/in_systm.h>
-#include <netinet/in_var.h>
-#include <netinet/ip.h>
-#endif
-
-#ifdef IPX
-#include <netipx/ipx.h>
-#include <netipx/ipx_if.h>
-#endif
-
-#ifdef VJC
-#include <net/slcompress.h>
-#endif
-
-#include <net/if_ppp.h>
-#include <net/if_pppvar.h>
-
-#include <security/mac/mac_framework.h>
-
-/* minimise diffs */
-#ifndef splsoftnet
-#define splsoftnet	splnet
-#endif
-
-#ifdef PPP_COMPRESS
-#define PACKETPTR	struct mbuf *
-#include <net/ppp_comp.h>
-#endif
-
-#define PPPNAME		"ppp"
-static MALLOC_DEFINE(M_PPP, PPPNAME, "PPP interface");
-
-static struct mtx ppp_softc_list_mtx;
-static LIST_HEAD(, ppp_softc) ppp_softc_list;
-
-#define	PPP_LIST_LOCK_INIT()	mtx_init(&ppp_softc_list_mtx,		\
-				    "ppp_softc_list_mtx", NULL, MTX_DEF)
-#define	PPP_LIST_LOCK_DESTROY()	mtx_destroy(&ppp_softc_list_mtx)
-#define	PPP_LIST_LOCK()		mtx_lock(&ppp_softc_list_mtx)
-#define	PPP_LIST_UNLOCK()	mtx_unlock(&ppp_softc_list_mtx)
-
-/* XXX layering violation */
-extern void	pppasyncattach(void *);
-extern void	pppasyncdetach(void);
-
-static int	pppsioctl(struct ifnet *ifp, u_long cmd, caddr_t data);
-static void	pppintr(void);
-
-static void	ppp_requeue(struct ppp_softc *);
-static void	ppp_ccp(struct ppp_softc *, struct mbuf *m, int rcvd);
-static void	ppp_ccp_closed(struct ppp_softc *);
-static void	ppp_inproc(struct ppp_softc *, struct mbuf *);
-static void	pppdumpm(struct mbuf *m0);
-static int	ppp_clone_create(struct if_clone *, int, caddr_t);
-static void	ppp_clone_destroy(struct ifnet *);
-
-IFC_SIMPLE_DECLARE(ppp, 0);
-
-/*
- * Some useful mbuf macros not in mbuf.h.
- */
-#define M_IS_CLUSTER(m)	((m)->m_flags & M_EXT)
-
-#define M_DATASTART(m)	\
-	(M_IS_CLUSTER(m) ? (m)->m_ext.ext_buf : \
-	    (m)->m_flags & M_PKTHDR ? (m)->m_pktdat : (m)->m_dat)
-
-#define M_DATASIZE(m)	\
-	(M_IS_CLUSTER(m) ? (m)->m_ext.ext_size : \
-	    (m)->m_flags & M_PKTHDR ? MHLEN: MLEN)
-
-/*
- * We steal two bits in the mbuf m_flags, to mark high-priority packets
- * for output, and received packets following lost/corrupted packets.
- */
-#define M_HIGHPRI	0x2000	/* output packet for sc_fastq */
-#define M_ERRMARK	0x4000	/* steal a bit in mbuf m_flags */
-
-
-#ifdef PPP_COMPRESS
-/*
- * List of compressors we know about.
- * We leave some space so maybe we can modload compressors.
- */
-
-extern struct compressor ppp_bsd_compress;
-extern struct compressor ppp_deflate, ppp_deflate_draft;
-
-static struct compressor *ppp_compressors[8] = {
-#if defined(PPP_BSDCOMP)
-    &ppp_bsd_compress,
-#endif
-#if defined(PPP_DEFLATE)
-    &ppp_deflate,
-    &ppp_deflate_draft,
-#endif
-    NULL
-};
-#endif /* PPP_COMPRESS */
-
-static int
-ppp_clone_create(struct if_clone *ifc, int unit, caddr_t params)
-{
-	struct ifnet		*ifp;
-	struct ppp_softc	*sc;
-
-	sc = malloc(sizeof(struct ppp_softc), M_PPP, M_WAITOK | M_ZERO);
-	ifp = sc->sc_ifp = if_alloc(IFT_PPP);
-	if (ifp == NULL) {
-		free(sc, M_PPP);
-		return (ENOSPC);
-	}
-
-	callout_init(&sc->sc_timo_ch, 0);
-	ifp->if_softc = sc;
-	if_initname(ifp, ifc->ifc_name, unit);
-	ifp->if_mtu = PPP_MTU;
-	ifp->if_flags = IFF_POINTOPOINT | IFF_MULTICAST | IFF_NEEDSGIANT;
-	ifp->if_hdrlen = PPP_HDRLEN;
-	ifp->if_ioctl = pppsioctl;
-	ifp->if_output = pppoutput;
-	ifp->if_snd.ifq_maxlen = IFQ_MAXLEN;
-	sc->sc_inq.ifq_maxlen = IFQ_MAXLEN;
-	sc->sc_fastq.ifq_maxlen = IFQ_MAXLEN;
-	sc->sc_rawq.ifq_maxlen = IFQ_MAXLEN;
-	mtx_init(&sc->sc_inq.ifq_mtx, "ppp_inq", NULL, MTX_DEF);
-	mtx_init(&sc->sc_fastq.ifq_mtx, "ppp_fastq", NULL, MTX_DEF);
-	mtx_init(&sc->sc_rawq.ifq_mtx, "ppp_rawq", NULL, MTX_DEF);
-	if_attach(ifp);
-	bpfattach(ifp, DLT_PPP, PPP_HDRLEN);
-
-	PPP_LIST_LOCK();
-	LIST_INSERT_HEAD(&ppp_softc_list, sc, sc_list);
-	PPP_LIST_UNLOCK();
-
-	return (0);
-}
-
-static void
-ppp_clone_destroy(struct ifnet *ifp)
-{
-	struct ppp_softc *sc;
-
-	sc = ifp->if_softc;
-	PPP_LIST_LOCK();
-	LIST_REMOVE(sc, sc_list);
-	PPP_LIST_UNLOCK();
-
-	bpfdetach(ifp);
-	if_detach(ifp);
-	if_free(ifp);
-	mtx_destroy(&sc->sc_rawq.ifq_mtx);
-	mtx_destroy(&sc->sc_fastq.ifq_mtx);
-	mtx_destroy(&sc->sc_inq.ifq_mtx);
-	free(sc, M_PPP);
-}
-
-static int
-ppp_modevent(module_t mod, int type, void *data) 
-{
-
-	switch (type) { 
-	case MOD_LOAD: 
-		PPP_LIST_LOCK_INIT();
-		LIST_INIT(&ppp_softc_list);
-		if_clone_attach(&ppp_cloner);
-
-		netisr_register(NETISR_PPP, (netisr_t *)pppintr, NULL,
-		    NETISR_FORCEQUEUE);
-		/*
-		 * XXX layering violation - if_ppp can work over any lower
-		 * level transport that cares to attach to it.
-		 */
-		pppasyncattach(NULL);
-		break; 
-	case MOD_UNLOAD: 
-		/* XXX: layering violation */
-		pppasyncdetach();
-
-		netisr_unregister(NETISR_PPP);
-
-		if_clone_detach(&ppp_cloner);
-		PPP_LIST_LOCK_DESTROY();
-		break; 
-	default:
-		return EOPNOTSUPP;
-	} 
-	return 0; 
-} 
-
-static moduledata_t ppp_mod = { 
-	"if_ppp", 
-	ppp_modevent, 
-	0
-}; 
-
-DECLARE_MODULE(if_ppp, ppp_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
-
-/*
- * Allocate a ppp interface unit and initialize it.
- */
-struct ppp_softc *
-pppalloc(pid)
-    pid_t pid;
-{
-    int i;
-    char tmpname[IFNAMSIZ];
-    struct ifnet *ifp;
-    struct ppp_softc *sc;
-
-    PPP_LIST_LOCK();
-    LIST_FOREACH(sc, &ppp_softc_list, sc_list) {
-	if (sc->sc_xfer == pid) {
-	    sc->sc_xfer = 0;
-	    PPP_LIST_UNLOCK();
-	    return sc;
-	}
-    }
-    LIST_FOREACH(sc, &ppp_softc_list, sc_list) {
-	if (sc->sc_devp == NULL)
-	    break;
-    }
-    PPP_LIST_UNLOCK();
-    /* Try to clone an interface if we don't have a free one */
-    if (sc == NULL) {
-	strcpy(tmpname, PPPNAME);
-	if (if_clone_create(tmpname, sizeof(tmpname), (caddr_t) 0) != 0)
-	    return NULL;
-	ifp = ifunit(tmpname);
-	if (ifp == NULL)
-	    return NULL;
-	sc = ifp->if_softc;
-    }
-    if (sc == NULL || sc->sc_devp != NULL)
-	return NULL;
-
-    sc->sc_flags = 0;
-    sc->sc_mru = PPP_MRU;
-    sc->sc_relinq = NULL;
-    bzero((char *)&sc->sc_stats, sizeof(sc->sc_stats));
-#ifdef VJC
-    sc->sc_comp = malloc(sizeof(struct slcompress),
-	   M_DEVBUF, M_NOWAIT);
-    if (sc->sc_comp)
-	sl_compress_init(sc->sc_comp, -1);
-#endif
-#ifdef PPP_COMPRESS
-    sc->sc_xc_state = NULL;
-    sc->sc_rc_state = NULL;
-#endif /* PPP_COMPRESS */
-    for (i = 0; i < NUM_NP; ++i)
-	sc->sc_npmode[i] = NPMODE_ERROR;
-    sc->sc_npqueue = NULL;
-    sc->sc_npqtail = &sc->sc_npqueue;
-    sc->sc_last_sent = sc->sc_last_recv = time_uptime;
-
-    return sc;
-}
-
-/*
- * Deallocate a ppp unit.  Must be called at splsoftnet or higher.
- */
-void
-pppdealloc(sc)
-    struct ppp_softc *sc;
-{
-    struct mbuf *m;
-
-    if_down(PPP2IFP(sc));
-    PPP2IFP(sc)->if_flags &= ~IFF_UP;
-    PPP2IFP(sc)->if_drv_flags &= ~IFF_DRV_RUNNING;
-    getmicrotime(&PPP2IFP(sc)->if_lastchange);
-    sc->sc_devp = NULL;
-    sc->sc_xfer = 0;
-    IF_DRAIN(&sc->sc_rawq);
-    IF_DRAIN(&sc->sc_inq);
-    IF_DRAIN(&sc->sc_fastq);
-    while ((m = sc->sc_npqueue) != NULL) {
-	sc->sc_npqueue = m->m_nextpkt;
-	m_freem(m);
-    }
-#ifdef PPP_COMPRESS
-    ppp_ccp_closed(sc);
-    sc->sc_xc_state = NULL;
-    sc->sc_rc_state = NULL;
-#endif /* PPP_COMPRESS */
-#ifdef PPP_FILTER
-    if (sc->sc_pass_filt.bf_insns != 0) {
-	free(sc->sc_pass_filt.bf_insns, M_DEVBUF);
-	sc->sc_pass_filt.bf_insns = 0;
-	sc->sc_pass_filt.bf_len = 0;
-    }
-    if (sc->sc_active_filt.bf_insns != 0) {
-	free(sc->sc_active_filt.bf_insns, M_DEVBUF);
-	sc->sc_active_filt.bf_insns = 0;
-	sc->sc_active_filt.bf_len = 0;
-    }
-#endif /* PPP_FILTER */
-#ifdef VJC
-    if (sc->sc_comp != 0) {
-	free(sc->sc_comp, M_DEVBUF);
-	sc->sc_comp = 0;
-    }
-#endif
-}
-
-/*
- * Ioctl routine for generic ppp devices.
- */
-int
-pppioctl(sc, cmd, data, flag, td)
-    struct ppp_softc *sc;
-    u_long cmd;
-    caddr_t data;
-    int flag;
-    struct thread *td;
-{
-    struct proc *p = td->td_proc;
-    int s, flags, mru, npx;
-    u_int nb;
-    int error = 0;
-    struct ppp_option_data *odp;
-    struct compressor **cp;
-    struct npioctl *npi;
-    time_t t;
-#ifdef PPP_FILTER
-    struct bpf_program *bp, *nbp;
-    struct bpf_insn *newcode, *oldcode;
-    int newcodelen;
-#endif /* PPP_FILTER */
-#ifdef	PPP_COMPRESS
-    u_char ccp_option[CCP_MAX_OPTION_LENGTH];
-#endif
-
-    switch (cmd) {
-    case FIONREAD:
-	*(int *)data = sc->sc_inq.ifq_len;
-	break;
-
-    case PPPIOCGUNIT:
-	*(int *)data = PPP2IFP(sc)->if_dunit;
-	break;
-
-    case PPPIOCGFLAGS:
-	*(u_int *)data = sc->sc_flags;
-	break;
-
-    case PPPIOCSFLAGS:
-	error = priv_check(td, PRIV_NET_PPP);
-	if (error)
-	    break;
-	flags = *(int *)data & SC_MASK;
-	s = splsoftnet();
-#ifdef PPP_COMPRESS
-	if (sc->sc_flags & SC_CCP_OPEN && !(flags & SC_CCP_OPEN))
-	    ppp_ccp_closed(sc);
-#endif
-	splimp();
-	sc->sc_flags = (sc->sc_flags & ~SC_MASK) | flags;
-	splx(s);
-	break;
-
-    case PPPIOCSMRU:
-	error = priv_check(td, PRIV_NET_PPP);
-	if (error)
-		return (error);
-	mru = *(int *)data;
-	if (mru >= PPP_MRU && mru <= PPP_MAXMRU)
-	    sc->sc_mru = mru;
-	break;
-
-    case PPPIOCGMRU:
-	*(int *)data = sc->sc_mru;
-	break;
-
-#ifdef VJC
-    case PPPIOCSMAXCID:
-	error = priv_check(td, PRIV_NET_PPP);
-	if (error)
-	    break;
-	if (sc->sc_comp) {
-	    s = splsoftnet();
-	    sl_compress_init(sc->sc_comp, *(int *)data);
-	    splx(s);
-	}
-	break;
-#endif
-
-    case PPPIOCXFERUNIT:
-	error = priv_check(td, PRIV_NET_PPP);
-	if (error)
-	    break;
-	sc->sc_xfer = p->p_pid;
-	break;
-
-#ifdef PPP_COMPRESS
-    case PPPIOCSCOMPRESS:
-	error = priv_check(td, PRIV_NET_PPP);
-	if (error)
-	    break;
-	odp = (struct ppp_option_data *) data;
-	nb = odp->length;
-	if (nb > sizeof(ccp_option))
-	    nb = sizeof(ccp_option);
-	if ((error = copyin(odp->ptr, ccp_option, nb)) != 0)
-	    break;
-	if (ccp_option[1] < 2) {  /* preliminary check on the length byte */
-	    error = EINVAL;
-	    break;
-	}
-	for (cp = ppp_compressors; *cp != NULL; ++cp)
-	    if ((*cp)->compress_proto == ccp_option[0]) {
-		/*
-		 * Found a handler for the protocol - try to allocate
-		 * a compressor or decompressor.
-		 */
-		error = 0;
-		if (odp->transmit) {
-		    s = splsoftnet();
-		    if (sc->sc_xc_state != NULL)
-			(*sc->sc_xcomp->comp_free)(sc->sc_xc_state);
-		    sc->sc_xcomp = *cp;
-		    sc->sc_xc_state = (*cp)->comp_alloc(ccp_option, nb);
-		    if (sc->sc_xc_state == NULL) {
-			if (sc->sc_flags & SC_DEBUG)
-			    if_printf(PPP2IFP(sc), "comp_alloc failed\n");
-			error = ENOBUFS;
-		    }
-		    splimp();
-		    sc->sc_flags &= ~SC_COMP_RUN;
-		    splx(s);
-		} else {
-		    s = splsoftnet();
-		    if (sc->sc_rc_state != NULL)
-			(*sc->sc_rcomp->decomp_free)(sc->sc_rc_state);
-		    sc->sc_rcomp = *cp;
-		    sc->sc_rc_state = (*cp)->decomp_alloc(ccp_option, nb);
-		    if (sc->sc_rc_state == NULL) {
-			if (sc->sc_flags & SC_DEBUG)
-			    if_printf(PPP2IFP(sc), "decomp_alloc failed\n");
-			error = ENOBUFS;
-		    }
-		    splimp();
-		    sc->sc_flags &= ~SC_DECOMP_RUN;
-		    splx(s);
-		}
-		return (error);
-	    }
-	if (sc->sc_flags & SC_DEBUG)
-	    if_printf(PPP2IFP(sc), "no compressor for [%x %x %x], %x\n",
-		   ccp_option[0], ccp_option[1], ccp_option[2], nb);
-	error = EINVAL;		/* no handler found */
-        break;
-#endif /* PPP_COMPRESS */
-
-    case PPPIOCGNPMODE:
-    case PPPIOCSNPMODE:
-	npi = (struct npioctl *) data;
-	npx = 0;	/* XXX: quiet gcc */
-	switch (npi->protocol) {
-	case PPP_IP:
-	    npx = NP_IP;
-	    break;
-	case PPP_IPV6:
-	    npx = NP_IPV6;
-	    break;
-	default:
-	    error = EINVAL;
-	}
-	if (error)
-	    break;
-	if (cmd == PPPIOCGNPMODE) {
-	    npi->mode = sc->sc_npmode[npx];
-	} else {
-	    error = priv_check(td, PRIV_NET_PPP);
-	    if (error)
-		break;
-	    if (npi->mode != sc->sc_npmode[npx]) {
-		s = splsoftnet();
-		sc->sc_npmode[npx] = npi->mode;
-		if (npi->mode != NPMODE_QUEUE) {
-		    ppp_requeue(sc);
-		    (*sc->sc_start)(sc);
-		}
-		splx(s);
-	    }
-	}
-	break;
-
-    case PPPIOCGIDLE:
-	s = splsoftnet();
-	t = time_uptime;
-	((struct ppp_idle *)data)->xmit_idle = t - sc->sc_last_sent;
-	((struct ppp_idle *)data)->recv_idle = t - sc->sc_last_recv;
-	splx(s);
-	break;
-
-#ifdef PPP_FILTER
-    case PPPIOCSPASS:
-    case PPPIOCSACTIVE:
-	nbp = (struct bpf_program *) data;
-	if ((unsigned) nbp->bf_len > BPF_MAXINSNS) {
-	    error = EINVAL;
-	    break;
-	}
-	newcodelen = nbp->bf_len * sizeof(struct bpf_insn);
-	if (newcodelen != 0) {
-	    newcode = malloc(newcodelen, M_DEVBUF, M_WAITOK);
-	    if (newcode == 0) {
-		error = EINVAL;		/* or sumpin */
-		break;
-	    }
-	    if ((error = copyin((caddr_t)nbp->bf_insns, (caddr_t)newcode,
-			       newcodelen)) != 0) {
-		free(newcode, M_DEVBUF);
-		break;
-	    }
-	    if (!bpf_validate(newcode, nbp->bf_len)) {
-		free(newcode, M_DEVBUF);
-		error = EINVAL;
-		break;
-	    }
-	} else
-	    newcode = 0;
-	bp = (cmd == PPPIOCSPASS)? &sc->sc_pass_filt: &sc->sc_active_filt;
-	oldcode = bp->bf_insns;
-	s = splimp();
-	bp->bf_len = nbp->bf_len;
-	bp->bf_insns = newcode;
-	splx(s);
-	if (oldcode != 0)
-	    free(oldcode, M_DEVBUF);
-	break;
-#endif
-
-    default:
-	error = ENOIOCTL;
-	break;
-    }
-    return (error);
-}
-
-/*
- * Process an ioctl request to the ppp network interface.
- */
-static int
-pppsioctl(ifp, cmd, data)
-    register struct ifnet *ifp;
-    u_long cmd;
-    caddr_t data;
-{
-    struct thread *td = curthread;	/* XXX */
-    register struct ppp_softc *sc = ifp->if_softc;
-    register struct ifaddr *ifa = (struct ifaddr *)data;
-    register struct ifreq *ifr = (struct ifreq *)data;
-    struct ppp_stats *psp;
-#ifdef	PPP_COMPRESS
-    struct ppp_comp_stats *pcp;
-#endif
-    int s = splimp(), error = 0;
-
-    switch (cmd) {
-    case SIOCSIFFLAGS:
-	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
-	    ifp->if_flags &= ~IFF_UP;
-	break;
-
-    case SIOCSIFADDR:
-    case SIOCAIFADDR:
-	switch(ifa->ifa_addr->sa_family) {
-#ifdef INET
-	case AF_INET:
-	    break;
-#endif
-#ifdef INET6
-	case AF_INET6:
-	    break;
-#endif
-#ifdef IPX
-	case AF_IPX:
-	    break;
-#endif
-	default:
-	    error = EAFNOSUPPORT;
-	    break;
-	}
-	break;
-
-    case SIOCSIFDSTADDR:
-	switch(ifa->ifa_addr->sa_family) {
-#ifdef INET
-	case AF_INET:
-	    break;
-#endif
-#ifdef INET6
-	case AF_INET6:
-	    break;
-#endif
-#ifdef IPX
-	case AF_IPX:
-	    break;
-#endif
-	default:
-	    error = EAFNOSUPPORT;
-	    break;
-	}
-	break;
-
-    case SIOCSIFMTU:
-	/*
-	 * XXXRW: Isn't this priv_check() check redundant to the one at the
-	 * ifnet layer?
-	 */
-	error = priv_check(td, PRIV_NET_SETIFMTU);
-	if (error)
-	    break;
-	if (ifr->ifr_mtu > PPP_MAXMTU)
-	    error = EINVAL;
-	else {
-	    PPP2IFP(sc)->if_mtu = ifr->ifr_mtu;
-	    if (sc->sc_setmtu)
-		    (*sc->sc_setmtu)(sc);
-	}
-	break;
-
-    case SIOCGIFMTU:
-	ifr->ifr_mtu = PPP2IFP(sc)->if_mtu;
-	break;
-
-    case SIOCADDMULTI:
-    case SIOCDELMULTI:
-	if (ifr == 0) {
-	    error = EAFNOSUPPORT;
-	    break;
-	}
-	switch(ifr->ifr_addr.sa_family) {
-#ifdef INET
-	case AF_INET:
-	    break;
-#endif
-#ifdef INET6
-	case AF_INET6:
-	    break;
-#endif
-	default:
-	    error = EAFNOSUPPORT;
-	    break;
-	}
-	break;
-
-    case SIOCGPPPSTATS:
-	psp = &((struct ifpppstatsreq *) data)->stats;
-	bzero(psp, sizeof(*psp));
-	psp->p = sc->sc_stats;
-#if defined(VJC) && !defined(SL_NO_STATS)
-	if (sc->sc_comp) {
-	    psp->vj.vjs_packets = sc->sc_comp->sls_packets;
-	    psp->vj.vjs_compressed = sc->sc_comp->sls_compressed;
-	    psp->vj.vjs_searches = sc->sc_comp->sls_searches;
-	    psp->vj.vjs_misses = sc->sc_comp->sls_misses;
-	    psp->vj.vjs_uncompressedin = sc->sc_comp->sls_uncompressedin;
-	    psp->vj.vjs_compressedin = sc->sc_comp->sls_compressedin;
-	    psp->vj.vjs_errorin = sc->sc_comp->sls_errorin;
-	    psp->vj.vjs_tossed = sc->sc_comp->sls_tossed;
-	}
-#endif /* VJC */
-	break;
-
-#ifdef PPP_COMPRESS
-    case SIOCGPPPCSTATS:
-	pcp = &((struct ifpppcstatsreq *) data)->stats;
-	bzero(pcp, sizeof(*pcp));
-	if (sc->sc_xc_state != NULL)
-	    (*sc->sc_xcomp->comp_stat)(sc->sc_xc_state, &pcp->c);
-	if (sc->sc_rc_state != NULL)
-	    (*sc->sc_rcomp->decomp_stat)(sc->sc_rc_state, &pcp->d);
-	break;
-#endif /* PPP_COMPRESS */
-
-    default:
-	error = ENOTTY;
-    }
-    splx(s);
-    return (error);
-}
-
-/*
- * Queue a packet.  Start transmission if not active.
- * Packet is placed in Information field of PPP frame.
- * Called at splnet as the if->if_output handler.
- * Called at splnet from pppwrite().
- */
-int
-pppoutput(ifp, m0, dst, rtp)
-    struct ifnet *ifp;
-    struct mbuf *m0;
-    struct sockaddr *dst;
-    struct rtentry *rtp;
-{
-    register struct ppp_softc *sc = ifp->if_softc;
-    int protocol, address, control;
-    u_char *cp;
-    int s, error;
-    struct ip *ip;
-    struct ifqueue *ifq;
-    enum NPmode mode;
-    int len;
-
-#ifdef MAC
-    error = mac_ifnet_check_transmit(ifp, m0);
-    if (error)
-	goto bad;
-#endif
-
-    if (sc->sc_devp == NULL || (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0
-	|| ((ifp->if_flags & IFF_UP) == 0 && dst->sa_family != AF_UNSPEC)) {
-	error = ENETDOWN;	/* sort of */
-	goto bad;
-    }
-
-    /*
-     * Compute PPP header.
-     */
-    m0->m_flags &= ~M_HIGHPRI;
-    switch (dst->sa_family) {
-#ifdef INET
-    case AF_INET:
-	address = PPP_ALLSTATIONS;
-	control = PPP_UI;
-	protocol = PPP_IP;
-	mode = sc->sc_npmode[NP_IP];
-
-	/*
-	 * If this packet has the "low delay" bit set in the IP header,
-	 * put it on the fastq instead.
-	 */
-	ip = mtod(m0, struct ip *);
-	if (ip->ip_tos & IPTOS_LOWDELAY)
-	    m0->m_flags |= M_HIGHPRI;
-	break;
-#endif
-#ifdef INET6
-    case AF_INET6:
-	address = PPP_ALLSTATIONS;	/*XXX*/
-	control = PPP_UI;		/*XXX*/
-	protocol = PPP_IPV6;
-	mode = sc->sc_npmode[NP_IPV6];
-
-#if 0	/* XXX flowinfo/traffic class, maybe? */
-	/*
-	 * If this packet has the "low delay" bit set in the IP header,
-	 * put it on the fastq instead.
-	 */
-	ip = mtod(m0, struct ip *);
-	if (ip->ip_tos & IPTOS_LOWDELAY)
-	    m0->m_flags |= M_HIGHPRI;
-#endif
-	break;
-#endif
-#ifdef IPX
-    case AF_IPX:
-	/*
-	 * This is pretty bogus.. We dont have an ipxcp module in pppd
-	 * yet to configure the link parameters.  Sigh. I guess a
-	 * manual ifconfig would do....  -Peter
-	 */
-	address = PPP_ALLSTATIONS;
-	control = PPP_UI;
-	protocol = PPP_IPX;
-	mode = NPMODE_PASS;
-	break;
-#endif
-    case AF_UNSPEC:
-	address = PPP_ADDRESS(dst->sa_data);
-	control = PPP_CONTROL(dst->sa_data);
-	protocol = PPP_PROTOCOL(dst->sa_data);
-	mode = NPMODE_PASS;
-	break;
-    default:
-	if_printf(ifp, "af%d not supported\n", dst->sa_family);
-	error = EAFNOSUPPORT;
-	goto bad;
-    }
-
-    /*
-     * Drop this packet, or return an error, if necessary.
-     */
-    if (mode == NPMODE_ERROR) {
-	error = ENETDOWN;
-	goto bad;
-    }
-    if (mode == NPMODE_DROP) {
-	error = 0;
-	goto bad;
-    }
-
-    /*
-     * Add PPP header.  If no space in first mbuf, allocate another.
-     * (This assumes M_LEADINGSPACE is always 0 for a cluster mbuf.)
-     */
-    if (M_LEADINGSPACE(m0) < PPP_HDRLEN) {
-	m0 = m_prepend(m0, PPP_HDRLEN, M_DONTWAIT);
-	if (m0 == 0) {
-	    error = ENOBUFS;
-	    goto bad;
-	}
-	m0->m_len = 0;
-    } else
-	m0->m_data -= PPP_HDRLEN;
-
-    cp = mtod(m0, u_char *);
-    *cp++ = address;
-    *cp++ = control;
-    *cp++ = protocol >> 8;
-    *cp++ = protocol & 0xff;
-    m0->m_len += PPP_HDRLEN;
-
-    len = m_length(m0, NULL);
-
-    if (sc->sc_flags & SC_LOG_OUTPKT) {
-	printf("%s output: ", ifp->if_xname);
-	pppdumpm(m0);
-    }
-
-    if ((protocol & 0x8000) == 0) {
-#ifdef PPP_FILTER
-	/*
-	 * Apply the pass and active filters to the packet,
-	 * but only if it is a data packet.
-	 */
-	*mtod(m0, u_char *) = 1;	/* indicates outbound */
-	if (sc->sc_pass_filt.bf_insns != 0
-	    && bpf_filter(sc->sc_pass_filt.bf_insns, (u_char *) m0,
-			  len, 0) == 0) {
-	    error = 0;		/* drop this packet */
-	    goto bad;
-	}
-
-	/*
-	 * Update the time we sent the most recent packet.
-	 */
-	if (sc->sc_active_filt.bf_insns == 0
-	    || bpf_filter(sc->sc_active_filt.bf_insns, (u_char *) m0, len, 0))
-	    sc->sc_last_sent = time_uptime;
-
-	*mtod(m0, u_char *) = address;
-#else
-	/*
-	 * Update the time we sent the most recent data packet.
-	 */
-	sc->sc_last_sent = time_uptime;
-#endif /* PPP_FILTER */
-    }
-
-    /*
-     * See if bpf wants to look at the packet.
-     */
-    BPF_MTAP(ifp, m0);
-
-    /*
-     * Put the packet on the appropriate queue.
-     */
-    s = splsoftnet();	/* redundant */
-    if (mode == NPMODE_QUEUE) {
-	/* XXX we should limit the number of packets on this queue */
-	*sc->sc_npqtail = m0;
-	m0->m_nextpkt = NULL;
-	sc->sc_npqtail = &m0->m_nextpkt;
-    } else {
-	/* fastq and if_snd are emptied at spl[soft]net now */
-	ifq = (m0->m_flags & M_HIGHPRI)? &sc->sc_fastq:
-	    (struct ifqueue *)&ifp->if_snd;
-        IF_LOCK(ifq);
-	if (_IF_QFULL(ifq) && dst->sa_family != AF_UNSPEC) {
-	    _IF_DROP(ifq);
-	    IF_UNLOCK(ifq);
-	    PPP2IFP(sc)->if_oerrors++;
-	    sc->sc_stats.ppp_oerrors++;
-	    error = ENOBUFS;
-	    goto bad;
-	}
-	_IF_ENQUEUE(ifq, m0);
-        IF_UNLOCK(ifq);
-	(*sc->sc_start)(sc);
-    }
-    getmicrotime(&ifp->if_lastchange);
-    ifp->if_opackets++;
-    ifp->if_obytes += len;
-
-    splx(s);
-    return (0);
-
-bad:
-    m_freem(m0);
-    return (error);
-}
-
-/*
- * After a change in the NPmode for some NP, move packets from the
- * npqueue to the send queue or the fast queue as appropriate.
- * Should be called at spl[soft]net.
- */
-static void
-ppp_requeue(sc)
-    struct ppp_softc *sc;
-{
-    struct mbuf *m, **mpp;
-    struct ifqueue *ifq;
-    enum NPmode mode;
-
-    for (mpp = &sc->sc_npqueue; (m = *mpp) != NULL; ) {
-	switch (PPP_PROTOCOL(mtod(m, u_char *))) {
-	case PPP_IP:
-	    mode = sc->sc_npmode[NP_IP];
-	    break;
-	case PPP_IPV6:
-	    mode = sc->sc_npmode[NP_IPV6];
-	    break;
-	default:
-	    mode = NPMODE_PASS;
-	}
-
-	switch (mode) {
-	case NPMODE_PASS:
-	    /*
-	     * This packet can now go on one of the queues to be sent.
-	     */
-	    *mpp = m->m_nextpkt;
-	    m->m_nextpkt = NULL;
-	    ifq = (m->m_flags & M_HIGHPRI)? &sc->sc_fastq:
-	        (struct ifqueue *)&PPP2IFP(sc)->if_snd;
-            if (! IF_HANDOFF(ifq, m, NULL)) {
-		PPP2IFP(sc)->if_oerrors++;
-		sc->sc_stats.ppp_oerrors++;
-	    }
-	    break;
-
-	case NPMODE_DROP:
-	case NPMODE_ERROR:
-	    *mpp = m->m_nextpkt;
-	    m_freem(m);
-	    break;
-
-	case NPMODE_QUEUE:
-	    mpp = &m->m_nextpkt;
-	    break;
-	}
-    }
-    sc->sc_npqtail = mpp;
-}
-
-/*
- * Transmitter has finished outputting some stuff;
- * remember to call sc->sc_start later at splsoftnet.
- */
-void
-ppp_restart(sc)
-    struct ppp_softc *sc;
-{
-    int s = splimp();
-
-    sc->sc_flags &= ~SC_TBUSY;
-    schednetisr(NETISR_PPP);
-    splx(s);
-}
-
-
-/*
- * Get a packet to send.  This procedure is intended to be called at
- * splsoftnet, since it may involve time-consuming operations such as
- * applying VJ compression, packet compression, address/control and/or
- * protocol field compression to the packet.
- */
-struct mbuf *
-ppp_dequeue(sc)
-    struct ppp_softc *sc;
-{
-    struct mbuf *m, *mp;
-    u_char *cp;
-    int address, control, protocol;
-
-    /*
-     * Grab a packet to send: first try the fast queue, then the
-     * normal queue.
-     */
-    IF_DEQUEUE(&sc->sc_fastq, m);
-    if (m == NULL)
-	IF_DEQUEUE(&PPP2IFP(sc)->if_snd, m);
-    if (m == NULL)
-	return NULL;
-
-    ++sc->sc_stats.ppp_opackets;
-
-    /*
-     * Extract the ppp header of the new packet.
-     * The ppp header will be in one mbuf.
-     */
-    cp = mtod(m, u_char *);
-    address = PPP_ADDRESS(cp);
-    control = PPP_CONTROL(cp);
-    protocol = PPP_PROTOCOL(cp);
-
-    switch (protocol) {
-    case PPP_IP:
-#ifdef VJC
-	/*
-	 * If the packet is a TCP/IP packet, see if we can compress it.
-	 */
-	if ((sc->sc_flags & SC_COMP_TCP) && sc->sc_comp != NULL) {
-	    struct ip *ip;
-	    int type;
-
-	    mp = m;
-	    ip = (struct ip *) (cp + PPP_HDRLEN);
-	    if (mp->m_len <= PPP_HDRLEN) {
-		mp = mp->m_next;
-		if (mp == NULL)
-		    break;
-		ip = mtod(mp, struct ip *);
-	    }
-	    /* this code assumes the IP/TCP header is in one non-shared mbuf */
-	    if (ip->ip_p == IPPROTO_TCP) {
-		type = sl_compress_tcp(mp, ip, sc->sc_comp,
-				       !(sc->sc_flags & SC_NO_TCP_CCID));
-		switch (type) {
-		case TYPE_UNCOMPRESSED_TCP:
-		    protocol = PPP_VJC_UNCOMP;
-		    break;
-		case TYPE_COMPRESSED_TCP:
-		    protocol = PPP_VJC_COMP;
-		    cp = mtod(m, u_char *);
-		    cp[0] = address;	/* header has moved */
-		    cp[1] = control;
-		    cp[2] = 0;
-		    break;
-		}
-		cp[3] = protocol;	/* update protocol in PPP header */
-	    }
-	}
-#endif	/* VJC */
-	break;
-
-#ifdef PPP_COMPRESS
-    case PPP_CCP:
-	ppp_ccp(sc, m, 0);
-	break;
-#endif	/* PPP_COMPRESS */
-    }
-
-#ifdef PPP_COMPRESS
-    if (protocol != PPP_LCP && protocol != PPP_CCP
-	&& sc->sc_xc_state && (sc->sc_flags & SC_COMP_RUN)) {
-	struct mbuf *mcomp = NULL;
-	int slen, clen;
-
-	slen = m_length(m, NULL);
-	clen = (*sc->sc_xcomp->compress)
-	    (sc->sc_xc_state, &mcomp, m, slen, PPP2IFP(sc)->if_mtu + PPP_HDRLEN);
-	if (mcomp != NULL) {
-	    if (sc->sc_flags & SC_CCP_UP) {
-		/* Send the compressed packet instead of the original. */
-		m_freem(m);
-		m = mcomp;
-		cp = mtod(m, u_char *);
-		protocol = cp[3];
-	    } else {
-		/* Can't transmit compressed packets until CCP is up. */
-		m_freem(mcomp);
-	    }
-	}
-    }
-#endif	/* PPP_COMPRESS */
-
-    /*
-     * Compress the address/control and protocol, if possible.
-     */
-    if (sc->sc_flags & SC_COMP_AC && address == PPP_ALLSTATIONS &&
-	control == PPP_UI && protocol != PPP_ALLSTATIONS &&
-	protocol != PPP_LCP) {
-	/* can compress address/control */
-	m->m_data += 2;
-	m->m_len -= 2;
-    }
-    if (sc->sc_flags & SC_COMP_PROT && protocol < 0xFF) {
-	/* can compress protocol */
-	if (mtod(m, u_char *) == cp) {
-	    cp[2] = cp[1];	/* move address/control up */
-	    cp[1] = cp[0];
-	}
-	++m->m_data;
-	--m->m_len;
-    }
-
-    return m;
-}
-
-/*
- * Software interrupt routine, called at spl[soft]net.
- */
-static void
-pppintr()
-{
-    struct ppp_softc *sc;
-    int s;
-    struct mbuf *m;
-
-    mtx_lock(&Giant);
-    PPP_LIST_LOCK();
-    LIST_FOREACH(sc, &ppp_softc_list, sc_list) {
-	s = splimp();
-	if (!(sc->sc_flags & SC_TBUSY)
-	    && (PPP2IFP(sc)->if_snd.ifq_head || sc->sc_fastq.ifq_head)) {
-	    sc->sc_flags |= SC_TBUSY;
-	    splx(s);
-	    (*sc->sc_start)(sc);
-	} else
-	    splx(s);
-	for (;;) {
-	    s = splimp();
-	    IF_DEQUEUE(&sc->sc_rawq, m);
-	    splx(s);
-	    if (m == NULL)
-		break;
-#ifdef MAC
-	    mac_ifnet_create_mbuf(PPP2IFP(sc), m);
-#endif
-	    ppp_inproc(sc, m);
-	}
-    }
-    PPP_LIST_UNLOCK();
-    mtx_unlock(&Giant);
-}
-
-#ifdef PPP_COMPRESS
-/*
- * Handle a CCP packet.  `rcvd' is 1 if the packet was received,
- * 0 if it is about to be transmitted.
- */
-static void
-ppp_ccp(sc, m, rcvd)
-    struct ppp_softc *sc;
-    struct mbuf *m;
-    int rcvd;
-{
-    u_char *dp, *ep;
-    struct mbuf *mp;
-    int slen, s;
-
-    /*
-     * Get a pointer to the data after the PPP header.
-     */
-    if (m->m_len <= PPP_HDRLEN) {
-	mp = m->m_next;
-	if (mp == NULL)
-	    return;
-	dp = (mp != NULL)? mtod(mp, u_char *): NULL;
-    } else {
-	mp = m;
-	dp = mtod(mp, u_char *) + PPP_HDRLEN;
-    }
-
-    ep = mtod(mp, u_char *) + mp->m_len;
-    if (dp + CCP_HDRLEN > ep)
-	return;
-    slen = CCP_LENGTH(dp);
-    if (dp + slen > ep) {
-	if (sc->sc_flags & SC_DEBUG)
-	    printf("if_ppp/ccp: not enough data in mbuf (%p+%x > %p+%x)\n",
-		   dp, slen, mtod(mp, u_char *), mp->m_len);
-	return;
-    }
-
-    switch (CCP_CODE(dp)) {
-    case CCP_CONFREQ:
-    case CCP_TERMREQ:
-    case CCP_TERMACK:
-	/* CCP must be going down - disable compression */
-	if (sc->sc_flags & SC_CCP_UP) {
-	    s = splimp();
-	    sc->sc_flags &= ~(SC_CCP_UP | SC_COMP_RUN | SC_DECOMP_RUN);
-	    splx(s);
-	}
-	break;
-
-    case CCP_CONFACK:
-	if (sc->sc_flags & SC_CCP_OPEN && !(sc->sc_flags & SC_CCP_UP)
-	    && slen >= CCP_HDRLEN + CCP_OPT_MINLEN
-	    && slen >= CCP_OPT_LENGTH(dp + CCP_HDRLEN) + CCP_HDRLEN) {
-	    if (!rcvd) {
-		/* we're agreeing to send compressed packets. */
-		if (sc->sc_xc_state != NULL
-		    && (*sc->sc_xcomp->comp_init)
-			(sc->sc_xc_state, dp + CCP_HDRLEN, slen - CCP_HDRLEN,
-			 PPP2IFP(sc)->if_dunit, 0, sc->sc_flags & SC_DEBUG)) {
-		    s = splimp();
-		    sc->sc_flags |= SC_COMP_RUN;
-		    splx(s);
-		}
-	    } else {
-		/* peer is agreeing to send compressed packets. */
-		if (sc->sc_rc_state != NULL
-		    && (*sc->sc_rcomp->decomp_init)
-			(sc->sc_rc_state, dp + CCP_HDRLEN, slen - CCP_HDRLEN,
-			 PPP2IFP(sc)->if_dunit, 0, sc->sc_mru,
-			 sc->sc_flags & SC_DEBUG)) {
-		    s = splimp();
-		    sc->sc_flags |= SC_DECOMP_RUN;
-		    sc->sc_flags &= ~(SC_DC_ERROR | SC_DC_FERROR);
-		    splx(s);
-		}
-	    }
-	}
-	break;
-
-    case CCP_RESETACK:
-	if (sc->sc_flags & SC_CCP_UP) {
-	    if (!rcvd) {
-		if (sc->sc_xc_state && (sc->sc_flags & SC_COMP_RUN))
-		    (*sc->sc_xcomp->comp_reset)(sc->sc_xc_state);
-	    } else {
-		if (sc->sc_rc_state && (sc->sc_flags & SC_DECOMP_RUN)) {
-		    (*sc->sc_rcomp->decomp_reset)(sc->sc_rc_state);
-		    s = splimp();
-		    sc->sc_flags &= ~SC_DC_ERROR;
-		    splx(s);
-		}
-	    }
-	}
-	break;
-    }
-}
-
-/*
- * CCP is down; free (de)compressor state if necessary.
- */
-static void
-ppp_ccp_closed(sc)
-    struct ppp_softc *sc;
-{
-    if (sc->sc_xc_state) {
-	(*sc->sc_xcomp->comp_free)(sc->sc_xc_state);
-	sc->sc_xc_state = NULL;
-    }
-    if (sc->sc_rc_state) {
-	(*sc->sc_rcomp->decomp_free)(sc->sc_rc_state);
-	sc->sc_rc_state = NULL;
-    }
-}
-#endif /* PPP_COMPRESS */
-
-/*
- * PPP packet input routine.
- * The caller has checked and removed the FCS and has inserted
- * the address/control bytes and the protocol high byte if they
- * were omitted.
- */
-void
-ppppktin(sc, m, lost)
-    struct ppp_softc *sc;
-    struct mbuf *m;
-    int lost;
-{
-    int s = splimp();
-
-    if (lost)
-	m->m_flags |= M_ERRMARK;
-    IF_ENQUEUE(&sc->sc_rawq, m);
-    schednetisr(NETISR_PPP);
-    splx(s);
-}
-
-/*
- * Process a received PPP packet, doing decompression as necessary.
- * Should be called at splsoftnet.
- */
-#define COMPTYPE(proto)	((proto) == PPP_VJC_COMP? TYPE_COMPRESSED_TCP: \
-			 TYPE_UNCOMPRESSED_TCP)
-
-static void
-ppp_inproc(sc, m)
-    struct ppp_softc *sc;
-    struct mbuf *m;
-{
-    struct ifnet *ifp = PPP2IFP(sc);
-    int isr;
-    int s, ilen = 0, xlen, proto, rv;
-    u_char *cp, adrs, ctrl;
-    struct mbuf *mp, *dmp = NULL;
-    u_char *iphdr;
-    u_int hlen;
-
-    sc->sc_stats.ppp_ipackets++;
-
-    if (sc->sc_flags & SC_LOG_INPKT) {
-	ilen = m_length(m, NULL);
-	if_printf(ifp, "got %d bytes\n", ilen);
-	pppdumpm(m);
-    }
-
-    cp = mtod(m, u_char *);
-    adrs = PPP_ADDRESS(cp);
-    ctrl = PPP_CONTROL(cp);
-    proto = PPP_PROTOCOL(cp);
-
-    if (m->m_flags & M_ERRMARK) {
-	m->m_flags &= ~M_ERRMARK;
-	s = splimp();
-	sc->sc_flags |= SC_VJ_RESET;
-	splx(s);
-    }
-
-#ifdef PPP_COMPRESS
-    /*
-     * Decompress this packet if necessary, update the receiver's
-     * dictionary, or take appropriate action on a CCP packet.
-     */
-    if (proto == PPP_COMP && sc->sc_rc_state && (sc->sc_flags & SC_DECOMP_RUN)
-	&& !(sc->sc_flags & SC_DC_ERROR) && !(sc->sc_flags & SC_DC_FERROR)) {
-	/* decompress this packet */
-	rv = (*sc->sc_rcomp->decompress)(sc->sc_rc_state, m, &dmp);
-	if (rv == DECOMP_OK) {
-	    m_freem(m);
-	    if (dmp == NULL) {
-		/* no error, but no decompressed packet produced */
-		return;
-	    }
-	    m = dmp;
-	    cp = mtod(m, u_char *);
-	    proto = PPP_PROTOCOL(cp);
-
-	} else {
-	    /*
-	     * An error has occurred in decompression.
-	     * Pass the compressed packet up to pppd, which may take
-	     * CCP down or issue a Reset-Req.
-	     */
-	    if (sc->sc_flags & SC_DEBUG)
-		if_printf(ifp, "decompress failed %d\n", rv);
-	    s = splimp();
-	    sc->sc_flags |= SC_VJ_RESET;
-	    if (rv == DECOMP_ERROR)
-		sc->sc_flags |= SC_DC_ERROR;
-	    else
-		sc->sc_flags |= SC_DC_FERROR;
-	    splx(s);
-	}
-
-    } else {
-	if (sc->sc_rc_state && (sc->sc_flags & SC_DECOMP_RUN)) {
-	    (*sc->sc_rcomp->incomp)(sc->sc_rc_state, m);
-	}
-	if (proto == PPP_CCP) {
-	    ppp_ccp(sc, m, 1);
-	}
-    }
-#endif
-
-    ilen = m_length(m, NULL);
-
-#ifdef VJC
-    if (sc->sc_flags & SC_VJ_RESET) {
-	/*
-	 * If we've missed a packet, we must toss subsequent compressed
-	 * packets which don't have an explicit connection ID.
-	 */
-	if (sc->sc_comp)
-	    sl_uncompress_tcp(NULL, 0, TYPE_ERROR, sc->sc_comp);
-	s = splimp();
-	sc->sc_flags &= ~SC_VJ_RESET;
-	splx(s);
-    }
-
-    /*
-     * See if we have a VJ-compressed packet to uncompress.
-     */
-    if (proto == PPP_VJC_COMP) {
-	if ((sc->sc_flags & SC_REJ_COMP_TCP) || sc->sc_comp == 0)
-	    goto bad;
-
-	xlen = sl_uncompress_tcp_core(cp + PPP_HDRLEN, m->m_len - PPP_HDRLEN,
-				      ilen - PPP_HDRLEN, TYPE_COMPRESSED_TCP,
-				      sc->sc_comp, &iphdr, &hlen);
-
-	if (xlen <= 0) {
-	    if (sc->sc_flags & SC_DEBUG)
-		if_printf(ifp, "VJ uncompress failed on type comp\n");
-	    goto bad;
-	}
-
-	/* Copy the PPP and IP headers into a new mbuf. */
-	MGETHDR(mp, M_DONTWAIT, MT_DATA);
-	if (mp == NULL)
-	    goto bad;
-	mp->m_len = 0;
-	mp->m_next = NULL;
-	if (hlen + PPP_HDRLEN > MHLEN) {
-	    MCLGET(mp, M_DONTWAIT);
-	    if (M_TRAILINGSPACE(mp) < hlen + PPP_HDRLEN) {
-		m_freem(mp);
-		goto bad;	/* lose if big headers and no clusters */
-	    }
-	}
-#ifdef MAC
-	mac_mbuf_copy(m, mp);
-#endif
-	cp = mtod(mp, u_char *);
-	cp[0] = adrs;
-	cp[1] = ctrl;
-	cp[2] = 0;
-	cp[3] = PPP_IP;
-	proto = PPP_IP;
-	bcopy(iphdr, cp + PPP_HDRLEN, hlen);
-	mp->m_len = hlen + PPP_HDRLEN;
-
-	/*
-	 * Trim the PPP and VJ headers off the old mbuf
-	 * and stick the new and old mbufs together.
-	 */
-	m->m_data += PPP_HDRLEN + xlen;
-	m->m_len -= PPP_HDRLEN + xlen;
-	if (m->m_len <= M_TRAILINGSPACE(mp)) {
-	    bcopy(mtod(m, u_char *), mtod(mp, u_char *) + mp->m_len, m->m_len);
-	    mp->m_len += m->m_len;
-	    mp->m_next = m_free(m);
-	} else {
-	    mp->m_next = m;
-	}
-	m = mp;
-	ilen += hlen - xlen;
-
-    } else if (proto == PPP_VJC_UNCOMP) {
-	if ((sc->sc_flags & SC_REJ_COMP_TCP) || sc->sc_comp == 0)
-	    goto bad;
-
-	xlen = sl_uncompress_tcp_core(cp + PPP_HDRLEN, m->m_len - PPP_HDRLEN,
-				      ilen - PPP_HDRLEN, TYPE_UNCOMPRESSED_TCP,
-				      sc->sc_comp, &iphdr, &hlen);
-
-	if (xlen < 0) {
-	    if (sc->sc_flags & SC_DEBUG)
-		if_printf(ifp, "VJ uncompress failed on type uncomp\n");
-	    goto bad;
-	}
-
-	proto = PPP_IP;
-	cp[3] = PPP_IP;
-    }
-#endif /* VJC */
-
-    /*
-     * If the packet will fit in a header mbuf, don't waste a
-     * whole cluster on it.
-     */
-    if (ilen <= MHLEN && M_IS_CLUSTER(m)) {
-	MGETHDR(mp, M_DONTWAIT, MT_DATA);
-	if (mp != NULL) {
-#ifdef MAC
-	    mac_mbuf_copy(m, mp);
-#endif
-	    m_copydata(m, 0, ilen, mtod(mp, caddr_t));
-	    m_freem(m);
-	    m = mp;
-	    m->m_len = ilen;
-	}
-    }
-    m->m_pkthdr.len = ilen;
-    m->m_pkthdr.rcvif = ifp;
-
-    if ((proto & 0x8000) == 0) {
-#ifdef PPP_FILTER
-	/*
-	 * See whether we want to pass this packet, and
-	 * if it counts as link activity.
-	 */
-	adrs = *mtod(m, u_char *);	/* save address field */
-	*mtod(m, u_char *) = 0;		/* indicate inbound */
-	if (sc->sc_pass_filt.bf_insns != 0
-	    && bpf_filter(sc->sc_pass_filt.bf_insns, (u_char *) m,
-			  ilen, 0) == 0) {
-	    /* drop this packet */
-	    m_freem(m);
-	    return;
-	}
-	if (sc->sc_active_filt.bf_insns == 0
-	    || bpf_filter(sc->sc_active_filt.bf_insns, (u_char *) m, ilen, 0))
-	    sc->sc_last_recv = time_uptime;
-
-	*mtod(m, u_char *) = adrs;
-#else
-	/*
-	 * Record the time that we received this packet.
-	 */
-	sc->sc_last_recv = time_uptime;
-#endif /* PPP_FILTER */
-    }
-
-    /* See if bpf wants to look at the packet. */
-    BPF_MTAP(PPP2IFP(sc), m);
-
-    isr = -1;
-    switch (proto) {
-#ifdef INET
-    case PPP_IP:
-	/*
-	 * IP packet - take off the ppp header and pass it up to IP.
-	 */
-	if ((ifp->if_flags & IFF_UP) == 0
-	    || sc->sc_npmode[NP_IP] != NPMODE_PASS) {
-	    /* interface is down - drop the packet. */
-	    m_freem(m);
-	    return;
-	}
-	m->m_pkthdr.len -= PPP_HDRLEN;
-	m->m_data += PPP_HDRLEN;
-	m->m_len -= PPP_HDRLEN;
-	if ((m = ip_fastforward(m)) == NULL)
-	    return;
-	isr = NETISR_IP;
-	break;
-#endif
-#ifdef INET6
-    case PPP_IPV6:
-	/*
-	 * IPv6 packet - take off the ppp header and pass it up to IPv6.
-	 */
-	if ((ifp->if_flags & IFF_UP) == 0
-	    || sc->sc_npmode[NP_IPV6] != NPMODE_PASS) {
-	    /* interface is down - drop the packet. */
-	    m_freem(m);
-	    return;
-	}
-	m->m_pkthdr.len -= PPP_HDRLEN;
-	m->m_data += PPP_HDRLEN;
-	m->m_len -= PPP_HDRLEN;
-	isr = NETISR_IPV6;
-	break;
-#endif
-#ifdef IPX
-    case PPP_IPX:
-	/*
-	 * IPX packet - take off the ppp header and pass it up to IPX.
-	 */
-	if ((PPP2IFP(sc)->if_flags & IFF_UP) == 0
-	    /* XXX: || sc->sc_npmode[NP_IPX] != NPMODE_PASS*/) {
-	    /* interface is down - drop the packet. */
-	    m_freem(m);
-	    return;
-	}
-	m->m_pkthdr.len -= PPP_HDRLEN;
-	m->m_data += PPP_HDRLEN;
-	m->m_len -= PPP_HDRLEN;
-	isr = NETISR_IPX;
-	sc->sc_last_recv = time_uptime;	/* update time of last pkt rcvd */
-	break;
-#endif
-
-    default:
-	/*
-	 * Some other protocol - place on input queue for read().
-	 */
-	break;
-    }
-
-    if (isr == -1)
-      rv = IF_HANDOFF(&sc->sc_inq, m, NULL);
-    else
-      rv = netisr_queue(isr, m);	/* (0) on success. */
-    if ((isr == -1 && !rv) || (isr != -1 && rv)) {
-	if (sc->sc_flags & SC_DEBUG)
-	    if_printf(ifp, "input queue full\n");
-	ifp->if_iqdrops++;
-	m = NULL;
-	goto bad;
-    }
-    ifp->if_ipackets++;
-    ifp->if_ibytes += ilen;
-    getmicrotime(&ifp->if_lastchange);
-
-    if (isr == -1)
-	(*sc->sc_ctlp)(sc);
-
-    return;
-
- bad:
-    if (m)
-        m_freem(m);
-    PPP2IFP(sc)->if_ierrors++;
-    sc->sc_stats.ppp_ierrors++;
-}
-
-#define MAX_DUMP_BYTES	128
-
-static void
-pppdumpm(m0)
-    struct mbuf *m0;
-{
-    char buf[3*MAX_DUMP_BYTES+4];
-    char *bp = buf;
-    struct mbuf *m;
-
-    for (m = m0; m; m = m->m_next) {
-	int l = m->m_len;
-	u_char *rptr = (u_char *)m->m_data;
-
-	while (l--) {
-	    if (bp > buf + (sizeof(buf) - 4))
-		goto done;
-	    *bp++ = hex2ascii(*rptr >> 4);
-	    *bp++ = hex2ascii(*rptr++ & 0xf);
-	}
-
-	if (m->m_next) {
-	    if (bp > buf + (sizeof(buf) - 3))
-		goto done;
-	    *bp++ = '|';
-	} else
-	    *bp++ = ' ';
-    }
-done:
-    if (m)
-	*bp++ = '>';
-    *bp = 0;
-    printf("%s\n", buf);
-}
diff --git a/sys/net/if_ppp.h b/sys/net/if_ppp.h
deleted file mode 100644
index d23255b..0000000
--- a/sys/net/if_ppp.h
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * if_ppp.h - Point-to-Point Protocol definitions.
- */
-
-/*-
- * Copyright (c) 1989 Carnegie Mellon University.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms are permitted
- * provided that the above copyright notice and this paragraph are
- * duplicated in all such forms and that any documentation,
- * advertising materials, and other materials related to such
- * distribution and use acknowledge that the software was developed
- * by Carnegie Mellon University.  The name of the
- * University may not be used to endorse or promote products derived
- * from this software without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
- *
- * $FreeBSD$
- */
-
-#ifndef _IF_PPP_H_
-#define _IF_PPP_H_
-
-/* XXX this used to be self-contained. */
-#include <net/ppp_defs.h>
-#include <net/if.h>
-
-/*
- * Packet sizes
- */
-#define	PPP_MTU		1500	/* Default MTU (size of Info field) */
-#define PPP_MAXMRU	65000	/* Largest MRU we allow */
-#define PPP_MAXMTU	16384	/* Largest MTU we allow */
-
-/*
- * Bit definitions for flags.
- */
-#define SC_COMP_PROT	0x00000001	/* protocol compression (output) */
-#define SC_COMP_AC	0x00000002	/* header compression (output) */
-#define	SC_COMP_TCP	0x00000004	/* TCP (VJ) compression (output) */
-#define SC_NO_TCP_CCID	0x00000008	/* disable VJ connection-id comp. */
-#define SC_REJ_COMP_AC	0x00000010	/* reject adrs/ctrl comp. on input */
-#define SC_REJ_COMP_TCP	0x00000020	/* reject TCP (VJ) comp. on input */
-#define SC_CCP_OPEN	0x00000040	/* Look at CCP packets */
-#define SC_CCP_UP	0x00000080	/* May send/recv compressed packets */
-#define SC_DEBUG	0x00010000	/* enable debug messages */
-#define SC_LOG_INPKT	0x00020000	/* log contents of good pkts recvd */
-#define SC_LOG_OUTPKT	0x00040000	/* log contents of pkts sent */
-#define SC_LOG_RAWIN	0x00080000	/* log all chars received */
-#define SC_LOG_FLUSH	0x00100000	/* log all chars flushed */
-#define SC_RCV_B7_0	0x01000000	/* have rcvd char with bit 7 = 0 */
-#define SC_RCV_B7_1	0x02000000	/* have rcvd char with bit 7 = 1 */
-#define SC_RCV_EVNP	0x04000000	/* have rcvd char with even parity */
-#define SC_RCV_ODDP	0x08000000	/* have rcvd char with odd parity */
-#define	SC_MASK		0x0fff00ff	/* bits that user can change */
-
-/*
- * State bits in sc_flags, not changeable by user.
- */
-#define SC_TIMEOUT	0x00000400	/* timeout is currently pending */
-#define SC_VJ_RESET	0x00000800	/* need to reset VJ decomp */
-#define SC_COMP_RUN	0x00001000	/* compressor has been initiated */
-#define SC_DECOMP_RUN	0x00002000	/* decompressor has been initiated */
-#define SC_DC_ERROR	0x00004000	/* non-fatal decomp error detected */
-#define SC_DC_FERROR	0x00008000	/* fatal decomp error detected */
-#define SC_TBUSY	0x10000000	/* xmitter doesn't need a packet yet */
-#define SC_PKTLOST	0x20000000	/* have lost or dropped a packet */
-#define	SC_FLUSH	0x40000000	/* flush input until next PPP_FLAG */
-#define	SC_ESCAPED	0x80000000	/* saw a PPP_ESCAPE */
-
-/*
- * Ioctl definitions.
- */
-
-struct npioctl {
-    int		protocol;	/* PPP procotol, e.g. PPP_IP */
-    enum NPmode	mode;
-};
-
-/* Structure describing a CCP configuration option, for PPPIOCSCOMPRESS */
-struct ppp_option_data {
-	u_char	*ptr;
-	u_int	length;
-	int	transmit;
-};
-
-struct ifpppstatsreq {
-    char ifr_name[IFNAMSIZ];
-    struct ppp_stats stats;
-};
-
-struct ifpppcstatsreq {
-    char ifr_name[IFNAMSIZ];
-    struct ppp_comp_stats stats;
-};
-
-/*
- * Ioctl definitions.
- */
-
-#define	PPPIOCGFLAGS	_IOR('t', 90, int)	/* get configuration flags */
-#define	PPPIOCSFLAGS	_IOW('t', 89, int)	/* set configuration flags */
-#define	PPPIOCGASYNCMAP	_IOR('t', 88, int)	/* get async map */
-#define	PPPIOCSASYNCMAP	_IOW('t', 87, int)	/* set async map */
-#define	PPPIOCGUNIT	_IOR('t', 86, int)	/* get ppp unit number */
-#define	PPPIOCGRASYNCMAP _IOR('t', 85, int)	/* get receive async map */
-#define	PPPIOCSRASYNCMAP _IOW('t', 84, int)	/* set receive async map */
-#define	PPPIOCGMRU	_IOR('t', 83, int)	/* get max receive unit */
-#define	PPPIOCSMRU	_IOW('t', 82, int)	/* set max receive unit */
-#define	PPPIOCSMAXCID	_IOW('t', 81, int)	/* set VJ max slot ID */
-#define PPPIOCGXASYNCMAP _IOR('t', 80, ext_accm) /* get extended ACCM */
-#define PPPIOCSXASYNCMAP _IOW('t', 79, ext_accm) /* set extended ACCM */
-#define PPPIOCXFERUNIT	_IO('t', 78)		/* transfer PPP unit */
-#define PPPIOCSCOMPRESS	_IOW('t', 77, struct ppp_option_data)
-#define PPPIOCGNPMODE	_IOWR('t', 76, struct npioctl) /* get NP mode */
-#define PPPIOCSNPMODE	_IOW('t', 75, struct npioctl)  /* set NP mode */
-#define PPPIOCGIDLE	_IOR('t', 74, struct ppp_idle) /* get idle time */
-#define PPPIOCSPASS	_IOW('t', 71, struct bpf_program) /* set pass filter */
-#define PPPIOCSACTIVE	_IOW('t', 70, struct bpf_program) /* set active filt */
-
-/* PPPIOC[GS]MTU are alternatives to SIOC[GS]IFMTU, used under Ultrix */
-#define PPPIOCGMTU	_IOR('t', 73, int)	/* get interface MTU */
-#define PPPIOCSMTU	_IOW('t', 72, int)	/* set interface MTU */
-
-/*
- * These two are interface ioctls so that pppstats can do them on
- * a socket without having to open the serial device.
- */
-#define SIOCGPPPSTATS	_IOWR('i', 123, struct ifpppstatsreq)
-#define SIOCGPPPCSTATS	_IOWR('i', 122, struct ifpppcstatsreq)
-
-#if !defined(ifr_mtu)
-#define ifr_mtu	ifr_ifru.ifru_metric
-#endif
-
-#endif /* _IF_PPP_H_ */
diff --git a/sys/net/if_pppvar.h b/sys/net/if_pppvar.h
deleted file mode 100644
index 77e1ade..0000000
--- a/sys/net/if_pppvar.h
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * if_pppvar.h - private structures and declarations for PPP.
- */
-/*-
- * Copyright (c) 1994 The Australian National University.
- * All rights reserved.
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation is hereby granted, provided that the above copyright
- * notice appears in all copies.  This software is provided without any
- * warranty, express or implied. The Australian National University
- * makes no representations about the suitability of this software for
- * any purpose.
- *
- * IN NO EVENT SHALL THE AUSTRALIAN NATIONAL UNIVERSITY BE LIABLE TO ANY
- * PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
- * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF
- * THE AUSTRALIAN NATIONAL UNIVERSITY HAVE BEEN ADVISED OF THE POSSIBILITY
- * OF SUCH DAMAGE.
- *
- * THE AUSTRALIAN NATIONAL UNIVERSITY SPECIFICALLY DISCLAIMS ANY WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
- * ON AN "AS IS" BASIS, AND THE AUSTRALIAN NATIONAL UNIVERSITY HAS NO
- * OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS,
- * OR MODIFICATIONS.
- *
- * Copyright (c) 1989 Carnegie Mellon University.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms are permitted
- * provided that the above copyright notice and this paragraph are
- * duplicated in all such forms and that any documentation,
- * advertising materials, and other materials related to such
- * distribution and use acknowledge that the software was developed
- * by Carnegie Mellon University.  The name of the
- * University may not be used to endorse or promote products derived
- * from this software without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
- *
- * $FreeBSD$
- */
-
-/*
- * Supported network protocols.  These values are used for
- * indexing sc_npmode.
- */
-#define NP_IP	0		/* Internet Protocol */
-#define NP_IPV6	1		/* Internet Protocol version 6 */
-#define NUM_NP	2		/* Number of NPs. */
-
-/*
- * Structure describing each ppp unit.
- */
-struct ppp_softc {
-	struct	ifnet *sc_ifp;		/* network-visible interface */
-/*hi*/	u_int	sc_flags;		/* control/status bits; see if_ppp.h */
-	struct	callout sc_timo_ch;	/* Used for scheduling timeouts */
-	void	*sc_devp;		/* pointer to device-dep structure */
-	void	(*sc_start)(struct ppp_softc *);	/* start output proc */
-	void	(*sc_ctlp)(struct ppp_softc *); /* rcvd control pkt */
-	void	(*sc_relinq)(struct ppp_softc *); /* relinquish ifunit */
-	void	(*sc_setmtu)(struct ppp_softc *); /* set mtu */
-	short	sc_mru;			/* max receive unit */
-	pid_t	sc_xfer;		/* used in transferring unit */
-/*hi*/	struct	ifqueue sc_rawq;	/* received packets */
-/*net*/	struct	ifqueue sc_inq;		/* queue of input packets for daemon */
-/*net*/	struct	ifqueue sc_fastq;	/* interactive output packet q */
-	struct	mbuf *sc_npqueue;	/* output packets not to be sent yet */
-	struct	mbuf **sc_npqtail;	/* ptr to last next ptr in npqueue */
-	struct	pppstat sc_stats;	/* count of bytes/pkts sent/rcvd */
-	enum	NPmode sc_npmode[NUM_NP]; /* what to do with each NP */
-	struct	compressor *sc_xcomp;	/* transmit compressor */
-	void	*sc_xc_state;		/* transmit compressor state */
-	struct	compressor *sc_rcomp;	/* receive decompressor */
-	void	*sc_rc_state;		/* receive decompressor state */
-	time_t	sc_last_sent;		/* time (secs) last NP pkt sent */
-	time_t	sc_last_recv;		/* time (secs) last NP pkt rcvd */
-#ifdef PPP_FILTER
-	struct	bpf_program sc_pass_filt;   /* filter for packets to pass */
-	struct	bpf_program sc_active_filt; /* filter for "non-idle" packets */
-#endif /* PPP_FILTER */
-#ifdef	VJC
-	struct	slcompress *sc_comp; 	/* vjc control buffer */
-#endif
-
-	/* Device-dependent part for async lines. */
-	ext_accm sc_asyncmap;		/* async control character map */
-	u_long	sc_rasyncmap;		/* receive async control char map */
-	struct	mbuf *sc_outm;		/* mbuf chain currently being output */
-	struct	mbuf *sc_m;		/* pointer to input mbuf chain */
-	struct	mbuf *sc_mc;		/* pointer to current input mbuf */
-	char	*sc_mp;			/* ptr to next char in input mbuf */
-	short	sc_ilen;		/* length of input packet so far */
-	u_short	sc_fcs;			/* FCS so far (input) */
-	u_short	sc_outfcs;		/* FCS so far for output packet */
-	u_char	sc_rawin[16];		/* chars as received */
-	int	sc_rawin_count;		/* # in sc_rawin */
-	LIST_ENTRY(ppp_softc) sc_list;
-};
-#define	PPP2IFP(sc)	((sc)->sc_ifp)
-
-struct	ppp_softc *pppalloc(pid_t pid);
-void	pppdealloc(struct ppp_softc *sc);
-int	pppioctl(struct ppp_softc *sc, u_long cmd, caddr_t data, int flag,
-	    struct thread *td);
-int	pppoutput(struct ifnet *ifp, struct mbuf *m0, struct sockaddr *dst,
-	    struct rtentry *rtp);
-void	ppp_restart(struct ppp_softc *sc);
-void	ppppktin(struct ppp_softc *sc, struct mbuf *m, int lost);
-struct	mbuf *ppp_dequeue(struct ppp_softc *sc);
diff --git a/sys/net/if_sl.c b/sys/net/if_sl.c
deleted file mode 100644
index 10853e9..0000000
--- a/sys/net/if_sl.c
+++ /dev/null
@@ -1,1110 +0,0 @@
-/*-
- * Copyright (c) 1987, 1989, 1992, 1993
- *	The Regents of the University of California.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)if_sl.c	8.6 (Berkeley) 2/1/94
- * $FreeBSD$
- */
-
-/*
- * Serial Line interface
- *
- * Rick Adams
- * Center for Seismic Studies
- * 1300 N 17th Street, Suite 1450
- * Arlington, Virginia 22209
- * (703)276-7900
- * rick@seismo.ARPA
- * seismo!rick
- *
- * Pounded on heavily by Chris Torek (chris@mimsy.umd.edu, umcp-cs!chris).
- * N.B.: this belongs in netinet, not net, the way it stands now.
- * Should have a link-layer type designation, but wouldn't be
- * backwards-compatible.
- *
- * Converted to 4.3BSD Beta by Chris Torek.
- * Other changes made at Berkeley, based in part on code by Kirk Smith.
- * W. Jolitz added slip abort.
- *
- * Hacked almost beyond recognition by Van Jacobson (van@helios.ee.lbl.gov).
- * Added priority queuing for "interactive" traffic; hooks for TCP
- * header compression; ICMP filtering (at 2400 baud, some cretin
- * pinging you can use up all your bandwidth).  Made low clist behavior
- * more robust and slightly less likely to hang serial line.
- * Sped up a bunch of things.
- *
- * Note that splimp() is used throughout to block both (tty) input
- * interrupts and network activity; thus, splimp must be >= spltty.
- */
-
-#include "opt_inet.h"
-#include "opt_slip.h"
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/malloc.h>
-#include <sys/mbuf.h>
-#include <sys/priv.h>
-#include <sys/proc.h>
-#include <sys/socket.h>
-#include <sys/sockio.h>
-#include <sys/fcntl.h>
-#include <sys/signalvar.h>
-#include <sys/tty.h>
-#include <sys/clist.h>
-#include <sys/kernel.h>
-#include <sys/conf.h>
-#include <sys/module.h>
-#include <sys/proc.h>
-
-#include <net/if.h>
-#include <net/if_types.h>
-#include <net/netisr.h>
-
-#if INET
-#include <netinet/in.h>
-#include <netinet/in_systm.h>
-#include <netinet/in_var.h>
-#include <netinet/ip.h>
-#else
-#error "Huh? Slip without inet?"
-#endif
-
-#include <net/slcompress.h>
-#include <net/if_slvar.h>
-#include <net/slip.h>
-
-#include <net/bpf.h>
-
-static MALLOC_DEFINE(M_SL, "sl", "SLIP Interface");
-
-/*
- * SLRMAX is a hard limit on input packet size.  To simplify the code
- * and improve performance, we require that packets fit in an mbuf
- * cluster, and if we get a compressed packet, there's enough extra
- * room to expand the header into a max length tcp/ip header (128
- * bytes).  So, SLRMAX can be at most
- *	MCLBYTES - 128
- *
- * SLMTU is the default transmit MTU. The transmit MTU should be kept
- * small enough so that interactive use doesn't suffer, but large
- * enough to provide good performance. 552 is a good choice for SLMTU
- * because it is high enough to not fragment TCP packets being routed
- * through this host. Packet fragmentation is bad with SLIP because
- * fragment headers aren't compressed. The previous assumptions about
- * the best MTU value don't really hold when using modern modems with
- * BTLZ data compression because the modem buffers play a much larger
- * role in interactive performance than the MTU. The MTU can be changed
- * at any time to suit the specific environment with ifconfig(8), and
- * its maximum value is defined as SLTMAX. SLTMAX must not be so large
- * that it would overflow the stack if BPF is configured (XXX; if_ppp.c
- * handles this better).
- *
- * SLIP_HIWAT is the amount of data that will be queued 'downstream'
- * of us (i.e., in clists waiting to be picked up by the tty output
- * interrupt).  If we queue a lot of data downstream, it's immune to
- * our t.o.s. queuing.
- * E.g., if SLIP_HIWAT is 1024, the interactive traffic in mixed
- * telnet/ftp will see a 1 sec wait, independent of the mtu (the
- * wait is dependent on the ftp window size but that's typically
- * 1k - 4k).  So, we want SLIP_HIWAT just big enough to amortize
- * the cost (in idle time on the wire) of the tty driver running
- * off the end of its clists & having to call back slstart for a
- * new packet.  For a tty interface with any buffering at all, this
- * cost will be zero.  Even with a totally brain dead interface (like
- * the one on a typical workstation), the cost will be <= 1 character
- * time.  So, setting SLIP_HIWAT to ~100 guarantees that we'll lose
- * at most 1% while maintaining good interactive response.
- */
-#define	BUFOFFSET	(128+sizeof(struct ifnet **)+SLIP_HDRLEN)
-#define	SLRMAX		(MCLBYTES - BUFOFFSET)
-#define	SLBUFSIZE	(SLRMAX + BUFOFFSET)
-#ifndef SLMTU
-#define	SLMTU		552		/* default MTU */
-#endif
-#define	SLTMAX		1500		/* maximum MTU */
-#define	SLIP_HIWAT	roundup(50,CBSIZE)
-#define	CLISTRESERVE	1024		/* Can't let clists get too low */
-
-/*
- * SLIP ABORT ESCAPE MECHANISM:
- *	(inspired by HAYES modem escape arrangement)
- *	1sec escape 1sec escape 1sec escape { 1sec escape 1sec escape }
- *	within window time signals a "soft" exit from slip mode by remote end
- *	if the IFF_DEBUG flag is on.
- */
-#define	ABT_ESC		'\033'	/* can't be t_intr - distant host must know it*/
-#define	ABT_IDLE	1	/* in seconds - idle before an escape */
-#define	ABT_COUNT	3	/* count of escapes for abort */
-#define	ABT_WINDOW	(ABT_COUNT*2+2)	/* in seconds - time to count */
-
-static LIST_HEAD(sl_list, sl_softc) sl_list;
-
-#define FRAME_END	 	0xc0		/* Frame End */
-#define FRAME_ESCAPE		0xdb		/* Frame Esc */
-#define TRANS_FRAME_END	 	0xdc		/* transposed frame end */
-#define TRANS_FRAME_ESCAPE 	0xdd		/* transposed frame esc */
-
-static int slisstatic(int);
-static void slmarkstatic(int);
-static struct sl_softc *slcreate(void);
-static void sldestroy(struct sl_softc *sc);
-static struct mbuf *sl_btom(struct sl_softc *, int);
-static timeout_t sl_keepalive;
-static timeout_t sl_outfill;
-static l_close_t	slclose;
-static l_rint_t		slinput;
-static l_ioctl_t	sltioctl;
-static l_start_t	sltstart;
-static int	slioctl(struct ifnet *, u_long, caddr_t);
-static int	slopen(struct cdev *, struct tty *);
-static int	sloutput(struct ifnet *,
-	    struct mbuf *, struct sockaddr *, struct rtentry *);
-static void	slstart(struct ifnet *);
-
-static struct linesw slipdisc = {
-	.l_open =	slopen,
-	.l_close =	slclose,
-	.l_read =	l_noread,
-	.l_write =	l_nowrite,
-	.l_ioctl =	sltioctl,
-	.l_rint =	slinput,
-	.l_start =	sltstart,
-	.l_modem =	ttymodem
-};
-
-/*
- * Called from boot code to establish sl interfaces.
- */
-static int
-sl_modevent(module_t mod, int type, void *data) 
-{ 
-	switch (type) { 
-	case MOD_LOAD: 
-		ldisc_register(SLIPDISC, &slipdisc);
-		LIST_INIT(&sl_list);
-		break; 
-	case MOD_UNLOAD: 
-		ldisc_deregister(SLIPDISC);
-		printf("if_sl module unload - not possible for this module type\n"); 
-		return EINVAL; 
-	default:
-		return EOPNOTSUPP;
-	} 
-	return 0; 
-} 
-
-static moduledata_t sl_mod = { 
-	"if_sl", 
-	sl_modevent, 
-	0
-}; 
-
-DECLARE_MODULE(if_sl, sl_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
-
-static int *st_unit_list;
-static size_t st_unit_max = 0;
-
-static int
-slisunitfree(int unit)
-{
-	struct sl_softc *sc;
-
-	LIST_FOREACH(sc, &sl_list, sl_next) {
-		if (SL2IFP(sc)->if_dunit == unit)
-			return (0);
-	}
-	return (1);
-}
-
-static struct sl_softc *
-sl_for_tty(struct tty *tp)
-{
-	struct sl_softc *nc;
-
-	LIST_FOREACH(nc, &sl_list, sl_next) {
-		if (nc->sc_ttyp == tp)
-			return (nc);
-	}
-	return (NULL);
-}
-static int
-slisstatic(int unit)
-{
-	size_t i;
-
-	for (i = 0; i < st_unit_max; i++)
-		if (st_unit_list[i] == unit)
-			return 1;
-	return 0;
-}
-
-static void
-slmarkstatic(int unit)
-{
-	int *t;
-
-	if (slisstatic(unit))
-		return;
-
-	t = malloc(sizeof(int) * (st_unit_max+1), M_SL, M_NOWAIT);
-	if (t == NULL)
-		return;
-
-	if (st_unit_list) {
-		bcopy(st_unit_list, t, sizeof(int) * st_unit_max);
-		free(st_unit_list, M_SL);
-	}
-	st_unit_list = t;
-	st_unit_list[st_unit_max] = unit;
-	st_unit_max++;
-}
-
-static struct sl_softc *
-slcreate(void)
-{
-	struct sl_softc *sc;
-	int unit;
-	struct mbuf *m;
-
-	sc = malloc(sizeof(*sc), M_SL, M_WAITOK | M_ZERO);
-	sc->sc_ifp = if_alloc(IFT_SLIP);
-	if (sc->sc_ifp == NULL) {
-		free(sc, M_SL);
-		return (NULL);
-	}
-
-	m = m_gethdr(M_WAIT, MT_DATA);
-	MCLGET(m, M_WAIT);
-
-	sc->sc_ep = mtod(m, u_char *) + SLBUFSIZE;
-	sc->sc_mbuf = m;
-	sc->sc_buf = sc->sc_ep - SLRMAX;
-	sc->sc_mp = sc->sc_buf;
-	sl_compress_init(&sc->sc_comp, -1);
-
-	SL2IFP(sc)->if_softc = sc;
-	SL2IFP(sc)->if_mtu = SLMTU;
-	SL2IFP(sc)->if_flags =
-#ifdef SLIP_IFF_OPTS
-	    SLIP_IFF_OPTS;
-#else
-	    IFF_POINTOPOINT | SC_AUTOCOMP | IFF_MULTICAST | IFF_NEEDSGIANT;
-#endif
-	SL2IFP(sc)->if_ioctl = slioctl;
-	SL2IFP(sc)->if_output = sloutput;
-	SL2IFP(sc)->if_start = slstart;
-	SL2IFP(sc)->if_snd.ifq_maxlen = 50;
-	sc->sc_fastq.ifq_maxlen = 32;
-	SL2IFP(sc)->if_linkmib = sc;
-	SL2IFP(sc)->if_linkmiblen = sizeof *sc;
-	mtx_init(&sc->sc_fastq.ifq_mtx, "sl_fastq", NULL, MTX_DEF);
-
-	/*
-	 * Find a suitable unit number.
-	 */
-	for (unit=0; ; unit++) {
-		if (slisstatic(unit))
-			continue;
-		if (!slisunitfree(unit))
-			continue;
-		break;
-	}
-	if_initname(SL2IFP(sc), "sl", unit);
-	LIST_INSERT_HEAD(&sl_list, sc, sl_next);
-
-	if_attach(SL2IFP(sc));
-	bpfattach(SL2IFP(sc), DLT_SLIP, SLIP_HDRLEN);
-
-	return sc;
-}
-
-
-/*
- * Line specific open routine.
- * Attach the given tty to the first available sl unit.
- */
-/* ARGSUSED */
-static int
-slopen(struct cdev *dev, register struct tty *tp)
-{
-	register struct sl_softc *sc;
-	int s, error;
-
-	error = priv_check(curthread, PRIV_NET_SLIP);
-	if (error)
-		return (error);
-
-	if ((sc = slcreate()) == NULL)
-		return (ENOBUFS);
-
-	tp->t_hotchar = FRAME_END;
-	sc->sc_ttyp = tp;
-	SL2IFP(sc)->if_baudrate = tp->t_ospeed;
-	ttyflush(tp, FREAD | FWRITE);
-
-	/*
-	 * We don't use t_canq or t_rawq, so reduce their
-	 * cblock resources to 0.  Reserve enough cblocks
-	 * for t_outq to guarantee that we can fit a full
-	 * packet if the SLIP_HIWAT check allows slstart()
-	 * to loop.  Use the same value for the cblock
-	 * limit since the reserved blocks should always
-	 * be enough.  Reserving cblocks probably makes
-	 * the CLISTRESERVE check unnecessary and wasteful.
-	 */
-	clist_alloc_cblocks(&tp->t_canq, 0, 0);
-	clist_alloc_cblocks(&tp->t_outq,
-	    SLIP_HIWAT + 2 * SL2IFP(sc)->if_mtu + 1,
-	    SLIP_HIWAT + 2 * SL2IFP(sc)->if_mtu + 1);
-	clist_alloc_cblocks(&tp->t_rawq, 0, 0);
-
-	s = splnet();
-	if_up(SL2IFP(sc));
-	splx(s);
-	return (0);
-}
-
-static void
-sldestroy(struct sl_softc *sc)
-{
-	bpfdetach(SL2IFP(sc));
-	if_detach(SL2IFP(sc));
-	if_free(SL2IFP(sc));
-	LIST_REMOVE(sc, sl_next);
-	m_free(sc->sc_mbuf);
-	mtx_destroy(&sc->sc_fastq.ifq_mtx);
-	if (sc->bpfbuf)
-		free(sc->bpfbuf, M_SL);
-	free(sc, M_SL);
-}
-
-/*
- * Line specific close routine.
- * Detach the tty from the sl unit.
- */
-static int
-slclose(struct tty *tp, int flag)
-{
-	register struct sl_softc *sc;
-	int s;
-
-	ttyflush(tp, FREAD | FWRITE);
-	/*
-	 * XXX the placement of the following spl is misleading.  tty
-	 * interrupts must be blocked across line discipline switches
-	 * and throughout closes to avoid races.
-	 */
-	s = splimp();		/* actually, max(spltty, splnet) */
-	clist_free_cblocks(&tp->t_outq);
-	sc = sl_for_tty(tp);
-	if (sc != NULL) {
-		if (sc->sc_outfill) {
-			sc->sc_outfill = 0;
-			untimeout(sl_outfill, sc, sc->sc_ofhandle);
-		}
-		if (sc->sc_keepalive) {
-			sc->sc_keepalive = 0;
-			untimeout(sl_keepalive, sc, sc->sc_kahandle);
-		}
-		if_down(SL2IFP(sc));
-		sc->sc_ttyp = NULL;
-		sldestroy(sc);
-	}
-	splx(s);
-	return 0;
-}
-
-/*
- * Line specific (tty) ioctl routine.
- * Provide a way to get the sl unit number.
- */
-/* ARGSUSED */
-static int
-sltioctl(struct tty *tp, u_long cmd, caddr_t data, int flag,
-    struct thread *td)
-{
-	struct sl_softc *sc = sl_for_tty(tp);
-	int s, unit, wasup;
-
-	s = splimp();
-	switch (cmd) {
-	case SLIOCGUNIT:
-		*(int *)data = SL2IFP(sc)->if_dunit;
-		break;
-
-	case SLIOCSUNIT:
-		unit = *(u_int *)data;
-		if (unit < 0) {
-			splx(s);
-			return (ENXIO);
-		}
-		if (SL2IFP(sc)->if_dunit != unit) {
-			if (!slisunitfree(unit)) {
-				splx(s);
-				return (ENXIO);
-			}
-
-			wasup = SL2IFP(sc)->if_flags & IFF_UP;
-			bpfdetach(SL2IFP(sc));
-			if_detach(SL2IFP(sc));
-			LIST_REMOVE(sc, sl_next);
-			if_initname(SL2IFP(sc), "sl", unit);
-			LIST_INSERT_HEAD(&sl_list, sc, sl_next);
-			if_attach(SL2IFP(sc));
-			bpfattach(SL2IFP(sc), DLT_SLIP, SLIP_HDRLEN);
-			if (wasup)
-				if_up(SL2IFP(sc));
-			else
-				if_down(SL2IFP(sc));
-			clist_alloc_cblocks(&tp->t_outq,
-			    SLIP_HIWAT + 2 * SL2IFP(sc)->if_mtu + 1,
-			    SLIP_HIWAT + 2 * SL2IFP(sc)->if_mtu + 1);
-		}
-		slmarkstatic(unit);
-		break;
-
-	case SLIOCSKEEPAL:
-		sc->sc_keepalive = *(u_int *)data * hz;
-		if (sc->sc_keepalive) {
-			sc->sc_flags |= SC_KEEPALIVE;
-			sc->sc_kahandle = timeout(sl_keepalive, sc,
-						  sc->sc_keepalive);
-		} else {
-			if ((sc->sc_flags & SC_KEEPALIVE) != 0) {
-				untimeout(sl_keepalive, sc, sc->sc_kahandle);
-				sc->sc_flags &= ~SC_KEEPALIVE;
-			}
-		}
-		break;
-
-	case SLIOCGKEEPAL:
-		*(int *)data = sc->sc_keepalive / hz;
-		break;
-
-	case SLIOCSOUTFILL:
-		sc->sc_outfill = *(u_int *)data * hz;
-		if (sc->sc_outfill) {
-			sc->sc_flags |= SC_OUTWAIT;
-			sc->sc_ofhandle = timeout(sl_outfill, sc,
-						  sc->sc_outfill);
-		} else {
-			if ((sc->sc_flags & SC_OUTWAIT) != 0) {
-				untimeout(sl_outfill, sc, sc->sc_ofhandle);
-				sc->sc_flags &= ~SC_OUTWAIT;
-			}
-		}
-		break;
-
-	case SLIOCGOUTFILL:
-		*(int *)data = sc->sc_outfill / hz;
-		break;
-
-	default:
-		splx(s);
-		return (ENOIOCTL);
-	}
-	splx(s);
-	return (0);
-}
-
-/*
- * Queue a packet.  Start transmission if not active.
- * Compression happens in slstart; if we do it here, IP TOS
- * will cause us to not compress "background" packets, because
- * ordering gets trashed.  It can be done for all packets in slstart.
- */
-static int
-sloutput(struct ifnet *ifp, register struct mbuf *m, struct sockaddr *dst,
-    struct rtentry *rtp)
-{
-	register struct sl_softc *sc = ifp->if_softc;
-	register struct ip *ip;
-	int error;
-
-	/*
-	 * `Cannot happen' (see slioctl).  Someday we will extend
-	 * the line protocol to support other address families.
-	 */
-	if (dst->sa_family != AF_INET) {
-		if_printf(ifp, "af%d not supported\n", dst->sa_family);
-		m_freem(m);
-		SL2IFP(sc)->if_noproto++;
-		return (EAFNOSUPPORT);
-	}
-
-	if (sc->sc_ttyp == NULL || !(ifp->if_flags & IFF_UP)) {
-		m_freem(m);
-		return (ENETDOWN);
-	}
-	if ((sc->sc_ttyp->t_state & TS_CONNECTED) == 0) {
-		m_freem(m);
-		return (EHOSTUNREACH);
-	}
-	ip = mtod(m, struct ip *);
-	if (SL2IFP(sc)->if_flags & SC_NOICMP && ip->ip_p == IPPROTO_ICMP) {
-		m_freem(m);
-		return (ENETRESET);		/* XXX ? */
-	}
-	if (ip->ip_tos & IPTOS_LOWDELAY &&
-	    !ALTQ_IS_ENABLED(&SL2IFP(sc)->if_snd))
-		error = !(IF_HANDOFF(&sc->sc_fastq, m, SL2IFP(sc)));
-	else
-		IFQ_HANDOFF(SL2IFP(sc), m, error);
-	if (error) {
-		SL2IFP(sc)->if_oerrors++;
-		return (ENOBUFS);
-	}
-	return (0);
-}
-
-static void
-slstart(struct ifnet *ifp)
-{
-	struct sl_softc *sc = ifp->if_softc;
-	int s;
-
-	s = splimp();
-	if (sc->sc_ttyp->t_outq.c_cc == 0)
-		sltstart(sc->sc_ttyp);
-	splx(s);
-}
-
-/*
- * Start output on interface.  Get another datagram
- * to send from the interface queue and map it to
- * the interface before starting output.
- */
-static int
-sltstart(struct tty *tp)
-{
-	register struct sl_softc *sc = sl_for_tty(tp);
-	register struct mbuf *m;
-	register u_char *cp;
-	register struct ip *ip;
-	int s;
-	register int len = 0;
-
-	GIANT_REQUIRED;		/* tty */
-
-	for (;;) {
-		/*
-		 * Call output process whether or not there is more in the
-		 * output queue.  We are being called in lieu of ttstart
-		 * and must do what it would.
-		 */
-		tt_oproc(tp);
-
-		if (tp->t_outq.c_cc != 0) {
-			if (sc != NULL)
-				sc->sc_flags &= ~SC_OUTWAIT;
-			if (tp->t_outq.c_cc > SLIP_HIWAT)
-				return 0;
-		}
-
-		/*
-		 * This happens briefly when the line shuts down.
-		 */
-		if (sc == NULL)
-			return 0;
-
-		/*
-		 * Get a packet and send it to the interface.
-		 */
-		s = splimp();
-		IF_DEQUEUE(&sc->sc_fastq, m);
-		if (m)
-			SL2IFP(sc)->if_omcasts++;		/* XXX */
-		else
-			IF_DEQUEUE(&SL2IFP(sc)->if_snd, m);
-		splx(s);
-		if (m == NULL)
-			return 0;
-
-		/*
-		 * We do the header compression here rather than in sloutput
-		 * because the packets will be out of order if we are using TOS
-		 * queueing, and the connection id compression will get
-		 * munged when this happens.
-		 */
-		if (bpf_peers_present(SL2IFP(sc)->if_bpf)) {
-			/*
-			 * We need to save the TCP/IP header before it's
-			 * compressed.  To avoid complicated code, we just
-			 * copy the entire packet into a stack buffer (since
-			 * this is a serial line, packets should be short
-			 * and/or the copy should be negligible cost compared
-			 * to the packet transmission time).
-			 */
-			register struct mbuf *m1 = m;
-			register u_char *cp;
-
-			if (sc->bpfbuf == NULL)
-				sc->bpfbuf = malloc(SLTMAX + SLIP_HDRLEN,
-				    M_SL, M_NOWAIT);
-
-			if (sc->bpfbuf) {
-				cp = sc->bpfbuf + SLIP_HDRLEN;
-				len = 0;
-				do {
-					register int mlen = m1->m_len;
-
-					bcopy(mtod(m1, caddr_t), cp, mlen);
-					cp += mlen;
-					len += mlen;
-				} while ((m1 = m1->m_next) != NULL);
-			}
-		}
-		ip = mtod(m, struct ip *);
-		if (ip->ip_v == IPVERSION && ip->ip_p == IPPROTO_TCP) {
-			if (SL2IFP(sc)->if_flags & SC_COMPRESS)
-				*mtod(m, u_char *) |= sl_compress_tcp(m, ip,
-				    &sc->sc_comp, 1);
-		}
-		if (bpf_peers_present(SL2IFP(sc)->if_bpf) && sc->bpfbuf) {
-			/*
-			 * Put the SLIP pseudo-"link header" in place.  The
-			 * compressed header is now at the beginning of the
-			 * mbuf.
-			 */
-			sc->bpfbuf[SLX_DIR] = SLIPDIR_OUT;
-			bcopy(mtod(m, caddr_t), &sc->bpfbuf[SLX_CHDR], CHDR_LEN);
-			BPF_TAP(SL2IFP(sc), sc->bpfbuf, len + SLIP_HDRLEN);
-		}
-
-		/*
-		 * If system is getting low on clists, just flush our
-		 * output queue (if the stuff was important, it'll get
-		 * retransmitted). Note that SLTMAX is used instead of
-		 * the current if_mtu setting because connections that
-		 * have already been established still use the original
-		 * (possibly larger) mss.
-		 */
-		if (cfreecount < CLISTRESERVE + SLTMAX) {
-			m_freem(m);
-			SL2IFP(sc)->if_collisions++;
-			continue;
-		}
-
-		sc->sc_flags &= ~SC_OUTWAIT;
-		/*
-		 * The extra FRAME_END will start up a new packet, and thus
-		 * will flush any accumulated garbage.  We do this whenever
-		 * the line may have been idle for some time.
-		 */
-		if (tp->t_outq.c_cc == 0) {
-			++SL2IFP(sc)->if_obytes;
-			(void) putc(FRAME_END, &tp->t_outq);
-		}
-
-		while (m) {
-			register u_char *ep;
-
-			cp = mtod(m, u_char *); ep = cp + m->m_len;
-			while (cp < ep) {
-				/*
-				 * Find out how many bytes in the string we can
-				 * handle without doing something special.
-				 */
-				register u_char *bp = cp;
-
-				while (cp < ep) {
-					switch (*cp++) {
-					case FRAME_ESCAPE:
-					case FRAME_END:
-						--cp;
-						goto out;
-					}
-				}
-				out:
-				if (cp > bp) {
-					/*
-					 * Put n characters at once
-					 * into the tty output queue.
-					 */
-					if (b_to_q((char *)bp, cp - bp,
-					    &tp->t_outq))
-						break;
-					SL2IFP(sc)->if_obytes += cp - bp;
-				}
-				/*
-				 * If there are characters left in the mbuf,
-				 * the first one must be special..
-				 * Put it out in a different form.
-				 */
-				if (cp < ep) {
-					if (putc(FRAME_ESCAPE, &tp->t_outq))
-						break;
-					if (putc(*cp++ == FRAME_ESCAPE ?
-					   TRANS_FRAME_ESCAPE : TRANS_FRAME_END,
-					   &tp->t_outq)) {
-						(void) unputc(&tp->t_outq);
-						break;
-					}
-					SL2IFP(sc)->if_obytes += 2;
-				}
-			}
-			m = m_free(m);
-		}
-
-		if (putc(FRAME_END, &tp->t_outq)) {
-			/*
-			 * Not enough room.  Remove a char to make room
-			 * and end the packet normally.
-			 * If you get many collisions (more than one or two
-			 * a day) you probably do not have enough clists
-			 * and you should increase "nclist" in param.c.
-			 */
-			(void) unputc(&tp->t_outq);
-			(void) putc(FRAME_END, &tp->t_outq);
-			SL2IFP(sc)->if_collisions++;
-		} else {
-			++SL2IFP(sc)->if_obytes;
-			SL2IFP(sc)->if_opackets++;
-		}
-	}
-	return 0;
-}
-
-/*
- * Copy data buffer to mbuf chain; add ifnet pointer.
- */
-static struct mbuf *
-sl_btom(struct sl_softc *sc, register int len)
-{
-	struct mbuf *m, *newm;
-
-	MGETHDR(m, M_DONTWAIT, MT_DATA);
-	if (m == NULL)
-		return (NULL);
-
-	/*
-	 * If we have more than MHLEN bytes, it's cheaper to
-	 * queue the cluster we just filled & allocate a new one
-	 * for the input buffer.  Otherwise, fill the mbuf we
-	 * allocated above.  Note that code in the input routine
-	 * guarantees that packet will fit in a cluster.
-	 */
-	if (len >= MHLEN) {
-		MCLGET(m, M_DONTWAIT);
-		if ((m->m_flags & M_EXT) == 0) {
-			/*
-			 * we couldn't get a cluster - if memory's this
-			 * low, it's time to start dropping packets.
-			 */
-			(void) m_free(m);
-			return (NULL);
-		}
-		/* Swap the new and old clusters */
-		newm = m;
-		m = sc->sc_mbuf;
-		sc->sc_mbuf = newm;
-		sc->sc_ep = mtod(newm, u_char *) + SLBUFSIZE;
-		
-		m->m_data = (caddr_t)sc->sc_buf;
-	} else
-		bcopy((caddr_t)sc->sc_buf, mtod(m, caddr_t), len);
-
-	m->m_len = len;
-	m->m_pkthdr.len = len;
-	m->m_pkthdr.rcvif = SL2IFP(sc);
-	return (m);
-}
-
-/*
- * tty interface receiver interrupt.
- */
-static int
-slinput(int c, struct tty *tp)
-{
-	register struct sl_softc *sc;
-	register struct mbuf *m;
-	register int len;
-	u_char chdr[CHDR_LEN];
-
-	tk_nin++;
-	sc = sl_for_tty(tp);
-	if (sc == NULL)
-		return 0;
-	if (c & TTY_ERRORMASK || (tp->t_state & TS_CONNECTED) == 0) {
-		sc->sc_flags |= SC_ERROR;
-		return 0;
-	}
-	c &= TTY_CHARMASK;
-
-	++SL2IFP(sc)->if_ibytes;
-
-	if (SL2IFP(sc)->if_flags & IFF_DEBUG) {
-		if (c == ABT_ESC) {
-			/*
-			 * If we have a previous abort, see whether
-			 * this one is within the time limit.
-			 */
-			if (sc->sc_abortcount &&
-			    time_uptime >= sc->sc_starttime + ABT_WINDOW)
-				sc->sc_abortcount = 0;
-			/*
-			 * If we see an abort after "idle" time, count it;
-			 * record when the first abort escape arrived.
-			 */
-			if (time_uptime >= sc->sc_lasttime + ABT_IDLE) {
-				if (++sc->sc_abortcount == 1)
-					sc->sc_starttime = time_uptime;
-				if (sc->sc_abortcount >= ABT_COUNT) {
-					slclose(tp,0);
-					return 0;
-				}
-			}
-		} else
-			sc->sc_abortcount = 0;
-		sc->sc_lasttime = time_uptime;
-	}
-
-	switch (c) {
-
-	case TRANS_FRAME_ESCAPE:
-		if (sc->sc_escape)
-			c = FRAME_ESCAPE;
-		break;
-
-	case TRANS_FRAME_END:
-		if (sc->sc_escape)
-			c = FRAME_END;
-		break;
-
-	case FRAME_ESCAPE:
-		sc->sc_escape = 1;
-		return 0;
-
-	case FRAME_END:
-		sc->sc_flags &= ~SC_KEEPALIVE;
-		if(sc->sc_flags & SC_ERROR) {
-			sc->sc_flags &= ~SC_ERROR;
-			goto newpack;
-		}
-		len = sc->sc_mp - sc->sc_buf;
-		if (len < 3)
-			/* less than min length packet - ignore */
-			goto newpack;
-
-		if (bpf_peers_present(SL2IFP(sc)->if_bpf)) {
-			/*
-			 * Save the compressed header, so we
-			 * can tack it on later.  Note that we
-			 * will end up copying garbage in some
-			 * cases but this is okay.  We remember
-			 * where the buffer started so we can
-			 * compute the new header length.
-			 */
-			bcopy(sc->sc_buf, chdr, CHDR_LEN);
-		}
-
-		if ((c = (*sc->sc_buf & 0xf0)) != (IPVERSION << 4)) {
-			if (c & 0x80)
-				c = TYPE_COMPRESSED_TCP;
-			else if (c == TYPE_UNCOMPRESSED_TCP)
-				*sc->sc_buf &= 0x4f; /* XXX */
-			/*
-			 * We've got something that's not an IP packet.
-			 * If compression is enabled, try to decompress it.
-			 * Otherwise, if `auto-enable' compression is on and
-			 * it's a reasonable packet, decompress it and then
-			 * enable compression.  Otherwise, drop it.
-			 */
-			if (SL2IFP(sc)->if_flags & SC_COMPRESS) {
-				len = sl_uncompress_tcp(&sc->sc_buf, len,
-							(u_int)c, &sc->sc_comp);
-				if (len <= 0)
-					goto error;
-			} else if ((SL2IFP(sc)->if_flags & SC_AUTOCOMP) &&
-			    c == TYPE_UNCOMPRESSED_TCP && len >= 40) {
-				len = sl_uncompress_tcp(&sc->sc_buf, len,
-							(u_int)c, &sc->sc_comp);
-				if (len <= 0)
-					goto error;
-				SL2IFP(sc)->if_flags |= SC_COMPRESS;
-			} else
-				goto error;
-		}
-		if (bpf_peers_present(SL2IFP(sc)->if_bpf)) {
-			/*
-			 * Put the SLIP pseudo-"link header" in place.
-			 * We couldn't do this any earlier since
-			 * decompression probably moved the buffer
-			 * pointer.  Then, invoke BPF.
-			 */
-			register u_char *hp = sc->sc_buf - SLIP_HDRLEN;
-
-			hp[SLX_DIR] = SLIPDIR_IN;
-			bcopy(chdr, &hp[SLX_CHDR], CHDR_LEN);
-			BPF_TAP(SL2IFP(sc), hp, len + SLIP_HDRLEN);
-		}
-		m = sl_btom(sc, len);
-		if (m == NULL)
-			goto error;
-
-		SL2IFP(sc)->if_ipackets++;
-
-		if ((SL2IFP(sc)->if_flags & IFF_UP) == 0) {
-			m_freem(m);
-			goto newpack;
-		}
-		if (netisr_queue(NETISR_IP, m)) {	/* (0) on success. */
-			SL2IFP(sc)->if_ierrors++;
-			SL2IFP(sc)->if_iqdrops++;
-		}
-		goto newpack;
-	}
-	if (sc->sc_mp < sc->sc_ep) {
-		*sc->sc_mp++ = c;
-		sc->sc_escape = 0;
-		return 0;
-	}
-
-	/* can't put lower; would miss an extra frame */
-	sc->sc_flags |= SC_ERROR;
-
-error:
-	SL2IFP(sc)->if_ierrors++;
-newpack:
-	sc->sc_mp = sc->sc_buf = sc->sc_ep - SLRMAX;
-	sc->sc_escape = 0;
-	return 0;
-}
-
-/*
- * Process an ioctl request.
- */
-static int
-slioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
-{
-	register struct ifaddr *ifa = (struct ifaddr *)data;
-	register struct ifreq *ifr = (struct ifreq *)data;
-	register int s, error = 0;
-	struct sl_softc *sc = ifp->if_softc;
-
-	s = splimp();
-
-	switch (cmd) {
-
-	case SIOCSIFFLAGS:
-		/*
-		 * if.c will set the interface up even if we
-		 * don't want it to.
-		 */
-		if (sc->sc_ttyp == NULL) {
-			ifp->if_flags &= ~IFF_UP;
-		}
-		break;
-	case SIOCSIFADDR:
-		/*
-		 * This is "historical" - set the interface up when
-		 * setting the address.
-		 */
-		if (ifa->ifa_addr->sa_family == AF_INET) {
-			if (sc->sc_ttyp != NULL)
-				ifp->if_flags |= IFF_UP;
-		} else {
-			error = EAFNOSUPPORT;
-		}
-		break;
-
-	case SIOCSIFDSTADDR:
-		if (ifa->ifa_addr->sa_family != AF_INET)
-			error = EAFNOSUPPORT;
-		break;
-
-	case SIOCADDMULTI:
-	case SIOCDELMULTI:
-		break;
-
-	case SIOCSIFMTU:
-		/*
-		 * Set the interface MTU.
-		 */
-		if (ifr->ifr_mtu > SLTMAX)
-			error = EINVAL;
-		else {
-			struct tty *tp;
-
-			ifp->if_mtu = ifr->ifr_mtu;
-			tp = sc->sc_ttyp;
-			if (tp != NULL)
-				clist_alloc_cblocks(&tp->t_outq,
-				    SLIP_HIWAT + 2 * ifp->if_mtu + 1,
-				    SLIP_HIWAT + 2 * ifp->if_mtu + 1);
-		}
-		break;
-
-	default:
-		error = EINVAL;
-	}
-	splx(s);
-	return (error);
-}
-
-static void
-sl_keepalive(void *chan)
-{
-	struct sl_softc *sc = chan;
-
-	if (sc->sc_keepalive) {
-		if (sc->sc_flags & SC_KEEPALIVE) {
-			if (sc->sc_ttyp->t_pgrp != NULL) {
-				PGRP_LOCK(sc->sc_ttyp->t_pgrp);
-				pgsignal (sc->sc_ttyp->t_pgrp, SIGURG, 1);
-				PGRP_UNLOCK(sc->sc_ttyp->t_pgrp);
-			}
-		} else
-			sc->sc_flags |= SC_KEEPALIVE;
-		sc->sc_kahandle = timeout(sl_keepalive, sc, sc->sc_keepalive);
-	} else {
-		sc->sc_flags &= ~SC_KEEPALIVE;
-	}
-}
-
-static void
-sl_outfill(void *chan)
-{
-	struct sl_softc *sc = chan;
-	register struct tty *tp = sc->sc_ttyp;
-	int s;
-
-	if (sc->sc_outfill && tp != NULL) {
-		if (sc->sc_flags & SC_OUTWAIT) {
-			s = splimp ();
-			++SL2IFP(sc)->if_obytes;
-			(void) putc(FRAME_END, &tp->t_outq);
-			tt_oproc(tp);
-			splx (s);
-		} else
-			sc->sc_flags |= SC_OUTWAIT;
-		sc->sc_ofhandle = timeout(sl_outfill, sc, sc->sc_outfill);
-	} else {
-		sc->sc_flags &= ~SC_OUTWAIT;
-	}
-}
diff --git a/sys/net/if_slvar.h b/sys/net/if_slvar.h
deleted file mode 100644
index e6b6d2c..0000000
--- a/sys/net/if_slvar.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/*-
- * Copyright (c) 1991, 1993
- *	The Regents of the University of California.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)if_slvar.h	8.3 (Berkeley) 2/1/94
- *
- * $FreeBSD$
- */
-
-#ifndef _NET_IF_SLVAR_H_
-#define _NET_IF_SLVAR_H_
-
-#include <sys/callout.h>
-
-/*
- * Definitions for SLIP interface data structures
- *
- * (This exists so programs like slstats can get at the definition
- *  of sl_softc.)
- */
-struct sl_softc {
-	struct	ifnet *sc_ifp;		/* network-visible interface */
-	struct	ifqueue sc_fastq;	/* interactive output queue */
-	struct	tty *sc_ttyp;		/* pointer to tty structure */
-	struct	mbuf *sc_mbuf;		/* pointer to mbuf containing buffer */
-	u_char	*sc_mp;			/* pointer to next available buf char */
-	u_char	*sc_ep;			/* pointer to last available buf char */
-	u_char	*sc_buf;		/* input buffer */
-	u_int	sc_flags;		/* see below */
-	u_int	sc_escape;	/* =1 if last char input was FRAME_ESCAPE */
-	time_t	sc_lasttime;		/* last time a char arrived */
-	long	sc_abortcount;		/* number of abort escape chars */
-	time_t	sc_starttime;		/* time of first abort in window */
-	u_int	sc_keepalive;		/* time	to decide link hang */
-	u_int	sc_outfill;	/* time	to send	FRAME_END when output idle */
-					/*
-					 * Handles for scheduling outfill and
-					 * keepalive timeouts.
-					 */
-	struct	callout_handle	sc_ofhandle;
-	struct	callout_handle	sc_kahandle;
-	struct	slcompress sc_comp;	/* tcp compression data */
-	LIST_ENTRY(sl_softc) sl_next;
-	u_char	*bpfbuf;		/* hang buffer for bpf here */
-};
-#define	SL2IFP(sc)	((sc)->sc_ifp)
-
-/* internal flags */
-#define	SC_ERROR	0x0001		/* had an input error */
-#define	SC_OUTWAIT	0x0002		/* waiting for output fill */
-#define	SC_KEEPALIVE	0x0004		/* input keepalive */
-
-/* visible flags */
-#define	SC_COMPRESS	IFF_LINK0	/* compress TCP traffic */
-#define	SC_NOICMP	IFF_LINK1	/* suppress ICMP traffic */
-#define	SC_AUTOCOMP	IFF_LINK2	/* auto-enable TCP compression */
-
-
-#endif
diff --git a/sys/net/if_spppsubr.c b/sys/net/if_spppsubr.c
index 9a5a11d..b1c7125 100644
--- a/sys/net/if_spppsubr.c
+++ b/sys/net/if_spppsubr.c
@@ -239,7 +239,7 @@ struct cp {
 #define SPPP_LOCK(sp)	mtx_lock (&(sp)->mtx)
 #define SPPP_UNLOCK(sp)	mtx_unlock (&(sp)->mtx)
 #define SPPP_LOCK_ASSERT(sp)	mtx_assert (&(sp)->mtx, MA_OWNED)
-#define SPPP_LOCK_OWNED(sp)	mtx_owned (&sp->mtx)
+#define SPPP_LOCK_OWNED(sp)	mtx_owned (&(sp)->mtx)
 
 #ifdef INET
 /*
diff --git a/sys/net/if_tap.c b/sys/net/if_tap.c
index b4ee7e1..9256301 100644
--- a/sys/net/if_tap.c
+++ b/sys/net/if_tap.c
@@ -600,6 +600,7 @@ static int
 tapifioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 {
 	struct tap_softc	*tp = ifp->if_softc;
+	struct ifreq		*ifr = (struct ifreq *)data;
 	struct ifstat		*ifs = NULL;
 	int			 s, dummy;
 
@@ -609,6 +610,10 @@ tapifioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		case SIOCDELMULTI:
 			break;
 
+		case SIOCSIFMTU:
+			ifp->if_mtu = ifr->ifr_mtu;
+			break;
+
 		case SIOCGIFSTATUS:
 			s = splimp();
 			ifs = (struct ifstat *)data;
diff --git a/sys/net/if_var.h b/sys/net/if_var.h
index 3ff019f2..7640d78 100644
--- a/sys/net/if_var.h
+++ b/sys/net/if_var.h
@@ -181,7 +181,6 @@ struct ifnet {
 	void	*if_afdata[AF_MAX];
 	int	if_afdata_initialized;
 	struct	rwlock if_afdata_lock;
-	struct	task if_starttask;	/* task for IFF_NEEDSGIANT */
 	struct	task if_linktask;	/* task for link change events */
 	struct	mtx if_addr_mtx;	/* mutex to protect address lists */
 
@@ -379,16 +378,6 @@ EVENTHANDLER_DECLARE(group_change_event, group_change_event_handler_t);
 #define	IF_AFDATA_LOCK_ASSERT(ifp)	rw_assert(&(ifp)->if_afdata_lock, RA_LOCKED)
 #define	IF_AFDATA_UNLOCK_ASSERT(ifp)	rw_assert(&(ifp)->if_afdata_lock, RA_UNLOCKED)
 
-#define	IFF_LOCKGIANT(ifp) do {						\
-	if ((ifp)->if_flags & IFF_NEEDSGIANT)				\
-		mtx_lock(&Giant);					\
-} while (0)
-
-#define	IFF_UNLOCKGIANT(ifp) do {					\
-	if ((ifp)->if_flags & IFF_NEEDSGIANT)				\
-		mtx_unlock(&Giant);					\
-} while (0)
-
 int	if_handoff(struct ifqueue *ifq, struct mbuf *m, struct ifnet *ifp,
 	    int adjust);
 #define	IF_HANDOFF(ifq, m, ifp)			\
diff --git a/sys/net/ppp_comp.h b/sys/net/ppp_comp.h
deleted file mode 100644
index 00591ef..0000000
--- a/sys/net/ppp_comp.h
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * ppp_comp.h - Definitions for doing PPP packet compression.
- */
-
-/*-
- * Copyright (c) 1994 The Australian National University.
- * All rights reserved.
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation is hereby granted, provided that the above copyright
- * notice appears in all copies.  This software is provided without any
- * warranty, express or implied. The Australian National University
- * makes no representations about the suitability of this software for
- * any purpose.
- *
- * IN NO EVENT SHALL THE AUSTRALIAN NATIONAL UNIVERSITY BE LIABLE TO ANY
- * PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
- * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF
- * THE AUSTRALIAN NATIONAL UNIVERSITY HAVE BEEN ADVISED OF THE POSSIBILITY
- * OF SUCH DAMAGE.
- *
- * THE AUSTRALIAN NATIONAL UNIVERSITY SPECIFICALLY DISCLAIMS ANY WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
- * ON AN "AS IS" BASIS, AND THE AUSTRALIAN NATIONAL UNIVERSITY HAS NO
- * OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS,
- * OR MODIFICATIONS.
- *
- * $FreeBSD$
- */
-
-#ifndef _NET_PPP_COMP_H
-#define _NET_PPP_COMP_H
-
-/*
- * Structure giving methods for compression/decompression.
- */
-#ifdef PACKETPTR
-struct compressor {
-	int	compress_proto;	/* CCP compression protocol number */
-
-	/* Allocate space for a compressor (transmit side) */
-	void	*(*comp_alloc)(u_char *options, int opt_len);
-	/* Free space used by a compressor */
-	void	(*comp_free)(void *state);
-	/* Initialize a compressor */
-	int	(*comp_init)(void *state, u_char *options, int opt_len,
-		    int unit, int hdrlen, int debug);
-	/* Reset a compressor */
-	void	(*comp_reset)(void *state);
-	/* Compress a packet */
-	int	(*compress)(void *state, PACKETPTR *mret, PACKETPTR mp,
-		    int orig_len, int max_len);
-	/* Return compression statistics */
-	void	(*comp_stat)(void *state, struct compstat *stats);
-
-	/* Allocate space for a decompressor (receive side) */
-	void	*(*decomp_alloc)(u_char *options, int opt_len);
-	/* Free space used by a decompressor */
-	void	(*decomp_free)(void *state);
-	/* Initialize a decompressor */
-	int	(*decomp_init)(void *state, u_char *options, int opt_len,
-		    int unit, int hdrlen, int mru, int debug);
-	/* Reset a decompressor */
-	void	(*decomp_reset)(void *state);
-	/* Decompress a packet. */
-	int	(*decompress)(void *state, PACKETPTR mp, PACKETPTR *dmpp);
-	/* Update state for an incompressible packet received */
-	void	(*incomp)(void *state, PACKETPTR mp);
-	/* Return decompression statistics */
-	void	(*decomp_stat)(void *state, struct compstat *stats);
-};
-#endif /* PACKETPTR */
-
-/*
- * Return values for decompress routine.
- * We need to make these distinctions so that we can disable certain
- * useful functionality, namely sending a CCP reset-request as a result
- * of an error detected after decompression.  This is to avoid infringing
- * a patent held by Motorola.
- * Don't you just lurve software patents.
- */
-#define DECOMP_OK		0	/* everything went OK */
-#define DECOMP_ERROR		1	/* error detected before decomp. */
-#define DECOMP_FATALERROR	2	/* error detected after decomp. */
-
-/*
- * CCP codes.
- */
-#define CCP_CONFREQ	1
-#define CCP_CONFACK	2
-#define CCP_TERMREQ	5
-#define CCP_TERMACK	6
-#define CCP_RESETREQ	14
-#define CCP_RESETACK	15
-
-/*
- * Max # bytes for a CCP option
- */
-#define CCP_MAX_OPTION_LENGTH	32
-
-/*
- * Parts of a CCP packet.
- */
-#define CCP_CODE(dp)		((dp)[0])
-#define CCP_ID(dp)		((dp)[1])
-#define CCP_LENGTH(dp)		(((dp)[2] << 8) + (dp)[3])
-#define CCP_HDRLEN		4
-
-#define CCP_OPT_CODE(dp)	((dp)[0])
-#define CCP_OPT_LENGTH(dp)	((dp)[1])
-#define CCP_OPT_MINLEN		2
-
-/*
- * Definitions for BSD-Compress.
- */
-#define CI_BSD_COMPRESS		21	/* config. option for BSD-Compress */
-#define CILEN_BSD_COMPRESS	3	/* length of config. option */
-
-/* Macros for handling the 3rd byte of the BSD-Compress config option. */
-#define BSD_NBITS(x)		((x) & 0x1F)	/* number of bits requested */
-#define BSD_VERSION(x)		((x) >> 5)	/* version of option format */
-#define BSD_CURRENT_VERSION	1		/* current version number */
-#define BSD_MAKE_OPT(v, n)	(((v) << 5) | (n))
-
-#define BSD_MIN_BITS		9	/* smallest code size supported */
-#define BSD_MAX_BITS		15	/* largest code size supported */
-
-/*
- * Definitions for Deflate.
- */
-#define CI_DEFLATE		26	/* config option for Deflate */
-#define CI_DEFLATE_DRAFT	24	/* value used in original draft RFC */
-#define CILEN_DEFLATE		4	/* length of its config option */
-
-#define DEFLATE_MIN_SIZE	8
-#define DEFLATE_MAX_SIZE	15
-#define DEFLATE_METHOD_VAL	8
-#define DEFLATE_SIZE(x)		(((x) >> 4) + DEFLATE_MIN_SIZE)
-#define DEFLATE_METHOD(x)	((x) & 0x0F)
-#define DEFLATE_MAKE_OPT(w)	((((w) - DEFLATE_MIN_SIZE) << 4) \
-				 + DEFLATE_METHOD_VAL)
-#define DEFLATE_CHK_SEQUENCE	0
-
-/*
- * Definitions for other, as yet unsupported, compression methods.
- */
-#define CI_PREDICTOR_1		1	/* config option for Predictor-1 */
-#define CILEN_PREDICTOR_1	2	/* length of its config option */
-#define CI_PREDICTOR_2		2	/* config option for Predictor-2 */
-#define CILEN_PREDICTOR_2	2	/* length of its config option */
-
-#endif /* _NET_PPP_COMP_H */
diff --git a/sys/net/ppp_deflate.c b/sys/net/ppp_deflate.c
deleted file mode 100644
index e3015bd..0000000
--- a/sys/net/ppp_deflate.c
+++ /dev/null
@@ -1,680 +0,0 @@
-/* $FreeBSD$	*/
-
-/*-
- * ppp_deflate.c - interface the zlib procedures for Deflate compression
- * and decompression (as used by gzip) to the PPP code.
- * This version is for use with mbufs on BSD-derived systems.
- *
- * Copyright (c) 1994 The Australian National University.
- * All rights reserved.
- *
- * Permission to use, copy, modify, and distribute this software and its
- * documentation is hereby granted, provided that the above copyright
- * notice appears in all copies.  This software is provided without any
- * warranty, express or implied. The Australian National University
- * makes no representations about the suitability of this software for
- * any purpose.
- *
- * IN NO EVENT SHALL THE AUSTRALIAN NATIONAL UNIVERSITY BE LIABLE TO ANY
- * PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
- * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF
- * THE AUSTRALIAN NATIONAL UNIVERSITY HAS BEEN ADVISED OF THE POSSIBILITY
- * OF SUCH DAMAGE.
- *
- * THE AUSTRALIAN NATIONAL UNIVERSITY SPECIFICALLY DISCLAIMS ANY WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
- * AND FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
- * ON AN "AS IS" BASIS, AND THE AUSTRALIAN NATIONAL UNIVERSITY HAS NO
- * OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS,
- * OR MODIFICATIONS.
- */
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/lock.h>
-#include <sys/malloc.h>
-#include <sys/mbuf.h>
-#include <sys/module.h>
-#include <sys/mutex.h>
-
-#include <net/ppp_defs.h>
-#include <net/zlib.h>
-
-#define PACKETPTR	struct mbuf *
-#include <net/ppp_comp.h>
-
-#define DEFLATE_DEBUG	1
-
-/*
- * State for a Deflate (de)compressor.
- */
-struct deflate_state {
-    int		seqno;
-    int		w_size;
-    int		unit;
-    int		hdrlen;
-    int		mru;
-    int		debug;
-    z_stream	strm;
-    struct compstat stats;
-};
-
-#define DEFLATE_OVHD	2		/* Deflate overhead/packet */
-
-static void	*z_alloc(void *, u_int items, u_int size);
-static void	z_free(void *, void *ptr);
-static void	*z_comp_alloc(u_char *options, int opt_len);
-static void	*z_decomp_alloc(u_char *options, int opt_len);
-static void	z_comp_free(void *state);
-static void	z_decomp_free(void *state);
-static int	z_comp_init(void *state, u_char *options, int opt_len,
-		    int unit, int hdrlen, int debug);
-static int	z_decomp_init(void *state, u_char *options, int opt_len,
-		    int unit, int hdrlen, int mru, int debug);
-static int	z_compress(void *state, struct mbuf **mret, struct mbuf *mp,
-		    int slen, int maxolen);
-static void	z_incomp(void *state, struct mbuf *dmsg);
-static int	z_decompress(void *state, struct mbuf *cmp, struct mbuf **dmpp);
-static void	z_comp_reset(void *state);
-static void	z_decomp_reset(void *state);
-static void	z_comp_stats(void *state, struct compstat *stats);
-
-/*
- * Procedures exported to if_ppp.c.
- */
-struct compressor ppp_deflate = {
-    CI_DEFLATE,			/* compress_proto */
-    z_comp_alloc,		/* comp_alloc */
-    z_comp_free,		/* comp_free */
-    z_comp_init,		/* comp_init */
-    z_comp_reset,		/* comp_reset */
-    z_compress,			/* compress */
-    z_comp_stats,		/* comp_stat */
-    z_decomp_alloc,		/* decomp_alloc */
-    z_decomp_free,		/* decomp_free */
-    z_decomp_init,		/* decomp_init */
-    z_decomp_reset,		/* decomp_reset */
-    z_decompress,		/* decompress */
-    z_incomp,			/* incomp */
-    z_comp_stats,		/* decomp_stat */
-};
-
-struct compressor ppp_deflate_draft = {
-    CI_DEFLATE_DRAFT,		/* compress_proto */
-    z_comp_alloc,		/* comp_alloc */
-    z_comp_free,		/* comp_free */
-    z_comp_init,		/* comp_init */
-    z_comp_reset,		/* comp_reset */
-    z_compress,			/* compress */
-    z_comp_stats,		/* comp_stat */
-    z_decomp_alloc,		/* decomp_alloc */
-    z_decomp_free,		/* decomp_free */
-    z_decomp_init,		/* decomp_init */
-    z_decomp_reset,		/* decomp_reset */
-    z_decompress,		/* decompress */
-    z_incomp,			/* incomp */
-    z_comp_stats,		/* decomp_stat */
-};
-
-/*
- * Space allocation and freeing routines for use by zlib routines.
- */
-static void *
-z_alloc(notused, items, size)
-    void *notused;
-    u_int items, size;
-{
-    void *ptr;
-
-    ptr = malloc(items * size, M_DEVBUF, M_NOWAIT);
-    return ptr;
-}
-
-static void
-z_free(notused, ptr)
-    void *notused;
-    void *ptr;
-{
-    free(ptr, M_DEVBUF);
-}
-
-/*
- * Allocate space for a compressor.
- */
-static void *
-z_comp_alloc(options, opt_len)
-    u_char *options;
-    int opt_len;
-{
-    struct deflate_state *state;
-    int w_size;
-
-    if (opt_len != CILEN_DEFLATE
-	|| (options[0] != CI_DEFLATE && options[0] != CI_DEFLATE_DRAFT)
-	|| options[1] != CILEN_DEFLATE
-	|| DEFLATE_METHOD(options[2]) != DEFLATE_METHOD_VAL
-	|| options[3] != DEFLATE_CHK_SEQUENCE)
-	return NULL;
-    w_size = DEFLATE_SIZE(options[2]);
-    if (w_size < DEFLATE_MIN_SIZE || w_size > DEFLATE_MAX_SIZE)
-	return NULL;
-
-    state = malloc(sizeof(struct deflate_state),
-	   M_DEVBUF, M_NOWAIT);
-    if (state == NULL)
-	return NULL;
-
-    state->strm.next_in = NULL;
-    state->strm.zalloc = z_alloc;
-    state->strm.zfree = z_free;
-    if (deflateInit2(&state->strm, Z_DEFAULT_COMPRESSION, DEFLATE_METHOD_VAL,
-		     -w_size, 8, Z_DEFAULT_STRATEGY) != Z_OK) {
-	free(state, M_DEVBUF);
-	return NULL;
-    }
-
-    state->w_size = w_size;
-    bzero(&state->stats, sizeof(state->stats));
-    return (void *) state;
-}
-
-static void
-z_comp_free(arg)
-    void *arg;
-{
-    struct deflate_state *state = (struct deflate_state *) arg;
-
-    deflateEnd(&state->strm);
-    free(state, M_DEVBUF);
-}
-
-static int
-z_comp_init(arg, options, opt_len, unit, hdrlen, debug)
-    void *arg;
-    u_char *options;
-    int opt_len, unit, hdrlen, debug;
-{
-    struct deflate_state *state = (struct deflate_state *) arg;
-
-    if (opt_len < CILEN_DEFLATE
-	|| (options[0] != CI_DEFLATE && options[0] != CI_DEFLATE_DRAFT)
-	|| options[1] != CILEN_DEFLATE
-	|| DEFLATE_METHOD(options[2]) != DEFLATE_METHOD_VAL
-	|| DEFLATE_SIZE(options[2]) != state->w_size
-	|| options[3] != DEFLATE_CHK_SEQUENCE)
-	return 0;
-
-    state->seqno = 0;
-    state->unit = unit;
-    state->hdrlen = hdrlen;
-    state->debug = debug;
-
-    deflateReset(&state->strm);
-
-    return 1;
-}
-
-static void
-z_comp_reset(arg)
-    void *arg;
-{
-    struct deflate_state *state = (struct deflate_state *) arg;
-
-    state->seqno = 0;
-    deflateReset(&state->strm);
-}
-
-static int
-z_compress(arg, mret, mp, orig_len, maxolen)
-    void *arg;
-    struct mbuf **mret;		/* compressed packet (out) */
-    struct mbuf *mp;		/* uncompressed packet (in) */
-    int orig_len, maxolen;
-{
-    struct deflate_state *state = (struct deflate_state *) arg;
-    u_char *rptr, *wptr;
-    int proto, olen, wspace, r, flush;
-    struct mbuf *m;
-
-    /*
-     * Check that the protocol is in the range we handle.
-     */
-    rptr = mtod(mp, u_char *);
-    proto = PPP_PROTOCOL(rptr);
-    if (proto > 0x3fff || proto == 0xfd || proto == 0xfb) {
-	*mret = NULL;
-	return orig_len;
-    }
-
-    /* Allocate one mbuf initially. */
-    if (maxolen > orig_len)
-	maxolen = orig_len;
-    MGET(m, M_DONTWAIT, MT_DATA);
-    *mret = m;
-    if (m != NULL) {
-	m->m_len = 0;
-	if (maxolen + state->hdrlen > MLEN)
-	    MCLGET(m, M_DONTWAIT);
-	wspace = M_TRAILINGSPACE(m);
-	if (state->hdrlen + PPP_HDRLEN + 2 < wspace) {
-	    m->m_data += state->hdrlen;
-	    wspace -= state->hdrlen;
-	}
-	wptr = mtod(m, u_char *);
-
-	/*
-	 * Copy over the PPP header and store the 2-byte sequence number.
-	 */
-	wptr[0] = PPP_ADDRESS(rptr);
-	wptr[1] = PPP_CONTROL(rptr);
-	wptr[2] = PPP_COMP >> 8;
-	wptr[3] = PPP_COMP;
-	wptr += PPP_HDRLEN;
-	wptr[0] = state->seqno >> 8;
-	wptr[1] = state->seqno;
-	wptr += 2;
-	state->strm.next_out = wptr;
-	state->strm.avail_out = wspace - (PPP_HDRLEN + 2);
-    } else {
-	state->strm.next_out = NULL;
-	state->strm.avail_out = 1000000;
-	wptr = NULL;
-	wspace = 0;
-    }
-    ++state->seqno;
-
-    rptr += (proto > 0xff)? 2: 3;	/* skip 1st proto byte if 0 */
-    state->strm.next_in = rptr;
-    state->strm.avail_in = mtod(mp, u_char *) + mp->m_len - rptr;
-    mp = mp->m_next;
-    flush = (mp == NULL)? Z_PACKET_FLUSH: Z_NO_FLUSH;
-    olen = 0;
-    for (;;) {
-	r = deflate(&state->strm, flush);
-	if (r != Z_OK) {
-	    printf("z_compress: deflate returned %d (%s)\n",
-		   r, (state->strm.msg? state->strm.msg: ""));
-	    break;
-	}
-	if (flush != Z_NO_FLUSH && state->strm.avail_out != 0)
-	    break;		/* all done */
-	if (state->strm.avail_in == 0 && mp != NULL) {
-	    state->strm.next_in = mtod(mp, u_char *);
-	    state->strm.avail_in = mp->m_len;
-	    mp = mp->m_next;
-	    if (mp == NULL)
-		flush = Z_PACKET_FLUSH;
-	}
-	if (state->strm.avail_out == 0) {
-	    if (m != NULL) {
-		m->m_len = wspace;
-		olen += wspace;
-		MGET(m->m_next, M_DONTWAIT, MT_DATA);
-		m = m->m_next;
-		if (m != NULL) {
-		    m->m_len = 0;
-		    if (maxolen - olen > MLEN)
-			MCLGET(m, M_DONTWAIT);
-		    state->strm.next_out = mtod(m, u_char *);
-		    state->strm.avail_out = wspace = M_TRAILINGSPACE(m);
-		}
-	    }
-	    if (m == NULL) {
-		state->strm.next_out = NULL;
-		state->strm.avail_out = 1000000;
-	    }
-	}
-    }
-    if (m != NULL)
-	olen += (m->m_len = wspace - state->strm.avail_out);
-
-    /*
-     * See if we managed to reduce the size of the packet.
-     */
-    if (m != NULL && olen < orig_len) {
-	state->stats.comp_bytes += olen;
-	state->stats.comp_packets++;
-    } else {
-	if (*mret != NULL) {
-	    m_freem(*mret);
-	    *mret = NULL;
-	}
-	state->stats.inc_bytes += orig_len;
-	state->stats.inc_packets++;
-	olen = orig_len;
-    }
-    state->stats.unc_bytes += orig_len;
-    state->stats.unc_packets++;
-
-    return olen;
-}
-
-static void
-z_comp_stats(arg, stats)
-    void *arg;
-    struct compstat *stats;
-{
-    struct deflate_state *state = (struct deflate_state *) arg;
-    u_int out;
-
-    *stats = state->stats;
-    stats->ratio = stats->unc_bytes;
-    out = stats->comp_bytes + stats->inc_bytes;
-    if (stats->ratio <= 0x7ffffff)
-	stats->ratio <<= 8;
-    else
-	out >>= 8;
-    if (out != 0)
-	stats->ratio /= out;
-}
-
-/*
- * Allocate space for a decompressor.
- */
-static void *
-z_decomp_alloc(options, opt_len)
-    u_char *options;
-    int opt_len;
-{
-    struct deflate_state *state;
-    int w_size;
-
-    if (opt_len != CILEN_DEFLATE
-	|| (options[0] != CI_DEFLATE && options[0] != CI_DEFLATE_DRAFT)
-	|| options[1] != CILEN_DEFLATE
-	|| DEFLATE_METHOD(options[2]) != DEFLATE_METHOD_VAL
-	|| options[3] != DEFLATE_CHK_SEQUENCE)
-	return NULL;
-    w_size = DEFLATE_SIZE(options[2]);
-    if (w_size < DEFLATE_MIN_SIZE || w_size > DEFLATE_MAX_SIZE)
-	return NULL;
-
-    state = malloc(sizeof(struct deflate_state),
-	   M_DEVBUF, M_NOWAIT);
-    if (state == NULL)
-	return NULL;
-
-    state->strm.next_out = NULL;
-    state->strm.zalloc = z_alloc;
-    state->strm.zfree = z_free;
-    if (inflateInit2(&state->strm, -w_size) != Z_OK) {
-	free(state, M_DEVBUF);
-	return NULL;
-    }
-
-    state->w_size = w_size;
-    bzero(&state->stats, sizeof(state->stats));
-    return (void *) state;
-}
-
-static void
-z_decomp_free(arg)
-    void *arg;
-{
-    struct deflate_state *state = (struct deflate_state *) arg;
-
-    inflateEnd(&state->strm);
-    free(state, M_DEVBUF);
-}
-
-static int
-z_decomp_init(arg, options, opt_len, unit, hdrlen, mru, debug)
-    void *arg;
-    u_char *options;
-    int opt_len, unit, hdrlen, mru, debug;
-{
-    struct deflate_state *state = (struct deflate_state *) arg;
-
-    if (opt_len < CILEN_DEFLATE
-	|| (options[0] != CI_DEFLATE && options[0] != CI_DEFLATE_DRAFT)
-	|| options[1] != CILEN_DEFLATE
-	|| DEFLATE_METHOD(options[2]) != DEFLATE_METHOD_VAL
-	|| DEFLATE_SIZE(options[2]) != state->w_size
-	|| options[3] != DEFLATE_CHK_SEQUENCE)
-	return 0;
-
-    state->seqno = 0;
-    state->unit = unit;
-    state->hdrlen = hdrlen;
-    state->debug = debug;
-    state->mru = mru;
-
-    inflateReset(&state->strm);
-
-    return 1;
-}
-
-static void
-z_decomp_reset(arg)
-    void *arg;
-{
-    struct deflate_state *state = (struct deflate_state *) arg;
-
-    state->seqno = 0;
-    inflateReset(&state->strm);
-}
-
-/*
- * Decompress a Deflate-compressed packet.
- *
- * Because of patent problems, we return DECOMP_ERROR for errors
- * found by inspecting the input data and for system problems, but
- * DECOMP_FATALERROR for any errors which could possibly be said to
- * be being detected "after" decompression.  For DECOMP_ERROR,
- * we can issue a CCP reset-request; for DECOMP_FATALERROR, we may be
- * infringing a patent of Motorola's if we do, so we take CCP down
- * instead.
- *
- * Given that the frame has the correct sequence number and a good FCS,
- * errors such as invalid codes in the input most likely indicate a
- * bug, so we return DECOMP_FATALERROR for them in order to turn off
- * compression, even though they are detected by inspecting the input.
- */
-static int
-z_decompress(arg, mi, mop)
-    void *arg;
-    struct mbuf *mi, **mop;
-{
-    struct deflate_state *state = (struct deflate_state *) arg;
-    struct mbuf *mo, *mo_head;
-    u_char *rptr, *wptr;
-    int rlen, olen, ospace;
-    int seq, i, flush, r, decode_proto;
-    u_char hdr[PPP_HDRLEN + DEFLATE_OVHD];
-
-    *mop = NULL;
-    rptr = mtod(mi, u_char *);
-    rlen = mi->m_len;
-    for (i = 0; i < PPP_HDRLEN + DEFLATE_OVHD; ++i) {
-	while (rlen <= 0) {
-	    mi = mi->m_next;
-	    if (mi == NULL)
-		return DECOMP_ERROR;
-	    rptr = mtod(mi, u_char *);
-	    rlen = mi->m_len;
-	}
-	hdr[i] = *rptr++;
-	--rlen;
-    }
-
-    /* Check the sequence number. */
-    seq = (hdr[PPP_HDRLEN] << 8) + hdr[PPP_HDRLEN+1];
-    if (seq != state->seqno) {
-	if (state->debug)
-	    printf("z_decompress%d: bad seq # %d, expected %d\n",
-		   state->unit, seq, state->seqno);
-	return DECOMP_ERROR;
-    }
-    ++state->seqno;
-
-    /* Allocate an output mbuf. */
-    MGETHDR(mo, M_DONTWAIT, MT_DATA);
-    if (mo == NULL)
-	return DECOMP_ERROR;
-    mo_head = mo;
-    mo->m_len = 0;
-    mo->m_next = NULL;
-    MCLGET(mo, M_DONTWAIT);
-    ospace = M_TRAILINGSPACE(mo);
-    if (state->hdrlen + PPP_HDRLEN < ospace) {
-	mo->m_data += state->hdrlen;
-	ospace -= state->hdrlen;
-    }
-
-    /*
-     * Fill in the first part of the PPP header.  The protocol field
-     * comes from the decompressed data.
-     */
-    wptr = mtod(mo, u_char *);
-    wptr[0] = PPP_ADDRESS(hdr);
-    wptr[1] = PPP_CONTROL(hdr);
-    wptr[2] = 0;
-
-    /*
-     * Set up to call inflate.  We set avail_out to 1 initially so we can
-     * look at the first byte of the output and decide whether we have
-     * a 1-byte or 2-byte protocol field.
-     */
-    state->strm.next_in = rptr;
-    state->strm.avail_in = rlen;
-    mi = mi->m_next;
-    flush = (mi == NULL)? Z_PACKET_FLUSH: Z_NO_FLUSH;
-    rlen += PPP_HDRLEN + DEFLATE_OVHD;
-    state->strm.next_out = wptr + 3;
-    state->strm.avail_out = 1;
-    decode_proto = 1;
-    olen = PPP_HDRLEN;
-
-    /*
-     * Call inflate, supplying more input or output as needed.
-     */
-    for (;;) {
-	r = inflate(&state->strm, flush);
-	if (r != Z_OK) {
-#if !DEFLATE_DEBUG
-	    if (state->debug)
-#endif
-		printf("z_decompress%d: inflate returned %d (%s)\n",
-		       state->unit, r, (state->strm.msg? state->strm.msg: ""));
-	    m_freem(mo_head);
-	    return DECOMP_FATALERROR;
-	}
-	if (flush != Z_NO_FLUSH && state->strm.avail_out != 0)
-	    break;		/* all done */
-	if (state->strm.avail_in == 0 && mi != NULL) {
-	    state->strm.next_in = mtod(mi, u_char *);
-	    state->strm.avail_in = mi->m_len;
-	    rlen += mi->m_len;
-	    mi = mi->m_next;
-	    if (mi == NULL)
-		flush = Z_PACKET_FLUSH;
-	}
-	if (state->strm.avail_out == 0) {
-	    if (decode_proto) {
-		state->strm.avail_out = ospace - PPP_HDRLEN;
-		if ((wptr[3] & 1) == 0) {
-		    /* 2-byte protocol field */
-		    wptr[2] = wptr[3];
-		    --state->strm.next_out;
-		    ++state->strm.avail_out;
-		    --olen;
-		}
-		decode_proto = 0;
-	    } else {
-		mo->m_len = ospace;
-		olen += ospace;
-		MGET(mo->m_next, M_DONTWAIT, MT_DATA);
-		mo = mo->m_next;
-		if (mo == NULL) {
-		    m_freem(mo_head);
-		    return DECOMP_ERROR;
-		}
-		MCLGET(mo, M_DONTWAIT);
-		state->strm.next_out = mtod(mo, u_char *);
-		state->strm.avail_out = ospace = M_TRAILINGSPACE(mo);
-	    }
-	}
-    }
-    if (decode_proto) {
-	m_freem(mo_head);
-	return DECOMP_ERROR;
-    }
-    olen += (mo->m_len = ospace - state->strm.avail_out);
-#if DEFLATE_DEBUG
-    if (state->debug && olen > state->mru + PPP_HDRLEN)
-	printf("ppp_deflate%d: exceeded mru (%d > %d)\n",
-	       state->unit, olen, state->mru + PPP_HDRLEN);
-#endif
-
-    state->stats.unc_bytes += olen;
-    state->stats.unc_packets++;
-    state->stats.comp_bytes += rlen;
-    state->stats.comp_packets++;
-
-    *mop = mo_head;
-    return DECOMP_OK;
-}
-
-/*
- * Incompressible data has arrived - add it to the history.
- */
-static void
-z_incomp(arg, mi)
-    void *arg;
-    struct mbuf *mi;
-{
-    struct deflate_state *state = (struct deflate_state *) arg;
-    u_char *rptr;
-    int rlen, proto, r;
-
-    /*
-     * Check that the protocol is one we handle.
-     */
-    rptr = mtod(mi, u_char *);
-    proto = PPP_PROTOCOL(rptr);
-    if (proto > 0x3fff || proto == 0xfd || proto == 0xfb)
-	return;
-
-    ++state->seqno;
-
-    /*
-     * Iterate through the mbufs, adding the characters in them
-     * to the decompressor's history.  For the first mbuf, we start
-     * at the either the 1st or 2nd byte of the protocol field,
-     * depending on whether the protocol value is compressible.
-     */
-    rlen = mi->m_len;
-    state->strm.next_in = rptr + 3;
-    state->strm.avail_in = rlen - 3;
-    if (proto > 0xff) {
-	--state->strm.next_in;
-	++state->strm.avail_in;
-    }
-    for (;;) {
-	r = inflateIncomp(&state->strm);
-	if (r != Z_OK) {
-	    /* gak! */
-#if !DEFLATE_DEBUG
-	    if (state->debug)
-#endif
-		printf("z_incomp%d: inflateIncomp returned %d (%s)\n",
-		       state->unit, r, (state->strm.msg? state->strm.msg: ""));
-	    return;
-	}
-	mi = mi->m_next;
-	if (mi == NULL)
-	    break;
-	state->strm.next_in = mtod(mi, u_char *);
-	state->strm.avail_in = mi->m_len;
-	rlen += mi->m_len;
-    }
-
-    /*
-     * Update stats.
-     */
-    state->stats.inc_bytes += rlen;
-    state->stats.inc_packets++;
-    state->stats.unc_bytes += rlen;
-    state->stats.unc_packets++;
-}
-MODULE_DEPEND(ppp_deflate, zlib, 1, 1, 1);
diff --git a/sys/net/ppp_tty.c b/sys/net/ppp_tty.c
deleted file mode 100644
index ae37bd7..0000000
--- a/sys/net/ppp_tty.c
+++ /dev/null
@@ -1,1076 +0,0 @@
-/*
- * ppp_tty.c - Point-to-Point Protocol (PPP) driver for asynchronous
- * 	       tty devices.
- */
-/*-
- * Copyright (c) 1989 Carnegie Mellon University.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms are permitted
- * provided that the above copyright notice and this paragraph are
- * duplicated in all such forms and that any documentation,
- * advertising materials, and other materials related to such
- * distribution and use acknowledge that the software was developed
- * by Carnegie Mellon University.  The name of the
- * University may not be used to endorse or promote products derived
- * from this software without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
- * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
- *
- * Drew D. Perkins
- * Carnegie Mellon University
- * 4910 Forbes Ave.
- * Pittsburgh, PA 15213
- * (412) 268-8576
- * ddp@andrew.cmu.edu
- *
- * Based on:
- *	@(#)if_sl.c	7.6.1.2 (Berkeley) 2/15/89
- *
- * Copyright (c) 1987 Regents of the University of California.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms are permitted
- * provided that the above copyright notice and this paragraph are
- * duplicated in all such forms and that any documentation,
- * advertising materials, and other materials related to such
- * distribution and use acknowledge that the software was developed
- * by the University of California, Berkeley.  The name of the
- * University may not be used to endorse or promote products derived
- * from this software without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
- * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
- *
- * Serial Line interface
- *
- * Rick Adams
- * Center for Seismic Studies
- * 1300 N 17th Street, Suite 1450
- * Arlington, Virginia 22209
- * (703)276-7900
- * rick@seismo.ARPA
- * seismo!rick
- *
- * Pounded on heavily by Chris Torek (chris@mimsy.umd.edu, umcp-cs!chris).
- * Converted to 4.3BSD Beta by Chris Torek.
- * Other changes made at Berkeley, based in part on code by Kirk Smith.
- *
- * Converted to 4.3BSD+ 386BSD by Brad Parker (brad@cayman.com)
- * Added VJ tcp header compression; more unified ioctls
- *
- * Extensively modified by Paul Mackerras (paulus@cs.anu.edu.au).
- * Cleaned up a lot of the mbuf-related code to fix bugs that
- * caused system crashes and packet corruption.  Changed pppstart
- * so that it doesn't just give up with a "collision" if the whole
- * packet doesn't fit in the output ring buffer.
- *
- * Added priority queueing for interactive IP packets, following
- * the model of if_sl.c, plus hooks for bpf.
- * Paul Mackerras (paulus@cs.anu.edu.au).
- */
-
-/* $FreeBSD$ */
-
-#include "opt_ppp.h"		/* XXX for ppp_defs.h */
-
-#define VJC			/* XXX for ppp_defs.h */
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/priv.h>
-#include <sys/proc.h>
-#include <sys/mbuf.h>
-#include <sys/socket.h>
-#include <sys/fcntl.h>
-#include <sys/tty.h>
-#include <sys/conf.h>
-#include <sys/uio.h>
-
-#ifdef PPP_FILTER
-#include <net/bpf.h>
-#endif
-#include <net/if_ppp.h>
-#include <net/if_pppvar.h>
-
-static int	pppopen(struct cdev *dev, struct tty *tp);
-static int	pppclose(struct tty *tp, int flag);
-static int	pppread(struct tty *tp, struct uio *uio, int flag);
-static int	pppwrite(struct tty *tp, struct uio *uio, int flag);
-static int	ppptioctl(struct tty *tp, u_long cmd, caddr_t data,
-					int flag, struct thread *td);
-static int	pppinput(int c, struct tty *tp);
-static int	pppstart(struct tty *tp);
-
-static u_short	pppfcs(u_short fcs, u_char *cp, int len);
-static void	pppasyncstart(struct ppp_softc *);
-static void	pppasyncctlp(struct ppp_softc *);
-static void	pppasyncrelinq(struct ppp_softc *);
-static void	pppasyncsetmtu(struct ppp_softc *);
-static void	ppp_timeout(void *);
-static void	pppgetm(struct ppp_softc *sc);
-static void	ppplogchar(struct ppp_softc *, int);
-
-/* XXX called from if_ppp.c - layering violation */
-void		pppasyncattach(void *);
-void		pppasyncdetach(void);
-
-/*
- * Some useful mbuf macros not in mbuf.h.
- */
-#define M_IS_CLUSTER(m)	((m)->m_flags & M_EXT)
-
-#define M_DATASTART(m)	\
-	(M_IS_CLUSTER(m) ? (m)->m_ext.ext_buf : \
-	    (m)->m_flags & M_PKTHDR ? (m)->m_pktdat : (m)->m_dat)
-
-#define M_DATASIZE(m)	\
-	(M_IS_CLUSTER(m) ? (m)->m_ext.ext_size : \
-	    (m)->m_flags & M_PKTHDR ? MHLEN: MLEN)
-
-/*
- * Does c need to be escaped?
- */
-#define ESCAPE_P(c)	(sc->sc_asyncmap[(c) >> 5] & (1 << ((c) & 0x1F)))
-
-/*
- * Procedures for using an async tty interface for PPP.
- */
-
-/* This is a FreeBSD-2.X kernel. */
-#define CCOUNT(q)	((q)->c_cc)
-#define PPP_LOWAT	100	/* Process more output when < LOWAT on queue */
-#define	PPP_HIWAT	400	/* Don't start a new packet if HIWAT on que */
-
-/*
- * Define the PPP line discipline.
- */
-
-static struct linesw pppdisc = {
-	pppopen,	pppclose,	pppread,	pppwrite,
-	ppptioctl,	pppinput,	pppstart,	ttymodem
-};
-
-void
-pppasyncattach(dummy)
-    void *dummy;
-{
-    ldisc_register(PPPDISC, &pppdisc);
-}
-
-void
-pppasyncdetach()
-{
-    ldisc_deregister(PPPDISC);
-}
-
-/*
- * Line specific open routine for async tty devices.
- * Attach the given tty to the first available ppp unit.
- * Called from device open routine or ttioctl() at >= splsofttty()
- */
-/* ARGSUSED */
-static int
-pppopen(dev, tp)
-    struct cdev *dev;
-    register struct tty *tp;
-{
-    struct thread *td = curthread;		/* XXX */
-    register struct ppp_softc *sc;
-    int error, s;
-
-    error = priv_check(td, PRIV_NET_PPP);
-    if (error)
-	return (error);
-
-    s = spltty();
-
-    tp->t_hotchar = PPP_FLAG;
-
-    if ((sc = pppalloc(td->td_proc->p_pid)) == NULL) {
-	splx(s);
-	return ENXIO;
-    }
-
-    if (sc->sc_relinq)
-	(*sc->sc_relinq)(sc);	/* get previous owner to relinquish the unit */
-
-    sc->sc_ilen = 0;
-    sc->sc_m = NULL;
-    bzero(sc->sc_asyncmap, sizeof(sc->sc_asyncmap));
-    sc->sc_asyncmap[0] = 0xffffffff;
-    sc->sc_asyncmap[3] = 0x60000000;
-    sc->sc_rasyncmap = 0;
-    sc->sc_devp = (void *) tp;
-    sc->sc_start = pppasyncstart;
-    sc->sc_ctlp = pppasyncctlp;
-    sc->sc_relinq = pppasyncrelinq;
-    sc->sc_setmtu = pppasyncsetmtu;
-    sc->sc_outm = NULL;
-    pppgetm(sc);
-    PPP2IFP(sc)->if_drv_flags |= IFF_DRV_RUNNING;
-    getmicrotime(&PPP2IFP(sc)->if_lastchange);
-    PPP2IFP(sc)->if_baudrate = tp->t_ospeed;
-
-    tp->t_hotchar = PPP_FLAG;
-    tp->t_lsc = sc;
-    ttyflush(tp, FREAD | FWRITE);
-
-    /*
-     * Pre-allocate cblocks to the "just right" amount.  The 1 byte t_canq
-     * allocation helps avoid the need for select and/or FIONREAD.
-     * We also pass 1 byte tokens through t_canq...
-     */
-    clist_alloc_cblocks(&tp->t_canq, 1, 1);
-    clist_alloc_cblocks(&tp->t_outq, PPP2IFP(sc)->if_mtu + PPP_HIWAT,
-			PPP2IFP(sc)->if_mtu + PPP_HIWAT);
-    clist_alloc_cblocks(&tp->t_rawq, 0, 0);
-
-    splx(s);
-
-    return (0);
-}
-
-/*
- * Line specific close routine, called from device close routine
- * and from ttioctl at >= splsofttty().
- * Detach the tty from the ppp unit.
- * Mimics part of tty_close().
- */
-static int
-pppclose(tp, flag)
-    struct tty *tp;
-    int flag;
-{
-    register struct ppp_softc *sc = (struct ppp_softc *)tp->t_lsc;
-    int s;
-
-    s = spltty();
-    ttyflush(tp, FREAD | FWRITE);
-    clist_free_cblocks(&tp->t_canq);
-    clist_free_cblocks(&tp->t_outq);
-    if (sc != NULL) {
-	    pppasyncrelinq(sc);
-	    pppdealloc(sc);
-    }
-    splx(s);
-    return 0;
-}
-
-/*
- * Relinquish the interface unit to another device.
- */
-static void
-pppasyncrelinq(sc)
-    struct ppp_softc *sc;
-{
-    int s;
-
-    s = spltty();
-    if (sc->sc_outm) {
-	m_freem(sc->sc_outm);
-	sc->sc_outm = NULL;
-    }
-    if (sc->sc_m) {
-	m_freem(sc->sc_m);
-	sc->sc_m = NULL;
-    }
-    if (sc->sc_flags & SC_TIMEOUT) {
-	callout_stop(&sc->sc_timo_ch);
-	sc->sc_flags &= ~SC_TIMEOUT;
-    }
-    splx(s);
-}
-
-/*
- * This gets called from the upper layer to notify a mtu change
- */
-static void
-pppasyncsetmtu(sc)
-register struct ppp_softc *sc;
-{
-    register struct tty *tp = (struct tty *) sc->sc_devp;
-    int s;
-
-    s = spltty();
-    if (tp != NULL)
-	clist_alloc_cblocks(&tp->t_outq, PPP2IFP(sc)->if_mtu + PPP_HIWAT,
-			     PPP2IFP(sc)->if_mtu + PPP_HIWAT);
-    splx(s);
-}
-
-/*
- * Line specific (tty) read routine.
- * called at zero spl from the device driver in the response to user-level
- * reads on the tty file descriptor (ie: pppd).
- */
-static int
-pppread(tp, uio, flag)
-    register struct tty *tp;
-    struct uio *uio;
-    int flag;
-{
-    register struct ppp_softc *sc = (struct ppp_softc *)tp->t_lsc;
-    struct mbuf *m, *m0;
-    register int s;
-    int error = 0;
-
-    if (sc == NULL)
-	return 0;
-    /*
-     * Loop waiting for input, checking that nothing disasterous
-     * happens in the meantime.
-     */
-    s = spltty();
-    for (;;) {
-	if (tp != (struct tty *) sc->sc_devp || tp->t_line != PPPDISC) {
-	    splx(s);
-	    return 0;
-	}
-	if (sc->sc_inq.ifq_head != NULL)
-	    break;
-	if ((tp->t_state & TS_CONNECTED) == 0) {
-	    splx(s);
-	    return 0;		/* end of file */
-	}
-	if (tp->t_state & TS_ASYNC || flag & O_NONBLOCK) {
-	    splx(s);
-	    return (EWOULDBLOCK);
-	}
-	error = ttysleep(tp, TSA_HUP_OR_INPUT(tp), TTIPRI | PCATCH, "pppin", 0);
-	if (error) {
-	    splx(s);
-	    return error;
-	}
-    }
-
-    /* Pull place-holder byte out of canonical queue */
-    getc(&tp->t_canq);
-
-    /* Get the packet from the input queue */
-    IF_DEQUEUE(&sc->sc_inq, m0);
-    splx(s);
-
-    for (m = m0; m && uio->uio_resid; m = m->m_next)
-	if ((error = uiomove(mtod(m, u_char *), m->m_len, uio)) != 0)
-	    break;
-    m_freem(m0);
-    return (error);
-}
-
-/*
- * Line specific (tty) write routine.
- * called at zero spl from the device driver in the response to user-level
- * writes on the tty file descriptor (ie: pppd).
- */
-static int
-pppwrite(tp, uio, flag)
-    register struct tty *tp;
-    struct uio *uio;
-    int flag;
-{
-    register struct ppp_softc *sc = (struct ppp_softc *)tp->t_lsc;
-    struct mbuf *m;
-    struct sockaddr dst;
-    int error, s;
-
-    if ((tp->t_state & TS_CONNECTED) == 0)
-	return 0;		/* wrote 0 bytes */
-    if (tp->t_line != PPPDISC)
-	return (EINVAL);
-    if (sc == NULL)
-	return EIO;
-    if (uio->uio_resid > PPP2IFP(sc)->if_mtu + PPP_HDRLEN ||
-	uio->uio_resid < PPP_HDRLEN)
-	return (EMSGSIZE);
-
-    s = spltty();
-    if ((m = m_uiotombuf(uio, M_DONTWAIT, 0, 0, M_PKTHDR)) == NULL) {
-	splx(s);
-	return (ENOBUFS);
-    }
-
-    dst.sa_family = AF_UNSPEC;
-    bcopy(mtod(m, u_char *), dst.sa_data, PPP_HDRLEN);
-    m->m_data += PPP_HDRLEN;
-    m->m_len -= PPP_HDRLEN;
-
-    /* call the upper layer to "transmit" it... */
-    error = pppoutput(PPP2IFP(sc), m, &dst, NULL);
-    splx(s);
-    return (error);
-}
-
-/*
- * Line specific (tty) ioctl routine.
- * This discipline requires that tty device drivers call
- * the line specific l_ioctl routine from their ioctl routines.
- */
-/* ARGSUSED */
-static int
-ppptioctl(tp, cmd, data, flag, td)
-    struct tty *tp;
-    u_long cmd;
-    caddr_t data;
-    int flag;
-    struct thread *td;
-{
-    struct ppp_softc *sc = (struct ppp_softc *)tp->t_lsc;
-    int error, s;
-
-    if (sc == NULL || tp != (struct tty *) sc->sc_devp)
-	return (ENOIOCTL);
-
-    error = 0;
-    switch (cmd) {
-    case PPPIOCSASYNCMAP:
-	error = priv_check(td, PRIV_NET_PPP);
-	if (error)
-	    break;
-	sc->sc_asyncmap[0] = *(u_int *)data;
-	break;
-
-    case PPPIOCGASYNCMAP:
-	*(u_int *)data = sc->sc_asyncmap[0];
-	break;
-
-    case PPPIOCSRASYNCMAP:
-	error = priv_check(td, PRIV_NET_PPP);
-	if (error)
-	    break;
-	sc->sc_rasyncmap = *(u_int *)data;
-	break;
-
-    case PPPIOCGRASYNCMAP:
-	*(u_int *)data = sc->sc_rasyncmap;
-	break;
-
-    case PPPIOCSXASYNCMAP:
-	error = priv_check(td, PRIV_NET_PPP);
-	if (error)
-	    break;
-	s = spltty();
-	bcopy(data, sc->sc_asyncmap, sizeof(sc->sc_asyncmap));
-	sc->sc_asyncmap[1] = 0;		    /* mustn't escape 0x20 - 0x3f */
-	sc->sc_asyncmap[2] &= ~0x40000000;  /* mustn't escape 0x5e */
-	sc->sc_asyncmap[3] |= 0x60000000;   /* must escape 0x7d, 0x7e */
-	splx(s);
-	break;
-
-    case PPPIOCGXASYNCMAP:
-	bcopy(sc->sc_asyncmap, data, sizeof(sc->sc_asyncmap));
-	break;
-
-    default:
-	error = pppioctl(sc, cmd, data, flag, td);
-	if (error == 0 && cmd == PPPIOCSMRU)
-	    pppgetm(sc);
-    }
-
-    return error;
-}
-
-/*
- * FCS lookup table as calculated by genfcstab.
- */
-static u_short fcstab[256] = {
-	0x0000,	0x1189,	0x2312,	0x329b,	0x4624,	0x57ad,	0x6536,	0x74bf,
-	0x8c48,	0x9dc1,	0xaf5a,	0xbed3,	0xca6c,	0xdbe5,	0xe97e,	0xf8f7,
-	0x1081,	0x0108,	0x3393,	0x221a,	0x56a5,	0x472c,	0x75b7,	0x643e,
-	0x9cc9,	0x8d40,	0xbfdb,	0xae52,	0xdaed,	0xcb64,	0xf9ff,	0xe876,
-	0x2102,	0x308b,	0x0210,	0x1399,	0x6726,	0x76af,	0x4434,	0x55bd,
-	0xad4a,	0xbcc3,	0x8e58,	0x9fd1,	0xeb6e,	0xfae7,	0xc87c,	0xd9f5,
-	0x3183,	0x200a,	0x1291,	0x0318,	0x77a7,	0x662e,	0x54b5,	0x453c,
-	0xbdcb,	0xac42,	0x9ed9,	0x8f50,	0xfbef,	0xea66,	0xd8fd,	0xc974,
-	0x4204,	0x538d,	0x6116,	0x709f,	0x0420,	0x15a9,	0x2732,	0x36bb,
-	0xce4c,	0xdfc5,	0xed5e,	0xfcd7,	0x8868,	0x99e1,	0xab7a,	0xbaf3,
-	0x5285,	0x430c,	0x7197,	0x601e,	0x14a1,	0x0528,	0x37b3,	0x263a,
-	0xdecd,	0xcf44,	0xfddf,	0xec56,	0x98e9,	0x8960,	0xbbfb,	0xaa72,
-	0x6306,	0x728f,	0x4014,	0x519d,	0x2522,	0x34ab,	0x0630,	0x17b9,
-	0xef4e,	0xfec7,	0xcc5c,	0xddd5,	0xa96a,	0xb8e3,	0x8a78,	0x9bf1,
-	0x7387,	0x620e,	0x5095,	0x411c,	0x35a3,	0x242a,	0x16b1,	0x0738,
-	0xffcf,	0xee46,	0xdcdd,	0xcd54,	0xb9eb,	0xa862,	0x9af9,	0x8b70,
-	0x8408,	0x9581,	0xa71a,	0xb693,	0xc22c,	0xd3a5,	0xe13e,	0xf0b7,
-	0x0840,	0x19c9,	0x2b52,	0x3adb,	0x4e64,	0x5fed,	0x6d76,	0x7cff,
-	0x9489,	0x8500,	0xb79b,	0xa612,	0xd2ad,	0xc324,	0xf1bf,	0xe036,
-	0x18c1,	0x0948,	0x3bd3,	0x2a5a,	0x5ee5,	0x4f6c,	0x7df7,	0x6c7e,
-	0xa50a,	0xb483,	0x8618,	0x9791,	0xe32e,	0xf2a7,	0xc03c,	0xd1b5,
-	0x2942,	0x38cb,	0x0a50,	0x1bd9,	0x6f66,	0x7eef,	0x4c74,	0x5dfd,
-	0xb58b,	0xa402,	0x9699,	0x8710,	0xf3af,	0xe226,	0xd0bd,	0xc134,
-	0x39c3,	0x284a,	0x1ad1,	0x0b58,	0x7fe7,	0x6e6e,	0x5cf5,	0x4d7c,
-	0xc60c,	0xd785,	0xe51e,	0xf497,	0x8028,	0x91a1,	0xa33a,	0xb2b3,
-	0x4a44,	0x5bcd,	0x6956,	0x78df,	0x0c60,	0x1de9,	0x2f72,	0x3efb,
-	0xd68d,	0xc704,	0xf59f,	0xe416,	0x90a9,	0x8120,	0xb3bb,	0xa232,
-	0x5ac5,	0x4b4c,	0x79d7,	0x685e,	0x1ce1,	0x0d68,	0x3ff3,	0x2e7a,
-	0xe70e,	0xf687,	0xc41c,	0xd595,	0xa12a,	0xb0a3,	0x8238,	0x93b1,
-	0x6b46,	0x7acf,	0x4854,	0x59dd,	0x2d62,	0x3ceb,	0x0e70,	0x1ff9,
-	0xf78f,	0xe606,	0xd49d,	0xc514,	0xb1ab,	0xa022,	0x92b9,	0x8330,
-	0x7bc7,	0x6a4e,	0x58d5,	0x495c,	0x3de3,	0x2c6a,	0x1ef1,	0x0f78
-};
-
-/*
- * Calculate a new FCS given the current FCS and the new data.
- */
-static u_short
-pppfcs(u_short fcs, u_char *cp, int len)
-{
-    while (len--)
-	fcs = PPP_FCS(fcs, *cp++);
-    return (fcs);
-}
-
-/*
- * This gets called at splsoftnet from if_ppp.c at various times
- * when there is data ready to be sent.
- */
-static void
-pppasyncstart(sc)
-    register struct ppp_softc *sc;
-{
-    register struct tty *tp = (struct tty *) sc->sc_devp;
-    register struct mbuf *m;
-    register int len;
-    register u_char *start, *stop, *cp;
-    int n, ndone, done, idle;
-    int s;
-
-    idle = 0;
-    /* XXX assumes atomic access to *tp although we're not at spltty(). */
-    while (CCOUNT(&tp->t_outq) < PPP_HIWAT) {
-	/*
-	 * See if we have an existing packet partly sent.
-	 * If not, get a new packet and start sending it.
-	 */
-	m = sc->sc_outm;
-	if (m == NULL) {
-	    /*
-	     * Get another packet to be sent.
-	     */
-	    m = ppp_dequeue(sc);
-	    if (m == NULL) {
-		idle = 1;
-		break;
-	    }
-
-	    /*
-	     * The extra PPP_FLAG will start up a new packet, and thus
-	     * will flush any accumulated garbage.  We do this whenever
-	     * the line may have been idle for some time.
-	     */
-	    /* XXX as above. */
-	    if (CCOUNT(&tp->t_outq) == 0) {
-		++sc->sc_stats.ppp_obytes;
-		(void) putc(PPP_FLAG, &tp->t_outq);
-	    }
-
-	    /* Calculate the FCS for the first mbuf's worth. */
-	    sc->sc_outfcs = pppfcs(PPP_INITFCS, mtod(m, u_char *), m->m_len);
-	    getmicrotime(&PPP2IFP(sc)->if_lastchange);
-	}
-
-	for (;;) {
-	    start = mtod(m, u_char *);
-	    len = m->m_len;
-	    stop = start + len;
-	    while (len > 0) {
-		/*
-		 * Find out how many bytes in the string we can
-		 * handle without doing something special.
-		 */
-		for (cp = start; cp < stop; cp++)
-		    if (ESCAPE_P(*cp))
-			break;
-		n = cp - start;
-		if (n) {
-		    /* NetBSD (0.9 or later), 4.3-Reno or similar. */
-		    ndone = n - b_to_q(start, n, &tp->t_outq);
-		    len -= ndone;
-		    start += ndone;
-		    sc->sc_stats.ppp_obytes += ndone;
-
-		    if (ndone < n)
-			break;	/* packet doesn't fit */
-		}
-		/*
-		 * If there are characters left in the mbuf,
-		 * the first one must be special.
-		 * Put it out in a different form.
-		 */
-		if (len) {
-		    s = spltty();
-		    if (putc(PPP_ESCAPE, &tp->t_outq)) {
-			splx(s);
-			break;
-		    }
-		    if (putc(*start ^ PPP_TRANS, &tp->t_outq)) {
-			(void) unputc(&tp->t_outq);
-			splx(s);
-			break;
-		    }
-		    splx(s);
-		    sc->sc_stats.ppp_obytes += 2;
-		    start++;
-		    len--;
-		}
-	    }
-
-	    /*
-	     * If we didn't empty this mbuf, remember where we're up to.
-	     * If we emptied the last mbuf, try to add the FCS and closing
-	     * flag, and if we can't, leave sc_outm pointing to m, but with
-	     * m->m_len == 0, to remind us to output the FCS and flag later.
-	     */
-	    done = len == 0;
-	    if (done && m->m_next == NULL) {
-		u_char *p, *q;
-		int c;
-		u_char endseq[8];
-
-		/*
-		 * We may have to escape the bytes in the FCS.
-		 */
-		p = endseq;
-		c = ~sc->sc_outfcs & 0xFF;
-		if (ESCAPE_P(c)) {
-		    *p++ = PPP_ESCAPE;
-		    *p++ = c ^ PPP_TRANS;
-		} else
-		    *p++ = c;
-		c = (~sc->sc_outfcs >> 8) & 0xFF;
-		if (ESCAPE_P(c)) {
-		    *p++ = PPP_ESCAPE;
-		    *p++ = c ^ PPP_TRANS;
-		} else
-		    *p++ = c;
-		*p++ = PPP_FLAG;
-
-		/*
-		 * Try to output the FCS and flag.  If the bytes
-		 * don't all fit, back out.
-		 */
-		s = spltty();
-		for (q = endseq; q < p; ++q)
-		    if (putc(*q, &tp->t_outq)) {
-			done = 0;
-			for (; q > endseq; --q)
-			    unputc(&tp->t_outq);
-			break;
-		    }
-		splx(s);
-		if (done)
-		    sc->sc_stats.ppp_obytes += q - endseq;
-	    }
-
-	    if (!done) {
-		/* remember where we got to */
-		m->m_data = start;
-		m->m_len = len;
-		break;
-	    }
-
-	    /* Finished with this mbuf; free it and move on. */
-	    m = m_free(m);
-	    if (m == NULL) {
-		/* Finished a packet */
-		break;
-	    }
-	    sc->sc_outfcs = pppfcs(sc->sc_outfcs, mtod(m, u_char *), m->m_len);
-	}
-
-	/*
-	 * If m == NULL, we have finished a packet.
-	 * If m != NULL, we've either done as much work this time
-	 * as we need to, or else we've filled up the output queue.
-	 */
-	sc->sc_outm = m;
-	if (m)
-	    break;
-    }
-
-    /* Call pppstart to start output again if necessary. */
-    s = spltty();
-    pppstart(tp);
-
-    /*
-     * This timeout is needed for operation on a pseudo-tty,
-     * because the pty code doesn't call pppstart after it has
-     * drained the t_outq.
-     */
-    if (!idle && (sc->sc_flags & SC_TIMEOUT) == 0) {
-	callout_reset(&sc->sc_timo_ch, 1, ppp_timeout, sc);
-	sc->sc_flags |= SC_TIMEOUT;
-    }
-
-    splx(s);
-}
-
-/*
- * This gets called when a received packet is placed on
- * the inq, at splsoftnet. The pppd daemon is to be woken up to do a read().
- */
-static void
-pppasyncctlp(sc)
-    struct ppp_softc *sc;
-{
-    struct tty *tp;
-    int s;
-
-    /* Put a placeholder byte in canq for ttselect()/ttnread(). */
-    s = spltty();
-    tp = (struct tty *) sc->sc_devp;
-    putc(0, &tp->t_canq);
-    ttwakeup(tp);
-    splx(s);
-}
-
-/*
- * Start output on async tty interface.  If the transmit queue
- * has drained sufficiently, arrange for pppasyncstart to be
- * called later at splsoftnet.
- * Called at spltty or higher.
- */
-static int
-pppstart(tp)
-    register struct tty *tp;
-{
-    register struct ppp_softc *sc = (struct ppp_softc *)tp->t_lsc;
-
-    /*
-     * Call output process whether or not there is any output.
-     * We are being called in lieu of ttstart and must do what it would.
-     */
-    tt_oproc(tp);
-
-    /*
-     * If the transmit queue has drained and the tty has not hung up
-     * or been disconnected from the ppp unit, then tell if_ppp.c that
-     * we need more output.
-     */
-    if (CCOUNT(&tp->t_outq) < PPP_LOWAT
-	&& !((tp->t_state & TS_CONNECTED) == 0)
-	&& sc != NULL) {
-	ppp_restart(sc);
-    }
-
-    return 0;
-}
-
-/*
- * Timeout routine - try to start some more output.
- */
-static void
-ppp_timeout(x)
-    void *x;
-{
-    struct ppp_softc *sc = (struct ppp_softc *) x;
-    struct tty *tp = (struct tty *) sc->sc_devp;
-    int s;
-
-    s = spltty();
-    sc->sc_flags &= ~SC_TIMEOUT;
-    pppstart(tp);
-    splx(s);
-}
-
-/*
- * Allocate enough mbuf to handle current MRU.
- */
-static void
-pppgetm(sc)
-    register struct ppp_softc *sc;
-{
-    struct mbuf *m, **mp;
-    int len;
-
-    mp = &sc->sc_m;
-    for (len = sc->sc_mru + PPP_HDRLEN + PPP_FCSLEN; len > 0; ){
-	if ((m = *mp) == NULL) {
-	    MGETHDR(m, M_DONTWAIT, MT_DATA);
-	    if (m == NULL)
-		break;
-	    *mp = m;
-	    MCLGET(m, M_DONTWAIT);
-	}
-	len -= M_DATASIZE(m);
-	mp = &m->m_next;
-    }
-}
-
-/*
- * tty interface receiver interrupt.
- */
-static unsigned paritytab[8] = {
-    0x96696996, 0x69969669, 0x69969669, 0x96696996,
-    0x69969669, 0x96696996, 0x96696996, 0x69969669
-};
-
-/*
- * Called when character is available from device driver.
- * Only guaranteed to be at splsofttty() or spltty()
- * This is safe to be called while the upper half's netisr is preempted.
- */
-static int
-pppinput(c, tp)
-    int c;
-    register struct tty *tp;
-{
-    register struct ppp_softc *sc;
-    struct mbuf *m;
-    int ilen, s;
-
-    sc = (struct ppp_softc *)tp->t_lsc;
-    if (sc == NULL)
-	return 0;
-
-    ++tk_nin;
-    ++sc->sc_stats.ppp_ibytes;
-
-    if ((tp->t_state & TS_CONNECTED) == 0) {
-	if (sc->sc_flags & SC_DEBUG)
-	    if_printf(PPP2IFP(sc), "no carrier\n");
-	goto flush;
-    }
-
-    if (c & TTY_ERRORMASK) {
-	/* framing error or overrun on this char - abort packet */
-	if (sc->sc_flags & SC_DEBUG)
-	    if_printf(PPP2IFP(sc), "line error %x\n", c & TTY_ERRORMASK);
-	goto flush;
-    }
-
-    c &= TTY_CHARMASK;
-
-    /*
-     * Handle software flow control of output.
-     */
-    if (tp->t_iflag & IXON) {
-	if (c == tp->t_cc[VSTOP] && tp->t_cc[VSTOP] != _POSIX_VDISABLE) {
-	    if ((tp->t_state & TS_TTSTOP) == 0) {
-		tp->t_state |= TS_TTSTOP;
-		tt_stop(tp, 0);
-	    }
-	    return 0;
-	}
-	if (c == tp->t_cc[VSTART] && tp->t_cc[VSTART] != _POSIX_VDISABLE) {
-	    tp->t_state &= ~TS_TTSTOP;
-	    tt_oproc(tp);
-	    return 0;
-	}
-    }
-
-    s = spltty();
-    if (c & 0x80)
-	sc->sc_flags |= SC_RCV_B7_1;
-    else
-	sc->sc_flags |= SC_RCV_B7_0;
-    if (paritytab[c >> 5] & (1 << (c & 0x1F)))
-	sc->sc_flags |= SC_RCV_ODDP;
-    else
-	sc->sc_flags |= SC_RCV_EVNP;
-    splx(s);
-
-    if (sc->sc_flags & SC_LOG_RAWIN)
-	ppplogchar(sc, c);
-
-    if (c == PPP_FLAG) {
-	ilen = sc->sc_ilen;
-	sc->sc_ilen = 0;
-
-	if (sc->sc_rawin_count > 0) 
-	    ppplogchar(sc, -1);
-
-	/*
-	 * If SC_ESCAPED is set, then we've seen the packet
-	 * abort sequence "}~".
-	 */
-	if (sc->sc_flags & (SC_FLUSH | SC_ESCAPED)
-	    || (ilen > 0 && sc->sc_fcs != PPP_GOODFCS)) {
-	    s = spltty();
-	    sc->sc_flags |= SC_PKTLOST;	/* note the dropped packet */
-	    if ((sc->sc_flags & (SC_FLUSH | SC_ESCAPED)) == 0){
-		if (sc->sc_flags & SC_DEBUG)
-		    if_printf(PPP2IFP(sc), "bad fcs %x, pkt len %d\n",
-			   sc->sc_fcs, ilen);
-		PPP2IFP(sc)->if_ierrors++;
-		sc->sc_stats.ppp_ierrors++;
-	    } else
-		sc->sc_flags &= ~(SC_FLUSH | SC_ESCAPED);
-	    splx(s);
-	    return 0;
-	}
-
-	if (ilen < PPP_HDRLEN + PPP_FCSLEN) {
-	    if (ilen) {
-		if (sc->sc_flags & SC_DEBUG)
-		    if_printf(PPP2IFP(sc), "too short (%d)\n", ilen);
-		s = spltty();
-		PPP2IFP(sc)->if_ierrors++;
-		sc->sc_stats.ppp_ierrors++;
-		sc->sc_flags |= SC_PKTLOST;
-		splx(s);
-	    }
-	    return 0;
-	}
-
-	/*
-	 * Remove FCS trailer.  Somewhat painful...
-	 */
-	ilen -= 2;
-	if (--sc->sc_mc->m_len == 0) {
-	    for (m = sc->sc_m; m->m_next != sc->sc_mc; m = m->m_next)
-		;
-	    sc->sc_mc = m;
-	}
-	sc->sc_mc->m_len--;
-
-	/* excise this mbuf chain */
-	m = sc->sc_m;
-	sc->sc_m = sc->sc_mc->m_next;
-	sc->sc_mc->m_next = NULL;
-
-	ppppktin(sc, m, sc->sc_flags & SC_PKTLOST);
-	if (sc->sc_flags & SC_PKTLOST) {
-	    s = spltty();
-	    sc->sc_flags &= ~SC_PKTLOST;
-	    splx(s);
-	}
-
-	pppgetm(sc);
-	return 0;
-    }
-
-    if (sc->sc_flags & SC_FLUSH) {
-	if (sc->sc_flags & SC_LOG_FLUSH)
-	    ppplogchar(sc, c);
-	return 0;
-    }
-
-    if (c < 0x20 && (sc->sc_rasyncmap & (1 << c)))
-	return 0;
-
-    s = spltty();
-    if (sc->sc_flags & SC_ESCAPED) {
-	sc->sc_flags &= ~SC_ESCAPED;
-	c ^= PPP_TRANS;
-    } else if (c == PPP_ESCAPE) {
-	sc->sc_flags |= SC_ESCAPED;
-	splx(s);
-	return 0;
-    }
-    splx(s);
-
-    /*
-     * Initialize buffer on first octet received.
-     * First octet could be address or protocol (when compressing
-     * address/control).
-     * Second octet is control.
-     * Third octet is first or second (when compressing protocol)
-     * octet of protocol.
-     * Fourth octet is second octet of protocol.
-     */
-    if (sc->sc_ilen == 0) {
-	/* reset the first input mbuf */
-	if (sc->sc_m == NULL) {
-	    pppgetm(sc);
-	    if (sc->sc_m == NULL) {
-		if (sc->sc_flags & SC_DEBUG)
-		    if_printf(PPP2IFP(sc), "no input mbufs!\n");
-		goto flush;
-	    }
-	}
-	m = sc->sc_m;
-	m->m_len = 0;
-	m->m_data = M_DATASTART(sc->sc_m);
-	sc->sc_mc = m;
-	sc->sc_mp = mtod(m, char *);
-	sc->sc_fcs = PPP_INITFCS;
-	if (c != PPP_ALLSTATIONS) {
-	    if (sc->sc_flags & SC_REJ_COMP_AC) {
-		if (sc->sc_flags & SC_DEBUG)
-		    if_printf(PPP2IFP(sc),
-		              "garbage received: 0x%x (need 0xFF)\n", c);
-		goto flush;
-	    }
-	    *sc->sc_mp++ = PPP_ALLSTATIONS;
-	    *sc->sc_mp++ = PPP_UI;
-	    sc->sc_ilen += 2;
-	    m->m_len += 2;
-	}
-    }
-    if (sc->sc_ilen == 1 && c != PPP_UI) {
-	if (sc->sc_flags & SC_DEBUG)
-	    if_printf(PPP2IFP(sc), "missing UI (0x3), got 0x%x\n", c);
-	goto flush;
-    }
-    if (sc->sc_ilen == 2 && (c & 1) == 1) {
-	/* a compressed protocol */
-	*sc->sc_mp++ = 0;
-	sc->sc_ilen++;
-	sc->sc_mc->m_len++;
-    }
-    if (sc->sc_ilen == 3 && (c & 1) == 0) {
-	if (sc->sc_flags & SC_DEBUG)
-	    if_printf(PPP2IFP(sc), "bad protocol %x\n",
-		      (sc->sc_mp[-1] << 8) + c);
-	goto flush;
-    }
-
-    /* packet beyond configured mru? */
-    if (++sc->sc_ilen > sc->sc_mru + PPP_HDRLEN + PPP_FCSLEN) {
-	if (sc->sc_flags & SC_DEBUG)
-	    if_printf(PPP2IFP(sc), "packet too big\n");
-	goto flush;
-    }
-
-    /* is this mbuf full? */
-    m = sc->sc_mc;
-    if (M_TRAILINGSPACE(m) <= 0) {
-	if (m->m_next == NULL) {
-	    pppgetm(sc);
-	    if (m->m_next == NULL) {
-		if (sc->sc_flags & SC_DEBUG)
-		    if_printf(PPP2IFP(sc), "too few input mbufs!\n");
-		goto flush;
-	    }
-	}
-	sc->sc_mc = m = m->m_next;
-	m->m_len = 0;
-	m->m_data = M_DATASTART(m);
-	sc->sc_mp = mtod(m, char *);
-    }
-
-    ++m->m_len;
-    *sc->sc_mp++ = c;
-    sc->sc_fcs = PPP_FCS(sc->sc_fcs, c);
-    return 0;
-
- flush:
-    if (!(sc->sc_flags & SC_FLUSH)) {
-	s = spltty();
-	PPP2IFP(sc)->if_ierrors++;
-	sc->sc_stats.ppp_ierrors++;
-	sc->sc_flags |= SC_FLUSH;
-	splx(s);
-	if (sc->sc_flags & SC_LOG_FLUSH)
-	    ppplogchar(sc, c);
-    }
-    return 0;
-}
-
-#define MAX_DUMP_BYTES	128
-
-static void
-ppplogchar(sc, c)
-    struct ppp_softc *sc;
-    int c;
-{
-    if (c >= 0)
-	sc->sc_rawin[sc->sc_rawin_count++] = c;
-    if (sc->sc_rawin_count >= sizeof(sc->sc_rawin)
-	|| (c < 0 && sc->sc_rawin_count > 0)) {
-	printf("%s input: %*D", PPP2IFP(sc)->if_xname,
-		sc->sc_rawin_count, sc->sc_rawin, " ");
-	sc->sc_rawin_count = 0;
-    }
-}
diff --git a/sys/net/route.c b/sys/net/route.c
index e06c059..f1e13ad 100644
--- a/sys/net/route.c
+++ b/sys/net/route.c
@@ -106,6 +106,15 @@ static int	rttrash;		/* routes not in table but not freed */
 
 static void rt_maskedcopy(struct sockaddr *,
 	    struct sockaddr *, struct sockaddr *);
+static int vnet_route_iattach(const void *);
+
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_rtable_modinfo = {
+	.vmi_id		= VNET_MOD_RTABLE,
+	.vmi_name	= "rtable",
+	.vmi_iattach	= vnet_route_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
 
 /* compare two sockaddr structures */
 #define	sa_equal(a1, a2) (bcmp((a1), (a2), (a1)->sa_len) == 0)
@@ -122,7 +131,9 @@ static void rt_maskedcopy(struct sockaddr *,
  */
 #define RNTORT(p)	((struct rtentry *)(p))
 
+#ifdef VIMAGE_GLOBALS
 static uma_zone_t rtzone;		/* Routing table UMA zone. */
+#endif
 
 #if 0
 /* default fib for tunnels to use */
@@ -150,20 +161,30 @@ SYSCTL_PROC(_net, OID_AUTO, my_fibnum, CTLTYPE_INT|CTLFLAG_RD,
 static void
 route_init(void)
 {
-	INIT_VNET_INET(curvnet);
-	int table;
-	struct domain *dom;
-	int fam;
 
 	/* whack the tunable ints into  line. */
 	if (rt_numfibs > RT_MAXFIBS)
 		rt_numfibs = RT_MAXFIBS;
 	if (rt_numfibs == 0)
 		rt_numfibs = 1;
-	rtzone = uma_zcreate("rtentry", sizeof(struct rtentry), NULL, NULL,
-	    NULL, NULL, UMA_ALIGN_PTR, 0);
 	rn_init();	/* initialize all zeroes, all ones, mask table */
 
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register(&vnet_rtable_modinfo);
+#else
+	vnet_route_iattach(NULL);
+#endif
+}
+
+static int vnet_route_iattach(const void *unused __unused)
+{
+	INIT_VNET_INET(curvnet);
+	int table;
+	struct domain *dom;
+	int fam;
+
+	V_rtzone = uma_zcreate("rtentry", sizeof(struct rtentry), NULL, NULL,
+	    NULL, NULL, UMA_ALIGN_PTR, 0);
 	for (dom = domains; dom; dom = dom->dom_next) {
 		if (dom->dom_rtattach)  {
 			for  (table = 0; table < rt_numfibs; table++) {
@@ -186,6 +207,8 @@ route_init(void)
 			}
 		}
 	}
+
+	return (0);
 }
 
 #ifndef _SYS_SYSPROTO_H_
@@ -402,7 +425,7 @@ rtfree(struct rtentry *rt)
 		 * and the rtentry itself of course
 		 */
 		RT_LOCK_DESTROY(rt);
-		uma_zfree(rtzone, rt);
+		uma_zfree(V_rtzone, rt);
 		return;
 	}
 done:
@@ -958,7 +981,7 @@ deldone:
 		if (info->rti_ifa == NULL && (error = rt_getifa_fib(info, fibnum)))
 			senderr(error);
 		ifa = info->rti_ifa;
-		rt = uma_zalloc(rtzone, M_NOWAIT | M_ZERO);
+		rt = uma_zalloc(V_rtzone, M_NOWAIT | M_ZERO);
 		if (rt == NULL)
 			senderr(ENOBUFS);
 		RT_LOCK_INIT(rt);
@@ -971,7 +994,7 @@ deldone:
 		RT_LOCK(rt);
 		if ((error = rt_setgate(rt, dst, gateway)) != 0) {
 			RT_LOCK_DESTROY(rt);
-			uma_zfree(rtzone, rt);
+			uma_zfree(V_rtzone, rt);
 			senderr(error);
 		}
 
@@ -1006,7 +1029,7 @@ deldone:
 			}
 			Free(rt_key(rt));
 			RT_LOCK_DESTROY(rt);
-			uma_zfree(rtzone, rt);
+			uma_zfree(V_rtzone, rt);
 			senderr(EEXIST);
 		}
 #endif
@@ -1022,7 +1045,7 @@ deldone:
 				IFAFREE(rt->rt_ifa);
 			Free(rt_key(rt));
 			RT_LOCK_DESTROY(rt);
-			uma_zfree(rtzone, rt);
+			uma_zfree(V_rtzone, rt);
 			senderr(EEXIST);
 		}
 
diff --git a/sys/net/vnet.h b/sys/net/vnet.h
index 9354610..0b3ef8e 100644
--- a/sys/net/vnet.h
+++ b/sys/net/vnet.h
@@ -47,6 +47,7 @@ struct vnet_net {
 	struct	rtstat _rtstat;
 	struct	radix_node_head *_rt_tables[RT_MAXFIBS][AF_MAX+1];
 	int	_rttrash;
+	uma_zone_t _rtzone;
 
 	struct	ifnet *_loif;
 	LIST_HEAD(, lo_softc) _lo_list;
@@ -86,5 +87,6 @@ extern struct vnet_net vnet_net_0;
 #define	V_rt_tables	VNET_NET(rt_tables)
 #define	V_rtstat	VNET_NET(rtstat)
 #define	V_rttrash	VNET_NET(rttrash)
+#define	V_rtzone	VNET_NET(rtzone)
 
 #endif /* !_NET_VNET_H_ */
diff --git a/sys/net80211/ieee80211.c b/sys/net80211/ieee80211.c
index 217f04d..bd3c1ae 100644
--- a/sys/net80211/ieee80211.c
+++ b/sys/net80211/ieee80211.c
@@ -46,6 +46,9 @@ __FBSDID("$FreeBSD$");
 
 #include <net80211/ieee80211_var.h>
 #include <net80211/ieee80211_regdomain.h>
+#ifdef IEEE80211_SUPPORT_SUPERG
+#include <net80211/ieee80211_superg.h>
+#endif
 
 #include <net/bpf.h>
 
@@ -183,6 +186,7 @@ ieee80211_chan_init(struct ieee80211com *ic)
 	ic->ic_csa_newchan = NULL;
 	/* arbitrarily pick the first channel */
 	ic->ic_curchan = &ic->ic_channels[0];
+	ic->ic_rt = ieee80211_get_ratetable(ic->ic_curchan);
 
 	/* fillin well-known rate sets if driver has not specified */
 	DEFAULTRATES(IEEE80211_MODE_11B,	 ieee80211_rateset_11b);
@@ -236,7 +240,8 @@ null_input(struct ifnet *ifp, struct mbuf *m)
  * the driver on attach to prior to creating any vap's.
  */
 void
-ieee80211_ifattach(struct ieee80211com *ic)
+ieee80211_ifattach(struct ieee80211com *ic,
+	const uint8_t macaddr[IEEE80211_ADDR_LEN])
 {
 	struct ifnet *ifp = ic->ic_ifp;
 	struct sockaddr_dl *sdl;
@@ -264,6 +269,9 @@ ieee80211_ifattach(struct ieee80211com *ic)
 	ieee80211_node_attach(ic);
 	ieee80211_power_attach(ic);
 	ieee80211_proto_attach(ic);
+#ifdef IEEE80211_SUPPORT_SUPERG
+	ieee80211_superg_attach(ic);
+#endif
 	ieee80211_ht_attach(ic);
 	ieee80211_scan_attach(ic);
 	ieee80211_regdomain_attach(ic);
@@ -284,7 +292,7 @@ ieee80211_ifattach(struct ieee80211com *ic)
 	sdl = (struct sockaddr_dl *)ifa->ifa_addr;
 	sdl->sdl_type = IFT_ETHER;		/* XXX IFT_IEEE80211? */
 	sdl->sdl_alen = IEEE80211_ADDR_LEN;
-	IEEE80211_ADDR_COPY(LLADDR(sdl), ic->ic_myaddr);
+	IEEE80211_ADDR_COPY(LLADDR(sdl), macaddr);
 }
 
 /*
@@ -306,6 +314,9 @@ ieee80211_ifdetach(struct ieee80211com *ic)
 	ieee80211_sysctl_detach(ic);
 	ieee80211_regdomain_detach(ic);
 	ieee80211_scan_detach(ic);
+#ifdef IEEE80211_SUPPORT_SUPERG
+	ieee80211_superg_detach(ic);
+#endif
 	ieee80211_ht_detach(ic);
 	/* NB: must be called before ieee80211_node_detach */
 	ieee80211_proto_detach(ic);
@@ -415,10 +426,6 @@ ieee80211_vap_setup(struct ieee80211com *ic, struct ieee80211vap *vap,
 		vap->iv_flags |= IEEE80211_F_WME;
 	if (vap->iv_caps & IEEE80211_C_BURST)
 		vap->iv_flags |= IEEE80211_F_BURST;
-	if (vap->iv_caps & IEEE80211_C_FF)
-		vap->iv_flags |= IEEE80211_F_FF;
-	if (vap->iv_caps & IEEE80211_C_TURBOP)
-		vap->iv_flags |= IEEE80211_F_TURBOP;
 	/* NB: bg scanning only makes sense for station mode right now */
 	if (vap->iv_opmode == IEEE80211_M_STA &&
 	    (vap->iv_caps & IEEE80211_C_BGSCAN))
@@ -445,6 +452,9 @@ ieee80211_vap_setup(struct ieee80211com *ic, struct ieee80211vap *vap,
 	ieee80211_node_vattach(vap);
 	ieee80211_power_vattach(vap);
 	ieee80211_proto_vattach(vap);
+#ifdef IEEE80211_SUPPORT_SUPERG
+	ieee80211_superg_vattach(vap);
+#endif
 	ieee80211_ht_vattach(vap);
 	ieee80211_scan_vattach(vap);
 	ieee80211_regdomain_vattach(vap);
@@ -497,7 +507,9 @@ ieee80211_vap_attach(struct ieee80211vap *vap,
 	IEEE80211_LOCK(ic);
 	TAILQ_INSERT_TAIL(&ic->ic_vaps, vap, iv_next);
 	ieee80211_syncflag_locked(ic, IEEE80211_F_WME);
+#ifdef IEEE80211_SUPPORT_SUPERG
 	ieee80211_syncflag_locked(ic, IEEE80211_F_TURBOP);
+#endif
 	ieee80211_syncflag_locked(ic, IEEE80211_F_PCF);
 	ieee80211_syncflag_locked(ic, IEEE80211_F_BURST);
 	ieee80211_syncflag_ext_locked(ic, IEEE80211_FEXT_HT);
@@ -545,7 +557,9 @@ ieee80211_vap_detach(struct ieee80211vap *vap)
 
 	TAILQ_REMOVE(&ic->ic_vaps, vap, iv_next);
 	ieee80211_syncflag_locked(ic, IEEE80211_F_WME);
+#ifdef IEEE80211_SUPPORT_SUPERG
 	ieee80211_syncflag_locked(ic, IEEE80211_F_TURBOP);
+#endif
 	ieee80211_syncflag_locked(ic, IEEE80211_F_PCF);
 	ieee80211_syncflag_locked(ic, IEEE80211_F_BURST);
 	ieee80211_syncflag_ext_locked(ic, IEEE80211_FEXT_HT);
@@ -562,6 +576,9 @@ ieee80211_vap_detach(struct ieee80211vap *vap)
 
 	ieee80211_regdomain_vdetach(vap);
 	ieee80211_scan_vdetach(vap);
+#ifdef IEEE80211_SUPPORT_SUPERG
+	ieee80211_superg_vdetach(vap);
+#endif
 	ieee80211_ht_vdetach(vap);
 	/* NB: must be before ieee80211_node_vdetach */
 	ieee80211_proto_vdetach(vap);
diff --git a/sys/net80211/ieee80211.h b/sys/net80211/ieee80211.h
index fb63507..226fb05 100644
--- a/sys/net80211/ieee80211.h
+++ b/sys/net80211/ieee80211.h
@@ -733,28 +733,6 @@ struct ieee80211_csa_ie {
 	uint8_t		csa_count;		/* Channel Switch Count */
 } __packed;
 
-/*
- * Atheros advanced capability information element.
- */
-struct ieee80211_ath_ie {
-	uint8_t		ath_id;			/* IEEE80211_ELEMID_VENDOR */
-	uint8_t		ath_len;		/* length in bytes */
-	uint8_t		ath_oui[3];		/* 0x00, 0x03, 0x7f */
-	uint8_t		ath_oui_type;		/* OUI type */
-	uint8_t		ath_oui_subtype;	/* OUI subtype */
-	uint8_t		ath_version;		/* spec revision */
-	uint8_t		ath_capability;		/* capability info */
-#define	ATHEROS_CAP_TURBO_PRIME		0x01	/* dynamic turbo--aka Turbo' */
-#define	ATHEROS_CAP_COMPRESSION		0x02	/* data compression */
-#define	ATHEROS_CAP_FAST_FRAME		0x04	/* fast (jumbo) frames */
-#define	ATHEROS_CAP_XR			0x08	/* Xtended Range support */
-#define	ATHEROS_CAP_AR			0x10	/* Advanded Radar support */
-#define	ATHEROS_CAP_BURST		0x20	/* Bursting - not negotiated */
-#define	ATHEROS_CAP_WME			0x40	/* CWMin tuning */
-#define	ATHEROS_CAP_BOOST		0x80	/* use turbo/!turbo mode */
-	uint8_t		ath_defkeyix[2];
-} __packed;
-
 /* rate set entries are in .5 Mb/s units, and potentially marked as basic */
 #define	IEEE80211_RATE_BASIC		0x80
 #define	IEEE80211_RATE_VAL		0x7f
@@ -768,9 +746,11 @@ struct ieee80211_ath_ie {
 	"\20\1NON_ERP_PRESENT\2USE_PROTECTION\3LONG_PREAMBLE"
 
 #define	ATH_OUI			0x7f0300	/* Atheros OUI */
-#define	ATH_OUI_TYPE		0x01
-#define	ATH_OUI_SUBTYPE		0x01
-#define	ATH_OUI_VERSION		0x00
+#define	ATH_OUI_TYPE		0x01		/* Atheros protocol ie */
+
+/* NB: Atheros allocated the OUI for this purpose ~2005 but beware ... */
+#define	TDMA_OUI		ATH_OUI
+#define	TDMA_OUI_TYPE		0x02		/* TDMA protocol ie */
 
 #define	BCM_OUI			0x4c9000	/* Broadcom OUI */
 #define	BCM_OUI_HTCAP		51		/* pre-draft HTCAP ie */
@@ -1048,55 +1028,4 @@ struct ieee80211_duration {
 				 IEEE80211_DUR_DS_SLOW_PLCPHDR + \
 				 IEEE80211_DUR_DIFS)
 
-/*
- * Atheros fast-frame encapsulation format.
- * FF max payload:
- * 802.2 + FFHDR + HPAD + 802.3 + 802.2 + 1500 + SPAD + 802.3 + 802.2 + 1500:
- *   8   +   4   +  4   +   14  +   8   + 1500 +  6   +   14  +   8   + 1500
- * = 3066
- */
-/* fast frame header is 32-bits */
-#define	ATH_FF_PROTO	0x0000003f	/* protocol */
-#define	ATH_FF_PROTO_S	0
-#define	ATH_FF_FTYPE	0x000000c0	/* frame type */
-#define	ATH_FF_FTYPE_S	6
-#define	ATH_FF_HLEN32	0x00000300	/* optional hdr length */
-#define	ATH_FF_HLEN32_S	8
-#define	ATH_FF_SEQNUM	0x001ffc00	/* sequence number */
-#define	ATH_FF_SEQNUM_S	10
-#define	ATH_FF_OFFSET	0xffe00000	/* offset to 2nd payload */
-#define	ATH_FF_OFFSET_S	21
-
-#define	ATH_FF_MAX_HDR_PAD	4
-#define	ATH_FF_MAX_SEP_PAD	6
-#define	ATH_FF_MAX_HDR		30
-
-#define	ATH_FF_PROTO_L2TUNNEL	0	/* L2 tunnel protocol */
-#define	ATH_FF_ETH_TYPE		0x88bd	/* Ether type for encapsulated frames */
-#define	ATH_FF_SNAP_ORGCODE_0	0x00
-#define	ATH_FF_SNAP_ORGCODE_1	0x03
-#define	ATH_FF_SNAP_ORGCODE_2	0x7f
-
-struct ieee80211_tdma_param {
-	u_int8_t	tdma_id;	/* IEEE80211_ELEMID_VENDOR */
-	u_int8_t	tdma_len;
-	u_int8_t	tdma_oui[3];	/* 0x00, 0x03, 0x7f */
-	u_int8_t	tdma_type;	/* OUI type */
-	u_int8_t	tdma_subtype;	/* OUI subtype */
-	u_int8_t	tdma_version;	/* spec revision */
-	u_int8_t	tdma_slot;	/* station slot # */
-	u_int8_t	tdma_slotcnt;	/* bss slot count */
-	u_int16_t	tdma_slotlen;	/* bss slot len (100us) */
-	u_int8_t	tdma_bintval;	/* beacon interval (superframes) */
-	u_int8_t	tdma_inuse[1];	/* slot occupancy map */
-	u_int8_t	tdma_pad[2];
-	u_int8_t	tdma_tstamp[8];	/* timestamp from last beacon */
-} __packed;
-
-/* NB: Atheros allocated the OUI for this purpose ~3 years ago but beware ... */
-#define	TDMA_OUI		ATH_OUI
-#define	TDMA_OUI_TYPE		0x02
-#define	TDMA_SUBTYPE_PARAM	0x01
-#define	TDMA_VERSION		2
-
 #endif /* _NET80211_IEEE80211_H_ */
diff --git a/sys/net80211/ieee80211_adhoc.c b/sys/net80211/ieee80211_adhoc.c
index e72aef1..445b1bb 100644
--- a/sys/net80211/ieee80211_adhoc.c
+++ b/sys/net80211/ieee80211_adhoc.c
@@ -57,6 +57,9 @@ __FBSDID("$FreeBSD$");
 #include <net80211/ieee80211_var.h>
 #include <net80211/ieee80211_adhoc.h>
 #include <net80211/ieee80211_input.h>
+#ifdef IEEE80211_SUPPORT_SUPERG
+#include <net80211/ieee80211_superg.h>
+#endif
 #ifdef IEEE80211_SUPPORT_TDMA
 #include <net80211/ieee80211_tdma.h>
 #endif
@@ -590,32 +593,13 @@ adhoc_input(struct ieee80211_node *ni, struct mbuf *m,
 			m = ieee80211_decap_amsdu(ni, m);
 			if (m == NULL)
 				return IEEE80211_FC0_TYPE_DATA;
-		} else if (IEEE80211_ATH_CAP(vap, ni, IEEE80211_NODE_FF) &&
-#define	FF_LLC_SIZE	(sizeof(struct ether_header) + sizeof(struct llc))
-		    m->m_pkthdr.len >= 3*FF_LLC_SIZE) {
-			struct llc *llc;
-
-			/*
-			 * Check for fast-frame tunnel encapsulation.
-			 */
-			if (m->m_len < FF_LLC_SIZE &&
-			    (m = m_pullup(m, FF_LLC_SIZE)) == NULL) {
-				IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
-				    ni->ni_macaddr, "fast-frame",
-				    "%s", "m_pullup(llc) failed");
-				vap->iv_stats.is_rx_tooshort++;
+		} else {
+#ifdef IEEE80211_SUPPORT_SUPERG
+			m = ieee80211_decap_fastframe(vap, ni, m);
+			if (m == NULL)
 				return IEEE80211_FC0_TYPE_DATA;
-			}
-			llc = (struct llc *)(mtod(m, uint8_t *) + 
-				sizeof(struct ether_header));
-			if (llc->llc_snap.ether_type == htons(ATH_FF_ETH_TYPE)) {
-				m_adj(m, FF_LLC_SIZE);
-				m = ieee80211_decap_fastframe(ni, m);
-				if (m == NULL)
-					return IEEE80211_FC0_TYPE_DATA;
-			}
+#endif
 		}
-#undef FF_LLC_SIZE
 		if (dir == IEEE80211_FC1_DIR_DSTODS && ni->ni_wdsvap != NULL)
 			ieee80211_deliver_data(ni->ni_wdsvap, ni, m);
 		else
diff --git a/sys/net80211/ieee80211_ddb.c b/sys/net80211/ieee80211_ddb.c
index 09ea89c..0dd27b1 100644
--- a/sys/net80211/ieee80211_ddb.c
+++ b/sys/net80211/ieee80211_ddb.c
@@ -445,7 +445,6 @@ _db_show_com(const struct ieee80211com *ic, int showvaps, int showsta, int showp
 	db_printf(" opmode %s", ieee80211_opmode_name[ic->ic_opmode]);
 	db_printf("\n");
 	db_printf("\tmedia %p", &ic->ic_media);
-	db_printf(" myaddr %s", ether_sprintf(ic->ic_myaddr));
 	db_printf(" inact %p", &ic->ic_inact);
 	db_printf("\n");
 
diff --git a/sys/net80211/ieee80211_freebsd.c b/sys/net80211/ieee80211_freebsd.c
index 5f44f1c..8d38614 100644
--- a/sys/net80211/ieee80211_freebsd.c
+++ b/sys/net80211/ieee80211_freebsd.c
@@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/socket.h>
 
 #include <net/if.h>
+#include <net/if_dl.h>
 #include <net/if_clone.h>
 #include <net/if_media.h>
 #include <net/if_types.h>
@@ -137,7 +138,7 @@ wlan_clone_create(struct if_clone *ifc, int unit, caddr_t params)
 	vap = ic->ic_vap_create(ic, ifc->ifc_name, unit,
 			cp.icp_opmode, cp.icp_flags, cp.icp_bssid,
 			cp.icp_flags & IEEE80211_CLONE_MACADDR ?
-			    cp.icp_macaddr : ic->ic_myaddr);
+			    cp.icp_macaddr : (const uint8_t *)IF_LLADDR(ifp));
 	return (vap == NULL ? EIO : 0);
 }
 
@@ -188,6 +189,15 @@ SYSCTL_PROC(_net_wlan, OID_AUTO, addba_backoff, CTLFLAG_RW,
 extern int ieee80211_addba_maxtries;
 SYSCTL_INT(_net_wlan, OID_AUTO, addba_maxtries, CTLFLAG_RW,
 	&ieee80211_addba_maxtries, 0, "max ADDBA requests sent before backoff");
+#ifdef IEEE80211_SUPPORT_SUPERG
+extern int ieee80211_ffppsmin;
+SYSCTL_INT(_net_wlan, OID_AUTO, ffppsmin, CTLFLAG_RW,
+	&ieee80211_ffppsmin, 0, "min packet rate before fast-frame staging");
+extern int ieee80211_ffagemax;
+SYSCTL_PROC(_net_wlan, OID_AUTO, ffagemax, CTLFLAG_RW,
+	&ieee80211_ffagemax, 0, ieee80211_sysctl_msecs_ticks, "I",
+	"max hold time for fast-frame staging (ms)");
+#endif /* IEEE80211_SUPPORT_SUPERG */
 
 static int
 ieee80211_sysctl_inact(SYSCTL_HANDLER_ARGS)
diff --git a/sys/net80211/ieee80211_freebsd.h b/sys/net80211/ieee80211_freebsd.h
index 519e37d..db6648d 100644
--- a/sys/net80211/ieee80211_freebsd.h
+++ b/sys/net80211/ieee80211_freebsd.h
@@ -211,7 +211,6 @@ struct mbuf *ieee80211_getmgtframe(uint8_t **frm, int headroom, int pktlen);
 
 /* tx path usage */
 #define	M_ENCAP		M_PROTO1		/* 802.11 encap done */
-#define	M_WDS		M_PROTO2		/* WDS frame */
 #define	M_EAPOL		M_PROTO3		/* PAE/EAPOL frame */
 #define	M_PWR_SAV	M_PROTO4		/* bypass PS handling */
 #define	M_MORE_DATA	M_PROTO5		/* more data frames to follow */
@@ -219,7 +218,7 @@ struct mbuf *ieee80211_getmgtframe(uint8_t **frm, int headroom, int pktlen);
 #define	M_TXCB		M_PROTO7		/* do tx complete callback */
 #define	M_AMPDU_MPDU	M_PROTO8		/* ok for A-MPDU aggregation */
 #define	M_80211_TX \
-	(M_FRAG|M_FIRSTFRAG|M_LASTFRAG|M_ENCAP|M_WDS|M_EAPOL|M_PWR_SAV|\
+	(M_FRAG|M_FIRSTFRAG|M_LASTFRAG|M_ENCAP|M_EAPOL|M_PWR_SAV|\
 	 M_MORE_DATA|M_FF|M_TXCB|M_AMPDU_MPDU)
 
 /* rx path usage */
@@ -391,6 +390,17 @@ alg##_modevent(int type)						\
 	/* XXX nothing to do until the rate control framework arrives */\
 }									\
 TEXT_SET(rate##_set, alg##_modevent)
+
+struct ieee80211req;
+typedef int ieee80211_ioctl_getfunc(struct ieee80211vap *,
+    struct ieee80211req *);
+SET_DECLARE(ieee80211_ioctl_getset, ieee80211_ioctl_getfunc);
+#define	IEEE80211_IOCTL_GET(_name, _get) TEXT_SET(ieee80211_ioctl_getset, _get)
+
+typedef int ieee80211_ioctl_setfunc(struct ieee80211vap *,
+    struct ieee80211req *);
+SET_DECLARE(ieee80211_ioctl_setset, ieee80211_ioctl_setfunc);
+#define	IEEE80211_IOCTL_SET(_name, _set) TEXT_SET(ieee80211_ioctl_setset, _set)
 #endif /* _KERNEL */
 
 /* XXX this stuff belongs elsewhere */
diff --git a/sys/net80211/ieee80211_hostap.c b/sys/net80211/ieee80211_hostap.c
index e0f831b..5b16a8d 100644
--- a/sys/net80211/ieee80211_hostap.c
+++ b/sys/net80211/ieee80211_hostap.c
@@ -57,6 +57,9 @@ __FBSDID("$FreeBSD$");
 #include <net80211/ieee80211_var.h>
 #include <net80211/ieee80211_hostap.h>
 #include <net80211/ieee80211_input.h>
+#ifdef IEEE80211_SUPPORT_SUPERG
+#include <net80211/ieee80211_superg.h>
+#endif
 #include <net80211/ieee80211_wds.h>
 
 #define	IEEE80211_RATE2MBS(r)	(((r) & IEEE80211_RATE_VAL) / 2)
@@ -752,32 +755,13 @@ hostap_input(struct ieee80211_node *ni, struct mbuf *m,
 			m = ieee80211_decap_amsdu(ni, m);
 			if (m == NULL)
 				return IEEE80211_FC0_TYPE_DATA;
-		} else if (IEEE80211_ATH_CAP(vap, ni, IEEE80211_NODE_FF) &&
-#define	FF_LLC_SIZE	(sizeof(struct ether_header) + sizeof(struct llc))
-		    m->m_pkthdr.len >= 3*FF_LLC_SIZE) {
-			struct llc *llc;
-
-			/*
-			 * Check for fast-frame tunnel encapsulation.
-			 */
-			if (m->m_len < FF_LLC_SIZE &&
-			    (m = m_pullup(m, FF_LLC_SIZE)) == NULL) {
-				IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
-				    ni->ni_macaddr, "fast-frame",
-				    "%s", "m_pullup(llc) failed");
-				vap->iv_stats.is_rx_tooshort++;
+		} else {
+#ifdef IEEE80211_SUPPORT_SUPERG
+			m = ieee80211_decap_fastframe(vap, ni, m);
+			if (m == NULL)
 				return IEEE80211_FC0_TYPE_DATA;
-			}
-			llc = (struct llc *)(mtod(m, uint8_t *) + 
-				sizeof(struct ether_header));
-			if (llc->llc_snap.ether_type == htons(ATH_FF_ETH_TYPE)) {
-				m_adj(m, FF_LLC_SIZE);
-				m = ieee80211_decap_fastframe(ni, m);
-				if (m == NULL)
-					return IEEE80211_FC0_TYPE_DATA;
-			}
+#endif
 		}
-#undef FF_LLC_SIZE
 		if (dir == IEEE80211_FC1_DIR_DSTODS && ni->ni_wdsvap != NULL)
 			ieee80211_deliver_data(ni->ni_wdsvap, ni, m);
 		else
@@ -1967,8 +1951,10 @@ hostap_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0,
 					wpa = frm;
 				else if (iswmeinfo(frm))
 					wme = frm;
+#ifdef IEEE80211_SUPPORT_SUPERG
 				else if (isatherosoui(frm))
 					ath = frm;
+#endif
 				else if (vap->iv_flags_ext & IEEE80211_FEXT_HTCOMPAT) {
 					if (ishtcapoui(frm) && htcap == NULL)
 						htcap = frm;
@@ -2052,6 +2038,10 @@ hostap_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0,
 			ieee80211_ht_updatehtcap(ni, htcap);
 		} else if (ni->ni_flags & IEEE80211_NODE_HT)
 			ieee80211_ht_node_cleanup(ni);
+#ifdef IEEE80211_SUPPORT_SUPERG
+		else if (ni->ni_ath_flags & IEEE80211_NODE_ATH)
+			ieee80211_ff_node_cleanup(ni);
+#endif
 		/*
 		 * Allow AMPDU operation only with unencrypted traffic
 		 * or AES-CCM; the 11n spec only specifies these ciphers
@@ -2106,6 +2096,7 @@ hostap_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0,
 				ni->ni_flags |= IEEE80211_NODE_QOS;
 			} else
 				ni->ni_flags &= ~IEEE80211_NODE_QOS;
+#ifdef IEEE80211_SUPPORT_SUPERG
 			if (ath != NULL) {
 				setie(ath_ie, ath - sfrm);
 				/* 
@@ -2113,6 +2104,7 @@ hostap_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0,
 				 */
 				ieee80211_parse_ath(ni, ni->ni_ies.ath_ie);
 			} else
+#endif
 				ni->ni_ath_flags = 0;
 #undef setie
 		} else {
diff --git a/sys/net80211/ieee80211_input.c b/sys/net80211/ieee80211_input.c
index 23f770b..2134e29d 100644
--- a/sys/net80211/ieee80211_input.c
+++ b/sys/net80211/ieee80211_input.c
@@ -98,7 +98,7 @@ ieee80211_input_all(struct ieee80211com *ic,
 }
 
 /*
- * This function reassemble fragments.
+ * This function reassembles fragments.
  *
  * XXX should handle 3 concurrent reassemblies per-spec.
  */
@@ -358,73 +358,6 @@ ieee80211_decap1(struct mbuf *m, int *framelen)
 }
 
 /*
- * Decap the encapsulated frame pair and dispatch the first
- * for delivery.  The second frame is returned for delivery
- * via the normal path.
- */
-struct mbuf *
-ieee80211_decap_fastframe(struct ieee80211_node *ni, struct mbuf *m)
-{
-#define	MS(x,f)	(((x) & f) >> f##_S)
-	struct ieee80211vap *vap = ni->ni_vap;
-	uint32_t ath;
-	struct mbuf *n;
-	int framelen;
-
-	m_copydata(m, 0, sizeof(uint32_t), (caddr_t) &ath);
-	if (MS(ath, ATH_FF_PROTO) != ATH_FF_PROTO_L2TUNNEL) {
-		IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
-		    ni->ni_macaddr, "fast-frame",
-		    "unsupport tunnel protocol, header 0x%x", ath);
-		vap->iv_stats.is_ff_badhdr++;
-		m_freem(m);
-		return NULL;
-	}
-	/* NB: skip header and alignment padding */
-	m_adj(m, roundup(sizeof(uint32_t) - 2, 4) + 2);
-
-	vap->iv_stats.is_ff_decap++;
-
-	/*
-	 * Decap the first frame, bust it apart from the
-	 * second and deliver; then decap the second frame
-	 * and return it to the caller for normal delivery.
-	 */
-	m = ieee80211_decap1(m, &framelen);
-	if (m == NULL) {
-		IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
-		    ni->ni_macaddr, "fast-frame", "%s", "first decap failed");
-		vap->iv_stats.is_ff_tooshort++;
-		return NULL;
-	}
-	n = m_split(m, framelen, M_NOWAIT);
-	if (n == NULL) {
-		IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
-		    ni->ni_macaddr, "fast-frame",
-		    "%s", "unable to split encapsulated frames");
-		vap->iv_stats.is_ff_split++;
-		m_freem(m);			/* NB: must reclaim */
-		return NULL;
-	}
-	/* XXX not right for WDS */
-	vap->iv_deliver_data(vap, ni, m);	/* 1st of pair */
-
-	/*
-	 * Decap second frame.
-	 */
-	m_adj(n, roundup2(framelen, 4) - framelen);	/* padding */
-	n = ieee80211_decap1(n, &framelen);
-	if (n == NULL) {
-		IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
-		    ni->ni_macaddr, "fast-frame", "%s", "second decap failed");
-		vap->iv_stats.is_ff_tooshort++;
-	}
-	/* XXX verify framelen against mbuf contents */
-	return n;				/* 2nd delivered by caller */
-#undef MS
-}
-
-/*
  * Install received rate set information in the node's state block.
  */
 int
@@ -510,16 +443,6 @@ ieee80211_alloc_challenge(struct ieee80211_node *ni)
 	return (ni->ni_challenge != NULL);
 }
 
-void
-ieee80211_parse_ath(struct ieee80211_node *ni, uint8_t *ie)
-{
-	const struct ieee80211_ath_ie *ath =
-		(const struct ieee80211_ath_ie *) ie;
-
-	ni->ni_ath_flags = ath->ath_capability;
-	ni->ni_ath_defkeyix = LE_READ_2(&ath->ath_defkeyix);
-}
-
 /*
  * Parse a Beacon or ProbeResponse frame and return the
  * useful information in an ieee80211_scanparams structure.
@@ -633,8 +556,10 @@ ieee80211_parse_beacon(struct ieee80211_node *ni, struct mbuf *m,
 				scan->wpa = frm;
 			else if (iswmeparam(frm) || iswmeinfo(frm))
 				scan->wme = frm;
+#ifdef IEEE80211_SUPPORT_SUPERG
 			else if (isatherosoui(frm))
 				scan->ath = frm;
+#endif
 #ifdef IEEE80211_SUPPORT_TDMA
 			else if (istdmaoui(frm))
 				scan->tdma = frm;
diff --git a/sys/net80211/ieee80211_input.h b/sys/net80211/ieee80211_input.h
index b656c55..f19b934 100644
--- a/sys/net80211/ieee80211_input.h
+++ b/sys/net80211/ieee80211_input.h
@@ -148,14 +148,11 @@ struct mbuf *ieee80211_defrag(struct ieee80211_node *,
 		struct mbuf *, int);
 struct mbuf *ieee80211_decap(struct ieee80211vap *, struct mbuf *, int);
 struct mbuf *ieee80211_decap1(struct mbuf *, int *);
-struct mbuf *ieee80211_decap_fastframe(struct ieee80211_node *,
-		struct mbuf *);
 int	ieee80211_setup_rates(struct ieee80211_node *ni,
 		const uint8_t *rates, const uint8_t *xrates, int flags);
 void ieee80211_send_error(struct ieee80211_node *,
 		const uint8_t mac[IEEE80211_ADDR_LEN], int subtype, int arg);
 int	ieee80211_alloc_challenge(struct ieee80211_node *);
-void	ieee80211_parse_ath(struct ieee80211_node *, uint8_t *);
 int	ieee80211_parse_beacon(struct ieee80211_node *, struct mbuf *,
 		struct ieee80211_scanparams *);
 int	ieee80211_parse_action(struct ieee80211_node *, struct mbuf *);
diff --git a/sys/net80211/ieee80211_ioctl.c b/sys/net80211/ieee80211_ioctl.c
index 21d5637..c148214 100644
--- a/sys/net80211/ieee80211_ioctl.c
+++ b/sys/net80211/ieee80211_ioctl.c
@@ -63,9 +63,6 @@ __FBSDID("$FreeBSD$");
 #include <net80211/ieee80211_ioctl.h>
 #include <net80211/ieee80211_regdomain.h>
 #include <net80211/ieee80211_input.h>
-#ifdef IEEE80211_SUPPORT_TDMA
-#include <net80211/ieee80211_tdma.h>
-#endif
 
 #define	IS_UP_AUTO(_vap) \
 	(IFNET_IS_UP_RUNNING(vap->iv_ifp) && \
@@ -587,21 +584,6 @@ ieee80211_ioctl_getmaccmd(struct ieee80211vap *vap, struct ieee80211req *ireq)
 	return (acl == NULL ? EINVAL : acl->iac_getioctl(vap, ireq));
 }
 
-/*
- * Return the current ``state'' of an Atheros capbility.
- * If associated in station mode report the negotiated
- * setting. Otherwise report the current setting.
- */
-static int
-getathcap(struct ieee80211vap *vap, int cap)
-{
-	if (vap->iv_opmode == IEEE80211_M_STA &&
-	    vap->iv_state == IEEE80211_S_RUN)
-		return IEEE80211_ATH_CAP(vap, vap->iv_bss, cap) != 0;
-	else
-		return (vap->iv_flags & cap) != 0;
-}
-
 static __noinline int
 ieee80211_ioctl_getcurchan(struct ieee80211vap *vap, struct ieee80211req *ireq)
 {
@@ -752,6 +734,30 @@ ieee80211_ioctl_getstavlan(struct ieee80211vap *vap, struct ieee80211req *ireq)
 }
 
 /*
+ * Dummy ioctl get handler so the linker set is defined.
+ */
+static int
+dummy_ioctl_get(struct ieee80211vap *vap, struct ieee80211req *ireq)
+{
+	return ENOSYS;
+}
+IEEE80211_IOCTL_GET(dummy, dummy_ioctl_get);
+
+static int
+ieee80211_ioctl_getdefault(struct ieee80211vap *vap, struct ieee80211req *ireq)
+{
+	ieee80211_ioctl_getfunc * const *get;
+	int error;
+
+	SET_FOREACH(get, ieee80211_ioctl_getset) {
+		error = (*get)(vap, ireq);
+		if (error != ENOSYS)
+			return error;
+	}
+	return EINVAL;
+}
+
+/*
  * When building the kernel with -O2 on the i386 architecture, gcc
  * seems to want to inline this function into ieee80211_ioctl()
  * (which is the only routine that calls it). When this happens,
@@ -953,12 +959,6 @@ ieee80211_ioctl_get80211(struct ieee80211vap *vap, u_long cmd,
 	case IEEE80211_IOC_PUREG:
 		ireq->i_val = (vap->iv_flags & IEEE80211_F_PUREG) != 0;
 		break;
-	case IEEE80211_IOC_FF:
-		ireq->i_val = getathcap(vap, IEEE80211_F_FF);
-		break;
-	case IEEE80211_IOC_TURBOP:
-		ireq->i_val = getathcap(vap, IEEE80211_F_TURBOP);
-		break;
 	case IEEE80211_IOC_BGSCAN:
 		ireq->i_val = (vap->iv_flags & IEEE80211_F_BGSCAN) != 0;
 		break;
@@ -1104,16 +1104,8 @@ ieee80211_ioctl_get80211(struct ieee80211vap *vap, u_long cmd,
 			ireq->i_val =
 			    (vap->iv_flags_ext & IEEE80211_FEXT_RIFS) != 0;
 		break;
-#ifdef IEEE80211_SUPPORT_TDMA
-	case IEEE80211_IOC_TDMA_SLOT:
-	case IEEE80211_IOC_TDMA_SLOTCNT:
-	case IEEE80211_IOC_TDMA_SLOTLEN:
-	case IEEE80211_IOC_TDMA_BINTERVAL:
-		error = ieee80211_tdma_ioctl_get80211(vap, ireq);
-		break;
-#endif
 	default:
-		error = EINVAL;
+		error = ieee80211_ioctl_getdefault(vap, ireq);
 		break;
 	}
 	return error;
@@ -1879,6 +1871,7 @@ setcurchan(struct ieee80211vap *vap, struct ieee80211_channel *c)
 			vap->iv_bss->ni_chan = ic->ic_curchan;
 		} else
 			ic->ic_curchan = vap->iv_des_chan;
+			ic->ic_rt = ieee80211_get_ratetable(ic->ic_curchan);
 	} else {
 		/*
 		 * Need to go through the state machine in case we
@@ -1894,6 +1887,7 @@ setcurchan(struct ieee80211vap *vap, struct ieee80211_channel *c)
 			 * there is immediate feedback; e.g. via ifconfig.
 			 */
 			ic->ic_curchan = vap->iv_des_chan;
+			ic->ic_rt = ieee80211_get_ratetable(ic->ic_curchan);
 		}
 	}
 	return error;
@@ -2476,6 +2470,30 @@ isvapht(const struct ieee80211vap *vap)
 	    IEEE80211_IS_CHAN_HT(bss->ni_chan);
 }
 
+/*
+ * Dummy ioctl set handler so the linker set is defined.
+ */
+static int
+dummy_ioctl_set(struct ieee80211vap *vap, struct ieee80211req *ireq)
+{
+	return ENOSYS;
+}
+IEEE80211_IOCTL_SET(dummy, dummy_ioctl_set);
+
+static int
+ieee80211_ioctl_setdefault(struct ieee80211vap *vap, struct ieee80211req *ireq)
+{
+	ieee80211_ioctl_setfunc * const *set;
+	int error;
+
+	SET_FOREACH(set, ieee80211_ioctl_setset) {
+		error = (*set)(vap, ireq);
+		if (error != ENOSYS)
+			return error;
+	}
+	return EINVAL;
+}
+
 static __noinline int
 ieee80211_ioctl_set80211(struct ieee80211vap *vap, u_long cmd, struct ieee80211req *ireq)
 {
@@ -2852,24 +2870,6 @@ ieee80211_ioctl_set80211(struct ieee80211vap *vap, u_long cmd, struct ieee80211r
 		if (isvap11g(vap))
 			error = ENETRESET;
 		break;
-	case IEEE80211_IOC_FF:
-		if (ireq->i_val) {
-			if ((vap->iv_caps & IEEE80211_C_FF) == 0)
-				return EOPNOTSUPP;
-			vap->iv_flags |= IEEE80211_F_FF;
-		} else
-			vap->iv_flags &= ~IEEE80211_F_FF;
-		error = ERESTART;
-		break;
-	case IEEE80211_IOC_TURBOP:
-		if (ireq->i_val) {
-			if ((vap->iv_caps & IEEE80211_C_TURBOP) == 0)
-				return EOPNOTSUPP;
-			vap->iv_flags |= IEEE80211_F_TURBOP;
-		} else
-			vap->iv_flags &= ~IEEE80211_F_TURBOP;
-		error = ENETRESET;
-		break;
 	case IEEE80211_IOC_BGSCAN:
 		if (ireq->i_val) {
 			if ((vap->iv_caps & IEEE80211_C_BGSCAN) == 0)
@@ -3131,16 +3131,8 @@ ieee80211_ioctl_set80211(struct ieee80211vap *vap, u_long cmd, struct ieee80211r
 		if (isvapht(vap))
 			error = ERESTART;
 		break;
-#ifdef IEEE80211_SUPPORT_TDMA
-	case IEEE80211_IOC_TDMA_SLOT:
-	case IEEE80211_IOC_TDMA_SLOTCNT:
-	case IEEE80211_IOC_TDMA_SLOTLEN:
-	case IEEE80211_IOC_TDMA_BINTERVAL:
-		error = ieee80211_tdma_ioctl_set80211(vap, ireq);
-		break;
-#endif
 	default:
-		error = EINVAL;
+		error = ieee80211_ioctl_setdefault(vap, ireq);
 		break;
 	}
 	/*
diff --git a/sys/net80211/ieee80211_ioctl.h b/sys/net80211/ieee80211_ioctl.h
index 367f505..6a81375 100644
--- a/sys/net80211/ieee80211_ioctl.h
+++ b/sys/net80211/ieee80211_ioctl.h
@@ -217,7 +217,8 @@ struct ieee80211_stats {
 	uint8_t		is_rx_authfail_code;	/* last rx'd auth fail reason */
 	uint32_t	is_beacon_miss;		/* beacon miss notification */
 	uint32_t	is_rx_badstate;		/* rx discard state != RUN */
-	uint32_t	is_spare[12];
+	uint32_t	is_ff_flush;		/* ff's flush'd from stageq */
+	uint32_t	is_spare[11];
 };
 
 /*
diff --git a/sys/net80211/ieee80211_node.c b/sys/net80211/ieee80211_node.c
index 761c969..1632b19 100644
--- a/sys/net80211/ieee80211_node.c
+++ b/sys/net80211/ieee80211_node.c
@@ -43,6 +43,9 @@ __FBSDID("$FreeBSD$");
 
 #include <net80211/ieee80211_var.h>
 #include <net80211/ieee80211_input.h>
+#ifdef IEEE80211_SUPPORT_SUPERG
+#include <net80211/ieee80211_superg.h>
+#endif
 #ifdef IEEE80211_SUPPORT_TDMA
 #include <net80211/ieee80211_tdma.h>
 #endif
@@ -624,6 +627,7 @@ ieee80211_sync_curchan(struct ieee80211com *ic)
 	if (c != ic->ic_curchan) {
 		ic->ic_curchan = c;
 		ic->ic_curmode = ieee80211_chan2mode(ic->ic_curchan);
+		ic->ic_rt = ieee80211_get_ratetable(ic->ic_curchan);
 		ic->ic_set_channel(ic);
 	}
 }
@@ -648,6 +652,7 @@ ieee80211_setcurchan(struct ieee80211com *ic, struct ieee80211_channel *c)
 	}
 	ic->ic_bsschan = ic->ic_curchan = c;
 	ic->ic_curmode = ieee80211_chan2mode(ic->ic_curchan);
+	ic->ic_rt = ieee80211_get_ratetable(ic->ic_curchan);
 	ic->ic_set_channel(ic);
 }
 
@@ -754,8 +759,10 @@ ieee80211_sta_join(struct ieee80211vap *vap, struct ieee80211_channel *chan,
 
 	if (ieee80211_ies_init(&ni->ni_ies, se->se_ies.data, se->se_ies.len)) {
 		ieee80211_ies_expand(&ni->ni_ies);
+#ifdef IEEE80211_SUPPORT_SUPERG
 		if (ni->ni_ies.ath_ie != NULL)
 			ieee80211_parse_ath(ni, ni->ni_ies.ath_ie);
+#endif
 		if (ni->ni_ies.htcap_ie != NULL)
 			ieee80211_parse_htcap(ni, ni->ni_ies.htcap_ie);
 		if (ni->ni_ies.htinfo_ie != NULL)
@@ -875,8 +882,10 @@ ieee80211_ies_expand(struct ieee80211_ies *ies)
 				ies->wpa_ie = ie;
 			else if (iswmeoui(ie))
 				ies->wme_ie = ie;
+#ifdef IEEE80211_SUPPORT_SUPERG
 			else if (isatherosoui(ie))
 				ies->ath_ie = ie;
+#endif
 #ifdef IEEE80211_SUPPORT_TDMA
 			else if (istdmaoui(ie))
 				ies->tdma_ie = ie;
@@ -920,6 +929,10 @@ node_cleanup(struct ieee80211_node *ni)
 	 */
 	if (ni->ni_flags & IEEE80211_NODE_HT)
 		ieee80211_ht_node_cleanup(ni);
+#ifdef IEEE80211_SUPPORT_SUPERG
+	else if (ni->ni_ath_flags & IEEE80211_NODE_ATH)
+		ieee80211_ff_node_cleanup(ni);
+#endif
 	/*
 	 * Clear AREF flag that marks the authorization refcnt bump
 	 * has happened.  This is probably not needed as the node
@@ -1173,8 +1186,10 @@ ieee80211_node_create_wds(struct ieee80211vap *vap,
 		 */
 		if (vap->iv_flags & IEEE80211_F_WME)
 			ni->ni_flags |= IEEE80211_NODE_QOS;
+#ifdef IEEE80211_SUPPORT_SUPERG
 		if (vap->iv_flags & IEEE80211_F_FF)
 			ni->ni_flags |= IEEE80211_NODE_FF;
+#endif
 		if ((ic->ic_htcaps & IEEE80211_HTC_HT) &&
 		    (vap->iv_flags_ext & IEEE80211_FEXT_HT)) {
 			/*
@@ -1331,8 +1346,10 @@ ieee80211_fakeup_adhoc_node(struct ieee80211vap *vap,
 			 */
 			if (vap->iv_flags & IEEE80211_F_WME)
 				ni->ni_flags |= IEEE80211_NODE_QOS;
+#ifdef IEEE80211_SUPPORT_SUPERG
 			if (vap->iv_flags & IEEE80211_F_FF)
 				ni->ni_flags |= IEEE80211_NODE_FF;
+#endif
 		}
 		node_setuptxparms(ni);
 		if (ic->ic_newassoc != NULL)
@@ -1366,8 +1383,10 @@ ieee80211_init_neighbor(struct ieee80211_node *ni,
 			ni->ni_flags |= IEEE80211_NODE_QOS;
 		else
 			ni->ni_flags &= ~IEEE80211_NODE_QOS;
+#ifdef IEEE80211_SUPPORT_SUPERG
 		if (ni->ni_ies.ath_ie != NULL)
 			ieee80211_parse_ath(ni, ni->ni_ies.ath_ie);
+#endif
 	}
 
 	/* NB: must be after ni_chan is setup */
diff --git a/sys/net80211/ieee80211_node.h b/sys/net80211/ieee80211_node.h
index 589ad52..9e9d254 100644
--- a/sys/net80211/ieee80211_node.h
+++ b/sys/net80211/ieee80211_node.h
@@ -69,7 +69,7 @@ struct ieee80211vap;
 /*
  * Information element ``blob''.  We use this structure
  * to capture management frame payloads that need to be
- * retained.  Information elemnts within the payload that
+ * retained.  Information elements within the payload that
  * we need to consult have references recorded.
  */
 struct ieee80211_ies {
diff --git a/sys/net80211/ieee80211_output.c b/sys/net80211/ieee80211_output.c
index f471c5c..21e0370 100644
--- a/sys/net80211/ieee80211_output.c
+++ b/sys/net80211/ieee80211_output.c
@@ -47,6 +47,9 @@ __FBSDID("$FreeBSD$");
 
 #include <net80211/ieee80211_var.h>
 #include <net80211/ieee80211_regdomain.h>
+#ifdef IEEE80211_SUPPORT_SUPERG
+#include <net80211/ieee80211_superg.h>
+#endif
 #ifdef IEEE80211_SUPPORT_TDMA
 #include <net80211/ieee80211_tdma.h>
 #endif
@@ -62,9 +65,6 @@ __FBSDID("$FreeBSD$");
 #define	ETHER_HEADER_COPY(dst, src) \
 	memcpy(dst, src, sizeof(struct ether_header))
 
-static struct mbuf *ieee80211_encap_fastframe(struct ieee80211vap *,
-	struct mbuf *m1, const struct ether_header *eh1,
-	struct mbuf *m2, const struct ether_header *eh2);
 static int ieee80211_fragment(struct ieee80211vap *, struct mbuf *,
 	u_int hdrsize, u_int ciphdrsize, u_int mtu);
 static	void ieee80211_tx_mgt_cb(struct ieee80211_node *, void *, int);
@@ -205,7 +205,6 @@ ieee80211_start(struct ifnet *ifp)
 			m_freem(m);
 			continue;
 		}
-		/* XXX AUTH'd */
 		if (ni->ni_associd == 0 &&
 		    (ni->ni_flags & IEEE80211_NODE_ASSOCID)) {
 			IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_OUTPUT,
@@ -218,6 +217,7 @@ ieee80211_start(struct ifnet *ifp)
 			ieee80211_free_node(ni);
 			continue;
 		}
+
 		if ((ni->ni_flags & IEEE80211_NODE_PWR_MGT) &&
 		    (m->m_flags & M_PWR_SAV) == 0) {
 			/*
@@ -242,23 +242,28 @@ ieee80211_start(struct ifnet *ifp)
 			continue;
 		}
 
-		BPF_MTAP(ifp, m);		/* 802.11 tx path */
-
-		/*
-		 * XXX When ni is associated with a WDS link then
-		 * the vap will be the WDS vap but ni_vap will point
-		 * to the ap vap the station associated to.  Once
-		 * we handoff the packet to the driver the callback
-		 * to ieee80211_encap won't be able to tell if the
-		 * packet should be encapsulated for WDS or not (e.g.
-		 * multicast frames will not be handled correctly).
-		 * We hack this by marking the mbuf so ieee80211_encap
-		 * can do the right thing.
-		 */
-		if (vap->iv_opmode == IEEE80211_M_WDS)
-			m->m_flags |= M_WDS;
-		else
-			m->m_flags &= ~M_WDS;
+		BPF_MTAP(ifp, m);		/* 802.3 tx */
+ 
+#ifdef IEEE80211_SUPPORT_SUPERG
+		if (IEEE80211_ATH_CAP(vap, ni, IEEE80211_NODE_FF)) {
+			m = ieee80211_ff_check(ni, m);
+			if (m == NULL) {
+				/* NB: any ni ref held on stageq */
+				continue;
+			}
+		}
+#endif /* IEEE80211_SUPPORT_SUPERG */
+		if (__predict_true((vap->iv_caps & IEEE80211_C_8023ENCAP) == 0)) {
+			/*
+			 * Encapsulate the packet in prep for transmission.
+			 */
+			m = ieee80211_encap(vap, ni, m);
+			if (m == NULL) {
+				/* NB: stat+msg handled in ieee80211_encap */
+				ieee80211_free_node(ni);
+				continue;
+			}
+		}
 
 		/*
 		 * Stash the node pointer and hand the frame off to
@@ -268,7 +273,6 @@ ieee80211_start(struct ifnet *ifp)
 		 */
 		m->m_pkthdr.rcvif = (void *)ni;
 
-		/* XXX defer if_start calls? */
 		error = parent->if_transmit(parent, m);
 		if (error != 0) {
 			/* NB: IFQ_HANDOFF reclaims mbuf */
@@ -732,7 +736,7 @@ done:
  * Drivers and cipher modules assume we have done the necessary work
  * and fail rudely if they don't find the space they need.
  */
-static struct mbuf *
+struct mbuf *
 ieee80211_mbuf_adjust(struct ieee80211vap *vap, int hdrsize,
 	struct ieee80211_key *key, struct mbuf *m)
 {
@@ -853,16 +857,16 @@ ieee80211_crypto_getmcastkey(struct ieee80211vap *vap,
  *     marked EAPOL frames w/ M_EAPOL.
  */
 struct mbuf *
-ieee80211_encap(struct ieee80211_node *ni, struct mbuf *m)
+ieee80211_encap(struct ieee80211vap *vap, struct ieee80211_node *ni,
+    struct mbuf *m)
 {
 #define	WH4(wh)	((struct ieee80211_frame_addr4 *)(wh))
-	struct ieee80211vap *vap = ni->ni_vap;
 	struct ieee80211com *ic = ni->ni_ic;
 	struct ether_header eh;
 	struct ieee80211_frame *wh;
 	struct ieee80211_key *key;
 	struct llc *llc;
-	int hdrsize, hdrspace, datalen, addqos, txfrag, isff, is4addr;
+	int hdrsize, hdrspace, datalen, addqos, txfrag, is4addr;
 
 	/*
 	 * Copy existing Ethernet header to a safe place.  The
@@ -917,12 +921,11 @@ ieee80211_encap(struct ieee80211_node *ni, struct mbuf *m)
 		hdrsize = sizeof(struct ieee80211_frame);
 	/*
 	 * 4-address frames need to be generated for:
-	 * o packets sent through a WDS vap (M_WDS || IEEE80211_M_WDS)
+	 * o packets sent through a WDS vap (IEEE80211_M_WDS)
 	 * o packets relayed by a station operating with dynamic WDS
 	 *   (IEEE80211_M_STA+IEEE80211_F_DWDS and src address)
 	 */
-	is4addr = (m->m_flags & M_WDS) ||
-	    vap->iv_opmode == IEEE80211_M_WDS ||	/* XXX redundant? */
+	is4addr = vap->iv_opmode == IEEE80211_M_WDS ||
 	    (vap->iv_opmode == IEEE80211_M_STA &&
 	     (vap->iv_flags & IEEE80211_F_DWDS) &&
 	     !IEEE80211_ADDR_EQ(eh.ether_shost, vap->iv_myaddr));
@@ -936,56 +939,7 @@ ieee80211_encap(struct ieee80211_node *ni, struct mbuf *m)
 	else
 		hdrspace = hdrsize;
 
-	if ((isff = m->m_flags & M_FF) != 0) {
-		struct mbuf *m2;
-		struct ether_header eh2;
-
-		/*
-		 * Fast frame encapsulation.  There must be two packets
-		 * chained with m_nextpkt.  We do header adjustment for
-		 * each, add the tunnel encapsulation, and then concatenate
-		 * the mbuf chains to form a single frame for transmission.
-		 */
-		m2 = m->m_nextpkt;
-		if (m2 == NULL) {
-			IEEE80211_DPRINTF(vap, IEEE80211_MSG_SUPERG,
-				"%s: only one frame\n", __func__);
-			goto bad;
-		}
-		m->m_nextpkt = NULL;
-		/*
-		 * Include fast frame headers in adjusting header
-		 * layout; this allocates space according to what
-		 * ieee80211_encap_fastframe will do.
-		 */
-		m = ieee80211_mbuf_adjust(vap,
-			hdrspace + sizeof(struct llc) + sizeof(uint32_t) + 2 +
-			    sizeof(struct ether_header),
-			key, m);
-		if (m == NULL) {
-			/* NB: ieee80211_mbuf_adjust handles msgs+statistics */
-			m_freem(m2);
-			goto bad;
-		}
-		/*
-		 * Copy second frame's Ethernet header out of line
-		 * and adjust for encapsulation headers.  Note that
-		 * we make room for padding in case there isn't room
-		 * at the end of first frame.
-		 */
-		KASSERT(m2->m_len >= sizeof(eh2), ("no ethernet header!"));
-		ETHER_HEADER_COPY(&eh2, mtod(m2, caddr_t));
-		m2 = ieee80211_mbuf_adjust(vap,
-			ATH_FF_MAX_HDR_PAD + sizeof(struct ether_header),
-			NULL, m2);
-		if (m2 == NULL) {
-			/* NB: ieee80211_mbuf_adjust handles msgs+statistics */
-			goto bad;
-		}
-		m = ieee80211_encap_fastframe(vap, m, &eh, m2, &eh2);
-		if (m == NULL)
-			goto bad;
-	} else {
+	if (__predict_true((m->m_flags & M_FF) == 0)) {
 		/*
 		 * Normal frame.
 		 */
@@ -1003,6 +957,15 @@ ieee80211_encap(struct ieee80211_node *ni, struct mbuf *m)
 		llc->llc_snap.org_code[1] = 0;
 		llc->llc_snap.org_code[2] = 0;
 		llc->llc_snap.ether_type = eh.ether_type;
+	} else {
+#ifdef IEEE80211_SUPPORT_SUPERG
+		/*
+		 * Aggregated frame.
+		 */
+		m = ieee80211_ff_encap(vap, m, hdrspace, key);
+		if (m == NULL)
+#endif
+			goto bad;
 	}
 	datalen = m->m_pkthdr.len;		/* NB: w/o 802.11 header */
 
@@ -1129,7 +1092,7 @@ ieee80211_encap(struct ieee80211_node *ni, struct mbuf *m)
 	txfrag = (m->m_pkthdr.len > vap->iv_fragthreshold &&
 	    !IEEE80211_IS_MULTICAST(wh->i_addr1) &&
 	    (vap->iv_caps & IEEE80211_C_TXFRAG) &&
-	    !isff);		/* NB: don't fragment ff's */
+	    (m->m_flags & M_FF) == 0);		/* NB: don't fragment ff's */
 	if (key != NULL) {
 		/*
 		 * IEEE 802.1X: send EAPOL frames always in the clear.
@@ -1176,135 +1139,6 @@ bad:
 }
 
 /*
- * Do Ethernet-LLC encapsulation for each payload in a fast frame
- * tunnel encapsulation.  The frame is assumed to have an Ethernet
- * header at the front that must be stripped before prepending the
- * LLC followed by the Ethernet header passed in (with an Ethernet
- * type that specifies the payload size).
- */
-static struct mbuf *
-ieee80211_encap1(struct ieee80211vap *vap, struct mbuf *m,
-	const struct ether_header *eh)
-{
-	struct llc *llc;
-	uint16_t payload;
-
-	/* XXX optimize by combining m_adj+M_PREPEND */
-	m_adj(m, sizeof(struct ether_header) - sizeof(struct llc));
-	llc = mtod(m, struct llc *);
-	llc->llc_dsap = llc->llc_ssap = LLC_SNAP_LSAP;
-	llc->llc_control = LLC_UI;
-	llc->llc_snap.org_code[0] = 0;
-	llc->llc_snap.org_code[1] = 0;
-	llc->llc_snap.org_code[2] = 0;
-	llc->llc_snap.ether_type = eh->ether_type;
-	payload = m->m_pkthdr.len;		/* NB: w/o Ethernet header */
-
-	M_PREPEND(m, sizeof(struct ether_header), M_DONTWAIT);
-	if (m == NULL) {		/* XXX cannot happen */
-		IEEE80211_DPRINTF(vap, IEEE80211_MSG_SUPERG,
-			"%s: no space for ether_header\n", __func__);
-		vap->iv_stats.is_tx_nobuf++;
-		return NULL;
-	}
-	ETHER_HEADER_COPY(mtod(m, void *), eh);
-	mtod(m, struct ether_header *)->ether_type = htons(payload);
-	return m;
-}
-
-/*
- * Do fast frame tunnel encapsulation.  The two frames and
- * Ethernet headers are supplied.  The caller is assumed to
- * have arrange for space in the mbuf chains for encapsulating
- * headers (to avoid major mbuf fragmentation).
- *
- * The encapsulated frame is returned or NULL if there is a
- * problem (should not happen).
- */
-static struct mbuf *
-ieee80211_encap_fastframe(struct ieee80211vap *vap,
-	struct mbuf *m1, const struct ether_header *eh1,
-	struct mbuf *m2, const struct ether_header *eh2)
-{
-	struct llc *llc;
-	struct mbuf *m;
-	int pad;
-
-	/*
-	 * First, each frame gets a standard encapsulation.
-	 */
-	m1 = ieee80211_encap1(vap, m1, eh1);
-	if (m1 == NULL) {
-		m_freem(m2);
-		return NULL;
-	}
-	m2 = ieee80211_encap1(vap, m2, eh2);
-	if (m2 == NULL) {
-		m_freem(m1);
-		return NULL;
-	}
-
-	/*
-	 * Pad leading frame to a 4-byte boundary.  If there
-	 * is space at the end of the first frame, put it
-	 * there; otherwise prepend to the front of the second
-	 * frame.  We know doing the second will always work
-	 * because we reserve space above.  We prefer appending
-	 * as this typically has better DMA alignment properties.
-	 */
-	for (m = m1; m->m_next != NULL; m = m->m_next)
-		;
-	pad = roundup2(m1->m_pkthdr.len, 4) - m1->m_pkthdr.len;
-	if (pad) {
-		if (M_TRAILINGSPACE(m) < pad) {		/* prepend to second */
-			m2->m_data -= pad;
-			m2->m_len += pad;
-			m2->m_pkthdr.len += pad;
-		} else {				/* append to first */
-			m->m_len += pad;
-			m1->m_pkthdr.len += pad;
-		}
-	}
-
-	/*
-	 * Now, stick 'em together and prepend the tunnel headers;
-	 * first the Atheros tunnel header (all zero for now) and
-	 * then a special fast frame LLC.
-	 *
-	 * XXX optimize by prepending together
-	 */
-	m->m_next = m2;			/* NB: last mbuf from above */
-	m1->m_pkthdr.len += m2->m_pkthdr.len;
-	M_PREPEND(m1, sizeof(uint32_t)+2, M_DONTWAIT);
-	if (m1 == NULL) {		/* XXX cannot happen */
-		IEEE80211_DPRINTF(vap, IEEE80211_MSG_SUPERG,
-			"%s: no space for tunnel header\n", __func__);
-		vap->iv_stats.is_tx_nobuf++;
-		return NULL;
-	}
-	memset(mtod(m1, void *), 0, sizeof(uint32_t)+2);
-
-	M_PREPEND(m1, sizeof(struct llc), M_DONTWAIT);
-	if (m1 == NULL) {		/* XXX cannot happen */
-		IEEE80211_DPRINTF(vap, IEEE80211_MSG_SUPERG,
-			"%s: no space for llc header\n", __func__);
-		vap->iv_stats.is_tx_nobuf++;
-		return NULL;
-	}
-	llc = mtod(m1, struct llc *);
-	llc->llc_dsap = llc->llc_ssap = LLC_SNAP_LSAP;
-	llc->llc_control = LLC_UI;
-	llc->llc_snap.org_code[0] = ATH_FF_SNAP_ORGCODE_0;
-	llc->llc_snap.org_code[1] = ATH_FF_SNAP_ORGCODE_1;
-	llc->llc_snap.org_code[2] = ATH_FF_SNAP_ORGCODE_2;
-	llc->llc_snap.ether_type = htons(ATH_FF_ETH_TYPE);
-
-	vap->iv_stats.is_ff_encap++;
-
-	return m1;
-}
-
-/*
  * Fragment the frame according to the specified mtu.
  * The size of the 802.11 header (w/o padding) is provided
  * so we don't need to recalculate it.  We create a new
@@ -1569,31 +1403,6 @@ ieee80211_add_wme_param(uint8_t *frm, struct ieee80211_wme_state *wme)
 }
 #undef WME_OUI_BYTES
 
-#define	ATH_OUI_BYTES		0x00, 0x03, 0x7f
-/*
- * Add a WME information element to a frame.
- */
-static uint8_t *
-ieee80211_add_ath(uint8_t *frm, uint8_t caps, uint16_t defkeyix)
-{
-	static const struct ieee80211_ath_ie info = {
-		.ath_id		= IEEE80211_ELEMID_VENDOR,
-		.ath_len	= sizeof(struct ieee80211_ath_ie) - 2,
-		.ath_oui	= { ATH_OUI_BYTES },
-		.ath_oui_type	= ATH_OUI_TYPE,
-		.ath_oui_subtype= ATH_OUI_SUBTYPE,
-		.ath_version	= ATH_OUI_VERSION,
-	};
-	struct ieee80211_ath_ie *ath = (struct ieee80211_ath_ie *) frm;
-
-	memcpy(frm, &info, sizeof(info));
-	ath->ath_capability = caps;
-	ath->ath_defkeyix[0] = (defkeyix & 0xff);
-	ath->ath_defkeyix[1] = ((defkeyix >> 8) & 0xff);
-	return frm + sizeof(info); 
-}
-#undef ATH_OUI_BYTES
-
 /*
  * Add an 11h Power Constraint element to a frame.
  */
@@ -1978,7 +1787,9 @@ ieee80211_send_mgmt(struct ieee80211_node *ni, int type, int arg)
 		       + sizeof(struct ieee80211_wme_info)
 		       + sizeof(struct ieee80211_ie_htcap)
 		       + 4 + sizeof(struct ieee80211_ie_htcap)
+#ifdef IEEE80211_SUPPORT_SUPERG
 		       + sizeof(struct ieee80211_ath_ie)
+#endif
 		       + (vap->iv_appie_wpa != NULL ?
 				vap->iv_appie_wpa->ie_len : 0)
 		       + (vap->iv_appie_assocreq != NULL ?
@@ -2047,13 +1858,15 @@ ieee80211_send_mgmt(struct ieee80211_node *ni, int type, int arg)
 		    ni->ni_ies.htcap_ie != NULL &&
 		    ni->ni_ies.htcap_ie[0] == IEEE80211_ELEMID_VENDOR)
 			frm = ieee80211_add_htcap_vendor(frm, ni);
-		if (IEEE80211_ATH_CAP(vap, ni, IEEE80211_F_ATHEROS))
-			frm = ieee80211_add_ath(frm,
+#ifdef IEEE80211_SUPPORT_SUPERG
+		if (IEEE80211_ATH_CAP(vap, ni, IEEE80211_F_ATHEROS)) {
+			frm = ieee80211_add_ath(frm, 
 				IEEE80211_ATH_CAP(vap, ni, IEEE80211_F_ATHEROS),
-				(vap->iv_flags & IEEE80211_F_WPA) == 0 &&
-				ni->ni_authmode != IEEE80211_AUTH_8021X &&
-				vap->iv_def_txkey != IEEE80211_KEYIX_NONE ?
-				vap->iv_def_txkey : 0x7fff);
+				((vap->iv_flags & IEEE80211_F_WPA) == 0 &&
+				 ni->ni_authmode != IEEE80211_AUTH_8021X) ?
+				vap->iv_def_txkey : IEEE80211_KEYIX_NONE);
+		}
+#endif /* IEEE80211_SUPPORT_SUPERG */
 		if (vap->iv_appie_assocreq != NULL)
 			frm = add_appie(frm, vap->iv_appie_assocreq);
 		m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *);
@@ -2089,7 +1902,9 @@ ieee80211_send_mgmt(struct ieee80211_node *ni, int type, int arg)
 		       + sizeof(struct ieee80211_ie_htcap) + 4
 		       + sizeof(struct ieee80211_ie_htinfo) + 4
 		       + sizeof(struct ieee80211_wme_param)
+#ifdef IEEE80211_SUPPORT_SUPERG
 		       + sizeof(struct ieee80211_ath_ie)
+#endif
 		       + (vap->iv_appie_assocresp != NULL ?
 				vap->iv_appie_assocresp->ie_len : 0)
 		);
@@ -2124,10 +1939,14 @@ ieee80211_send_mgmt(struct ieee80211_node *ni, int type, int arg)
 			frm = ieee80211_add_htcap_vendor(frm, ni);
 			frm = ieee80211_add_htinfo_vendor(frm, ni);
 		}
+#ifdef IEEE80211_SUPPORT_SUPERG
 		if (IEEE80211_ATH_CAP(vap, ni, IEEE80211_F_ATHEROS))
-			frm = ieee80211_add_ath(frm,
+			frm = ieee80211_add_ath(frm, 
 				IEEE80211_ATH_CAP(vap, ni, IEEE80211_F_ATHEROS),
-				ni->ni_ath_defkeyix);
+				((vap->iv_flags & IEEE80211_F_WPA) == 0 &&
+				 ni->ni_authmode != IEEE80211_AUTH_8021X) ?
+				vap->iv_def_txkey : IEEE80211_KEYIX_NONE);
+#endif /* IEEE80211_SUPPORT_SUPERG */
 		if (vap->iv_appie_assocresp != NULL)
 			frm = add_appie(frm, vap->iv_appie_assocresp);
 		m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *);
@@ -2228,7 +2047,9 @@ ieee80211_alloc_proberesp(struct ieee80211_node *bss, int legacy)
 	       + sizeof(struct ieee80211_wme_param)
 	       + 4 + sizeof(struct ieee80211_ie_htcap)
 	       + 4 + sizeof(struct ieee80211_ie_htinfo)
+#ifdef IEEE80211_SUPPORT_SUPERG
 	       + sizeof(struct ieee80211_ath_ie)
+#endif
 	       + (vap->iv_appie_proberesp != NULL ?
 			vap->iv_appie_proberesp->ie_len : 0)
 	);
@@ -2311,9 +2132,11 @@ ieee80211_alloc_proberesp(struct ieee80211_node *bss, int legacy)
 		frm = ieee80211_add_htcap_vendor(frm, bss);
 		frm = ieee80211_add_htinfo_vendor(frm, bss);
 	}
-	if (bss->ni_ies.ath_ie != NULL && legacy != IEEE80211_SEND_LEGACY_11B)
-		frm = ieee80211_add_ath(frm, bss->ni_ath_flags,
-			bss->ni_ath_defkeyix);
+#ifdef IEEE80211_SUPPORT_SUPERG
+	if ((vap->iv_flags & IEEE80211_F_ATHEROS) &&
+	    legacy != IEEE80211_SEND_LEGACY_11B)
+		frm = ieee80211_add_athcaps(frm, bss);
+#endif
 	if (vap->iv_appie_proberesp != NULL)
 		frm = add_appie(frm, vap->iv_appie_proberesp);
 	m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *);
@@ -2506,6 +2329,7 @@ ieee80211_beacon_construct(struct mbuf *m, uint8_t *frm,
 	 *	[tlv] WME parameters
 	 *	[tlv] Vendor OUI HT capabilities (optional)
 	 *	[tlv] Vendor OUI HT information (optional)
+	 *	[tlv] Atheros capabilities (optional)
 	 *	[tlv] TDMA parameters (optional)
 	 *	[tlv] application data (optional)
 	 */
@@ -2596,6 +2420,12 @@ ieee80211_beacon_construct(struct mbuf *m, uint8_t *frm,
 		frm = ieee80211_add_htcap_vendor(frm, ni);
 		frm = ieee80211_add_htinfo_vendor(frm, ni);
 	}
+#ifdef IEEE80211_SUPPORT_SUPERG
+	if (vap->iv_flags & IEEE80211_F_ATHEROS) {
+		bo->bo_ath = frm;
+		frm = ieee80211_add_athcaps(frm, ni);
+	}
+#endif
 #ifdef IEEE80211_SUPPORT_TDMA
 	if (vap->iv_caps & IEEE80211_C_TDMA) {
 		bo->bo_tdma = frm;
@@ -2675,6 +2505,9 @@ ieee80211_beacon_alloc(struct ieee80211_node *ni,
 		 + 4+2*sizeof(struct ieee80211_ie_htinfo)/* HT info */
 		 + (vap->iv_caps & IEEE80211_C_WME ?	/* WME */
 			sizeof(struct ieee80211_wme_param) : 0)
+#ifdef IEEE80211_SUPPORT_SUPERG
+		 + sizeof(struct ieee80211_ath_ie)	/* ATH */
+#endif
 #ifdef IEEE80211_SUPPORT_TDMA
 		 + (vap->iv_caps & IEEE80211_C_TDMA ?	/* TDMA */
 			sizeof(struct ieee80211_tdma_param) : 0)
@@ -2852,6 +2685,12 @@ ieee80211_beacon_update(struct ieee80211_node *ni,
 				bo->bo_tim_trailer += adjust;
 				bo->bo_erp += adjust;
 				bo->bo_htinfo += adjust;
+#ifdef IEEE80211_SUPERG_SUPPORT
+				bo->bo_ath += adjust;
+#endif
+#ifdef IEEE80211_TDMA_SUPPORT
+				bo->bo_tdma += adjust;
+#endif
 				bo->bo_appie += adjust;
 				bo->bo_wme += adjust;
 				bo->bo_csa += adjust;
@@ -2895,7 +2734,14 @@ ieee80211_beacon_update(struct ieee80211_node *ni,
 			if (vap->iv_csa_count == 0) {
 				memmove(&csa[1], csa, bo->bo_csa_trailer_len);
 				bo->bo_erp += sizeof(*csa);
+				bo->bo_htinfo += sizeof(*csa);
 				bo->bo_wme += sizeof(*csa);
+#ifdef IEEE80211_SUPERG_SUPPORT
+				bo->bo_ath += sizeof(*csa);
+#endif
+#ifdef IEEE80211_TDMA_SUPPORT
+				bo->bo_tdma += sizeof(*csa);
+#endif
 				bo->bo_appie += sizeof(*csa);
 				bo->bo_csa_trailer_len += sizeof(*csa);
 				bo->bo_tim_trailer_len += sizeof(*csa);
@@ -2915,6 +2761,12 @@ ieee80211_beacon_update(struct ieee80211_node *ni,
 			(void) ieee80211_add_erp(bo->bo_erp, ic);
 			clrbit(bo->bo_flags, IEEE80211_BEACON_ERP);
 		}
+#ifdef IEEE80211_SUPPORT_SUPERG
+		if (isset(bo->bo_flags,  IEEE80211_BEACON_ATH)) {
+			ieee80211_add_athcaps(bo->bo_ath, ni);
+			clrbit(bo->bo_flags, IEEE80211_BEACON_ATH);
+		}
+#endif
 	}
 	if (isset(bo->bo_flags, IEEE80211_BEACON_APPIE)) {
 		const struct ieee80211_appie *aie = vap->iv_appie_beacon;
diff --git a/sys/net80211/ieee80211_proto.c b/sys/net80211/ieee80211_proto.c
index f870c3c..572580f 100644
--- a/sys/net80211/ieee80211_proto.c
+++ b/sys/net80211/ieee80211_proto.c
@@ -1316,39 +1316,6 @@ ieee80211_resume_all(struct ieee80211com *ic)
 	IEEE80211_UNLOCK(ic);
 }
 
-/*
- * Switch between turbo and non-turbo operating modes.
- * Use the specified channel flags to locate the new
- * channel, update 802.11 state, and then call back into
- * the driver to effect the change.
- */
-void
-ieee80211_dturbo_switch(struct ieee80211vap *vap, int newflags)
-{
-	struct ieee80211com *ic = vap->iv_ic;
-	struct ieee80211_channel *chan;
-
-	chan = ieee80211_find_channel(ic, ic->ic_bsschan->ic_freq, newflags);
-	if (chan == NULL) {		/* XXX should not happen */
-		IEEE80211_DPRINTF(vap, IEEE80211_MSG_SUPERG,
-		    "%s: no channel with freq %u flags 0x%x\n",
-		    __func__, ic->ic_bsschan->ic_freq, newflags);
-		return;
-	}
-
-	IEEE80211_DPRINTF(vap, IEEE80211_MSG_SUPERG,
-	    "%s: %s -> %s (freq %u flags 0x%x)\n", __func__,
-	    ieee80211_phymode_name[ieee80211_chan2mode(ic->ic_bsschan)],
-	    ieee80211_phymode_name[ieee80211_chan2mode(chan)],
-	    chan->ic_freq, chan->ic_flags);
-
-	ic->ic_bsschan = chan;
-	ic->ic_prevchan = ic->ic_curchan;
-	ic->ic_curchan = chan;
-	ic->ic_set_channel(ic);
-	/* NB: do not need to reset ERP state 'cuz we're in sta mode */
-}
-
 void
 ieee80211_beacon_miss(struct ieee80211com *ic)
 {
diff --git a/sys/net80211/ieee80211_proto.h b/sys/net80211/ieee80211_proto.h
index fa50758..2d32d1e 100644
--- a/sys/net80211/ieee80211_proto.h
+++ b/sys/net80211/ieee80211_proto.h
@@ -74,7 +74,10 @@ int	ieee80211_output(struct ifnet *, struct mbuf *,
 void	ieee80211_start(struct ifnet *);
 int	ieee80211_send_nulldata(struct ieee80211_node *);
 int	ieee80211_classify(struct ieee80211_node *, struct mbuf *m);
-struct mbuf *ieee80211_encap(struct ieee80211_node *, struct mbuf *);
+struct mbuf *ieee80211_mbuf_adjust(struct ieee80211vap *, int,
+		struct ieee80211_key *, struct mbuf *);
+struct mbuf *ieee80211_encap(struct ieee80211vap *, struct ieee80211_node *,
+		struct mbuf *);
 int	ieee80211_send_mgmt(struct ieee80211_node *, int, int);
 struct ieee80211_appie;
 int	ieee80211_send_probereq(struct ieee80211_node *ni,
@@ -298,6 +301,7 @@ struct ieee80211_beacon_offsets {
 	uint16_t	bo_tim_trailer_len;/* tim trailer length in bytes */
 	uint8_t		*bo_erp;	/* start of ERP element */
 	uint8_t		*bo_htinfo;	/* start of HT info element */
+	uint8_t		*bo_ath;	/* start of ATH parameters */
 	uint8_t		*bo_appie;	/* start of AppIE element */
 	uint16_t	bo_appie_len;	/* AppIE length in bytes */
 	uint16_t	bo_csa_trailer_len;;
@@ -328,6 +332,7 @@ enum {
 	IEEE80211_BEACON_CFP	= 6,	/* CFParms */
 	IEEE80211_BEACON_CSA	= 7,	/* Channel Switch Announcement */
 	IEEE80211_BEACON_TDMA	= 9,	/* TDMA Info */
+	IEEE80211_BEACON_ATH	= 10,	/* ATH parameters */
 };
 int	ieee80211_beacon_update(struct ieee80211_node *,
 		struct ieee80211_beacon_offsets *, struct mbuf *, int mcast);
diff --git a/sys/net80211/ieee80211_scan.c b/sys/net80211/ieee80211_scan.c
index be3c08a..7ce2d31 100644
--- a/sys/net80211/ieee80211_scan.c
+++ b/sys/net80211/ieee80211_scan.c
@@ -298,6 +298,7 @@ change_channel(struct ieee80211com *ic,
 	struct ieee80211_channel *chan)
 {
 	ic->ic_curchan = chan;
+	ic->ic_rt = ieee80211_get_ratetable(chan);
 	ic->ic_set_channel(ic);
 }
 
diff --git a/sys/net80211/ieee80211_scan.h b/sys/net80211/ieee80211_scan.h
index 3b58cff..78e5fe4 100644
--- a/sys/net80211/ieee80211_scan.h
+++ b/sys/net80211/ieee80211_scan.h
@@ -42,7 +42,7 @@
  * a callback when scanning on a ``passive channel'' when the
  * IEEE80211_FEXT_PROBECHAN flag is set.
  *
- * A scan operation involves constructing a set of channels to inspec
+ * A scan operation involves constructing a set of channels to inspect
  * (the scan set), visiting each channel and collecting information
  * (e.g. what bss are present), and then analyzing the results to make
  * decisions like which bss to join.  This process needs to be as fast
diff --git a/sys/net80211/ieee80211_scan_sta.c b/sys/net80211/ieee80211_scan_sta.c
index bc99817..607c36d 100644
--- a/sys/net80211/ieee80211_scan_sta.c
+++ b/sys/net80211/ieee80211_scan_sta.c
@@ -126,6 +126,7 @@ static void sta_flush_table(struct sta_table *);
 #define	MATCH_TDMA_NOTMASTER	0x0800	/* not TDMA master */
 #define	MATCH_TDMA_NOSLOT	0x1000	/* all TDMA slots occupied */
 #define	MATCH_TDMA_LOCAL	0x2000	/* local address */
+#define	MATCH_TDMA_VERSION	0x4000	/* protocol version mismatch */
 static int match_bss(struct ieee80211vap *,
 	const struct ieee80211_scan_state *, struct sta_entry *, int);
 static void adhoc_age(struct ieee80211_scan_state *);
@@ -970,9 +971,12 @@ match_bss(struct ieee80211vap *vap,
 		if (vap->iv_caps & IEEE80211_C_TDMA) {
 			const struct ieee80211_tdma_param *tdma =
 			    (const struct ieee80211_tdma_param *)se->se_ies.tdma_ie;
+			const struct ieee80211_tdma_state *ts = vap->iv_tdma;
 
 			if (tdma == NULL)
 				fail |= MATCH_TDMA_NOIE;
+			else if (tdma->tdma_version != ts->tdma_version)
+				fail |= MATCH_TDMA_VERSION;
 			else if (tdma->tdma_slot != 0)
 				fail |= MATCH_TDMA_NOTMASTER;
 			else if (tdma_isfull(tdma))
@@ -1062,9 +1066,10 @@ match_bss(struct ieee80211vap *vap,
 		    fail & MATCH_CC ? '$' :
 #ifdef IEEE80211_SUPPORT_TDMA
 		    fail & MATCH_TDMA_NOIE ? '&' :
-		    fail & MATCH_TDMA_NOTMASTER ? ':' :
-		    fail & MATCH_TDMA_NOSLOT ? '@' :
-		    fail & MATCH_TDMA_LOCAL ? '#' :
+		    fail & MATCH_TDMA_VERSION ? 'v' :
+		    fail & MATCH_TDMA_NOTMASTER ? 's' :
+		    fail & MATCH_TDMA_NOSLOT ? 'f' :
+		    fail & MATCH_TDMA_LOCAL ? 'l' :
 #endif
 		    fail ? '-' : '+', ether_sprintf(se->se_macaddr));
 		printf(" %s%c", ether_sprintf(se->se_bssid),
@@ -1076,8 +1081,7 @@ match_bss(struct ieee80211vap *vap,
 		    fail & MATCH_RATE ? '!' : ' ');
 		printf(" %4s%c",
 		    (se->se_capinfo & IEEE80211_CAPINFO_ESS) ? "ess" :
-		    (se->se_capinfo & IEEE80211_CAPINFO_IBSS) ? "ibss" :
-		    "????",
+		    (se->se_capinfo & IEEE80211_CAPINFO_IBSS) ? "ibss" : "",
 		    fail & MATCH_CAPINFO ? '!' : ' ');
 		printf(" %3s%c ",
 		    (se->se_capinfo & IEEE80211_CAPINFO_PRIVACY) ?
diff --git a/sys/net80211/ieee80211_sta.c b/sys/net80211/ieee80211_sta.c
index 54733c9..ff3365f 100644
--- a/sys/net80211/ieee80211_sta.c
+++ b/sys/net80211/ieee80211_sta.c
@@ -57,6 +57,9 @@ __FBSDID("$FreeBSD$");
 #include <net80211/ieee80211_var.h>
 #include <net80211/ieee80211_sta.h>
 #include <net80211/ieee80211_input.h>
+#ifdef IEEE80211_SUPPORT_SUPERG
+#include <net80211/ieee80211_superg.h>
+#endif
 
 #define	IEEE80211_RATE2MBS(r)	(((r) & IEEE80211_RATE_VAL) / 2)
 
@@ -102,9 +105,7 @@ sta_vattach(struct ieee80211vap *vap)
 static void
 sta_beacon_miss(struct ieee80211vap *vap)
 {
-	struct ieee80211com *ic = vap->iv_ic;
-
-	KASSERT((ic->ic_flags & IEEE80211_F_SCAN) == 0, ("scanning"));
+	KASSERT((vap->iv_ic->ic_flags & IEEE80211_F_SCAN) == 0, ("scanning"));
 	KASSERT(vap->iv_state == IEEE80211_S_RUN,
 	    ("wrong state %d", vap->iv_state));
 
@@ -129,6 +130,9 @@ sta_beacon_miss(struct ieee80211vap *vap)
 	vap->iv_bmiss_count = 0;
 	vap->iv_stats.is_beacon_miss++;
 	if (vap->iv_roaming == IEEE80211_ROAMING_AUTO) {
+#ifdef IEEE80211_SUPPORT_SUPERG
+		struct ieee80211com *ic = vap->iv_ic;
+
 		/*
 		 * If we receive a beacon miss interrupt when using
 		 * dynamic turbo, attempt to switch modes before
@@ -137,6 +141,7 @@ sta_beacon_miss(struct ieee80211vap *vap)
 		if (IEEE80211_ATH_CAP(vap, vap->iv_bss, IEEE80211_NODE_TURBOP))
 			ieee80211_dturbo_switch(vap,
 			    ic->ic_bsschan->ic_flags ^ IEEE80211_CHAN_TURBO);
+#endif
 		/*
 		 * Try to reassociate before scanning for a new ap.
 		 */
@@ -795,32 +800,13 @@ sta_input(struct ieee80211_node *ni, struct mbuf *m,
 			m = ieee80211_decap_amsdu(ni, m);
 			if (m == NULL)
 				return IEEE80211_FC0_TYPE_DATA;
-		} else if (IEEE80211_ATH_CAP(vap, ni, IEEE80211_NODE_FF) &&
-#define	FF_LLC_SIZE	(sizeof(struct ether_header) + sizeof(struct llc))
-		    m->m_pkthdr.len >= 3*FF_LLC_SIZE) {
-			struct llc *llc;
-
-			/*
-			 * Check for fast-frame tunnel encapsulation.
-			 */
-			if (m->m_len < FF_LLC_SIZE &&
-			    (m = m_pullup(m, FF_LLC_SIZE)) == NULL) {
-				IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
-				    ni->ni_macaddr, "fast-frame",
-				    "%s", "m_pullup(llc) failed");
-				vap->iv_stats.is_rx_tooshort++;
+		} else {
+#ifdef IEEE80211_SUPPORT_SUPERG
+			m = ieee80211_decap_fastframe(vap, ni, m);
+			if (m == NULL)
 				return IEEE80211_FC0_TYPE_DATA;
-			}
-			llc = (struct llc *)(mtod(m, uint8_t *) + 
-				sizeof(struct ether_header));
-			if (llc->llc_snap.ether_type == htons(ATH_FF_ETH_TYPE)) {
-				m_adj(m, FF_LLC_SIZE);
-				m = ieee80211_decap_fastframe(ni, m);
-				if (m == NULL)
-					return IEEE80211_FC0_TYPE_DATA;
-			}
+#endif
 		}
-#undef FF_LLC_SIZE
 		ieee80211_deliver_data(vap, ni, m);
 		return IEEE80211_FC0_TYPE_DATA;
 
@@ -1094,51 +1080,6 @@ ieee80211_parse_wmeparams(struct ieee80211vap *vap, uint8_t *frm,
 #undef MS
 }
 
-static int
-ieee80211_parse_athparams(struct ieee80211_node *ni, uint8_t *frm,
-	const struct ieee80211_frame *wh)
-{
-	struct ieee80211vap *vap = ni->ni_vap;
-	const struct ieee80211_ath_ie *ath;
-	u_int len = frm[1];
-	int capschanged;
-	uint16_t defkeyix;
-
-	if (len < sizeof(struct ieee80211_ath_ie)-2) {
-		IEEE80211_DISCARD_IE(vap,
-		    IEEE80211_MSG_ELEMID | IEEE80211_MSG_SUPERG,
-		    wh, "Atheros", "too short, len %u", len);
-		return -1;
-	}
-	ath = (const struct ieee80211_ath_ie *)frm;
-	capschanged = (ni->ni_ath_flags != ath->ath_capability);
-	defkeyix = LE_READ_2(ath->ath_defkeyix);
-	if (capschanged || defkeyix != ni->ni_ath_defkeyix) {
-		ni->ni_ath_flags = ath->ath_capability;
-		ni->ni_ath_defkeyix = defkeyix;
-		IEEE80211_NOTE(vap, IEEE80211_MSG_SUPERG, ni,
-		    "ath ie change: new caps 0x%x defkeyix 0x%x",
-		    ni->ni_ath_flags, ni->ni_ath_defkeyix);
-	}
-	if (IEEE80211_ATH_CAP(vap, ni, ATHEROS_CAP_TURBO_PRIME)) {
-		uint16_t curflags, newflags;
-
-		/*
-		 * Check for turbo mode switch.  Calculate flags
-		 * for the new mode and effect the switch.
-		 */
-		newflags = curflags = vap->iv_ic->ic_bsschan->ic_flags;
-		/* NB: BOOST is not in ic_flags, so get it from the ie */
-		if (ath->ath_capability & ATHEROS_CAP_BOOST) 
-			newflags |= IEEE80211_CHAN_TURBO;
-		else
-			newflags &= ~IEEE80211_CHAN_TURBO;
-		if (newflags != curflags)
-			ieee80211_dturbo_switch(vap, newflags);
-	}
-	return capschanged;
-}
-
 /*
  * Return non-zero if a background scan may be continued:
  * o bg scan is active
@@ -1176,7 +1117,9 @@ startbgscan(struct ieee80211vap *vap)
 
 	return ((vap->iv_flags & IEEE80211_F_BGSCAN) &&
 	    (ic->ic_flags & IEEE80211_F_CSAPENDING) == 0 &&
+#ifdef IEEE80211_SUPPORT_SUPERG
 	    !IEEE80211_IS_CHAN_DTURBO(ic->ic_curchan) &&
+#endif
 	    time_after(ticks, ic->ic_lastscan + vap->iv_bgscanintvl) &&
 	    time_after(ticks, ic->ic_lastdata + vap->iv_bgscanidle));
 }
@@ -1271,8 +1214,10 @@ sta_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0,
 			    (ni->ni_flags & IEEE80211_NODE_QOS) &&
 			    ieee80211_parse_wmeparams(vap, scan.wme, wh) > 0)
 				ieee80211_wme_updateparams(vap);
+#ifdef IEEE80211_SUPPORT_SUPERG
 			if (scan.ath != NULL)
 				ieee80211_parse_athparams(ni, scan.ath, wh);
+#endif
 			if (scan.htcap != NULL && scan.htinfo != NULL &&
 			    (vap->iv_flags_ext & IEEE80211_FEXT_HT)) {
 				ieee80211_ht_updateparams(ni,
@@ -1514,6 +1459,11 @@ sta_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0,
 			ieee80211_setup_htrates(ni, htcap,
 			     IEEE80211_F_JOIN | IEEE80211_F_DOBRS);
 			ieee80211_setup_basic_htrates(ni, htinfo);
+		} else {
+#ifdef IEEE80211_SUPPORT_SUPERG
+			if (IEEE80211_ATH_CAP(vap, ni, IEEE80211_NODE_ATH))
+				ieee80211_ff_node_init(ni);
+#endif
 		}
 		/*
 		 * Configure state now that we are associated.
diff --git a/sys/net80211/ieee80211_superg.c b/sys/net80211/ieee80211_superg.c
new file mode 100644
index 0000000..4bae885
--- /dev/null
+++ b/sys/net80211/ieee80211_superg.c
@@ -0,0 +1,872 @@
+/*-
+ * Copyright (c) 2002-2009 Sam Leffler, Errno Consulting
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_wlan.h"
+
+#include <sys/param.h>
+#include <sys/systm.h> 
+#include <sys/mbuf.h>   
+#include <sys/kernel.h>
+#include <sys/endian.h>
+
+#include <sys/socket.h>
+ 
+#include <net/bpf.h>
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_llc.h>
+#include <net/if_media.h>
+
+#include <net80211/ieee80211_var.h>
+#include <net80211/ieee80211_input.h>
+#include <net80211/ieee80211_phy.h>
+#include <net80211/ieee80211_superg.h>
+
+/*
+ * Atheros fast-frame encapsulation format.
+ * FF max payload:
+ * 802.2 + FFHDR + HPAD + 802.3 + 802.2 + 1500 + SPAD + 802.3 + 802.2 + 1500:
+ *   8   +   4   +  4   +   14  +   8   + 1500 +  6   +   14  +   8   + 1500
+ * = 3066
+ */
+/* fast frame header is 32-bits */
+#define	ATH_FF_PROTO	0x0000003f	/* protocol */
+#define	ATH_FF_PROTO_S	0
+#define	ATH_FF_FTYPE	0x000000c0	/* frame type */
+#define	ATH_FF_FTYPE_S	6
+#define	ATH_FF_HLEN32	0x00000300	/* optional hdr length */
+#define	ATH_FF_HLEN32_S	8
+#define	ATH_FF_SEQNUM	0x001ffc00	/* sequence number */
+#define	ATH_FF_SEQNUM_S	10
+#define	ATH_FF_OFFSET	0xffe00000	/* offset to 2nd payload */
+#define	ATH_FF_OFFSET_S	21
+
+#define	ATH_FF_MAX_HDR_PAD	4
+#define	ATH_FF_MAX_SEP_PAD	6
+#define	ATH_FF_MAX_HDR		30
+
+#define	ATH_FF_PROTO_L2TUNNEL	0	/* L2 tunnel protocol */
+#define	ATH_FF_ETH_TYPE		0x88bd	/* Ether type for encapsulated frames */
+#define	ATH_FF_SNAP_ORGCODE_0	0x00
+#define	ATH_FF_SNAP_ORGCODE_1	0x03
+#define	ATH_FF_SNAP_ORGCODE_2	0x7f
+
+#define	ATH_FF_TXQMIN	2		/* min txq depth for staging */
+#define	ATH_FF_TXQMAX	50		/* maximum # of queued frames allowed */
+#define	ATH_FF_STAGEMAX	5		/* max waiting period for staged frame*/
+
+#define	ETHER_HEADER_COPY(dst, src) \
+	memcpy(dst, src, sizeof(struct ether_header))
+
+/* XXX public for sysctl hookup */
+int	ieee80211_ffppsmin = 2;		/* pps threshold for ff aggregation */
+int	ieee80211_ffagemax = -1;	/* max time frames held on stage q */
+
+void
+ieee80211_superg_attach(struct ieee80211com *ic)
+{
+	ieee80211_ffagemax = msecs_to_ticks(150);
+}
+
+void
+ieee80211_superg_detach(struct ieee80211com *ic)
+{
+}
+
+void
+ieee80211_superg_vattach(struct ieee80211vap *vap)
+{
+	if (vap->iv_caps & IEEE80211_C_FF)
+		vap->iv_flags |= IEEE80211_F_FF;
+	/* NB: we only implement sta mode */
+	if (vap->iv_opmode == IEEE80211_M_STA &&
+	    (vap->iv_caps & IEEE80211_C_TURBOP))
+		vap->iv_flags |= IEEE80211_F_TURBOP;
+}
+
+void
+ieee80211_superg_vdetach(struct ieee80211vap *vap)
+{
+}
+
+#define	ATH_OUI_BYTES		0x00, 0x03, 0x7f
+/*
+ * Add a WME information element to a frame.
+ */
+uint8_t *
+ieee80211_add_ath(uint8_t *frm, uint8_t caps, ieee80211_keyix defkeyix)
+{
+	static const struct ieee80211_ath_ie info = {
+		.ath_id		= IEEE80211_ELEMID_VENDOR,
+		.ath_len	= sizeof(struct ieee80211_ath_ie) - 2,
+		.ath_oui	= { ATH_OUI_BYTES },
+		.ath_oui_type	= ATH_OUI_TYPE,
+		.ath_oui_subtype= ATH_OUI_SUBTYPE,
+		.ath_version	= ATH_OUI_VERSION,
+	};
+	struct ieee80211_ath_ie *ath = (struct ieee80211_ath_ie *) frm;
+
+	memcpy(frm, &info, sizeof(info));
+	ath->ath_capability = caps;
+	if (defkeyix != IEEE80211_KEYIX_NONE) {
+		ath->ath_defkeyix[0] = (defkeyix & 0xff);
+		ath->ath_defkeyix[1] = ((defkeyix >> 8) & 0xff);
+	} else {
+		ath->ath_defkeyix[0] = 0xff;
+		ath->ath_defkeyix[1] = 0x7f;
+	}
+	return frm + sizeof(info); 
+}
+#undef ATH_OUI_BYTES
+
+uint8_t *
+ieee80211_add_athcaps(uint8_t *frm, const struct ieee80211_node *bss)
+{
+	const struct ieee80211vap *vap = bss->ni_vap;
+
+	return ieee80211_add_ath(frm,
+	    vap->iv_flags & IEEE80211_F_ATHEROS,
+	    ((vap->iv_flags & IEEE80211_F_WPA) == 0 &&
+	    bss->ni_authmode != IEEE80211_AUTH_8021X) ?
+	    vap->iv_def_txkey : IEEE80211_KEYIX_NONE);
+}
+
+void
+ieee80211_parse_ath(struct ieee80211_node *ni, uint8_t *ie)
+{
+	const struct ieee80211_ath_ie *ath =
+		(const struct ieee80211_ath_ie *) ie;
+
+	ni->ni_ath_flags = ath->ath_capability;
+	ni->ni_ath_defkeyix = LE_READ_2(&ath->ath_defkeyix);
+}
+
+int
+ieee80211_parse_athparams(struct ieee80211_node *ni, uint8_t *frm,
+	const struct ieee80211_frame *wh)
+{
+	struct ieee80211vap *vap = ni->ni_vap;
+	const struct ieee80211_ath_ie *ath;
+	u_int len = frm[1];
+	int capschanged;
+	uint16_t defkeyix;
+
+	if (len < sizeof(struct ieee80211_ath_ie)-2) {
+		IEEE80211_DISCARD_IE(vap,
+		    IEEE80211_MSG_ELEMID | IEEE80211_MSG_SUPERG,
+		    wh, "Atheros", "too short, len %u", len);
+		return -1;
+	}
+	ath = (const struct ieee80211_ath_ie *)frm;
+	capschanged = (ni->ni_ath_flags != ath->ath_capability);
+	defkeyix = LE_READ_2(ath->ath_defkeyix);
+	if (capschanged || defkeyix != ni->ni_ath_defkeyix) {
+		ni->ni_ath_flags = ath->ath_capability;
+		ni->ni_ath_defkeyix = defkeyix;
+		IEEE80211_NOTE(vap, IEEE80211_MSG_SUPERG, ni,
+		    "ath ie change: new caps 0x%x defkeyix 0x%x",
+		    ni->ni_ath_flags, ni->ni_ath_defkeyix);
+	}
+	if (IEEE80211_ATH_CAP(vap, ni, ATHEROS_CAP_TURBO_PRIME)) {
+		uint16_t curflags, newflags;
+
+		/*
+		 * Check for turbo mode switch.  Calculate flags
+		 * for the new mode and effect the switch.
+		 */
+		newflags = curflags = vap->iv_ic->ic_bsschan->ic_flags;
+		/* NB: BOOST is not in ic_flags, so get it from the ie */
+		if (ath->ath_capability & ATHEROS_CAP_BOOST) 
+			newflags |= IEEE80211_CHAN_TURBO;
+		else
+			newflags &= ~IEEE80211_CHAN_TURBO;
+		if (newflags != curflags)
+			ieee80211_dturbo_switch(vap, newflags);
+	}
+	return capschanged;
+}
+
+/*
+ * Decap the encapsulated frame pair and dispatch the first
+ * for delivery.  The second frame is returned for delivery
+ * via the normal path.
+ */
+struct mbuf *
+ieee80211_ff_decap(struct ieee80211_node *ni, struct mbuf *m)
+{
+#define	FF_LLC_SIZE	(sizeof(struct ether_header) + sizeof(struct llc))
+#define	MS(x,f)	(((x) & f) >> f##_S)
+	struct ieee80211vap *vap = ni->ni_vap;
+	struct llc *llc;
+	uint32_t ath;
+	struct mbuf *n;
+	int framelen;
+
+	/* NB: we assume caller does this check for us */
+	KASSERT(IEEE80211_ATH_CAP(vap, ni, IEEE80211_NODE_FF),
+	    ("ff not negotiated"));
+	/*
+	 * Check for fast-frame tunnel encapsulation.
+	 */
+	if (m->m_pkthdr.len < 3*FF_LLC_SIZE)
+		return m;
+	if (m->m_len < FF_LLC_SIZE &&
+	    (m = m_pullup(m, FF_LLC_SIZE)) == NULL) {
+		IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
+		    ni->ni_macaddr, "fast-frame",
+		    "%s", "m_pullup(llc) failed");
+		vap->iv_stats.is_rx_tooshort++;
+		return NULL;
+	}
+	llc = (struct llc *)(mtod(m, uint8_t *) +
+	    sizeof(struct ether_header));
+	if (llc->llc_snap.ether_type != htons(ATH_FF_ETH_TYPE))
+		return m;
+	m_adj(m, FF_LLC_SIZE);
+	m_copydata(m, 0, sizeof(uint32_t), (caddr_t) &ath);
+	if (MS(ath, ATH_FF_PROTO) != ATH_FF_PROTO_L2TUNNEL) {
+		IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
+		    ni->ni_macaddr, "fast-frame",
+		    "unsupport tunnel protocol, header 0x%x", ath);
+		vap->iv_stats.is_ff_badhdr++;
+		m_freem(m);
+		return NULL;
+	}
+	/* NB: skip header and alignment padding */
+	m_adj(m, roundup(sizeof(uint32_t) - 2, 4) + 2);
+
+	vap->iv_stats.is_ff_decap++;
+
+	/*
+	 * Decap the first frame, bust it apart from the
+	 * second and deliver; then decap the second frame
+	 * and return it to the caller for normal delivery.
+	 */
+	m = ieee80211_decap1(m, &framelen);
+	if (m == NULL) {
+		IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
+		    ni->ni_macaddr, "fast-frame", "%s", "first decap failed");
+		vap->iv_stats.is_ff_tooshort++;
+		return NULL;
+	}
+	n = m_split(m, framelen, M_NOWAIT);
+	if (n == NULL) {
+		IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
+		    ni->ni_macaddr, "fast-frame",
+		    "%s", "unable to split encapsulated frames");
+		vap->iv_stats.is_ff_split++;
+		m_freem(m);			/* NB: must reclaim */
+		return NULL;
+	}
+	/* XXX not right for WDS */
+	vap->iv_deliver_data(vap, ni, m);	/* 1st of pair */
+
+	/*
+	 * Decap second frame.
+	 */
+	m_adj(n, roundup2(framelen, 4) - framelen);	/* padding */
+	n = ieee80211_decap1(n, &framelen);
+	if (n == NULL) {
+		IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
+		    ni->ni_macaddr, "fast-frame", "%s", "second decap failed");
+		vap->iv_stats.is_ff_tooshort++;
+	}
+	/* XXX verify framelen against mbuf contents */
+	return n;				/* 2nd delivered by caller */
+#undef MS
+#undef FF_LLC_SIZE
+}
+
+/*
+ * Do Ethernet-LLC encapsulation for each payload in a fast frame
+ * tunnel encapsulation.  The frame is assumed to have an Ethernet
+ * header at the front that must be stripped before prepending the
+ * LLC followed by the Ethernet header passed in (with an Ethernet
+ * type that specifies the payload size).
+ */
+static struct mbuf *
+ff_encap1(struct ieee80211vap *vap, struct mbuf *m,
+	const struct ether_header *eh)
+{
+	struct llc *llc;
+	uint16_t payload;
+
+	/* XXX optimize by combining m_adj+M_PREPEND */
+	m_adj(m, sizeof(struct ether_header) - sizeof(struct llc));
+	llc = mtod(m, struct llc *);
+	llc->llc_dsap = llc->llc_ssap = LLC_SNAP_LSAP;
+	llc->llc_control = LLC_UI;
+	llc->llc_snap.org_code[0] = 0;
+	llc->llc_snap.org_code[1] = 0;
+	llc->llc_snap.org_code[2] = 0;
+	llc->llc_snap.ether_type = eh->ether_type;
+	payload = m->m_pkthdr.len;		/* NB: w/o Ethernet header */
+
+	M_PREPEND(m, sizeof(struct ether_header), M_DONTWAIT);
+	if (m == NULL) {		/* XXX cannot happen */
+		IEEE80211_DPRINTF(vap, IEEE80211_MSG_SUPERG,
+			"%s: no space for ether_header\n", __func__);
+		vap->iv_stats.is_tx_nobuf++;
+		return NULL;
+	}
+	ETHER_HEADER_COPY(mtod(m, void *), eh);
+	mtod(m, struct ether_header *)->ether_type = htons(payload);
+	return m;
+}
+
+/*
+ * Fast frame encapsulation.  There must be two packets
+ * chained with m_nextpkt.  We do header adjustment for
+ * each, add the tunnel encapsulation, and then concatenate
+ * the mbuf chains to form a single frame for transmission.
+ */
+struct mbuf *
+ieee80211_ff_encap(struct ieee80211vap *vap, struct mbuf *m1, int hdrspace,
+	struct ieee80211_key *key)
+{
+	struct mbuf *m2;
+	struct ether_header eh1, eh2;
+	struct llc *llc;
+	struct mbuf *m;
+	int pad;
+
+	m2 = m1->m_nextpkt;
+	if (m2 == NULL) {
+		IEEE80211_DPRINTF(vap, IEEE80211_MSG_SUPERG,
+		    "%s: only one frame\n", __func__);
+		goto bad;
+	}
+	m1->m_nextpkt = NULL;
+	/*
+	 * Include fast frame headers in adjusting header layout.
+	 */
+	KASSERT(m1->m_len >= sizeof(eh1), ("no ethernet header!"));
+	ETHER_HEADER_COPY(&eh1, mtod(m1, caddr_t));
+	m1 = ieee80211_mbuf_adjust(vap,
+		hdrspace + sizeof(struct llc) + sizeof(uint32_t) + 2 +
+		    sizeof(struct ether_header),
+		key, m1);
+	if (m1 == NULL) {
+		/* NB: ieee80211_mbuf_adjust handles msgs+statistics */
+		m_freem(m2);
+		goto bad;
+	}
+
+	/*
+	 * Copy second frame's Ethernet header out of line
+	 * and adjust for encapsulation headers.  Note that
+	 * we make room for padding in case there isn't room
+	 * at the end of first frame.
+	 */
+	KASSERT(m2->m_len >= sizeof(eh2), ("no ethernet header!"));
+	ETHER_HEADER_COPY(&eh2, mtod(m2, caddr_t));
+	m2 = ieee80211_mbuf_adjust(vap,
+		ATH_FF_MAX_HDR_PAD + sizeof(struct ether_header),
+		NULL, m2);
+	if (m2 == NULL) {
+		/* NB: ieee80211_mbuf_adjust handles msgs+statistics */
+		goto bad;
+	}
+
+	/*
+	 * Now do tunnel encapsulation.  First, each
+	 * frame gets a standard encapsulation.
+	 */
+	m1 = ff_encap1(vap, m1, &eh1);
+	if (m1 == NULL)
+		goto bad;
+	m2 = ff_encap1(vap, m2, &eh2);
+	if (m2 == NULL)
+		goto bad;
+
+	/*
+	 * Pad leading frame to a 4-byte boundary.  If there
+	 * is space at the end of the first frame, put it
+	 * there; otherwise prepend to the front of the second
+	 * frame.  We know doing the second will always work
+	 * because we reserve space above.  We prefer appending
+	 * as this typically has better DMA alignment properties.
+	 */
+	for (m = m1; m->m_next != NULL; m = m->m_next)
+		;
+	pad = roundup2(m1->m_pkthdr.len, 4) - m1->m_pkthdr.len;
+	if (pad) {
+		if (M_TRAILINGSPACE(m) < pad) {		/* prepend to second */
+			m2->m_data -= pad;
+			m2->m_len += pad;
+			m2->m_pkthdr.len += pad;
+		} else {				/* append to first */
+			m->m_len += pad;
+			m1->m_pkthdr.len += pad;
+		}
+	}
+
+	/*
+	 * Now, stick 'em together and prepend the tunnel headers;
+	 * first the Atheros tunnel header (all zero for now) and
+	 * then a special fast frame LLC.
+	 *
+	 * XXX optimize by prepending together
+	 */
+	m->m_next = m2;			/* NB: last mbuf from above */
+	m1->m_pkthdr.len += m2->m_pkthdr.len;
+	M_PREPEND(m1, sizeof(uint32_t)+2, M_DONTWAIT);
+	if (m1 == NULL) {		/* XXX cannot happen */
+		IEEE80211_DPRINTF(vap, IEEE80211_MSG_SUPERG,
+		    "%s: no space for tunnel header\n", __func__);
+		vap->iv_stats.is_tx_nobuf++;
+		return NULL;
+	}
+	memset(mtod(m1, void *), 0, sizeof(uint32_t)+2);
+
+	M_PREPEND(m1, sizeof(struct llc), M_DONTWAIT);
+	if (m1 == NULL) {		/* XXX cannot happen */
+		IEEE80211_DPRINTF(vap, IEEE80211_MSG_SUPERG,
+		    "%s: no space for llc header\n", __func__);
+		vap->iv_stats.is_tx_nobuf++;
+		return NULL;
+	}
+	llc = mtod(m1, struct llc *);
+	llc->llc_dsap = llc->llc_ssap = LLC_SNAP_LSAP;
+	llc->llc_control = LLC_UI;
+	llc->llc_snap.org_code[0] = ATH_FF_SNAP_ORGCODE_0;
+	llc->llc_snap.org_code[1] = ATH_FF_SNAP_ORGCODE_1;
+	llc->llc_snap.org_code[2] = ATH_FF_SNAP_ORGCODE_2;
+	llc->llc_snap.ether_type = htons(ATH_FF_ETH_TYPE);
+
+	vap->iv_stats.is_ff_encap++;
+
+	return m1;
+bad:
+	if (m1 != NULL)
+		m_freem(m1);
+	if (m2 != NULL)
+		m_freem(m2);
+	return NULL;
+}
+
+static void
+ff_transmit(struct ieee80211_node *ni, struct mbuf *m)
+{
+	struct ieee80211vap *vap = ni->ni_vap;
+	int error;
+
+	/* encap and xmit */
+	m = ieee80211_encap(vap, ni, m);
+	if (m != NULL) {
+		struct ifnet *ifp = vap->iv_ifp;
+		struct ifnet *parent = ni->ni_ic->ic_ifp;
+
+		error = parent->if_transmit(parent, m);
+		if (error != 0) {
+			/* NB: IFQ_HANDOFF reclaims mbuf */
+			ieee80211_free_node(ni);
+		} else {
+			ifp->if_opackets++;
+		}
+	} else
+		ieee80211_free_node(ni);
+}
+
+/*
+ * Flush frames to device; note we re-use the linked list
+ * the frames were stored on and use the sentinel (unchanged)
+ * which may be non-NULL.
+ */
+static void
+ff_flush(struct mbuf *head, struct mbuf *last)
+{
+	struct mbuf *m, *next;
+	struct ieee80211_node *ni;
+	struct ieee80211vap *vap;
+
+	for (m = head; m != last; m = next) {
+		next = m->m_nextpkt;
+		m->m_nextpkt = NULL;
+
+		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
+		vap = ni->ni_vap;
+
+		IEEE80211_NOTE(vap, IEEE80211_MSG_SUPERG, ni,
+		    "%s: flush frame, age %u", __func__, M_AGE_GET(m));
+		vap->iv_stats.is_ff_flush++;
+
+		ff_transmit(ni, m);
+	}
+}
+
+/*
+ * Age frames on the staging queue.
+ */
+void
+ieee80211_ff_age(struct ieee80211com *ic, struct ieee80211_stageq *sq, int quanta)
+{
+	struct mbuf *m, *head;
+	struct ieee80211_node *ni;
+	struct ieee80211_tx_ampdu *tap;
+
+	KASSERT(sq->head != NULL, ("stageq empty"));
+
+	IEEE80211_LOCK(ic);
+	head = sq->head;
+	while ((m = sq->head) != NULL && M_AGE_GET(m) < quanta) {
+		/* clear tap ref to frame */
+		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
+		tap = &ni->ni_tx_ampdu[M_WME_GETAC(m)];
+		KASSERT(tap->txa_private == m, ("staging queue empty"));
+		tap->txa_private = NULL;
+
+		sq->head = m->m_nextpkt;
+		sq->depth--;
+		ic->ic_stageqdepth--;
+	}
+	if (m == NULL)
+		sq->tail = NULL;
+	else
+		M_AGE_SUB(m, quanta);
+	IEEE80211_UNLOCK(ic);
+
+	ff_flush(head, m);
+}
+
+static void
+stageq_add(struct ieee80211_stageq *sq, struct mbuf *m)
+{
+	int age = ieee80211_ffagemax;
+	if (sq->tail != NULL) {
+		sq->tail->m_nextpkt = m;
+		age -= M_AGE_GET(sq->head);
+	} else
+		sq->head = m;
+	KASSERT(age >= 0, ("age %d", age));
+	M_AGE_SET(m, age);
+	m->m_nextpkt = NULL;
+	sq->tail = m;
+	sq->depth++;
+}
+
+static void
+stageq_remove(struct ieee80211_stageq *sq, struct mbuf *mstaged)
+{
+	struct mbuf *m, *mprev;
+
+	mprev = NULL;
+	for (m = sq->head; m != NULL; m = m->m_nextpkt) {
+		if (m == mstaged) {
+			if (mprev == NULL)
+				sq->head = m->m_nextpkt;
+			else
+				mprev->m_nextpkt = m->m_nextpkt;
+			if (sq->tail == m)
+				sq->tail = mprev;
+			sq->depth--;
+			return;
+		}
+		mprev = m;
+	}
+	printf("%s: packet not found\n", __func__);
+}
+
+static uint32_t
+ff_approx_txtime(struct ieee80211_node *ni,
+	const struct mbuf *m1, const struct mbuf *m2)
+{
+	struct ieee80211com *ic = ni->ni_ic;
+	struct ieee80211vap *vap = ni->ni_vap;
+	uint32_t framelen;
+
+	/*
+	 * Approximate the frame length to be transmitted. A swag to add
+	 * the following maximal values to the skb payload:
+	 *   - 32: 802.11 encap + CRC
+	 *   - 24: encryption overhead (if wep bit)
+	 *   - 4 + 6: fast-frame header and padding
+	 *   - 16: 2 LLC FF tunnel headers
+	 *   - 14: 1 802.3 FF tunnel header (mbuf already accounts for 2nd)
+	 */
+	framelen = m1->m_pkthdr.len + 32 +
+	    ATH_FF_MAX_HDR_PAD + ATH_FF_MAX_SEP_PAD + ATH_FF_MAX_HDR;
+	if (vap->iv_flags & IEEE80211_F_PRIVACY)
+		framelen += 24;
+	if (m2 != NULL)
+		framelen += m2->m_pkthdr.len;
+	return ieee80211_compute_duration(ic->ic_rt, framelen, ni->ni_txrate, 0);
+}
+
+/*
+ * Check if the supplied frame can be partnered with an existing
+ * or pending frame.  Return a reference to any frame that should be
+ * sent on return; otherwise return NULL.
+ */
+struct mbuf *
+ieee80211_ff_check(struct ieee80211_node *ni, struct mbuf *m)
+{
+	struct ieee80211vap *vap = ni->ni_vap;
+	struct ieee80211com *ic = ni->ni_ic;
+	const int pri = M_WME_GETAC(m);
+	struct ieee80211_stageq *sq;
+	struct ieee80211_tx_ampdu *tap;
+	struct mbuf *mstaged;
+	uint32_t txtime, limit;
+
+	/*
+	 * Check if the supplied frame can be aggregated.
+	 *
+	 * NB: we allow EAPOL frames to be aggregated with other ucast traffic.
+	 *     Do 802.1x EAPOL frames proceed in the clear? Then they couldn't
+	 *     be aggregated with other types of frames when encryption is on?
+	 */
+	IEEE80211_LOCK(ic);
+	tap = &ni->ni_tx_ampdu[pri];
+	mstaged = tap->txa_private;		/* NB: we reuse AMPDU state */
+	ieee80211_txampdu_count_packet(tap);
+
+	/*
+	 * When not in station mode never aggregate a multicast
+	 * frame; this insures, for example, that a combined frame
+	 * does not require multiple encryption keys.
+	 */
+	if (vap->iv_opmode != IEEE80211_M_STA &&
+	    ETHER_IS_MULTICAST(mtod(m, struct ether_header *)->ether_dhost)) {
+		/* XXX flush staged frame? */
+		IEEE80211_UNLOCK(ic);
+		return m;
+	}
+	/*
+	 * If there is no frame to combine with and the pps is
+	 * too low; then do not attempt to aggregate this frame.
+	 */
+	if (mstaged == NULL &&
+	    ieee80211_txampdu_getpps(tap) < ieee80211_ffppsmin) {
+		IEEE80211_UNLOCK(ic);
+		return m;
+	}
+	sq = &ic->ic_ff_stageq[pri];
+	/*
+	 * Check the txop limit to insure the aggregate fits.
+	 */
+	limit = IEEE80211_TXOP_TO_US(
+		ic->ic_wme.wme_chanParams.cap_wmeParams[pri].wmep_txopLimit);
+	if (limit != 0 &&
+	    (txtime = ff_approx_txtime(ni, m, mstaged)) > limit) {
+		/*
+		 * Aggregate too long, return to the caller for direct
+		 * transmission.  In addition, flush any pending frame
+		 * before sending this one.
+		 */
+		IEEE80211_DPRINTF(vap, IEEE80211_MSG_SUPERG,
+		    "%s: txtime %u exceeds txop limit %u\n",
+		    __func__, txtime, limit);
+
+		tap->txa_private = NULL;
+		if (mstaged != NULL)
+			stageq_remove(sq, mstaged);
+		IEEE80211_UNLOCK(ic);
+
+		if (mstaged != NULL) {
+			IEEE80211_NOTE(vap, IEEE80211_MSG_SUPERG, ni,
+			    "%s: flush staged frame", __func__);
+			/* encap and xmit */
+			ff_transmit(ni, mstaged);
+		}
+		return m;		/* NB: original frame */
+	}
+	/*
+	 * An aggregation candidate.  If there's a frame to partner
+	 * with then combine and return for processing.  Otherwise
+	 * save this frame and wait for a partner to show up (or
+	 * the frame to be flushed).  Note that staged frames also
+	 * hold their node reference.
+	 */
+	if (mstaged != NULL) {
+		tap->txa_private = NULL;
+		stageq_remove(sq, mstaged);
+		IEEE80211_UNLOCK(ic);
+
+		IEEE80211_NOTE(vap, IEEE80211_MSG_SUPERG, ni,
+		    "%s: aggregate fast-frame", __func__);
+		/*
+		 * Release the node reference; we only need
+		 * the one already in mstaged.
+		 */
+		KASSERT(mstaged->m_pkthdr.rcvif == (void *)ni,
+		    ("rcvif %p ni %p", mstaged->m_pkthdr.rcvif, ni));
+		ieee80211_free_node(ni);
+
+		m->m_nextpkt = NULL;
+		mstaged->m_nextpkt = m;
+		mstaged->m_flags |= M_FF; /* NB: mark for encap work */
+	} else {
+		m->m_pkthdr.rcvif = (void *)ni;	/* NB: hold node reference */
+
+		KASSERT(tap->txa_private == NULL,
+		    ("txa_private %p", tap->txa_private));
+		tap->txa_private = m;
+
+		stageq_add(sq, m);
+		ic->ic_stageqdepth++;
+		IEEE80211_UNLOCK(ic);
+
+		IEEE80211_NOTE(vap, IEEE80211_MSG_SUPERG, ni,
+		    "%s: stage frame, %u queued", __func__, sq->depth);
+		/* NB: mstaged is NULL */
+	}
+	return mstaged;
+}
+
+void
+ieee80211_ff_node_init(struct ieee80211_node *ni)
+{
+	/*
+	 * Clean FF state on re-associate.  This handles the case
+	 * where a station leaves w/o notifying us and then returns
+	 * before node is reaped for inactivity.
+	 */
+	ieee80211_ff_node_cleanup(ni);
+}
+
+void
+ieee80211_ff_node_cleanup(struct ieee80211_node *ni)
+{
+	struct ieee80211com *ic = ni->ni_ic;
+	struct ieee80211_tx_ampdu *tap;
+	struct mbuf *m, *head;
+	int ac;
+
+	IEEE80211_LOCK(ic);
+	head = NULL;
+	for (ac = 0; ac < WME_NUM_AC; ac++) {
+		tap = &ni->ni_tx_ampdu[ac];
+		m = tap->txa_private;
+		if (m != NULL) {
+			tap->txa_private = NULL;
+			stageq_remove(&ic->ic_ff_stageq[ac], m);
+			m->m_nextpkt = head;
+			head = m;
+		}
+	}
+	IEEE80211_UNLOCK(ic);
+
+	for (m = head; m != NULL; m = m->m_nextpkt) {
+		m_freem(m);
+		ieee80211_free_node(ni);
+	}
+}
+
+/*
+ * Switch between turbo and non-turbo operating modes.
+ * Use the specified channel flags to locate the new
+ * channel, update 802.11 state, and then call back into
+ * the driver to effect the change.
+ */
+void
+ieee80211_dturbo_switch(struct ieee80211vap *vap, int newflags)
+{
+	struct ieee80211com *ic = vap->iv_ic;
+	struct ieee80211_channel *chan;
+
+	chan = ieee80211_find_channel(ic, ic->ic_bsschan->ic_freq, newflags);
+	if (chan == NULL) {		/* XXX should not happen */
+		IEEE80211_DPRINTF(vap, IEEE80211_MSG_SUPERG,
+		    "%s: no channel with freq %u flags 0x%x\n",
+		    __func__, ic->ic_bsschan->ic_freq, newflags);
+		return;
+	}
+
+	IEEE80211_DPRINTF(vap, IEEE80211_MSG_SUPERG,
+	    "%s: %s -> %s (freq %u flags 0x%x)\n", __func__,
+	    ieee80211_phymode_name[ieee80211_chan2mode(ic->ic_bsschan)],
+	    ieee80211_phymode_name[ieee80211_chan2mode(chan)],
+	    chan->ic_freq, chan->ic_flags);
+
+	ic->ic_bsschan = chan;
+	ic->ic_prevchan = ic->ic_curchan;
+	ic->ic_curchan = chan;
+	ic->ic_rt = ieee80211_get_ratetable(chan);
+	ic->ic_set_channel(ic);
+	/* NB: do not need to reset ERP state 'cuz we're in sta mode */
+}
+
+/*
+ * Return the current ``state'' of an Atheros capbility.
+ * If associated in station mode report the negotiated
+ * setting. Otherwise report the current setting.
+ */
+static int
+getathcap(struct ieee80211vap *vap, int cap)
+{
+	if (vap->iv_opmode == IEEE80211_M_STA &&
+	    vap->iv_state == IEEE80211_S_RUN)
+		return IEEE80211_ATH_CAP(vap, vap->iv_bss, cap) != 0;
+	else
+		return (vap->iv_flags & cap) != 0;
+}
+
+static int
+superg_ioctl_get80211(struct ieee80211vap *vap, struct ieee80211req *ireq)
+{
+	switch (ireq->i_type) {
+	case IEEE80211_IOC_FF:
+		ireq->i_val = getathcap(vap, IEEE80211_F_FF);
+		break;
+	case IEEE80211_IOC_TURBOP:
+		ireq->i_val = getathcap(vap, IEEE80211_F_TURBOP);
+		break;
+	default:
+		return ENOSYS;
+	}
+	return 0;
+}
+IEEE80211_IOCTL_GET(superg, superg_ioctl_get80211);
+
+static int
+superg_ioctl_set80211(struct ieee80211vap *vap, struct ieee80211req *ireq)
+{
+	switch (ireq->i_type) {
+	case IEEE80211_IOC_FF:
+		if (ireq->i_val) {
+			if ((vap->iv_caps & IEEE80211_C_FF) == 0)
+				return EOPNOTSUPP;
+			vap->iv_flags |= IEEE80211_F_FF;
+		} else
+			vap->iv_flags &= ~IEEE80211_F_FF;
+		return ENETRESET;
+	case IEEE80211_IOC_TURBOP:
+		if (ireq->i_val) {
+			if ((vap->iv_caps & IEEE80211_C_TURBOP) == 0)
+				return EOPNOTSUPP;
+			vap->iv_flags |= IEEE80211_F_TURBOP;
+		} else
+			vap->iv_flags &= ~IEEE80211_F_TURBOP;
+		return ENETRESET;
+	default:
+		return ENOSYS;
+	}
+	return 0;
+}
+IEEE80211_IOCTL_SET(superg, superg_ioctl_set80211);
diff --git a/sys/net80211/ieee80211_superg.h b/sys/net80211/ieee80211_superg.h
new file mode 100644
index 0000000..d627ab0
--- /dev/null
+++ b/sys/net80211/ieee80211_superg.h
@@ -0,0 +1,104 @@
+/*-
+ * Copyright (c) 2009 Sam Leffler, Errno Consulting
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+#ifndef _NET80211_IEEE80211_SUPERG_H_
+#define _NET80211_IEEE80211_SUPERG_H_
+
+/*
+ * Atheros' 802.11 SuperG protocol support.
+ */
+
+/*
+ * Atheros advanced capability information element.
+ */
+struct ieee80211_ath_ie {
+	uint8_t		ath_id;			/* IEEE80211_ELEMID_VENDOR */
+	uint8_t		ath_len;		/* length in bytes */
+	uint8_t		ath_oui[3];		/* ATH_OUI */
+	uint8_t		ath_oui_type;		/* ATH_OUI_TYPE */
+	uint8_t		ath_oui_subtype;	/* ATH_OUI_SUBTYPE */
+	uint8_t		ath_version;		/* spec revision */
+	uint8_t		ath_capability;		/* capability info */
+#define	ATHEROS_CAP_TURBO_PRIME		0x01	/* dynamic turbo--aka Turbo' */
+#define	ATHEROS_CAP_COMPRESSION		0x02	/* data compression */
+#define	ATHEROS_CAP_FAST_FRAME		0x04	/* fast (jumbo) frames */
+#define	ATHEROS_CAP_XR			0x08	/* Xtended Range support */
+#define	ATHEROS_CAP_AR			0x10	/* Advanded Radar support */
+#define	ATHEROS_CAP_BURST		0x20	/* Bursting - not negotiated */
+#define	ATHEROS_CAP_WME			0x40	/* CWMin tuning */
+#define	ATHEROS_CAP_BOOST		0x80	/* use turbo/!turbo mode */
+	uint8_t		ath_defkeyix[2];
+} __packed;
+
+#define	ATH_OUI_VERSION		0x00
+#define	ATH_OUI_SUBTYPE		0x01
+
+#ifdef _KERNEL
+void	ieee80211_superg_attach(struct ieee80211com *);
+void	ieee80211_superg_detach(struct ieee80211com *);
+void	ieee80211_superg_vattach(struct ieee80211vap *);
+void	ieee80211_superg_vdetach(struct ieee80211vap *);
+
+uint8_t *ieee80211_add_ath(uint8_t *, uint8_t, ieee80211_keyix);
+uint8_t *ieee80211_add_athcaps(uint8_t *, const struct ieee80211_node *);
+void	ieee80211_parse_ath(struct ieee80211_node *, uint8_t *);
+int	ieee80211_parse_athparams(struct ieee80211_node *, uint8_t *,
+	    const struct ieee80211_frame *);
+
+void	ieee80211_ff_node_init(struct ieee80211_node *);
+void	ieee80211_ff_node_cleanup(struct ieee80211_node *);
+
+struct mbuf *ieee80211_ff_check(struct ieee80211_node *, struct mbuf *);
+void	ieee80211_ff_age(struct ieee80211com *, struct ieee80211_stageq *, int);
+
+static __inline void
+ieee80211_flush_stageq(struct ieee80211com *ic, int ac)
+{
+	if (ic->ic_ff_stageq[ac].depth)
+		ieee80211_ff_age(ic, &ic->ic_ff_stageq[ac], 0x7fffffff);
+}
+
+static __inline void
+ieee80211_age_stageq(struct ieee80211com *ic, int ac, int quanta)
+{
+	if (ic->ic_ff_stageq[ac].depth)
+		ieee80211_ff_age(ic, &ic->ic_ff_stageq[ac], quanta);
+}
+
+struct mbuf *ieee80211_ff_encap(struct ieee80211vap *, struct mbuf *,
+	    int, struct ieee80211_key *);
+
+struct mbuf *ieee80211_ff_decap(struct ieee80211_node *, struct mbuf *);
+
+static __inline struct mbuf *
+ieee80211_decap_fastframe(struct ieee80211vap *vap, struct ieee80211_node *ni,
+    struct mbuf *m)
+{
+	return IEEE80211_ATH_CAP(vap, ni, IEEE80211_NODE_FF) ?
+	    ieee80211_ff_decap(ni, m) : m;
+}
+#endif /* _KERNEL */
+#endif /* _NET80211_IEEE80211_SUPERG_H_ */
diff --git a/sys/net80211/ieee80211_tdma.c b/sys/net80211/ieee80211_tdma.c
index 82ba29e..88c52e0 100644
--- a/sys/net80211/ieee80211_tdma.c
+++ b/sys/net80211/ieee80211_tdma.c
@@ -33,9 +33,9 @@ __FBSDID("$FreeBSD$");
  * IEEE 802.11 TDMA mode support.
  */
 #include "opt_inet.h"
+#include "opt_tdma.h"
 #include "opt_wlan.h"
 
-#ifdef IEEE80211_SUPPORT_TDMA
 #include <sys/param.h>
 #include <sys/systm.h> 
 #include <sys/mbuf.h>   
@@ -60,7 +60,6 @@ __FBSDID("$FreeBSD$");
 #include <net80211/ieee80211_tdma.h>
 #include <net80211/ieee80211_input.h>
 
-#include "opt_tdma.h"
 #ifndef TDMA_SLOTLEN_DEFAULT
 #define	TDMA_SLOTLEN_DEFAULT	10*1000		/* 10ms */
 #endif
@@ -95,6 +94,16 @@ __FBSDID("$FreeBSD$");
 #define	TDMA_TXRATE_11NG_DEFAULT	(4 | IEEE80211_RATE_MCS)
 #endif
 
+#define	TDMA_VERSION_VALID(_version) \
+	(TDMA_VERSION_V2 <= (_version) && (_version) <= TDMA_VERSION)
+#define	TDMA_SLOTCNT_VALID(_slotcnt) \
+	(2 <= (_slotcnt) && (_slotcnt) <= TDMA_MAXSLOTS)
+/* XXX magic constants */
+#define	TDMA_SLOTLEN_VALID(_slotlen) \
+	(2*100 <= (_slotlen) && (unsigned)(_slotlen) <= 0xfffff)
+/* XXX probably should set a max */
+#define	TDMA_BINTVAL_VALID(_bintval)	(1 <= (_bintval))
+
 static void tdma_vdetach(struct ieee80211vap *vap);
 static int tdma_newstate(struct ieee80211vap *, enum ieee80211_state, int);
 static void tdma_beacon_miss(struct ieee80211vap *vap);
@@ -142,6 +151,7 @@ ieee80211_tdma_vattach(struct ieee80211vap *vap)
 		return;
 	}
 	/* NB: default configuration is passive so no beacons */
+	ts->tdma_version = TDMA_VERSION;
 	ts->tdma_slotlen = TDMA_SLOTLEN_DEFAULT;
 	ts->tdma_slotcnt = TDMA_SLOTCNT_DEFAULT;
 	ts->tdma_bintval = TDMA_BINTVAL_DEFAULT;
@@ -371,71 +381,93 @@ tdma_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0,
  * a TDMA information element.  The sender's identity
  * is provided so we can track who our peer is.  If pickslot
  * is non-zero we scan the slot allocation state in the ie
- * locate a free slot for our use.
+ * to locate a free slot for our use.
  */
 static int
 tdma_update(struct ieee80211vap *vap, const struct ieee80211_tdma_param *tdma,
 	struct ieee80211_node *ni, int pickslot)
 {
 	struct ieee80211_tdma_state *ts = vap->iv_tdma;
-	int slotlen, slotcnt, slot, bintval;
+	int slot, slotlen, update;
 
 	KASSERT(vap->iv_caps & IEEE80211_C_TDMA,
 	     ("not a tdma vap, caps 0x%x", vap->iv_caps));
 
-	slotlen = le16toh(tdma->tdma_slotlen);
-	slotcnt = tdma->tdma_slotcnt;
-	bintval = tdma->tdma_bintval;
-
-	/* XXX rate-limit printf's */
-	if (!(2 <= slotcnt && slotcnt <= IEEE80211_TDMA_MAXSLOTS)) {
-		printf("%s: bogus slot cnt %u\n", __func__, slotcnt);
-		return 0;
-	}
-	/* XXX magic constants */
-	if (slotlen < 2 || slotlen > (0xfffff/100)) {
-		printf("%s: bogus slot len %u\n", __func__, slotlen);
-		return 0;
-	}
-	if (bintval < 1) {
-		printf("%s: bogus beacon interval %u\n", __func__, bintval);
-		return 0;
-	}
+	update = 0;
+	if (tdma->tdma_slotcnt != ts->tdma_slotcnt) {
+		if (!TDMA_SLOTCNT_VALID(tdma->tdma_slotcnt)) {
+			if (ppsratecheck(&ts->tdma_lastprint, &ts->tdma_fails, 1))
+				printf("%s: bad slot cnt %u\n",
+				    __func__, tdma->tdma_slotcnt);
+			return 0;
+		}
+		update |= TDMA_UPDATE_SLOTCNT;
+ 	}
+	slotlen = le16toh(tdma->tdma_slotlen) * 100;
+	if (slotlen != ts->tdma_slotlen) {
+		if (!TDMA_SLOTLEN_VALID(slotlen)) {
+			if (ppsratecheck(&ts->tdma_lastprint, &ts->tdma_fails, 1))
+				printf("%s: bad slot len %u\n",
+				    __func__, slotlen);
+			return 0;
+		}
+		update |= TDMA_UPDATE_SLOTLEN;
+ 	}
+	if (tdma->tdma_bintval != ts->tdma_bintval) {
+		if (!TDMA_BINTVAL_VALID(tdma->tdma_bintval)) {
+			if (ppsratecheck(&ts->tdma_lastprint, &ts->tdma_fails, 1))
+				printf("%s: bad beacon interval %u\n",
+				    __func__, tdma->tdma_bintval);
+			return 0;
+		}
+		update |= TDMA_UPDATE_BINTVAL;
+ 	}
+	slot = ts->tdma_slot;
 	if (pickslot) {
 		/*
 		 * Pick unoccupied slot.  Note we never choose slot 0.
 		 */
-		for (slot = slotcnt-1; slot > 0; slot--)
+		for (slot = tdma->tdma_slotcnt-1; slot > 0; slot--)
 			if (isclr(tdma->tdma_inuse, slot))
 				break;
 		if (slot <= 0) {
 			printf("%s: no free slot, slotcnt %u inuse: 0x%x\n",
-				__func__, slotcnt, tdma->tdma_inuse[0]);
+				__func__, tdma->tdma_slotcnt,
+				tdma->tdma_inuse[0]);
 			/* XXX need to do something better */
 			return 0;
 		}
-	} else
-		slot = ts->tdma_slot;
+		if (slot != ts->tdma_slot)
+			update |= TDMA_UPDATE_SLOT;
+	}
+	if (ni != ts->tdma_peer) {
+		/* update everything */
+		update = TDMA_UPDATE_SLOT
+		       | TDMA_UPDATE_SLOTCNT
+		       | TDMA_UPDATE_SLOTLEN
+		       | TDMA_UPDATE_BINTVAL;
+	}
 
-	if (slotcnt != ts->tdma_slotcnt ||
-	    100*slotlen != ts->tdma_slotlen ||
-	    bintval != ts->tdma_bintval ||
-	    slot != ts->tdma_slot ||
-	    ts->tdma_peer != ni) {
+	if (update) {
 		/*
 		 * New/changed parameters; update runtime state.
 		 */
 		/* XXX overwrites user parameters */
-		ts->tdma_slotcnt = slotcnt;
-		ts->tdma_slotlen = 100*slotlen;
-		ts->tdma_slot = slot;
-		ts->tdma_bintval = bintval;
+		if (update & TDMA_UPDATE_SLOTCNT)
+			ts->tdma_slotcnt = tdma->tdma_slotcnt;
+		if (update & TDMA_UPDATE_SLOTLEN)
+			ts->tdma_slotlen = slotlen;
+		if (update & TDMA_UPDATE_SLOT)
+			ts->tdma_slot = slot;
+		if (update & TDMA_UPDATE_BINTVAL)
+			ts->tdma_bintval = tdma->tdma_bintval;
 		/* mark beacon to be updated before next xmit */
 		ieee80211_beacon_notify(vap, IEEE80211_BEACON_TDMA);
 
 		IEEE80211_DPRINTF(vap, IEEE80211_MSG_TDMA,
 		    "%s: slot %u slotcnt %u slotlen %u us bintval %u\n",
-		    __func__, slot, slotcnt, 100*slotlen, tdma->tdma_bintval);
+		    __func__, ts->tdma_slot, ts->tdma_slotcnt,
+		    100*ts->tdma_slotlen, ts->tdma_bintval);
 	}
 	/*
 	 * Notify driver.  Note we can be called before
@@ -445,7 +477,7 @@ tdma_update(struct ieee80211vap *vap, const struct ieee80211_tdma_param *tdma,
 	 * has been setup.  The next beacon will dtrt.
 	 */
 	if (vap->iv_state == IEEE80211_S_RUN)
-		vap->iv_ic->ic_tdma_update(ni, tdma);
+		vap->iv_ic->ic_tdma_update(ni, tdma, update);
 	/*
 	 * Dispatch join event on first beacon from new master.
 	 */
@@ -481,10 +513,23 @@ tdma_process_params(struct ieee80211_node *ni,
 		    wh, "tdma", "too short, len %u", len);
 		return IEEE80211_REASON_IE_INVALID;
 	}
-	if (tdma->tdma_version != TDMA_VERSION) {
+	if (tdma->tdma_version != ts->tdma_version) {
+		IEEE80211_DISCARD_IE(vap,
+		    IEEE80211_MSG_ELEMID | IEEE80211_MSG_TDMA,
+		    wh, "tdma", "bad version %u (ours %u)",
+		    tdma->tdma_version, ts->tdma_version);
+		return IEEE80211_REASON_IE_INVALID;
+	}
+ 	/*
+	 * NB: ideally we'd check against tdma_slotcnt, but that
+	 * would require extra effort so do this easy check that
+	 * covers the work below; more stringent checks are done
+	 * before we make more extensive use of the ie contents.
+	 */
+	if (tdma->tdma_slot >= TDMA_MAXSLOTS) {
 		IEEE80211_DISCARD_IE(vap,
 		    IEEE80211_MSG_ELEMID | IEEE80211_MSG_TDMA,
-		    wh, "tdma", "bad version %u", tdma->tdma_version);
+		    wh, "tdma", "invalid slot %u", tdma->tdma_slot);
 		return IEEE80211_REASON_IE_INVALID;
 	}
 	/*
@@ -612,7 +657,7 @@ ieee80211_add_tdma(uint8_t *frm, struct ieee80211vap *vap)
 		.tdma_subtype	= TDMA_SUBTYPE_PARAM,
 		.tdma_version	= TDMA_VERSION,
 	};
-	const struct ieee80211_tdma_state *tdma = vap->iv_tdma;
+	const struct ieee80211_tdma_state *ts = vap->iv_tdma;
 	uint16_t slotlen;
 
 	KASSERT(vap->iv_caps & IEEE80211_C_TDMA,
@@ -620,13 +665,13 @@ ieee80211_add_tdma(uint8_t *frm, struct ieee80211vap *vap)
 
 	memcpy(frm, &param, sizeof(param));
 	frm += __offsetof(struct ieee80211_tdma_param, tdma_slot);
-	*frm++ = tdma->tdma_slot;
-	*frm++ = tdma->tdma_slotcnt;
+	*frm++ = ts->tdma_slot;
+	*frm++ = ts->tdma_slotcnt;
 	/* NB: convert units to fit in 16-bits */
-	slotlen = tdma->tdma_slotlen / 100;	/* 100us units */
+	slotlen = ts->tdma_slotlen / 100;	/* 100us units */
 	ADDSHORT(frm, slotlen);
-	*frm++ = tdma->tdma_bintval;
-	*frm++ = tdma->tdma_inuse[0];
+	*frm++ = ts->tdma_bintval;
+	*frm++ = ts->tdma_inuse[0];
 	frm += 10;				/* pad+timestamp */
 	return frm; 
 #undef ADDSHORT
@@ -670,9 +715,8 @@ ieee80211_tdma_update_beacon(struct ieee80211vap *vap,
 		ts->tdma_count--;
 }
 
-int
-ieee80211_tdma_ioctl_get80211(struct ieee80211vap *vap,
-	struct ieee80211req *ireq)
+static int
+tdma_ioctl_get80211(struct ieee80211vap *vap, struct ieee80211req *ireq)
 {
 	struct ieee80211_tdma_state *ts = vap->iv_tdma;
 
@@ -693,14 +737,14 @@ ieee80211_tdma_ioctl_get80211(struct ieee80211vap *vap,
 		ireq->i_val = ts->tdma_bintval;
 		break;
 	default:
-		return EINVAL;
+		return ENOSYS;
 	}
 	return 0;
 }
+IEEE80211_IOCTL_GET(tdma, tdma_ioctl_get80211);
 
-int
-ieee80211_tdma_ioctl_set80211(struct ieee80211vap *vap,
-	struct ieee80211req *ireq)
+static int
+tdma_ioctl_set80211(struct ieee80211vap *vap, struct ieee80211req *ireq)
 {
 	struct ieee80211_tdma_state *ts = vap->iv_tdma;
 
@@ -717,8 +761,7 @@ ieee80211_tdma_ioctl_set80211(struct ieee80211vap *vap,
 		}
 		break;
 	case IEEE80211_IOC_TDMA_SLOTCNT:
-		if (!(2 <= ireq->i_val &&
-		      ireq->i_val <= IEEE80211_TDMA_MAXSLOTS))
+		if (!TDMA_SLOTCNT_VALID(ireq->i_val))
 			return EINVAL;
 		if (ireq->i_val != ts->tdma_slotcnt) {
 			ts->tdma_slotcnt = ireq->i_val;
@@ -732,7 +775,7 @@ ieee80211_tdma_ioctl_set80211(struct ieee80211vap *vap,
 		 * 0xfffff is the max duration for bursting
 		 * (implict by way of 16-bit data type for i_val)
 		 */
-		if (ireq->i_val < 150)
+		if (!TDMA_SLOTLEN_VALID(ireq->i_val))
 			return EINVAL;
 		if (ireq->i_val != ts->tdma_slotlen) {
 			ts->tdma_slotlen = ireq->i_val;
@@ -740,7 +783,7 @@ ieee80211_tdma_ioctl_set80211(struct ieee80211vap *vap,
 		}
 		break;
 	case IEEE80211_IOC_TDMA_BINTERVAL:
-		if (ireq->i_val < 1)
+		if (!TDMA_BINTVAL_VALID(ireq->i_val))
 			return EINVAL;
 		if (ireq->i_val != ts->tdma_bintval) {
 			ts->tdma_bintval = ireq->i_val;
@@ -748,8 +791,8 @@ ieee80211_tdma_ioctl_set80211(struct ieee80211vap *vap,
 		}
 		break;
 	default:
-		return EINVAL;
+		return ENOSYS;
 	}
 	return 0;
 }
-#endif /* IEEE80211_SUPPORT_TDMA */
+IEEE80211_IOCTL_SET(tdma, tdma_ioctl_set80211);
diff --git a/sys/net80211/ieee80211_tdma.h b/sys/net80211/ieee80211_tdma.h
index c5ac3a2..41ca09e 100644
--- a/sys/net80211/ieee80211_tdma.h
+++ b/sys/net80211/ieee80211_tdma.h
@@ -31,16 +31,51 @@
 /*
  * TDMA-mode implementation definitions.
  */
+
+#define	TDMA_SUBTYPE_PARAM	0x01
+#define	TDMA_VERSION_V2		2
+#define	TDMA_VERSION		TDMA_VERSION_V2
+
+/* NB: we only support 2 right now but protocol handles up to 8 */
+#define	TDMA_MAXSLOTS		2	/* max slots/sta's */
+
+#define	TDMA_PARAM_LEN_V2	sizeof(struct ieee80211_tdma_param)
+
+/*
+ * TDMA information element.
+ */
+struct ieee80211_tdma_param {
+	u_int8_t	tdma_id;	/* IEEE80211_ELEMID_VENDOR */
+	u_int8_t	tdma_len;
+	u_int8_t	tdma_oui[3];	/* TDMA_OUI */
+	u_int8_t	tdma_type;	/* TDMA_OUI_TYPE */
+	u_int8_t	tdma_subtype;	/* TDMA_SUBTYPE_PARAM */
+	u_int8_t	tdma_version;	/* spec revision */
+	u_int8_t	tdma_slot;	/* station slot # [0..7] */
+	u_int8_t	tdma_slotcnt;	/* bss slot count [1..8] */
+	u_int16_t	tdma_slotlen;	/* bss slot len (100us) */
+	u_int8_t	tdma_bintval;	/* beacon interval (superframes) */
+	u_int8_t	tdma_inuse[1];	/* slot occupancy map */
+	u_int8_t	tdma_pad[2];
+	u_int8_t	tdma_tstamp[8];	/* timestamp from last beacon */
+} __packed;
+
+#ifdef _KERNEL
+/*
+ * Implementation state.
+ */
 struct ieee80211_tdma_state {
 	u_int	tdma_slotlen;		/* bss slot length (us) */
+	uint8_t	tdma_version;		/* protocol version to use */
 	uint8_t	tdma_slotcnt;		/* bss slot count */
 	uint8_t	tdma_bintval;		/* beacon interval (slots) */
 	uint8_t	tdma_slot;		/* station slot # */
 	uint8_t	tdma_inuse[1];		/* mask of slots in use */
-#define	IEEE80211_TDMA_MAXSLOTS	8
-	void	*tdma_peer;		/* peer station cookie */
 	uint8_t	tdma_active[1];		/* mask of active slots */
 	int	tdma_count;		/* active/inuse countdown */
+	void	*tdma_peer;		/* peer station cookie */
+	struct timeval tdma_lastprint;	/* time of last rate-limited printf */
+	int	tdma_fails;		/* fail count for rate-limiting */
 
 	/* parent method pointers */
 	int	(*tdma_newstate)(struct ieee80211vap *, enum ieee80211_state,
@@ -49,6 +84,11 @@ struct ieee80211_tdma_state {
 		    struct mbuf *, int, int, int, uint32_t);
 	void	(*tdma_opdetach)(struct ieee80211vap *);
 };
+ 
+#define	TDMA_UPDATE_SLOT	0x0001	/* tdma_slot changed */
+#define	TDMA_UPDATE_SLOTCNT	0x0002	/* tdma_slotcnt changed */
+#define	TDMA_UPDATE_SLOTLEN	0x0004	/* tdma_slotlen changed */
+#define	TDMA_UPDATE_BINTVAL	0x0008	/* tdma_bintval changed */
 
 void	ieee80211_tdma_vattach(struct ieee80211vap *);
 
@@ -58,9 +98,5 @@ uint8_t *ieee80211_add_tdma(uint8_t *frm, struct ieee80211vap *vap);
 struct ieee80211_beacon_offsets;
 void	ieee80211_tdma_update_beacon(struct ieee80211vap *vap,
 	    struct ieee80211_beacon_offsets *bo);
-struct ieee80211req;
-int	ieee80211_tdma_ioctl_get80211(struct ieee80211vap *vap,
-	    struct ieee80211req *ireq);
-int	ieee80211_tdma_ioctl_set80211(struct ieee80211vap *vap,
-	    struct ieee80211req *ireq);
+#endif /* _KERNEL */
 #endif /* !_NET80211_IEEE80211_TDMA_H_ */
diff --git a/sys/net80211/ieee80211_var.h b/sys/net80211/ieee80211_var.h
index ace2c2f..4705047 100644
--- a/sys/net80211/ieee80211_var.h
+++ b/sys/net80211/ieee80211_var.h
@@ -47,6 +47,7 @@
 #include <net80211/ieee80211_crypto.h>
 #include <net80211/ieee80211_dfs.h>
 #include <net80211/ieee80211_ioctl.h>		/* for ieee80211_stats */
+#include <net80211/ieee80211_phy.h>
 #include <net80211/ieee80211_power.h>
 #include <net80211/ieee80211_node.h>
 #include <net80211/ieee80211_proto.h>
@@ -106,6 +107,13 @@ struct ieee80211_appie {
 };
 
 struct ieee80211_tdma_param;
+struct ieee80211_rate_table;
+
+struct ieee80211_stageq {
+	struct mbuf		*head;		/* frames linked w/ m_nextpkt */
+	struct mbuf		*tail;		/* last frame in queue */
+	int			depth;		/* # items on head */
+};
 
 struct ieee80211com {
 	struct ifnet		*ic_ifp;	/* associated device */
@@ -115,7 +123,6 @@ struct ieee80211com {
 	enum ieee80211_phytype	ic_phytype;	/* XXX wrong for multi-mode */
 	enum ieee80211_opmode	ic_opmode;	/* operation mode */
 	struct ifmedia		ic_media;	/* interface media config */
-	uint8_t			ic_myaddr[IEEE80211_ADDR_LEN];
 	struct callout		ic_inact;	/* inactivity processing */
 	struct task		ic_parent_task;	/* deferred parent processing */
 
@@ -162,6 +169,7 @@ struct ieee80211com {
 	uint8_t			ic_chan_active[IEEE80211_CHAN_BYTES];
 	uint8_t			ic_chan_scan[IEEE80211_CHAN_BYTES];
 	struct ieee80211_channel *ic_curchan;	/* current channel */
+	const struct ieee80211_rate_table *ic_rt; /* table for ic_curchan */
 	struct ieee80211_channel *ic_bsschan;	/* bss channel */
 	struct ieee80211_channel *ic_prevchan;	/* previous channel */
 	struct ieee80211_regdomain ic_regdomain;/* regulatory data */
@@ -196,6 +204,10 @@ struct ieee80211com {
 	int			ic_lastnonerp;	/* last time non-ERP sta noted*/
 	int			ic_lastnonht;	/* last time non-HT sta noted */
 
+	/* fast-frames staging q */
+	struct ieee80211_stageq	ic_ff_stageq[WME_NUM_AC];
+	int			ic_stageqdepth;	/* cumulative depth */
+
 	/* virtual ap create/delete */
 	struct ieee80211vap*	(*ic_vap_create)(struct ieee80211com *,
 				    const char name[IFNAMSIZ], int unit,
@@ -229,7 +241,7 @@ struct ieee80211com {
 	void			(*ic_newassoc)(struct ieee80211_node *, int);
 	/* TDMA update notification */
 	void			(*ic_tdma_update)(struct ieee80211_node *,
-				    const struct ieee80211_tdma_param *);
+				    const struct ieee80211_tdma_param *, int);
 	/* node state management */
 	struct ieee80211_node*	(*ic_node_alloc)(struct ieee80211vap *,
 				    const uint8_t [IEEE80211_ADDR_LEN]);
@@ -520,8 +532,9 @@ MALLOC_DECLARE(M_80211_VAP);
 #define	IEEE80211_FVEN_BITS	"\20"
 
 /* ic_caps/iv_caps: device driver capabilities */
-/* 0x2f available */
+/* 0x2e available */
 #define	IEEE80211_C_STA		0x00000001	/* CAPABILITY: STA available */
+#define	IEEE80211_C_8023ENCAP	0x00000002	/* CAPABILITY: 802.3 encap */
 #define	IEEE80211_C_FF		0x00000040	/* CAPABILITY: ATH FF avail */
 #define	IEEE80211_C_TURBOP	0x00000080	/* CAPABILITY: ATH Turbo avail*/
 #define	IEEE80211_C_IBSS	0x00000100	/* CAPABILITY: IBSS available */
@@ -553,7 +566,7 @@ MALLOC_DECLARE(M_80211_VAP);
 	 IEEE80211_C_TDMA)
 
 #define	IEEE80211_C_BITS \
-	"\20\1STA\7FF\10TURBOP\11IBSS\12PMGT" \
+	"\20\1STA\002803ENCAP\7FF\10TURBOP\11IBSS\12PMGT" \
 	"\13HOSTAP\14AHDEMO\15SWRETRY\16TXPMGT\17SHSLOT\20SHPREAMBLE" \
 	"\21MONITOR\22DFS\30WPA1\31WPA2\32BURST\33WME\34WDS\36BGSCAN" \
 	"\37TXFRAG\40TDMA"
@@ -575,7 +588,8 @@ MALLOC_DECLARE(M_80211_VAP);
 	"\20\1LDPC\2CHWIDTH40\5GREENFIELD\6SHORTGI20\7SHORTGI40\10TXSTBC" \
 	"\21AMPDU\22AMSDU\23HT\24SMPS\25RIFS"
 
-void	ieee80211_ifattach(struct ieee80211com *);
+void	ieee80211_ifattach(struct ieee80211com *,
+		const uint8_t macaddr[IEEE80211_ADDR_LEN]);
 void	ieee80211_ifdetach(struct ieee80211com *);
 int	ieee80211_vap_setup(struct ieee80211com *, struct ieee80211vap *,
 		const char name[IFNAMSIZ], int unit, int opmode, int flags,
@@ -702,7 +716,7 @@ ieee80211_htchanflags(const struct ieee80211_channel *c)
 #define	IEEE80211_MSG_DOT1XSM	0x00010000	/* 802.1x state machine */
 #define	IEEE80211_MSG_RADIUS	0x00008000	/* 802.1x radius client */
 #define	IEEE80211_MSG_RADDUMP	0x00004000	/* dump 802.1x radius packets */
-#define	IEEE80211_MSG_RADKEYS	0x00002000	/* dump 802.1x keys */
+#define	IEEE80211_MSG_MESH	0x00002000	/* mesh networking */
 #define	IEEE80211_MSG_WPA	0x00001000	/* WPA/RSN protocol */
 #define	IEEE80211_MSG_ACL	0x00000800	/* ACL handling */
 #define	IEEE80211_MSG_WME	0x00000400	/* WME protocol */
diff --git a/sys/net80211/ieee80211_wds.c b/sys/net80211/ieee80211_wds.c
index 2501cb50..27b995a 100644
--- a/sys/net80211/ieee80211_wds.c
+++ b/sys/net80211/ieee80211_wds.c
@@ -57,6 +57,9 @@ __FBSDID("$FreeBSD$");
 #include <net80211/ieee80211_var.h>
 #include <net80211/ieee80211_wds.h>
 #include <net80211/ieee80211_input.h>
+#ifdef IEEE80211_SUPPORT_SUPERG
+#include <net80211/ieee80211_superg.h>
+#endif
 
 static void wds_vattach(struct ieee80211vap *);
 static int wds_newstate(struct ieee80211vap *, enum ieee80211_state, int);
@@ -247,9 +250,7 @@ ieee80211_dwds_mcast(struct ieee80211vap *vap0, struct mbuf *m)
 		if (ifp == m->m_pkthdr.rcvif)
 			continue;
 		/*
-		 * Duplicate the frame and send it.  We don't need
-		 * to classify or lookup the tx node; this was already
-		 * done by the caller so we can just re-use the info.
+		 * Duplicate the frame and send it.
 		 */
 		mcopy = m_copypacket(m, M_DONTWAIT);
 		if (mcopy == NULL) {
@@ -264,6 +265,7 @@ ieee80211_dwds_mcast(struct ieee80211vap *vap0, struct mbuf *m)
 			m_freem(mcopy);
 			continue;
 		}
+		/* calculate priority so drivers can find the tx queue */
 		if (ieee80211_classify(ni, mcopy)) {
 			IEEE80211_DISCARD_MAC(vap,
 			    IEEE80211_MSG_OUTPUT | IEEE80211_MSG_WDS,
@@ -275,7 +277,16 @@ ieee80211_dwds_mcast(struct ieee80211vap *vap0, struct mbuf *m)
 			ieee80211_free_node(ni);
 			continue;
 		}
-		mcopy->m_flags |= M_MCAST | M_WDS;
+		/*
+		 * Encapsulate the packet in prep for transmission.
+		 */
+		mcopy = ieee80211_encap(vap, ni, mcopy);
+		if (m == NULL) {
+			/* NB: stat+msg handled in ieee80211_encap */
+			ieee80211_free_node(ni);
+			continue;
+		}
+		mcopy->m_flags |= M_MCAST;
 		mcopy->m_pkthdr.rcvif = (void *) ni;
 
 		err = parent->if_transmit(parent, mcopy);
@@ -728,32 +739,13 @@ wds_input(struct ieee80211_node *ni, struct mbuf *m,
 			m = ieee80211_decap_amsdu(ni, m);
 			if (m == NULL)
 				return IEEE80211_FC0_TYPE_DATA;
-		} else if (IEEE80211_ATH_CAP(vap, ni, IEEE80211_NODE_FF) &&
-#define	FF_LLC_SIZE	(sizeof(struct ether_header) + sizeof(struct llc))
-		    m->m_pkthdr.len >= 3*FF_LLC_SIZE) {
-			struct llc *llc;
-
-			/*
-			 * Check for fast-frame tunnel encapsulation.
-			 */
-			if (m->m_len < FF_LLC_SIZE &&
-			    (m = m_pullup(m, FF_LLC_SIZE)) == NULL) {
-				IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY,
-				    ni->ni_macaddr, "fast-frame",
-				    "%s", "m_pullup(llc) failed");
-				vap->iv_stats.is_rx_tooshort++;
+		} else {
+#ifdef IEEE80211_SUPPORT_SUPERG
+			m = ieee80211_decap_fastframe(vap, ni, m);
+			if (m == NULL)
 				return IEEE80211_FC0_TYPE_DATA;
-			}
-			llc = (struct llc *)(mtod(m, uint8_t *) + 
-				sizeof(struct ether_header));
-			if (llc->llc_snap.ether_type == htons(ATH_FF_ETH_TYPE)) {
-				m_adj(m, FF_LLC_SIZE);
-				m = ieee80211_decap_fastframe(ni, m);
-				if (m == NULL)
-					return IEEE80211_FC0_TYPE_DATA;
-			}
+#endif
 		}
-#undef FF_LLC_SIZE
 		ieee80211_deliver_data(vap, ni, m);
 		return IEEE80211_FC0_TYPE_DATA;
 
diff --git a/sys/netinet/icmp6.h b/sys/netinet/icmp6.h
index fcdc9ce..ad5f7bc 100644
--- a/sys/netinet/icmp6.h
+++ b/sys/netinet/icmp6.h
@@ -596,6 +596,11 @@ struct icmp6stat {
 	u_quad_t icp6s_badredirect;	/* bad redirect message */
 };
 
+#ifdef _KERNEL
+#define	ICMP6STAT_ADD(name, val)	V_icmp6stat.name += (val)
+#define	ICMP6STAT_INC(name)		ICMP6STAT_ADD(name, 1)
+#endif
+
 /*
  * Names for ICMP sysctl objects
  */
diff --git a/sys/netinet/icmp_var.h b/sys/netinet/icmp_var.h
index cd9ee00..8d7b9de 100644
--- a/sys/netinet/icmp_var.h
+++ b/sys/netinet/icmp_var.h
@@ -57,6 +57,11 @@ struct	icmpstat {
 	u_long	icps_noroute;		/* no route back */
 };
 
+#ifdef _KERNEL
+#define	ICMPSTAT_ADD(name, val)	V_icmpstat.name += (val)
+#define	ICMPSTAT_INC(name)	ICMPSTAT_ADD(name, 1)
+#endif
+
 /*
  * Names for ICMP sysctl objects
  */
diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c
index a918415..4aae7d3 100644
--- a/sys/netinet/if_ether.c
+++ b/sys/netinet/if_ether.c
@@ -111,6 +111,7 @@ SYSCTL_V_INT(V_NET, vnet_inet, _net_link_ether_inet, OID_AUTO, proxyall,
 	"Enable proxy ARP for all suitable requests");
 
 static void	arp_init(void);
+static int	arp_iattach(const void *);
 void		arprequest(struct ifnet *,
 			struct in_addr *, struct in_addr *, u_char *);
 static void	arpintr(struct mbuf *);
@@ -119,6 +120,15 @@ static void	arptimer(void *);
 static void	in_arpinput(struct mbuf *);
 #endif
 
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_arp_modinfo = {
+	.vmi_id		= VNET_MOD_ARP,
+	.vmi_name	= "arp",
+	.vmi_dependson	= VNET_MOD_INET,
+	.vmi_iattach	= arp_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
+
 #ifdef AF_INET
 void arp_ifscrub(struct ifnet *ifp, uint32_t addr);
 
@@ -790,8 +800,8 @@ arp_ifinit2(struct ifnet *ifp, struct ifaddr *ifa, u_char *enaddr)
 	ifa->ifa_rtrequest = NULL;
 }
 
-static void
-arp_init(void)
+static int
+arp_iattach(const void *unused __unused)
 {
 	INIT_VNET_INET(curvnet);
 
@@ -800,6 +810,19 @@ arp_init(void)
 	V_useloopback = 1; /* use loopback interface for local traffic */
 	V_arp_proxyall = 0;
 
+	return (0);
+}
+
+static void
+arp_init(void)
+{
+
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register(&vnet_arp_modinfo);
+#else
+	arp_iattach(NULL);
+#endif
+
 	arpintrq.ifq_maxlen = 50;
 	mtx_init(&arpintrq.ifq_mtx, "arp_inq", NULL, MTX_DEF);
 	netisr_register(NETISR_ARP, arpintr, &arpintrq, 0);
diff --git a/sys/netinet/igmp.c b/sys/netinet/igmp.c
index 31e8306..6f3286f 100644
--- a/sys/netinet/igmp.c
+++ b/sys/netinet/igmp.c
@@ -183,6 +183,11 @@ static int	vnet_igmp_idetach(const void *);
  * VIMAGE: Each in_multi corresponds to an ifp, and each ifp corresponds
  * to a vnet in ifp->if_vnet.
  *
+ * SMPng: XXX We may potentially race operations on ifma_protospec.
+ * The problem is that we currently lack a clean way of taking the
+ * IF_ADDR_LOCK() between the ifnet and in layers w/o recursing,
+ * as anything which modifies ifma needs to be covered by that lock.
+ * So check for ifma_protospec being NULL before proceeding.
  */
 struct mtx		 igmp_mtx;
 int			 mpsafe_igmp = 0;
@@ -601,6 +606,7 @@ out:
  * is detached, but also before the link layer does its cleanup.
  *
  * SMPNG: igmp_ifdetach() needs to take IF_ADDR_LOCK().
+ * XXX This is also bitten by unlocked ifma_protospec access.
  *
  * VIMAGE: curvnet should have been set by caller, but let's not assume
  * that for now.
@@ -623,8 +629,13 @@ igmp_ifdetach(struct ifnet *ifp)
 	if (igi->igi_version == IGMP_VERSION_3) {
 		IF_ADDR_LOCK(ifp);
 		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
-			if (ifma->ifma_addr->sa_family != AF_INET)
+			if (ifma->ifma_addr->sa_family != AF_INET ||
+			    ifma->ifma_protospec == NULL)
 				continue;
+#if 0
+			KASSERT(ifma->ifma_protospec != NULL,
+			    ("%s: ifma_protospec is NULL", __func__));
+#endif
 			inm = (struct in_multi *)ifma->ifma_protospec;
 			if (inm->inm_state == IGMP_LEAVING_MEMBER) {
 				SLIST_INSERT_HEAD(&igi->igi_relinmhead,
@@ -751,11 +762,11 @@ igmp_input_v1_query(struct ifnet *ifp, const struct ip *ip)
 	 * daemon may wish to see it.
 	 */
 	if (!in_allhosts(ip->ip_dst)) {
-		++V_igmpstat.igps_rcv_badqueries;
+		IGMPSTAT_INC(igps_rcv_badqueries);
 		return (0);
 	}
 
-	++V_igmpstat.igps_rcv_gen_queries;
+	IGMPSTAT_INC(igps_rcv_gen_queries);
 
 	/*
 	 * Switch to IGMPv1 host compatibility mode.
@@ -783,7 +794,8 @@ igmp_input_v1_query(struct ifnet *ifp, const struct ip *ip)
 	 */
 	IF_ADDR_LOCK(ifp);
 	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
-		if (ifma->ifma_addr->sa_family != AF_INET)
+		if (ifma->ifma_addr->sa_family != AF_INET ||
+		    ifma->ifma_protospec == NULL)
 			continue;
 		inm = (struct in_multi *)ifma->ifma_protospec;
 		if (inm->inm_timer != 0)
@@ -863,7 +875,7 @@ igmp_input_v2_query(struct ifnet *ifp, const struct ip *ip,
 			    inet_ntoa(igmp->igmp_group), ifp, ifp->if_xname);
 			igmp_v2_update_group(inm, timer);
 		}
-		++V_igmpstat.igps_rcv_group_queries;
+		IGMPSTAT_INC(igps_rcv_group_queries);
 	} else {
 		/*
 		 * IGMPv2 General Query.
@@ -880,14 +892,15 @@ igmp_input_v2_query(struct ifnet *ifp, const struct ip *ip,
 
 			IF_ADDR_LOCK(ifp);
 			TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
-				if (ifma->ifma_addr->sa_family != AF_INET)
+				if (ifma->ifma_addr->sa_family != AF_INET ||
+				    ifma->ifma_protospec == NULL)
 					continue;
 				inm = (struct in_multi *)ifma->ifma_protospec;
 				igmp_v2_update_group(inm, timer);
 			}
 			IF_ADDR_UNLOCK(ifp);
 		}
-		++V_igmpstat.igps_rcv_gen_queries;
+		IGMPSTAT_INC(igps_rcv_gen_queries);
 	}
 
 out_locked:
@@ -991,8 +1004,8 @@ igmp_input_v3_query(struct ifnet *ifp, const struct ip *ip,
 
 	qqi = igmpv3->igmp_qqi;
 	if (qqi >= 128) {
-		maxresp = IGMP_MANT(igmpv3->igmp_qqi) <<
-			  (IGMP_EXP(igmpv3->igmp_qqi) + 3);
+		qqi = IGMP_MANT(igmpv3->igmp_qqi) <<
+		     (IGMP_EXP(igmpv3->igmp_qqi) + 3);
 	}
 
 	timer = maxresp * PR_FASTHZ / IGMP_TIMER_SCALE;
@@ -1028,7 +1041,7 @@ igmp_input_v3_query(struct ifnet *ifp, const struct ip *ip,
 		 * Schedule a current-state report on this ifp for
 		 * all groups, possibly containing source lists.
 		 */
-		++V_igmpstat.igps_rcv_gen_queries;
+		IGMPSTAT_INC(igps_rcv_gen_queries);
 
 		if (!in_allhosts(ip->ip_dst) || nsrc > 0) {
 			/*
@@ -1036,7 +1049,7 @@ igmp_input_v3_query(struct ifnet *ifp, const struct ip *ip,
 			 * A general query with a source list has undefined
 			 * behaviour; discard it.
 			 */
-			++V_igmpstat.igps_rcv_badqueries;
+			IGMPSTAT_INC(igps_rcv_badqueries);
 			goto out_locked;
 		}
 
@@ -1066,16 +1079,16 @@ igmp_input_v3_query(struct ifnet *ifp, const struct ip *ip,
 		if (inm == NULL)
 			goto out_locked;
 		if (nsrc > 0) {
-			++V_igmpstat.igps_rcv_gsr_queries;
+			IGMPSTAT_INC(igps_rcv_gsr_queries);
 			if (!ratecheck(&inm->inm_lastgsrtv,
 			    &V_igmp_gsrdelay)) {
 				CTR1(KTR_IGMPV3, "%s: GS query throttled.",
 				    __func__);
-				++V_igmpstat.igps_drop_gsr_queries;
+				IGMPSTAT_INC(igps_drop_gsr_queries);
 				goto out_locked;
 			}
 		} else {
-			++V_igmpstat.igps_rcv_group_queries;
+			IGMPSTAT_INC(igps_rcv_group_queries);
 		}
 		CTR3(KTR_IGMPV3, "process v3 %s query on ifp %p(%s)",
 		     inet_ntoa(igmpv3->igmp_group), ifp, ifp->if_xname);
@@ -1211,14 +1224,14 @@ igmp_input_v1_report(struct ifnet *ifp, /*const*/ struct ip *ip,
 	struct in_ifaddr *ia;
 	struct in_multi *inm;
 
-	++V_igmpstat.igps_rcv_reports;
+	IGMPSTAT_INC(igps_rcv_reports);
 
 	if (ifp->if_flags & IFF_LOOPBACK)
 		return (0);
 
 	if (!IN_MULTICAST(ntohl(igmp->igmp_group.s_addr) ||
 	    !in_hosteq(igmp->igmp_group, ip->ip_dst))) {
-		++V_igmpstat.igps_rcv_badreports;
+		IGMPSTAT_INC(igps_rcv_badreports);
 		return (EINVAL);
 	}
 
@@ -1255,7 +1268,7 @@ igmp_input_v1_report(struct ifnet *ifp, /*const*/ struct ip *ip,
 			goto out_locked;
 		}
 
-		++V_igmpstat.igps_rcv_ourreports;
+		IGMPSTAT_INC(igps_rcv_ourreports);
 
 		/*
 		 * If we are in IGMPv3 host mode, do not allow the
@@ -1326,14 +1339,14 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip,
 	if (ia != NULL && in_hosteq(ip->ip_src, IA_SIN(ia)->sin_addr))
 		return (0);
 
-	++V_igmpstat.igps_rcv_reports;
+	IGMPSTAT_INC(igps_rcv_reports);
 
 	if (ifp->if_flags & IFF_LOOPBACK)
 		return (0);
 
 	if (!IN_MULTICAST(ntohl(igmp->igmp_group.s_addr)) ||
 	    !in_hosteq(igmp->igmp_group, ip->ip_dst)) {
-		++V_igmpstat.igps_rcv_badreports;
+		IGMPSTAT_INC(igps_rcv_badreports);
 		return (EINVAL);
 	}
 
@@ -1366,7 +1379,7 @@ igmp_input_v2_report(struct ifnet *ifp, /*const*/ struct ip *ip,
 		igi = inm->inm_igi;
 		KASSERT(igi != NULL, ("%s: no igi for ifp %p", __func__, ifp));
 
-		++V_igmpstat.igps_rcv_ourreports;
+		IGMPSTAT_INC(igps_rcv_ourreports);
 
 		/*
 		 * If we are in IGMPv3 host mode, do not allow the
@@ -1424,7 +1437,7 @@ igmp_input(struct mbuf *m, int off)
 	ifp = m->m_pkthdr.rcvif;
 	INIT_VNET_INET(ifp->if_vnet);
 
-	++V_igmpstat.igps_rcv_total;
+	IGMPSTAT_INC(igps_rcv_total);
 
 	ip = mtod(m, struct ip *);
 	iphlen = off;
@@ -1434,7 +1447,7 @@ igmp_input(struct mbuf *m, int off)
 	 * Validate lengths.
 	 */
 	if (igmplen < IGMP_MINLEN) {
-		++V_igmpstat.igps_rcv_tooshort;
+		IGMPSTAT_INC(igps_rcv_tooshort);
 		m_freem(m);
 		return;
 	}
@@ -1450,13 +1463,13 @@ igmp_input(struct mbuf *m, int off)
 		minlen += IGMP_MINLEN;
 	if ((m->m_flags & M_EXT || m->m_len < minlen) &&
 	    (m = m_pullup(m, minlen)) == 0) {
-		++V_igmpstat.igps_rcv_tooshort;
+		IGMPSTAT_INC(igps_rcv_tooshort);
 		return;
 	}
 	ip = mtod(m, struct ip *);
 
 	if (ip->ip_ttl != 1) {
-		++V_igmpstat.igps_rcv_badttl;
+		IGMPSTAT_INC(igps_rcv_badttl);
 		m_freem(m);
 		return;
 	}
@@ -1468,7 +1481,7 @@ igmp_input(struct mbuf *m, int off)
 	m->m_len -= iphlen;
 	igmp = mtod(m, struct igmp *);
 	if (in_cksum(m, igmplen)) {
-		++V_igmpstat.igps_rcv_badsum;
+		IGMPSTAT_INC(igps_rcv_badsum);
 		m_freem(m);
 		return;
 	}
@@ -1485,14 +1498,14 @@ igmp_input(struct mbuf *m, int off)
 		} else if (igmplen >= IGMP_V3_QUERY_MINLEN) {
 			queryver = IGMP_VERSION_3;
 		} else {
-			++V_igmpstat.igps_rcv_tooshort;
+			IGMPSTAT_INC(igps_rcv_tooshort);
 			m_freem(m);
 			return;
 		}
 
 		switch (queryver) {
 		case IGMP_VERSION_1:
-			++V_igmpstat.igps_rcv_v1v2_queries;
+			IGMPSTAT_INC(igps_rcv_v1v2_queries);
 			if (!V_igmp_v1enable)
 				break;
 			if (igmp_input_v1_query(ifp, ip) != 0) {
@@ -1502,7 +1515,7 @@ igmp_input(struct mbuf *m, int off)
 			break;
 
 		case IGMP_VERSION_2:
-			++V_igmpstat.igps_rcv_v1v2_queries;
+			IGMPSTAT_INC(igps_rcv_v1v2_queries);
 			if (!V_igmp_v2enable)
 				break;
 			if (igmp_input_v2_query(ifp, ip, igmp) != 0) {
@@ -1517,7 +1530,7 @@ igmp_input(struct mbuf *m, int off)
 				uint16_t srclen;
 				int nsrc;
 
-				++V_igmpstat.igps_rcv_v3_queries;
+				IGMPSTAT_INC(igps_rcv_v3_queries);
 				igmpv3 = (struct igmpv3 *)igmp;
 				/*
 				 * Validate length based on source count.
@@ -1525,7 +1538,7 @@ igmp_input(struct mbuf *m, int off)
 				nsrc = ntohs(igmpv3->igmp_numsrc);
 				srclen = sizeof(struct in_addr) * nsrc;
 				if (nsrc * sizeof(in_addr_t) > srclen) {
-					++V_igmpstat.igps_rcv_tooshort;
+					IGMPSTAT_INC(igps_rcv_tooshort);
 					return;
 				}
 				/*
@@ -1537,7 +1550,7 @@ igmp_input(struct mbuf *m, int off)
 				if ((m->m_flags & M_EXT ||
 				     m->m_len < igmpv3len) &&
 				    (m = m_pullup(m, igmpv3len)) == NULL) {
-					++V_igmpstat.igps_rcv_tooshort;
+					IGMPSTAT_INC(igps_rcv_tooshort);
 					return;
 				}
 				igmpv3 = (struct igmpv3 *)(mtod(m, uint8_t *)
@@ -1564,7 +1577,7 @@ igmp_input(struct mbuf *m, int off)
 		if (!V_igmp_v2enable)
 			break;
 		if (!ip_checkrouteralert(m))
-			++V_igmpstat.igps_rcv_nora;
+			IGMPSTAT_INC(igps_rcv_nora);
 		if (igmp_input_v2_report(ifp, ip, igmp) != 0) {
 			m_freem(m);
 			return;
@@ -1577,7 +1590,7 @@ igmp_input(struct mbuf *m, int off)
 		 * as report suppression is no longer required.
 		 */
 		if (!ip_checkrouteralert(m))
-			++V_igmpstat.igps_rcv_nora;
+			IGMPSTAT_INC(igps_rcv_nora);
 		break;
 
 	default:
@@ -1701,7 +1714,8 @@ igmp_fasttimo_vnet(void)
 		IF_ADDR_LOCK(ifp);
 		TAILQ_FOREACH_SAFE(ifma, &ifp->if_multiaddrs, ifma_link,
 		    tifma) {
-			if (ifma->ifma_addr->sa_family != AF_INET)
+			if (ifma->ifma_addr->sa_family != AF_INET ||
+			    ifma->ifma_protospec == NULL)
 				continue;
 			inm = (struct in_multi *)ifma->ifma_protospec;
 			switch (igi->igi_version) {
@@ -2770,8 +2784,11 @@ igmp_v3_enqueue_group_record(struct ifqueue *ifq, struct in_multi *inm,
 		m = NULL;
 		m0srcs = (ifp->if_mtu - IGMP_LEADINGSPACE -
 		    sizeof(struct igmp_grouprec)) / sizeof(in_addr_t);
-		if (!is_state_change && !is_group_query)
+		if (!is_state_change && !is_group_query) {
 			m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
+			if (m)
+				m->m_data += IGMP_LEADINGSPACE;
+		}
 		if (m == NULL) {
 			m = m_gethdr(M_DONTWAIT, MT_DATA);
 			if (m)
@@ -2779,7 +2796,6 @@ igmp_v3_enqueue_group_record(struct ifqueue *ifq, struct in_multi *inm,
 		}
 		if (m == NULL)
 			return (-ENOMEM);
-		m->m_data += IGMP_LEADINGSPACE;
 
 		igmp_save_context(m, ifp);
 
@@ -2895,6 +2911,8 @@ igmp_v3_enqueue_group_record(struct ifqueue *ifq, struct in_multi *inm,
 			return (-ENOMEM);
 		}
 		m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
+		if (m)
+			m->m_data += IGMP_LEADINGSPACE;
 		if (m == NULL) {
 			m = m_gethdr(M_DONTWAIT, MT_DATA);
 			if (m)
@@ -2903,7 +2921,6 @@ igmp_v3_enqueue_group_record(struct ifqueue *ifq, struct in_multi *inm,
 		if (m == NULL)
 			return (-ENOMEM);
 		igmp_save_context(m, ifp);
-		m->m_data += IGMP_LEADINGSPACE;
 		md = m_getptr(m, 0, &off);
 		pig = (struct igmp_grouprec *)(mtod(md, uint8_t *) + off);
 		CTR1(KTR_IGMPV3, "%s: allocated next packet", __func__);
@@ -3049,6 +3066,8 @@ igmp_v3_enqueue_filter_change(struct ifqueue *ifq, struct in_multi *inm)
 				    "%s: use previous packet", __func__);
 			} else {
 				m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
+				if (m)
+					m->m_data += IGMP_LEADINGSPACE;
 				if (m == NULL) {
 					m = m_gethdr(M_DONTWAIT, MT_DATA);
 					if (m)
@@ -3061,7 +3080,6 @@ igmp_v3_enqueue_filter_change(struct ifqueue *ifq, struct in_multi *inm)
 				}
 				m->m_pkthdr.PH_vt.vt_nrecs = 0;
 				igmp_save_context(m, ifp);
-				m->m_data += IGMP_LEADINGSPACE;
 				m0srcs = (ifp->if_mtu - IGMP_LEADINGSPACE -
 				    sizeof(struct igmp_grouprec)) /
 				    sizeof(in_addr_t);
@@ -3311,7 +3329,8 @@ igmp_v3_dispatch_general_query(struct igmp_ifinfo *igi)
 
 	IF_ADDR_LOCK(ifp);
 	TAILQ_FOREACH_SAFE(ifma, &ifp->if_multiaddrs, ifma_link, tifma) {
-		if (ifma->ifma_addr->sa_family != AF_INET)
+		if (ifma->ifma_addr->sa_family != AF_INET ||
+		    ifma->ifma_protospec == NULL)
 			continue;
 
 		inm = (struct in_multi *)ifma->ifma_protospec;
@@ -3394,7 +3413,7 @@ igmp_intr(struct mbuf *m)
 		CTR3(KTR_IGMPV3, "%s: dropped %p as ifindex %u went away.",
 		    __func__, m, ifindex);
 		m_freem(m);
-		V_ipstat.ips_noroute++;
+		IPSTAT_INC(ips_noroute);
 		goto out;
 	}
 
@@ -3422,7 +3441,7 @@ igmp_intr(struct mbuf *m)
 		if (m0 == NULL) {
 			CTR2(KTR_IGMPV3, "%s: dropped %p", __func__, m);
 			m_freem(m);
-			V_ipstat.ips_odropped++;
+			IPSTAT_INC(ips_odropped);
 			goto out;
 		}
 	}
@@ -3436,11 +3455,10 @@ igmp_intr(struct mbuf *m)
 	error = ip_output(m0, ipopts, NULL, 0, &imo, NULL);
 	if (error) {
 		CTR3(KTR_IGMPV3, "%s: ip_output(%p) = %d", __func__, m0, error);
-		m_freem(m0);
 		goto out;
 	}
 
-	++V_igmpstat.igps_snd_reports;
+	IGMPSTAT_INC(igps_snd_reports);
 
 out:
 	/*
diff --git a/sys/netinet/igmp.h b/sys/netinet/igmp.h
index 186142d..ab5d4cf 100644
--- a/sys/netinet/igmp.h
+++ b/sys/netinet/igmp.h
@@ -108,7 +108,7 @@ struct igmp_report {
 #define IGMP_HOST_MEMBERSHIP_QUERY	0x11	/* membership query         */
 #define IGMP_v1_HOST_MEMBERSHIP_REPORT	0x12	/* Ver. 1 membership report */
 #define IGMP_DVMRP			0x13	/* DVMRP routing message    */
-#define IGMP_PIM			0x14	/* PIM routing message     */
+#define IGMP_PIM			0x14	/* PIMv1 message (historic) */
 #define IGMP_v2_HOST_MEMBERSHIP_REPORT	0x16	/* Ver. 2 membership report */
 #define IGMP_HOST_LEAVE_MESSAGE		0x17	/* Leave-group message     */
 #define IGMP_MTRACE_REPLY		0x1e	/* mtrace(8) reply */
diff --git a/sys/netinet/igmp_var.h b/sys/netinet/igmp_var.h
index 9c9c7d4..ca17158 100644
--- a/sys/netinet/igmp_var.h
+++ b/sys/netinet/igmp_var.h
@@ -106,6 +106,11 @@ struct igmpstat {
 #define IGPS_VERSION_3	3		/* as of FreeBSD 8.x */
 #define IGPS_VERSION3_LEN		168
 
+#ifdef _KERNEL
+#define	IGMPSTAT_ADD(name, val)		V_igmpstat.name += (val)
+#define	IGMPSTAT_INC(name)		IGMPSTAT_ADD(name, 1)
+#endif
+
 #ifdef CTASSERT
 CTASSERT(sizeof(struct igmpstat) == 168);
 #endif
diff --git a/sys/netinet/in.c b/sys/netinet/in.c
index e2834bf..1423e81 100644
--- a/sys/netinet/in.c
+++ b/sys/netinet/in.c
@@ -397,10 +397,8 @@ in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp,
 		oldaddr = ia->ia_dstaddr;
 		ia->ia_dstaddr = *(struct sockaddr_in *)&ifr->ifr_dstaddr;
 		if (ifp->if_ioctl != NULL) {
-			IFF_LOCKGIANT(ifp);
 			error = (*ifp->if_ioctl)(ifp, SIOCSIFDSTADDR,
 			    (caddr_t)ia);
-			IFF_UNLOCKGIANT(ifp);
 			if (error) {
 				ia->ia_dstaddr = oldaddr;
 				return (error);
@@ -507,10 +505,7 @@ in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp,
 	default:
 		if (ifp == NULL || ifp->if_ioctl == NULL)
 			return (EOPNOTSUPP);
-		IFF_LOCKGIANT(ifp);
-		error = (*ifp->if_ioctl)(ifp, cmd, data);
-		IFF_UNLOCKGIANT(ifp);
-		return (error);
+		return ((*ifp->if_ioctl)(ifp, cmd, data));
 	}
 
 	/*
@@ -531,7 +526,6 @@ in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp,
 		IFP_TO_IA(ifp, oia);
 		if (oia == NULL) {
 			ii = ((struct in_ifinfo *)ifp->if_afdata[AF_INET]);
-			IFF_LOCKGIANT(ifp);
 			IN_MULTI_LOCK();
 			if (ii->ii_allhosts) {
 				(void)in_leavegroup_locked(ii->ii_allhosts,
@@ -539,7 +533,6 @@ in_control(struct socket *so, u_long cmd, caddr_t data, struct ifnet *ifp,
 				ii->ii_allhosts = NULL;
 			}
 			IN_MULTI_UNLOCK();
-			IFF_UNLOCKGIANT(ifp);
 		}
 	}
 	IFAFREE(&ia->ia_ifa);
@@ -753,9 +746,7 @@ in_ifinit(struct ifnet *ifp, struct in_ifaddr *ia, struct sockaddr_in *sin,
 	 * and to validate the address if necessary.
 	 */
 	if (ifp->if_ioctl != NULL) {
-		IFF_LOCKGIANT(ifp);
 		error = (*ifp->if_ioctl)(ifp, SIOCSIFADDR, (caddr_t)ia);
-		IFF_UNLOCKGIANT(ifp);
 		if (error) {
 			splx(s);
 			/* LIST_REMOVE(ia, ia_hash) is done in in_control */
@@ -1020,6 +1011,8 @@ in_ifdetach(struct ifnet *ifp)
  * Delete all IPv4 multicast address records, and associated link-layer
  * multicast address records, associated with ifp.
  * XXX It looks like domifdetach runs AFTER the link layer cleanup.
+ * XXX This should not race with ifma_protospec being set during
+ * a new allocation, if it does, we have bigger problems.
  */
 static void
 in_purgemaddrs(struct ifnet *ifp)
@@ -1040,8 +1033,13 @@ in_purgemaddrs(struct ifnet *ifp)
 	 */
 	IF_ADDR_LOCK(ifp);
 	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
-		if (ifma->ifma_addr->sa_family != AF_INET)
+		if (ifma->ifma_addr->sa_family != AF_INET ||
+		    ifma->ifma_protospec == NULL)
 			continue;
+#if 0
+		KASSERT(ifma->ifma_protospec != NULL,
+		    ("%s: ifma_protospec is NULL", __func__));
+#endif
 		inm = (struct in_multi *)ifma->ifma_protospec;
 		LIST_INSERT_HEAD(&purgeinms, inm, inm_link);
 	}
diff --git a/sys/netinet/in.h b/sys/netinet/in.h
index 4b2c827..dd302a5 100644
--- a/sys/netinet/in.h
+++ b/sys/netinet/in.h
@@ -120,7 +120,7 @@ struct sockaddr_in {
 	char	sin_zero[8];
 };
 
-#ifndef _KERNEL
+#if !defined(_KERNEL) && __BSD_VISIBLE
 
 #ifndef _BYTEORDER_PROTOTYPED
 #define	_BYTEORDER_PROTOTYPED
@@ -140,7 +140,7 @@ __END_DECLS
 #define	ntohs(x)	__ntohs(x)
 #endif
 
-#endif /* !_KERNEL */
+#endif /* !_KERNEL && __BSD_VISIBLE */
 
 #if __POSIX_VISIBLE >= 200112
 #define	IPPROTO_RAW		255		/* raw IP packet */
diff --git a/sys/netinet/in_gif.c b/sys/netinet/in_gif.c
index a05c04b..bc62f5f 100644
--- a/sys/netinet/in_gif.c
+++ b/sys/netinet/in_gif.c
@@ -273,14 +273,14 @@ in_gif_input(struct mbuf *m, int off)
 	sc = (struct gif_softc *)encap_getarg(m);
 	if (sc == NULL) {
 		m_freem(m);
-		V_ipstat.ips_nogif++;
+		IPSTAT_INC(ips_nogif);
 		return;
 	}
 
 	gifp = GIF2IFP(sc);
 	if (gifp == NULL || (gifp->if_flags & IFF_UP) == 0) {
 		m_freem(m);
-		V_ipstat.ips_nogif++;
+		IPSTAT_INC(ips_nogif);
 		return;
 	}
 
@@ -340,7 +340,7 @@ in_gif_input(struct mbuf *m, int off)
  		break;	
 
 	default:
-		V_ipstat.ips_nogif++;
+		IPSTAT_INC(ips_nogif);
 		m_freem(m);
 		return;
 	}
diff --git a/sys/netinet/in_mcast.c b/sys/netinet/in_mcast.c
index 0c7e7e1..9c04916 100644
--- a/sys/netinet/in_mcast.c
+++ b/sys/netinet/in_mcast.c
@@ -432,6 +432,9 @@ in_getmulti(struct ifnet *ifp, const struct in_addr *group,
 	if (error != 0)
 		return (error);
 
+	/* XXX ifma_protospec must be covered by IF_ADDR_LOCK */
+	IF_ADDR_LOCK(ifp);
+
 	/*
 	 * If something other than netinet is occupying the link-layer
 	 * group, print a meaningful error message and back out of
@@ -454,9 +457,12 @@ in_getmulti(struct ifnet *ifp, const struct in_addr *group,
 #endif
 		++inm->inm_refcount;
 		*pinm = inm;
+		IF_ADDR_UNLOCK(ifp);
 		return (0);
 	}
 
+	IF_ADDR_LOCK_ASSERT(ifp);
+
 	/*
 	 * A new in_multi record is needed; allocate and initialize it.
 	 * We DO NOT perform an IGMP join as the in_ layer may need to
@@ -467,6 +473,7 @@ in_getmulti(struct ifnet *ifp, const struct in_addr *group,
 	inm = malloc(sizeof(*inm), M_IPMADDR, M_NOWAIT | M_ZERO);
 	if (inm == NULL) {
 		if_delmulti_ifma(ifma);
+		IF_ADDR_UNLOCK(ifp);
 		return (ENOMEM);
 	}
 	inm->inm_addr = *group;
@@ -489,6 +496,7 @@ in_getmulti(struct ifnet *ifp, const struct in_addr *group,
 
 	*pinm = inm;
 
+	IF_ADDR_UNLOCK(ifp);
 	return (0);
 }
 
@@ -522,6 +530,7 @@ inm_release_locked(struct in_multi *inm)
 
 	ifma = inm->inm_ifma;
 
+	/* XXX this access is not covered by IF_ADDR_LOCK */
 	CTR2(KTR_IGMPV3, "%s: purging ifma %p", __func__, ifma);
 	KASSERT(ifma->ifma_protospec == inm,
 	    ("%s: ifma_protospec != inm", __func__));
@@ -1100,11 +1109,9 @@ in_joingroup(struct ifnet *ifp, const struct in_addr *gina,
 {
 	int error;
 
-	IFF_LOCKGIANT(ifp);
 	IN_MULTI_LOCK();
 	error = in_joingroup_locked(ifp, gina, imf, pinm);
 	IN_MULTI_UNLOCK();
-	IFF_UNLOCKGIANT(ifp);
 
 	return (error);
 }
@@ -1181,20 +1188,14 @@ int
 in_leavegroup(struct in_multi *inm, /*const*/ struct in_mfilter *imf)
 {
 	struct ifnet *ifp;
-	int detached, error;
+	int error;
 
-	detached = inm_is_ifp_detached(inm);
 	ifp = inm->inm_ifp;
-	if (!detached)
-		IFF_LOCKGIANT(ifp);
 
 	IN_MULTI_LOCK();
 	error = in_leavegroup_locked(inm, imf);
 	IN_MULTI_UNLOCK();
 
-	if (!detached)
-		IFF_UNLOCKGIANT(ifp);
-
 	return (error);
 }
 
@@ -1396,8 +1397,6 @@ inp_block_unblock_source(struct inpcb *inp, struct sockopt *sopt)
 	if (!IN_MULTICAST(ntohl(gsa->sin.sin_addr.s_addr)))
 		return (EINVAL);
 
-	IFF_LOCKGIANT(ifp);
-
 	/*
 	 * Check if we are actually a member of this group.
 	 */
@@ -1486,7 +1485,6 @@ out_imf_rollback:
 
 out_inp_locked:
 	INP_WUNLOCK(inp);
-	IFF_UNLOCKGIANT(ifp);
 	return (error);
 }
 
@@ -1978,8 +1976,6 @@ inp_join_group(struct inpcb *inp, struct sockopt *sopt)
 	if (ifp == NULL || (ifp->if_flags & IFF_MULTICAST) == 0)
 		return (EADDRNOTAVAIL);
 
-	IFF_LOCKGIANT(ifp);
-
 	/*
 	 * MCAST_JOIN_SOURCE on an exclusive membership is an error.
 	 * On an existing inclusive membership, it just adds the
@@ -2102,7 +2098,6 @@ out_imo_free:
 
 out_inp_locked:
 	INP_WUNLOCK(inp);
-	IFF_UNLOCKGIANT(ifp);
 	return (error);
 }
 
@@ -2215,9 +2210,6 @@ inp_leave_group(struct inpcb *inp, struct sockopt *sopt)
 	if (!IN_MULTICAST(ntohl(gsa->sin.sin_addr.s_addr)))
 		return (EINVAL);
 
-	if (ifp)
-		IFF_LOCKGIANT(ifp);
-
 	/*
 	 * Find the membership in the membership array.
 	 */
@@ -2312,8 +2304,6 @@ out_imf_rollback:
 
 out_inp_locked:
 	INP_WUNLOCK(inp);
-	if (ifp)
-		IFF_UNLOCKGIANT(ifp);
 	return (error);
 }
 
@@ -2432,8 +2422,6 @@ inp_set_source_filters(struct inpcb *inp, struct sockopt *sopt)
 	if (ifp == NULL)
 		return (EADDRNOTAVAIL);
 
-	IFF_LOCKGIANT(ifp);
-
 	/*
 	 * Take the INP write lock.
 	 * Check if this socket is a member of this group.
@@ -2551,7 +2539,6 @@ out_imf_rollback:
 
 out_inp_locked:
 	INP_WUNLOCK(inp);
-	IFF_UNLOCKGIANT(ifp);
 	return (error);
 }
 
@@ -2851,7 +2838,7 @@ inm_print(const struct in_multi *inm)
 {
 	int t;
 
-	if ((KTR_COMPILE & KTR_IGMPV3) == 0)
+	if ((ktr_mask & KTR_IGMPV3) == 0)
 		return;
 
 	printf("%s: --- begin inm %p ---\n", __func__, inm);
diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
index 6703c88..a10da084 100644
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -384,7 +384,7 @@ in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp,
 	 * This entire block sorely needs a rewrite.
 	 */
 				if (t &&
-				    ((t->inp_vflag & INP_TIMEWAIT) == 0) &&
+				    ((t->inp_flags & INP_TIMEWAIT) == 0) &&
 				    (so->so_type != SOCK_STREAM ||
 				     ntohl(t->inp_faddr.s_addr) == INADDR_ANY) &&
 				    (ntohl(sin->sin_addr.s_addr) != INADDR_ANY ||
@@ -397,7 +397,7 @@ in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp,
 			}
 			t = in_pcblookup_local(pcbinfo, sin->sin_addr,
 			    lport, wild, cred);
-			if (t && (t->inp_vflag & INP_TIMEWAIT)) {
+			if (t && (t->inp_flags & INP_TIMEWAIT)) {
 				/*
 				 * XXXRW: If an incpb has had its timewait
 				 * state recycled, we treat the address as
@@ -1031,7 +1031,7 @@ in_pcbdrop(struct inpcb *inp)
 	INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo);
 	INP_WLOCK_ASSERT(inp);
 
-	inp->inp_vflag |= INP_DROPPED;
+	inp->inp_flags |= INP_DROPPED;
 	if (inp->inp_flags & INP_INHASHLIST) {
 		struct inpcbport *phd = inp->inp_phd;
 
@@ -1818,6 +1818,22 @@ db_print_inpflags(int inp_flags)
 		db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : "");
 		comma = 1;
 	}
+	if (inp_flags & INP_TIMEWAIT) {
+		db_printf("%sINP_TIMEWAIT", comma ? ", " : "");
+		comma  = 1;
+	}
+	if (inp_flags & INP_ONESBCAST) {
+		db_printf("%sINP_ONESBCAST", comma ? ", " : "");
+		comma  = 1;
+	}
+	if (inp_flags & INP_DROPPED) {
+		db_printf("%sINP_DROPPED", comma ? ", " : "");
+		comma  = 1;
+	}
+	if (inp_flags & INP_SOCKREF) {
+		db_printf("%sINP_SOCKREF", comma ? ", " : "");
+		comma  = 1;
+	}
 	if (inp_flags & IN6P_RFC2292) {
 		db_printf("%sIN6P_RFC2292", comma ? ", " : "");
 		comma = 1;
@@ -1846,22 +1862,6 @@ db_print_inpvflag(u_char inp_vflag)
 		db_printf("%sINP_IPV6PROTO", comma ? ", " : "");
 		comma  = 1;
 	}
-	if (inp_vflag & INP_TIMEWAIT) {
-		db_printf("%sINP_TIMEWAIT", comma ? ", " : "");
-		comma  = 1;
-	}
-	if (inp_vflag & INP_ONESBCAST) {
-		db_printf("%sINP_ONESBCAST", comma ? ", " : "");
-		comma  = 1;
-	}
-	if (inp_vflag & INP_DROPPED) {
-		db_printf("%sINP_DROPPED", comma ? ", " : "");
-		comma  = 1;
-	}
-	if (inp_vflag & INP_SOCKREF) {
-		db_printf("%sINP_SOCKREF", comma ? ", " : "");
-		comma  = 1;
-	}
 }
 
 void
diff --git a/sys/netinet/in_pcb.h b/sys/netinet/in_pcb.h
index 677b974..aada126 100644
--- a/sys/netinet/in_pcb.h
+++ b/sys/netinet/in_pcb.h
@@ -167,7 +167,7 @@ struct inpcb {
 	u_char	inp_ip_ttl;		/* (i) time to live proto */
 	u_char	inp_ip_p;		/* (c) protocol proto */
 	u_char	inp_ip_minttl;		/* (i) minimum TTL or drop */
-	uint32_t inp_ispare1;		/* (x) connection id / queue id */
+	uint32_t inp_flowid;		/* (x) flow id / queue id */
 	u_int	inp_refcount;		/* (i) refcount */
 	void	*inp_pspare[2];		/* (x) rtentry / general use */
 
@@ -384,40 +384,40 @@ void 	inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
 #define	INP_IPV4	0x1
 #define	INP_IPV6	0x2
 #define	INP_IPV6PROTO	0x4		/* opened under IPv6 protocol */
-#define	INP_TIMEWAIT	0x8		/* inpcb in TIMEWAIT, ppcb is tcptw */
-#define	INP_ONESBCAST	0x10		/* send all-ones broadcast */
-#define	INP_DROPPED	0x20		/* protocol drop flag */
-#define	INP_SOCKREF	0x40		/* strong socket reference */
 
 /*
  * Flags for inp_flag.
  */
-#define	INP_RECVOPTS		0x01	/* receive incoming IP options */
-#define	INP_RECVRETOPTS		0x02	/* receive IP options for reply */
-#define	INP_RECVDSTADDR		0x04	/* receive IP dst address */
-#define	INP_HDRINCL		0x08	/* user supplies entire IP header */
-#define	INP_HIGHPORT		0x10	/* user wants "high" port binding */
-#define	INP_LOWPORT		0x20	/* user wants "low" port binding */
-#define	INP_ANONPORT		0x40	/* port chosen for user */
-#define	INP_RECVIF		0x80	/* receive incoming interface */
-#define	INP_MTUDISC		0x100	/* user can do MTU discovery */
-#define	INP_FAITH		0x200	/* accept FAITH'ed connections */
-#define	INP_RECVTTL		0x400	/* receive incoming IP TTL */
-#define	INP_DONTFRAG		0x800	/* don't fragment packet */
-#define	INP_NONLOCALOK		0x1000	/* Allow bind to spoof any address */
+#define	INP_RECVOPTS		0x00000001 /* receive incoming IP options */
+#define	INP_RECVRETOPTS		0x00000002 /* receive IP options for reply */
+#define	INP_RECVDSTADDR		0x00000004 /* receive IP dst address */
+#define	INP_HDRINCL		0x00000008 /* user supplies entire IP header */
+#define	INP_HIGHPORT		0x00000010 /* user wants "high" port binding */
+#define	INP_LOWPORT		0x00000020 /* user wants "low" port binding */
+#define	INP_ANONPORT		0x00000040 /* port chosen for user */
+#define	INP_RECVIF		0x00000080 /* receive incoming interface */
+#define	INP_MTUDISC		0x00000100 /* user can do MTU discovery */
+#define	INP_FAITH		0x00000200 /* accept FAITH'ed connections */
+#define	INP_RECVTTL		0x00000400 /* receive incoming IP TTL */
+#define	INP_DONTFRAG		0x00000800 /* don't fragment packet */
+#define	INP_NONLOCALOK		0x00001000 /* Allow bind to spoof any address */
 					/* - requires options IP_NONLOCALBIND */
-#define	INP_INHASHLIST		0x2000	/* in_pcbinshash() has been called */
-
-#define IN6P_IPV6_V6ONLY	0x008000 /* restrict AF_INET6 socket for v6 */
-
-#define	IN6P_PKTINFO		0x010000 /* receive IP6 dst and I/F */
-#define	IN6P_HOPLIMIT		0x020000 /* receive hoplimit */
-#define	IN6P_HOPOPTS		0x040000 /* receive hop-by-hop options */
-#define	IN6P_DSTOPTS		0x080000 /* receive dst options after rthdr */
-#define	IN6P_RTHDR		0x100000 /* receive routing header */
-#define	IN6P_RTHDRDSTOPTS	0x200000 /* receive dstoptions before rthdr */
-#define	IN6P_TCLASS		0x400000 /* receive traffic class value */
-#define	IN6P_AUTOFLOWLABEL	0x800000 /* attach flowlabel automatically */
+#define	INP_INHASHLIST		0x00002000 /* in_pcbinshash() has been called */
+#define	IN6P_IPV6_V6ONLY	0x00008000 /* restrict AF_INET6 socket for v6 */
+#define	IN6P_PKTINFO		0x00010000 /* receive IP6 dst and I/F */
+#define	IN6P_HOPLIMIT		0x00020000 /* receive hoplimit */
+#define	IN6P_HOPOPTS		0x00040000 /* receive hop-by-hop options */
+#define	IN6P_DSTOPTS		0x00080000 /* receive dst options after rthdr */
+#define	IN6P_RTHDR		0x00100000 /* receive routing header */
+#define	IN6P_RTHDRDSTOPTS	0x00200000 /* receive dstoptions before rthdr */
+#define	IN6P_TCLASS		0x00400000 /* receive traffic class value */
+#define	IN6P_AUTOFLOWLABEL	0x00800000 /* attach flowlabel automatically */
+#define	INP_TIMEWAIT		0x01000000 /* in TIMEWAIT, ppcb is tcptw */
+#define	INP_ONESBCAST		0x02000000 /* send all-ones broadcast */
+#define	INP_DROPPED		0x04000000 /* protocol drop flag */
+#define	INP_SOCKREF		0x08000000 /* strong socket reference */
+#define	INP_SW_FLOWID           0x10000000 /* software generated flow id */
+#define	INP_HW_FLOWID           0x20000000 /* hardware generated flow id */
 #define	IN6P_RFC2292		0x40000000 /* used RFC2292 API on the socket */
 #define	IN6P_MTU		0x80000000 /* receive path MTU */
 
diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c
index 6d96163..cc74580 100644
--- a/sys/netinet/ip_carp.c
+++ b/sys/netinet/ip_carp.c
@@ -530,7 +530,7 @@ carp_input(struct mbuf *m, int hlen)
 	struct carp_header *ch;
 	int iplen, len;
 
-	carpstats.carps_ipackets++;
+	CARPSTATS_INC(carps_ipackets);
 
 	if (!carp_opts[CARPCTL_ALLOW]) {
 		m_freem(m);
@@ -539,7 +539,7 @@ carp_input(struct mbuf *m, int hlen)
 
 	/* check if received on a valid carp interface */
 	if (m->m_pkthdr.rcvif->if_carp == NULL) {
-		carpstats.carps_badif++;
+		CARPSTATS_INC(carps_badif);
 		CARP_LOG("carp_input: packet received on non-carp "
 		    "interface: %s\n",
 		    m->m_pkthdr.rcvif->if_xname);
@@ -549,7 +549,7 @@ carp_input(struct mbuf *m, int hlen)
 
 	/* verify that the IP TTL is 255.  */
 	if (ip->ip_ttl != CARP_DFLTTL) {
-		carpstats.carps_badttl++;
+		CARPSTATS_INC(carps_badttl);
 		CARP_LOG("carp_input: received ttl %d != 255i on %s\n",
 		    ip->ip_ttl,
 		    m->m_pkthdr.rcvif->if_xname);
@@ -560,7 +560,7 @@ carp_input(struct mbuf *m, int hlen)
 	iplen = ip->ip_hl << 2;
 
 	if (m->m_pkthdr.len < iplen + sizeof(*ch)) {
-		carpstats.carps_badlen++;
+		CARPSTATS_INC(carps_badlen);
 		CARP_LOG("carp_input: received len %zd < "
 		    "sizeof(struct carp_header)\n",
 		    m->m_len - sizeof(struct ip));
@@ -570,7 +570,7 @@ carp_input(struct mbuf *m, int hlen)
 
 	if (iplen + sizeof(*ch) < m->m_len) {
 		if ((m = m_pullup(m, iplen + sizeof(*ch))) == NULL) {
-			carpstats.carps_hdrops++;
+			CARPSTATS_INC(carps_hdrops);
 			CARP_LOG("carp_input: pullup failed\n");
 			return;
 		}
@@ -584,7 +584,7 @@ carp_input(struct mbuf *m, int hlen)
 	 */
 	len = iplen + sizeof(*ch);
 	if (len > m->m_pkthdr.len) {
-		carpstats.carps_badlen++;
+		CARPSTATS_INC(carps_badlen);
 		CARP_LOG("carp_input: packet too short %d on %s\n",
 		    m->m_pkthdr.len,
 		    m->m_pkthdr.rcvif->if_xname);
@@ -593,7 +593,7 @@ carp_input(struct mbuf *m, int hlen)
 	}
 
 	if ((m = m_pullup(m, len)) == NULL) {
-		carpstats.carps_hdrops++;
+		CARPSTATS_INC(carps_hdrops);
 		return;
 	}
 	ip = mtod(m, struct ip *);
@@ -602,7 +602,7 @@ carp_input(struct mbuf *m, int hlen)
 	/* verify the CARP checksum */
 	m->m_data += iplen;
 	if (carp_cksum(m, len - iplen)) {
-		carpstats.carps_badsum++;
+		CARPSTATS_INC(carps_badsum);
 		CARP_LOG("carp_input: checksum failed on %s\n",
 		    m->m_pkthdr.rcvif->if_xname);
 		m_freem(m);
@@ -622,7 +622,7 @@ carp6_input(struct mbuf **mp, int *offp, int proto)
 	struct carp_header *ch;
 	u_int len;
 
-	carpstats.carps_ipackets6++;
+	CARPSTATS_INC(carps_ipackets6);
 
 	if (!carp_opts[CARPCTL_ALLOW]) {
 		m_freem(m);
@@ -631,7 +631,7 @@ carp6_input(struct mbuf **mp, int *offp, int proto)
 
 	/* check if received on a valid carp interface */
 	if (m->m_pkthdr.rcvif->if_carp == NULL) {
-		carpstats.carps_badif++;
+		CARPSTATS_INC(carps_badif);
 		CARP_LOG("carp6_input: packet received on non-carp "
 		    "interface: %s\n",
 		    m->m_pkthdr.rcvif->if_xname);
@@ -641,7 +641,7 @@ carp6_input(struct mbuf **mp, int *offp, int proto)
 
 	/* verify that the IP TTL is 255 */
 	if (ip6->ip6_hlim != CARP_DFLTTL) {
-		carpstats.carps_badttl++;
+		CARPSTATS_INC(carps_badttl);
 		CARP_LOG("carp6_input: received ttl %d != 255 on %s\n",
 		    ip6->ip6_hlim,
 		    m->m_pkthdr.rcvif->if_xname);
@@ -653,7 +653,7 @@ carp6_input(struct mbuf **mp, int *offp, int proto)
 	len = m->m_len;
 	IP6_EXTHDR_GET(ch, struct carp_header *, m, *offp, sizeof(*ch));
 	if (ch == NULL) {
-		carpstats.carps_badlen++;
+		CARPSTATS_INC(carps_badlen);
 		CARP_LOG("carp6_input: packet size %u too small\n", len);
 		return (IPPROTO_DONE);
 	}
@@ -662,7 +662,7 @@ carp6_input(struct mbuf **mp, int *offp, int proto)
 	/* verify the CARP checksum */
 	m->m_data += *offp;
 	if (carp_cksum(m, sizeof(*ch))) {
-		carpstats.carps_badsum++;
+		CARPSTATS_INC(carps_badsum);
 		CARP_LOG("carp6_input: checksum failed, on %s\n",
 		    m->m_pkthdr.rcvif->if_xname);
 		m_freem(m);
@@ -691,7 +691,7 @@ carp_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af)
 
 	if (!sc || !((SC2IFP(sc)->if_flags & IFF_UP) &&
 	    (SC2IFP(sc)->if_drv_flags & IFF_DRV_RUNNING))) {
-		carpstats.carps_badvhid++;
+		CARPSTATS_INC(carps_badvhid);
 		CARP_UNLOCK(ifp->if_carp);
 		m_freem(m);
 		return;
@@ -713,7 +713,7 @@ carp_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af)
 
 	/* verify the CARP version. */
 	if (ch->carp_version != CARP_VERSION) {
-		carpstats.carps_badver++;
+		CARPSTATS_INC(carps_badver);
 		SC2IFP(sc)->if_ierrors++;
 		CARP_UNLOCK(ifp->if_carp);
 		CARP_LOG("%s; invalid version %d\n",
@@ -725,7 +725,7 @@ carp_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af)
 
 	/* verify the hash */
 	if (carp_hmac_verify(sc, ch->carp_counter, ch->carp_md)) {
-		carpstats.carps_badauth++;
+		CARPSTATS_INC(carps_badauth);
 		SC2IFP(sc)->if_ierrors++;
 		CARP_UNLOCK(ifp->if_carp);
 		CARP_LOG("%s: incorrect hash\n", SC2IFP(sc)->if_xname);
@@ -915,7 +915,7 @@ carp_send_ad_locked(struct carp_softc *sc)
 		MGETHDR(m, M_DONTWAIT, MT_HEADER);
 		if (m == NULL) {
 			SC2IFP(sc)->if_oerrors++;
-			carpstats.carps_onomem++;
+			CARPSTATS_INC(carps_onomem);
 			/* XXX maybe less ? */
 			if (advbase != 255 || advskew != 255)
 				callout_reset(&sc->sc_ad_tmo, tvtohz(&tv),
@@ -953,7 +953,7 @@ carp_send_ad_locked(struct carp_softc *sc)
 		getmicrotime(&SC2IFP(sc)->if_lastchange);
 		SC2IFP(sc)->if_opackets++;
 		SC2IFP(sc)->if_obytes += len;
-		carpstats.carps_opackets++;
+		CARPSTATS_INC(carps_opackets);
 
 		if (ip_output(m, NULL, NULL, IP_RAWOUTPUT, &sc->sc_imo, NULL)) {
 			SC2IFP(sc)->if_oerrors++;
@@ -987,7 +987,7 @@ carp_send_ad_locked(struct carp_softc *sc)
 		MGETHDR(m, M_DONTWAIT, MT_HEADER);
 		if (m == NULL) {
 			SC2IFP(sc)->if_oerrors++;
-			carpstats.carps_onomem++;
+			CARPSTATS_INC(carps_onomem);
 			/* XXX maybe less ? */
 			if (advbase != 255 || advskew != 255)
 				callout_reset(&sc->sc_ad_tmo, tvtohz(&tv),
@@ -1030,7 +1030,7 @@ carp_send_ad_locked(struct carp_softc *sc)
 		getmicrotime(&SC2IFP(sc)->if_lastchange);
 		SC2IFP(sc)->if_opackets++;
 		SC2IFP(sc)->if_obytes += len;
-		carpstats.carps_opackets6++;
+		CARPSTATS_INC(carps_opackets6);
 
 		if (ip6_output(m, NULL, NULL, 0, &sc->sc_im6o, NULL, NULL)) {
 			SC2IFP(sc)->if_oerrors++;
diff --git a/sys/netinet/ip_carp.h b/sys/netinet/ip_carp.h
index 2dd9544..47bb494 100644
--- a/sys/netinet/ip_carp.h
+++ b/sys/netinet/ip_carp.h
@@ -117,6 +117,11 @@ struct carpstats {
 	uint64_t	carps_preempt;		/* if enabled, preemptions */
 };
 
+#ifdef _KERNEL
+#define	CARPSTATS_ADD(name, val)	carpstats.name += (val)
+#define	CARPSTATS_INC(name)		CARPSTATS_ADD(name, 1)
+#endif
+
 /*
  * Configuration structure for SIOCSVH SIOCGVH
  */
diff --git a/sys/netinet/ip_divert.c b/sys/netinet/ip_divert.c
index c2e3191..f39c655 100644
--- a/sys/netinet/ip_divert.c
+++ b/sys/netinet/ip_divert.c
@@ -186,7 +186,7 @@ div_input(struct mbuf *m, int off)
 {
 	INIT_VNET_INET(curvnet);
 
-	V_ipstat.ips_noproto++;
+	IPSTAT_INC(ips_noproto);
 	m_freem(m);
 }
 
@@ -307,8 +307,8 @@ divert_packet(struct mbuf *m, int incoming)
 	INP_INFO_RUNLOCK(&V_divcbinfo);
 	if (sa == NULL) {
 		m_freem(m);
-		V_ipstat.ips_noproto++;
-		V_ipstat.ips_delivered--;
+		IPSTAT_INC(ips_noproto);
+		IPSTAT_DEC(ips_delivered);
         }
 }
 
@@ -394,7 +394,7 @@ div_output(struct socket *so, struct mbuf *m, struct sockaddr_in *sin,
 			ip->ip_off = ntohs(ip->ip_off);
 
 			/* Send packet to output processing */
-			V_ipstat.ips_rawout++;			/* XXX */
+			IPSTAT_INC(ips_rawout);			/* XXX */
 
 #ifdef MAC
 			mac_inpcb_create_mbuf(inp, m);
@@ -570,7 +570,7 @@ div_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
 	/* Packet must have a header (but that's about it) */
 	if (m->m_len < sizeof (struct ip) &&
 	    (m = m_pullup(m, sizeof (struct ip))) == 0) {
-		V_ipstat.ips_toosmall++;
+		IPSTAT_INC(ips_toosmall);
 		m_freem(m);
 		return EINVAL;
 	}
diff --git a/sys/netinet/ip_dummynet.c b/sys/netinet/ip_dummynet.c
index 4c7f60a..5816be9 100644
--- a/sys/netinet/ip_dummynet.c
+++ b/sys/netinet/ip_dummynet.c
@@ -519,14 +519,64 @@ transmit_event(struct dn_pipe *pipe, struct mbuf **head, struct mbuf **tail)
 	}
 }
 
+#define div64(a, b)	((int64_t)(a) / (int64_t)(b))
+#define DN_TO_DROP	0xffff
 /*
- * the following macro computes how many ticks we have to wait
- * before being able to transmit a packet. The credit is taken from
- * either a pipe (WF2Q) or a flow_queue (per-flow queueing)
+ * Compute how many ticks we have to wait before being able to send
+ * a packet. This is computed as the "wire time" for the packet
+ * (length + extra bits), minus the credit available, scaled to ticks.
+ * Check that the result is not be negative (it could be if we have
+ * too much leftover credit in q->numbytes).
  */
-#define SET_TICKS(_m, q, p)	\
-    ((_m)->m_pkthdr.len * 8 * hz - (q)->numbytes + p->bandwidth - 1) / \
-    p->bandwidth;
+static inline dn_key
+set_ticks(struct mbuf *m, struct dn_flow_queue *q, struct dn_pipe *p)
+{
+	int64_t ret;
+
+	ret = div64( (m->m_pkthdr.len * 8 + q->extra_bits) * hz
+		- q->numbytes + p->bandwidth - 1 , p->bandwidth);
+#if 0
+	printf("%s %d extra_bits %d numb %d ret %d\n",
+		__FUNCTION__, __LINE__,
+		(int)(q->extra_bits & 0xffffffff),
+		(int)(q->numbytes & 0xffffffff),
+		(int)(ret & 0xffffffff));
+#endif
+	if (ret < 0)
+		ret = 0;
+	return ret;
+}
+
+/*
+ * Convert the additional MAC overheads/delays into an equivalent
+ * number of bits for the given data rate. The samples are in milliseconds
+ * so we need to divide by 1000.
+ */
+static dn_key
+compute_extra_bits(struct mbuf *pkt, struct dn_pipe *p)
+{
+	int index;
+	dn_key extra_bits;
+
+	if (!p->samples || p->samples_no == 0)
+		return 0;
+	index  = random() % p->samples_no;
+	extra_bits = ((dn_key)p->samples[index] * p->bandwidth) / 1000;
+	if (index >= p->loss_level) {
+		struct dn_pkt_tag *dt = dn_tag_get(pkt);
+		if (dt)
+			dt->dn_dir = DN_TO_DROP;
+	}
+	return extra_bits;
+}
+
+static void
+free_pipe(struct dn_pipe *p)
+{
+	if (p->samples)
+		free(p->samples, M_DUMMYNET);
+	free(p, M_DUMMYNET);
+}
 
 /*
  * extract pkt from queue, compute output time (could be now)
@@ -585,12 +635,16 @@ ready_event(struct dn_flow_queue *q, struct mbuf **head, struct mbuf **tail)
 	q->numbytes += (curr_time - q->sched_time) * p->bandwidth;
 	while ((pkt = q->head) != NULL) {
 		int len = pkt->m_pkthdr.len;
-		int len_scaled = p->bandwidth ? len * 8 * hz : 0;
+		dn_key len_scaled = p->bandwidth ? len*8*hz
+			+ q->extra_bits*hz
+			: 0;
 
-		if (len_scaled > q->numbytes)
+		if (DN_KEY_GT(len_scaled, q->numbytes))
 			break;
 		q->numbytes -= len_scaled;
 		move_pkt(pkt, q, p, len);
+		if (q->head)
+			q->extra_bits = compute_extra_bits(q->head, p);
 	}
 	/*
 	 * If we have more packets queued, schedule next ready event
@@ -600,7 +654,7 @@ ready_event(struct dn_flow_queue *q, struct mbuf **head, struct mbuf **tail)
 	 * ticks to go for the finish time of the packet.
 	 */
 	if ((pkt = q->head) != NULL) {	/* this implies bandwidth != 0 */
-		dn_key t = SET_TICKS(pkt, q, p); /* ticks i have to wait */
+		dn_key t = set_ticks(pkt, q, p); /* ticks i have to wait */
 
 		q->sched_time = curr_time;
 		heap_insert(&ready_heap, curr_time + t, (void *)q);
@@ -933,6 +987,12 @@ dummynet_send(struct mbuf *m)
 		case DN_TO_ETH_OUT:
 			ether_output_frame(pkt->ifp, m);
 			break;
+
+		case DN_TO_DROP:
+			/* drop the packet after some time */
+			m_freem(m);
+			break;
+
 		default:
 			printf("dummynet: bad switch %d!\n", pkt->dn_dir);
 			m_freem(m);
@@ -1367,8 +1427,10 @@ dummynet_io(struct mbuf **m0, int dir, struct ip_fw_args *fwa)
 		/* Fixed-rate queue: just insert into the ready_heap. */
 		dn_key t = 0;
 
-		if (pipe->bandwidth && m->m_pkthdr.len * 8 * hz > q->numbytes)
-			t = SET_TICKS(m, q, pipe);
+		if (pipe->bandwidth) {
+			q->extra_bits = compute_extra_bits(m, pipe);
+			t = set_ticks(m, q, pipe);
+		}
 		q->sched_time = curr_time;
 		if (t == 0)		/* Must process it now. */
 			ready_event(q, &head, &tail);
@@ -1555,7 +1617,7 @@ dummynet_flush(void)
 		SLIST_FOREACH_SAFE(pipe, &pipehash[i], next, pipe1) {
 			SLIST_REMOVE(&pipehash[i], pipe, dn_pipe, next);
 			purge_pipe(pipe);
-			free(pipe, M_DUMMYNET);
+			free_pipe(pipe);
 		}
 	DUMMYNET_UNLOCK();
 }
@@ -1775,11 +1837,38 @@ config_pipe(struct dn_pipe *p)
 		pipe->delay = p->delay;
 		set_fs_parms(&(pipe->fs), pfs);
 
+		/* Handle changes in the delay profile. */
+		if (p->samples_no > 0) {
+			if (pipe->samples_no != p->samples_no) {
+				if (pipe->samples != NULL)
+					free(pipe->samples, M_DUMMYNET);
+				pipe->samples =
+				    malloc(p->samples_no*sizeof(dn_key),
+					M_DUMMYNET, M_NOWAIT | M_ZERO);
+				if (pipe->samples == NULL) {
+					DUMMYNET_UNLOCK();
+					printf("dummynet: no memory "
+						"for new samples\n");
+					return (ENOMEM);
+				}
+				pipe->samples_no = p->samples_no;
+			}
+
+			strncpy(pipe->name,p->name,sizeof(pipe->name));
+			pipe->loss_level = p->loss_level;
+			for (i = 0; i<pipe->samples_no; ++i)
+				pipe->samples[i] = p->samples[i];
+		} else if (pipe->samples != NULL) {
+			free(pipe->samples, M_DUMMYNET);
+			pipe->samples = NULL;
+			pipe->samples_no = 0;
+		}
+
 		if (pipe->fs.rq == NULL) {	/* a new pipe */
 			error = alloc_hash(&(pipe->fs), pfs);
 			if (error) {
 				DUMMYNET_UNLOCK();
-				free(pipe, M_DUMMYNET);
+				free_pipe(pipe);
 				return (error);
 			}
 			SLIST_INSERT_HEAD(&pipehash[HASH(pipe->pipe_nr)],
@@ -1957,7 +2046,7 @@ delete_pipe(struct dn_pipe *p)
 	pipe_remove_from_heap(&wfq_ready_heap, pipe);
 	DUMMYNET_UNLOCK();
 
-	free(pipe, M_DUMMYNET);
+	free_pipe(pipe);
     } else { /* this is a WF2Q queue (dn_flow_set) */
 	struct dn_flow_set *fs;
 
@@ -2095,6 +2184,7 @@ dummynet_get(struct sockopt *sopt)
 		pipe_bp->fs.next.sle_next = NULL;
 		pipe_bp->fs.pipe = NULL;
 		pipe_bp->fs.rq = NULL;
+		pipe_bp->samples = NULL;
 
 		bp += sizeof(*pipe) ;
 		bp = dn_copy_set(&(pipe->fs), bp);
@@ -2127,7 +2217,8 @@ static int
 ip_dn_ctl(struct sockopt *sopt)
 {
     int error = 0 ;
-    struct dn_pipe *p, tmp_pipe;
+    struct dn_pipe *p;
+    struct dn_pipe_max tmp_pipe;	/* pipe + large buffer */
 
     error = priv_check(sopt->sopt_td, PRIV_NETINET_DUMMYNET);
     if (error)
@@ -2159,15 +2250,18 @@ ip_dn_ctl(struct sockopt *sopt)
 	break ;
 
     case IP_DUMMYNET_CONFIGURE :
-	p = &tmp_pipe ;
-	error = sooptcopyin(sopt, p, sizeof *p, sizeof *p);
+	p = (struct dn_pipe *)&tmp_pipe ;
+	error = sooptcopyin(sopt, p, sizeof(tmp_pipe), sizeof *p);
 	if (error)
 	    break ;
+	if (p->samples_no > 0)
+	    p->samples = &tmp_pipe.samples[0];
+
 	error = config_pipe(p);
 	break ;
 
     case IP_DUMMYNET_DEL :	/* remove a pipe or queue */
-	p = &tmp_pipe ;
+	p = (struct dn_pipe *)&tmp_pipe ;
 	error = sooptcopyin(sopt, p, sizeof *p, sizeof *p);
 	if (error)
 	    break ;
diff --git a/sys/netinet/ip_dummynet.h b/sys/netinet/ip_dummynet.h
index 0a37e6c..6e102f4 100644
--- a/sys/netinet/ip_dummynet.h
+++ b/sys/netinet/ip_dummynet.h
@@ -204,7 +204,18 @@ struct dn_flow_queue {
     struct mbuf *head, *tail ;	/* queue of packets */
     u_int len ;
     u_int len_bytes ;
-    u_long numbytes ;		/* credit for transmission (dynamic queues) */
+
+    /*
+     * When we emulate MAC overheads, or channel unavailability due
+     * to other traffic on a shared medium, we augment the packet at
+     * the head of the queue with an 'extra_bits' field representsing
+     * the additional delay the packet will be subject to:
+     *		extra_bits = bw*unavailable_time.
+     * With large bandwidth and large delays, extra_bits (and also numbytes)
+     * can become very large, so better play safe and use 64 bit
+     */
+    dn_key numbytes ;		/* credit for transmission (dynamic queues) */
+    dn_key extra_bits;		/* extra bits simulating unavailable channel */
 
     u_int64_t tot_pkts ;	/* statistics counters	*/
     u_int64_t tot_bytes ;
@@ -252,6 +263,7 @@ struct dn_flow_set {
 #define DN_IS_GENTLE_RED	0x0004
 #define DN_QSIZE_IS_BYTES	0x0008	/* queue size is measured in bytes */
 #define DN_NOERROR		0x0010	/* do not report ENOBUFS on drops  */
+#define	DN_HAS_PROFILE		0x0020	/* the pipe has a delay profile. */
 #define DN_IS_PIPE		0x4000
 #define DN_IS_QUEUE		0x8000
 
@@ -324,7 +336,9 @@ struct dn_pipe {		/* a pipe */
 
     dn_key V ;			/* virtual time */
     int sum;			/* sum of weights of all active sessions */
-    int numbytes;		/* bits I can transmit (more or less). */
+
+    /* Same as in dn_flow_queue, numbytes can become large */
+    dn_key numbytes;		/* bits I can transmit (more or less). */
 
     dn_key sched_time ;		/* time pipe was scheduled in ready_heap */
 
@@ -337,7 +351,25 @@ struct dn_pipe {		/* a pipe */
     int ready ; /* set if ifp != NULL and we got a signal from it */
 
     struct dn_flow_set fs ; /* used with fixed-rate flows */
+
+    /* fields to simulate a delay profile */
+
+#define ED_MAX_NAME_LEN		32
+    char name[ED_MAX_NAME_LEN];
+    int loss_level;
+    int samples_no;
+    int *samples;
 };
+
+/* dn_pipe_max is used to pass pipe configuration from userland onto
+ * kernel space and back
+ */
+#define ED_MAX_SAMPLES_NO	1024
+struct dn_pipe_max {
+	struct dn_pipe pipe;
+	int samples[ED_MAX_SAMPLES_NO];
+};
+
 SLIST_HEAD(dn_pipe_head, dn_pipe);
 
 #ifdef _KERNEL
diff --git a/sys/netinet/ip_fastfwd.c b/sys/netinet/ip_fastfwd.c
index ea106df..0479064 100644
--- a/sys/netinet/ip_fastfwd.c
+++ b/sys/netinet/ip_fastfwd.c
@@ -140,8 +140,8 @@ ip_findroute(struct route *ro, struct in_addr dest, struct mbuf *m)
 		if (rt->rt_flags & RTF_GATEWAY)
 			dst = (struct sockaddr_in *)rt->rt_gateway;
 	} else {
-		V_ipstat.ips_noroute++;
-		V_ipstat.ips_cantforward++;
+		IPSTAT_INC(ips_noroute);
+		IPSTAT_INC(ips_cantforward);
 		if (rt)
 			RTFREE(rt);
 		icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_HOST, 0, 0);
@@ -193,7 +193,7 @@ ip_fastforward(struct mbuf *m)
 	 * Is entire packet big enough?
 	 */
 	if (m->m_pkthdr.len < sizeof(struct ip)) {
-		V_ipstat.ips_tooshort++;
+		IPSTAT_INC(ips_tooshort);
 		goto drop;
 	}
 
@@ -202,7 +202,7 @@ ip_fastforward(struct mbuf *m)
 	 */
 	if (m->m_len < sizeof (struct ip) &&
 	   (m = m_pullup(m, sizeof (struct ip))) == NULL) {
-		V_ipstat.ips_toosmall++;
+		IPSTAT_INC(ips_toosmall);
 		return NULL;	/* mbuf already free'd */
 	}
 
@@ -212,7 +212,7 @@ ip_fastforward(struct mbuf *m)
 	 * Is it IPv4?
 	 */
 	if (ip->ip_v != IPVERSION) {
-		V_ipstat.ips_badvers++;
+		IPSTAT_INC(ips_badvers);
 		goto drop;
 	}
 
@@ -221,12 +221,12 @@ ip_fastforward(struct mbuf *m)
 	 */
 	hlen = ip->ip_hl << 2;
 	if (hlen < sizeof(struct ip)) {	/* minimum header length */
-		V_ipstat.ips_badlen++;
+		IPSTAT_INC(ips_badlen);
 		goto drop;
 	}
 	if (hlen > m->m_len) {
 		if ((m = m_pullup(m, hlen)) == NULL) {
-			V_ipstat.ips_badhlen++;
+			IPSTAT_INC(ips_badhlen);
 			return NULL;	/* mbuf already free'd */
 		}
 		ip = mtod(m, struct ip *);
@@ -244,7 +244,7 @@ ip_fastforward(struct mbuf *m)
 			sum = in_cksum(m, hlen);
 	}
 	if (sum) {
-		V_ipstat.ips_badsum++;
+		IPSTAT_INC(ips_badsum);
 		goto drop;
 	}
 
@@ -259,7 +259,7 @@ ip_fastforward(struct mbuf *m)
 	 * Is IP length longer than packet we have got?
 	 */
 	if (m->m_pkthdr.len < ip_len) {
-		V_ipstat.ips_tooshort++;
+		IPSTAT_INC(ips_tooshort);
 		goto drop;
 	}
 
@@ -279,7 +279,7 @@ ip_fastforward(struct mbuf *m)
 	 */
 	if ((ntohl(ip->ip_dst.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET ||
 	    (ntohl(ip->ip_src.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET) {
-		V_ipstat.ips_badaddr++;
+		IPSTAT_INC(ips_badaddr);
 		goto drop;
 	}
 
@@ -337,7 +337,7 @@ ip_fastforward(struct mbuf *m)
 	if (in_localip(ip->ip_dst))
 		return m;
 
-	V_ipstat.ips_total++;
+	IPSTAT_INC(ips_total);
 
 	/*
 	 * Step 3: incoming packet firewall processing
@@ -519,7 +519,7 @@ passout:
 	 */
 	if ((ifp->if_snd.ifq_len + ip->ip_len / ifp->if_mtu + 1) >=
 	    ifp->if_snd.ifq_maxlen) {
-		V_ipstat.ips_odropped++;
+		IPSTAT_INC(ips_odropped);
 		/* would send source quench here but that is depreciated */
 		goto drop;
 	}
@@ -558,7 +558,7 @@ passout:
 		 * Handle EMSGSIZE with icmp reply needfrag for TCP MTU discovery
 		 */
 		if (ip->ip_off & IP_DF) {
-			V_ipstat.ips_cantfrag++;
+			IPSTAT_INC(ips_cantfrag);
 			icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_NEEDFRAG,
 				0, mtu);
 			goto consumed;
@@ -596,16 +596,16 @@ passout:
 					m_freem(m);
 				}
 			} else
-				V_ipstat.ips_fragmented++;
+				IPSTAT_INC(ips_fragmented);
 		}
 	}
 
 	if (error != 0)
-		V_ipstat.ips_odropped++;
+		IPSTAT_INC(ips_odropped);
 	else {
 		ro.ro_rt->rt_rmx.rmx_pksent++;
-		V_ipstat.ips_forward++;
-		V_ipstat.ips_fastforward++;
+		IPSTAT_INC(ips_forward);
+		IPSTAT_INC(ips_fastforward);
 	}
 consumed:
 	RTFREE(ro.ro_rt);
diff --git a/sys/netinet/ip_fw.h b/sys/netinet/ip_fw.h
index 0cf1a52..fa37a73 100644
--- a/sys/netinet/ip_fw.h
+++ b/sys/netinet/ip_fw.h
@@ -139,7 +139,8 @@ enum ipfw_opcodes {		/* arguments (4 byte each)	*/
 	O_FORWARD_IP,		/* fwd sockaddr			*/
 	O_FORWARD_MAC,		/* fwd mac			*/
 	O_NAT,                  /* nope                         */
-
+	O_REASS,                /* none                         */
+	
 	/*
 	 * More opcodes.
 	 */
@@ -574,6 +575,7 @@ enum {
 	IP_FW_NETGRAPH,
 	IP_FW_NGTEE,
 	IP_FW_NAT,
+	IP_FW_REASS,
 };
 
 /* flags for divert mtag */
@@ -696,6 +698,7 @@ struct vnet_ipfw {
 	int	_fw_debug;		/* actually unused */
 	int	_autoinc_step;
 	ipfw_dyn_rule **_ipfw_dyn_v;
+	uma_zone_t _ipfw_dyn_rule_zone;
 	struct ip_fw_chain _layer3_chain;
 	u_int32_t _dyn_buckets;
 	u_int32_t _curr_dyn_buckets;
@@ -740,6 +743,7 @@ extern struct vnet_ipfw vnet_ipfw_0;
 #define	V_fw_debug		VNET_IPFW(fw_debug)
 #define	V_autoinc_step		VNET_IPFW(autoinc_step)
 #define	V_ipfw_dyn_v		VNET_IPFW(ipfw_dyn_v)
+#define	V_ipfw_dyn_rule_zone	VNET_IPFW(ipfw_dyn_rule_zone)
 #define	V_layer3_chain		VNET_IPFW(layer3_chain)
 #define	V_dyn_buckets		VNET_IPFW(dyn_buckets)
 #define	V_curr_dyn_buckets	VNET_IPFW(curr_dyn_buckets)
diff --git a/sys/netinet/ip_fw2.c b/sys/netinet/ip_fw2.c
index 1405a27..fa6e6f1 100644
--- a/sys/netinet/ip_fw2.c
+++ b/sys/netinet/ip_fw2.c
@@ -898,6 +898,9 @@ ipfw_log(struct ip_fw *f, u_int hlen, struct ip_fw_args *args,
 		case O_NAT:
 			action = "Nat";
  			break;
+		case O_REASS:
+			action = "Reass";
+			break;
 		default:
 			action = "UNKNOWN";
 			break;
@@ -3375,6 +3378,55 @@ check_body:
 				goto done;
 			}
 
+			case O_REASS: {
+				int ip_off;
+
+				f->pcnt++;
+				f->bcnt += pktlen;
+				ip_off = (args->eh != NULL) ? ntohs(ip->ip_off) : ip->ip_off;
+				if (ip_off & (IP_MF | IP_OFFMASK)) {
+					/* 
+					 * ip_reass() expects len & off in host
+					 * byte order: fix them in case we come
+					 * from layer2.
+					 */
+					if (args->eh != NULL) {
+						ip->ip_len = ntohs(ip->ip_len);
+						ip->ip_off = ntohs(ip->ip_off);
+					}
+
+					m = ip_reass(m);
+					args->m = m;
+					
+					/*
+					 * IP header checksum fixup after 
+					 * reassembly and leave header
+					 * in network byte order.
+					 */
+					if (m != NULL) {
+						int hlen;
+					
+						ip = mtod(m, struct ip *);
+						hlen = ip->ip_hl << 2;
+						/* revert len & off for layer2 pkts */
+						if (args->eh != NULL)
+							ip->ip_len = htons(ip->ip_len);
+						ip->ip_sum = 0;
+						if (hlen == sizeof(struct ip))
+							ip->ip_sum = in_cksum_hdr(ip);
+						else
+							ip->ip_sum = in_cksum(m, hlen);
+						retval = IP_FW_REASS;
+						args->rule = f;
+						goto done;
+					} else {
+						retval = IP_FW_DENY;
+						goto done;
+					}
+				}
+				goto next_rule;
+			}
+
 			default:
 				panic("-- unknown opcode %d\n", cmd->opcode);
 			} /* end of switch() on opcodes */
@@ -4024,6 +4076,7 @@ check_ipfw_struct(struct ip_fw *rule, int size)
 		case O_UNREACH6:
 #endif
 		case O_SKIPTO:
+		case O_REASS:
 check_size:
 			if (cmdlen != F_INSN_SIZE(ipfw_insn))
 				goto bad_size;
diff --git a/sys/netinet/ip_fw_pfil.c b/sys/netinet/ip_fw_pfil.c
index 1e6d2b0..11560a7 100644
--- a/sys/netinet/ip_fw_pfil.c
+++ b/sys/netinet/ip_fw_pfil.c
@@ -200,6 +200,9 @@ again:
 	case IP_FW_NAT:
 		goto again;		/* continue with packet */
 
+	case IP_FW_REASS:
+		goto again;
+
 	default:
 		KASSERT(0, ("%s: unknown retval", __func__));
 	}
@@ -329,6 +332,9 @@ again:
 	case IP_FW_NAT:
 		goto again;		/* continue with packet */
 		
+	case IP_FW_REASS:
+		goto again;	
+	
 	default:
 		KASSERT(0, ("%s: unknown retval", __func__));
 	}
diff --git a/sys/netinet/ip_icmp.c b/sys/netinet/ip_icmp.c
index baf07d3..fe06b9a 100644
--- a/sys/netinet/ip_icmp.c
+++ b/sys/netinet/ip_icmp.c
@@ -180,7 +180,7 @@ icmp_error(struct mbuf *n, int type, int code, uint32_t dest, int mtu)
 		printf("icmp_error(%p, %x, %d)\n", oip, type, code);
 #endif
 	if (type != ICMP_REDIRECT)
-		V_icmpstat.icps_error++;
+		ICMPSTAT_INC(icps_error);
 	/*
 	 * Don't send error:
 	 *  if the original packet was encrypted.
@@ -197,7 +197,7 @@ icmp_error(struct mbuf *n, int type, int code, uint32_t dest, int mtu)
 	if (oip->ip_p == IPPROTO_ICMP && type != ICMP_REDIRECT &&
 	  n->m_len >= oiphlen + ICMP_MINLEN &&
 	  !ICMP_INFOTYPE(((struct icmp *)((caddr_t)oip + oiphlen))->icmp_type)) {
-		V_icmpstat.icps_oldicmp++;
+		ICMPSTAT_INC(icps_oldicmp);
 		goto freeit;
 	}
 	/* Drop if IP header plus 8 bytes is not contignous in first mbuf. */
@@ -257,7 +257,7 @@ stdreply:	icmpelen = max(8, min(V_icmp_quotelen, oip->ip_len - oiphlen));
 	 */
 	M_SETFIB(m, M_GETFIB(n));
 	icp = mtod(m, struct icmp *);
-	V_icmpstat.icps_outhist[type]++;
+	ICMPSTAT_INC(icps_outhist[type]);
 	icp->icmp_type = type;
 	if (type == ICMP_REDIRECT)
 		icp->icmp_gwaddr.s_addr = dest;
@@ -340,12 +340,12 @@ icmp_input(struct mbuf *m, int off)
 	}
 #endif
 	if (icmplen < ICMP_MINLEN) {
-		V_icmpstat.icps_tooshort++;
+		ICMPSTAT_INC(icps_tooshort);
 		goto freeit;
 	}
 	i = hlen + min(icmplen, ICMP_ADVLENMIN);
 	if (m->m_len < i && (m = m_pullup(m, i)) == 0)  {
-		V_icmpstat.icps_tooshort++;
+		ICMPSTAT_INC(icps_tooshort);
 		return;
 	}
 	ip = mtod(m, struct ip *);
@@ -353,7 +353,7 @@ icmp_input(struct mbuf *m, int off)
 	m->m_data += hlen;
 	icp = mtod(m, struct icmp *);
 	if (in_cksum(m, icmplen)) {
-		V_icmpstat.icps_checksum++;
+		ICMPSTAT_INC(icps_checksum);
 		goto freeit;
 	}
 	m->m_len += hlen;
@@ -395,7 +395,7 @@ icmp_input(struct mbuf *m, int off)
 	icmpgw.sin_len = sizeof(struct sockaddr_in);
 	icmpgw.sin_family = AF_INET;
 
-	V_icmpstat.icps_inhist[icp->icmp_type]++;
+	ICMPSTAT_INC(icps_inhist[icp->icmp_type]);
 	code = icp->icmp_code;
 	switch (icp->icmp_type) {
 
@@ -460,7 +460,7 @@ icmp_input(struct mbuf *m, int off)
 		 */
 		if (icmplen < ICMP_ADVLENMIN || icmplen < ICMP_ADVLEN(icp) ||
 		    icp->icmp_ip.ip_hl < (sizeof(struct ip) >> 2)) {
-			V_icmpstat.icps_badlen++;
+			ICMPSTAT_INC(icps_badlen);
 			goto freeit;
 		}
 		icp->icmp_ip.ip_len = ntohs(icp->icmp_ip.ip_len);
@@ -483,13 +483,13 @@ icmp_input(struct mbuf *m, int off)
 		break;
 
 	badcode:
-		V_icmpstat.icps_badcode++;
+		ICMPSTAT_INC(icps_badcode);
 		break;
 
 	case ICMP_ECHO:
 		if (!V_icmpbmcastecho
 		    && (m->m_flags & (M_MCAST | M_BCAST)) != 0) {
-			V_icmpstat.icps_bmcastecho++;
+			ICMPSTAT_INC(icps_bmcastecho);
 			break;
 		}
 		icp->icmp_type = ICMP_ECHOREPLY;
@@ -501,11 +501,11 @@ icmp_input(struct mbuf *m, int off)
 	case ICMP_TSTAMP:
 		if (!V_icmpbmcastecho
 		    && (m->m_flags & (M_MCAST | M_BCAST)) != 0) {
-			V_icmpstat.icps_bmcasttstamp++;
+			ICMPSTAT_INC(icps_bmcasttstamp);
 			break;
 		}
 		if (icmplen < ICMP_TSLEN) {
-			V_icmpstat.icps_badlen++;
+			ICMPSTAT_INC(icps_badlen);
 			break;
 		}
 		icp->icmp_type = ICMP_TSTAMPREPLY;
@@ -554,8 +554,8 @@ icmp_input(struct mbuf *m, int off)
 		}
 reflect:
 		ip->ip_len += hlen;	/* since ip_input deducts this */
-		V_icmpstat.icps_reflect++;
-		V_icmpstat.icps_outhist[icp->icmp_type]++;
+		ICMPSTAT_INC(icps_reflect);
+		ICMPSTAT_INC(icps_outhist[icp->icmp_type]);
 		icmp_reflect(m);
 		return;
 
@@ -585,7 +585,7 @@ reflect:
 			goto badcode;
 		if (icmplen < ICMP_ADVLENMIN || icmplen < ICMP_ADVLEN(icp) ||
 		    icp->icmp_ip.ip_hl < (sizeof(struct ip) >> 2)) {
-			V_icmpstat.icps_badlen++;
+			ICMPSTAT_INC(icps_badlen);
 			break;
 		}
 		/*
@@ -660,7 +660,7 @@ icmp_reflect(struct mbuf *m)
 	    IN_EXPERIMENTAL(ntohl(ip->ip_src.s_addr)) ||
 	    IN_ZERONET(ntohl(ip->ip_src.s_addr)) ) {
 		m_freem(m);	/* Bad return address */
-		V_icmpstat.icps_badaddr++;
+		ICMPSTAT_INC(icps_badaddr);
 		goto done;	/* Ip_output() will check for broadcast */
 	}
 
@@ -729,7 +729,7 @@ icmp_reflect(struct mbuf *m)
 	ia = ip_rtaddr(ip->ip_dst, M_GETFIB(m));
 	if (ia == NULL) {
 		m_freem(m);
-		V_icmpstat.icps_noroute++;
+		ICMPSTAT_INC(icps_noroute);
 		goto done;
 	}
 match:
diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c
index a75ee72..920d741 100644
--- a/sys/netinet/ip_input.c
+++ b/sys/netinet/ip_input.c
@@ -224,6 +224,23 @@ int fw_one_pass;
 
 static void	ip_freef(struct ipqhead *, struct ipq *);
 
+#ifndef VIMAGE_GLOBALS
+static void vnet_inet_register(void);
+ 
+static const vnet_modinfo_t vnet_inet_modinfo = {
+	.vmi_id		= VNET_MOD_INET,
+	.vmi_name	= "inet",
+};
+ 
+static void vnet_inet_register()
+{
+  
+	vnet_mod_register(&vnet_inet_modinfo);
+}
+ 
+SYSINIT(inet, SI_SUB_PROTO_BEGIN, SI_ORDER_FIRST, vnet_inet_register, 0);
+#endif
+
 /*
  * IP initialization: fill in IP protocol switch table.
  * All protocols not implemented in kernel go to raw IP protocol handler.
@@ -242,6 +259,7 @@ ip_init(void)
 	V_rsvp_on = 0;
 	V_ip_defttl = IPDEFTTL;
 	V_ip_do_randomid = 0;
+	V_ip_id = time_second & 0xffff;
 	V_ipforwarding = 0;
 	V_ipstealth = 0;
 	V_nipq = 0;	/* Total # of reass queues */
@@ -270,6 +288,20 @@ ip_init(void)
 
 	TAILQ_INIT(&V_in_ifaddrhead);
 	V_in_ifaddrhashtbl = hashinit(INADDR_NHASH, M_IFADDR, &V_in_ifaddrhmask);
+
+	/* Initialize IP reassembly queue. */
+	for (i = 0; i < IPREASS_NHASH; i++)
+		TAILQ_INIT(&V_ipq[i]);
+	V_maxnipq = nmbclusters / 32;
+	V_maxfragsperpacket = 16;
+	V_ipq_zone = uma_zcreate("ipq", sizeof(struct ipq), NULL, NULL, NULL,
+	    NULL, UMA_ALIGN_PTR, 0);
+	maxnipq_update();
+
+	/* Skip initialization of globals for non-default instances. */
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
+
 	pr = pffindproto(PF_INET, IPPROTO_RAW, SOCK_RAW);
 	if (pr == NULL)
 		panic("ip_init: PF_INET not found");
@@ -297,16 +329,6 @@ ip_init(void)
 		printf("%s: WARNING: unable to register pfil hook, "
 			"error %d\n", __func__, i);
 
-	/* Initialize IP reassembly queue. */
-	IPQ_LOCK_INIT();
-	for (i = 0; i < IPREASS_NHASH; i++)
-	    TAILQ_INIT(&V_ipq[i]);
-	V_maxnipq = nmbclusters / 32;
-	V_maxfragsperpacket = 16;
-	V_ipq_zone = uma_zcreate("ipq", sizeof(struct ipq), NULL, NULL, NULL,
-	    NULL, UMA_ALIGN_PTR, 0);
-	maxnipq_update();
-
 	/* Start ipport_tick. */
 	callout_init(&ipport_tick_callout, CALLOUT_MPSAFE);
 	ipport_tick(NULL);
@@ -316,7 +338,7 @@ ip_init(void)
 		NULL, EVENTHANDLER_PRI_ANY);
 
 	/* Initialize various other remaining things. */
-	V_ip_id = time_second & 0xffff;
+	IPQ_LOCK_INIT();
 	ipintrq.ifq_maxlen = ipqmaxlen;
 	mtx_init(&ipintrq.ifq_mtx, "ip_inq", NULL, MTX_DEF);
 	netisr_register(NETISR_IP, ip_input, &ipintrq, 0);
@@ -359,31 +381,31 @@ ip_input(struct mbuf *m)
 		goto ours;
 	}
 
-	V_ipstat.ips_total++;
+	IPSTAT_INC(ips_total);
 
 	if (m->m_pkthdr.len < sizeof(struct ip))
 		goto tooshort;
 
 	if (m->m_len < sizeof (struct ip) &&
 	    (m = m_pullup(m, sizeof (struct ip))) == NULL) {
-		V_ipstat.ips_toosmall++;
+		IPSTAT_INC(ips_toosmall);
 		return;
 	}
 	ip = mtod(m, struct ip *);
 
 	if (ip->ip_v != IPVERSION) {
-		V_ipstat.ips_badvers++;
+		IPSTAT_INC(ips_badvers);
 		goto bad;
 	}
 
 	hlen = ip->ip_hl << 2;
 	if (hlen < sizeof(struct ip)) {	/* minimum header length */
-		V_ipstat.ips_badhlen++;
+		IPSTAT_INC(ips_badhlen);
 		goto bad;
 	}
 	if (hlen > m->m_len) {
 		if ((m = m_pullup(m, hlen)) == NULL) {
-			V_ipstat.ips_badhlen++;
+			IPSTAT_INC(ips_badhlen);
 			return;
 		}
 		ip = mtod(m, struct ip *);
@@ -393,7 +415,7 @@ ip_input(struct mbuf *m)
 	if ((ntohl(ip->ip_dst.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET ||
 	    (ntohl(ip->ip_src.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET) {
 		if ((m->m_pkthdr.rcvif->if_flags & IFF_LOOPBACK) == 0) {
-			V_ipstat.ips_badaddr++;
+			IPSTAT_INC(ips_badaddr);
 			goto bad;
 		}
 	}
@@ -408,7 +430,7 @@ ip_input(struct mbuf *m)
 		}
 	}
 	if (sum) {
-		V_ipstat.ips_badsum++;
+		IPSTAT_INC(ips_badsum);
 		goto bad;
 	}
 
@@ -423,7 +445,7 @@ ip_input(struct mbuf *m)
 	 */
 	ip->ip_len = ntohs(ip->ip_len);
 	if (ip->ip_len < hlen) {
-		V_ipstat.ips_badlen++;
+		IPSTAT_INC(ips_badlen);
 		goto bad;
 	}
 	ip->ip_off = ntohs(ip->ip_off);
@@ -436,7 +458,7 @@ ip_input(struct mbuf *m)
 	 */
 	if (m->m_pkthdr.len < ip->ip_len) {
 tooshort:
-		V_ipstat.ips_tooshort++;
+		IPSTAT_INC(ips_tooshort);
 		goto bad;
 	}
 	if (m->m_pkthdr.len > ip->ip_len) {
@@ -587,7 +609,7 @@ passin:
 	}
 	/* RFC 3927 2.7: Do not forward datagrams for 169.254.0.0/16. */
 	if (IN_LINKLOCAL(ntohl(ip->ip_dst.s_addr))) {
-		V_ipstat.ips_cantforward++;
+		IPSTAT_INC(ips_cantforward);
 		m_freem(m);
 		return;
 	}
@@ -603,7 +625,7 @@ passin:
 			 */
 			if (ip_mforward &&
 			    ip_mforward(ip, m->m_pkthdr.rcvif, m, 0) != 0) {
-				V_ipstat.ips_cantforward++;
+				IPSTAT_INC(ips_cantforward);
 				m_freem(m);
 				return;
 			}
@@ -615,7 +637,7 @@ passin:
 			 */
 			if (ip->ip_p == IPPROTO_IGMP)
 				goto ours;
-			V_ipstat.ips_forward++;
+			IPSTAT_INC(ips_forward);
 		}
 		/*
 		 * Assume the packet is for us, to avoid prematurely taking
@@ -645,7 +667,7 @@ passin:
 	 * Not for us; forward if possible and desirable.
 	 */
 	if (V_ipforwarding == 0) {
-		V_ipstat.ips_cantforward++;
+		IPSTAT_INC(ips_cantforward);
 		m_freem(m);
 	} else {
 #ifdef IPSEC
@@ -705,7 +727,7 @@ ours:
 	/*
 	 * Switch out to protocol's input routine.
 	 */
-	V_ipstat.ips_delivered++;
+	IPSTAT_INC(ips_delivered);
 
 	(*inetsw[ip_protox[ip->ip_p]].pr_input)(m, hlen);
 	return;
@@ -803,8 +825,8 @@ ip_reass(struct mbuf *m)
 
 	/* If maxnipq or maxfragsperpacket are 0, never accept fragments. */
 	if (V_maxnipq == 0 || V_maxfragsperpacket == 0) {
-		V_ipstat.ips_fragments++;
-		V_ipstat.ips_fragdropped++;
+		IPSTAT_INC(ips_fragments);
+		IPSTAT_INC(ips_fragdropped);
 		m_freem(m);
 		return (NULL);
 	}
@@ -846,14 +868,14 @@ ip_reass(struct mbuf *m)
 			for (i = 0; i < IPREASS_NHASH; i++) {
 				struct ipq *r = TAILQ_LAST(&V_ipq[i], ipqhead);
 				if (r) {
-					V_ipstat.ips_fragtimeout +=
-					    r->ipq_nfrags;
+					IPSTAT_ADD(ips_fragtimeout,
+					    r->ipq_nfrags);
 					ip_freef(&V_ipq[i], r);
 					break;
 				}
 			}
 		} else {
-			V_ipstat.ips_fragtimeout += q->ipq_nfrags;
+			IPSTAT_ADD(ips_fragtimeout, q->ipq_nfrags);
 			ip_freef(head, q);
 		}
 	}
@@ -870,7 +892,7 @@ found:
 		 * that's a non-zero multiple of 8 bytes.
 		 */
 		if (ip->ip_len == 0 || (ip->ip_len & 0x7) != 0) {
-			V_ipstat.ips_toosmall++; /* XXX */
+			IPSTAT_INC(ips_toosmall); /* XXX */
 			goto dropfrag;
 		}
 		m->m_flags |= M_FRAG;
@@ -883,7 +905,7 @@ found:
 	 * Attempt reassembly; if it succeeds, proceed.
 	 * ip_reass() will return a different mbuf.
 	 */
-	V_ipstat.ips_fragments++;
+	IPSTAT_INC(ips_fragments);
 	m->m_pkthdr.header = ip;
 
 	/* Previous ip_reass() started here. */
@@ -994,7 +1016,7 @@ found:
 		}
 		nq = q->m_nextpkt;
 		m->m_nextpkt = nq;
-		V_ipstat.ips_fragdropped++;
+		IPSTAT_INC(ips_fragdropped);
 		fp->ipq_nfrags--;
 		m_freem(q);
 	}
@@ -1013,7 +1035,7 @@ found:
 	for (p = NULL, q = fp->ipq_frags; q; p = q, q = q->m_nextpkt) {
 		if (GETIP(q)->ip_off != next) {
 			if (fp->ipq_nfrags > V_maxfragsperpacket) {
-				V_ipstat.ips_fragdropped += fp->ipq_nfrags;
+				IPSTAT_ADD(ips_fragdropped, fp->ipq_nfrags);
 				ip_freef(head, fp);
 			}
 			goto done;
@@ -1023,7 +1045,7 @@ found:
 	/* Make sure the last packet didn't have the IP_MF flag */
 	if (p->m_flags & M_FRAG) {
 		if (fp->ipq_nfrags > V_maxfragsperpacket) {
-			V_ipstat.ips_fragdropped += fp->ipq_nfrags;
+			IPSTAT_ADD(ips_fragdropped, fp->ipq_nfrags);
 			ip_freef(head, fp);
 		}
 		goto done;
@@ -1035,8 +1057,8 @@ found:
 	q = fp->ipq_frags;
 	ip = GETIP(q);
 	if (next + (ip->ip_hl << 2) > IP_MAXPACKET) {
-		V_ipstat.ips_toolong++;
-		V_ipstat.ips_fragdropped += fp->ipq_nfrags;
+		IPSTAT_INC(ips_toolong);
+		IPSTAT_ADD(ips_fragdropped, fp->ipq_nfrags);
 		ip_freef(head, fp);
 		goto done;
 	}
@@ -1085,12 +1107,12 @@ found:
 	/* some debugging cruft by sklower, below, will go away soon */
 	if (m->m_flags & M_PKTHDR)	/* XXX this should be done elsewhere */
 		m_fixhdr(m);
-	V_ipstat.ips_reassembled++;
+	IPSTAT_INC(ips_reassembled);
 	IPQ_UNLOCK();
 	return (m);
 
 dropfrag:
-	V_ipstat.ips_fragdropped++;
+	IPSTAT_INC(ips_fragdropped);
 	if (fp != NULL)
 		fp->ipq_nfrags--;
 	m_freem(m);
@@ -1147,8 +1169,8 @@ ip_slowtimo(void)
 				fpp = fp;
 				fp = TAILQ_NEXT(fp, ipq_list);
 				if(--fpp->ipq_ttl == 0) {
-					V_ipstat.ips_fragtimeout +=
-					    fpp->ipq_nfrags;
+					IPSTAT_ADD(ips_fragtimeout,
+					    fpp->ipq_nfrags);
 					ip_freef(&V_ipq[i], fpp);
 				}
 			}
@@ -1162,8 +1184,8 @@ ip_slowtimo(void)
 			for (i = 0; i < IPREASS_NHASH; i++) {
 				while (V_nipq > V_maxnipq &&
 				    !TAILQ_EMPTY(&V_ipq[i])) {
-					V_ipstat.ips_fragdropped +=
-					    TAILQ_FIRST(&V_ipq[i])->ipq_nfrags;
+					IPSTAT_ADD(ips_fragdropped,
+					    TAILQ_FIRST(&V_ipq[i])->ipq_nfrags);
 					ip_freef(&V_ipq[i],
 					    TAILQ_FIRST(&V_ipq[i]));
 				}
@@ -1191,8 +1213,8 @@ ip_drain(void)
 		INIT_VNET_INET(vnet_iter);
 		for (i = 0; i < IPREASS_NHASH; i++) {
 			while(!TAILQ_EMPTY(&V_ipq[i])) {
-				V_ipstat.ips_fragdropped +=
-				    TAILQ_FIRST(&V_ipq[i])->ipq_nfrags;
+				IPSTAT_ADD(ips_fragdropped,
+				    TAILQ_FIRST(&V_ipq[i])->ipq_nfrags);
 				ip_freef(&V_ipq[i], TAILQ_FIRST(&V_ipq[i]));
 			}
 		}
@@ -1324,7 +1346,7 @@ ip_forward(struct mbuf *m, int srcrt)
 	int error, type = 0, code = 0, mtu = 0;
 
 	if (m->m_flags & (M_BCAST|M_MCAST) || in_canforward(ip->ip_dst) == 0) {
-		V_ipstat.ips_cantforward++;
+		IPSTAT_INC(ips_cantforward);
 		m_freem(m);
 		return;
 	}
@@ -1443,11 +1465,11 @@ ip_forward(struct mbuf *m, int srcrt)
 		RTFREE(ro.ro_rt);
 
 	if (error)
-		V_ipstat.ips_cantforward++;
+		IPSTAT_INC(ips_cantforward);
 	else {
-		V_ipstat.ips_forward++;
+		IPSTAT_INC(ips_forward);
 		if (type)
-			V_ipstat.ips_redirectsent++;
+			IPSTAT_INC(ips_redirectsent);
 		else {
 			if (mcopy)
 				m_freem(mcopy);
@@ -1499,7 +1521,7 @@ ip_forward(struct mbuf *m, int srcrt)
 			else
 				mtu = ip_next_mtu(ip->ip_len, 0);
 		}
-		V_ipstat.ips_cantfrag++;
+		IPSTAT_INC(ips_cantfrag);
 		break;
 
 	case ENOBUFS:
diff --git a/sys/netinet/ip_ipsec.c b/sys/netinet/ip_ipsec.c
index 4e23a2b..b9dc483 100644
--- a/sys/netinet/ip_ipsec.c
+++ b/sys/netinet/ip_ipsec.c
@@ -129,7 +129,7 @@ ip_ipsec_fwd(struct mbuf *m)
 	KEY_FREESP(&sp);
 	splx(s);
 	if (error) {
-		V_ipstat.ips_cantforward++;
+		IPSTAT_INC(ips_cantforward);
 		return 1;
 	}
 #endif /* IPSEC */
diff --git a/sys/netinet/ip_mroute.c b/sys/netinet/ip_mroute.c
index 36a5efd..476ad20 100644
--- a/sys/netinet/ip_mroute.c
+++ b/sys/netinet/ip_mroute.c
@@ -53,11 +53,23 @@
  * bandwidth metering and signaling
  */
 
+/*
+ * TODO: Prefix functions with ipmf_.
+ * TODO: Maintain a refcount on if_allmulti() in ifnet or in the protocol
+ * domain attachment (if_afdata) so we can track consumers of that service.
+ * TODO: Deprecate routing socket path for SIOCGETSGCNT and SIOCGETVIFCNT,
+ * move it to socket options.
+ * TODO: Cleanup LSRR removal further.
+ * TODO: Push RSVP stubs into raw_ip.c.
+ * TODO: Use bitstring.h for vif set.
+ * TODO: Fix mrt6_ioctl dangling ref when dynamically loaded.
+ * TODO: Sync ip6_mroute.c with this file.
+ */
+
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
-#include "opt_inet6.h"
 #include "opt_mac.h"
 #include "opt_mrouting.h"
 
@@ -65,7 +77,9 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/kernel.h>
+#include <sys/stddef.h>
 #include <sys/lock.h>
+#include <sys/ktr.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/module.h>
@@ -81,9 +95,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/time.h>
 #include <sys/vimage.h>
+
 #include <net/if.h>
 #include <net/netisr.h>
 #include <net/route.h>
+
 #include <netinet/in.h>
 #include <netinet/igmp.h>
 #include <netinet/in_systm.h>
@@ -98,96 +114,79 @@ __FBSDID("$FreeBSD$");
 #include <netinet/udp.h>
 #include <netinet/vinet.h>
 
-#ifdef INET6
-#include <netinet/ip6.h>
-#include <netinet6/in6_var.h>
-#include <netinet6/ip6_mroute.h>
-#include <netinet6/ip6_var.h>
-#endif
 #include <machine/in_cksum.h>
 
 #include <security/mac/mac_framework.h>
 
-/*
- * Control debugging code for rsvp and multicast routing code.
- * Can only set them with the debugger.
- */
-static u_int    rsvpdebug;		/* non-zero enables debugging	*/
-
-static u_int	mrtdebug;		/* any set of the flags below	*/
-#define		DEBUG_MFC	0x02
-#define		DEBUG_FORWARD	0x04
-#define		DEBUG_EXPIRE	0x08
-#define		DEBUG_XMIT	0x10
-#define		DEBUG_PIM	0x20
+#ifndef KTR_IPMF
+#define KTR_IPMF KTR_SUBSYS
+#endif
 
 #define		VIFI_INVALID	((vifi_t) -1)
+#define		M_HASCL(m)	((m)->m_flags & M_EXT)
 
-#define M_HASCL(m)	((m)->m_flags & M_EXT)
-
-static MALLOC_DEFINE(M_MRTABLE, "mroutetbl", "multicast routing tables");
+static MALLOC_DEFINE(M_MRTABLE, "mroutetbl", "multicast forwarding cache");
 
 /*
  * Locking.  We use two locks: one for the virtual interface table and
  * one for the forwarding table.  These locks may be nested in which case
  * the VIF lock must always be taken first.  Note that each lock is used
  * to cover not only the specific data structure but also related data
- * structures.  It may be better to add more fine-grained locking later;
- * it's not clear how performance-critical this code is.
- *
- * XXX: This module could particularly benefit from being cleaned
- *      up to use the <sys/queue.h> macros.
- *
+ * structures.
  */
 
-static struct mrtstat	mrtstat;
-SYSCTL_STRUCT(_net_inet_ip, OID_AUTO, mrtstat, CTLFLAG_RW,
-    &mrtstat, mrtstat,
-    "Multicast Routing Statistics (struct mrtstat, netinet/ip_mroute.h)");
-
-static struct mfc	*mfctable[MFCTBLSIZ];
-SYSCTL_OPAQUE(_net_inet_ip, OID_AUTO, mfctable, CTLFLAG_RD,
-    &mfctable, sizeof(mfctable), "S,*mfc[MFCTBLSIZ]",
-    "Multicast Forwarding Table (struct *mfc[MFCTBLSIZ], netinet/ip_mroute.h)");
-
 static struct mtx mrouter_mtx;
 #define	MROUTER_LOCK()		mtx_lock(&mrouter_mtx)
 #define	MROUTER_UNLOCK()	mtx_unlock(&mrouter_mtx)
 #define	MROUTER_LOCK_ASSERT()	mtx_assert(&mrouter_mtx, MA_OWNED)
-#define	MROUTER_LOCK_INIT()	\
+#define	MROUTER_LOCK_INIT()						\
 	mtx_init(&mrouter_mtx, "IPv4 multicast forwarding", NULL, MTX_DEF)
 #define	MROUTER_LOCK_DESTROY()	mtx_destroy(&mrouter_mtx)
 
+static struct mrtstat	mrtstat;
+SYSCTL_STRUCT(_net_inet_ip, OID_AUTO, mrtstat, CTLFLAG_RW,
+    &mrtstat, mrtstat,
+    "IPv4 Multicast Forwarding Statistics (struct mrtstat, "
+    "netinet/ip_mroute.h)");
+
+static u_long			 mfchash;
+#define MFCHASH(a, g)							\
+	((((a).s_addr >> 20) ^ ((a).s_addr >> 10) ^ (a).s_addr ^ \
+	  ((g).s_addr >> 20) ^ ((g).s_addr >> 10) ^ (g).s_addr) & mfchash)
+#define MFCHASHSIZE	256
+
+static u_char			*nexpire;	/* 0..mfchashsize-1 */
+static u_long			 mfchashsize;	/* Hash size */
+LIST_HEAD(mfchashhdr, mfc)	*mfchashtbl;
+
 static struct mtx mfc_mtx;
-#define	MFC_LOCK()	mtx_lock(&mfc_mtx)
-#define	MFC_UNLOCK()	mtx_unlock(&mfc_mtx)
+#define	MFC_LOCK()		mtx_lock(&mfc_mtx)
+#define	MFC_UNLOCK()		mtx_unlock(&mfc_mtx)
 #define	MFC_LOCK_ASSERT()	mtx_assert(&mfc_mtx, MA_OWNED)
-#define	MFC_LOCK_INIT()	mtx_init(&mfc_mtx, "mroute mfc table", NULL, MTX_DEF)
+#define	MFC_LOCK_INIT()							\
+	mtx_init(&mfc_mtx, "IPv4 multicast forwarding cache", NULL, MTX_DEF)
 #define	MFC_LOCK_DESTROY()	mtx_destroy(&mfc_mtx)
 
+static vifi_t		numvifs;
 static struct vif	viftable[MAXVIFS];
 SYSCTL_OPAQUE(_net_inet_ip, OID_AUTO, viftable, CTLFLAG_RD,
     &viftable, sizeof(viftable), "S,vif[MAXVIFS]",
-    "Multicast Virtual Interfaces (struct vif[MAXVIFS], netinet/ip_mroute.h)");
+    "IPv4 Multicast Interfaces (struct vif[MAXVIFS], netinet/ip_mroute.h)");
 
 static struct mtx vif_mtx;
-#define	VIF_LOCK()	mtx_lock(&vif_mtx)
-#define	VIF_UNLOCK()	mtx_unlock(&vif_mtx)
+#define	VIF_LOCK()		mtx_lock(&vif_mtx)
+#define	VIF_UNLOCK()		mtx_unlock(&vif_mtx)
 #define	VIF_LOCK_ASSERT()	mtx_assert(&vif_mtx, MA_OWNED)
-#define	VIF_LOCK_INIT()	mtx_init(&vif_mtx, "mroute vif table", NULL, MTX_DEF)
+#define	VIF_LOCK_INIT()							\
+	mtx_init(&vif_mtx, "IPv4 multicast interfaces", NULL, MTX_DEF)
 #define	VIF_LOCK_DESTROY()	mtx_destroy(&vif_mtx)
 
-static u_char		nexpire[MFCTBLSIZ];
-
 static eventhandler_tag if_detach_event_tag = NULL;
 
 static struct callout expire_upcalls_ch;
-
 #define		EXPIRE_TIMEOUT	(hz / 4)	/* 4x / second		*/
 #define		UPCALL_EXPIRE	6		/* number of timeouts	*/
 
-#define ENCAP_TTL 64
-
 /*
  * Bandwidth meter variables and constants
  */
@@ -223,7 +222,7 @@ SYSCTL_ULONG(_net_inet_pim, OID_AUTO, squelch_wholepkt, CTLFLAG_RW,
     "Disable IGMP_WHOLEPKT notifications if rendezvous point is unspecified");
 
 extern  struct domain inetdomain;
-struct protosw in_pim_protosw = {
+static const struct protosw in_pim_protosw = {
 	.pr_type =		SOCK_RAW,
 	.pr_domain =		&inetdomain,
 	.pr_protocol =		IPPROTO_PIM,
@@ -235,18 +234,6 @@ struct protosw in_pim_protosw = {
 };
 static const struct encaptab *pim_encap_cookie;
 
-#ifdef INET6
-/* ip6_mroute.c glue */
-extern struct in6_protosw in6_pim_protosw;
-static const struct encaptab *pim6_encap_cookie;
-
-extern int X_ip6_mrouter_set(struct socket *, struct sockopt *);
-extern int X_ip6_mrouter_get(struct socket *, struct sockopt *);
-extern int X_ip6_mrouter_done(void);
-extern int X_ip6_mforward(struct ip6_hdr *, struct ifnet *, struct mbuf *);
-extern int X_mrt6_ioctl(int, caddr_t);
-#endif
-
 static int pim_encapcheck(const struct mbuf *, int, int, void *);
 
 /*
@@ -264,6 +251,7 @@ struct pim_encap_pimhdr {
 	struct pim pim;
 	uint32_t   flags;
 };
+#define		PIM_ENCAP_TTL	64
 
 static struct ip pim_encap_iphdr = {
 #if BYTE_ORDER == LITTLE_ENDIAN
@@ -277,7 +265,7 @@ static struct ip pim_encap_iphdr = {
 	sizeof(struct ip),	/* total length */
 	0,			/* id */
 	0,			/* frag offset */
-	ENCAP_TTL,
+	PIM_ENCAP_TTL,
 	IPPROTO_PIM,
 	0,			/* checksum */
 };
@@ -297,129 +285,97 @@ static vifi_t reg_vif_num = VIFI_INVALID;
 /*
  * Private variables.
  */
-static vifi_t	   numvifs;
 
-static u_long	X_ip_mcast_src(int vifi);
-static int	X_ip_mforward(struct ip *ip, struct ifnet *ifp,
-			struct mbuf *m, struct ip_moptions *imo);
+static u_long	X_ip_mcast_src(int);
+static int	X_ip_mforward(struct ip *, struct ifnet *, struct mbuf *,
+		    struct ip_moptions *);
 static int	X_ip_mrouter_done(void);
-static int	X_ip_mrouter_get(struct socket *so, struct sockopt *m);
-static int	X_ip_mrouter_set(struct socket *so, struct sockopt *m);
-static int	X_legal_vif_num(int vif);
-static int	X_mrt_ioctl(int cmd, caddr_t data, int fibnum);
-
-static int get_sg_cnt(struct sioc_sg_req *);
-static int get_vif_cnt(struct sioc_vif_req *);
-static void if_detached_event(void *arg __unused, struct ifnet *);
-static int ip_mrouter_init(struct socket *, int);
-static int add_vif(struct vifctl *);
-static int del_vif_locked(vifi_t);
-static int del_vif(vifi_t);
-static int add_mfc(struct mfcctl2 *);
-static int del_mfc(struct mfcctl2 *);
-static int set_api_config(uint32_t *); /* chose API capabilities */
-static int socket_send(struct socket *, struct mbuf *, struct sockaddr_in *);
-static int set_assert(int);
-static void expire_upcalls(void *);
-static int ip_mdq(struct mbuf *, struct ifnet *, struct mfc *, vifi_t);
-static void phyint_send(struct ip *, struct vif *, struct mbuf *);
-static void send_packet(struct vif *, struct mbuf *);
-
-/*
- * Bandwidth monitoring
- */
-static void free_bw_list(struct bw_meter *list);
-static int add_bw_upcall(struct bw_upcall *);
-static int del_bw_upcall(struct bw_upcall *);
-static void bw_meter_receive_packet(struct bw_meter *x, int plen,
-		struct timeval *nowp);
-static void bw_meter_prepare_upcall(struct bw_meter *x, struct timeval *nowp);
-static void bw_upcalls_send(void);
-static void schedule_bw_meter(struct bw_meter *x, struct timeval *nowp);
-static void unschedule_bw_meter(struct bw_meter *x);
-static void bw_meter_process(void);
-static void expire_bw_upcalls_send(void *);
-static void expire_bw_meter_process(void *);
-
-static int pim_register_send(struct ip *, struct vif *,
-		struct mbuf *, struct mfc *);
-static int pim_register_send_rp(struct ip *, struct vif *,
-		struct mbuf *, struct mfc *);
-static int pim_register_send_upcall(struct ip *, struct vif *,
-		struct mbuf *, struct mfc *);
-static struct mbuf *pim_register_prepare(struct ip *, struct mbuf *);
-
-/*
- * whether or not special PIM assert processing is enabled.
- */
-static int pim_assert;
-/*
- * Rate limit for assert notification messages, in usec
- */
-#define ASSERT_MSG_TIME		3000000
+static int	X_ip_mrouter_get(struct socket *, struct sockopt *);
+static int	X_ip_mrouter_set(struct socket *, struct sockopt *);
+static int	X_legal_vif_num(int);
+static int	X_mrt_ioctl(int, caddr_t, int);
+
+static int	add_bw_upcall(struct bw_upcall *);
+static int	add_mfc(struct mfcctl2 *);
+static int	add_vif(struct vifctl *);
+static void	bw_meter_prepare_upcall(struct bw_meter *, struct timeval *);
+static void	bw_meter_process(void);
+static void	bw_meter_receive_packet(struct bw_meter *, int,
+		    struct timeval *);
+static void	bw_upcalls_send(void);
+static int	del_bw_upcall(struct bw_upcall *);
+static int	del_mfc(struct mfcctl2 *);
+static int	del_vif(vifi_t);
+static int	del_vif_locked(vifi_t);
+static void	expire_bw_meter_process(void *);
+static void	expire_bw_upcalls_send(void *);
+static void	expire_mfc(struct mfc *);
+static void	expire_upcalls(void *);
+static void	free_bw_list(struct bw_meter *);
+static int	get_sg_cnt(struct sioc_sg_req *);
+static int	get_vif_cnt(struct sioc_vif_req *);
+static void	if_detached_event(void *, struct ifnet *);
+static int	ip_mdq(struct mbuf *, struct ifnet *, struct mfc *, vifi_t);
+static int	ip_mrouter_init(struct socket *, int);
+static __inline struct mfc *
+		mfc_find(struct in_addr *, struct in_addr *);
+static void	phyint_send(struct ip *, struct vif *, struct mbuf *);
+static struct mbuf *
+		pim_register_prepare(struct ip *, struct mbuf *);
+static int	pim_register_send(struct ip *, struct vif *,
+		    struct mbuf *, struct mfc *);
+static int	pim_register_send_rp(struct ip *, struct vif *,
+		    struct mbuf *, struct mfc *);
+static int	pim_register_send_upcall(struct ip *, struct vif *,
+		    struct mbuf *, struct mfc *);
+static void	schedule_bw_meter(struct bw_meter *, struct timeval *);
+static void	send_packet(struct vif *, struct mbuf *);
+static int	set_api_config(uint32_t *);
+static int	set_assert(int);
+static int	socket_send(struct socket *, struct mbuf *,
+		    struct sockaddr_in *);
+static void	unschedule_bw_meter(struct bw_meter *);
 
 /*
- * Kernel multicast routing API capabilities and setup.
+ * Kernel multicast forwarding API capabilities and setup.
  * If more API capabilities are added to the kernel, they should be
  * recorded in `mrt_api_support'.
  */
+#define MRT_API_VERSION		0x0305
+
+static const int mrt_api_version = MRT_API_VERSION;
 static const uint32_t mrt_api_support = (MRT_MFC_FLAGS_DISABLE_WRONGVIF |
 					 MRT_MFC_FLAGS_BORDER_VIF |
 					 MRT_MFC_RP |
 					 MRT_MFC_BW_UPCALL);
 static uint32_t mrt_api_config = 0;
 
-/*
- * Hash function for a source, group entry
- */
-#define MFCHASH(a, g) MFCHASHMOD(((a) >> 20) ^ ((a) >> 10) ^ (a) ^ \
-			((g) >> 20) ^ ((g) >> 10) ^ (g))
+static int pim_assert_enabled;
+static struct timeval pim_assert_interval = { 3, 0 };	/* Rate limit */
 
 /*
- * Find a route for a given origin IP address and Multicast group address
- * Statistics are updated by the caller if needed
- * (mrtstat.mrts_mfc_lookups and mrtstat.mrts_mfc_misses)
+ * Find a route for a given origin IP address and multicast group address.
+ * Statistics must be updated by the caller.
  */
-static struct mfc *
-mfc_find(in_addr_t o, in_addr_t g)
+static __inline struct mfc *
+mfc_find(struct in_addr *o, struct in_addr *g)
 {
-    struct mfc *rt;
+	struct mfc *rt;
 
-    MFC_LOCK_ASSERT();
+	MFC_LOCK_ASSERT();
 
-    for (rt = mfctable[MFCHASH(o,g)]; rt; rt = rt->mfc_next)
-	if ((rt->mfc_origin.s_addr == o) &&
-		(rt->mfc_mcastgrp.s_addr == g) && (rt->mfc_stall == NULL))
-	    break;
-    return rt;
-}
+	LIST_FOREACH(rt, &mfchashtbl[MFCHASH(*o, *g)], mfc_hash) {
+		if (in_hosteq(rt->mfc_origin, *o) &&
+		    in_hosteq(rt->mfc_mcastgrp, *g) &&
+		    TAILQ_EMPTY(&rt->mfc_stall))
+			break;
+	}
 
-/*
- * Macros to compute elapsed time efficiently
- * Borrowed from Van Jacobson's scheduling code
- */
-#define TV_DELTA(a, b, delta) {					\
-	int xxs;						\
-	delta = (a).tv_usec - (b).tv_usec;			\
-	if ((xxs = (a).tv_sec - (b).tv_sec)) {			\
-		switch (xxs) {					\
-		case 2:						\
-		      delta += 1000000;				\
-		      /* FALLTHROUGH */				\
-		case 1:						\
-		      delta += 1000000;				\
-		      break;					\
-		default:					\
-		      delta += (1000000 * xxs);			\
-		}						\
-	}							\
+	return (rt);
 }
 
-#define TV_LT(a, b) (((a).tv_usec < (b).tv_usec && \
-	      (a).tv_sec <= (b).tv_sec) || (a).tv_sec < (b).tv_sec)
-
 /*
- * Handle MRT setsockopt commands to modify the multicast routing tables.
+ * Handle MRT setsockopt commands to modify the multicast forwarding tables.
  */
 static int
 X_ip_mrouter_set(struct socket *so, struct sockopt *sopt)
@@ -526,15 +482,15 @@ static int
 X_ip_mrouter_get(struct socket *so, struct sockopt *sopt)
 {
     int error;
-    static int version = 0x0305; /* !!! why is this here? XXX */
 
     switch (sopt->sopt_name) {
     case MRT_VERSION:
-	error = sooptcopyout(sopt, &version, sizeof version);
+	error = sooptcopyout(sopt, &mrt_api_version, sizeof mrt_api_version);
 	break;
 
     case MRT_ASSERT:
-	error = sooptcopyout(sopt, &pim_assert, sizeof pim_assert);
+	error = sooptcopyout(sopt, &pim_assert_enabled,
+	    sizeof pim_assert_enabled);
 	break;
 
     case MRT_API_SUPPORT:
@@ -556,7 +512,7 @@ X_ip_mrouter_get(struct socket *so, struct sockopt *sopt)
  * Handle ioctl commands to obtain information from the cache
  */
 static int
-X_mrt_ioctl(int cmd, caddr_t data, int fibnum)
+X_mrt_ioctl(int cmd, caddr_t data, int fibnum __unused)
 {
     int error = 0;
 
@@ -593,7 +549,7 @@ get_sg_cnt(struct sioc_sg_req *req)
     struct mfc *rt;
 
     MFC_LOCK();
-    rt = mfc_find(req->src.s_addr, req->grp.s_addr);
+    rt = mfc_find(&req->src, &req->grp);
     if (rt == NULL) {
 	MFC_UNLOCK();
 	req->pktcnt = req->bytecnt = req->wrong_if = 0xffffffff;
@@ -632,10 +588,8 @@ get_vif_cnt(struct sioc_vif_req *req)
 static void
 ip_mrouter_reset(void)
 {
-    bzero((caddr_t)mfctable, sizeof(mfctable));
-    bzero((caddr_t)nexpire, sizeof(nexpire));
 
-    pim_assert = 0;
+    pim_assert_enabled = 0;
     mrt_api_config = 0;
 
     callout_init(&expire_upcalls_ch, CALLOUT_MPSAFE);
@@ -652,55 +606,40 @@ if_detached_event(void *arg __unused, struct ifnet *ifp)
     INIT_VNET_INET(curvnet);
     vifi_t vifi;
     int i;
-    struct mfc *mfc;
-    struct mfc *nmfc;
-    struct mfc **ppmfc;	/* Pointer to previous node's next-pointer */
-    struct rtdetq *pq;
-    struct rtdetq *npq;
 
     MROUTER_LOCK();
+
     if (V_ip_mrouter == NULL) {
 	MROUTER_UNLOCK();
+	return;
     }
 
+    VIF_LOCK();
+    MFC_LOCK();
+
     /*
      * Tear down multicast forwarder state associated with this ifnet.
      * 1. Walk the vif list, matching vifs against this ifnet.
      * 2. Walk the multicast forwarding cache (mfc) looking for
      *    inner matches with this vif's index.
-     * 3. Free any pending mbufs for this mfc.
-     * 4. Free the associated mfc entry and state associated with this vif.
-     *    Be very careful about unlinking from a singly-linked list whose
-     *    "head node" is a pointer in a simple array.
-     * 5. Free vif state. This should disable ALLMULTI on the interface.
+     * 3. Expire any matching multicast forwarding cache entries.
+     * 4. Free vif state. This should disable ALLMULTI on the interface.
      */
-    VIF_LOCK();
-    MFC_LOCK();
     for (vifi = 0; vifi < numvifs; vifi++) {
 	if (viftable[vifi].v_ifp != ifp)
 		continue;
-	for (i = 0; i < MFCTBLSIZ; i++) {
-	    ppmfc = &mfctable[i];
-	    for (mfc = mfctable[i]; mfc != NULL; ) {
-		nmfc = mfc->mfc_next;
-		if (mfc->mfc_parent == vifi) {
-		    for (pq = mfc->mfc_stall; pq != NULL; ) {
-			npq = pq->next;
-			m_freem(pq->m);
-			free(pq, M_MRTABLE);
-			pq = npq;
-		    }
-		    free_bw_list(mfc->mfc_bw_meter);
-		    free(mfc, M_MRTABLE);
-		    *ppmfc = nmfc;
-		} else {
-		    ppmfc = &mfc->mfc_next;
+	for (i = 0; i < mfchashsize; i++) {
+		struct mfc *rt, *nrt;
+		for (rt = LIST_FIRST(&mfchashtbl[i]); rt; rt = nrt) {
+			nrt = LIST_NEXT(rt, mfc_hash);
+			if (rt->mfc_parent == vifi) {
+				expire_mfc(rt);
+			}
 		}
-		mfc = nmfc;
-	    }
 	}
 	del_vif_locked(vifi);
     }
+
     MFC_UNLOCK();
     VIF_UNLOCK();
 
@@ -708,16 +647,15 @@ if_detached_event(void *arg __unused, struct ifnet *ifp)
 }
                         
 /*
- * Enable multicast routing
+ * Enable multicast forwarding.
  */
 static int
 ip_mrouter_init(struct socket *so, int version)
 {
     INIT_VNET_INET(curvnet);
 
-    if (mrtdebug)
-	log(LOG_DEBUG, "ip_mrouter_init: so_type = %d, pr_protocol = %d\n",
-	    so->so_type, so->so_proto->pr_protocol);
+    CTR3(KTR_IPMF, "%s: so_type %d, pr_protocol %d", __func__,
+        so->so_type, so->so_proto->pr_protocol);
 
     if (so->so_type != SOCK_RAW || so->so_proto->pr_protocol != IPPROTO_IGMP)
 	return EOPNOTSUPP;
@@ -739,6 +677,8 @@ ip_mrouter_init(struct socket *so, int version)
 	return (ENOMEM);
     }
 
+    mfchashtbl = hashinit_flags(mfchashsize, M_MRTABLE, &mfchash, HASH_NOWAIT);
+
     callout_reset(&expire_upcalls_ch, EXPIRE_TIMEOUT, expire_upcalls, NULL);
 
     callout_reset(&bw_upcalls_ch, BW_UPCALLS_PERIOD,
@@ -749,14 +689,13 @@ ip_mrouter_init(struct socket *so, int version)
 
     MROUTER_UNLOCK();
 
-    if (mrtdebug)
-	log(LOG_DEBUG, "ip_mrouter_init\n");
+    CTR1(KTR_IPMF, "%s: done", __func__);
 
     return 0;
 }
 
 /*
- * Disable multicast routing
+ * Disable multicast forwarding.
  */
 static int
 X_ip_mrouter_done(void)
@@ -766,8 +705,6 @@ X_ip_mrouter_done(void)
     int i;
     struct ifnet *ifp;
     struct ifreq ifr;
-    struct mfc *rt;
-    struct rtdetq *rte;
 
     MROUTER_LOCK();
 
@@ -783,12 +720,13 @@ X_ip_mrouter_done(void)
     mrt_api_config = 0;
 
     VIF_LOCK();
+
     /*
      * For each phyint in use, disable promiscuous reception of all IP
      * multicasts.
      */
     for (vifi = 0; vifi < numvifs; vifi++) {
-	if (viftable[vifi].v_lcl_addr.s_addr != 0 &&
+	if (!in_nullhost(viftable[vifi].v_lcl_addr) &&
 		!(viftable[vifi].v_flags & (VIFF_TUNNEL | VIFF_REGISTER))) {
 	    struct sockaddr_in *so = (struct sockaddr_in *)&(ifr.ifr_addr);
 
@@ -801,46 +739,44 @@ X_ip_mrouter_done(void)
     }
     bzero((caddr_t)viftable, sizeof(viftable));
     numvifs = 0;
-    pim_assert = 0;
+    pim_assert_enabled = 0;
+
     VIF_UNLOCK();
+
     EVENTHANDLER_DEREGISTER(ifnet_departure_event, if_detach_event_tag);
 
-    /*
-     * Free all multicast forwarding cache entries.
-     */
     callout_stop(&expire_upcalls_ch);
     callout_stop(&bw_upcalls_ch);
     callout_stop(&bw_meter_ch);
 
     MFC_LOCK();
-    for (i = 0; i < MFCTBLSIZ; i++) {
-	for (rt = mfctable[i]; rt != NULL; ) {
-	    struct mfc *nr = rt->mfc_next;
 
-	    for (rte = rt->mfc_stall; rte != NULL; ) {
-		struct rtdetq *n = rte->next;
-
-		m_freem(rte->m);
-		free(rte, M_MRTABLE);
-		rte = n;
-	    }
-	    free_bw_list(rt->mfc_bw_meter);
-	    free(rt, M_MRTABLE);
-	    rt = nr;
+    /*
+     * Free all multicast forwarding cache entries.
+     * Do not use hashdestroy(), as we must perform other cleanup.
+     */
+    for (i = 0; i < mfchashsize; i++) {
+	struct mfc *rt, *nrt;
+	for (rt = LIST_FIRST(&mfchashtbl[i]); rt; rt = nrt) {
+		nrt = LIST_NEXT(rt, mfc_hash);
+		expire_mfc(rt);
 	}
     }
-    bzero((caddr_t)mfctable, sizeof(mfctable));
-    bzero((caddr_t)nexpire, sizeof(nexpire));
+    free(mfchashtbl, M_MRTABLE);
+    mfchashtbl = NULL;
+
+    bzero(nexpire, sizeof(nexpire[0]) * mfchashsize);
+
     bw_upcalls_n = 0;
     bzero(bw_meter_timers, sizeof(bw_meter_timers));
+
     MFC_UNLOCK();
 
     reg_vif_num = VIFI_INVALID;
 
     MROUTER_UNLOCK();
 
-    if (mrtdebug)
-	log(LOG_DEBUG, "ip_mrouter_done\n");
+    CTR1(KTR_IPMF, "%s: done", __func__);
 
     return 0;
 }
@@ -854,7 +790,7 @@ set_assert(int i)
     if ((i != 1) && (i != 0))
 	return EINVAL;
 
-    pim_assert = i;
+    pim_assert_enabled = i;
 
     return 0;
 }
@@ -878,17 +814,22 @@ set_api_config(uint32_t *apival)
 	*apival = 0;
 	return EPERM;
     }
-    if (pim_assert) {
+    if (pim_assert_enabled) {
 	*apival = 0;
 	return EPERM;
     }
-    for (i = 0; i < MFCTBLSIZ; i++) {
-	if (mfctable[i] != NULL) {
+
+    MFC_LOCK();
+
+    for (i = 0; i < mfchashsize; i++) {
+	if (LIST_FIRST(&mfchashtbl[i]) != NULL) {
 	    *apival = 0;
 	    return EPERM;
 	}
     }
 
+    MFC_UNLOCK();
+
     mrt_api_config = *apival & mrt_api_support;
     *apival = mrt_api_config;
 
@@ -918,11 +859,11 @@ add_vif(struct vifctl *vifcp)
 	VIF_UNLOCK();
 	return EINVAL;
     }
-    if (vifp->v_lcl_addr.s_addr != INADDR_ANY) {
+    if (!in_nullhost(vifp->v_lcl_addr)) {
 	VIF_UNLOCK();
 	return EADDRINUSE;
     }
-    if (vifcp->vifc_lcl_addr.s_addr == INADDR_ANY) {
+    if (in_nullhost(vifcp->vifc_lcl_addr)) {
 	VIF_UNLOCK();
 	return EADDRNOTAVAIL;
     }
@@ -946,14 +887,12 @@ add_vif(struct vifctl *vifcp)
     }
 
     if ((vifcp->vifc_flags & VIFF_TUNNEL) != 0) {
-	log(LOG_ERR, "tunnels are no longer supported\n");
+	CTR1(KTR_IPMF, "%s: tunnels are no longer supported", __func__);
 	VIF_UNLOCK();
 	return EOPNOTSUPP;
     } else if (vifcp->vifc_flags & VIFF_REGISTER) {
 	ifp = &multicast_register_if;
-	if (mrtdebug)
-	    log(LOG_DEBUG, "Adding a register vif, ifp: %p\n",
-		    (void *)&multicast_register_if);
+	CTR2(KTR_IPMF, "%s: add register vif for ifp %p", __func__, ifp);
 	if (reg_vif_num == VIFI_INVALID) {
 	    if_initname(&multicast_register_if, "register_vif", 0);
 	    multicast_register_if.if_flags = IFF_LOOPBACK;
@@ -978,8 +917,6 @@ add_vif(struct vifctl *vifcp)
     vifp->v_lcl_addr  = vifcp->vifc_lcl_addr;
     vifp->v_rmt_addr  = vifcp->vifc_rmt_addr;
     vifp->v_ifp       = ifp;
-    vifp->v_rsvp_on   = 0;
-    vifp->v_rsvpd     = NULL;
     /* initialize per vif pkt counters */
     vifp->v_pkt_in    = 0;
     vifp->v_pkt_out   = 0;
@@ -988,17 +925,14 @@ add_vif(struct vifctl *vifcp)
     bzero(&vifp->v_route, sizeof(vifp->v_route));
 
     /* Adjust numvifs up if the vifi is higher than numvifs */
-    if (numvifs <= vifcp->vifc_vifi) numvifs = vifcp->vifc_vifi + 1;
+    if (numvifs <= vifcp->vifc_vifi)
+	numvifs = vifcp->vifc_vifi + 1;
 
     VIF_UNLOCK();
 
-    if (mrtdebug)
-	log(LOG_DEBUG, "add_vif #%d, lcladdr %lx, %s %lx, thresh %x\n",
-	    vifcp->vifc_vifi,
-	    (u_long)ntohl(vifcp->vifc_lcl_addr.s_addr),
-	    (vifcp->vifc_flags & VIFF_TUNNEL) ? "rmtaddr" : "mask",
-	    (u_long)ntohl(vifcp->vifc_rmt_addr.s_addr),
-	    vifcp->vifc_threshold);
+    CTR4(KTR_IPMF, "%s: add vif %d laddr %s thresh %x", __func__,
+	(int)vifcp->vifc_vifi, inet_ntoa(vifcp->vifc_lcl_addr),
+	(int)vifcp->vifc_threshold);
 
     return 0;
 }
@@ -1017,7 +951,7 @@ del_vif_locked(vifi_t vifi)
 	return EINVAL;
     }
     vifp = &viftable[vifi];
-    if (vifp->v_lcl_addr.s_addr == INADDR_ANY) {
+    if (in_nullhost(vifp->v_lcl_addr)) {
 	return EADDRNOTAVAIL;
     }
 
@@ -1029,12 +963,11 @@ del_vif_locked(vifi_t vifi)
 
     bzero((caddr_t)vifp, sizeof (*vifp));
 
-    if (mrtdebug)
-	log(LOG_DEBUG, "del_vif %d, numvifs %d\n", vifi, numvifs);
+    CTR2(KTR_IPMF, "%s: delete vif %d", __func__, (int)vifi);
 
     /* Adjust numvifs down */
     for (vifi = numvifs; vifi > 0; vifi--)
-	if (viftable[vifi-1].v_lcl_addr.s_addr != INADDR_ANY)
+	if (!in_nullhost(viftable[vifi-1].v_lcl_addr))
 	    break;
     numvifs = vifi;
 
@@ -1089,9 +1022,25 @@ init_mfc_params(struct mfc *rt, struct mfcctl2 *mfccp)
     rt->mfc_pkt_cnt    = 0;
     rt->mfc_byte_cnt   = 0;
     rt->mfc_wrong_if   = 0;
-    rt->mfc_last_assert.tv_sec = rt->mfc_last_assert.tv_usec = 0;
+    timevalclear(&rt->mfc_last_assert);
 }
 
+static void
+expire_mfc(struct mfc *rt)
+{
+	struct rtdetq *rte, *nrte;
+
+	free_bw_list(rt->mfc_bw_meter);
+
+	TAILQ_FOREACH_SAFE(rte, &rt->mfc_stall, rte_link, nrte) {
+		m_freem(rte->m);
+		TAILQ_REMOVE(&rt->mfc_stall, rte, rte_link);
+		free(rte, M_MRTABLE);
+	}
+
+	LIST_REMOVE(rt, mfc_hash);
+	free(rt, M_MRTABLE);
+}
 
 /*
  * Add an mfc entry
@@ -1100,67 +1049,58 @@ static int
 add_mfc(struct mfcctl2 *mfccp)
 {
     struct mfc *rt;
-    u_long hash;
-    struct rtdetq *rte;
+    struct rtdetq *rte, *nrte;
+    u_long hash = 0;
     u_short nstl;
 
     VIF_LOCK();
     MFC_LOCK();
 
-    rt = mfc_find(mfccp->mfcc_origin.s_addr, mfccp->mfcc_mcastgrp.s_addr);
+    rt = mfc_find(&mfccp->mfcc_origin, &mfccp->mfcc_mcastgrp);
 
     /* If an entry already exists, just update the fields */
     if (rt) {
-	if (mrtdebug & DEBUG_MFC)
-	    log(LOG_DEBUG,"add_mfc update o %lx g %lx p %x\n",
-		(u_long)ntohl(mfccp->mfcc_origin.s_addr),
-		(u_long)ntohl(mfccp->mfcc_mcastgrp.s_addr),
-		mfccp->mfcc_parent);
-
+	CTR4(KTR_IPMF, "%s: update mfc orig %s group %lx parent %x",
+	    __func__, inet_ntoa(mfccp->mfcc_origin),
+	    (u_long)ntohl(mfccp->mfcc_mcastgrp.s_addr),
+	    mfccp->mfcc_parent);
 	update_mfc_params(rt, mfccp);
 	MFC_UNLOCK();
 	VIF_UNLOCK();
-	return 0;
+	return (0);
     }
 
     /*
      * Find the entry for which the upcall was made and update
      */
-    hash = MFCHASH(mfccp->mfcc_origin.s_addr, mfccp->mfcc_mcastgrp.s_addr);
-    for (rt = mfctable[hash], nstl = 0; rt; rt = rt->mfc_next) {
-
-	if ((rt->mfc_origin.s_addr == mfccp->mfcc_origin.s_addr) &&
-		(rt->mfc_mcastgrp.s_addr == mfccp->mfcc_mcastgrp.s_addr) &&
-		(rt->mfc_stall != NULL)) {
-
-	    if (nstl++)
-		log(LOG_ERR, "add_mfc %s o %lx g %lx p %x dbx %p\n",
-		    "multiple kernel entries",
-		    (u_long)ntohl(mfccp->mfcc_origin.s_addr),
-		    (u_long)ntohl(mfccp->mfcc_mcastgrp.s_addr),
-		    mfccp->mfcc_parent, (void *)rt->mfc_stall);
-
-	    if (mrtdebug & DEBUG_MFC)
-		log(LOG_DEBUG,"add_mfc o %lx g %lx p %x dbg %p\n",
-		    (u_long)ntohl(mfccp->mfcc_origin.s_addr),
+    nstl = 0;
+    hash = MFCHASH(mfccp->mfcc_origin, mfccp->mfcc_mcastgrp);
+    LIST_FOREACH(rt, &mfchashtbl[hash], mfc_hash) {
+	if (in_hosteq(rt->mfc_origin, mfccp->mfcc_origin) &&
+	    in_hosteq(rt->mfc_mcastgrp, mfccp->mfcc_mcastgrp) &&
+	    !TAILQ_EMPTY(&rt->mfc_stall)) {
+		CTR5(KTR_IPMF,
+		    "%s: add mfc orig %s group %lx parent %x qh %p",
+		    __func__, inet_ntoa(mfccp->mfcc_origin),
 		    (u_long)ntohl(mfccp->mfcc_mcastgrp.s_addr),
-		    mfccp->mfcc_parent, (void *)rt->mfc_stall);
-
-	    init_mfc_params(rt, mfccp);
-
-	    rt->mfc_expire = 0;	/* Don't clean this guy up */
-	    nexpire[hash]--;
-
-	    /* free packets Qed at the end of this entry */
-	    for (rte = rt->mfc_stall; rte != NULL; ) {
-		struct rtdetq *n = rte->next;
+		    mfccp->mfcc_parent,
+		    TAILQ_FIRST(&rt->mfc_stall));
+		if (nstl++)
+			CTR1(KTR_IPMF, "%s: multiple matches", __func__);
 
-		ip_mdq(rte->m, rte->ifp, rt, -1);
-		m_freem(rte->m);
-		free(rte, M_MRTABLE);
-		rte = n;
-	    }
-	    rt->mfc_stall = NULL;
+		init_mfc_params(rt, mfccp);
+		rt->mfc_expire = 0;	/* Don't clean this guy up */
+		nexpire[hash]--;
+
+		/* Free queued packets, but attempt to forward them first. */
+		TAILQ_FOREACH_SAFE(rte, &rt->mfc_stall, rte_link, nrte) {
+			if (rte->ifp != NULL)
+				ip_mdq(rte->m, rte->ifp, rt, -1);
+			m_freem(rte->m);
+			TAILQ_REMOVE(&rt->mfc_stall, rte, rte_link);
+			rt->mfc_nstall--;
+			free(rte, M_MRTABLE);
+		}
 	}
     }
 
@@ -1168,43 +1108,42 @@ add_mfc(struct mfcctl2 *mfccp)
      * It is possible that an entry is being inserted without an upcall
      */
     if (nstl == 0) {
-	if (mrtdebug & DEBUG_MFC)
-	    log(LOG_DEBUG,"add_mfc no upcall h %lu o %lx g %lx p %x\n",
-		hash, (u_long)ntohl(mfccp->mfcc_origin.s_addr),
-		(u_long)ntohl(mfccp->mfcc_mcastgrp.s_addr),
-		mfccp->mfcc_parent);
-
-	for (rt = mfctable[hash]; rt != NULL; rt = rt->mfc_next) {
-	    if ((rt->mfc_origin.s_addr == mfccp->mfcc_origin.s_addr) &&
-		    (rt->mfc_mcastgrp.s_addr == mfccp->mfcc_mcastgrp.s_addr)) {
-		init_mfc_params(rt, mfccp);
-		if (rt->mfc_expire)
-		    nexpire[hash]--;
-		rt->mfc_expire = 0;
-		break; /* XXX */
-	    }
+	CTR1(KTR_IPMF, "%s: adding mfc w/o upcall", __func__);
+	LIST_FOREACH(rt, &mfchashtbl[hash], mfc_hash) {
+		if (in_hosteq(rt->mfc_origin, mfccp->mfcc_origin) &&
+		    in_hosteq(rt->mfc_mcastgrp, mfccp->mfcc_mcastgrp)) {
+			init_mfc_params(rt, mfccp);
+			if (rt->mfc_expire)
+			    nexpire[hash]--;
+			rt->mfc_expire = 0;
+			break; /* XXX */
+		}
 	}
+
 	if (rt == NULL) {		/* no upcall, so make a new entry */
 	    rt = (struct mfc *)malloc(sizeof(*rt), M_MRTABLE, M_NOWAIT);
 	    if (rt == NULL) {
 		MFC_UNLOCK();
 		VIF_UNLOCK();
-		return ENOBUFS;
+		return (ENOBUFS);
 	    }
 
 	    init_mfc_params(rt, mfccp);
-	    rt->mfc_expire     = 0;
-	    rt->mfc_stall      = NULL;
+	    TAILQ_INIT(&rt->mfc_stall);
+	    rt->mfc_nstall = 0;
 
+	    rt->mfc_expire     = 0;
 	    rt->mfc_bw_meter = NULL;
+
 	    /* insert new entry at head of hash chain */
-	    rt->mfc_next = mfctable[hash];
-	    mfctable[hash] = rt;
+	    LIST_INSERT_HEAD(&mfchashtbl[hash], rt, mfc_hash);
 	}
     }
+
     MFC_UNLOCK();
     VIF_UNLOCK();
-    return 0;
+
+    return (0);
 }
 
 /*
@@ -1216,49 +1155,37 @@ del_mfc(struct mfcctl2 *mfccp)
     struct in_addr	origin;
     struct in_addr	mcastgrp;
     struct mfc		*rt;
-    struct mfc		**nptr;
-    u_long		hash;
-    struct bw_meter	*list;
 
     origin = mfccp->mfcc_origin;
     mcastgrp = mfccp->mfcc_mcastgrp;
 
-    if (mrtdebug & DEBUG_MFC)
-	log(LOG_DEBUG,"del_mfc orig %lx mcastgrp %lx\n",
-	    (u_long)ntohl(origin.s_addr), (u_long)ntohl(mcastgrp.s_addr));
+    CTR3(KTR_IPMF, "%s: delete mfc orig %s group %lx", __func__,
+	inet_ntoa(origin), (u_long)ntohl(mcastgrp.s_addr));
 
     MFC_LOCK();
 
-    hash = MFCHASH(origin.s_addr, mcastgrp.s_addr);
-    for (nptr = &mfctable[hash]; (rt = *nptr) != NULL; nptr = &rt->mfc_next)
-	if (origin.s_addr == rt->mfc_origin.s_addr &&
-		mcastgrp.s_addr == rt->mfc_mcastgrp.s_addr &&
-		rt->mfc_stall == NULL)
-	    break;
+    rt = mfc_find(&origin, &mcastgrp);
     if (rt == NULL) {
 	MFC_UNLOCK();
 	return EADDRNOTAVAIL;
     }
 
-    *nptr = rt->mfc_next;
-
     /*
      * free the bw_meter entries
      */
-    list = rt->mfc_bw_meter;
+    free_bw_list(rt->mfc_bw_meter);
     rt->mfc_bw_meter = NULL;
 
+    LIST_REMOVE(rt, mfc_hash);
     free(rt, M_MRTABLE);
 
-    free_bw_list(list);
-
     MFC_UNLOCK();
 
-    return 0;
+    return (0);
 }
 
 /*
- * Send a message to the routing daemon on the multicast routing socket
+ * Send a message to the routing daemon on the multicast routing socket.
  */
 static int
 socket_send(struct socket *s, struct mbuf *mm, struct sockaddr_in *src)
@@ -1298,10 +1225,8 @@ X_ip_mforward(struct ip *ip, struct ifnet *ifp, struct mbuf *m,
     int error;
     vifi_t vifi;
 
-    if (mrtdebug & DEBUG_FORWARD)
-	log(LOG_DEBUG, "ip_mforward: src %lx, dst %lx, ifp %p\n",
-	    (u_long)ntohl(ip->ip_src.s_addr), (u_long)ntohl(ip->ip_dst.s_addr),
-	    (void *)ifp);
+    CTR3(KTR_IPMF, "ip_mforward: delete mfc orig %s group %lx ifp %p",
+	inet_ntoa(ip->ip_src), (u_long)ntohl(ip->ip_dst.s_addr), ifp);
 
     if (ip->ip_hl < (sizeof(struct ip) + TUNNEL_LEN) >> 2 ||
 		((u_char *)(ip + 1))[1] != IPOPT_LSRR ) {
@@ -1314,14 +1239,7 @@ X_ip_mforward(struct ip *ip, struct ifnet *ifp, struct mbuf *m,
 	 * Packet arrived through a source-route tunnel.
 	 * Source-route tunnels are no longer supported.
 	 */
-	static int last_log;
-	if (last_log != time_uptime) {
-	    last_log = time_uptime;
-	    log(LOG_ERR,
-		"ip_mforward: received source-routed packet from %lx\n",
-		(u_long)ntohl(ip->ip_src.s_addr));
-	}
-	return 1;
+	return (1);
     }
 
     VIF_LOCK();
@@ -1329,26 +1247,11 @@ X_ip_mforward(struct ip *ip, struct ifnet *ifp, struct mbuf *m,
     if (imo && ((vifi = imo->imo_multicast_vif) < numvifs)) {
 	if (ip->ip_ttl < MAXTTL)
 	    ip->ip_ttl++;	/* compensate for -1 in *_send routines */
-	if (rsvpdebug && ip->ip_p == IPPROTO_RSVP) {
-	    struct vif *vifp = viftable + vifi;
-
-	    printf("Sending IPPROTO_RSVP from %lx to %lx on vif %d (%s%s)\n",
-		(long)ntohl(ip->ip_src.s_addr), (long)ntohl(ip->ip_dst.s_addr),
-		vifi,
-		(vifp->v_flags & VIFF_TUNNEL) ? "tunnel on " : "",
-		vifp->v_ifp->if_xname);
-	}
 	error = ip_mdq(m, ifp, NULL, vifi);
 	MFC_UNLOCK();
 	VIF_UNLOCK();
 	return error;
     }
-    if (rsvpdebug && ip->ip_p == IPPROTO_RSVP) {
-	printf("Warning: IPPROTO_RSVP from %lx to %lx without vif option\n",
-	    (long)ntohl(ip->ip_src.s_addr), (long)ntohl(ip->ip_dst.s_addr));
-	if (!imo)
-	    printf("In fact, no options were specified at all\n");
-    }
 
     /*
      * Don't forward a packet with time-to-live of zero or one,
@@ -1363,8 +1266,8 @@ X_ip_mforward(struct ip *ip, struct ifnet *ifp, struct mbuf *m,
     /*
      * Determine forwarding vifs from the forwarding cache table
      */
-    ++mrtstat.mrts_mfc_lookups;
-    rt = mfc_find(ip->ip_src.s_addr, ip->ip_dst.s_addr);
+    MRTSTAT_INC(mrts_mfc_lookups);
+    rt = mfc_find(&ip->ip_src, &ip->ip_dst);
 
     /* Entry exists, so forward if necessary */
     if (rt != NULL) {
@@ -1383,25 +1286,24 @@ X_ip_mforward(struct ip *ip, struct ifnet *ifp, struct mbuf *m,
 	u_long hash;
 	int hlen = ip->ip_hl << 2;
 
-	++mrtstat.mrts_mfc_misses;
-
-	mrtstat.mrts_no_route++;
-	if (mrtdebug & (DEBUG_FORWARD | DEBUG_MFC))
-	    log(LOG_DEBUG, "ip_mforward: no rte s %lx g %lx\n",
-		(u_long)ntohl(ip->ip_src.s_addr),
-		(u_long)ntohl(ip->ip_dst.s_addr));
+	MRTSTAT_INC(mrts_mfc_misses);
+	MRTSTAT_INC(mrts_no_route);
+	CTR2(KTR_IPMF, "ip_mforward: no mfc for (%s,%lx)",
+	    inet_ntoa(ip->ip_src), (u_long)ntohl(ip->ip_dst.s_addr));
 
 	/*
 	 * Allocate mbufs early so that we don't do extra work if we are
 	 * just going to fail anyway.  Make sure to pullup the header so
 	 * that other people can't step on it.
 	 */
-	rte = (struct rtdetq *)malloc((sizeof *rte), M_MRTABLE, M_NOWAIT);
+	rte = (struct rtdetq *)malloc((sizeof *rte), M_MRTABLE,
+	    M_NOWAIT|M_ZERO);
 	if (rte == NULL) {
 	    MFC_UNLOCK();
 	    VIF_UNLOCK();
 	    return ENOBUFS;
 	}
+
 	mb0 = m_copypacket(m, M_DONTWAIT);
 	if (mb0 && (M_HASCL(mb0) || mb0->m_len < hlen))
 	    mb0 = m_pullup(mb0, hlen);
@@ -1413,12 +1315,12 @@ X_ip_mforward(struct ip *ip, struct ifnet *ifp, struct mbuf *m,
 	}
 
 	/* is there an upcall waiting for this flow ? */
-	hash = MFCHASH(ip->ip_src.s_addr, ip->ip_dst.s_addr);
-	for (rt = mfctable[hash]; rt; rt = rt->mfc_next) {
-	    if ((ip->ip_src.s_addr == rt->mfc_origin.s_addr) &&
-		    (ip->ip_dst.s_addr == rt->mfc_mcastgrp.s_addr) &&
-		    (rt->mfc_stall != NULL))
-		break;
+	hash = MFCHASH(ip->ip_src, ip->ip_dst);
+	LIST_FOREACH(rt, &mfchashtbl[hash], mfc_hash) {
+		if (in_hosteq(ip->ip_src, rt->mfc_origin) &&
+		    in_hosteq(ip->ip_dst, rt->mfc_mcastgrp) &&
+		    !TAILQ_EMPTY(&rt->mfc_stall))
+			break;
 	}
 
 	if (rt == NULL) {
@@ -1431,7 +1333,8 @@ X_ip_mforward(struct ip *ip, struct ifnet *ifp, struct mbuf *m,
 	     * Locate the vifi for the incoming interface for this packet.
 	     * If none found, drop packet.
 	     */
-	    for (vifi=0; vifi < numvifs && viftable[vifi].v_ifp != ifp; vifi++)
+	    for (vifi = 0; vifi < numvifs &&
+		    viftable[vifi].v_ifp != ifp; vifi++)
 		;
 	    if (vifi >= numvifs)	/* vif not found, drop packet */
 		goto non_fatal;
@@ -1440,6 +1343,7 @@ X_ip_mforward(struct ip *ip, struct ifnet *ifp, struct mbuf *m,
 	    rt = (struct mfc *)malloc(sizeof(*rt), M_MRTABLE, M_NOWAIT);
 	    if (rt == NULL)
 		goto fail;
+
 	    /* Make a copy of the header to send to the user level process */
 	    mm = m_copy(mb0, 0, hlen);
 	    if (mm == NULL)
@@ -1455,12 +1359,12 @@ X_ip_mforward(struct ip *ip, struct ifnet *ifp, struct mbuf *m,
 	    im->im_mbz = 0;
 	    im->im_vif = vifi;
 
-	    mrtstat.mrts_upcalls++;
+	    MRTSTAT_INC(mrts_upcalls);
 
 	    k_igmpsrc.sin_addr = ip->ip_src;
 	    if (socket_send(V_ip_mrouter, mm, &k_igmpsrc) < 0) {
-		log(LOG_WARNING, "ip_mforward: ip_mrouter socket queue full\n");
-		++mrtstat.mrts_upq_sockfull;
+		CTR0(KTR_IPMF, "ip_mforward: socket queue full");
+		MRTSTAT_INC(mrts_upq_sockfull);
 fail1:
 		free(rt, M_MRTABLE);
 fail:
@@ -1482,45 +1386,32 @@ fail:
 	    }
 	    rt->mfc_parent = -1;
 
-	    rt->mfc_rp.s_addr = INADDR_ANY; /* clear the RP address */
-
+	    /* clear the RP address */
+	    rt->mfc_rp.s_addr = INADDR_ANY;
 	    rt->mfc_bw_meter = NULL;
 
 	    /* link into table */
-	    rt->mfc_next   = mfctable[hash];
-	    mfctable[hash] = rt;
-	    rt->mfc_stall = rte;
+	    LIST_INSERT_HEAD(&mfchashtbl[hash], rt, mfc_hash);
+	    TAILQ_INSERT_HEAD(&rt->mfc_stall, rte, rte_link);
+	    rt->mfc_nstall++;
 
 	} else {
-	    /* determine if q has overflowed */
-	    int npkts = 0;
-	    struct rtdetq **p;
-
-	    /*
-	     * XXX ouch! we need to append to the list, but we
-	     * only have a pointer to the front, so we have to
-	     * scan the entire list every time.
-	     */
-	    for (p = &rt->mfc_stall; *p != NULL; p = &(*p)->next)
-		npkts++;
-
-	    if (npkts > MAX_UPQ) {
-		mrtstat.mrts_upq_ovflw++;
+	    /* determine if queue has overflowed */
+	    if (rt->mfc_nstall > MAX_UPQ) {
+		MRTSTAT_INC(mrts_upq_ovflw);
 non_fatal:
 		free(rte, M_MRTABLE);
 		m_freem(mb0);
 		MFC_UNLOCK();
 		VIF_UNLOCK();
-		return 0;
+		return (0);
 	    }
-
-	    /* Add this entry to the end of the queue */
-	    *p = rte;
+	    TAILQ_INSERT_TAIL(&rt->mfc_stall, rte, rte_link);
+	    rt->mfc_nstall++;
 	}
 
 	rte->m			= mb0;
 	rte->ifp		= ifp;
-	rte->next		= NULL;
 
 	MFC_UNLOCK();
 	VIF_UNLOCK();
@@ -1535,58 +1426,44 @@ non_fatal:
 static void
 expire_upcalls(void *unused)
 {
-    struct rtdetq *rte;
-    struct mfc *mfc, **nptr;
     int i;
 
     MFC_LOCK();
-    for (i = 0; i < MFCTBLSIZ; i++) {
+
+    for (i = 0; i < mfchashsize; i++) {
+	struct mfc *rt, *nrt;
+
 	if (nexpire[i] == 0)
 	    continue;
-	nptr = &mfctable[i];
-	for (mfc = *nptr; mfc != NULL; mfc = *nptr) {
-	    /*
-	     * Skip real cache entries
-	     * Make sure it wasn't marked to not expire (shouldn't happen)
-	     * If it expires now
-	     */
-	    if (mfc->mfc_stall != NULL && mfc->mfc_expire != 0 &&
-		    --mfc->mfc_expire == 0) {
-		if (mrtdebug & DEBUG_EXPIRE)
-		    log(LOG_DEBUG, "expire_upcalls: expiring (%lx %lx)\n",
-			(u_long)ntohl(mfc->mfc_origin.s_addr),
-			(u_long)ntohl(mfc->mfc_mcastgrp.s_addr));
-		/*
-		 * drop all the packets
-		 * free the mbuf with the pkt, if, timing info
-		 */
-		for (rte = mfc->mfc_stall; rte; ) {
-		    struct rtdetq *n = rte->next;
 
-		    m_freem(rte->m);
-		    free(rte, M_MRTABLE);
-		    rte = n;
-		}
-		++mrtstat.mrts_cache_cleanups;
-		nexpire[i]--;
+	for (rt = LIST_FIRST(&mfchashtbl[i]); rt; rt = nrt) {
+		nrt = LIST_NEXT(rt, mfc_hash);
+
+		if (TAILQ_EMPTY(&rt->mfc_stall))
+			continue;
+
+		if (rt->mfc_expire == 0 || --rt->mfc_expire > 0)
+			continue;
 
 		/*
 		 * free the bw_meter entries
 		 */
-		while (mfc->mfc_bw_meter != NULL) {
-		    struct bw_meter *x = mfc->mfc_bw_meter;
+		while (rt->mfc_bw_meter != NULL) {
+		    struct bw_meter *x = rt->mfc_bw_meter;
 
-		    mfc->mfc_bw_meter = x->bm_mfc_next;
+		    rt->mfc_bw_meter = x->bm_mfc_next;
 		    free(x, M_BWMETER);
 		}
 
-		*nptr = mfc->mfc_next;
-		free(mfc, M_MRTABLE);
-	    } else {
-		nptr = &mfc->mfc_next;
+		MRTSTAT_INC(mrts_cache_cleanups);
+		CTR3(KTR_IPMF, "%s: expire (%lx, %lx)", __func__,
+		    (u_long)ntohl(rt->mfc_origin.s_addr),
+		    (u_long)ntohl(rt->mfc_mcastgrp.s_addr));
+
+		expire_mfc(rt);
 	    }
-	}
     }
+
     MFC_UNLOCK();
 
     callout_reset(&expire_upcalls_ch, EXPIRE_TIMEOUT, expire_upcalls, NULL);
@@ -1623,11 +1500,9 @@ ip_mdq(struct mbuf *m, struct ifnet *ifp, struct mfc *rt, vifi_t xmt_vif)
      */
     vifi = rt->mfc_parent;
     if ((vifi >= numvifs) || (viftable[vifi].v_ifp != ifp)) {
-	/* came in the wrong interface */
-	if (mrtdebug & DEBUG_FORWARD)
-	    log(LOG_DEBUG, "wrong if: ifp %p vifi %d vififp %p\n",
-		(void *)ifp, vifi, (void *)viftable[vifi].v_ifp);
-	++mrtstat.mrts_wrong_if;
+	CTR4(KTR_IPMF, "%s: rx on wrong ifp %p (vifi %d, v_ifp %p)",
+	    __func__, ifp, (int)vifi, viftable[vifi].v_ifp);
+	MRTSTAT_INC(mrts_wrong_if);
 	++rt->mfc_wrong_if;
 	/*
 	 * If we are doing PIM assert processing, send a message
@@ -1637,12 +1512,10 @@ ip_mdq(struct mbuf *m, struct ifnet *ifp, struct mfc *rt, vifi_t xmt_vif)
 	 * can complete the SPT switch, regardless of the type
 	 * of the iif (broadcast media, GRE tunnel, etc).
 	 */
-	if (pim_assert && (vifi < numvifs) && viftable[vifi].v_ifp) {
-	    struct timeval now;
-	    u_long delta;
+	if (pim_assert_enabled && (vifi < numvifs) && viftable[vifi].v_ifp) {
 
 	    if (ifp == &multicast_register_if)
-		pimstat.pims_rcv_registers_wrongiif++;
+		PIMSTAT_INC(pims_rcv_registers_wrongiif);
 
 	    /* Get vifi for the incoming packet */
 	    for (vifi=0; vifi < numvifs && viftable[vifi].v_ifp != ifp; vifi++)
@@ -1653,11 +1526,7 @@ ip_mdq(struct mbuf *m, struct ifnet *ifp, struct mfc *rt, vifi_t xmt_vif)
 	    if (rt->mfc_flags[vifi] & MRT_MFC_FLAGS_DISABLE_WRONGVIF)
 		return 0;	/* WRONGVIF disabled: ignore the packet */
 
-	    GET_TIME(now);
-
-	    TV_DELTA(now, rt->mfc_last_assert, delta);
-
-	    if (delta > ASSERT_MSG_TIME) {
+	    if (ratecheck(&rt->mfc_last_assert, &pim_assert_interval)) {
 		struct sockaddr_in k_igmpsrc = { sizeof k_igmpsrc, AF_INET };
 		struct igmpmsg *im;
 		int hlen = ip->ip_hl << 2;
@@ -1668,20 +1537,17 @@ ip_mdq(struct mbuf *m, struct ifnet *ifp, struct mfc *rt, vifi_t xmt_vif)
 		if (mm == NULL)
 		    return ENOBUFS;
 
-		rt->mfc_last_assert = now;
-
 		im = mtod(mm, struct igmpmsg *);
 		im->im_msgtype	= IGMPMSG_WRONGVIF;
 		im->im_mbz		= 0;
 		im->im_vif		= vifi;
 
-		mrtstat.mrts_upcalls++;
+		MRTSTAT_INC(mrts_upcalls);
 
 		k_igmpsrc.sin_addr = im->im_src;
 		if (socket_send(V_ip_mrouter, mm, &k_igmpsrc) < 0) {
-		    log(LOG_WARNING,
-			"ip_mforward: ip_mrouter socket queue full\n");
-		    ++mrtstat.mrts_upq_sockfull;
+		    CTR1(KTR_IPMF, "%s: socket queue full", __func__);
+		    MRTSTAT_INC(mrts_upq_sockfull);
 		    return ENOBUFS;
 		}
 	    }
@@ -1689,8 +1555,9 @@ ip_mdq(struct mbuf *m, struct ifnet *ifp, struct mfc *rt, vifi_t xmt_vif)
 	return 0;
     }
 
+
     /* If I sourced this packet, it counts as output, else it was input. */
-    if (ip->ip_src.s_addr == viftable[vifi].v_lcl_addr.s_addr) {
+    if (in_hosteq(ip->ip_src, viftable[vifi].v_lcl_addr)) {
 	viftable[vifi].v_pkt_out++;
 	viftable[vifi].v_bytes_out += plen;
     } else {
@@ -1723,7 +1590,7 @@ ip_mdq(struct mbuf *m, struct ifnet *ifp, struct mfc *rt, vifi_t xmt_vif)
 	struct bw_meter *x;
 	struct timeval now;
 
-	GET_TIME(now);
+	microtime(&now);
 	MFC_LOCK_ASSERT();
 	for (x = rt->mfc_bw_meter; x != NULL; x = x->bm_mfc_next)
 	    bw_meter_receive_packet(x, plen, &now);
@@ -1733,13 +1600,23 @@ ip_mdq(struct mbuf *m, struct ifnet *ifp, struct mfc *rt, vifi_t xmt_vif)
 }
 
 /*
- * check if a vif number is legal/ok. This is used by ip_output.
+ * Check if a vif number is legal/ok. This is used by in_mcast.c.
  */
 static int
 X_legal_vif_num(int vif)
 {
-    /* XXX unlocked, matter? */
-    return (vif >= 0 && vif < numvifs);
+	int ret;
+
+	ret = 0;
+	if (vif < 0)
+		return (ret);
+
+	VIF_LOCK();
+	if (vif < numvifs)
+		ret = 1;
+	VIF_UNLOCK();
+
+	return (ret);
 }
 
 /*
@@ -1748,11 +1625,18 @@ X_legal_vif_num(int vif)
 static u_long
 X_ip_mcast_src(int vifi)
 {
-    /* XXX unlocked, matter? */
-    if (vifi >= 0 && vifi < numvifs)
-	return viftable[vifi].v_lcl_addr.s_addr;
-    else
-	return INADDR_ANY;
+	in_addr_t addr;
+
+	addr = INADDR_ANY;
+	if (vifi < 0)
+		return (addr);
+
+	VIF_LOCK();
+	if (vifi < numvifs)
+		addr = viftable[vifi].v_lcl_addr.s_addr;
+	VIF_UNLOCK();
+
+	return (addr);
 }
 
 static void
@@ -1801,173 +1685,33 @@ send_packet(struct vif *vifp, struct mbuf *m)
 	 * the loopback interface, thus preventing looping.
 	 */
 	error = ip_output(m, NULL, &vifp->v_route, IP_FORWARDING, &imo, NULL);
-	if (mrtdebug & DEBUG_XMIT) {
-	    log(LOG_DEBUG, "phyint_send on vif %td err %d\n",
-		vifp - viftable, error);
-	}
+	CTR3(KTR_IPMF, "%s: vif %td err %d", __func__,
+	    (ptrdiff_t)(vifp - viftable), error);
 }
 
+/*
+ * Stubs for old RSVP socket shim implementation.
+ */
+
 static int
-X_ip_rsvp_vif(struct socket *so, struct sockopt *sopt)
+X_ip_rsvp_vif(struct socket *so __unused, struct sockopt *sopt __unused)
 {
-    INIT_VNET_INET(curvnet);
-    int error, vifi;
-
-    if (so->so_type != SOCK_RAW || so->so_proto->pr_protocol != IPPROTO_RSVP)
-	return EOPNOTSUPP;
-
-    error = sooptcopyin(sopt, &vifi, sizeof vifi, sizeof vifi);
-    if (error)
-	return error;
-
-    VIF_LOCK();
 
-    if (vifi < 0 || vifi >= numvifs) {	/* Error if vif is invalid */
-	VIF_UNLOCK();
-	return EADDRNOTAVAIL;
-    }
-
-    if (sopt->sopt_name == IP_RSVP_VIF_ON) {
-	/* Check if socket is available. */
-	if (viftable[vifi].v_rsvpd != NULL) {
-	    VIF_UNLOCK();
-	    return EADDRINUSE;
-	}
-
-	viftable[vifi].v_rsvpd = so;
-	/* This may seem silly, but we need to be sure we don't over-increment
-	 * the RSVP counter, in case something slips up.
-	 */
-	if (!viftable[vifi].v_rsvp_on) {
-	    viftable[vifi].v_rsvp_on = 1;
-	    V_rsvp_on++;
-	}
-    } else { /* must be VIF_OFF */
-	/*
-	 * XXX as an additional consistency check, one could make sure
-	 * that viftable[vifi].v_rsvpd == so, otherwise passing so as
-	 * first parameter is pretty useless.
-	 */
-	viftable[vifi].v_rsvpd = NULL;
-	/*
-	 * This may seem silly, but we need to be sure we don't over-decrement
-	 * the RSVP counter, in case something slips up.
-	 */
-	if (viftable[vifi].v_rsvp_on) {
-	    viftable[vifi].v_rsvp_on = 0;
-	    V_rsvp_on--;
-	}
-    }
-    VIF_UNLOCK();
-    return 0;
+	return (EOPNOTSUPP);
 }
 
 static void
-X_ip_rsvp_force_done(struct socket *so)
+X_ip_rsvp_force_done(struct socket *so __unused)
 {
-    INIT_VNET_INET(curvnet);
-    int vifi;
 
-    /* Don't bother if it is not the right type of socket. */
-    if (so->so_type != SOCK_RAW || so->so_proto->pr_protocol != IPPROTO_RSVP)
-	return;
-
-    VIF_LOCK();
-
-    /* The socket may be attached to more than one vif...this
-     * is perfectly legal.
-     */
-    for (vifi = 0; vifi < numvifs; vifi++) {
-	if (viftable[vifi].v_rsvpd == so) {
-	    viftable[vifi].v_rsvpd = NULL;
-	    /* This may seem silly, but we need to be sure we don't
-	     * over-decrement the RSVP counter, in case something slips up.
-	     */
-	    if (viftable[vifi].v_rsvp_on) {
-		viftable[vifi].v_rsvp_on = 0;
-		V_rsvp_on--;
-	    }
-	}
-    }
-
-    VIF_UNLOCK();
 }
 
 static void
-X_rsvp_input(struct mbuf *m, int off)
+X_rsvp_input(struct mbuf *m, int off __unused)
 {
-    INIT_VNET_INET(curvnet);
-    int vifi;
-    struct ip *ip = mtod(m, struct ip *);
-    struct sockaddr_in rsvp_src = { sizeof rsvp_src, AF_INET };
-    struct ifnet *ifp;
 
-    if (rsvpdebug)
-	printf("rsvp_input: rsvp_on %d\n", V_rsvp_on);
-
-    /* Can still get packets with rsvp_on = 0 if there is a local member
-     * of the group to which the RSVP packet is addressed.  But in this
-     * case we want to throw the packet away.
-     */
-    if (!V_rsvp_on) {
-	m_freem(m);
-	return;
-    }
-
-    if (rsvpdebug)
-	printf("rsvp_input: check vifs\n");
-
-#ifdef DIAGNOSTIC
-    M_ASSERTPKTHDR(m);
-#endif
-
-    ifp = m->m_pkthdr.rcvif;
-
-    VIF_LOCK();
-    /* Find which vif the packet arrived on. */
-    for (vifi = 0; vifi < numvifs; vifi++)
-	if (viftable[vifi].v_ifp == ifp)
-	    break;
-
-    if (vifi == numvifs || viftable[vifi].v_rsvpd == NULL) {
-	/*
-	 * Drop the lock here to avoid holding it across rip_input.
-	 * This could make rsvpdebug printfs wrong.  If you care,
-	 * record the state of stuff before dropping the lock.
-	 */
-	VIF_UNLOCK();
-	/*
-	 * If the old-style non-vif-associated socket is set,
-	 * then use it.  Otherwise, drop packet since there
-	 * is no specific socket for this vif.
-	 */
-	if (V_ip_rsvpd != NULL) {
-	    if (rsvpdebug)
-		printf("rsvp_input: Sending packet up old-style socket\n");
-	    rip_input(m, off);  /* xxx */
-	} else {
-	    if (rsvpdebug && vifi == numvifs)
-		printf("rsvp_input: Can't find vif for packet.\n");
-	    else if (rsvpdebug && viftable[vifi].v_rsvpd == NULL)
-		printf("rsvp_input: No socket defined for vif %d\n",vifi);
-	    m_freem(m);
-	}
-	return;
-    }
-    rsvp_src.sin_addr = ip->ip_src;
-
-    if (rsvpdebug && m)
-	printf("rsvp_input: m->m_len = %d, sbspace() = %ld\n",
-	       m->m_len,sbspace(&(viftable[vifi].v_rsvpd->so_rcv)));
-
-    if (socket_send(viftable[vifi].v_rsvpd, m, &rsvp_src) < 0) {
-	if (rsvpdebug)
-	    printf("rsvp_input: Failed to append to socket\n");
-    } else {
-	if (rsvpdebug)
-	    printf("rsvp_input: send packet up\n");
-    }
-    VIF_UNLOCK();
+	if (!V_rsvp_on)
+		m_freem(m);
 }
 
 /*
@@ -2033,7 +1777,7 @@ add_bw_upcall(struct bw_upcall *req)
      * Find if we have already same bw_meter entry
      */
     MFC_LOCK();
-    mfc = mfc_find(req->bu_src.s_addr, req->bu_dst.s_addr);
+    mfc = mfc_find(&req->bu_src, &req->bu_dst);
     if (mfc == NULL) {
 	MFC_UNLOCK();
 	return EADDRNOTAVAIL;
@@ -2058,7 +1802,7 @@ add_bw_upcall(struct bw_upcall *req)
 
     /* Set the new bw_meter entry */
     x->bm_threshold.b_time = req->bu_threshold.b_time;
-    GET_TIME(now);
+    microtime(&now);
     x->bm_start_time = now;
     x->bm_threshold.b_packets = req->bu_threshold.b_packets;
     x->bm_threshold.b_bytes = req->bu_threshold.b_bytes;
@@ -2103,8 +1847,9 @@ del_bw_upcall(struct bw_upcall *req)
 	return EOPNOTSUPP;
 
     MFC_LOCK();
+
     /* Find the corresponding MFC entry */
-    mfc = mfc_find(req->bu_src.s_addr, req->bu_dst.s_addr);
+    mfc = mfc_find(&req->bu_src, &req->bu_dst);
     if (mfc == NULL) {
 	MFC_UNLOCK();
 	return EADDRNOTAVAIL;
@@ -2336,10 +2081,10 @@ bw_upcalls_send(void)
      * Send the upcalls
      * XXX do we need to set the address in k_igmpsrc ?
      */
-    mrtstat.mrts_upcalls++;
+    MRTSTAT_INC(mrts_upcalls);
     if (socket_send(V_ip_mrouter, m, &k_igmpsrc) < 0) {
 	log(LOG_WARNING, "bw_upcalls_send: ip_mrouter socket queue full\n");
-	++mrtstat.mrts_upq_sockfull;
+	MRTSTAT_INC(mrts_upq_sockfull);
     }
 }
 
@@ -2446,7 +2191,7 @@ bw_meter_process()
     int i;
     struct timeval now, process_endtime;
 
-    GET_TIME(now);
+    microtime(&now);
     if (last_tv_sec == now.tv_sec)
 	return;		/* nothing to do */
 
@@ -2564,15 +2309,12 @@ pim_register_send(struct ip *ip, struct vif *vifp, struct mbuf *m,
 {
     struct mbuf *mb_copy, *mm;
 
-    if (mrtdebug & DEBUG_PIM)
-	log(LOG_DEBUG, "pim_register_send: ");
-
     /*
      * Do not send IGMP_WHOLEPKT notifications to userland, if the
      * rendezvous point was unspecified, and we were told not to.
      */
     if (pim_squelch_wholepkt != 0 && (mrt_api_config & MRT_MFC_RP) &&
-	(rt->mfc_rp.s_addr == INADDR_ANY))
+	in_nullhost(rt->mfc_rp))
 	return 0;
 
     mb_copy = pim_register_prepare(ip, m);
@@ -2589,8 +2331,7 @@ pim_register_send(struct ip *ip, struct vif *vifp, struct mbuf *m,
 	mm = m_pullup(mm, sizeof(struct ip));
 	if (mm != NULL) {
 	    ip = mtod(mm, struct ip *);
-	    if ((mrt_api_config & MRT_MFC_RP) &&
-		(rt->mfc_rp.s_addr != INADDR_ANY)) {
+	    if ((mrt_api_config & MRT_MFC_RP) && !in_nullhost(rt->mfc_rp)) {
 		pim_register_send_rp(ip, vifp, mm, rt);
 	    } else {
 		pim_register_send_upcall(ip, vifp, mm, rt);
@@ -2690,19 +2431,17 @@ pim_register_send_upcall(struct ip *ip, struct vif *vifp,
 
     k_igmpsrc.sin_addr	= ip->ip_src;
 
-    mrtstat.mrts_upcalls++;
+    MRTSTAT_INC(mrts_upcalls);
 
     if (socket_send(V_ip_mrouter, mb_first, &k_igmpsrc) < 0) {
-	if (mrtdebug & DEBUG_PIM)
-	    log(LOG_WARNING,
-		"mcast: pim_register_send_upcall: ip_mrouter socket queue full");
-	++mrtstat.mrts_upq_sockfull;
+	CTR1(KTR_IPMF, "%s: socket queue full", __func__);
+	MRTSTAT_INC(mrts_upq_sockfull);
 	return ENOBUFS;
     }
 
     /* Keep statistics */
-    pimstat.pims_snd_registers_msgs++;
-    pimstat.pims_snd_registers_bytes += len;
+    PIMSTAT_INC(pims_snd_registers_msgs);
+    PIMSTAT_ADD(pims_snd_registers_bytes, len);
 
     return 0;
 }
@@ -2723,7 +2462,7 @@ pim_register_send_rp(struct ip *ip, struct vif *vifp, struct mbuf *mb_copy,
 
     VIF_LOCK_ASSERT();
 
-    if ((vifi >= numvifs) || (viftable[vifi].v_lcl_addr.s_addr == 0)) {
+    if ((vifi >= numvifs) || in_nullhost(viftable[vifi].v_lcl_addr)) {
 	m_freem(mb_copy);
 	return EADDRNOTAVAIL;		/* The iif vif is invalid */
     }
@@ -2772,14 +2511,14 @@ pim_register_send_rp(struct ip *ip, struct vif *vifp, struct mbuf *mb_copy,
     send_packet(vifp, mb_first);
 
     /* Keep statistics */
-    pimstat.pims_snd_registers_msgs++;
-    pimstat.pims_snd_registers_bytes += len;
+    PIMSTAT_INC(pims_snd_registers_msgs);
+    PIMSTAT_ADD(pims_snd_registers_bytes, len);
 
     return 0;
 }
 
 /*
- * pim_encapcheck() is called by the encap[46]_input() path at runtime to
+ * pim_encapcheck() is called by the encap4_input() path at runtime to
  * determine if a packet is for PIM; allowing PIM to be dynamically loaded
  * into the kernel.
  */
@@ -2815,16 +2554,16 @@ pim_input(struct mbuf *m, int off)
     int iphlen = off;
 
     /* Keep statistics */
-    pimstat.pims_rcv_total_msgs++;
-    pimstat.pims_rcv_total_bytes += datalen;
+    PIMSTAT_INC(pims_rcv_total_msgs);
+    PIMSTAT_ADD(pims_rcv_total_bytes, datalen);
 
     /*
      * Validate lengths
      */
     if (datalen < PIM_MINLEN) {
-	pimstat.pims_rcv_tooshort++;
-	log(LOG_ERR, "pim_input: packet size too small %d from %lx\n",
-	    datalen, (u_long)ip->ip_src.s_addr);
+	PIMSTAT_INC(pims_rcv_tooshort);
+	CTR3(KTR_IPMF, "%s: short packet (%d) from %s",
+	    __func__, datalen, inet_ntoa(ip->ip_src));
 	m_freem(m);
 	return;
     }
@@ -2844,9 +2583,10 @@ pim_input(struct mbuf *m, int off)
      */
     if ((m->m_flags & M_EXT || m->m_len < minlen) &&
 	(m = m_pullup(m, minlen)) == 0) {
-	log(LOG_ERR, "pim_input: m_pullup failure\n");
+	CTR1(KTR_IPMF, "%s: m_pullup() failed", __func__);
 	return;
     }
+
     /* m_pullup() may have given us a new mbuf so reset ip. */
     ip = mtod(m, struct ip *);
     ip_tos = ip->ip_tos;
@@ -2866,18 +2606,17 @@ pim_input(struct mbuf *m, int off)
     if (PIM_VT_T(pim->pim_vt) == PIM_REGISTER && in_cksum(m, PIM_MINLEN) == 0) {
 	/* do nothing, checksum okay */
     } else if (in_cksum(m, datalen)) {
-	pimstat.pims_rcv_badsum++;
-	if (mrtdebug & DEBUG_PIM)
-	    log(LOG_DEBUG, "pim_input: invalid checksum");
+	PIMSTAT_INC(pims_rcv_badsum);
+	CTR1(KTR_IPMF, "%s: invalid checksum", __func__);
 	m_freem(m);
 	return;
     }
 
     /* PIM version check */
     if (PIM_VT_V(pim->pim_vt) < PIM_VERSION) {
-	pimstat.pims_rcv_badversion++;
-	log(LOG_ERR, "pim_input: incorrect version %d, expecting %d\n",
-	    PIM_VT_V(pim->pim_vt), PIM_VERSION);
+	PIMSTAT_INC(pims_rcv_badversion);
+	CTR3(KTR_IPMF, "%s: bad version %d expect %d", __func__,
+	    (int)PIM_VT_V(pim->pim_vt), PIM_VERSION);
 	m_freem(m);
 	return;
     }
@@ -2901,9 +2640,8 @@ pim_input(struct mbuf *m, int off)
 	VIF_LOCK();
 	if ((reg_vif_num >= numvifs) || (reg_vif_num == VIFI_INVALID)) {
 	    VIF_UNLOCK();
-	    if (mrtdebug & DEBUG_PIM)
-		log(LOG_DEBUG,
-		    "pim_input: register vif not set: %d\n", reg_vif_num);
+	    CTR2(KTR_IPMF, "%s: register vif not set: %d", __func__,
+		(int)reg_vif_num);
 	    m_freem(m);
 	    return;
 	}
@@ -2915,11 +2653,9 @@ pim_input(struct mbuf *m, int off)
 	 * Validate length
 	 */
 	if (datalen < PIM_REG_MINLEN) {
-	    pimstat.pims_rcv_tooshort++;
-	    pimstat.pims_rcv_badregisters++;
-	    log(LOG_ERR,
-		"pim_input: register packet size too small %d from %lx\n",
-		datalen, (u_long)ip->ip_src.s_addr);
+	    PIMSTAT_INC(pims_rcv_tooshort);
+	    PIMSTAT_INC(pims_rcv_badregisters);
+	    CTR1(KTR_IPMF, "%s: register packet size too small", __func__);
 	    m_freem(m);
 	    return;
 	}
@@ -2927,33 +2663,22 @@ pim_input(struct mbuf *m, int off)
 	reghdr = (u_int32_t *)(pim + 1);
 	encap_ip = (struct ip *)(reghdr + 1);
 
-	if (mrtdebug & DEBUG_PIM) {
-	    log(LOG_DEBUG,
-		"pim_input[register], encap_ip: %lx -> %lx, encap_ip len %d\n",
-		(u_long)ntohl(encap_ip->ip_src.s_addr),
-		(u_long)ntohl(encap_ip->ip_dst.s_addr),
-		ntohs(encap_ip->ip_len));
-	}
+	CTR3(KTR_IPMF, "%s: register: encap ip src %s len %d",
+	    __func__, inet_ntoa(encap_ip->ip_src), ntohs(encap_ip->ip_len));
 
 	/* verify the version number of the inner packet */
 	if (encap_ip->ip_v != IPVERSION) {
-	    pimstat.pims_rcv_badregisters++;
-	    if (mrtdebug & DEBUG_PIM) {
-		log(LOG_DEBUG, "pim_input: invalid IP version (%d) "
-		    "of the inner packet\n", encap_ip->ip_v);
-	    }
+	    PIMSTAT_INC(pims_rcv_badregisters);
+	    CTR1(KTR_IPMF, "%s: bad encap ip version", __func__);
 	    m_freem(m);
 	    return;
 	}
 
 	/* verify the inner packet is destined to a mcast group */
 	if (!IN_MULTICAST(ntohl(encap_ip->ip_dst.s_addr))) {
-	    pimstat.pims_rcv_badregisters++;
-	    if (mrtdebug & DEBUG_PIM)
-		log(LOG_DEBUG,
-		    "pim_input: inner packet of register is not "
-		    "multicast %lx\n",
-		    (u_long)ntohl(encap_ip->ip_dst.s_addr));
+	    PIMSTAT_INC(pims_rcv_badregisters);
+	    CTR2(KTR_IPMF, "%s: bad encap ip dest %s", __func__,
+		inet_ntoa(encap_ip->ip_dst));
 	    m_freem(m);
 	    return;
 	}
@@ -2992,30 +2717,28 @@ pim_input(struct mbuf *m, int off)
 	 */
 	mcp = m_copy(m, 0, iphlen + PIM_REG_MINLEN);
 	if (mcp == NULL) {
-	    log(LOG_ERR,
-		"pim_input: pim register: could not copy register head\n");
+	    CTR1(KTR_IPMF, "%s: m_copy() failed", __func__);
 	    m_freem(m);
 	    return;
 	}
 
 	/* Keep statistics */
 	/* XXX: registers_bytes include only the encap. mcast pkt */
-	pimstat.pims_rcv_registers_msgs++;
-	pimstat.pims_rcv_registers_bytes += ntohs(encap_ip->ip_len);
+	PIMSTAT_INC(pims_rcv_registers_msgs);
+	PIMSTAT_ADD(pims_rcv_registers_bytes, ntohs(encap_ip->ip_len));
 
 	/*
 	 * forward the inner ip packet; point m_data at the inner ip.
 	 */
 	m_adj(m, iphlen + PIM_MINLEN);
 
-	if (mrtdebug & DEBUG_PIM) {
-	    log(LOG_DEBUG,
-		"pim_input: forwarding decapsulated register: "
-		"src %lx, dst %lx, vif %d\n",
-		(u_long)ntohl(encap_ip->ip_src.s_addr),
-		(u_long)ntohl(encap_ip->ip_dst.s_addr),
-		reg_vif_num);
-	}
+	CTR4(KTR_IPMF,
+	    "%s: forward decap'd REGISTER: src %lx dst %lx vif %d",
+	    __func__,
+	    (u_long)ntohl(encap_ip->ip_src.s_addr),
+	    (u_long)ntohl(encap_ip->ip_dst.s_addr),
+	    (int)reg_vif_num);
+
 	/* NB: vifp was collected above; can it change on us? */
 	if_simloop(vifp, m, dst.sin_family, 0);
 
@@ -3037,10 +2760,37 @@ pim_input_to_daemon:
     return;
 }
 
-/*
- * XXX: This is common code for dealing with initialization for both
- * the IPv4 and IPv6 multicast forwarding paths. It could do with cleanup.
- */
+static int
+sysctl_mfctable(SYSCTL_HANDLER_ARGS)
+{
+	struct mfc	*rt;
+	int		 error, i;
+
+	if (req->newptr)
+		return (EPERM);
+	if (mfchashtbl == NULL)	/* XXX unlocked */
+		return (0);
+	error = sysctl_wire_old_buffer(req, 0);
+	if (error)
+		return (error);
+
+	MFC_LOCK();
+	for (i = 0; i < mfchashsize; i++) {
+		LIST_FOREACH(rt, &mfchashtbl[i], mfc_hash) {
+			error = SYSCTL_OUT(req, rt, sizeof(struct mfc));
+			if (error)
+				goto out_locked;
+		}
+	}
+out_locked:
+	MFC_UNLOCK();
+	return (error);
+}
+
+SYSCTL_NODE(_net_inet_ip, OID_AUTO, mfctable, CTLFLAG_RD, sysctl_mfctable,
+    "IPv4 Multicast Forwarding Table (struct *mfc[mfchashsize], "
+    "netinet/ip_mroute.h)");
+
 static int
 ip_mroute_modevent(module_t mod, int type, void *unused)
 {
@@ -3051,9 +2801,20 @@ ip_mroute_modevent(module_t mod, int type, void *unused)
 	MROUTER_LOCK_INIT();
 	MFC_LOCK_INIT();
 	VIF_LOCK_INIT();
-	ip_mrouter_reset();
+
+	mfchashsize = MFCHASHSIZE;
+	if (TUNABLE_ULONG_FETCH("net.inet.ip.mfchashsize", &mfchashsize) &&
+	    !powerof2(mfchashsize)) {
+		printf("WARNING: %s not a power of 2; using default\n",
+		    "net.inet.ip.mfchashsize");
+		mfchashsize = MFCHASHSIZE;
+	}
+	MALLOC(nexpire, u_char *, mfchashsize, M_MRTABLE, M_WAITOK|M_ZERO);
+
+	pim_squelch_wholepkt = 0;
 	TUNABLE_ULONG_FETCH("net.inet.pim.squelch_wholepkt",
 	    &pim_squelch_wholepkt);
+	ip_mrouter_reset();
 
 	pim_encap_cookie = encap_attach_func(AF_INET, IPPROTO_PIM,
 	    pim_encapcheck, &in_pim_protosw, NULL);
@@ -3065,36 +2826,12 @@ ip_mroute_modevent(module_t mod, int type, void *unused)
 		return (EINVAL);
 	}
 
-#ifdef INET6
-	pim6_encap_cookie = encap_attach_func(AF_INET6, IPPROTO_PIM,
-	    pim_encapcheck, (struct protosw *)&in6_pim_protosw, NULL);
-	if (pim6_encap_cookie == NULL) {
-		printf("ip_mroute: unable to attach pim6 encap\n");
-		if (pim_encap_cookie) {
-		    encap_detach(pim_encap_cookie);
-		    pim_encap_cookie = NULL;
-		}
-		VIF_LOCK_DESTROY();
-		MFC_LOCK_DESTROY();
-		MROUTER_LOCK_DESTROY();
-		return (EINVAL);
-	}
-#endif
-
 	ip_mcast_src = X_ip_mcast_src;
 	ip_mforward = X_ip_mforward;
 	ip_mrouter_done = X_ip_mrouter_done;
 	ip_mrouter_get = X_ip_mrouter_get;
 	ip_mrouter_set = X_ip_mrouter_set;
 
-#ifdef INET6
-	ip6_mforward = X_ip6_mforward;
-	ip6_mrouter_done = X_ip6_mrouter_done;
-	ip6_mrouter_get = X_ip6_mrouter_get;
-	ip6_mrouter_set = X_ip6_mrouter_set;
-	mrt6_ioctl = X_mrt6_ioctl;
-#endif
-
 	ip_rsvp_force_done = X_ip_rsvp_force_done;
 	ip_rsvp_vif = X_ip_rsvp_vif;
 
@@ -3112,31 +2849,18 @@ ip_mroute_modevent(module_t mod, int type, void *unused)
 	 * just loaded and then unloaded w/o starting up a user
 	 * process we still need to cleanup.
 	 */
-	if (V_ip_mrouter
-#ifdef INET6
-	    || ip6_mrouter
-#endif
-	)
-	    return EINVAL;
-
-#ifdef INET6
-	if (pim6_encap_cookie) {
-	    encap_detach(pim6_encap_cookie);
-	    pim6_encap_cookie = NULL;
-	}
-	X_ip6_mrouter_done();
-	ip6_mforward = NULL;
-	ip6_mrouter_done = NULL;
-	ip6_mrouter_get = NULL;
-	ip6_mrouter_set = NULL;
-	mrt6_ioctl = NULL;
-#endif
+	if (V_ip_mrouter != NULL)
+	    return (EINVAL);
 
 	if (pim_encap_cookie) {
 	    encap_detach(pim_encap_cookie);
 	    pim_encap_cookie = NULL;
 	}
 	X_ip_mrouter_done();
+
+	FREE(nexpire, M_MRTABLE);
+	nexpire = NULL;
+
 	ip_mcast_src = NULL;
 	ip_mforward = NULL;
 	ip_mrouter_done = NULL;
@@ -3166,4 +2890,5 @@ static moduledata_t ip_mroutemod = {
     ip_mroute_modevent,
     0
 };
+
 DECLARE_MODULE(ip_mroute, ip_mroutemod, SI_SUB_PSEUDO, SI_ORDER_ANY);
diff --git a/sys/netinet/ip_mroute.h b/sys/netinet/ip_mroute.h
index 4043e44..d5ef5e0 100644
--- a/sys/netinet/ip_mroute.h
+++ b/sys/netinet/ip_mroute.h
@@ -70,9 +70,6 @@
 #define MRT_ADD_BW_UPCALL 111	/* create bandwidth monitor */
 #define MRT_DEL_BW_UPCALL 112	/* delete bandwidth monitor */
 
-
-#define GET_TIME(t)	microtime(&t)
-
 /*
  * Types and macros for handling bitmaps with one bit per virtual interface.
  */
@@ -224,6 +221,11 @@ struct mrtstat {
     u_long	mrts_upq_sockfull;	/* upcalls dropped - socket full */
 };
 
+#ifdef _KERNEL
+#define	MRTSTAT_ADD(name, val)	mrtstat.name += (val)
+#define	MRTSTAT_INC(name)	MRTSTAT_ADD(name, 1)
+#endif
+
 /*
  * Argument structure used by mrouted to get src-grp pkt counts
  */
@@ -253,8 +255,6 @@ struct sioc_vif_req {
 struct vif {
     u_char		v_flags;	/* VIFF_ flags defined above         */
     u_char		v_threshold;	/* min ttl required to forward on vif*/
-    u_int		v_rate_limit;	/* ignored; kept for compatibility */
-    struct tbf         *v_tbf;		/* ignored; kept for compatibility */
     struct in_addr	v_lcl_addr;	/* local interface address           */
     struct in_addr	v_rmt_addr;	/* remote address (tunnels only)     */
     struct ifnet       *v_ifp;		/* pointer to interface              */
@@ -263,16 +263,13 @@ struct vif {
     u_long		v_bytes_in;	/* # bytes in on interface	     */
     u_long		v_bytes_out;	/* # bytes out on interface	     */
     struct route	v_route;	/* cached route */
-    u_int		v_rsvp_on;	/* RSVP listening on this vif */
-    struct socket      *v_rsvpd;	/* RSVP daemon socket */
 };
 
 /*
  * The kernel's multicast forwarding cache entry structure
- * (A field for the type of service (mfc_tos) is to be added
- * at a future point)
  */
 struct mfc {
+	LIST_ENTRY(mfc)	mfc_hash;
 	struct in_addr	mfc_origin;		/* IP origin of mcasts	     */
 	struct in_addr  mfc_mcastgrp;		/* multicast group associated*/
 	vifi_t		mfc_parent;		/* incoming vif              */
@@ -282,11 +279,11 @@ struct mfc {
 	u_long		mfc_wrong_if;		/* wrong if for src-grp	     */
 	int		mfc_expire;		/* time to clean entry up    */
 	struct timeval	mfc_last_assert;	/* last time I sent an assert*/
-	struct rtdetq	*mfc_stall;		/* q of packets awaiting mfc */
-	struct mfc	*mfc_next;		/* next mfc entry            */
 	uint8_t		mfc_flags[MAXVIFS];	/* the MRT_MFC_FLAGS_* flags */
 	struct in_addr	mfc_rp;			/* the RP address	     */
 	struct bw_meter	*mfc_bw_meter;		/* list of bandwidth meters  */
+	u_long		mfc_nstall;		/* # of packets awaiting mfc */
+	TAILQ_HEAD(, rtdetq) mfc_stall;		/* q of packets awaiting mfc */
 };
 
 /*
@@ -311,19 +308,11 @@ struct igmpmsg {
  * Argument structure used for pkt info. while upcall is made
  */
 struct rtdetq {
+    TAILQ_ENTRY(rtdetq)	rte_link;
     struct mbuf		*m;		/* A copy of the packet		    */
     struct ifnet	*ifp;		/* Interface pkt came in on	    */
     vifi_t		xmt_vif;	/* Saved copy of imo_multicast_vif  */
-    struct rtdetq	*next;		/* Next in list of packets          */
 };
-
-#define MFCTBLSIZ	256
-#if (MFCTBLSIZ & (MFCTBLSIZ - 1)) == 0	  /* from sys:route.h */
-#define MFCHASHMOD(h)	((h) & (MFCTBLSIZ - 1))
-#else
-#define MFCHASHMOD(h)	((h) % MFCTBLSIZ)
-#endif
-
 #define MAX_UPQ	4		/* max. no of pkts in upcall Q */
 
 /*
diff --git a/sys/netinet/ip_options.c b/sys/netinet/ip_options.c
index 74d54f2..dbb7d81 100644
--- a/sys/netinet/ip_options.c
+++ b/sys/netinet/ip_options.c
@@ -218,7 +218,7 @@ nosourcerouting:
 #ifdef IPSTEALTH
 dropit:
 #endif
-					V_ipstat.ips_cantforward++;
+					IPSTAT_INC(ips_cantforward);
 					m_freem(m);
 					return (1);
 				}
@@ -366,7 +366,7 @@ dropit:
 	return (0);
 bad:
 	icmp_error(m, type, code, 0, 0);
-	V_ipstat.ips_badoptions++;
+	IPSTAT_INC(ips_badoptions);
 	return (1);
 }
 
diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c
index 90df601..16dba61 100644
--- a/sys/netinet/ip_output.c
+++ b/sys/netinet/ip_output.c
@@ -153,6 +153,10 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags,
 	if (inp != NULL) {
 		M_SETFIB(m, inp->inp_inc.inc_fibnum);
 		INP_LOCK_ASSERT(inp);
+		if (inp->inp_flags & (INP_HW_FLOWID|INP_SW_FLOWID)) {
+			m->m_pkthdr.flowid = inp->inp_flowid;
+			m->m_flags |= M_FLOWID;
+		}
 	}
 
 	if (opt) {
@@ -178,7 +182,7 @@ ip_output(struct mbuf *m, struct mbuf *opt, struct route *ro, int flags,
 		ip->ip_v = IPVERSION;
 		ip->ip_hl = hlen >> 2;
 		ip->ip_id = ip_newid();
-		V_ipstat.ips_localout++;
+		IPSTAT_INC(ips_localout);
 	} else {
 		hlen = ip->ip_hl << 2;
 	}
@@ -217,7 +221,7 @@ again:
 	if (flags & IP_SENDONES) {
 		if ((ia = ifatoia(ifa_ifwithbroadaddr(sintosa(dst)))) == NULL &&
 		    (ia = ifatoia(ifa_ifwithdstaddr(sintosa(dst)))) == NULL) {
-			V_ipstat.ips_noroute++;
+			IPSTAT_INC(ips_noroute);
 			error = ENETUNREACH;
 			goto bad;
 		}
@@ -229,7 +233,7 @@ again:
 	} else if (flags & IP_ROUTETOIF) {
 		if ((ia = ifatoia(ifa_ifwithdstaddr(sintosa(dst)))) == NULL &&
 		    (ia = ifatoia(ifa_ifwithnet(sintosa(dst)))) == NULL) {
-			V_ipstat.ips_noroute++;
+			IPSTAT_INC(ips_noroute);
 			error = ENETUNREACH;
 			goto bad;
 		}
@@ -261,7 +265,7 @@ again:
 			    inp ? inp->inp_inc.inc_fibnum : M_GETFIB(m));
 #endif
 		if (ro->ro_rt == NULL) {
-			V_ipstat.ips_noroute++;
+			IPSTAT_INC(ips_noroute);
 			error = EHOSTUNREACH;
 			goto bad;
 		}
@@ -318,7 +322,7 @@ again:
 		 */
 		if ((imo == NULL) || (imo->imo_multicast_vif == -1)) {
 			if ((ifp->if_flags & IFF_MULTICAST) == 0) {
-				V_ipstat.ips_noroute++;
+				IPSTAT_INC(ips_noroute);
 				error = ENETUNREACH;
 				goto bad;
 			}
@@ -416,7 +420,7 @@ again:
 #endif /* ALTQ */
 	{
 		error = ENOBUFS;
-		V_ipstat.ips_odropped++;
+		IPSTAT_INC(ips_odropped);
 		ifp->if_snd.ifq_drops += (ip->ip_len / ifp->if_mtu + 1);
 		goto bad;
 	}
@@ -534,7 +538,7 @@ passout:
 	if ((ntohl(ip->ip_dst.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET ||
 	    (ntohl(ip->ip_src.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET) {
 		if ((ifp->if_flags & IFF_LOOPBACK) == 0) {
-			V_ipstat.ips_badaddr++;
+			IPSTAT_INC(ips_badaddr);
 			error = EADDRNOTAVAIL;
 			goto bad;
 		}
@@ -598,7 +602,7 @@ passout:
 	/* Balk when DF bit is set or the interface didn't support TSO. */
 	if ((ip->ip_off & IP_DF) || (m->m_pkthdr.csum_flags & CSUM_TSO)) {
 		error = EMSGSIZE;
-		V_ipstat.ips_cantfrag++;
+		IPSTAT_INC(ips_cantfrag);
 		goto bad;
 	}
 
@@ -631,7 +635,7 @@ passout:
 	}
 
 	if (error == 0)
-		V_ipstat.ips_fragmented++;
+		IPSTAT_INC(ips_fragmented);
 
 done:
 	if (ro == &iproute && ro->ro_rt) {
@@ -667,7 +671,7 @@ ip_fragment(struct ip *ip, struct mbuf **m_frag, int mtu,
 	int nfrags;
 
 	if (ip->ip_off & IP_DF) {	/* Fragmentation not allowed */
-		V_ipstat.ips_cantfrag++;
+		IPSTAT_INC(ips_cantfrag);
 		return EMSGSIZE;
 	}
 
@@ -748,7 +752,7 @@ smart_frag_failure:
 		MGETHDR(m, M_DONTWAIT, MT_DATA);
 		if (m == NULL) {
 			error = ENOBUFS;
-			V_ipstat.ips_odropped++;
+			IPSTAT_INC(ips_odropped);
 			goto done;
 		}
 		m->m_flags |= (m0->m_flags & M_MCAST) | M_FRAG;
@@ -778,7 +782,7 @@ smart_frag_failure:
 		if (m->m_next == NULL) {	/* copy failed */
 			m_free(m);
 			error = ENOBUFS;	/* ??? */
-			V_ipstat.ips_odropped++;
+			IPSTAT_INC(ips_odropped);
 			goto done;
 		}
 		m->m_pkthdr.len = mhlen + len;
@@ -794,7 +798,7 @@ smart_frag_failure:
 		*mnext = m;
 		mnext = &m->m_nextpkt;
 	}
-	V_ipstat.ips_ofragments += nfrags;
+	IPSTAT_ADD(ips_ofragments, nfrags);
 
 	/* set first marker for fragment chain */
 	m0->m_flags |= M_FIRSTFRAG | M_FRAG;
diff --git a/sys/netinet/ip_var.h b/sys/netinet/ip_var.h
index b68f481..472a458 100644
--- a/sys/netinet/ip_var.h
+++ b/sys/netinet/ip_var.h
@@ -132,6 +132,11 @@ struct	ipstat {
 
 #ifdef _KERNEL
 
+#define	IPSTAT_ADD(name, val)	V_ipstat.name += (val)
+#define	IPSTAT_SUB(name, val)	V_ipstat.name -= (val)
+#define	IPSTAT_INC(name)	IPSTAT_ADD(name, 1)
+#define	IPSTAT_DEC(name)	IPSTAT_SUB(name, 1)
+
 /* flags passed to ip_output as last parameter */
 #define	IP_FORWARDING		0x1		/* most of ip header exists */
 #define	IP_RAWOUTPUT		0x2		/* raw ip header exists */
diff --git a/sys/netinet/libalias/alias.c b/sys/netinet/libalias/alias.c
index 9d80da9..2e469d7 100644
--- a/sys/netinet/libalias/alias.c
+++ b/sys/netinet/libalias/alias.c
@@ -742,7 +742,7 @@ UdpAliasIn(struct libalias *la, struct ip *pip)
 		u_short alias_port;
 		u_short proxy_port;
 		int accumulate;
-		int r = 0, error;
+		int error;
 		struct alias_data ad = {
 			.lnk = lnk, 
 			.oaddr = &original_address, 
@@ -762,6 +762,9 @@ UdpAliasIn(struct libalias *la, struct ip *pip)
 
 		/* Walk out chain. */		
 		error = find_handler(IN, UDP, la, pip, &ad);
+		/* If we cannot figure out the packet, ignore it. */
+		if (error < 0)
+			return (PKT_ALIAS_IGNORED);
 
 /* If UDP checksum is not zero, then adjust since destination port */
 /* is being unaliased and destination address is being altered.    */
@@ -801,13 +804,7 @@ UdpAliasIn(struct libalias *la, struct ip *pip)
 		    &original_address, &pip->ip_dst, 2);
 		pip->ip_dst = original_address;
 
-		/*
-		 * If we cannot figure out the packet, ignore it.
-		 */
-		if (r < 0)
-			return (PKT_ALIAS_IGNORED);
-		else
-			return (PKT_ALIAS_OK);
+		return (PKT_ALIAS_OK);
 	}
 	return (PKT_ALIAS_IGNORED);
 }
@@ -1226,7 +1223,6 @@ FragmentOut(struct libalias *la, struct in_addr *ip_src, u_short *ip_sum)
 (prototypes in alias.h)
 */
 
-// XXX ip free
 int
 LibAliasSaveFragment(struct libalias *la, char *ptr)
 {
@@ -1246,7 +1242,6 @@ LibAliasSaveFragment(struct libalias *la, char *ptr)
 	return (iresult);
 }
 
-// XXX ip free
 char           *
 LibAliasGetFragment(struct libalias *la, char *ptr)
 {
@@ -1268,7 +1263,6 @@ LibAliasGetFragment(struct libalias *la, char *ptr)
 	return (fptr);
 }
 
-// XXX ip free
 void
 LibAliasFragmentIn(struct libalias *la, char *ptr,	/* Points to correctly
 							 * de-aliased header
diff --git a/sys/netinet/libalias/alias_cuseeme.c b/sys/netinet/libalias/alias_cuseeme.c
index f255942..28f5e93 100644
--- a/sys/netinet/libalias/alias_cuseeme.c
+++ b/sys/netinet/libalias/alias_cuseeme.c
@@ -64,7 +64,7 @@ AliasHandleCUSeeMeIn(struct libalias *la, struct ip *pip,
 		     struct in_addr original_addr);
 
 static int 
-fingerprint(struct libalias *la, struct ip *pip, struct alias_data *ah)
+fingerprint(struct libalias *la, struct alias_data *ah)
 {
 
 	if (ah->dport == NULL || ah->oaddr == NULL)
diff --git a/sys/netinet/libalias/alias_dummy.c b/sys/netinet/libalias/alias_dummy.c
index fc6a0f9..4283aaf 100644
--- a/sys/netinet/libalias/alias_dummy.c
+++ b/sys/netinet/libalias/alias_dummy.c
@@ -60,7 +60,7 @@ static void
 AliasHandleDummy(struct libalias *la, struct ip *ip, struct alias_data *ah);
 
 static int 
-fingerprint(struct libalias *la, struct ip *pip, struct alias_data *ah)
+fingerprint(struct libalias *la, struct alias_data *ah)
 {
 
 	/* 
diff --git a/sys/netinet/libalias/alias_ftp.c b/sys/netinet/libalias/alias_ftp.c
index 50fa8d0..ef0e52c 100644
--- a/sys/netinet/libalias/alias_ftp.c
+++ b/sys/netinet/libalias/alias_ftp.c
@@ -104,7 +104,7 @@ AliasHandleFtpOut(struct libalias *, struct ip *, struct alias_link *,
 		  int maxpacketsize);
 
 static int 
-fingerprint(struct libalias *la, struct ip *pip, struct alias_data *ah)
+fingerprint(struct libalias *la, struct alias_data *ah)
 {
 
 	if (ah->dport == NULL || ah->sport == NULL || ah->lnk == NULL || 
diff --git a/sys/netinet/libalias/alias_irc.c b/sys/netinet/libalias/alias_irc.c
index 8197fb8..c01e8e6 100644
--- a/sys/netinet/libalias/alias_irc.c
+++ b/sys/netinet/libalias/alias_irc.c
@@ -93,7 +93,7 @@ AliasHandleIrcOut(struct libalias *, struct ip *, struct alias_link *,
 		  int maxpacketsize);
 
 static int 
-fingerprint(struct libalias *la, struct ip *pip, struct alias_data *ah)
+fingerprint(struct libalias *la, struct alias_data *ah)
 {
 
 	if (ah->dport == NULL || ah->dport == NULL || ah->lnk == NULL || 
diff --git a/sys/netinet/libalias/alias_mod.c b/sys/netinet/libalias/alias_mod.c
index 643fe5f..2713137 100644
--- a/sys/netinet/libalias/alias_mod.c
+++ b/sys/netinet/libalias/alias_mod.c
@@ -133,9 +133,10 @@ handler_chain_destroy(void)
 static int
 _attach_handler(struct proto_handler *p)
 {
-	struct proto_handler *b = NULL;
+	struct proto_handler *b;
 
-	LIBALIAS_WLOCK_ASSERT();	
+	LIBALIAS_WLOCK_ASSERT();
+	b = NULL;
 	LIST_FOREACH(b, &handler_chain, entries) {
 		if ((b->pri == p->pri) && 
 		    (b->dir == p->dir) &&
@@ -172,10 +173,11 @@ _detach_handler(struct proto_handler *p)
 int
 LibAliasAttachHandlers(struct proto_handler *_p)
 {
-	int i, error = -1;
+	int i, error;
 
 	LIBALIAS_WLOCK();
-	for (i=0; 1; i++) {
+	error = -1;
+	for (i = 0; 1; i++) {
 		if (*((int *)&_p[i]) == EOH) 
 			break;
 		error = _attach_handler(&_p[i]);
@@ -189,10 +191,11 @@ LibAliasAttachHandlers(struct proto_handler *_p)
 int
 LibAliasDetachHandlers(struct proto_handler *_p)
 {
-	int i, error = -1;
+	int i, error;
 
 	LIBALIAS_WLOCK();
-	for (i=0; 1; i++) {
+	error = -1;
+	for (i = 0; 1; i++) {
 		if (*((int *)&_p[i]) == EOH) 
 			break;
 		error = _detach_handler(&_p[i]);
@@ -206,26 +209,27 @@ LibAliasDetachHandlers(struct proto_handler *_p)
 int
 detach_handler(struct proto_handler *_p)
 {
-	int error = -1;
+	int error;
 
 	LIBALIAS_WLOCK();
+	error = -1;
 	error = _detach_handler(_p);
 	LIBALIAS_WUNLOCK();
 	return (error);
 }
 
 int
-find_handler(int8_t dir, int8_t proto, struct libalias *la, struct ip *pip, 
-	     struct alias_data *ad)
+find_handler(int8_t dir, int8_t proto, struct libalias *la, __unused struct ip *pip, 
+    struct alias_data *ad)
 {
 	struct proto_handler *p;
-	int error = ENOENT;
+	int error;
 
 	LIBALIAS_RLOCK();
-	
+	error = ENOENT;
 	LIST_FOREACH(p, &handler_chain, entries) {
 		if ((p->dir & dir) && (p->proto & proto))
-			if (p->fingerprint(la, pip, ad) == 0) {
+			if (p->fingerprint(la, ad) == 0) {
 				error = p->protohandler(la, pip, ad);
 				break;
 			}
@@ -259,9 +263,11 @@ attach_dll(struct dll *p)
 void *
 detach_dll(char *p)
 {
-	struct dll *b = NULL, *b_tmp;
-	void *error = NULL;
+	struct dll *b, *b_tmp;
+	void *error;
 
+	b = NULL;
+	error = NULL;
 	SLIST_FOREACH_SAFE(b, &dll_chain, next, b_tmp)
 		if (!strncmp(b->name, p, DLL_LEN)) {
 			SLIST_REMOVE(&dll_chain, b, dll, next); 
diff --git a/sys/netinet/libalias/alias_mod.h b/sys/netinet/libalias/alias_mod.h
index 5e193e9..f2f679d 100644
--- a/sys/netinet/libalias/alias_mod.h
+++ b/sys/netinet/libalias/alias_mod.h
@@ -80,10 +80,10 @@ struct proto_handler {
 	u_int pri;                                              /* Handler priority. */
         int16_t dir;                                            /* Flow direction. */
 	uint8_t proto;                                          /* Working protocol. */	
-	int (*fingerprint)(struct libalias *la,                 /* Fingerprint * function. */
-		 struct ip *pip, struct alias_data *ah);
-	int (*protohandler)(struct libalias *la,                /* Aliasing * function. */
-		 struct ip *pip, struct alias_data *ah);                 
+	int (*fingerprint)(struct libalias *,                   /* Fingerprint * function. */
+	    struct alias_data *);                
+	int (*protohandler)(struct libalias *,                  /* Aliasing * function. */
+	    struct ip *, struct alias_data *);                 
 	LIST_ENTRY(proto_handler) entries;
 };
 
diff --git a/sys/netinet/libalias/alias_nbt.c b/sys/netinet/libalias/alias_nbt.c
index 924ee6a..6fcf727 100644
--- a/sys/netinet/libalias/alias_nbt.c
+++ b/sys/netinet/libalias/alias_nbt.c
@@ -77,7 +77,7 @@ static int
 AliasHandleUdpNbtNS(struct libalias *, struct ip *, struct alias_link *,
 		    struct in_addr *, u_short *, struct in_addr *, u_short *);
 static int 
-fingerprint1(struct libalias *la, struct ip *pip, struct alias_data *ah)
+fingerprint1(struct libalias *la, struct alias_data *ah)
 {
 
 	if (ah->dport == NULL || ah->sport == NULL || ah->lnk == NULL || 
@@ -93,12 +93,11 @@ static int
 protohandler1(struct libalias *la, struct ip *pip, struct alias_data *ah)
 {
 	
-	AliasHandleUdpNbt(la, pip, ah->lnk, ah->aaddr, *ah->aport);
-	return (0);
+	return (AliasHandleUdpNbt(la, pip, ah->lnk, ah->aaddr, *ah->aport));
 }
 
 static int 
-fingerprint2(struct libalias *la, struct ip *pip, struct alias_data *ah)
+fingerprint2(struct libalias *la, struct alias_data *ah)
 {
 
 	if (ah->dport == NULL || ah->sport == NULL || ah->lnk == NULL || 
@@ -123,9 +122,8 @@ static int
 protohandler2out(struct libalias *la, struct ip *pip, struct alias_data *ah)
 {
 	
-	AliasHandleUdpNbtNS(la, pip, ah->lnk, &pip->ip_src, ah->sport,
- 			    ah->aaddr, ah->aport);
-	return (0);
+	return (AliasHandleUdpNbtNS(la, pip, ah->lnk, &pip->ip_src, ah->sport,
+ 	    ah->aaddr, ah->aport));
 }
 
 /* Kernel module definition. */
diff --git a/sys/netinet/libalias/alias_pptp.c b/sys/netinet/libalias/alias_pptp.c
index b4fd9d0..3465c78 100644
--- a/sys/netinet/libalias/alias_pptp.c
+++ b/sys/netinet/libalias/alias_pptp.c
@@ -79,7 +79,7 @@ static int
 AliasHandlePptpGreIn(struct libalias *, struct ip *);
 
 static int 
-fingerprint(struct libalias *la, struct ip *pip, struct alias_data *ah)
+fingerprint(struct libalias *la, struct alias_data *ah)
 {
 
 	if (ah->dport == NULL || ah->sport == NULL || ah->lnk == NULL)
@@ -91,7 +91,7 @@ fingerprint(struct libalias *la, struct ip *pip, struct alias_data *ah)
 }
 
 static int 
-fingerprintgre(struct libalias *la, struct ip *pip, struct alias_data *ah)
+fingerprintgre(struct libalias *la, struct alias_data *ah)
 {
 
 	return (0);
diff --git a/sys/netinet/libalias/alias_skinny.c b/sys/netinet/libalias/alias_skinny.c
index e726e55..12be97a 100644
--- a/sys/netinet/libalias/alias_skinny.c
+++ b/sys/netinet/libalias/alias_skinny.c
@@ -57,7 +57,7 @@ static void
 AliasHandleSkinny(struct libalias *, struct ip *, struct alias_link *);
 
 static int 
-fingerprint(struct libalias *la, struct ip *pip, struct alias_data *ah)
+fingerprint(struct libalias *la, struct alias_data *ah)
 {
 
 	if (ah->dport == NULL || ah->sport == NULL || ah->lnk == NULL)
diff --git a/sys/netinet/libalias/alias_smedia.c b/sys/netinet/libalias/alias_smedia.c
index e748ad7..2e1b17e 100644
--- a/sys/netinet/libalias/alias_smedia.c
+++ b/sys/netinet/libalias/alias_smedia.c
@@ -132,7 +132,7 @@ static void
 AliasHandleRtspOut(struct libalias *, struct ip *, struct alias_link *,	
 		  int maxpacketsize);
 static int 
-fingerprint(struct libalias *la, struct ip *pip, struct alias_data *ah)
+fingerprint(struct libalias *la, struct alias_data *ah)
 {
 
 	if (ah->dport != NULL && ah->aport != NULL && ah->sport != NULL &&
diff --git a/sys/netinet/pim_var.h b/sys/netinet/pim_var.h
index f79edde..aecb673 100644
--- a/sys/netinet/pim_var.h
+++ b/sys/netinet/pim_var.h
@@ -59,6 +59,11 @@ struct pimstat {
 	u_quad_t pims_snd_registers_bytes; /* sent regs. bytes (data only)   */
 };
 
+#ifdef _KERNEL
+#define	PIMSTAT_ADD(name, val)	pimstat.name += (val)
+#define	PIMSTAT_INC(name)	PIMSTAT_ADD(name, 1)
+#endif
+
 /*
  * Names for PIM sysctl objects
  */
diff --git a/sys/netinet/raw_ip.c b/sys/netinet/raw_ip.c
index b536eb7..0775168 100644
--- a/sys/netinet/raw_ip.c
+++ b/sys/netinet/raw_ip.c
@@ -345,7 +345,7 @@ rip_input(struct mbuf *m, int off)
 			    (struct sockaddr *)&group,
 			    (struct sockaddr *)&ripsrc);
 			if (blocked != MCAST_PASS) {
-				V_ipstat.ips_notmember++;
+				IPSTAT_INC(ips_notmember);
 				continue;
 			}
 		}
@@ -364,12 +364,12 @@ rip_input(struct mbuf *m, int off)
 	INP_INFO_RUNLOCK(&V_ripcbinfo);
 	if (last != NULL) {
 		if (rip_append(last, ip, m, &ripsrc) != 0)
-			V_ipstat.ips_delivered--;
+			IPSTAT_INC(ips_delivered);
 		INP_RUNLOCK(last);
 	} else {
 		m_freem(m);
-		V_ipstat.ips_noproto++;
-		V_ipstat.ips_delivered--;
+		IPSTAT_INC(ips_noproto);
+		IPSTAT_DEC(ips_delivered);
 	}
 }
 
@@ -450,7 +450,7 @@ rip_output(struct mbuf *m, struct socket *so, u_long dst)
 		 * XXX prevent ip_output from overwriting header fields.
 		 */
 		flags |= IP_RAWOUTPUT;
-		V_ipstat.ips_rawout++;
+		IPSTAT_INC(ips_rawout);
 	}
 
 	if (inp->inp_flags & INP_ONESBCAST)
diff --git a/sys/netinet/sctp.h b/sys/netinet/sctp.h
index afbf5ea..83b939a 100644
--- a/sys/netinet/sctp.h
+++ b/sys/netinet/sctp.h
@@ -544,7 +544,7 @@ struct sctp_error_unrecognized_chunk {
 #define SCTP_THRESHOLD_LOGGING              0x02000000
 #define SCTP_LOG_AT_SEND_2_SCTP             0x04000000
 #define SCTP_LOG_AT_SEND_2_OUTQ             0x08000000
-
+#define SCTP_LOG_TRY_ADVANCE                0x10000000
 
 
 #undef SCTP_PACKED
diff --git a/sys/netinet/sctp_constants.h b/sys/netinet/sctp_constants.h
index bdd1849..8df3490 100644
--- a/sys/netinet/sctp_constants.h
+++ b/sys/netinet/sctp_constants.h
@@ -229,8 +229,8 @@ __FBSDID("$FreeBSD$");
 #define SCTP_THRESHOLD_CLEAR       120
 #define SCTP_THRESHOLD_INCR        121
 #define SCTP_FLIGHT_LOG_DWN_WP_FWD 122
-
-#define SCTP_LOG_MAX_TYPES 123
+#define SCTP_FWD_TSN_CHECK         123
+#define SCTP_LOG_MAX_TYPES 124
 /*
  * To turn on various logging, you must first enable 'options KTR' and
  * you might want to bump the entires 'options KTR_ENTRIES=80000'.
diff --git a/sys/netinet/sctp_indata.c b/sys/netinet/sctp_indata.c
index f9fd08f..98dc28e 100644
--- a/sys/netinet/sctp_indata.c
+++ b/sys/netinet/sctp_indata.c
@@ -423,12 +423,13 @@ abandon:
 		if (SCTP_BASE_SYSCTL(sctp_nr_sack_on_off) && asoc->peer_supports_nr_sack) {
 
 			nr_tsn = chk->rec.data.TSN_seq;
-			if (nr_tsn >= asoc->nr_mapping_array_base_tsn) {
+			if ((compare_with_wrap(nr_tsn, asoc->nr_mapping_array_base_tsn, MAX_TSN)) ||
+			    (nr_tsn == asoc->nr_mapping_array_base_tsn)) {
 				nr_gap = nr_tsn - asoc->nr_mapping_array_base_tsn;
 			} else {
 				nr_gap = (MAX_TSN - asoc->nr_mapping_array_base_tsn) + nr_tsn + 1;
 			}
-			if ((nr_gap >= (SCTP_NR_MAPPING_ARRAY << 3)) ||
+			if ((nr_gap >= (uint32_t) (asoc->nr_mapping_array_size << 3)) ||
 			    (nr_gap >= (uint32_t) (asoc->nr_mapping_array_size << 3))) {
 				/*
 				 * EY The 1st should never happen, as in
@@ -440,10 +441,11 @@ abandon:
 				 * nr_mapping_array is always expanded when
 				 * mapping_array is expanded
 				 */
+				printf("Impossible nr_gap ack range failed\n");
 			} else {
 				SCTP_TCB_LOCK_ASSERT(stcb);
 				SCTP_SET_TSN_PRESENT(asoc->nr_mapping_array, nr_gap);
-				if (nr_tsn > asoc->highest_tsn_inside_nr_map)
+				if (compare_with_wrap(nr_tsn, asoc->highest_tsn_inside_nr_map, MAX_TSN))
 					asoc->highest_tsn_inside_nr_map = nr_tsn;
 			}
 		}
@@ -550,7 +552,9 @@ abandon:
 							} else {
 								SCTP_TCB_LOCK_ASSERT(stcb);
 								SCTP_SET_TSN_PRESENT(asoc->nr_mapping_array, nr_gap);
-								if (nr_tsn > asoc->highest_tsn_inside_nr_map)
+								if (compare_with_wrap(nr_tsn,
+								    asoc->highest_tsn_inside_nr_map,
+								    MAX_TSN))
 									asoc->highest_tsn_inside_nr_map = nr_tsn;
 							}
 						}
@@ -699,7 +703,7 @@ protocol_error:
 			} else {
 				SCTP_TCB_LOCK_ASSERT(stcb);
 				SCTP_SET_TSN_PRESENT(asoc->nr_mapping_array, nr_gap);
-				if (nr_tsn > asoc->highest_tsn_inside_nr_map)
+				if (compare_with_wrap(nr_tsn, asoc->highest_tsn_inside_nr_map, MAX_TSN))
 					asoc->highest_tsn_inside_nr_map = nr_tsn;
 			}
 		}
@@ -760,7 +764,8 @@ protocol_error:
 					} else {
 						SCTP_TCB_LOCK_ASSERT(stcb);
 						SCTP_SET_TSN_PRESENT(asoc->nr_mapping_array, nr_gap);
-						if (nr_tsn > asoc->highest_tsn_inside_nr_map)
+						if (compare_with_wrap(nr_tsn, asoc->highest_tsn_inside_nr_map,
+						    MAX_TSN))
 							asoc->highest_tsn_inside_nr_map = nr_tsn;
 					}
 				}
@@ -2390,6 +2395,15 @@ finish_express_del:
 	}
 	SCTP_TCB_LOCK_ASSERT(stcb);
 	SCTP_SET_TSN_PRESENT(asoc->mapping_array, gap);
+
+	if (SCTP_BASE_SYSCTL(sctp_nr_sack_on_off) &&
+	    asoc->peer_supports_nr_sack &&
+	    (SCTP_BASE_SYSCTL(sctp_do_drain) == 0)) {
+		SCTP_SET_TSN_PRESENT(asoc->nr_mapping_array, gap);
+		if (compare_with_wrap(tsn, asoc->highest_tsn_inside_nr_map, MAX_TSN)) {
+			asoc->highest_tsn_inside_nr_map = tsn;
+		}
+	}
 	/* check the special flag for stream resets */
 	if (((liste = TAILQ_FIRST(&asoc->resetHead)) != NULL) &&
 	    ((compare_with_wrap(asoc->cumulative_tsn, liste->tsn, MAX_TSN)) ||
@@ -2498,9 +2512,9 @@ sctp_sack_check(struct sctp_tcb *stcb, int ok_to_sack, int was_a_gap, int *abort
 	int slide_from, slide_end, lgap, distance;
 
 	/* EY nr_mapping array variables */
-	int nr_at;
-	int nr_last_all_ones = 0;
-	int nr_slide_from, nr_slide_end, nr_lgap, nr_distance;
+	/* int nr_at; */
+	/* int nr_last_all_ones = 0; */
+	/* int nr_slide_from, nr_slide_end, nr_lgap, nr_distance; */
 
 	uint32_t old_cumack, old_base, old_highest;
 	unsigned char aux_array[64];
@@ -2683,102 +2697,19 @@ sctp_sack_check(struct sctp_tcb *stcb, int ok_to_sack, int was_a_gap, int *abort
 				    asoc->cumulative_tsn, asoc->highest_tsn_inside_map,
 				    SCTP_MAP_SLIDE_RESULT);
 			}
-		}
-	}
-	/*
-	 * EY if doing nr_sacks then slide the nr_mapping_array accordingly
-	 * please
-	 */
-	if (SCTP_BASE_SYSCTL(sctp_nr_sack_on_off) && asoc->peer_supports_nr_sack) {
-
-		nr_at = 0;
-		for (nr_slide_from = 0; nr_slide_from < stcb->asoc.nr_mapping_array_size; nr_slide_from++) {
-
-			if (asoc->nr_mapping_array[nr_slide_from] == 0xff) {
-				nr_at += 8;
-				nr_last_all_ones = 1;
-			} else {
-				/* there is a 0 bit */
-				nr_at += sctp_map_lookup_tab[asoc->nr_mapping_array[nr_slide_from]];
-				nr_last_all_ones = 0;
-				break;
-			}
-		}
-
-		nr_at++;
-
-		if (compare_with_wrap(asoc->cumulative_tsn,
-		    asoc->highest_tsn_inside_nr_map, MAX_TSN) && (at >= 8)) {
-			/* The complete array was completed by a single FR */
-			/* higest becomes the cum-ack */
-			int clr;
-
-			clr = (nr_at >> 3) + 1;
-
-			if (clr > asoc->nr_mapping_array_size)
-				clr = asoc->nr_mapping_array_size;
-
-			memset(asoc->nr_mapping_array, 0, clr);
-			/* base becomes one ahead of the cum-ack */
-			asoc->nr_mapping_array_base_tsn = asoc->cumulative_tsn + 1;
-			asoc->highest_tsn_inside_nr_map = asoc->cumulative_tsn;
-
-		} else if (nr_at >= 8) {
-			/* we can slide the mapping array down */
-			/* Calculate the new byte postion we can move down */
-
 			/*
-			 * now calculate the ceiling of the move using our
-			 * highest TSN value
+			 * EY if doing nr_sacks then slide the
+			 * nr_mapping_array accordingly please
 			 */
-			if (asoc->highest_tsn_inside_nr_map >= asoc->nr_mapping_array_base_tsn) {
-				nr_lgap = asoc->highest_tsn_inside_nr_map -
-				    asoc->nr_mapping_array_base_tsn;
-			} else {
-				nr_lgap = (MAX_TSN - asoc->nr_mapping_array_base_tsn) +
-				    asoc->highest_tsn_inside_nr_map + 1;
-			}
-			nr_slide_end = nr_lgap >> 3;
-			if (nr_slide_end < nr_slide_from) {
-#ifdef INVARIANTS
-				panic("impossible slide");
-#else
-				printf("impossible slide?\n");
-				return;
-#endif
-			}
-			if (nr_slide_end > asoc->nr_mapping_array_size) {
-#ifdef INVARIANTS
-				panic("would overrun buffer");
-#else
-				printf("Gak, would have overrun map end:%d nr_slide_end:%d\n",
-				    asoc->nr_mapping_array_size, nr_slide_end);
-				nr_slide_end = asoc->nr_mapping_array_size;
-#endif
-			}
-			nr_distance = (nr_slide_end - nr_slide_from) + 1;
-
-			if (nr_distance + nr_slide_from > asoc->nr_mapping_array_size ||
-			    nr_distance < 0) {
-				/*
-				 * Here we do NOT slide forward the array so
-				 * that hopefully when more data comes in to
-				 * fill it up we will be able to slide it
-				 * forward. Really I don't think this should
-				 * happen :-0
-				 */
-				;
-			} else {
-				int ii;
-
-				for (ii = 0; ii < nr_distance; ii++) {
+			if (SCTP_BASE_SYSCTL(sctp_nr_sack_on_off) && asoc->peer_supports_nr_sack) {
+				for (ii = 0; ii < distance; ii++) {
 					asoc->nr_mapping_array[ii] =
-					    asoc->nr_mapping_array[nr_slide_from + ii];
+					    asoc->nr_mapping_array[slide_from + ii];
 				}
-				for (ii = nr_distance; ii <= nr_slide_end; ii++) {
+				for (ii = distance; ii <= slide_end; ii++) {
 					asoc->nr_mapping_array[ii] = 0;
 				}
-				asoc->nr_mapping_array_base_tsn += (nr_slide_from << 3);
+				asoc->nr_mapping_array_base_tsn += (slide_from << 3);
 			}
 		}
 	}
@@ -2802,7 +2733,7 @@ sctp_sack_check(struct sctp_tcb *stcb, int ok_to_sack, int was_a_gap, int *abort
 			 * EY if nr_sacks used then send an nr-sack , a sack
 			 * otherwise
 			 */
-			if (SCTP_BASE_SYSCTL(sctp_nr_sack_on_off) && asoc->peer_supports_nr_sack)
+			if (SCTP_BASE_SYSCTL(sctp_nr_sack_on_off) && stcb->asoc.peer_supports_nr_sack)
 				sctp_send_nr_sack(stcb);
 			else
 				sctp_send_sack(stcb);
@@ -3496,9 +3427,13 @@ sctp_handle_segments(struct mbuf *m, int *offset, struct sctp_tcb *stcb, struct
 						/*
 						 * All chunks NOT UNSENT
 						 * fall through here and are
-						 * marked
+						 * marked (leave PR-SCTP
+						 * ones that are to skip
+						 * alone though)
 						 */
-						tp1->sent = SCTP_DATAGRAM_MARKED;
+						if (tp1->sent != SCTP_FORWARD_TSN_SKIP)
+							tp1->sent = SCTP_DATAGRAM_MARKED;
+
 						if (tp1->rec.data.chunk_was_revoked) {
 							/* deflate the cwnd */
 							tp1->whoTo->cwnd -= tp1->book_size;
@@ -3680,20 +3615,7 @@ sctp_strike_gap_ack_chunks(struct sctp_tcb *stcb, struct sctp_association *asoc,
 					if (tp1->data != NULL) {
 						(void)sctp_release_pr_sctp_chunk(stcb, tp1,
 						    (SCTP_RESPONSE_TO_USER_REQ | SCTP_NOTIFY_DATAGRAM_SENT),
-						    &asoc->sent_queue, SCTP_SO_NOT_LOCKED);
-					}
-					tp1 = TAILQ_NEXT(tp1, sctp_next);
-					continue;
-				}
-			}
-			if ((PR_SCTP_RTX_ENABLED(tp1->flags)) && tp1->sent < SCTP_DATAGRAM_ACKED) {
-				/* Has it been retransmitted tv_sec times? */
-				if (tp1->snd_count > tp1->rec.data.timetodrop.tv_sec) {
-					/* Yes, so drop it */
-					if (tp1->data != NULL) {
-						(void)sctp_release_pr_sctp_chunk(stcb, tp1,
-						    (SCTP_RESPONSE_TO_USER_REQ | SCTP_NOTIFY_DATAGRAM_SENT),
-						    &asoc->sent_queue, SCTP_SO_NOT_LOCKED);
+						    SCTP_SO_NOT_LOCKED);
 					}
 					tp1 = TAILQ_NEXT(tp1, sctp_next);
 					continue;
@@ -3914,6 +3836,25 @@ sctp_strike_gap_ack_chunks(struct sctp_tcb *stcb, struct sctp_association *asoc,
 			/* Increment the count to resend */
 			struct sctp_nets *alt;
 
+			if ((stcb->asoc.peer_supports_prsctp) &&
+			    (PR_SCTP_RTX_ENABLED(tp1->flags))) {
+				/*
+				 * Has it been retransmitted tv_sec times? -
+				 * we store the retran count there.
+				 */
+				if (tp1->snd_count > tp1->rec.data.timetodrop.tv_sec) {
+					/* Yes, so drop it */
+					if (tp1->data != NULL) {
+						(void)sctp_release_pr_sctp_chunk(stcb, tp1,
+						    (SCTP_RESPONSE_TO_USER_REQ | SCTP_NOTIFY_DATAGRAM_SENT),
+						    SCTP_SO_NOT_LOCKED);
+					}
+					/* Make sure to flag we had a FR */
+					tp1->whoTo->net_ack++;
+					tp1 = TAILQ_NEXT(tp1, sctp_next);
+					continue;
+				}
+			}
 			/* printf("OK, we are now ready to FR this guy\n"); */
 			if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_FR_LOGGING_ENABLE) {
 				sctp_log_fr(tp1->rec.data.TSN_seq, tp1->snd_count,
@@ -4078,6 +4019,13 @@ sctp_try_advance_peer_ack_point(struct sctp_tcb *stcb,
 			/* no chance to advance, out of here */
 			break;
 		}
+		if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_LOG_TRY_ADVANCE) {
+			if (tp1->sent == SCTP_FORWARD_TSN_SKIP) {
+				sctp_misc_ints(SCTP_FWD_TSN_CHECK,
+				    asoc->advanced_peer_ack_point,
+				    tp1->rec.data.TSN_seq, 0, 0);
+			}
+		}
 		if (!PR_SCTP_ENABLED(tp1->flags)) {
 			/*
 			 * We can't fwd-tsn past any that are reliable aka
@@ -4107,7 +4055,7 @@ sctp_try_advance_peer_ack_point(struct sctp_tcb *stcb,
 				if (tp1->data) {
 					(void)sctp_release_pr_sctp_chunk(stcb, tp1,
 					    (SCTP_RESPONSE_TO_USER_REQ | SCTP_NOTIFY_DATAGRAM_SENT),
-					    &asoc->sent_queue, SCTP_SO_NOT_LOCKED);
+					    SCTP_SO_NOT_LOCKED);
 				}
 			} else {
 				/*
@@ -4124,8 +4072,16 @@ sctp_try_advance_peer_ack_point(struct sctp_tcb *stcb,
 		 */
 		if (tp1->sent == SCTP_FORWARD_TSN_SKIP) {
 			/* advance PeerAckPoint goes forward */
-			asoc->advanced_peer_ack_point = tp1->rec.data.TSN_seq;
-			a_adv = tp1;
+			if (compare_with_wrap(tp1->rec.data.TSN_seq,
+			    asoc->advanced_peer_ack_point,
+			    MAX_TSN)) {
+
+				asoc->advanced_peer_ack_point = tp1->rec.data.TSN_seq;
+				a_adv = tp1;
+			} else if (tp1->rec.data.TSN_seq == asoc->advanced_peer_ack_point) {
+				/* No update but we do save the chk */
+				a_adv = tp1;
+			}
 		} else {
 			/*
 			 * If it is still in RESEND we can advance no
@@ -4142,14 +4098,27 @@ sctp_try_advance_peer_ack_point(struct sctp_tcb *stcb,
 	return (a_adv);
 }
 
-static void
+static int
 sctp_fs_audit(struct sctp_association *asoc)
 {
 	struct sctp_tmit_chunk *chk;
 	int inflight = 0, resend = 0, inbetween = 0, acked = 0, above = 0;
+	int entry_flight, entry_cnt, ret;
+
+	entry_flight = asoc->total_flight;
+	entry_cnt = asoc->total_flight_count;
+	ret = 0;
+
+	if (asoc->pr_sctp_cnt >= asoc->sent_queue_cnt)
+		return (0);
 
 	TAILQ_FOREACH(chk, &asoc->sent_queue, sctp_next) {
 		if (chk->sent < SCTP_DATAGRAM_RESEND) {
+			printf("Chk TSN:%u size:%d inflight cnt:%d\n",
+			    chk->rec.data.TSN_seq,
+			    chk->send_size,
+			    chk->snd_count
+			    );
 			inflight++;
 		} else if (chk->sent == SCTP_DATAGRAM_RESEND) {
 			resend++;
@@ -4166,10 +4135,15 @@ sctp_fs_audit(struct sctp_association *asoc)
 #ifdef INVARIANTS
 		panic("Flight size-express incorrect? \n");
 #else
-		SCTP_PRINTF("Flight size-express incorrect inflight:%d inbetween:%d\n",
-		    inflight, inbetween);
+		printf("asoc->total_flight:%d cnt:%d\n",
+		    entry_flight, entry_cnt);
+
+		SCTP_PRINTF("Flight size-express incorrect F:%d I:%d R:%d Ab:%d ACK:%d\n",
+		    inflight, inbetween, resend, above, acked);
+		ret = 1;
 #endif
 	}
+	return (ret);
 }
 
 
@@ -4590,20 +4564,26 @@ again:
 	    (asoc->sent_queue_retran_cnt == 0) &&
 	    (win_probe_recovered == 0) &&
 	    (done_once == 0)) {
-		/* huh, this should not happen */
-		sctp_fs_audit(asoc);
-		TAILQ_FOREACH(net, &asoc->nets, sctp_next) {
-			net->flight_size = 0;
-		}
-		asoc->total_flight = 0;
-		asoc->total_flight_count = 0;
-		asoc->sent_queue_retran_cnt = 0;
-		TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) {
-			if (tp1->sent < SCTP_DATAGRAM_RESEND) {
-				sctp_flight_size_increase(tp1);
-				sctp_total_flight_increase(stcb, tp1);
-			} else if (tp1->sent == SCTP_DATAGRAM_RESEND) {
-				asoc->sent_queue_retran_cnt++;
+		/*
+		 * huh, this should not happen unless all packets are
+		 * PR-SCTP and marked to skip of course.
+		 */
+		if (sctp_fs_audit(asoc)) {
+			TAILQ_FOREACH(net, &asoc->nets, sctp_next) {
+				if (net->flight_size) {
+					net->flight_size = 0;
+				}
+			}
+			asoc->total_flight = 0;
+			asoc->total_flight_count = 0;
+			asoc->sent_queue_retran_cnt = 0;
+			TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) {
+				if (tp1->sent < SCTP_DATAGRAM_RESEND) {
+					sctp_flight_size_increase(tp1);
+					sctp_total_flight_increase(stcb, tp1);
+				} else if (tp1->sent == SCTP_DATAGRAM_RESEND) {
+					asoc->sent_queue_retran_cnt++;
+				}
 			}
 		}
 		done_once = 1;
@@ -4728,6 +4708,13 @@ again:
 				 */
 				asoc->nonce_sum_check = 0;
 				asoc->nonce_resync_tsn = asoc->advanced_peer_ack_point;
+			} else if (lchk) {
+				/* try to FR fwd-tsn's that get lost too */
+				lchk->rec.data.fwd_tsn_cnt++;
+				if (lchk->rec.data.fwd_tsn_cnt > 3) {
+					send_forward_tsn(stcb, asoc);
+					lchk->rec.data.fwd_tsn_cnt = 0;
+				}
 			}
 		}
 		if (lchk) {
@@ -4813,10 +4800,6 @@ sctp_handle_sack(struct mbuf *m, int offset,
 	num_seg = ntohs(sack->num_gap_ack_blks);
 	a_rwnd = rwnd;
 
-	if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_LOG_SACK_ARRIVALS_ENABLE) {
-		sctp_misc_ints(SCTP_SACK_LOG_NORMAL, cum_ack,
-		    rwnd, stcb->asoc.last_acked_seq, stcb->asoc.peers_rwnd);
-	}
 	/* CMT DAC algo */
 	cmt_dac_flag = ch->ch.chunk_flags & SCTP_SACK_CMT_DAC;
 	num_dup = ntohs(sack->num_dup_tsns);
@@ -5605,20 +5588,24 @@ again:
 	    (asoc->sent_queue_retran_cnt == 0) &&
 	    (win_probe_recovered == 0) &&
 	    (done_once == 0)) {
-		/* huh, this should not happen */
-		sctp_fs_audit(asoc);
-		TAILQ_FOREACH(net, &asoc->nets, sctp_next) {
-			net->flight_size = 0;
-		}
-		asoc->total_flight = 0;
-		asoc->total_flight_count = 0;
-		asoc->sent_queue_retran_cnt = 0;
-		TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) {
-			if (tp1->sent < SCTP_DATAGRAM_RESEND) {
-				sctp_flight_size_increase(tp1);
-				sctp_total_flight_increase(stcb, tp1);
-			} else if (tp1->sent == SCTP_DATAGRAM_RESEND) {
-				asoc->sent_queue_retran_cnt++;
+		/*
+		 * huh, this should not happen unless all packets are
+		 * PR-SCTP and marked to skip of course.
+		 */
+		if (sctp_fs_audit(asoc)) {
+			TAILQ_FOREACH(net, &asoc->nets, sctp_next) {
+				net->flight_size = 0;
+			}
+			asoc->total_flight = 0;
+			asoc->total_flight_count = 0;
+			asoc->sent_queue_retran_cnt = 0;
+			TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) {
+				if (tp1->sent < SCTP_DATAGRAM_RESEND) {
+					sctp_flight_size_increase(tp1);
+					sctp_total_flight_increase(stcb, tp1);
+				} else if (tp1->sent == SCTP_DATAGRAM_RESEND) {
+					asoc->sent_queue_retran_cnt++;
+				}
 			}
 		}
 		done_once = 1;
@@ -5643,6 +5630,11 @@ again:
 			 * on issues that will occur when the ECN NONCE
 			 * stuff is put into SCTP for cross checking.
 			 */
+			if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_LOG_TRY_ADVANCE) {
+				sctp_misc_ints(SCTP_FWD_TSN_CHECK,
+				    0xee, cum_ack, asoc->advanced_peer_ack_point,
+				    old_adv_peer_ack_point);
+			}
 			if (compare_with_wrap(asoc->advanced_peer_ack_point, old_adv_peer_ack_point,
 			    MAX_TSN)) {
 				send_forward_tsn(stcb, asoc);
@@ -5652,6 +5644,13 @@ again:
 				 */
 				asoc->nonce_sum_check = 0;
 				asoc->nonce_resync_tsn = asoc->advanced_peer_ack_point;
+			} else if (lchk) {
+				/* try to FR fwd-tsn's that get lost too */
+				lchk->rec.data.fwd_tsn_cnt++;
+				if (lchk->rec.data.fwd_tsn_cnt > 3) {
+					send_forward_tsn(stcb, asoc);
+					lchk->rec.data.fwd_tsn_cnt = 0;
+				}
 			}
 		}
 		if (lchk) {
@@ -5740,7 +5739,9 @@ sctp_kick_prsctp_reorder_queue(struct sctp_tcb *stcb,
 					} else {
 						SCTP_TCB_LOCK_ASSERT(stcb);
 						SCTP_SET_TSN_PRESENT(asoc->nr_mapping_array, nr_gap);
-						if (nr_tsn > asoc->highest_tsn_inside_nr_map)
+						if (compare_with_wrap(nr_tsn,
+						    asoc->highest_tsn_inside_nr_map,
+						    MAX_TSN))
 							asoc->highest_tsn_inside_nr_map = nr_tsn;
 					}
 
@@ -5843,7 +5844,8 @@ sctp_kick_prsctp_reorder_queue(struct sctp_tcb *stcb,
 					} else {
 						SCTP_TCB_LOCK_ASSERT(stcb);
 						SCTP_SET_TSN_PRESENT(asoc->nr_mapping_array, nr_gap);
-						if (nr_tsn > asoc->highest_tsn_inside_nr_map)
+						if (compare_with_wrap(nr_tsn, asoc->highest_tsn_inside_nr_map,
+						    MAX_TSN))
 							asoc->highest_tsn_inside_nr_map = nr_tsn;
 					}
 
@@ -5905,6 +5907,91 @@ sctp_kick_prsctp_reorder_queue(struct sctp_tcb *stcb,
 	}
 }
 
+static void
+sctp_flush_reassm_for_str_seq(struct sctp_tcb *stcb,
+    struct sctp_association *asoc,
+    uint16_t stream, uint16_t seq)
+{
+	struct sctp_tmit_chunk *chk, *at;
+
+	if (!TAILQ_EMPTY(&asoc->reasmqueue)) {
+		/* For each one on here see if we need to toss it */
+		/*
+		 * For now large messages held on the reasmqueue that are
+		 * complete will be tossed too. We could in theory do more
+		 * work to spin through and stop after dumping one msg aka
+		 * seeing the start of a new msg at the head, and call the
+		 * delivery function... to see if it can be delivered... But
+		 * for now we just dump everything on the queue.
+		 */
+		chk = TAILQ_FIRST(&asoc->reasmqueue);
+		while (chk) {
+			at = TAILQ_NEXT(chk, sctp_next);
+			if (chk->rec.data.stream_number != stream) {
+				chk = at;
+				continue;
+			}
+			if (chk->rec.data.stream_seq == seq) {
+				/* It needs to be tossed */
+				TAILQ_REMOVE(&asoc->reasmqueue, chk, sctp_next);
+				if (compare_with_wrap(chk->rec.data.TSN_seq,
+				    asoc->tsn_last_delivered, MAX_TSN)) {
+					asoc->tsn_last_delivered =
+					    chk->rec.data.TSN_seq;
+					asoc->str_of_pdapi =
+					    chk->rec.data.stream_number;
+					asoc->ssn_of_pdapi =
+					    chk->rec.data.stream_seq;
+					asoc->fragment_flags =
+					    chk->rec.data.rcv_flags;
+				}
+				asoc->size_on_reasm_queue -= chk->send_size;
+				sctp_ucount_decr(asoc->cnt_on_reasm_queue);
+
+				/* Clear up any stream problem */
+				if ((chk->rec.data.rcv_flags & SCTP_DATA_UNORDERED) !=
+				    SCTP_DATA_UNORDERED &&
+				    (compare_with_wrap(chk->rec.data.stream_seq,
+				    asoc->strmin[chk->rec.data.stream_number].last_sequence_delivered,
+				    MAX_SEQ))) {
+					/*
+					 * We must dump forward this streams
+					 * sequence number if the chunk is
+					 * not unordered that is being
+					 * skipped. There is a chance that
+					 * if the peer does not include the
+					 * last fragment in its FWD-TSN we
+					 * WILL have a problem here since
+					 * you would have a partial chunk in
+					 * queue that may not be
+					 * deliverable. Also if a Partial
+					 * delivery API as started the user
+					 * may get a partial chunk. The next
+					 * read returning a new chunk...
+					 * really ugly but I see no way
+					 * around it! Maybe a notify??
+					 */
+					asoc->strmin[chk->rec.data.stream_number].last_sequence_delivered =
+					    chk->rec.data.stream_seq;
+				}
+				if (chk->data) {
+					sctp_m_freem(chk->data);
+					chk->data = NULL;
+				}
+				sctp_free_a_chunk(stcb, chk);
+			} else if (compare_with_wrap(chk->rec.data.stream_seq, seq, MAX_SEQ)) {
+				/*
+				 * If the stream_seq is > than the purging
+				 * one, we are done
+				 */
+				break;
+			}
+			chk = at;
+		}
+	}
+}
+
+
 void
 sctp_handle_forward_tsn(struct sctp_tcb *stcb,
     struct sctp_forward_tsn_chunk *fwd, int *abort_flag, struct mbuf *m, int offset)
@@ -5934,13 +6021,14 @@ sctp_handle_forward_tsn(struct sctp_tcb *stcb,
 	 */
 	struct sctp_association *asoc;
 	uint32_t new_cum_tsn, gap;
-	unsigned int i, cnt_gone, fwd_sz, cumack_set_flag, m_size;
+	unsigned int i, fwd_sz, cumack_set_flag, m_size;
+	uint32_t str_seq;
 	struct sctp_stream_in *strm;
 	struct sctp_tmit_chunk *chk, *at;
+	struct sctp_queued_to_read *ctl, *sv;
 
 	cumack_set_flag = 0;
 	asoc = &stcb->asoc;
-	cnt_gone = 0;
 	if ((fwd_sz = ntohs(fwd->ch.chunk_length)) < sizeof(struct sctp_forward_tsn_chunk)) {
 		SCTPDBG(SCTP_DEBUG_INDATA1,
 		    "Bad size too small/big fwd-tsn\n");
@@ -6019,7 +6107,6 @@ sctp_handle_forward_tsn(struct sctp_tcb *stcb,
 			return;
 		}
 		SCTP_STAT_INCR(sctps_fwdtsn_map_over);
-slide_out:
 		memset(stcb->asoc.mapping_array, 0, stcb->asoc.mapping_array_size);
 		cumack_set_flag = 1;
 		asoc->mapping_array_base_tsn = new_cum_tsn + 1;
@@ -6043,12 +6130,15 @@ slide_out:
 		asoc->last_echo_tsn = asoc->highest_tsn_inside_map;
 	} else {
 		SCTP_TCB_LOCK_ASSERT(stcb);
-		if ((compare_with_wrap(((uint32_t) asoc->cumulative_tsn + gap), asoc->highest_tsn_inside_map, MAX_TSN)) ||
-		    (((uint32_t) asoc->cumulative_tsn + gap) == asoc->highest_tsn_inside_map)) {
-			goto slide_out;
-		} else {
-			for (i = 0; i <= gap; i++) {
-				SCTP_SET_TSN_PRESENT(asoc->mapping_array, i);
+		for (i = 0; i <= gap; i++) {
+			SCTP_SET_TSN_PRESENT(asoc->mapping_array, i);
+			/*
+			 * EY if drain is off then every gap-ack is an
+			 * nr-gap-ack
+			 */
+			if (SCTP_BASE_SYSCTL(sctp_nr_sack_on_off) && asoc->peer_supports_nr_sack
+			    && SCTP_BASE_SYSCTL(sctp_do_drain) == 0) {
+				SCTP_SET_TSN_PRESENT(asoc->nr_mapping_array, i);
 			}
 		}
 		/*
@@ -6059,7 +6149,6 @@ slide_out:
 		if (*abort_flag)
 			return;
 	}
-
 	/*************************************************************/
 	/* 2. Clear up re-assembly queue                             */
 	/*************************************************************/
@@ -6083,9 +6172,9 @@ slide_out:
 		chk = TAILQ_FIRST(&asoc->reasmqueue);
 		while (chk) {
 			at = TAILQ_NEXT(chk, sctp_next);
-			if (compare_with_wrap(asoc->cumulative_tsn,
-			    chk->rec.data.TSN_seq, MAX_TSN) ||
-			    asoc->cumulative_tsn == chk->rec.data.TSN_seq) {
+			if ((compare_with_wrap(new_cum_tsn,
+			    chk->rec.data.TSN_seq, MAX_TSN)) ||
+			    (new_cum_tsn == chk->rec.data.TSN_seq)) {
 				/* It needs to be tossed */
 				TAILQ_REMOVE(&asoc->reasmqueue, chk, sctp_next);
 				if (compare_with_wrap(chk->rec.data.TSN_seq,
@@ -6101,7 +6190,6 @@ slide_out:
 				}
 				asoc->size_on_reasm_queue -= chk->send_size;
 				sctp_ucount_decr(asoc->cnt_on_reasm_queue);
-				cnt_gone++;
 
 				/* Clear up any stream problem */
 				if ((chk->rec.data.rcv_flags & SCTP_DATA_UNORDERED) !=
@@ -6137,45 +6225,17 @@ slide_out:
 			} else {
 				/*
 				 * Ok we have gone beyond the end of the
-				 * fwd-tsn's mark. Some checks...
+				 * fwd-tsn's mark.
 				 */
-				if ((asoc->fragmented_delivery_inprogress) &&
-				    (chk->rec.data.rcv_flags & SCTP_DATA_FIRST_FRAG)) {
-					uint32_t str_seq;
-
-					/*
-					 * Special case PD-API is up and
-					 * what we fwd-tsn' over includes
-					 * one that had the LAST_FRAG. We no
-					 * longer need to do the PD-API.
-					 */
-					asoc->fragmented_delivery_inprogress = 0;
-
-					str_seq = (asoc->str_of_pdapi << 16) | asoc->ssn_of_pdapi;
-					sctp_ulp_notify(SCTP_NOTIFY_PARTIAL_DELVIERY_INDICATION,
-					    stcb, SCTP_PARTIAL_DELIVERY_ABORTED, (void *)&str_seq, SCTP_SO_NOT_LOCKED);
-
-				}
 				break;
 			}
 			chk = at;
 		}
 	}
-	if (asoc->fragmented_delivery_inprogress) {
-		/*
-		 * Ok we removed cnt_gone chunks in the PD-API queue that
-		 * were being delivered. So now we must turn off the flag.
-		 */
-		uint32_t str_seq;
-
-		str_seq = (asoc->str_of_pdapi << 16) | asoc->ssn_of_pdapi;
-		sctp_ulp_notify(SCTP_NOTIFY_PARTIAL_DELVIERY_INDICATION,
-		    stcb, SCTP_PARTIAL_DELIVERY_ABORTED, (void *)&str_seq, SCTP_SO_NOT_LOCKED);
-		asoc->fragmented_delivery_inprogress = 0;
-	}
-	/*************************************************************/
-	/* 3. Update the PR-stream re-ordering queues                */
-	/*************************************************************/
+	/*******************************************************/
+	/* 3. Update the PR-stream re-ordering queues and fix  */
+	/* delivery issues as needed.                       */
+	/*******************************************************/
 	fwd_sz -= sizeof(*fwd);
 	if (m && fwd_sz) {
 		/* New method. */
@@ -6184,6 +6244,7 @@ slide_out:
 
 		offset += sizeof(*fwd);
 
+		SCTP_INP_READ_LOCK(stcb->sctp_ep);
 		num_str = fwd_sz / sizeof(struct sctp_strseq);
 		for (i = 0; i < num_str; i++) {
 			uint16_t st;
@@ -6200,11 +6261,49 @@ slide_out:
 			stseq->stream = st;
 			st = ntohs(stseq->sequence);
 			stseq->sequence = st;
+
 			/* now process */
+
+			/*
+			 * Ok we now look for the stream/seq on the read
+			 * queue where its not all delivered. If we find it
+			 * we transmute the read entry into a PDI_ABORTED.
+			 */
 			if (stseq->stream >= asoc->streamincnt) {
 				/* screwed up streams, stop!  */
 				break;
 			}
+			if ((asoc->str_of_pdapi == stseq->stream) &&
+			    (asoc->ssn_of_pdapi == stseq->sequence)) {
+				/*
+				 * If this is the one we were partially
+				 * delivering now then we no longer are.
+				 * Note this will change with the reassembly
+				 * re-write.
+				 */
+				asoc->fragmented_delivery_inprogress = 0;
+			}
+			sctp_flush_reassm_for_str_seq(stcb, asoc, stseq->stream, stseq->sequence);
+			TAILQ_FOREACH(ctl, &stcb->sctp_ep->read_queue, next) {
+				if ((ctl->sinfo_stream == stseq->stream) &&
+				    (ctl->sinfo_ssn == stseq->sequence)) {
+					str_seq = (stseq->stream << 16) | stseq->sequence;
+					ctl->end_added = 1;
+					ctl->pdapi_aborted = 1;
+					sv = stcb->asoc.control_pdapi;
+					stcb->asoc.control_pdapi = ctl;
+					sctp_notify_partial_delivery_indication(stcb,
+					    SCTP_PARTIAL_DELIVERY_ABORTED,
+					    SCTP_HOLDS_LOCK,
+					    str_seq);
+					stcb->asoc.control_pdapi = sv;
+					break;
+				} else if ((ctl->sinfo_stream == stseq->stream) &&
+				    (compare_with_wrap(ctl->sinfo_ssn, stseq->sequence, MAX_SEQ))) {
+					/* We are past our victim SSN */
+					break;
+				}
+			}
 			strm = &asoc->strmin[stseq->stream];
 			if (compare_with_wrap(stseq->sequence,
 			    strm->last_sequence_delivered, MAX_SEQ)) {
@@ -6216,6 +6315,7 @@ slide_out:
 			/* sa_ignore NO_NULL_CHK */
 			sctp_kick_prsctp_reorder_queue(stcb, strm);
 		}
+		SCTP_INP_READ_UNLOCK(stcb->sctp_ep);
 	}
 	if (TAILQ_FIRST(&asoc->reasmqueue)) {
 		/* now lets kick out and check for more fragmented delivery */
@@ -6614,20 +6714,24 @@ again:
 	    (asoc->sent_queue_retran_cnt == 0) &&
 	    (win_probe_recovered == 0) &&
 	    (done_once == 0)) {
-		/* huh, this should not happen */
-		sctp_fs_audit(asoc);
-		TAILQ_FOREACH(net, &asoc->nets, sctp_next) {
-			net->flight_size = 0;
-		}
-		asoc->total_flight = 0;
-		asoc->total_flight_count = 0;
-		asoc->sent_queue_retran_cnt = 0;
-		TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) {
-			if (tp1->sent < SCTP_DATAGRAM_RESEND) {
-				sctp_flight_size_increase(tp1);
-				sctp_total_flight_increase(stcb, tp1);
-			} else if (tp1->sent == SCTP_DATAGRAM_RESEND) {
-				asoc->sent_queue_retran_cnt++;
+		/*
+		 * huh, this should not happen unless all packets are
+		 * PR-SCTP and marked to skip of course.
+		 */
+		if (sctp_fs_audit(asoc)) {
+			TAILQ_FOREACH(net, &asoc->nets, sctp_next) {
+				net->flight_size = 0;
+			}
+			asoc->total_flight = 0;
+			asoc->total_flight_count = 0;
+			asoc->sent_queue_retran_cnt = 0;
+			TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) {
+				if (tp1->sent < SCTP_DATAGRAM_RESEND) {
+					sctp_flight_size_increase(tp1);
+					sctp_total_flight_increase(stcb, tp1);
+				} else if (tp1->sent == SCTP_DATAGRAM_RESEND) {
+					asoc->sent_queue_retran_cnt++;
+				}
 			}
 		}
 		done_once = 1;
@@ -7012,7 +7116,8 @@ sctp_handle_nr_sack_segments(struct mbuf *m, int *offset, struct sctp_tcb *stcb,
 						 * fall through here and are
 						 * marked
 						 */
-						tp1->sent = SCTP_DATAGRAM_MARKED;
+						if (tp1->sent != SCTP_FORWARD_TSN_SKIP)
+							tp1->sent = SCTP_DATAGRAM_NR_MARKED;
 						if (tp1->rec.data.chunk_was_revoked) {
 							/* deflate the cwnd */
 							tp1->whoTo->cwnd -= tp1->book_size;
@@ -7024,7 +7129,8 @@ sctp_handle_nr_sack_segments(struct mbuf *m, int *offset, struct sctp_tcb *stcb,
 						 * nr_marked
 						 */
 						if (all_bit) {
-							tp1->sent = SCTP_DATAGRAM_NR_MARKED;
+							if (tp1->sent != SCTP_FORWARD_TSN_SKIP)
+								tp1->sent = SCTP_DATAGRAM_NR_MARKED;
 							/*
 							 * TAILQ_REMOVE(&asoc
 							 * ->sent_queue,
@@ -7143,7 +7249,8 @@ sctp_handle_nr_sack_segments(struct mbuf *m, int *offset, struct sctp_tcb *stcb,
 				while (tp1) {
 					if (tp1->rec.data.TSN_seq == j) {
 						if (tp1->sent != SCTP_DATAGRAM_UNSENT) {
-							tp1->sent = SCTP_DATAGRAM_NR_MARKED;
+							if (tp1->sent != SCTP_FORWARD_TSN_SKIP)
+								tp1->sent = SCTP_DATAGRAM_NR_MARKED;
 							/*
 							 * TAILQ_REMOVE(&asoc
 							 * ->sent_queue,
@@ -8170,20 +8277,24 @@ again:
 	    (asoc->sent_queue_retran_cnt == 0) &&
 	    (win_probe_recovered == 0) &&
 	    (done_once == 0)) {
-		/* huh, this should not happen */
-		sctp_fs_audit(asoc);
-		TAILQ_FOREACH(net, &asoc->nets, sctp_next) {
-			net->flight_size = 0;
-		}
-		asoc->total_flight = 0;
-		asoc->total_flight_count = 0;
-		asoc->sent_queue_retran_cnt = 0;
-		TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) {
-			if (tp1->sent < SCTP_DATAGRAM_RESEND) {
-				sctp_flight_size_increase(tp1);
-				sctp_total_flight_increase(stcb, tp1);
-			} else if (tp1->sent == SCTP_DATAGRAM_RESEND) {
-				asoc->sent_queue_retran_cnt++;
+		/*
+		 * huh, this should not happen unless all packets are
+		 * PR-SCTP and marked to skip of course.
+		 */
+		if (sctp_fs_audit(asoc)) {
+			TAILQ_FOREACH(net, &asoc->nets, sctp_next) {
+				net->flight_size = 0;
+			}
+			asoc->total_flight = 0;
+			asoc->total_flight_count = 0;
+			asoc->sent_queue_retran_cnt = 0;
+			TAILQ_FOREACH(tp1, &asoc->sent_queue, sctp_next) {
+				if (tp1->sent < SCTP_DATAGRAM_RESEND) {
+					sctp_flight_size_increase(tp1);
+					sctp_total_flight_increase(stcb, tp1);
+				} else if (tp1->sent == SCTP_DATAGRAM_RESEND) {
+					asoc->sent_queue_retran_cnt++;
+				}
 			}
 		}
 		done_once = 1;
@@ -8221,6 +8332,13 @@ again:
 				 */
 				asoc->nonce_sum_check = 0;
 				asoc->nonce_resync_tsn = asoc->advanced_peer_ack_point;
+			} else if (lchk) {
+				/* try to FR fwd-tsn's that get lost too */
+				lchk->rec.data.fwd_tsn_cnt++;
+				if (lchk->rec.data.fwd_tsn_cnt > 3) {
+					send_forward_tsn(stcb, asoc);
+					lchk->rec.data.fwd_tsn_cnt = 0;
+				}
 			}
 		}
 		if (lchk) {
diff --git a/sys/netinet/sctp_input.c b/sys/netinet/sctp_input.c
index 494929b..c962e4d 100644
--- a/sys/netinet/sctp_input.c
+++ b/sys/netinet/sctp_input.c
@@ -3150,8 +3150,10 @@ process_chunk_drop(struct sctp_tcb *stcb, struct sctp_chunk_desc *desc,
 					    (uintptr_t) stcb,
 					    tp1->rec.data.TSN_seq);
 				}
-				sctp_flight_size_decrease(tp1);
-				sctp_total_flight_decrease(stcb, tp1);
+				if (tp1->sent < SCTP_DATAGRAM_RESEND) {
+					sctp_flight_size_decrease(tp1);
+					sctp_total_flight_decrease(stcb, tp1);
+				}
 			} {
 				/* audit code */
 				unsigned int audit;
@@ -5606,11 +5608,14 @@ sctp_common_input_processing(struct mbuf **mm, int iphlen, int offset,
 			/* there was a gap before this data was processed */
 			was_a_gap = 1;
 		}
+		stcb->asoc.send_sack = 1;
 		sctp_sack_check(stcb, 1, was_a_gap, &abort_flag);
 		if (abort_flag) {
 			/* Again, we aborted so NO UNLOCK needed */
 			goto out_now;
 		}
+	} else if (fwd_tsn_seen) {
+		stcb->asoc.send_sack = 1;
 	}
 	/* trigger send of any chunks in queue... */
 trigger_send:
diff --git a/sys/netinet/sctp_output.c b/sys/netinet/sctp_output.c
index 3729616..5885474 100644
--- a/sys/netinet/sctp_output.c
+++ b/sys/netinet/sctp_output.c
@@ -1859,1812 +1859,6 @@ struct sack_track sack_array[256] = {
 	}
 };
 
-/* EY  below are nr_sacks version of the preceeding two data structures, identical except their names */
-#define SCTP_MAX_NR_GAPS_INARRAY 4
-struct nr_sack_track {
-	uint8_t right_edge;	/* mergable on the right edge */
-	uint8_t left_edge;	/* mergable on the left edge */
-	uint8_t num_entries;
-	uint8_t spare;
-	struct sctp_nr_gap_ack_block nr_gaps[SCTP_MAX_NR_GAPS_INARRAY];
-};
-
-struct nr_sack_track nr_sack_array[256] = {
-	{0, 0, 0, 0,		/* 0x00 */
-		{{0, 0},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 1, 0,		/* 0x01 */
-		{{0, 0},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x02 */
-		{{1, 1},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 1, 0,		/* 0x03 */
-		{{0, 1},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x04 */
-		{{2, 2},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x05 */
-		{{0, 0},
-		{2, 2},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x06 */
-		{{1, 2},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 1, 0,		/* 0x07 */
-		{{0, 2},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x08 */
-		{{3, 3},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x09 */
-		{{0, 0},
-		{3, 3},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x0a */
-		{{1, 1},
-		{3, 3},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x0b */
-		{{0, 1},
-		{3, 3},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x0c */
-		{{2, 3},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x0d */
-		{{0, 0},
-		{2, 3},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x0e */
-		{{1, 3},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 1, 0,		/* 0x0f */
-		{{0, 3},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x10 */
-		{{4, 4},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x11 */
-		{{0, 0},
-		{4, 4},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x12 */
-		{{1, 1},
-		{4, 4},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x13 */
-		{{0, 1},
-		{4, 4},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x14 */
-		{{2, 2},
-		{4, 4},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x15 */
-		{{0, 0},
-		{2, 2},
-		{4, 4},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x16 */
-		{{1, 2},
-		{4, 4},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x17 */
-		{{0, 2},
-		{4, 4},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x18 */
-		{{3, 4},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x19 */
-		{{0, 0},
-		{3, 4},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x1a */
-		{{1, 1},
-		{3, 4},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x1b */
-		{{0, 1},
-		{3, 4},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x1c */
-		{{2, 4},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x1d */
-		{{0, 0},
-		{2, 4},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x1e */
-		{{1, 4},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 1, 0,		/* 0x1f */
-		{{0, 4},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x20 */
-		{{5, 5},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x21 */
-		{{0, 0},
-		{5, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x22 */
-		{{1, 1},
-		{5, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x23 */
-		{{0, 1},
-		{5, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x24 */
-		{{2, 2},
-		{5, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x25 */
-		{{0, 0},
-		{2, 2},
-		{5, 5},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x26 */
-		{{1, 2},
-		{5, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x27 */
-		{{0, 2},
-		{5, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x28 */
-		{{3, 3},
-		{5, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x29 */
-		{{0, 0},
-		{3, 3},
-		{5, 5},
-		{0, 0}
-		}
-	},
-	{0, 0, 3, 0,		/* 0x2a */
-		{{1, 1},
-		{3, 3},
-		{5, 5},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x2b */
-		{{0, 1},
-		{3, 3},
-		{5, 5},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x2c */
-		{{2, 3},
-		{5, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x2d */
-		{{0, 0},
-		{2, 3},
-		{5, 5},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x2e */
-		{{1, 3},
-		{5, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x2f */
-		{{0, 3},
-		{5, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x30 */
-		{{4, 5},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x31 */
-		{{0, 0},
-		{4, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x32 */
-		{{1, 1},
-		{4, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x33 */
-		{{0, 1},
-		{4, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x34 */
-		{{2, 2},
-		{4, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x35 */
-		{{0, 0},
-		{2, 2},
-		{4, 5},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x36 */
-		{{1, 2},
-		{4, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x37 */
-		{{0, 2},
-		{4, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x38 */
-		{{3, 5},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x39 */
-		{{0, 0},
-		{3, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x3a */
-		{{1, 1},
-		{3, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x3b */
-		{{0, 1},
-		{3, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x3c */
-		{{2, 5},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x3d */
-		{{0, 0},
-		{2, 5},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x3e */
-		{{1, 5},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 1, 0,		/* 0x3f */
-		{{0, 5},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x40 */
-		{{6, 6},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x41 */
-		{{0, 0},
-		{6, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x42 */
-		{{1, 1},
-		{6, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x43 */
-		{{0, 1},
-		{6, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x44 */
-		{{2, 2},
-		{6, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x45 */
-		{{0, 0},
-		{2, 2},
-		{6, 6},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x46 */
-		{{1, 2},
-		{6, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x47 */
-		{{0, 2},
-		{6, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x48 */
-		{{3, 3},
-		{6, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x49 */
-		{{0, 0},
-		{3, 3},
-		{6, 6},
-		{0, 0}
-		}
-	},
-	{0, 0, 3, 0,		/* 0x4a */
-		{{1, 1},
-		{3, 3},
-		{6, 6},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x4b */
-		{{0, 1},
-		{3, 3},
-		{6, 6},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x4c */
-		{{2, 3},
-		{6, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x4d */
-		{{0, 0},
-		{2, 3},
-		{6, 6},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x4e */
-		{{1, 3},
-		{6, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x4f */
-		{{0, 3},
-		{6, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x50 */
-		{{4, 4},
-		{6, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x51 */
-		{{0, 0},
-		{4, 4},
-		{6, 6},
-		{0, 0}
-		}
-	},
-	{0, 0, 3, 0,		/* 0x52 */
-		{{1, 1},
-		{4, 4},
-		{6, 6},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x53 */
-		{{0, 1},
-		{4, 4},
-		{6, 6},
-		{0, 0}
-		}
-	},
-	{0, 0, 3, 0,		/* 0x54 */
-		{{2, 2},
-		{4, 4},
-		{6, 6},
-		{0, 0}
-		}
-	},
-	{1, 0, 4, 0,		/* 0x55 */
-		{{0, 0},
-		{2, 2},
-		{4, 4},
-		{6, 6}
-		}
-	},
-	{0, 0, 3, 0,		/* 0x56 */
-		{{1, 2},
-		{4, 4},
-		{6, 6},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x57 */
-		{{0, 2},
-		{4, 4},
-		{6, 6},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x58 */
-		{{3, 4},
-		{6, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x59 */
-		{{0, 0},
-		{3, 4},
-		{6, 6},
-		{0, 0}
-		}
-	},
-	{0, 0, 3, 0,		/* 0x5a */
-		{{1, 1},
-		{3, 4},
-		{6, 6},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x5b */
-		{{0, 1},
-		{3, 4},
-		{6, 6},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x5c */
-		{{2, 4},
-		{6, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x5d */
-		{{0, 0},
-		{2, 4},
-		{6, 6},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x5e */
-		{{1, 4},
-		{6, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x5f */
-		{{0, 4},
-		{6, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x60 */
-		{{5, 6},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x61 */
-		{{0, 0},
-		{5, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x62 */
-		{{1, 1},
-		{5, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x63 */
-		{{0, 1},
-		{5, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x64 */
-		{{2, 2},
-		{5, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x65 */
-		{{0, 0},
-		{2, 2},
-		{5, 6},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x66 */
-		{{1, 2},
-		{5, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x67 */
-		{{0, 2},
-		{5, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x68 */
-		{{3, 3},
-		{5, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x69 */
-		{{0, 0},
-		{3, 3},
-		{5, 6},
-		{0, 0}
-		}
-	},
-	{0, 0, 3, 0,		/* 0x6a */
-		{{1, 1},
-		{3, 3},
-		{5, 6},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x6b */
-		{{0, 1},
-		{3, 3},
-		{5, 6},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x6c */
-		{{2, 3},
-		{5, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x6d */
-		{{0, 0},
-		{2, 3},
-		{5, 6},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x6e */
-		{{1, 3},
-		{5, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x6f */
-		{{0, 3},
-		{5, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x70 */
-		{{4, 6},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x71 */
-		{{0, 0},
-		{4, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x72 */
-		{{1, 1},
-		{4, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x73 */
-		{{0, 1},
-		{4, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x74 */
-		{{2, 2},
-		{4, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 3, 0,		/* 0x75 */
-		{{0, 0},
-		{2, 2},
-		{4, 6},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x76 */
-		{{1, 2},
-		{4, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x77 */
-		{{0, 2},
-		{4, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x78 */
-		{{3, 6},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x79 */
-		{{0, 0},
-		{3, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 2, 0,		/* 0x7a */
-		{{1, 1},
-		{3, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x7b */
-		{{0, 1},
-		{3, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x7c */
-		{{2, 6},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 2, 0,		/* 0x7d */
-		{{0, 0},
-		{2, 6},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 0, 1, 0,		/* 0x7e */
-		{{1, 6},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 0, 1, 0,		/* 0x7f */
-		{{0, 6},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 1, 0,		/* 0x80 */
-		{{7, 7},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0x81 */
-		{{0, 0},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0x82 */
-		{{1, 1},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0x83 */
-		{{0, 1},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0x84 */
-		{{2, 2},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0x85 */
-		{{0, 0},
-		{2, 2},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0x86 */
-		{{1, 2},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0x87 */
-		{{0, 2},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0x88 */
-		{{3, 3},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0x89 */
-		{{0, 0},
-		{3, 3},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 3, 0,		/* 0x8a */
-		{{1, 1},
-		{3, 3},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0x8b */
-		{{0, 1},
-		{3, 3},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0x8c */
-		{{2, 3},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0x8d */
-		{{0, 0},
-		{2, 3},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0x8e */
-		{{1, 3},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0x8f */
-		{{0, 3},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0x90 */
-		{{4, 4},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0x91 */
-		{{0, 0},
-		{4, 4},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 3, 0,		/* 0x92 */
-		{{1, 1},
-		{4, 4},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0x93 */
-		{{0, 1},
-		{4, 4},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 3, 0,		/* 0x94 */
-		{{2, 2},
-		{4, 4},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 4, 0,		/* 0x95 */
-		{{0, 0},
-		{2, 2},
-		{4, 4},
-		{7, 7}
-		}
-	},
-	{0, 1, 3, 0,		/* 0x96 */
-		{{1, 2},
-		{4, 4},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0x97 */
-		{{0, 2},
-		{4, 4},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0x98 */
-		{{3, 4},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0x99 */
-		{{0, 0},
-		{3, 4},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 3, 0,		/* 0x9a */
-		{{1, 1},
-		{3, 4},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0x9b */
-		{{0, 1},
-		{3, 4},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0x9c */
-		{{2, 4},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0x9d */
-		{{0, 0},
-		{2, 4},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0x9e */
-		{{1, 4},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0x9f */
-		{{0, 4},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xa0 */
-		{{5, 5},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xa1 */
-		{{0, 0},
-		{5, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xa2 */
-		{{1, 1},
-		{5, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xa3 */
-		{{0, 1},
-		{5, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xa4 */
-		{{2, 2},
-		{5, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 4, 0,		/* 0xa5 */
-		{{0, 0},
-		{2, 2},
-		{5, 5},
-		{7, 7}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xa6 */
-		{{1, 2},
-		{5, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xa7 */
-		{{0, 2},
-		{5, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xa8 */
-		{{3, 3},
-		{5, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 4, 0,		/* 0xa9 */
-		{{0, 0},
-		{3, 3},
-		{5, 5},
-		{7, 7}
-		}
-	},
-	{0, 1, 4, 0,		/* 0xaa */
-		{{1, 1},
-		{3, 3},
-		{5, 5},
-		{7, 7}
-		}
-	},
-	{1, 1, 4, 0,		/* 0xab */
-		{{0, 1},
-		{3, 3},
-		{5, 5},
-		{7, 7}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xac */
-		{{2, 3},
-		{5, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 4, 0,		/* 0xad */
-		{{0, 0},
-		{2, 3},
-		{5, 5},
-		{7, 7}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xae */
-		{{1, 3},
-		{5, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xaf */
-		{{0, 3},
-		{5, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xb0 */
-		{{4, 5},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xb1 */
-		{{0, 0},
-		{4, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xb2 */
-		{{1, 1},
-		{4, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xb3 */
-		{{0, 1},
-		{4, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xb4 */
-		{{2, 2},
-		{4, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 4, 0,		/* 0xb5 */
-		{{0, 0},
-		{2, 2},
-		{4, 5},
-		{7, 7}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xb6 */
-		{{1, 2},
-		{4, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xb7 */
-		{{0, 2},
-		{4, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xb8 */
-		{{3, 5},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xb9 */
-		{{0, 0},
-		{3, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xba */
-		{{1, 1},
-		{3, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xbb */
-		{{0, 1},
-		{3, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xbc */
-		{{2, 5},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xbd */
-		{{0, 0},
-		{2, 5},
-		{7, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xbe */
-		{{1, 5},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xbf */
-		{{0, 5},
-		{7, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 1, 0,		/* 0xc0 */
-		{{6, 7},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xc1 */
-		{{0, 0},
-		{6, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xc2 */
-		{{1, 1},
-		{6, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xc3 */
-		{{0, 1},
-		{6, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xc4 */
-		{{2, 2},
-		{6, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xc5 */
-		{{0, 0},
-		{2, 2},
-		{6, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xc6 */
-		{{1, 2},
-		{6, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xc7 */
-		{{0, 2},
-		{6, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xc8 */
-		{{3, 3},
-		{6, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xc9 */
-		{{0, 0},
-		{3, 3},
-		{6, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xca */
-		{{1, 1},
-		{3, 3},
-		{6, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xcb */
-		{{0, 1},
-		{3, 3},
-		{6, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xcc */
-		{{2, 3},
-		{6, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xcd */
-		{{0, 0},
-		{2, 3},
-		{6, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xce */
-		{{1, 3},
-		{6, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xcf */
-		{{0, 3},
-		{6, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xd0 */
-		{{4, 4},
-		{6, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xd1 */
-		{{0, 0},
-		{4, 4},
-		{6, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xd2 */
-		{{1, 1},
-		{4, 4},
-		{6, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xd3 */
-		{{0, 1},
-		{4, 4},
-		{6, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xd4 */
-		{{2, 2},
-		{4, 4},
-		{6, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 4, 0,		/* 0xd5 */
-		{{0, 0},
-		{2, 2},
-		{4, 4},
-		{6, 7}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xd6 */
-		{{1, 2},
-		{4, 4},
-		{6, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xd7 */
-		{{0, 2},
-		{4, 4},
-		{6, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xd8 */
-		{{3, 4},
-		{6, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xd9 */
-		{{0, 0},
-		{3, 4},
-		{6, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xda */
-		{{1, 1},
-		{3, 4},
-		{6, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xdb */
-		{{0, 1},
-		{3, 4},
-		{6, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xdc */
-		{{2, 4},
-		{6, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xdd */
-		{{0, 0},
-		{2, 4},
-		{6, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xde */
-		{{1, 4},
-		{6, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xdf */
-		{{0, 4},
-		{6, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 1, 0,		/* 0xe0 */
-		{{5, 7},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xe1 */
-		{{0, 0},
-		{5, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xe2 */
-		{{1, 1},
-		{5, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xe3 */
-		{{0, 1},
-		{5, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xe4 */
-		{{2, 2},
-		{5, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xe5 */
-		{{0, 0},
-		{2, 2},
-		{5, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xe6 */
-		{{1, 2},
-		{5, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xe7 */
-		{{0, 2},
-		{5, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xe8 */
-		{{3, 3},
-		{5, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xe9 */
-		{{0, 0},
-		{3, 3},
-		{5, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 3, 0,		/* 0xea */
-		{{1, 1},
-		{3, 3},
-		{5, 7},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xeb */
-		{{0, 1},
-		{3, 3},
-		{5, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xec */
-		{{2, 3},
-		{5, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xed */
-		{{0, 0},
-		{2, 3},
-		{5, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xee */
-		{{1, 3},
-		{5, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xef */
-		{{0, 3},
-		{5, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 1, 0,		/* 0xf0 */
-		{{4, 7},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xf1 */
-		{{0, 0},
-		{4, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xf2 */
-		{{1, 1},
-		{4, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xf3 */
-		{{0, 1},
-		{4, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xf4 */
-		{{2, 2},
-		{4, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 3, 0,		/* 0xf5 */
-		{{0, 0},
-		{2, 2},
-		{4, 7},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xf6 */
-		{{1, 2},
-		{4, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xf7 */
-		{{0, 2},
-		{4, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 1, 0,		/* 0xf8 */
-		{{3, 7},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xf9 */
-		{{0, 0},
-		{3, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 2, 0,		/* 0xfa */
-		{{1, 1},
-		{3, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xfb */
-		{{0, 1},
-		{3, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 1, 0,		/* 0xfc */
-		{{2, 7},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 2, 0,		/* 0xfd */
-		{{0, 0},
-		{2, 7},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{0, 1, 1, 0,		/* 0xfe */
-		{{1, 7},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	},
-	{1, 1, 1, 0,		/* 0xff */
-		{{0, 7},
-		{0, 0},
-		{0, 0},
-		{0, 0}
-		}
-	}
-};
-
-
 
 int
 sctp_is_address_in_scope(struct sctp_ifa *ifa,
@@ -7513,7 +5707,7 @@ sctp_prune_prsctp(struct sctp_tcb *stcb,
 							cause = SCTP_RESPONSE_TO_USER_REQ | SCTP_NOTIFY_DATAGRAM_UNSENT;
 						ret_spc = sctp_release_pr_sctp_chunk(stcb, chk,
 						    cause,
-						    &asoc->sent_queue, SCTP_SO_LOCKED);
+						    SCTP_SO_LOCKED);
 						freed_spc += ret_spc;
 						if (freed_spc >= dataout) {
 							return;
@@ -7538,7 +5732,7 @@ sctp_prune_prsctp(struct sctp_tcb *stcb,
 
 						ret_spc = sctp_release_pr_sctp_chunk(stcb, chk,
 						    SCTP_RESPONSE_TO_USER_REQ | SCTP_NOTIFY_DATAGRAM_UNSENT,
-						    &asoc->send_queue, SCTP_SO_LOCKED);
+						    SCTP_SO_LOCKED);
 
 						freed_spc += ret_spc;
 						if (freed_spc >= dataout) {
@@ -7920,7 +6114,7 @@ sctp_med_chunk_output(struct sctp_inpcb *inp,
     struct sctp_association *asoc,
     int *num_out,
     int *reason_code,
-    int control_only, int *cwnd_full, int from_where,
+    int control_only, int from_where,
     struct timeval *now, int *now_filled, int frag_point, int so_locked
 #if !defined(__APPLE__) && !defined(SCTP_SO_LOCK_TESTING)
     SCTP_UNUSED
@@ -8106,13 +6300,13 @@ sctp_sendall_iterator(struct sctp_inpcb *inp, struct sctp_tcb *stcb, void *ptr,
 	if (do_chunk_output)
 		sctp_chunk_output(inp, stcb, SCTP_OUTPUT_FROM_USR_SEND, SCTP_SO_NOT_LOCKED);
 	else if (added_control) {
-		int num_out = 0, reason = 0, cwnd_full = 0, now_filled = 0;
+		int num_out = 0, reason = 0, now_filled = 0;
 		struct timeval now;
 		int frag_point;
 
 		frag_point = sctp_get_frag_point(stcb, &stcb->asoc);
 		(void)sctp_med_chunk_output(inp, stcb, &stcb->asoc, &num_out,
-		    &reason, 1, &cwnd_full, 1, &now, &now_filled, frag_point, SCTP_SO_NOT_LOCKED);
+		    &reason, 1, 1, &now, &now_filled, frag_point, SCTP_SO_NOT_LOCKED);
 	}
 no_chunk_output:
 	if (ret) {
@@ -8405,6 +6599,7 @@ sctp_clean_up_ctl(struct sctp_tcb *stcb, struct sctp_association *asoc)
 		    (chk->rec.chunk_id.id == SCTP_NR_SELECTIVE_ACK) ||	/* EY */
 		    (chk->rec.chunk_id.id == SCTP_HEARTBEAT_REQUEST) ||
 		    (chk->rec.chunk_id.id == SCTP_HEARTBEAT_ACK) ||
+		    (chk->rec.chunk_id.id == SCTP_FORWARD_CUM_TSN) ||
 		    (chk->rec.chunk_id.id == SCTP_SHUTDOWN) ||
 		    (chk->rec.chunk_id.id == SCTP_SHUTDOWN_ACK) ||
 		    (chk->rec.chunk_id.id == SCTP_OPERATION_ERROR) ||
@@ -8547,7 +6742,7 @@ one_more_time:
 			 * when we took all the data the sender_all_done was
 			 * not set.
 			 */
-			if (sp->put_last_out == 0) {
+			if ((sp->put_last_out == 0) && (sp->discard_rest == 0)) {
 				SCTP_PRINTF("Gak, put out entire msg with NO end!-1\n");
 				SCTP_PRINTF("sender_done:%d len:%d msg_comp:%d put_last_out:%d send_lock:%d\n",
 				    sp->sender_all_done,
@@ -8568,7 +6763,6 @@ one_more_time:
 				sp->data = NULL;
 			}
 			sctp_free_a_strmoq(stcb, sp);
-
 			/* we can't be locked to it */
 			*locked = 0;
 			stcb->asoc.locked_on_sending = NULL;
@@ -8596,6 +6790,29 @@ one_more_time:
 			*giveup = 1;
 			to_move = 0;
 			goto out_of;
+		} else if (sp->discard_rest) {
+			if (send_lock_up == 0) {
+				SCTP_TCB_SEND_LOCK(stcb);
+				send_lock_up = 1;
+			}
+			/* Whack down the size */
+			atomic_subtract_int(&stcb->asoc.total_output_queue_size, sp->length);
+			if ((stcb->sctp_socket != NULL) && \
+			    ((stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_TCPTYPE) ||
+			    (stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_IN_TCPPOOL))) {
+				atomic_subtract_int(&stcb->sctp_socket->so_snd.sb_cc, sp->length);
+			}
+			if (sp->data) {
+				sctp_m_freem(sp->data);
+				sp->data = NULL;
+				sp->tail_mbuf = NULL;
+			}
+			sp->length = 0;
+			sp->some_taken = 1;
+			*locked = 1;
+			*giveup = 1;
+			to_move = 0;
+			goto out_of;
 		}
 	}
 	some_taken = sp->some_taken;
@@ -9172,7 +7389,7 @@ sctp_med_chunk_output(struct sctp_inpcb *inp,
     struct sctp_association *asoc,
     int *num_out,
     int *reason_code,
-    int control_only, int *cwnd_full, int from_where,
+    int control_only, int from_where,
     struct timeval *now, int *now_filled, int frag_point, int so_locked
 #if !defined(__APPLE__) && !defined(SCTP_SO_LOCK_TESTING)
     SCTP_UNUSED
@@ -9188,18 +7405,18 @@ sctp_med_chunk_output(struct sctp_inpcb *inp,
 	 * fomulate and send the low level chunks. Making sure to combine
 	 * any control in the control chunk queue also.
 	 */
-	struct sctp_nets *net;
+	struct sctp_nets *net, *start_at, *old_start_at = NULL;
 	struct mbuf *outchain, *endoutchain;
 	struct sctp_tmit_chunk *chk, *nchk;
 
 	/* temp arrays for unlinking */
 	struct sctp_tmit_chunk *data_list[SCTP_MAX_DATA_BUNDLING];
 	int no_fragmentflg, error;
+	unsigned int max_rwnd_per_dest;
 	int one_chunk, hbflag, skip_data_for_this_net;
 	int asconf, cookie, no_out_cnt;
-	int bundle_at, ctl_cnt, no_data_chunks, cwnd_full_ind, eeor_mode;
+	int bundle_at, ctl_cnt, no_data_chunks, eeor_mode;
 	unsigned int mtu, r_mtu, omtu, mx_mtu, to_out;
-	struct sctp_nets *start_at, *old_startat = NULL, *send_start_at;
 	int tsns_sent = 0;
 	uint32_t auth_offset = 0;
 	struct sctp_auth_chunk *auth = NULL;
@@ -9215,7 +7432,6 @@ sctp_med_chunk_output(struct sctp_inpcb *inp,
 	int quit_now = 0;
 
 	*num_out = 0;
-	cwnd_full_ind = 0;
 	auth_keyid = stcb->asoc.authinfo.active_keyid;
 
 	if ((asoc->state & SCTP_STATE_SHUTDOWN_PENDING) ||
@@ -9251,112 +7467,61 @@ sctp_med_chunk_output(struct sctp_inpcb *inp,
 	}
 	if (asoc->peers_rwnd == 0) {
 		/* No room in peers rwnd */
-		*cwnd_full = 1;
 		*reason_code = 1;
 		if (asoc->total_flight > 0) {
 			/* we are allowed one chunk in flight */
 			no_data_chunks = 1;
 		}
 	}
+	max_rwnd_per_dest = ((asoc->peers_rwnd + asoc->total_flight) / asoc->numnets);
 	if ((no_data_chunks == 0) && (!TAILQ_EMPTY(&asoc->out_wheel))) {
-		if (SCTP_BASE_SYSCTL(sctp_cmt_on_off)) {
-			/*
-			 * for CMT we start at the next one past the one we
-			 * last added data to.
-			 */
-			if (TAILQ_FIRST(&asoc->send_queue) != NULL) {
-				goto skip_the_fill_from_streams;
-			}
-			if (asoc->last_net_data_came_from) {
-				net = TAILQ_NEXT(asoc->last_net_data_came_from, sctp_next);
-				if (net == NULL) {
-					net = TAILQ_FIRST(&asoc->nets);
-				}
-			} else {
-				/* back to start */
-				net = TAILQ_FIRST(&asoc->nets);
-			}
-
+		TAILQ_FOREACH(net, &asoc->nets, sctp_next) {
 			/*
-			 * JRI-TODO: CMT-MPI. Simply set the first
-			 * destination (net) to be optimized for the next
-			 * message to be pulled out of the outwheel. 1. peek
-			 * at outwheel 2. If large message, set net =
-			 * highest_cwnd 3. If small message, set net =
-			 * lowest rtt
+			 * This for loop we are in takes in each net, if
+			 * its's got space in cwnd and has data sent to it
+			 * (when CMT is off) then it calls
+			 * sctp_fill_outqueue for the net. This gets data on
+			 * the send queue for that network.
+			 * 
+			 * In sctp_fill_outqueue TSN's are assigned and data is
+			 * copied out of the stream buffers. Note mostly
+			 * copy by reference (we hope).
 			 */
-		} else {
-			net = asoc->primary_destination;
-			if (net == NULL) {
-				/* TSNH */
-				net = TAILQ_FIRST(&asoc->nets);
-			}
-		}
-		start_at = net;
-
-one_more_time:
-		for (; net != NULL; net = TAILQ_NEXT(net, sctp_next)) {
 			net->window_probe = 0;
-			if (old_startat && (old_startat == net)) {
-				break;
-			}
-			/*
-			 * JRI: if dest is unreachable or unconfirmed, do
-			 * not send data to it
-			 */
 			if ((net->dest_state & SCTP_ADDR_NOT_REACHABLE) || (net->dest_state & SCTP_ADDR_UNCONFIRMED)) {
-				continue;
-			}
-			/*
-			 * JRI: if dest is in PF state, do not send data to
-			 * it
-			 */
-			if (SCTP_BASE_SYSCTL(sctp_cmt_on_off) &&
-			    SCTP_BASE_SYSCTL(sctp_cmt_pf) &&
-			    (net->dest_state & SCTP_ADDR_PF)) {
+				if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_CWND_LOGGING_ENABLE) {
+					sctp_log_cwnd(stcb, net, 1,
+					    SCTP_CWND_LOG_FILL_OUTQ_CALLED);
+				}
 				continue;
 			}
 			if ((SCTP_BASE_SYSCTL(sctp_cmt_on_off) == 0) && (net->ref_count < 2)) {
 				/* nothing can be in queue for this guy */
+				if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_CWND_LOGGING_ENABLE) {
+					sctp_log_cwnd(stcb, net, 2,
+					    SCTP_CWND_LOG_FILL_OUTQ_CALLED);
+				}
 				continue;
 			}
 			if (net->flight_size >= net->cwnd) {
-				/* skip this network, no room */
-				cwnd_full_ind++;
+				/* skip this network, no room - can't fill */
+				if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_CWND_LOGGING_ENABLE) {
+					sctp_log_cwnd(stcb, net, 3,
+					    SCTP_CWND_LOG_FILL_OUTQ_CALLED);
+				}
 				continue;
 			}
-			/*
-			 * JRI : this for loop we are in takes in each net,
-			 * if its's got space in cwnd and has data sent to
-			 * it (when CMT is off) then it calls
-			 * sctp_fill_outqueue for the net. This gets data on
-			 * the send queue for that network.
-			 * 
-			 * In sctp_fill_outqueue TSN's are assigned and data is
-			 * copied out of the stream buffers. Note mostly
-			 * copy by reference (we hope).
-			 */
 			if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_CWND_LOGGING_ENABLE) {
-				sctp_log_cwnd(stcb, net, 0, SCTP_CWND_LOG_FILL_OUTQ_CALLED);
+				sctp_log_cwnd(stcb, net, 4, SCTP_CWND_LOG_FILL_OUTQ_CALLED);
 			}
 			sctp_fill_outqueue(stcb, net, frag_point, eeor_mode, &quit_now);
 			if (quit_now) {
 				/* memory alloc failure */
 				no_data_chunks = 1;
-				goto skip_the_fill_from_streams;
+				break;
 			}
 		}
-		if (start_at != TAILQ_FIRST(&asoc->nets)) {
-			/* got to pick up the beginning stuff. */
-			old_startat = start_at;
-			start_at = net = TAILQ_FIRST(&asoc->nets);
-			if (old_startat)
-				goto one_more_time;
-		}
 	}
-skip_the_fill_from_streams:
-	*cwnd_full = cwnd_full_ind;
-
 	/* now service each destination and send out what we can for it */
 	/* Nothing to send? */
 	if ((TAILQ_FIRST(&asoc->control_send_queue) == NULL) &&
@@ -9365,32 +7530,45 @@ skip_the_fill_from_streams:
 		*reason_code = 8;
 		return (0);
 	}
-	if (no_data_chunks) {
-		chk = TAILQ_FIRST(&asoc->asconf_send_queue);
-		if (chk == NULL)
-			chk = TAILQ_FIRST(&asoc->control_send_queue);
-	} else {
-		chk = TAILQ_FIRST(&asoc->send_queue);
-	}
-	if (chk) {
-		send_start_at = chk->whoTo;
+	if (SCTP_BASE_SYSCTL(sctp_cmt_on_off)) {
+		/* get the last start point */
+		start_at = asoc->last_net_cmt_send_started;
+		if (start_at == NULL) {
+			/* null so to beginning */
+			start_at = TAILQ_FIRST(&asoc->nets);
+		} else {
+			start_at = TAILQ_NEXT(asoc->last_net_cmt_send_started, sctp_next);
+			if (start_at == NULL) {
+				start_at = TAILQ_FIRST(&asoc->nets);
+			}
+		}
+		asoc->last_net_cmt_send_started = start_at;
 	} else {
-		send_start_at = TAILQ_FIRST(&asoc->nets);
+		start_at = TAILQ_FIRST(&asoc->nets);
 	}
-	old_startat = NULL;
+	old_start_at = NULL;
 again_one_more_time:
-	for (net = send_start_at; net != NULL; net = TAILQ_NEXT(net, sctp_next)) {
+	for (net = start_at; net != NULL; net = TAILQ_NEXT(net, sctp_next)) {
 		/* how much can we send? */
 		/* SCTPDBG("Examine for sending net:%x\n", (uint32_t)net); */
-		if (old_startat && (old_startat == net)) {
+		if (old_start_at && (old_start_at == net)) {
 			/* through list ocmpletely. */
 			break;
 		}
-		tsns_sent = 0;
-		if (net->ref_count < 2) {
+		tsns_sent = 0xa;
+		if ((SCTP_BASE_SYSCTL(sctp_cmt_on_off) == 0) && (net->ref_count < 2)) {
 			/*
 			 * Ref-count of 1 so we cannot have data or control
-			 * queued to this address. Skip it.
+			 * queued to this address. Skip it (non-CMT).
+			 */
+			continue;
+		}
+		if ((TAILQ_FIRST(&asoc->control_send_queue) == NULL) &&
+		    (TAILQ_FIRST(&asoc->asconf_send_queue) == NULL) &&
+		    (net->flight_size >= net->cwnd)) {
+			/*
+			 * Nothing on control or asconf and flight is full,
+			 * we can skip even in the CMT case.
 			 */
 			continue;
 		}
@@ -9862,6 +8040,19 @@ again_one_more_time:
 				}
 			}
 		}
+		/* JRI: if dest is in PF state, do not send data to it */
+		if (SCTP_BASE_SYSCTL(sctp_cmt_on_off) &&
+		    SCTP_BASE_SYSCTL(sctp_cmt_pf) &&
+		    (net->dest_state & SCTP_ADDR_PF)) {
+			goto no_data_fill;
+		}
+		if (net->flight_size >= net->cwnd) {
+			goto no_data_fill;
+		}
+		if ((SCTP_BASE_SYSCTL(sctp_cmt_on_off)) &&
+		    (net->flight_size > max_rwnd_per_dest)) {
+			goto no_data_fill;
+		}
 		/*********************/
 		/* Data transmission */
 		/*********************/
@@ -9897,7 +8088,8 @@ again_one_more_time:
 			omtu = 0;
 			break;
 		}
-		if ((((asoc->state & SCTP_STATE_OPEN) == SCTP_STATE_OPEN) && (skip_data_for_this_net == 0)) ||
+		if ((((asoc->state & SCTP_STATE_OPEN) == SCTP_STATE_OPEN) &&
+		    (skip_data_for_this_net == 0)) ||
 		    (cookie)) {
 			for (chk = TAILQ_FIRST(&asoc->send_queue); chk; chk = nchk) {
 				if (no_data_chunks) {
@@ -9911,7 +8103,18 @@ again_one_more_time:
 					break;
 				}
 				nchk = TAILQ_NEXT(chk, sctp_next);
-				if (chk->whoTo != net) {
+				if (SCTP_BASE_SYSCTL(sctp_cmt_on_off)) {
+					if (chk->whoTo != net) {
+						/*
+						 * For CMT, steal the data
+						 * to this network if its
+						 * not set here.
+						 */
+						sctp_free_remote_addr(chk->whoTo);
+						chk->whoTo = net;
+						atomic_add_int(&chk->whoTo->ref_count, 1);
+					}
+				} else if (chk->whoTo != net) {
 					/* No, not sent to this net */
 					continue;
 				}
@@ -10051,6 +8254,7 @@ again_one_more_time:
 				}
 			}	/* for (chunk gather loop for this net) */
 		}		/* if asoc.state OPEN */
+no_data_fill:
 		/* Is there something to send for this destination? */
 		if (outchain) {
 			/* We may need to start a control timer or two */
@@ -10199,10 +8403,10 @@ again_one_more_time:
 			sctp_log_cwnd(stcb, net, tsns_sent, SCTP_CWND_LOG_FROM_SEND);
 		}
 	}
-	if (old_startat == NULL) {
-		old_startat = send_start_at;
-		send_start_at = TAILQ_FIRST(&asoc->nets);
-		if (old_startat)
+	if (old_start_at == NULL) {
+		old_start_at = start_at;
+		start_at = TAILQ_FIRST(&asoc->nets);
+		if (old_start_at)
 			goto again_one_more_time;
 	}
 	/*
@@ -11247,7 +9451,6 @@ sctp_chunk_output(struct sctp_inpcb *inp,
 	    burst_cnt = 0, burst_limit = 0;
 	struct timeval now;
 	int now_filled = 0;
-	int cwnd_full = 0;
 	int nagle_on = 0;
 	int frag_point = sctp_get_frag_point(stcb, &stcb->asoc);
 	int un_sent = 0;
@@ -11298,7 +9501,7 @@ sctp_chunk_output(struct sctp_inpcb *inp,
 			 * and then the next call with get the retran's.
 			 */
 			(void)sctp_med_chunk_output(inp, stcb, asoc, &num_out, &reason_code, 1,
-			    &cwnd_full, from_where,
+			    from_where,
 			    &now, &now_filled, frag_point, so_locked);
 			return;
 		} else if (from_where != SCTP_OUTPUT_FROM_HB_TMR) {
@@ -11323,7 +9526,7 @@ sctp_chunk_output(struct sctp_inpcb *inp,
 			 * if queued too.
 			 */
 			(void)sctp_med_chunk_output(inp, stcb, asoc, &num_out, &reason_code, 1,
-			    &cwnd_full, from_where,
+			    from_where,
 			    &now, &now_filled, frag_point, so_locked);
 #ifdef SCTP_AUDITING_ENABLED
 			sctp_auditing(8, inp, stcb, NULL);
@@ -11350,7 +9553,7 @@ sctp_chunk_output(struct sctp_inpcb *inp,
 			sctp_auditing(10, inp, stcb, NULL);
 #endif
 			/* Push out any control */
-			(void)sctp_med_chunk_output(inp, stcb, asoc, &num_out, &reason_code, 1, &cwnd_full, from_where,
+			(void)sctp_med_chunk_output(inp, stcb, asoc, &num_out, &reason_code, 1, from_where,
 			    &now, &now_filled, frag_point, so_locked);
 			return;
 		}
@@ -11421,10 +9624,9 @@ sctp_chunk_output(struct sctp_inpcb *inp,
 
 	}
 	burst_cnt = 0;
-	cwnd_full = 0;
 	do {
 		error = sctp_med_chunk_output(inp, stcb, asoc, &num_out,
-		    &reason_code, 0, &cwnd_full, from_where,
+		    &reason_code, 0, from_where,
 		    &now, &now_filled, frag_point, so_locked);
 		if (error) {
 			SCTPDBG(SCTP_DEBUG_OUTPUT1, "Error %d was returned from med-c-op\n", error);
@@ -11533,6 +9735,7 @@ send_forward_tsn(struct sctp_tcb *stcb,
 {
 	struct sctp_tmit_chunk *chk;
 	struct sctp_forward_tsn_chunk *fwdtsn;
+	uint32_t advance_peer_ack_point;
 
 	SCTP_TCB_LOCK_ASSERT(stcb);
 	TAILQ_FOREACH(chk, &asoc->control_send_queue, sctp_next) {
@@ -11610,11 +9813,23 @@ sctp_fill_in_rest:
 			/* trim to a mtu size */
 			cnt_of_space = asoc->smallest_mtu - ovh;
 		}
+		if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_LOG_TRY_ADVANCE) {
+			sctp_misc_ints(SCTP_FWD_TSN_CHECK,
+			    0xff, 0, cnt_of_skipped,
+			    asoc->advanced_peer_ack_point);
+
+		}
+		advance_peer_ack_point = asoc->advanced_peer_ack_point;
 		if (cnt_of_space < space_needed) {
 			/*-
 			 * ok we must trim down the chunk by lowering the
 			 * advance peer ack point.
 			 */
+			if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_LOG_TRY_ADVANCE) {
+				sctp_misc_ints(SCTP_FWD_TSN_CHECK,
+				    0xff, 0xff, cnt_of_space,
+				    space_needed);
+			}
 			cnt_of_skipped = (cnt_of_space -
 			    ((sizeof(struct sctp_forward_tsn_chunk)) /
 			    sizeof(struct sctp_strseq)));
@@ -11627,12 +9842,17 @@ sctp_fill_in_rest:
 				tp1 = TAILQ_NEXT(at, sctp_next);
 				at = tp1;
 			}
+			if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_LOG_TRY_ADVANCE) {
+				sctp_misc_ints(SCTP_FWD_TSN_CHECK,
+				    0xff, cnt_of_skipped, at->rec.data.TSN_seq,
+				    asoc->advanced_peer_ack_point);
+			}
 			last = at;
 			/*-
 			 * last now points to last one I can report, update
 			 * peer ack point
 			 */
-			asoc->advanced_peer_ack_point = last->rec.data.TSN_seq;
+			advance_peer_ack_point = last->rec.data.TSN_seq;
 			space_needed -= (cnt_of_skipped * sizeof(struct sctp_strseq));
 		}
 		chk->send_size = space_needed;
@@ -11641,7 +9861,7 @@ sctp_fill_in_rest:
 		fwdtsn->ch.chunk_length = htons(chk->send_size);
 		fwdtsn->ch.chunk_flags = 0;
 		fwdtsn->ch.chunk_type = SCTP_FORWARD_CUM_TSN;
-		fwdtsn->new_cumulative_tsn = htonl(asoc->advanced_peer_ack_point);
+		fwdtsn->new_cumulative_tsn = htonl(advance_peer_ack_point);
 		chk->send_size = (sizeof(struct sctp_forward_tsn_chunk) +
 		    (cnt_of_skipped * sizeof(struct sctp_strseq)));
 		SCTP_BUF_LEN(chk->data) = chk->send_size;
@@ -11672,6 +9892,9 @@ sctp_fill_in_rest:
 				at = tp1;
 				continue;
 			}
+			if (at->rec.data.TSN_seq == advance_peer_ack_point) {
+				at->rec.data.fwd_tsn_cnt = 0;
+			}
 			strseq->stream = ntohs(at->rec.data.stream_number);
 			strseq->sequence = ntohs(at->rec.data.stream_seq);
 			strseq++;
@@ -11862,7 +10085,11 @@ sctp_send_sack(struct sctp_tcb *stcb)
 
 	gap_descriptor = (struct sctp_gap_ack_block *)((caddr_t)sack + sizeof(struct sctp_sack_chunk));
 
-	siz = (((asoc->highest_tsn_inside_map - asoc->mapping_array_base_tsn) + 1) + 7) / 8;
+	if (asoc->highest_tsn_inside_map > asoc->mapping_array_base_tsn)
+		siz = (((asoc->highest_tsn_inside_map - asoc->mapping_array_base_tsn) + 1) + 7) / 8;
+	else
+		siz = (((MAX_TSN - asoc->mapping_array_base_tsn) + 1) + asoc->highest_tsn_inside_map + 7) / 8;
+
 	if (compare_with_wrap(asoc->mapping_array_base_tsn, asoc->cumulative_tsn, MAX_TSN)) {
 		offset = 1;
 		/*-
@@ -11990,7 +10217,7 @@ sctp_send_nr_sack(struct sctp_tcb *stcb)
 	struct sctp_nr_gap_ack_block *nr_gap_descriptor;
 
 	struct sack_track *selector;
-	struct nr_sack_track *nr_selector;
+	struct sack_track *nr_selector;
 
 	/* EY do we need nr_mergeable, NO */
 	int mergeable = 0;
@@ -12170,9 +10397,12 @@ sctp_send_nr_sack(struct sctp_tcb *stcb)
 	stcb->freed_by_sorcv_sincelast = 0;
 
 	gap_descriptor = (struct sctp_gap_ack_block *)((caddr_t)nr_sack + sizeof(struct sctp_nr_sack_chunk));
-	nr_gap_descriptor = (struct sctp_nr_gap_ack_block *)((caddr_t)nr_sack + sizeof(struct sctp_nr_sack_chunk));
 
-	siz = (((asoc->highest_tsn_inside_map - asoc->mapping_array_base_tsn) + 1) + 7) / 8;
+	if (asoc->highest_tsn_inside_map > asoc->mapping_array_base_tsn)
+		siz = (((asoc->highest_tsn_inside_map - asoc->mapping_array_base_tsn) + 1) + 7) / 8;
+	else
+		siz = (((MAX_TSN - asoc->mapping_array_base_tsn) + 1) + asoc->highest_tsn_inside_map + 7) / 8;
+
 	if (compare_with_wrap(asoc->mapping_array_base_tsn, asoc->cumulative_tsn, MAX_TSN)) {
 		offset = 1;
 		/*-
@@ -12255,11 +10485,15 @@ sctp_send_nr_sack(struct sctp_tcb *stcb)
 	nr_gap_descriptor = (struct sctp_nr_gap_ack_block *)gap_descriptor;
 
 	/* EY - there will be gaps + nr_gaps if draining is possible */
-	if (SCTP_BASE_SYSCTL(sctp_do_drain)) {
+	if ((SCTP_BASE_SYSCTL(sctp_do_drain)) && (limit_reached == 0)) {
 
 		mergeable = 0;
 
-		siz = (((asoc->highest_tsn_inside_nr_map - asoc->nr_mapping_array_base_tsn) + 1) + 7) / 8;
+		if (asoc->highest_tsn_inside_nr_map > asoc->nr_mapping_array_base_tsn)
+			siz = (((asoc->highest_tsn_inside_nr_map - asoc->nr_mapping_array_base_tsn) + 1) + 7) / 8;
+		else
+			siz = (((MAX_TSN - asoc->nr_mapping_array_base_tsn) + 1) + asoc->highest_tsn_inside_nr_map + 7) / 8;
+
 		if (compare_with_wrap(asoc->nr_mapping_array_base_tsn, asoc->cumulative_tsn, MAX_TSN)) {
 			offset = 1;
 			/*-
@@ -12278,7 +10512,7 @@ sctp_send_nr_sack(struct sctp_tcb *stcb)
 		if (compare_with_wrap(asoc->highest_tsn_inside_nr_map, asoc->cumulative_tsn, MAX_TSN)) {
 			/* we have a gap .. maybe */
 			for (i = 0; i < siz; i++) {
-				nr_selector = &nr_sack_array[asoc->nr_mapping_array[i]];
+				nr_selector = &sack_array[asoc->nr_mapping_array[i]];
 				if (mergeable && nr_selector->right_edge) {
 					/*
 					 * Backup, left and right edges were
@@ -12304,9 +10538,9 @@ sctp_send_nr_sack(struct sctp_tcb *stcb)
 							 * left side
 							 */
 							mergeable = 0;
-							nr_gap_descriptor->start = htons((nr_selector->nr_gaps[j].start + offset));
+							nr_gap_descriptor->start = htons((nr_selector->gaps[j].start + offset));
 						}
-						nr_gap_descriptor->end = htons((nr_selector->nr_gaps[j].end + offset));
+						nr_gap_descriptor->end = htons((nr_selector->gaps[j].end + offset));
 						num_nr_gap_blocks++;
 						nr_gap_descriptor++;
 						if (((caddr_t)nr_gap_descriptor + sizeof(struct sctp_nr_gap_ack_block)) > limit) {
@@ -12328,7 +10562,7 @@ sctp_send_nr_sack(struct sctp_tcb *stcb)
 			}
 		}
 	}
-	/*---------------------------------------------------End of---filling the nr_gap_ack blocks----------------------------------------------------*/
+	/*---------------------------------------------End of---filling the nr_gap_ack blocks----------------------------------------------------*/
 
 	/* now we must add any dups we are going to report. */
 	if ((limit_reached == 0) && (asoc->numduptsns)) {
@@ -15401,7 +13635,7 @@ skip_out_eof:
 		}
 		sctp_chunk_output(inp, stcb, SCTP_OUTPUT_FROM_USR_SEND, SCTP_SO_LOCKED);
 	} else if (some_on_control) {
-		int num_out, reason, cwnd_full, frag_point;
+		int num_out, reason, frag_point;
 
 		/* Here we do control only */
 		if (hold_tcblock == 0) {
@@ -15410,7 +13644,7 @@ skip_out_eof:
 		}
 		frag_point = sctp_get_frag_point(stcb, &stcb->asoc);
 		(void)sctp_med_chunk_output(inp, stcb, &stcb->asoc, &num_out,
-		    &reason, 1, &cwnd_full, 1, &now, &now_filled, frag_point, SCTP_SO_LOCKED);
+		    &reason, 1, 1, &now, &now_filled, frag_point, SCTP_SO_LOCKED);
 	}
 	SCTPDBG(SCTP_DEBUG_OUTPUT1, "USR Send complete qo:%d prw:%d unsent:%d tf:%d cooq:%d toqs:%d err:%d\n",
 	    queue_only, stcb->asoc.peers_rwnd, un_sent,
diff --git a/sys/netinet/sctp_pcb.c b/sys/netinet/sctp_pcb.c
index d98e760..df519cc 100644
--- a/sys/netinet/sctp_pcb.c
+++ b/sys/netinet/sctp_pcb.c
@@ -3157,8 +3157,9 @@ sctp_inpcb_free(struct sctp_inpcb *inp, int immediate, int from)
 				SCTP_TCB_UNLOCK(asoc);
 				continue;
 			}
-			if ((SCTP_GET_STATE(&asoc->asoc) == SCTP_STATE_COOKIE_WAIT) ||
-			    (SCTP_GET_STATE(&asoc->asoc) == SCTP_STATE_COOKIE_ECHOED)) {
+			if (((SCTP_GET_STATE(&asoc->asoc) == SCTP_STATE_COOKIE_WAIT) ||
+			    (SCTP_GET_STATE(&asoc->asoc) == SCTP_STATE_COOKIE_ECHOED)) &&
+			    (asoc->asoc.total_output_queue_size == 0)) {
 				/*
 				 * If we have data in queue, we don't want
 				 * to just free since the app may have done,
@@ -6029,6 +6030,7 @@ sctp_load_addresses_from_init(struct sctp_tcb *stcb, struct mbuf *m,
 			stcb->asoc.peer_supports_prsctp = 0;
 			stcb->asoc.peer_supports_pktdrop = 0;
 			stcb->asoc.peer_supports_strreset = 0;
+			stcb->asoc.peer_supports_nr_sack = 0;
 			stcb->asoc.peer_supports_auth = 0;
 			pr_supported = (struct sctp_supported_chunk_types_param *)phdr;
 			num_ent = plen - sizeof(struct sctp_paramhdr);
@@ -6044,6 +6046,12 @@ sctp_load_addresses_from_init(struct sctp_tcb *stcb, struct mbuf *m,
 				case SCTP_PACKET_DROPPED:
 					stcb->asoc.peer_supports_pktdrop = 1;
 					break;
+				case SCTP_NR_SELECTIVE_ACK:
+					if (SCTP_BASE_SYSCTL(sctp_nr_sack_on_off))
+						stcb->asoc.peer_supports_nr_sack = 1;
+					else
+						stcb->asoc.peer_supports_nr_sack = 0;
+					break;
 				case SCTP_STREAM_RESET:
 					stcb->asoc.peer_supports_strreset = 1;
 					break;
diff --git a/sys/netinet/sctp_structs.h b/sys/netinet/sctp_structs.h
index a03ae94..24ae10f 100644
--- a/sys/netinet/sctp_structs.h
+++ b/sys/netinet/sctp_structs.h
@@ -196,6 +196,7 @@ struct sctp_nets {
 	/* smoothed average things for RTT and RTO itself */
 	int lastsa;
 	int lastsv;
+	int rtt;		/* last measured rtt value in ms */
 	unsigned int RTO;
 
 	/* This is used for SHUTDOWN/SHUTDOWN-ACK/SEND or INIT timers */
@@ -309,7 +310,7 @@ struct sctp_data_chunkrec {
 
 	/* ECN Nonce: Nonce Value for this chunk */
 	uint8_t ect_nonce;
-
+	uint8_t fwd_tsn_cnt;
 	/*
 	 * part of the Highest sacked algorithm to be able to stroke counts
 	 * on ones that are FR'd.
@@ -445,6 +446,7 @@ struct sctp_stream_queue_pending {
 	uint8_t pr_sctp_on;
 	uint8_t sender_all_done;
 	uint8_t put_last_out;
+	uint8_t discard_rest;
 };
 
 /*
@@ -676,7 +678,7 @@ struct sctp_association {
 	/* primary destination to use */
 	struct sctp_nets *primary_destination;
 	/* For CMT */
-	struct sctp_nets *last_net_data_came_from;
+	struct sctp_nets *last_net_cmt_send_started;
 	/* last place I got a data chunk from */
 	struct sctp_nets *last_data_chunk_from;
 	/* last place I got a control from */
diff --git a/sys/netinet/sctp_sysctl.c b/sys/netinet/sctp_sysctl.c
index a64aec4..969ffdd 100644
--- a/sys/netinet/sctp_sysctl.c
+++ b/sys/netinet/sctp_sysctl.c
@@ -463,6 +463,7 @@ sctp_assoclist(SYSCTL_HANDLER_ARGS)
 				xraddr.cwnd = net->cwnd;
 				xraddr.flight_size = net->flight_size;
 				xraddr.mtu = net->mtu;
+				xraddr.rtt = net->rtt;
 				xraddr.start_time.tv_sec = (uint32_t) net->start_time.tv_sec;
 				xraddr.start_time.tv_usec = (uint32_t) net->start_time.tv_usec;
 				SCTP_INP_RUNLOCK(inp);
diff --git a/sys/netinet/sctp_sysctl.h b/sys/netinet/sctp_sysctl.h
index dd42d41..7419568 100644
--- a/sys/netinet/sctp_sysctl.h
+++ b/sys/netinet/sctp_sysctl.h
@@ -74,7 +74,7 @@ struct sctp_sysctl {
 	uint32_t sctp_nr_outgoing_streams_default;
 	uint32_t sctp_cmt_on_off;
 	uint32_t sctp_cmt_use_dac;
-/* EY 5/5/08 - nr_sack flag variable */
+	/* EY 5/5/08 - nr_sack flag variable */
 	uint32_t sctp_nr_sack_on_off;
 	uint32_t sctp_cmt_pf;
 	uint32_t sctp_use_cwnd_based_maxburst;
diff --git a/sys/netinet/sctp_timer.c b/sys/netinet/sctp_timer.c
index 1d2d5f2..477c0b1 100644
--- a/sys/netinet/sctp_timer.c
+++ b/sys/netinet/sctp_timer.c
@@ -767,7 +767,7 @@ start_again:
 						(void)sctp_release_pr_sctp_chunk(stcb,
 						    chk,
 						    (SCTP_RESPONSE_TO_USER_REQ | SCTP_NOTIFY_DATAGRAM_SENT),
-						    &stcb->asoc.sent_queue, SCTP_SO_NOT_LOCKED);
+						    SCTP_SO_NOT_LOCKED);
 					}
 					continue;
 				}
@@ -779,7 +779,7 @@ start_again:
 						(void)sctp_release_pr_sctp_chunk(stcb,
 						    chk,
 						    (SCTP_RESPONSE_TO_USER_REQ | SCTP_NOTIFY_DATAGRAM_SENT),
-						    &stcb->asoc.sent_queue, SCTP_SO_NOT_LOCKED);
+						    SCTP_SO_NOT_LOCKED);
 					}
 					continue;
 				}
diff --git a/sys/netinet/sctp_uio.h b/sys/netinet/sctp_uio.h
index b8d2ecd..efcd2c4 100644
--- a/sys/netinet/sctp_uio.h
+++ b/sys/netinet/sctp_uio.h
@@ -565,9 +565,9 @@ struct sctp_sack_info {
 struct sctp_cwnd_args {
 	struct sctp_nets *net;	/* network to *//* FIXME: LP64 issue */
 	uint32_t cwnd_new_value;/* cwnd in k */
-	uint32_t inflight;	/* flightsize in k */
 	uint32_t pseudo_cumack;
-	uint32_t cwnd_augment;	/* increment to it */
+	uint16_t inflight;	/* flightsize in k */
+	uint16_t cwnd_augment;	/* increment to it */
 	uint8_t meets_pseudo_cumack;
 	uint8_t need_new_pseudo_cumack;
 	uint8_t cnt_in_send;
@@ -986,7 +986,7 @@ struct xsctp_inpcb {
 	uint16_t local_port;
 	uint16_t qlen;
 	uint16_t maxqlen;
-	uint32_t extra_padding[8];	/* future */
+	uint32_t extra_padding[32];	/* future */
 };
 
 struct xsctp_tcb {
@@ -1017,14 +1017,14 @@ struct xsctp_tcb {
 	struct sctp_timeval discontinuity_time;	/* sctpAssocEntry 17  */
 	uint32_t peers_rwnd;
 	sctp_assoc_t assoc_id;	/* sctpAssocEntry 1   */
-	uint32_t extra_padding[8];	/* future */
+	uint32_t extra_padding[32];	/* future */
 };
 
 struct xsctp_laddr {
 	union sctp_sockstore address;	/* sctpAssocLocalAddrEntry 1/2 */
 	uint32_t last;
 	struct sctp_timeval start_time;	/* sctpAssocLocalAddrEntry 3   */
-	uint32_t extra_padding[8];	/* future */
+	uint32_t extra_padding[32];	/* future */
 };
 
 struct xsctp_raddr {
@@ -1041,7 +1041,8 @@ struct xsctp_raddr {
 	uint8_t confirmed;	/* */
 	uint8_t heartbeat_enabled;	/* sctpAssocLocalRemEntry 4   */
 	struct sctp_timeval start_time;	/* sctpAssocLocalRemEntry 8   */
-	uint32_t extra_padding[8];	/* future */
+	uint32_t rtt;
+	uint32_t extra_padding[32];	/* future */
 };
 
 #define SCTP_MAX_LOGGING_SIZE 30000
diff --git a/sys/netinet/sctp_usrreq.c b/sys/netinet/sctp_usrreq.c
index b3bfea5..c80cca0 100644
--- a/sys/netinet/sctp_usrreq.c
+++ b/sys/netinet/sctp_usrreq.c
@@ -126,6 +126,10 @@ sctp_pathmtu_adjustment(struct sctp_inpcb *inp,
 			 * since we sent to big of chunk
 			 */
 			chk->flags |= CHUNK_FLAGS_FRAGMENT_OK;
+			if (chk->sent < SCTP_DATAGRAM_RESEND) {
+				sctp_flight_size_decrease(chk);
+				sctp_total_flight_decrease(stcb, chk);
+			}
 			if (chk->sent != SCTP_DATAGRAM_RESEND) {
 				sctp_ucount_incr(stcb->asoc.sent_queue_retran_cnt);
 			}
@@ -140,8 +144,6 @@ sctp_pathmtu_adjustment(struct sctp_inpcb *inp,
 			}
 			/* Clear any time so NO RTT is being done */
 			chk->do_rtt = 0;
-			sctp_flight_size_decrease(chk);
-			sctp_total_flight_decrease(stcb, chk);
 		}
 	}
 }
diff --git a/sys/netinet/sctp_var.h b/sys/netinet/sctp_var.h
index 934af00..1ccccf3 100644
--- a/sys/netinet/sctp_var.h
+++ b/sys/netinet/sctp_var.h
@@ -96,7 +96,8 @@ extern struct pr_usrreqs sctp_usrreqs;
 
 #define sctp_alloc_a_strmoq(_stcb, _strmoq) { \
 	(_strmoq) = SCTP_ZONE_GET(SCTP_BASE_INFO(ipi_zone_strmoq), struct sctp_stream_queue_pending); \
-	if ((_strmoq)) { \
+         if ((_strmoq)) {			  \
+		memset(_strmoq, 0, sizeof(struct sctp_stream_queue_pending)); \
 		SCTP_INCR_STRMOQ_COUNT(); \
 		(_strmoq)->holds_key_ref = 0; \
  	} \
@@ -267,6 +268,7 @@ extern struct pr_usrreqs sctp_usrreqs;
 #else
 
 #define sctp_total_flight_decrease(stcb, tp1) do { \
+        tp1->window_probe = 0; \
 	if (stcb->asoc.total_flight >= tp1->book_size) { \
 		stcb->asoc.total_flight -= tp1->book_size; \
 		if (stcb->asoc.total_flight_count > 0) \
diff --git a/sys/netinet/sctputil.c b/sys/netinet/sctputil.c
index 1eac98a..aea7376 100644
--- a/sys/netinet/sctputil.c
+++ b/sys/netinet/sctputil.c
@@ -970,7 +970,7 @@ sctp_init_asoc(struct sctp_inpcb *m, struct sctp_tcb *stcb,
 	asoc->sent_queue_retran_cnt = 0;
 
 	/* for CMT */
-	asoc->last_net_data_came_from = NULL;
+	asoc->last_net_cmt_send_started = NULL;
 
 	/* This will need to be adjusted */
 	asoc->last_cwr_tsn = asoc->init_seq_number - 1;
@@ -1222,33 +1222,25 @@ sctp_expand_mapping_array(struct sctp_association *asoc, uint32_t needed)
 	SCTP_FREE(asoc->mapping_array, SCTP_M_MAP);
 	asoc->mapping_array = new_array;
 	asoc->mapping_array_size = new_size;
-	return (0);
-}
-
-/* EY - nr_sack version of the above method */
-int
-sctp_expand_nr_mapping_array(struct sctp_association *asoc, uint32_t needed)
-{
-	/* nr mapping array needs to grow */
-	uint8_t *new_array;
-	uint32_t new_size;
-
-	new_size = asoc->nr_mapping_array_size + ((needed + 7) / 8 + SCTP_NR_MAPPING_ARRAY_INCR);
-	SCTP_MALLOC(new_array, uint8_t *, new_size, SCTP_M_MAP);
-	if (new_array == NULL) {
-		/* can't get more, forget it */
-		SCTP_PRINTF("No memory for expansion of SCTP mapping array %d\n",
-		    new_size);
-		return (-1);
+	if (asoc->peer_supports_nr_sack) {
+		new_size = asoc->nr_mapping_array_size + ((needed + 7) / 8 + SCTP_NR_MAPPING_ARRAY_INCR);
+		SCTP_MALLOC(new_array, uint8_t *, new_size, SCTP_M_MAP);
+		if (new_array == NULL) {
+			/* can't get more, forget it */
+			SCTP_PRINTF("No memory for expansion of SCTP mapping array %d\n",
+			    new_size);
+			return (-1);
+		}
+		memset(new_array, 0, new_size);
+		memcpy(new_array, asoc->nr_mapping_array, asoc->nr_mapping_array_size);
+		SCTP_FREE(asoc->nr_mapping_array, SCTP_M_MAP);
+		asoc->nr_mapping_array = new_array;
+		asoc->nr_mapping_array_size = new_size;
 	}
-	memset(new_array, 0, new_size);
-	memcpy(new_array, asoc->nr_mapping_array, asoc->nr_mapping_array_size);
-	SCTP_FREE(asoc->nr_mapping_array, SCTP_M_MAP);
-	asoc->nr_mapping_array = new_array;
-	asoc->nr_mapping_array_size = new_size;
 	return (0);
 }
 
+
 #if defined(SCTP_USE_THREAD_BASED_ITERATOR)
 static void
 sctp_iterator_work(struct sctp_iterator *it)
@@ -2589,7 +2581,7 @@ sctp_calculate_rto(struct sctp_tcb *stcb,
 	/***************************/
 	/* 2. update RTTVAR & SRTT */
 	/***************************/
-	o_calctime = calc_time;
+	net->rtt = o_calctime = calc_time;
 	/* this is Van Jacobson's integer version */
 	if (net->RTO_measured) {
 		calc_time -= (net->lastsa >> SCTP_RTT_SHIFT);	/* take away 1/8th when
@@ -3686,10 +3678,10 @@ sctp_report_all_outbound(struct sctp_tcb *stcb, int holds_lock, int so_locked
 		while (chk) {
 			TAILQ_REMOVE(&asoc->sent_queue, chk, sctp_next);
 			asoc->sent_queue_cnt--;
-			sctp_free_bufspace(stcb, asoc, chk, 1);
-			sctp_ulp_notify(SCTP_NOTIFY_DG_FAIL, stcb,
-			    SCTP_NOTIFY_DATAGRAM_SENT, chk, so_locked);
-			if (chk->data) {
+			if (chk->data != NULL) {
+				sctp_free_bufspace(stcb, asoc, chk, 1);
+				sctp_ulp_notify(SCTP_NOTIFY_DG_FAIL, stcb,
+				    SCTP_NOTIFY_DATAGRAM_SENT, chk, so_locked);
 				sctp_m_freem(chk->data);
 				chk->data = NULL;
 			}
@@ -3704,9 +3696,10 @@ sctp_report_all_outbound(struct sctp_tcb *stcb, int holds_lock, int so_locked
 		while (chk) {
 			TAILQ_REMOVE(&asoc->send_queue, chk, sctp_next);
 			asoc->send_queue_cnt--;
-			sctp_free_bufspace(stcb, asoc, chk, 1);
-			sctp_ulp_notify(SCTP_NOTIFY_DG_FAIL, stcb, SCTP_NOTIFY_DATAGRAM_UNSENT, chk, so_locked);
-			if (chk->data) {
+			if (chk->data != NULL) {
+				sctp_free_bufspace(stcb, asoc, chk, 1);
+				sctp_ulp_notify(SCTP_NOTIFY_DG_FAIL, stcb,
+				    SCTP_NOTIFY_DATAGRAM_UNSENT, chk, so_locked);
 				sctp_m_freem(chk->data);
 				chk->data = NULL;
 			}
@@ -4630,65 +4623,42 @@ sctp_free_bufspace(struct sctp_tcb *stcb, struct sctp_association *asoc,
 
 int
 sctp_release_pr_sctp_chunk(struct sctp_tcb *stcb, struct sctp_tmit_chunk *tp1,
-    int reason, struct sctpchunk_listhead *queue, int so_locked
+    int reason, int so_locked
 #if !defined(__APPLE__) && !defined(SCTP_SO_LOCK_TESTING)
     SCTP_UNUSED
 #endif
 )
 {
+	struct sctp_stream_out *strq;
+	struct sctp_tmit_chunk *chk = NULL;
+	struct sctp_stream_queue_pending *sp;
+	uint16_t stream = 0, seq = 0;
+	uint8_t foundeom = 0;
 	int ret_sz = 0;
 	int notdone;
-	uint8_t foundeom = 0;
+	int do_wakeup_routine = 0;
 
+	stream = tp1->rec.data.stream_number;
+	seq = tp1->rec.data.stream_seq;
 	do {
 		ret_sz += tp1->book_size;
-		tp1->sent = SCTP_FORWARD_TSN_SKIP;
-		if (tp1->data) {
-#if defined (__APPLE__) || defined(SCTP_SO_LOCK_TESTING)
-			struct socket *so;
-
-#endif
+		if (tp1->data != NULL) {
+			if (tp1->sent < SCTP_DATAGRAM_RESEND) {
+				sctp_flight_size_decrease(tp1);
+				sctp_total_flight_decrease(stcb, tp1);
+			}
 			sctp_free_bufspace(stcb, &stcb->asoc, tp1, 1);
-			sctp_flight_size_decrease(tp1);
-			sctp_total_flight_decrease(stcb, tp1);
+			stcb->asoc.peers_rwnd += tp1->send_size;
+			stcb->asoc.peers_rwnd += SCTP_BASE_SYSCTL(sctp_peer_chunk_oh);
 			sctp_ulp_notify(SCTP_NOTIFY_DG_FAIL, stcb, reason, tp1, so_locked);
 			sctp_m_freem(tp1->data);
 			tp1->data = NULL;
-#if defined (__APPLE__) || defined(SCTP_SO_LOCK_TESTING)
-			so = SCTP_INP_SO(stcb->sctp_ep);
-			if (!so_locked) {
-				atomic_add_int(&stcb->asoc.refcnt, 1);
-				SCTP_TCB_UNLOCK(stcb);
-				SCTP_SOCKET_LOCK(so, 1);
-				SCTP_TCB_LOCK(stcb);
-				atomic_subtract_int(&stcb->asoc.refcnt, 1);
-				if (stcb->asoc.state & SCTP_STATE_CLOSED_SOCKET) {
-					/*
-					 * assoc was freed while we were
-					 * unlocked
-					 */
-					SCTP_SOCKET_UNLOCK(so, 1);
-					return (ret_sz);
-				}
-			}
-#endif
-			sctp_sowwakeup(stcb->sctp_ep, stcb->sctp_socket);
-#if defined (__APPLE__) || defined(SCTP_SO_LOCK_TESTING)
-			if (!so_locked) {
-				SCTP_SOCKET_UNLOCK(so, 1);
+			do_wakeup_routine = 1;
+			if (PR_SCTP_BUF_ENABLED(tp1->flags)) {
+				stcb->asoc.sent_queue_cnt_removeable--;
 			}
-#endif
-		}
-		if (PR_SCTP_BUF_ENABLED(tp1->flags)) {
-			stcb->asoc.sent_queue_cnt_removeable--;
-		}
-		if (queue == &stcb->asoc.send_queue) {
-			TAILQ_REMOVE(&stcb->asoc.send_queue, tp1, sctp_next);
-			/* on to the sent queue */
-			TAILQ_INSERT_TAIL(&stcb->asoc.sent_queue, tp1,
-			    sctp_next);
-			stcb->asoc.sent_queue_cnt++;
 		}
+		tp1->sent = SCTP_FORWARD_TSN_SKIP;
 		if ((tp1->rec.data.rcv_flags & SCTP_DATA_NOT_FRAG) ==
 		    SCTP_DATA_NOT_FRAG) {
 			/* not frag'ed we ae done   */
@@ -4707,22 +4677,151 @@ sctp_release_pr_sctp_chunk(struct sctp_tcb *stcb, struct sctp_tmit_chunk *tp1,
 			tp1 = TAILQ_NEXT(tp1, sctp_next);
 		}
 	} while (tp1 && notdone);
-	if ((foundeom == 0) && (queue == &stcb->asoc.sent_queue)) {
+	if (foundeom == 0) {
 		/*
 		 * The multi-part message was scattered across the send and
 		 * sent queue.
 		 */
+next_on_sent:
 		tp1 = TAILQ_FIRST(&stcb->asoc.send_queue);
 		/*
 		 * recurse throught the send_queue too, starting at the
 		 * beginning.
 		 */
-		if (tp1) {
-			ret_sz += sctp_release_pr_sctp_chunk(stcb, tp1, reason,
-			    &stcb->asoc.send_queue, so_locked);
-		} else {
-			SCTP_PRINTF("hmm, nothing on the send queue and no EOM?\n");
+		if ((tp1) &&
+		    (tp1->rec.data.stream_number == stream) &&
+		    (tp1->rec.data.stream_seq == seq)
+		    ) {
+			/*
+			 * save to chk in case we have some on stream out
+			 * queue. If so and we have an un-transmitted one we
+			 * don't have to fudge the TSN.
+			 */
+			chk = tp1;
+			ret_sz += tp1->book_size;
+			sctp_ulp_notify(SCTP_NOTIFY_DG_FAIL, stcb, reason, tp1, so_locked);
+			sctp_free_bufspace(stcb, &stcb->asoc, tp1, 1);
+			sctp_m_freem(tp1->data);
+			/* No flight involved here book the size to 0 */
+			tp1->book_size = 0;
+			tp1->data = NULL;
+			if (tp1->rec.data.rcv_flags & SCTP_DATA_LAST_FRAG) {
+				foundeom = 1;
+			}
+			do_wakeup_routine = 1;
+			tp1->sent = SCTP_FORWARD_TSN_SKIP;
+			TAILQ_REMOVE(&stcb->asoc.send_queue, tp1, sctp_next);
+			/*
+			 * on to the sent queue so we can wait for it to be
+			 * passed by.
+			 */
+			TAILQ_INSERT_TAIL(&stcb->asoc.sent_queue, tp1,
+			    sctp_next);
+			stcb->asoc.send_queue_cnt--;
+			stcb->asoc.sent_queue_cnt++;
+			goto next_on_sent;
+		}
+	}
+	if (foundeom == 0) {
+		/*
+		 * Still no eom found. That means there is stuff left on the
+		 * stream out queue.. yuck.
+		 */
+		strq = &stcb->asoc.strmout[stream];
+		SCTP_TCB_SEND_LOCK(stcb);
+		sp = TAILQ_FIRST(&strq->outqueue);
+		while (sp->strseq <= seq) {
+			/* Check if its our SEQ */
+			if (sp->strseq == seq) {
+				sp->discard_rest = 1;
+				/*
+				 * We may need to put a chunk on the queue
+				 * that holds the TSN that would have been
+				 * sent with the LAST bit.
+				 */
+				if (chk == NULL) {
+					/* Yep, we have to */
+					sctp_alloc_a_chunk(stcb, chk);
+					if (chk == NULL) {
+						/*
+						 * we are hosed. All we can
+						 * do is nothing.. which
+						 * will cause an abort if
+						 * the peer is paying
+						 * attention.
+						 */
+						goto oh_well;
+					}
+					memset(chk, 0, sizeof(*chk));
+					chk->rec.data.rcv_flags = SCTP_DATA_LAST_FRAG;
+					chk->sent = SCTP_FORWARD_TSN_SKIP;
+					chk->asoc = &stcb->asoc;
+					chk->rec.data.stream_seq = sp->strseq;
+					chk->rec.data.stream_number = sp->stream;
+					chk->rec.data.payloadtype = sp->ppid;
+					chk->rec.data.context = sp->context;
+					chk->flags = sp->act_flags;
+					chk->addr_over = sp->addr_over;
+					chk->whoTo = sp->net;
+					atomic_add_int(&chk->whoTo->ref_count, 1);
+					chk->rec.data.TSN_seq = atomic_fetchadd_int(&stcb->asoc.sending_seq, 1);
+					stcb->asoc.pr_sctp_cnt++;
+					chk->pr_sctp_on = 1;
+					TAILQ_INSERT_TAIL(&stcb->asoc.sent_queue, chk, sctp_next);
+					stcb->asoc.sent_queue_cnt++;
+					stcb->asoc.pr_sctp_cnt++;
+				} else {
+					chk->rec.data.rcv_flags |= SCTP_DATA_LAST_FRAG;
+				}
+		oh_well:
+				if (sp->data) {
+					/*
+					 * Pull any data to free up the SB
+					 * and allow sender to "add more"
+					 * whilc we will throw away :-)
+					 */
+					sctp_free_spbufspace(stcb, &stcb->asoc,
+					    sp);
+					ret_sz += sp->length;
+					do_wakeup_routine = 1;
+					sp->some_taken = 1;
+					sctp_m_freem(sp->data);
+					sp->length = 0;
+					sp->data = NULL;
+					sp->tail_mbuf = NULL;
+				}
+				break;
+			} else {
+				/* Next one please */
+				sp = TAILQ_NEXT(sp, next);
+			}
+		}		/* End while */
+		SCTP_TCB_SEND_UNLOCK(stcb);
+	}
+	if (do_wakeup_routine) {
+#if defined (__APPLE__) || defined(SCTP_SO_LOCK_TESTING)
+		struct socket *so;
+
+		so = SCTP_INP_SO(stcb->sctp_ep);
+		if (!so_locked) {
+			atomic_add_int(&stcb->asoc.refcnt, 1);
+			SCTP_TCB_UNLOCK(stcb);
+			SCTP_SOCKET_LOCK(so, 1);
+			SCTP_TCB_LOCK(stcb);
+			atomic_subtract_int(&stcb->asoc.refcnt, 1);
+			if (stcb->asoc.state & SCTP_STATE_CLOSED_SOCKET) {
+				/* assoc was freed while we were unlocked */
+				SCTP_SOCKET_UNLOCK(so, 1);
+				return (ret_sz);
+			}
 		}
+#endif
+		sctp_sowwakeup(stcb->sctp_ep, stcb->sctp_socket);
+#if defined (__APPLE__) || defined(SCTP_SO_LOCK_TESTING)
+		if (!so_locked) {
+			SCTP_SOCKET_UNLOCK(so, 1);
+		}
+#endif
 	}
 	return (ret_sz);
 }
diff --git a/sys/netinet/sctputil.h b/sys/netinet/sctputil.h
index 56c6729..c5a2ad9 100644
--- a/sys/netinet/sctputil.h
+++ b/sys/netinet/sctputil.h
@@ -237,7 +237,7 @@ sctp_notify_partial_delivery_indication(struct sctp_tcb *stcb,
 
 int
 sctp_release_pr_sctp_chunk(struct sctp_tcb *, struct sctp_tmit_chunk *,
-    int, struct sctpchunk_listhead *, int
+    int, int
 #if !defined(__APPLE__) && !defined(SCTP_SO_LOCK_TESTING)
     SCTP_UNUSED
 #endif
@@ -287,7 +287,6 @@ do { \
 #define sctp_free_spbufspace(stcb, asoc, sp)  \
 do { \
  	if (sp->data != NULL) { \
-                atomic_subtract_int(&(asoc)->chunks_on_out_queue, 1); \
 		if ((asoc)->total_output_queue_size >= sp->length) { \
 			atomic_subtract_int(&(asoc)->total_output_queue_size, sp->length); \
 		} else { \
diff --git a/sys/netinet/tcp_hostcache.c b/sys/netinet/tcp_hostcache.c
index c9b5b8c..71f4626 100644
--- a/sys/netinet/tcp_hostcache.c
+++ b/sys/netinet/tcp_hostcache.c
@@ -341,7 +341,7 @@ tcp_hc_insert(struct in_conninfo *inc)
 		TAILQ_REMOVE(&hc_head->hch_bucket, hc_entry, rmx_q);
 		V_tcp_hostcache.hashbase[hash].hch_length--;
 		V_tcp_hostcache.cache_count--;
-		V_tcpstat.tcps_hc_bucketoverflow++;
+		TCPSTAT_INC(tcps_hc_bucketoverflow);
 #if 0
 		uma_zfree(V_tcp_hostcache.zone, hc_entry);
 #endif
@@ -373,7 +373,7 @@ tcp_hc_insert(struct in_conninfo *inc)
 	TAILQ_INSERT_HEAD(&hc_head->hch_bucket, hc_entry, rmx_q);
 	V_tcp_hostcache.hashbase[hash].hch_length++;
 	V_tcp_hostcache.cache_count++;
-	V_tcpstat.tcps_hc_added++;
+	TCPSTAT_INC(tcps_hc_added);
 
 	return hc_entry;
 }
@@ -508,7 +508,7 @@ tcp_hc_update(struct in_conninfo *inc, struct hc_metrics_lite *hcml)
 		else
 			hc_entry->rmx_rtt =
 			    (hc_entry->rmx_rtt + hcml->rmx_rtt) / 2;
-		V_tcpstat.tcps_cachedrtt++;
+		TCPSTAT_INC(tcps_cachedrtt);
 	}
 	if (hcml->rmx_rttvar != 0) {
 	        if (hc_entry->rmx_rttvar == 0)
@@ -516,7 +516,7 @@ tcp_hc_update(struct in_conninfo *inc, struct hc_metrics_lite *hcml)
 		else
 			hc_entry->rmx_rttvar =
 			    (hc_entry->rmx_rttvar + hcml->rmx_rttvar) / 2;
-		V_tcpstat.tcps_cachedrttvar++;
+		TCPSTAT_INC(tcps_cachedrttvar);
 	}
 	if (hcml->rmx_ssthresh != 0) {
 		if (hc_entry->rmx_ssthresh == 0)
@@ -524,7 +524,7 @@ tcp_hc_update(struct in_conninfo *inc, struct hc_metrics_lite *hcml)
 		else
 			hc_entry->rmx_ssthresh =
 			    (hc_entry->rmx_ssthresh + hcml->rmx_ssthresh) / 2;
-		V_tcpstat.tcps_cachedssthresh++;
+		TCPSTAT_INC(tcps_cachedssthresh);
 	}
 	if (hcml->rmx_bandwidth != 0) {
 		if (hc_entry->rmx_bandwidth == 0)
@@ -532,7 +532,7 @@ tcp_hc_update(struct in_conninfo *inc, struct hc_metrics_lite *hcml)
 		else
 			hc_entry->rmx_bandwidth =
 			    (hc_entry->rmx_bandwidth + hcml->rmx_bandwidth) / 2;
-		/* V_tcpstat.tcps_cachedbandwidth++; */
+		/* TCPSTAT_INC(tcps_cachedbandwidth); */
 	}
 	if (hcml->rmx_cwnd != 0) {
 		if (hc_entry->rmx_cwnd == 0)
@@ -540,7 +540,7 @@ tcp_hc_update(struct in_conninfo *inc, struct hc_metrics_lite *hcml)
 		else
 			hc_entry->rmx_cwnd =
 			    (hc_entry->rmx_cwnd + hcml->rmx_cwnd) / 2;
-		/* V_tcpstat.tcps_cachedcwnd++; */
+		/* TCPSTAT_INC(tcps_cachedcwnd); */
 	}
 	if (hcml->rmx_sendpipe != 0) {
 		if (hc_entry->rmx_sendpipe == 0)
@@ -548,7 +548,7 @@ tcp_hc_update(struct in_conninfo *inc, struct hc_metrics_lite *hcml)
 		else
 			hc_entry->rmx_sendpipe =
 			    (hc_entry->rmx_sendpipe + hcml->rmx_sendpipe) /2;
-		/* V_tcpstat.tcps_cachedsendpipe++; */
+		/* TCPSTAT_INC(tcps_cachedsendpipe); */
 	}
 	if (hcml->rmx_recvpipe != 0) {
 		if (hc_entry->rmx_recvpipe == 0)
@@ -556,7 +556,7 @@ tcp_hc_update(struct in_conninfo *inc, struct hc_metrics_lite *hcml)
 		else
 			hc_entry->rmx_recvpipe =
 			    (hc_entry->rmx_recvpipe + hcml->rmx_recvpipe) /2;
-		/* V_tcpstat.tcps_cachedrecvpipe++; */
+		/* TCPSTAT_INC(tcps_cachedrecvpipe); */
 	}
 
 	TAILQ_REMOVE(&hc_entry->rmx_head->hch_bucket, hc_entry, rmx_q);
diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c
index d3b91b6..d4a9f70 100644
--- a/sys/netinet/tcp_input.c
+++ b/sys/netinet/tcp_input.c
@@ -352,7 +352,7 @@ tcp_input(struct mbuf *m, int off0)
 #endif
 
 	to.to_flags = 0;
-	V_tcpstat.tcps_rcvtotal++;
+	TCPSTAT_INC(tcps_rcvtotal);
 
 	if (isipv6) {
 #ifdef INET6
@@ -360,7 +360,7 @@ tcp_input(struct mbuf *m, int off0)
 		ip6 = mtod(m, struct ip6_hdr *);
 		tlen = sizeof(*ip6) + ntohs(ip6->ip6_plen) - off0;
 		if (in6_cksum(m, IPPROTO_TCP, off0, tlen)) {
-			V_tcpstat.tcps_rcvbadsum++;
+			TCPSTAT_INC(tcps_rcvbadsum);
 			goto drop;
 		}
 		th = (struct tcphdr *)((caddr_t)ip6 + off0);
@@ -392,7 +392,7 @@ tcp_input(struct mbuf *m, int off0)
 		if (m->m_len < sizeof (struct tcpiphdr)) {
 			if ((m = m_pullup(m, sizeof (struct tcpiphdr)))
 			    == NULL) {
-				V_tcpstat.tcps_rcvshort++;
+				TCPSTAT_INC(tcps_rcvshort);
 				return;
 			}
 		}
@@ -426,7 +426,7 @@ tcp_input(struct mbuf *m, int off0)
 			th->th_sum = in_cksum(m, len);
 		}
 		if (th->th_sum) {
-			V_tcpstat.tcps_rcvbadsum++;
+			TCPSTAT_INC(tcps_rcvbadsum);
 			goto drop;
 		}
 		/* Re-initialization for later version check */
@@ -446,7 +446,7 @@ tcp_input(struct mbuf *m, int off0)
 	 */
 	off = th->th_off << 2;
 	if (off < sizeof (struct tcphdr) || off > tlen) {
-		V_tcpstat.tcps_rcvbadoff++;
+		TCPSTAT_INC(tcps_rcvbadoff);
 		goto drop;
 	}
 	tlen -= off;	/* tlen is used instead of ti->ti_len */
@@ -461,7 +461,7 @@ tcp_input(struct mbuf *m, int off0)
 			if (m->m_len < sizeof(struct ip) + off) {
 				if ((m = m_pullup(m, sizeof (struct ip) + off))
 				    == NULL) {
-					V_tcpstat.tcps_rcvshort++;
+					TCPSTAT_INC(tcps_rcvshort);
 					return;
 				}
 				ip = mtod(m, struct ip *);
@@ -594,7 +594,14 @@ findpcb:
 		goto dropwithreset;
 	}
 	INP_WLOCK(inp);
-
+	if (!(inp->inp_flags & INP_HW_FLOWID)
+	    && (m->m_flags & M_FLOWID)
+	    && ((inp->inp_socket == NULL)
+		|| !(inp->inp_socket->so_options & SO_ACCEPTCONN))) {
+		inp->inp_flags |= INP_HW_FLOWID;
+		inp->inp_flags &= ~INP_SW_FLOWID;
+		inp->inp_flowid = m->m_pkthdr.flowid;
+	}
 #ifdef IPSEC
 #ifdef INET6
 	if (isipv6 && ipsec6_in_reject(m, inp)) {
@@ -635,7 +642,7 @@ findpcb:
 	 * tried to free the inpcb, in which case we need to loop back and
 	 * try to find a new inpcb to deliver to.
 	 */
-	if (inp->inp_vflag & INP_TIMEWAIT) {
+	if (inp->inp_flags & INP_TIMEWAIT) {
 		KASSERT(ti_locked == TI_RLOCKED || ti_locked == TI_WLOCKED,
 		    ("%s: INP_TIMEWAIT ti_locked %d", __func__, ti_locked));
 
@@ -853,7 +860,7 @@ findpcb:
 				log(LOG_DEBUG, "%s; %s: Listen socket: "
 				    "SYN is missing, segment ignored\n",
 				    s, __func__);
-			V_tcpstat.tcps_badsyn++;
+			TCPSTAT_INC(tcps_badsyn);
 			goto dropunlock;
 		}
 		/*
@@ -865,7 +872,7 @@ findpcb:
 				    "SYN|ACK invalid, segment rejected\n",
 				    s, __func__);
 			syncache_badack(&inc);	/* XXX: Not needed! */
-			V_tcpstat.tcps_badsyn++;
+			TCPSTAT_INC(tcps_badsyn);
 			rstreason = BANDLIM_RST_OPENPORT;
 			goto dropwithreset;
 		}
@@ -885,7 +892,7 @@ findpcb:
 				log(LOG_DEBUG, "%s; %s: Listen socket: "
 				    "SYN|FIN segment ignored (based on "
 				    "sysctl setting)\n", s, __func__);
-			V_tcpstat.tcps_badsyn++;
+			TCPSTAT_INC(tcps_badsyn);
                 	goto dropunlock;
 		}
 		/*
@@ -1148,13 +1155,13 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 		switch (iptos & IPTOS_ECN_MASK) {
 		case IPTOS_ECN_CE:
 			tp->t_flags |= TF_ECN_SND_ECE;
-			V_tcpstat.tcps_ecn_ce++;
+			TCPSTAT_INC(tcps_ecn_ce);
 			break;
 		case IPTOS_ECN_ECT0:
-			V_tcpstat.tcps_ecn_ect0++;
+			TCPSTAT_INC(tcps_ecn_ect0);
 			break;
 		case IPTOS_ECN_ECT1:
-			V_tcpstat.tcps_ecn_ect1++;
+			TCPSTAT_INC(tcps_ecn_ect1);
 			break;
 		}
 
@@ -1167,7 +1174,7 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 		 */
 		if ((thflags & TH_ECE) &&
 		    SEQ_LEQ(th->th_ack, tp->snd_recover)) {
-			V_tcpstat.tcps_ecn_rcwnd++;
+			TCPSTAT_INC(tcps_ecn_rcwnd);
 			tcp_congestion_exp(tp);
 		}
 	}
@@ -1284,14 +1291,14 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 					    __func__, ti_locked);
 				ti_locked = TI_UNLOCKED;
 
-				++V_tcpstat.tcps_predack;
+				TCPSTAT_INC(tcps_predack);
 
 				/*
 				 * "bad retransmit" recovery.
 				 */
 				if (tp->t_rxtshift == 1 &&
 				    ticks < tp->t_badrxtwin) {
-					++V_tcpstat.tcps_sndrexmitbad;
+					TCPSTAT_INC(tcps_sndrexmitbad);
 					tp->snd_cwnd = tp->snd_cwnd_prev;
 					tp->snd_ssthresh =
 					    tp->snd_ssthresh_prev;
@@ -1327,8 +1334,8 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 				}
 				tcp_xmit_bandwidth_limit(tp, th->th_ack);
 				acked = th->th_ack - tp->snd_una;
-				V_tcpstat.tcps_rcvackpack++;
-				V_tcpstat.tcps_rcvackbyte += acked;
+				TCPSTAT_INC(tcps_rcvackpack);
+				TCPSTAT_ADD(tcps_rcvackbyte, acked);
 				sbdrop(&so->so_snd, acked);
 				if (SEQ_GT(tp->snd_una, tp->snd_recover) &&
 				    SEQ_LEQ(th->th_ack, tp->snd_recover))
@@ -1389,7 +1396,7 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 			/* Clean receiver SACK report if present */
 			if ((tp->t_flags & TF_SACK_PERMIT) && tp->rcv_numsacks)
 				tcp_clean_sackreport(tp);
-			++V_tcpstat.tcps_preddat;
+			TCPSTAT_INC(tcps_preddat);
 			tp->rcv_nxt += tlen;
 			/*
 			 * Pull snd_wl1 up to prevent seq wrap relative to
@@ -1401,8 +1408,8 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 			 * rcv_nxt.
 			 */
 			tp->rcv_up = tp->rcv_nxt;
-			V_tcpstat.tcps_rcvpack++;
-			V_tcpstat.tcps_rcvbyte += tlen;
+			TCPSTAT_INC(tcps_rcvpack);
+			TCPSTAT_ADD(tcps_rcvbyte, tlen);
 			ND6_HINT(tp);	/* Some progress has been made */
 #ifdef TCPDEBUG
 			if (so->so_options & SO_DEBUG)
@@ -1552,7 +1559,7 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 		tp->irs = th->th_seq;
 		tcp_rcvseqinit(tp);
 		if (thflags & TH_ACK) {
-			V_tcpstat.tcps_connects++;
+			TCPSTAT_INC(tcps_connects);
 			soisconnected(so);
 #ifdef MAC
 			SOCK_LOCK(so);
@@ -1578,7 +1585,7 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 
 			if ((thflags & TH_ECE) && V_tcp_do_ecn) {
 				tp->t_flags |= TF_ECN_PERMIT;
-				V_tcpstat.tcps_ecn_shs++;
+				TCPSTAT_INC(tcps_ecn_shs);
 			}
 			
 			/*
@@ -1628,8 +1635,8 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 			m_adj(m, -todrop);
 			tlen = tp->rcv_wnd;
 			thflags &= ~TH_FIN;
-			V_tcpstat.tcps_rcvpackafterwin++;
-			V_tcpstat.tcps_rcvbyteafterwin += todrop;
+			TCPSTAT_INC(tcps_rcvpackafterwin);
+			TCPSTAT_ADD(tcps_rcvbyteafterwin, todrop);
 		}
 		tp->snd_wl1 = th->th_seq - 1;
 		tp->rcv_up = th->th_seq;
@@ -1733,7 +1740,7 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 				    SEQ_LEQ(th->th_seq, tp->rcv_nxt + 1)) &&
 				    !(SEQ_GEQ(th->th_seq, tp->last_ack_sent - 1) &&
 				    SEQ_LEQ(th->th_seq, tp->last_ack_sent + 1))) {
-					V_tcpstat.tcps_badrst++;
+					TCPSTAT_INC(tcps_badrst);
 					goto drop;
 				}
 				/* FALLTHROUGH */
@@ -1748,7 +1755,7 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 				INP_INFO_WLOCK_ASSERT(&V_tcbinfo);
 
 				tp->t_state = TCPS_CLOSED;
-				V_tcpstat.tcps_drops++;
+				TCPSTAT_INC(tcps_drops);
 				tp = tcp_close(tp);
 				break;
 
@@ -1788,9 +1795,9 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 			 */
 			tp->ts_recent = 0;
 		} else {
-			V_tcpstat.tcps_rcvduppack++;
-			V_tcpstat.tcps_rcvdupbyte += tlen;
-			V_tcpstat.tcps_pawsdrop++;
+			TCPSTAT_INC(tcps_rcvduppack);
+			TCPSTAT_ADD(tcps_rcvdupbyte, tlen);
+			TCPSTAT_INC(tcps_pawsdrop);
 			if (tlen)
 				goto dropafterack;
 			goto drop;
@@ -1838,11 +1845,11 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 			 */
 			tp->t_flags |= TF_ACKNOW;
 			todrop = tlen;
-			V_tcpstat.tcps_rcvduppack++;
-			V_tcpstat.tcps_rcvdupbyte += todrop;
+			TCPSTAT_INC(tcps_rcvduppack);
+			TCPSTAT_ADD(tcps_rcvdupbyte, todrop);
 		} else {
-			V_tcpstat.tcps_rcvpartduppack++;
-			V_tcpstat.tcps_rcvpartdupbyte += todrop;
+			TCPSTAT_INC(tcps_rcvpartduppack);
+			TCPSTAT_ADD(tcps_rcvpartdupbyte, todrop);
 		}
 		drop_hdrlen += todrop;	/* drop from the top afterwards */
 		th->th_seq += todrop;
@@ -1874,7 +1881,7 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 			free(s, M_TCPLOG);
 		}
 		tp = tcp_close(tp);
-		V_tcpstat.tcps_rcvafterclose++;
+		TCPSTAT_INC(tcps_rcvafterclose);
 		rstreason = BANDLIM_UNLIMITED;
 		goto dropwithreset;
 	}
@@ -1885,9 +1892,9 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 	 */
 	todrop = (th->th_seq + tlen) - (tp->rcv_nxt + tp->rcv_wnd);
 	if (todrop > 0) {
-		V_tcpstat.tcps_rcvpackafterwin++;
+		TCPSTAT_INC(tcps_rcvpackafterwin);
 		if (todrop >= tlen) {
-			V_tcpstat.tcps_rcvbyteafterwin += tlen;
+			TCPSTAT_ADD(tcps_rcvbyteafterwin, tlen);
 			/*
 			 * If window is closed can only take segments at
 			 * window edge, and have to drop data and PUSH from
@@ -1897,11 +1904,11 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 			 */
 			if (tp->rcv_wnd == 0 && th->th_seq == tp->rcv_nxt) {
 				tp->t_flags |= TF_ACKNOW;
-				V_tcpstat.tcps_rcvwinprobe++;
+				TCPSTAT_INC(tcps_rcvwinprobe);
 			} else
 				goto dropafterack;
 		} else
-			V_tcpstat.tcps_rcvbyteafterwin += todrop;
+			TCPSTAT_ADD(tcps_rcvbyteafterwin, todrop);
 		m_adj(m, -todrop);
 		tlen -= todrop;
 		thflags &= ~(TH_PUSH|TH_FIN);
@@ -1974,7 +1981,7 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 	 */
 	case TCPS_SYN_RECEIVED:
 
-		V_tcpstat.tcps_connects++;
+		TCPSTAT_INC(tcps_connects);
 		soisconnected(so);
 		/* Do window scaling? */
 		if ((tp->t_flags & (TF_RCVD_SCALE|TF_REQ_SCALE)) ==
@@ -2020,7 +2027,7 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 	case TCPS_CLOSING:
 	case TCPS_LAST_ACK:
 		if (SEQ_GT(th->th_ack, tp->snd_max)) {
-			V_tcpstat.tcps_rcvacktoomuch++;
+			TCPSTAT_INC(tcps_rcvacktoomuch);
 			goto dropafterack;
 		}
 		if ((tp->t_flags & TF_SACK_PERMIT) &&
@@ -2029,7 +2036,7 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 			tcp_sack_doack(tp, &to, th->th_ack);
 		if (SEQ_LEQ(th->th_ack, tp->snd_una)) {
 			if (tlen == 0 && tiwin == tp->snd_wnd) {
-				V_tcpstat.tcps_rcvdupack++;
+				TCPSTAT_INC(tcps_rcvdupack);
 				/*
 				 * If we have outstanding data (other than
 				 * a window probe), this is a completely
@@ -2112,7 +2119,8 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 					tcp_timer_activate(tp, TT_REXMT, 0);
 					tp->t_rtttime = 0;
 					if (tp->t_flags & TF_SACK_PERMIT) {
-						V_tcpstat.tcps_sack_recovery_episode++;
+						TCPSTAT_INC(
+						    tcps_sack_recovery_episode);
 						tp->sack_newdata = tp->snd_nxt;
 						tp->snd_cwnd = tp->t_maxseg;
 						(void) tcp_output(tp);
@@ -2234,8 +2242,8 @@ process_ACK:
 		INP_WLOCK_ASSERT(tp->t_inpcb);
 
 		acked = th->th_ack - tp->snd_una;
-		V_tcpstat.tcps_rcvackpack++;
-		V_tcpstat.tcps_rcvackbyte += acked;
+		TCPSTAT_INC(tcps_rcvackpack);
+		TCPSTAT_ADD(tcps_rcvackbyte, acked);
 
 		/*
 		 * If we just performed our first retransmit, and the ACK
@@ -2245,7 +2253,7 @@ process_ACK:
 		 * we left off.
 		 */
 		if (tp->t_rxtshift == 1 && ticks < tp->t_badrxtwin) {
-			++V_tcpstat.tcps_sndrexmitbad;
+			TCPSTAT_INC(tcps_sndrexmitbad);
 			tp->snd_cwnd = tp->snd_cwnd_prev;
 			tp->snd_ssthresh = tp->snd_ssthresh_prev;
 			tp->snd_recover = tp->snd_recover_prev;
@@ -2469,7 +2477,7 @@ step6:
 		/* keep track of pure window updates */
 		if (tlen == 0 &&
 		    tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd)
-			V_tcpstat.tcps_rcvwinupd++;
+			TCPSTAT_INC(tcps_rcvwinupd);
 		tp->snd_wnd = tiwin;
 		tp->snd_wl1 = th->th_seq;
 		tp->snd_wl2 = th->th_ack;
@@ -2579,8 +2587,8 @@ dodata:							/* XXX */
 				tp->t_flags |= TF_ACKNOW;
 			tp->rcv_nxt += tlen;
 			thflags = th->th_flags & TH_FIN;
-			V_tcpstat.tcps_rcvpack++;
-			V_tcpstat.tcps_rcvbyte += tlen;
+			TCPSTAT_INC(tcps_rcvpack);
+			TCPSTAT_ADD(tcps_rcvbyte, tlen);
 			ND6_HINT(tp);
 			SOCKBUF_LOCK(&so->so_rcv);
 			if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
@@ -2933,7 +2941,7 @@ tcp_dooptions(struct tcpopt *to, u_char *cp, int cnt, int flags)
 			to->to_flags |= TOF_SACK;
 			to->to_nsacks = (optlen - 2) / TCPOLEN_SACK;
 			to->to_sacks = cp + 2;
-			V_tcpstat.tcps_sack_rcv_blocks++;
+			TCPSTAT_INC(tcps_sack_rcv_blocks);
 			break;
 		default:
 			continue;
@@ -2988,7 +2996,7 @@ tcp_xmit_timer(struct tcpcb *tp, int rtt)
 
 	INP_WLOCK_ASSERT(tp->t_inpcb);
 
-	V_tcpstat.tcps_rttupdated++;
+	TCPSTAT_INC(tcps_rttupdated);
 	tp->t_rttupdated++;
 	if (tp->t_srtt != 0) {
 		/*
@@ -3310,10 +3318,10 @@ tcp_mss(struct tcpcb *tp, int offer)
 	if (tp->t_srtt == 0 && (rtt = metrics.rmx_rtt)) {
 		tp->t_srtt = rtt;
 		tp->t_rttbest = tp->t_srtt + TCP_RTT_SCALE;
-		V_tcpstat.tcps_usedrtt++;
+		TCPSTAT_INC(tcps_usedrtt);
 		if (metrics.rmx_rttvar) {
 			tp->t_rttvar = metrics.rmx_rttvar;
-			V_tcpstat.tcps_usedrttvar++;
+			TCPSTAT_INC(tcps_usedrttvar);
 		} else {
 			/* default variation is +- 1 rtt */
 			tp->t_rttvar =
@@ -3331,7 +3339,7 @@ tcp_mss(struct tcpcb *tp, int offer)
 		 * threshold to no less than 2*mss.
 		 */
 		tp->snd_ssthresh = max(2 * mss, metrics.rmx_ssthresh);
-		V_tcpstat.tcps_usedssthresh++;
+		TCPSTAT_INC(tcps_usedssthresh);
 	}
 	if (metrics.rmx_bandwidth)
 		tp->snd_bandwidth = metrics.rmx_bandwidth;
diff --git a/sys/netinet/tcp_output.c b/sys/netinet/tcp_output.c
index 2347305..d014fdd 100644
--- a/sys/netinet/tcp_output.c
+++ b/sys/netinet/tcp_output.c
@@ -264,9 +264,9 @@ again:
 		if (len > 0) {
 			sack_rxmit = 1;
 			sendalot = 1;
-			V_tcpstat.tcps_sack_rexmits++;
-			V_tcpstat.tcps_sack_rexmit_bytes +=
-			    min(len, tp->t_maxseg);
+			TCPSTAT_INC(tcps_sack_rexmits);
+			TCPSTAT_ADD(tcps_sack_rexmit_bytes,
+			    min(len, tp->t_maxseg));
 		}
 	}
 after_sack_rexmit:
@@ -768,13 +768,13 @@ send:
 		u_int moff;
 
 		if ((tp->t_flags & TF_FORCEDATA) && len == 1)
-			V_tcpstat.tcps_sndprobe++;
+			TCPSTAT_INC(tcps_sndprobe);
 		else if (SEQ_LT(tp->snd_nxt, tp->snd_max) || sack_rxmit) {
-			V_tcpstat.tcps_sndrexmitpack++;
-			V_tcpstat.tcps_sndrexmitbyte += len;
+			TCPSTAT_INC(tcps_sndrexmitpack);
+			TCPSTAT_ADD(tcps_sndrexmitbyte, len);
 		} else {
-			V_tcpstat.tcps_sndpack++;
-			V_tcpstat.tcps_sndbyte += len;
+			TCPSTAT_INC(tcps_sndpack);
+			TCPSTAT_ADD(tcps_sndbyte, len);
 		}
 #ifdef notyet
 		if ((m = m_copypack(so->so_snd.sb_mb, off,
@@ -841,13 +841,13 @@ send:
 	} else {
 		SOCKBUF_UNLOCK(&so->so_snd);
 		if (tp->t_flags & TF_ACKNOW)
-			V_tcpstat.tcps_sndacks++;
+			TCPSTAT_INC(tcps_sndacks);
 		else if (flags & (TH_SYN|TH_FIN|TH_RST))
-			V_tcpstat.tcps_sndctrl++;
+			TCPSTAT_INC(tcps_sndctrl);
 		else if (SEQ_GT(tp->snd_up, tp->snd_una))
-			V_tcpstat.tcps_sndurg++;
+			TCPSTAT_INC(tcps_sndurg);
 		else
-			V_tcpstat.tcps_sndwinup++;
+			TCPSTAT_INC(tcps_sndwinup);
 
 		MGETHDR(m, M_DONTWAIT, MT_DATA);
 		if (m == NULL) {
@@ -919,7 +919,7 @@ send:
 			else
 #endif
 				ip->ip_tos |= IPTOS_ECN_ECT0;
-			V_tcpstat.tcps_ecn_ect0++;
+			TCPSTAT_INC(tcps_ecn_ect0);
 		}
 		
 		/*
@@ -1085,7 +1085,7 @@ send:
 			if (tp->t_rtttime == 0) {
 				tp->t_rtttime = ticks;
 				tp->t_rtseq = startseq;
-				V_tcpstat.tcps_segstimed++;
+				TCPSTAT_INC(tcps_segstimed);
 			}
 		}
 
@@ -1262,7 +1262,7 @@ out:
 			return (error);
 		}
 	}
-	V_tcpstat.tcps_sndtotal++;
+	TCPSTAT_INC(tcps_sndtotal);
 
 	/*
 	 * Data sent (as far as we can tell).
@@ -1437,7 +1437,7 @@ tcp_addoptions(struct tcpopt *to, u_char *optp)
 				optlen += TCPOLEN_SACK;
 				sack++;
 			}
-			V_tcpstat.tcps_sack_send_blocks++;
+			TCPSTAT_INC(tcps_sack_send_blocks);
 			break;
 			}
 		default:
diff --git a/sys/netinet/tcp_reass.c b/sys/netinet/tcp_reass.c
index 172abc5..080bb02 100644
--- a/sys/netinet/tcp_reass.c
+++ b/sys/netinet/tcp_reass.c
@@ -108,10 +108,12 @@ tcp_reass_zone_change(void *tag)
 	INIT_VNET_INET(curvnet);
 
 	V_tcp_reass_maxseg = nmbclusters / 16;
-	uma_zone_set_max(tcp_reass_zone, V_tcp_reass_maxseg);
+	uma_zone_set_max(V_tcp_reass_zone, V_tcp_reass_maxseg);
 }
 
+#ifdef VIMAGE_GLOBALS
 uma_zone_t	tcp_reass_zone;
+#endif
 
 void
 tcp_reass_init(void)
@@ -126,9 +128,9 @@ tcp_reass_init(void)
 	V_tcp_reass_maxseg = nmbclusters / 16;
 	TUNABLE_INT_FETCH("net.inet.tcp.reass.maxsegments",
 	    &V_tcp_reass_maxseg);
-	tcp_reass_zone = uma_zcreate("tcpreass", sizeof (struct tseg_qent),
+	V_tcp_reass_zone = uma_zcreate("tcpreass", sizeof (struct tseg_qent),
 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
-	uma_zone_set_max(tcp_reass_zone, V_tcp_reass_maxseg);
+	uma_zone_set_max(V_tcp_reass_zone, V_tcp_reass_maxseg);
 	EVENTHANDLER_REGISTER(nmbclusters_change,
 	    tcp_reass_zone_change, NULL, EVENTHANDLER_PRI_ANY);
 }
@@ -170,7 +172,7 @@ tcp_reass(struct tcpcb *tp, struct tcphdr *th, int *tlenp, struct mbuf *m)
 	    (V_tcp_reass_qsize + 1 >= V_tcp_reass_maxseg ||
 	     tp->t_segqlen >= V_tcp_reass_maxqlen)) {
 		V_tcp_reass_overflows++;
-		V_tcpstat.tcps_rcvmemdrop++;
+		TCPSTAT_INC(tcps_rcvmemdrop);
 		m_freem(m);
 		*tlenp = 0;
 		return (0);
@@ -180,9 +182,9 @@ tcp_reass(struct tcpcb *tp, struct tcphdr *th, int *tlenp, struct mbuf *m)
 	 * Allocate a new queue entry. If we can't, or hit the zone limit
 	 * just drop the pkt.
 	 */
-	te = uma_zalloc(tcp_reass_zone, M_NOWAIT);
+	te = uma_zalloc(V_tcp_reass_zone, M_NOWAIT);
 	if (te == NULL) {
-		V_tcpstat.tcps_rcvmemdrop++;
+		TCPSTAT_INC(tcps_rcvmemdrop);
 		m_freem(m);
 		*tlenp = 0;
 		return (0);
@@ -210,10 +212,10 @@ tcp_reass(struct tcpcb *tp, struct tcphdr *th, int *tlenp, struct mbuf *m)
 		i = p->tqe_th->th_seq + p->tqe_len - th->th_seq;
 		if (i > 0) {
 			if (i >= *tlenp) {
-				V_tcpstat.tcps_rcvduppack++;
-				V_tcpstat.tcps_rcvdupbyte += *tlenp;
+				TCPSTAT_INC(tcps_rcvduppack);
+				TCPSTAT_ADD(tcps_rcvdupbyte, *tlenp);
 				m_freem(m);
-				uma_zfree(tcp_reass_zone, te);
+				uma_zfree(V_tcp_reass_zone, te);
 				tp->t_segqlen--;
 				V_tcp_reass_qsize--;
 				/*
@@ -229,8 +231,8 @@ tcp_reass(struct tcpcb *tp, struct tcphdr *th, int *tlenp, struct mbuf *m)
 			th->th_seq += i;
 		}
 	}
-	V_tcpstat.tcps_rcvoopack++;
-	V_tcpstat.tcps_rcvoobyte += *tlenp;
+	TCPSTAT_INC(tcps_rcvoopack);
+	TCPSTAT_ADD(tcps_rcvoobyte, *tlenp);
 
 	/*
 	 * While we overlap succeeding segments trim them or,
@@ -250,7 +252,7 @@ tcp_reass(struct tcpcb *tp, struct tcphdr *th, int *tlenp, struct mbuf *m)
 		nq = LIST_NEXT(q, tqe_q);
 		LIST_REMOVE(q, tqe_q);
 		m_freem(q->tqe_m);
-		uma_zfree(tcp_reass_zone, q);
+		uma_zfree(V_tcp_reass_zone, q);
 		tp->t_segqlen--;
 		V_tcp_reass_qsize--;
 		q = nq;
@@ -287,7 +289,7 @@ present:
 			m_freem(q->tqe_m);
 		else
 			sbappendstream_locked(&so->so_rcv, q->tqe_m);
-		uma_zfree(tcp_reass_zone, q);
+		uma_zfree(V_tcp_reass_zone, q);
 		tp->t_segqlen--;
 		V_tcp_reass_qsize--;
 		q = nq;
diff --git a/sys/netinet/tcp_sack.c b/sys/netinet/tcp_sack.c
index 4ca10af..29bf50c 100644
--- a/sys/netinet/tcp_sack.c
+++ b/sys/netinet/tcp_sack.c
@@ -123,9 +123,8 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/in_cksum.h>
 
-extern struct uma_zone *sack_hole_zone;
-
 #ifdef VIMAGE_GLOBALS
+extern struct uma_zone *sack_hole_zone;
 int tcp_do_sack;
 int tcp_sack_maxholes;
 int tcp_sack_globalmaxholes;
@@ -261,11 +260,11 @@ tcp_sackhole_alloc(struct tcpcb *tp, tcp_seq start, tcp_seq end)
 
 	if (tp->snd_numholes >= V_tcp_sack_maxholes ||
 	    V_tcp_sack_globalholes >= V_tcp_sack_globalmaxholes) {
-		V_tcpstat.tcps_sack_sboverflow++;
+		TCPSTAT_INC(tcps_sack_sboverflow);
 		return NULL;
 	}
 
-	hole = (struct sackhole *)uma_zalloc(sack_hole_zone, M_NOWAIT);
+	hole = (struct sackhole *)uma_zalloc(V_sack_hole_zone, M_NOWAIT);
 	if (hole == NULL)
 		return NULL;
 
@@ -287,7 +286,7 @@ tcp_sackhole_free(struct tcpcb *tp, struct sackhole *hole)
 {
 	INIT_VNET_INET(tp->t_vnet);
 
-	uma_zfree(sack_hole_zone, hole);
+	uma_zfree(V_sack_hole_zone, hole);
 
 	tp->snd_numholes--;
 	V_tcp_sack_globalholes--;
diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c
index 11f4a81..d2d38a0 100644
--- a/sys/netinet/tcp_subr.c
+++ b/sys/netinet/tcp_subr.c
@@ -243,7 +243,9 @@ SYSCTL_V_INT(V_NET, vnet_inet, _net_inet_tcp_inflight, OID_AUTO, stab,
     CTLFLAG_RW, tcp_inflight_stab, 0,
     "Inflight Algorithm Stabilization 20 = 2 packets");
 
+#ifdef VIMAGE_GLOBALS
 uma_zone_t sack_hole_zone;
+#endif
 
 static struct inpcb *tcp_notify(struct inpcb *, int);
 static void	tcp_isn_tick(void *);
@@ -269,7 +271,9 @@ struct tcpcb_mem {
 	struct	tcp_timer	tt;
 };
 
+#ifdef VIMAGE_GLOBALS
 static uma_zone_t tcpcb_zone;
+#endif
 MALLOC_DEFINE(M_TCPLOG, "tcplog", "TCP address and flags print buffers");
 struct callout isn_callout;
 static struct mtx isn_mtx;
@@ -286,7 +290,7 @@ tcp_zone_change(void *tag)
 {
 
 	uma_zone_set_max(V_tcbinfo.ipi_zone, maxsockets);
-	uma_zone_set_max(tcpcb_zone, maxsockets);
+	uma_zone_set_max(V_tcpcb_zone, maxsockets);
 	tcp_tw_zone_change();
 }
 
@@ -348,18 +352,7 @@ tcp_init(void)
 	V_tcp_sack_globalmaxholes = 65536;
 	V_tcp_sack_globalholes = 0;
 
-	tcp_delacktime = TCPTV_DELACK;
-	tcp_keepinit = TCPTV_KEEP_INIT;
-	tcp_keepidle = TCPTV_KEEP_IDLE;
-	tcp_keepintvl = TCPTV_KEEPINTVL;
-	tcp_maxpersistidle = TCPTV_KEEP_IDLE;
-	tcp_msl = TCPTV_MSL;
-	tcp_rexmit_min = TCPTV_MIN;
-	if (tcp_rexmit_min < 1)
-		tcp_rexmit_min = 1;
-	tcp_rexmit_slop = TCPTV_CPU_VAR;
 	V_tcp_inflight_rttthresh = TCPTV_INFLIGHT_RTTTHRESH;
-	tcp_finwait2_timeout = TCPTV_FINWAIT2_TIMEOUT;
 
 	TUNABLE_INT_FETCH("net.inet.tcp.sack.enable", &V_tcp_do_sack);
 
@@ -372,7 +365,6 @@ tcp_init(void)
 		printf("WARNING: TCB hash size not a power of 2\n");
 		hashsize = 512; /* safe default */
 	}
-	tcp_tcbhashsize = hashsize;
 	V_tcbinfo.ipi_hashbase = hashinit(hashsize, M_PCB,
 	    &V_tcbinfo.ipi_hashmask);
 	V_tcbinfo.ipi_porthashbase = hashinit(hashsize, M_PCB,
@@ -380,6 +372,37 @@ tcp_init(void)
 	V_tcbinfo.ipi_zone = uma_zcreate("inpcb", sizeof(struct inpcb),
 	    NULL, NULL, tcp_inpcb_init, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
 	uma_zone_set_max(V_tcbinfo.ipi_zone, maxsockets);
+	/*
+	 * These have to be type stable for the benefit of the timers.
+	 */
+	V_tcpcb_zone = uma_zcreate("tcpcb", sizeof(struct tcpcb_mem),
+	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
+	uma_zone_set_max(V_tcpcb_zone, maxsockets);
+	tcp_tw_init();
+	syncache_init();
+	tcp_hc_init();
+	tcp_reass_init();
+	V_sack_hole_zone = uma_zcreate("sackhole", sizeof(struct sackhole),
+	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
+
+	/* Skip initialization of globals for non-default instances. */
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
+
+	/* XXX virtualize those bellow? */
+	tcp_delacktime = TCPTV_DELACK;
+	tcp_keepinit = TCPTV_KEEP_INIT;
+	tcp_keepidle = TCPTV_KEEP_IDLE;
+	tcp_keepintvl = TCPTV_KEEPINTVL;
+	tcp_maxpersistidle = TCPTV_KEEP_IDLE;
+	tcp_msl = TCPTV_MSL;
+	tcp_rexmit_min = TCPTV_MIN;
+	if (tcp_rexmit_min < 1)
+		tcp_rexmit_min = 1;
+	tcp_rexmit_slop = TCPTV_CPU_VAR;
+	tcp_finwait2_timeout = TCPTV_FINWAIT2_TIMEOUT;
+	tcp_tcbhashsize = hashsize;
+
 #ifdef INET6
 #define TCP_MINPROTOHDR (sizeof(struct ip6_hdr) + sizeof(struct tcphdr))
 #else /* INET6 */
@@ -390,23 +413,12 @@ tcp_init(void)
 	if (max_linkhdr + TCP_MINPROTOHDR > MHLEN)
 		panic("tcp_init");
 #undef TCP_MINPROTOHDR
-	/*
-	 * These have to be type stable for the benefit of the timers.
-	 */
-	tcpcb_zone = uma_zcreate("tcpcb", sizeof(struct tcpcb_mem),
-	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
-	uma_zone_set_max(tcpcb_zone, maxsockets);
-	tcp_tw_init();
-	syncache_init();
-	tcp_hc_init();
-	tcp_reass_init();
+
 	ISN_LOCK_INIT();
 	callout_init(&isn_callout, CALLOUT_MPSAFE);
-	tcp_isn_tick(NULL);
+	callout_reset(&isn_callout, hz/100, tcp_isn_tick, NULL);
 	EVENTHANDLER_REGISTER(shutdown_pre_sync, tcp_fini, NULL,
 		SHUTDOWN_PRI_DEFAULT);
-	sack_hole_zone = uma_zcreate("sackhole", sizeof(struct sackhole),
-	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
 	EVENTHANDLER_REGISTER(maxsockets_change, tcp_zone_change, NULL,
 		EVENTHANDLER_PRI_ANY);
 }
@@ -686,7 +698,7 @@ tcp_newtcpcb(struct inpcb *inp)
 	int isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
 #endif /* INET6 */
 
-	tm = uma_zalloc(tcpcb_zone, M_NOWAIT | M_ZERO);
+	tm = uma_zalloc(V_tcpcb_zone, M_NOWAIT | M_ZERO);
 	if (tm == NULL)
 		return (NULL);
 	tp = &tm->tcb;
@@ -752,9 +764,9 @@ tcp_drop(struct tcpcb *tp, int errno)
 	if (TCPS_HAVERCVDSYN(tp->t_state)) {
 		tp->t_state = TCPS_CLOSED;
 		(void) tcp_output_reset(tp);
-		V_tcpstat.tcps_drops++;
+		TCPSTAT_INC(tcps_drops);
 	} else
-		V_tcpstat.tcps_conndrops++;
+		TCPSTAT_INC(tcps_conndrops);
 	if (errno == ETIMEDOUT && tp->t_softerror)
 		errno = tp->t_softerror;
 	so->so_error = errno;
@@ -846,7 +858,7 @@ tcp_discardcb(struct tcpcb *tp)
 	while ((q = LIST_FIRST(&tp->t_segq)) != NULL) {
 		LIST_REMOVE(q, tqe_q);
 		m_freem(q->tqe_m);
-		uma_zfree(tcp_reass_zone, q);
+		uma_zfree(V_tcp_reass_zone, q);
 		tp->t_segqlen--;
 		V_tcp_reass_qsize--;
 	}
@@ -856,7 +868,7 @@ tcp_discardcb(struct tcpcb *tp)
 	tcp_free_sackholes(tp);
 	inp->inp_ppcb = NULL;
 	tp->t_inpcb = NULL;
-	uma_zfree(tcpcb_zone, tp);
+	uma_zfree(V_tcpcb_zone, tp);
 }
 
 /*
@@ -877,14 +889,14 @@ tcp_close(struct tcpcb *tp)
 	if (tp->t_state == TCPS_LISTEN)
 		tcp_offload_listen_close(tp);
 	in_pcbdrop(inp);
-	V_tcpstat.tcps_closed++;
+	TCPSTAT_INC(tcps_closed);
 	KASSERT(inp->inp_socket != NULL, ("tcp_close: inp_socket NULL"));
 	so = inp->inp_socket;
 	soisdisconnected(so);
-	if (inp->inp_vflag & INP_SOCKREF) {
+	if (inp->inp_flags & INP_SOCKREF) {
 		KASSERT(so->so_state & SS_PROTOREF,
 		    ("tcp_close: !SS_PROTOREF"));
-		inp->inp_vflag &= ~INP_SOCKREF;
+		inp->inp_flags &= ~INP_SOCKREF;
 		INP_WUNLOCK(inp);
 		ACCEPT_LOCK();
 		SOCK_LOCK(so);
@@ -921,7 +933,7 @@ tcp_drain(void)
 	 */
 		INP_INFO_RLOCK(&V_tcbinfo);
 		LIST_FOREACH(inpb, V_tcbinfo.ipi_listhead, inp_list) {
-			if (inpb->inp_vflag & INP_TIMEWAIT)
+			if (inpb->inp_flags & INP_TIMEWAIT)
 				continue;
 			INP_WLOCK(inpb);
 			if ((tcpb = intotcpcb(inpb)) != NULL) {
@@ -929,7 +941,7 @@ tcp_drain(void)
 			            != NULL) {
 					LIST_REMOVE(te, tqe_q);
 					m_freem(te->tqe_m);
-					uma_zfree(tcp_reass_zone, te);
+					uma_zfree(V_tcp_reass_zone, te);
 					tcpb->t_segqlen--;
 					V_tcp_reass_qsize--;
 				}
@@ -962,8 +974,8 @@ tcp_notify(struct inpcb *inp, int error)
 	INP_INFO_WLOCK_ASSERT(&V_tcbinfo);
 	INP_WLOCK_ASSERT(inp);
 
-	if ((inp->inp_vflag & INP_TIMEWAIT) ||
-	    (inp->inp_vflag & INP_DROPPED))
+	if ((inp->inp_flags & INP_TIMEWAIT) ||
+	    (inp->inp_flags & INP_DROPPED))
 		return (inp);
 
 	tp = intotcpcb(inp);
@@ -1063,7 +1075,7 @@ tcp_pcblist(SYSCTL_HANDLER_ARGS)
 			 * TCP state changes, is not quite right, but for
 			 * now, better than nothing.
 			 */
-			if (inp->inp_vflag & INP_TIMEWAIT) {
+			if (inp->inp_flags & INP_TIMEWAIT) {
 				if (intotw(inp) != NULL)
 					error = cr_cansee(req->td->td_ucred,
 					    intotw(inp)->tw_cred);
@@ -1094,7 +1106,7 @@ tcp_pcblist(SYSCTL_HANDLER_ARGS)
 			inp_ppcb = inp->inp_ppcb;
 			if (inp_ppcb == NULL)
 				bzero((char *) &xt.xt_tp, sizeof xt.xt_tp);
-			else if (inp->inp_vflag & INP_TIMEWAIT) {
+			else if (inp->inp_flags & INP_TIMEWAIT) {
 				bzero((char *) &xt.xt_tp, sizeof xt.xt_tp);
 				xt.xt_tp.t_state = TCPS_TIME_WAIT;
 			} else
@@ -1293,8 +1305,8 @@ tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip)
 		    ip->ip_src, th->th_sport, 0, NULL);
 		if (inp != NULL)  {
 			INP_WLOCK(inp);
-			if (!(inp->inp_vflag & INP_TIMEWAIT) &&
-			    !(inp->inp_vflag & INP_DROPPED) &&
+			if (!(inp->inp_flags & INP_TIMEWAIT) &&
+			    !(inp->inp_flags & INP_DROPPED) &&
 			    !(inp->inp_socket == NULL)) {
 				icmp_tcp_seq = htonl(th->th_seq);
 				tp = intotcpcb(inp);
@@ -1546,8 +1558,8 @@ tcp_isn_tick(void *xtp)
 	VNET_ITERATOR_DECL(vnet_iter);
 	u_int32_t projected_offset;
 
-	ISN_LOCK();
 	VNET_LIST_RLOCK();
+	ISN_LOCK();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter); /* XXX appease INVARIANTS */
 		INIT_VNET_INET(curvnet);
@@ -1560,9 +1572,9 @@ tcp_isn_tick(void *xtp)
 		V_isn_offset_old = V_isn_offset;
 		CURVNET_RESTORE();
 	}
+	ISN_UNLOCK();
 	VNET_LIST_RUNLOCK();
 	callout_reset(&isn_callout, hz/100, tcp_isn_tick, NULL);
-	ISN_UNLOCK();
 }
 
 /*
@@ -1581,8 +1593,8 @@ tcp_drop_syn_sent(struct inpcb *inp, int errno)
 	INP_INFO_WLOCK_ASSERT(&V_tcbinfo);
 	INP_WLOCK_ASSERT(inp);
 
-	if ((inp->inp_vflag & INP_TIMEWAIT) ||
-	    (inp->inp_vflag & INP_DROPPED))
+	if ((inp->inp_flags & INP_TIMEWAIT) ||
+	    (inp->inp_flags & INP_DROPPED))
 		return (inp);
 
 	tp = intotcpcb(inp);
@@ -1610,8 +1622,8 @@ tcp_mtudisc(struct inpcb *inp, int errno)
 	struct socket *so;
 
 	INP_WLOCK_ASSERT(inp);
-	if ((inp->inp_vflag & INP_TIMEWAIT) ||
-	    (inp->inp_vflag & INP_DROPPED))
+	if ((inp->inp_flags & INP_TIMEWAIT) ||
+	    (inp->inp_flags & INP_DROPPED))
 		return (inp);
 
 	tp = intotcpcb(inp);
@@ -1626,7 +1638,7 @@ tcp_mtudisc(struct inpcb *inp, int errno)
 		tp->t_maxseg = so->so_snd.sb_hiwat;
 	SOCKBUF_UNLOCK(&so->so_snd);
 
-	V_tcpstat.tcps_mturesent++;
+	TCPSTAT_INC(tcps_mturesent);
 	tp->t_rtttime = 0;
 	tp->snd_nxt = tp->snd_una;
 	tcp_free_sackholes(tp);
@@ -2114,7 +2126,6 @@ sysctl_drop(SYSCTL_HANDLER_ARGS)
 	struct sockaddr_in *fin, *lin;
 #ifdef INET6
 	struct sockaddr_in6 *fin6, *lin6;
-	struct in6_addr f6, l6;
 #endif
 	int error;
 
@@ -2174,8 +2185,9 @@ sysctl_drop(SYSCTL_HANDLER_ARGS)
 	switch (addrs[0].ss_family) {
 #ifdef INET6
 	case AF_INET6:
-		inp = in6_pcblookup_hash(&V_tcbinfo, &f6, fin6->sin6_port,
-		    &l6, lin6->sin6_port, 0, NULL);
+		inp = in6_pcblookup_hash(&V_tcbinfo, &fin6->sin6_addr,
+		    fin6->sin6_port, &lin6->sin6_addr, lin6->sin6_port, 0,
+		    NULL);
 		break;
 #endif
 	case AF_INET:
@@ -2185,7 +2197,7 @@ sysctl_drop(SYSCTL_HANDLER_ARGS)
 	}
 	if (inp != NULL) {
 		INP_WLOCK(inp);
-		if (inp->inp_vflag & INP_TIMEWAIT) {
+		if (inp->inp_flags & INP_TIMEWAIT) {
 			/*
 			 * XXXRW: There currently exists a state where an
 			 * inpcb is present, but its timewait state has been
@@ -2197,7 +2209,7 @@ sysctl_drop(SYSCTL_HANDLER_ARGS)
 				tcp_twclose(tw, 0);
 			else
 				INP_WUNLOCK(inp);
-		} else if (!(inp->inp_vflag & INP_DROPPED) &&
+		} else if (!(inp->inp_flags & INP_DROPPED) &&
 			   !(inp->inp_socket->so_options & SO_ACCEPTCONN)) {
 			tp = intotcpcb(inp);
 			tp = tcp_drop(tp, ECONNABORTED);
diff --git a/sys/netinet/tcp_syncache.c b/sys/netinet/tcp_syncache.c
index 9faca96..18f3fb4 100644
--- a/sys/netinet/tcp_syncache.c
+++ b/sys/netinet/tcp_syncache.c
@@ -294,7 +294,7 @@ syncache_insert(struct syncache *sc, struct syncache_head *sch)
 			("sch->sch_length incorrect"));
 		sc2 = TAILQ_LAST(&sch->sch_bucket, sch_head);
 		syncache_drop(sc2, sch);
-		V_tcpstat.tcps_sc_bucketoverflow++;
+		TCPSTAT_INC(tcps_sc_bucketoverflow);
 	}
 
 	/* Put it into the bucket. */
@@ -309,7 +309,7 @@ syncache_insert(struct syncache *sc, struct syncache_head *sch)
 	SCH_UNLOCK(sch);
 
 	V_tcp_syncache.cache_count++;
-	V_tcpstat.tcps_sc_added++;
+	TCPSTAT_INC(tcps_sc_added);
 }
 
 /*
@@ -398,7 +398,7 @@ syncache_timer(void *xsch)
 				free(s, M_TCPLOG);
 			}
 			syncache_drop(sc, sch);
-			V_tcpstat.tcps_sc_stale++;
+			TCPSTAT_INC(tcps_sc_stale);
 			continue;
 		}
 		if ((s = tcp_log_addrs(&sc->sc_inc, NULL, NULL, NULL))) {
@@ -409,7 +409,7 @@ syncache_timer(void *xsch)
 		}
 
 		(void) syncache_respond(sc);
-		V_tcpstat.tcps_sc_retransmitted++;
+		TCPSTAT_INC(tcps_sc_retransmitted);
 		syncache_timeout(sc, sch, 0);
 	}
 	if (!TAILQ_EMPTY(&(sch)->sch_bucket))
@@ -489,7 +489,7 @@ syncache_chkrst(struct in_conninfo *inc, struct tcphdr *th)
 		if ((s = tcp_log_addrs(inc, th, NULL, NULL)))
 			log(LOG_DEBUG, "%s; %s: Spurious RST with ACK, SYN or "
 			    "FIN flag set, segment ignored\n", s, __func__);
-		V_tcpstat.tcps_badrst++;
+		TCPSTAT_INC(tcps_badrst);
 		goto done;
 	}
 
@@ -506,7 +506,7 @@ syncache_chkrst(struct in_conninfo *inc, struct tcphdr *th)
 			log(LOG_DEBUG, "%s; %s: Spurious RST without matching "
 			    "syncache entry (possibly syncookie only), "
 			    "segment ignored\n", s, __func__);
-		V_tcpstat.tcps_badrst++;
+		TCPSTAT_INC(tcps_badrst);
 		goto done;
 	}
 
@@ -530,13 +530,13 @@ syncache_chkrst(struct in_conninfo *inc, struct tcphdr *th)
 			log(LOG_DEBUG, "%s; %s: Our SYN|ACK was rejected, "
 			    "connection attempt aborted by remote endpoint\n",
 			    s, __func__);
-		V_tcpstat.tcps_sc_reset++;
+		TCPSTAT_INC(tcps_sc_reset);
 	} else {
 		if ((s = tcp_log_addrs(inc, th, NULL, NULL)))
 			log(LOG_DEBUG, "%s; %s: RST with invalid SEQ %u != "
 			    "IRS %u (+WND %u), segment ignored\n",
 			    s, __func__, th->th_seq, sc->sc_irs, sc->sc_wnd);
-		V_tcpstat.tcps_badrst++;
+		TCPSTAT_INC(tcps_badrst);
 	}
 
 done:
@@ -556,7 +556,7 @@ syncache_badack(struct in_conninfo *inc)
 	SCH_LOCK_ASSERT(sch);
 	if (sc != NULL) {
 		syncache_drop(sc, sch);
-		V_tcpstat.tcps_sc_badack++;
+		TCPSTAT_INC(tcps_sc_badack);
 	}
 	SCH_UNLOCK(sch);
 }
@@ -590,7 +590,7 @@ syncache_unreach(struct in_conninfo *inc, struct tcphdr *th)
 		goto done;
 	}
 	syncache_drop(sc, sch);
-	V_tcpstat.tcps_sc_unreach++;
+	TCPSTAT_INC(tcps_sc_unreach);
 done:
 	SCH_UNLOCK(sch);
 }
@@ -622,7 +622,7 @@ syncache_socket(struct syncache *sc, struct socket *lso, struct mbuf *m)
 		 * have the peer retransmit its SYN again after its
 		 * RTO and try again.
 		 */
-		V_tcpstat.tcps_listendrop++;
+		TCPSTAT_INC(tcps_listendrop);
 		if ((s = tcp_log_addrs(&sc->sc_inc, NULL, NULL, NULL))) {
 			log(LOG_DEBUG, "%s; %s: Socket create failed "
 			    "due to limits or memory shortage\n",
@@ -792,7 +792,7 @@ syncache_socket(struct syncache *sc, struct socket *lso, struct mbuf *m)
 
 	INP_WUNLOCK(inp);
 
-	V_tcpstat.tcps_accepts++;
+	TCPSTAT_INC(tcps_accepts);
 	return (so);
 
 abort:
@@ -912,9 +912,9 @@ syncache_expand(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th,
 	*lsop = syncache_socket(sc, *lsop, m);
 
 	if (*lsop == NULL)
-		V_tcpstat.tcps_sc_aborted++;
+		TCPSTAT_INC(tcps_sc_aborted);
 	else
-		V_tcpstat.tcps_sc_completed++;
+		TCPSTAT_INC(tcps_sc_completed);
 
 /* how do we find the inp for the new socket? */
 	if (sc != &scs)
@@ -1046,7 +1046,7 @@ _syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th,
 			sc->sc_tu->tu_syncache_event(TOE_SC_ENTRY_PRESENT,
 			    sc->sc_toepcb);
 #endif		    
-		V_tcpstat.tcps_sc_dupsyn++;
+		TCPSTAT_INC(tcps_sc_dupsyn);
 		if (ipopts) {
 			/*
 			 * If we were remembering a previous source route,
@@ -1081,8 +1081,8 @@ _syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th,
 		if (!TOEPCB_ISSET(sc) && syncache_respond(sc) == 0) {
 			sc->sc_rxmits = 0;
 			syncache_timeout(sc, sch, 1);
-			V_tcpstat.tcps_sndacks++;
-			V_tcpstat.tcps_sndtotal++;
+			TCPSTAT_INC(tcps_sndacks);
+			TCPSTAT_INC(tcps_sndtotal);
 		}
 		SCH_UNLOCK(sch);
 		goto done;
@@ -1095,7 +1095,7 @@ _syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th,
 		 * Treat this as if the cache was full; drop the oldest
 		 * entry and insert the new one.
 		 */
-		V_tcpstat.tcps_sc_zonefail++;
+		TCPSTAT_INC(tcps_sc_zonefail);
 		if ((sc = TAILQ_LAST(&sch->sch_bucket, sch_head)) != NULL)
 			syncache_drop(sc, sch);
 		sc = uma_zalloc(V_tcp_syncache.zone, M_NOWAIT | M_ZERO);
@@ -1233,12 +1233,12 @@ _syncache_add(struct in_conninfo *inc, struct tcpopt *to, struct tcphdr *th,
 			syncache_free(sc);
 		else if (sc != &scs)
 			syncache_insert(sc, sch);   /* locks and unlocks sch */
-		V_tcpstat.tcps_sndacks++;
-		V_tcpstat.tcps_sndtotal++;
+		TCPSTAT_INC(tcps_sndacks);
+		TCPSTAT_INC(tcps_sndtotal);
 	} else {
 		if (sc != &scs)
 			syncache_free(sc);
-		V_tcpstat.tcps_sc_dropped++;
+		TCPSTAT_INC(tcps_sc_dropped);
 	}
 
 done:
@@ -1351,7 +1351,7 @@ syncache_respond(struct syncache *sc)
 
 	if (sc->sc_flags & SCF_ECN) {
 		th->th_flags |= TH_ECE;
-		V_tcpstat.tcps_ecn_shs++;
+		TCPSTAT_INC(tcps_ecn_shs);
 	}
 
 	/* Tack on the TCP options. */
@@ -1584,7 +1584,7 @@ syncookie_generate(struct syncache_head *sch, struct syncache *sc,
 		sc->sc_tsoff = data - ticks;		/* after XOR */
 	}
 
-	V_tcpstat.tcps_sc_sendcookie++;
+	TCPSTAT_INC(tcps_sc_sendcookie);
 }
 
 static struct syncache *
@@ -1687,7 +1687,7 @@ syncookie_lookup(struct in_conninfo *inc, struct syncache_head *sch,
 	sc->sc_rxmits = 0;
 	sc->sc_peer_mss = tcp_sc_msstab[mss];
 
-	V_tcpstat.tcps_sc_recvcookie++;
+	TCPSTAT_INC(tcps_sc_recvcookie);
 	return (sc);
 }
 
diff --git a/sys/netinet/tcp_timer.c b/sys/netinet/tcp_timer.c
index 6963d9c..67fc23f 100644
--- a/sys/netinet/tcp_timer.c
+++ b/sys/netinet/tcp_timer.c
@@ -182,7 +182,7 @@ tcp_timer_delack(void *xtp)
 	}
 	INP_WLOCK(inp);
 	INP_INFO_RUNLOCK(&V_tcbinfo);
-	if ((inp->inp_vflag & INP_DROPPED) || callout_pending(&tp->t_timers->tt_delack)
+	if ((inp->inp_flags & INP_DROPPED) || callout_pending(&tp->t_timers->tt_delack)
 	    || !callout_active(&tp->t_timers->tt_delack)) {
 		INP_WUNLOCK(inp);
 		CURVNET_RESTORE();
@@ -191,7 +191,7 @@ tcp_timer_delack(void *xtp)
 	callout_deactivate(&tp->t_timers->tt_delack);
 
 	tp->t_flags |= TF_ACKNOW;
-	V_tcpstat.tcps_delack++;
+	TCPSTAT_INC(tcps_delack);
 	(void) tcp_output(tp);
 	INP_WUNLOCK(inp);
 	CURVNET_RESTORE();
@@ -229,7 +229,7 @@ tcp_timer_2msl(void *xtp)
 	}
 	INP_WLOCK(inp);
 	tcp_free_sackholes(tp);
-	if ((inp->inp_vflag & INP_DROPPED) || callout_pending(&tp->t_timers->tt_2msl) ||
+	if ((inp->inp_flags & INP_DROPPED) || callout_pending(&tp->t_timers->tt_2msl) ||
 	    !callout_active(&tp->t_timers->tt_2msl)) {
 		INP_WUNLOCK(tp->t_inpcb);
 		INP_INFO_WUNLOCK(&V_tcbinfo);
@@ -250,7 +250,7 @@ tcp_timer_2msl(void *xtp)
 	if (tcp_fast_finwait2_recycle && tp->t_state == TCPS_FIN_WAIT_2 &&
 	    tp->t_inpcb && tp->t_inpcb->inp_socket && 
 	    (tp->t_inpcb->inp_socket->so_rcv.sb_state & SBS_CANTRCVMORE)) {
-		V_tcpstat.tcps_finwait2_drops++;
+		TCPSTAT_INC(tcps_finwait2_drops);
 		tp = tcp_close(tp);             
 	} else {
 		if (tp->t_state != TCPS_TIME_WAIT &&
@@ -301,7 +301,7 @@ tcp_timer_keep(void *xtp)
 		return;
 	}
 	INP_WLOCK(inp);
-	if ((inp->inp_vflag & INP_DROPPED) || callout_pending(&tp->t_timers->tt_keep)
+	if ((inp->inp_flags & INP_DROPPED) || callout_pending(&tp->t_timers->tt_keep)
 	    || !callout_active(&tp->t_timers->tt_keep)) {
 		INP_WUNLOCK(inp);
 		INP_INFO_WUNLOCK(&V_tcbinfo);
@@ -313,7 +313,7 @@ tcp_timer_keep(void *xtp)
 	 * Keep-alive timer went off; send something
 	 * or drop connection if idle for too long.
 	 */
-	V_tcpstat.tcps_keeptimeo++;
+	TCPSTAT_INC(tcps_keeptimeo);
 	if (tp->t_state < TCPS_ESTABLISHED)
 		goto dropit;
 	if ((always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&
@@ -332,7 +332,7 @@ tcp_timer_keep(void *xtp)
 		 * by the protocol spec, this requires the
 		 * correspondent TCP to respond.
 		 */
-		V_tcpstat.tcps_keepprobe++;
+		TCPSTAT_INC(tcps_keepprobe);
 		t_template = tcpip_maketemplate(inp);
 		if (t_template) {
 			tcp_respond(tp, t_template->tt_ipgen,
@@ -355,7 +355,7 @@ tcp_timer_keep(void *xtp)
 	return;
 
 dropit:
-	V_tcpstat.tcps_keepdrops++;
+	TCPSTAT_INC(tcps_keepdrops);
 	tp = tcp_drop(tp, ETIMEDOUT);
 
 #ifdef TCPDEBUG
@@ -397,7 +397,7 @@ tcp_timer_persist(void *xtp)
 		return;
 	}
 	INP_WLOCK(inp);
-	if ((inp->inp_vflag & INP_DROPPED) || callout_pending(&tp->t_timers->tt_persist)
+	if ((inp->inp_flags & INP_DROPPED) || callout_pending(&tp->t_timers->tt_persist)
 	    || !callout_active(&tp->t_timers->tt_persist)) {
 		INP_WUNLOCK(inp);
 		INP_INFO_WUNLOCK(&V_tcbinfo);
@@ -409,7 +409,7 @@ tcp_timer_persist(void *xtp)
 	 * Persistance timer into zero window.
 	 * Force a byte to be output, if possible.
 	 */
-	V_tcpstat.tcps_persisttimeo++;
+	TCPSTAT_INC(tcps_persisttimeo);
 	/*
 	 * Hack: if the peer is dead/unreachable, we do not
 	 * time out if the window is closed.  After a full
@@ -420,7 +420,7 @@ tcp_timer_persist(void *xtp)
 	if (tp->t_rxtshift == TCP_MAXRXTSHIFT &&
 	    ((ticks - tp->t_rcvtime) >= tcp_maxpersistidle ||
 	     (ticks - tp->t_rcvtime) >= TCP_REXMTVAL(tp) * tcp_totbackoff)) {
-		V_tcpstat.tcps_persistdrop++;
+		TCPSTAT_INC(tcps_persistdrop);
 		tp = tcp_drop(tp, ETIMEDOUT);
 		goto out;
 	}
@@ -471,7 +471,7 @@ tcp_timer_rexmt(void * xtp)
 		return;
 	}
 	INP_WLOCK(inp);
-	if ((inp->inp_vflag & INP_DROPPED) || callout_pending(&tp->t_timers->tt_rexmt)
+	if ((inp->inp_flags & INP_DROPPED) || callout_pending(&tp->t_timers->tt_rexmt)
 	    || !callout_active(&tp->t_timers->tt_rexmt)) {
 		INP_WUNLOCK(inp);
 		INP_INFO_WUNLOCK(&V_tcbinfo);
@@ -487,7 +487,7 @@ tcp_timer_rexmt(void * xtp)
 	 */
 	if (++tp->t_rxtshift > TCP_MAXRXTSHIFT) {
 		tp->t_rxtshift = TCP_MAXRXTSHIFT;
-		V_tcpstat.tcps_timeoutdrop++;
+		TCPSTAT_INC(tcps_timeoutdrop);
 		tp = tcp_drop(tp, tp->t_softerror ?
 			      tp->t_softerror : ETIMEDOUT);
 		goto out;
@@ -513,7 +513,7 @@ tcp_timer_rexmt(void * xtp)
 		  tp->t_flags &= ~TF_WASFRECOVERY;
 		tp->t_badrxtwin = ticks + (tp->t_srtt >> (TCP_RTT_SHIFT + 1));
 	}
-	V_tcpstat.tcps_rexmttimeo++;
+	TCPSTAT_INC(tcps_rexmttimeo);
 	if (tp->t_state == TCPS_SYN_SENT)
 		rexmt = TCP_REXMTVAL(tp) * tcp_syn_backoff[tp->t_rxtshift];
 	else
diff --git a/sys/netinet/tcp_timewait.c b/sys/netinet/tcp_timewait.c
index cc3170c..839b25d 100644
--- a/sys/netinet/tcp_timewait.c
+++ b/sys/netinet/tcp_timewait.c
@@ -94,7 +94,6 @@ __FBSDID("$FreeBSD$");
 
 #include <security/mac/mac_framework.h>
 
-static uma_zone_t tcptw_zone;
 static int	maxtcptw;
 
 /*
@@ -104,6 +103,7 @@ static int	maxtcptw;
  * tcbinfo lock, which must be held over queue iteration and modification.
  */
 #ifdef VIMAGE_GLOBALS
+static uma_zone_t tcptw_zone;
 static TAILQ_HEAD(, tcptw)	twq_2msl;
 int	nolocaltimewait;
 #endif
@@ -142,7 +142,7 @@ sysctl_maxtcptw(SYSCTL_HANDLER_ARGS)
 	if (error == 0 && req->newptr)
 		if (new >= 32) {
 			maxtcptw = new;
-			uma_zone_set_max(tcptw_zone, maxtcptw);
+			uma_zone_set_max(V_tcptw_zone, maxtcptw);
 		}
 	return (error);
 }
@@ -160,7 +160,7 @@ tcp_tw_zone_change(void)
 {
 
 	if (maxtcptw == 0)
-		uma_zone_set_max(tcptw_zone, tcptw_auto_size());
+		uma_zone_set_max(V_tcptw_zone, tcptw_auto_size());
 }
 
 void
@@ -168,13 +168,13 @@ tcp_tw_init(void)
 {
 	INIT_VNET_INET(curvnet);
 
-	tcptw_zone = uma_zcreate("tcptw", sizeof(struct tcptw),
+	V_tcptw_zone = uma_zcreate("tcptw", sizeof(struct tcptw),
 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
 	TUNABLE_INT_FETCH("net.inet.tcp.maxtcptw", &maxtcptw);
 	if (maxtcptw == 0)
-		uma_zone_set_max(tcptw_zone, tcptw_auto_size());
+		uma_zone_set_max(V_tcptw_zone, tcptw_auto_size());
 	else
-		uma_zone_set_max(tcptw_zone, maxtcptw);
+		uma_zone_set_max(V_tcptw_zone, maxtcptw);
 	TAILQ_INIT(&V_twq_2msl);
 }
 
@@ -204,7 +204,7 @@ tcp_twstart(struct tcpcb *tp)
 		return;
 	}
 
-	tw = uma_zalloc(tcptw_zone, M_NOWAIT);
+	tw = uma_zalloc(V_tcptw_zone, M_NOWAIT);
 	if (tw == NULL) {
 		tw = tcp_tw_2msl_scan(1);
 		if (tw == NULL) {
@@ -265,17 +265,17 @@ tcp_twstart(struct tcpcb *tp)
 	if (acknow)
 		tcp_twrespond(tw, TH_ACK);
 	inp->inp_ppcb = tw;
-	inp->inp_vflag |= INP_TIMEWAIT;
+	inp->inp_flags |= INP_TIMEWAIT;
 	tcp_tw_2msl_reset(tw, 0);
 
 	/*
 	 * If the inpcb owns the sole reference to the socket, then we can
 	 * detach and free the socket as it is not needed in time wait.
 	 */
-	if (inp->inp_vflag & INP_SOCKREF) {
+	if (inp->inp_flags & INP_SOCKREF) {
 		KASSERT(so->so_state & SS_PROTOREF,
 		    ("tcp_twstart: !SS_PROTOREF"));
-		inp->inp_vflag &= ~INP_SOCKREF;
+		inp->inp_flags &= ~INP_SOCKREF;
 		INP_WUNLOCK(inp);
 		ACCEPT_LOCK();
 		SOCK_LOCK(so);
@@ -435,7 +435,7 @@ tcp_twclose(struct tcptw *tw, int reuse)
 	 *     notify the socket layer.
 	 */
 	inp = tw->tw_inpcb;
-	KASSERT((inp->inp_vflag & INP_TIMEWAIT), ("tcp_twclose: !timewait"));
+	KASSERT((inp->inp_flags & INP_TIMEWAIT), ("tcp_twclose: !timewait"));
 	KASSERT(intotw(inp) == tw, ("tcp_twclose: inp_ppcb != tw"));
 	INP_INFO_WLOCK_ASSERT(&V_tcbinfo);	/* tcp_tw_2msl_stop(). */
 	INP_WLOCK_ASSERT(inp);
@@ -453,8 +453,8 @@ tcp_twclose(struct tcptw *tw, int reuse)
 		 * in which case another reference exists (XXXRW: think
 		 * about this more), and we don't need to take action.
 		 */
-		if (inp->inp_vflag & INP_SOCKREF) {
-			inp->inp_vflag &= ~INP_SOCKREF;
+		if (inp->inp_flags & INP_SOCKREF) {
+			inp->inp_flags &= ~INP_SOCKREF;
 			INP_WUNLOCK(inp);
 			ACCEPT_LOCK();
 			SOCK_LOCK(so);
@@ -472,12 +472,12 @@ tcp_twclose(struct tcptw *tw, int reuse)
 		}
 	} else
 		in_pcbfree(inp);
-	V_tcpstat.tcps_closed++;
+	TCPSTAT_INC(tcps_closed);
 	crfree(tw->tw_cred);
 	tw->tw_cred = NULL;
 	if (reuse)
 		return;
-	uma_zfree(tcptw_zone, tw);
+	uma_zfree(V_tcptw_zone, tw);
 }
 
 int
@@ -567,10 +567,10 @@ tcp_twrespond(struct tcptw *tw, int flags)
 		    NULL, inp);
 	}
 	if (flags & TH_ACK)
-		V_tcpstat.tcps_sndacks++;
+		TCPSTAT_INC(tcps_sndacks);
 	else
-		V_tcpstat.tcps_sndctrl++;
-	V_tcpstat.tcps_sndtotal++;
+		TCPSTAT_INC(tcps_sndctrl);
+	TCPSTAT_INC(tcps_sndtotal);
 	return (error);
 }
 
diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c
index 05e22be..089ef1f 100644
--- a/sys/netinet/tcp_usrreq.c
+++ b/sys/netinet/tcp_usrreq.c
@@ -169,7 +169,7 @@ tcp_detach(struct socket *so, struct inpcb *inp)
 
 	tp = intotcpcb(inp);
 
-	if (inp->inp_vflag & INP_TIMEWAIT) {
+	if (inp->inp_flags & INP_TIMEWAIT) {
 		/*
 		 * There are two cases to handle: one in which the time wait
 		 * state is being discarded (INP_DROPPED), and one in which
@@ -182,7 +182,7 @@ tcp_detach(struct socket *so, struct inpcb *inp)
 		 *
 		 * XXXRW: Would it be cleaner to free the tcptw here?
 		 */
-		if (inp->inp_vflag & INP_DROPPED) {
+		if (inp->inp_flags & INP_DROPPED) {
 			KASSERT(tp == NULL, ("tcp_detach: INP_TIMEWAIT && "
 			    "INP_DROPPED && tp != NULL"));
 			in_pcbdetach(inp);
@@ -201,7 +201,7 @@ tcp_detach(struct socket *so, struct inpcb *inp)
 		 *
 		 * XXXRW: Does the second case still occur?
 		 */
-		if (inp->inp_vflag & INP_DROPPED ||
+		if (inp->inp_flags & INP_DROPPED ||
 		    tp->t_state < TCPS_SYN_SENT) {
 			tcp_discardcb(tp);
 			in_pcbdetach(inp);
@@ -262,7 +262,7 @@ tcp_usr_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("tcp_usr_bind: inp == NULL"));
 	INP_WLOCK(inp);
-	if (inp->inp_vflag & (INP_TIMEWAIT | INP_DROPPED)) {
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		error = EINVAL;
 		goto out;
 	}
@@ -303,7 +303,7 @@ tcp6_usr_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("tcp6_usr_bind: inp == NULL"));
 	INP_WLOCK(inp);
-	if (inp->inp_vflag & (INP_TIMEWAIT | INP_DROPPED)) {
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		error = EINVAL;
 		goto out;
 	}
@@ -350,7 +350,7 @@ tcp_usr_listen(struct socket *so, int backlog, struct thread *td)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("tcp_usr_listen: inp == NULL"));
 	INP_WLOCK(inp);
-	if (inp->inp_vflag & (INP_TIMEWAIT | INP_DROPPED)) {
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		error = EINVAL;
 		goto out;
 	}
@@ -388,7 +388,7 @@ tcp6_usr_listen(struct socket *so, int backlog, struct thread *td)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("tcp6_usr_listen: inp == NULL"));
 	INP_WLOCK(inp);
-	if (inp->inp_vflag & (INP_TIMEWAIT | INP_DROPPED)) {
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		error = EINVAL;
 		goto out;
 	}
@@ -449,7 +449,7 @@ tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("tcp_usr_connect: inp == NULL"));
 	INP_WLOCK(inp);
-	if (inp->inp_vflag & (INP_TIMEWAIT | INP_DROPPED)) {
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		error = EINVAL;
 		goto out;
 	}
@@ -491,7 +491,7 @@ tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("tcp6_usr_connect: inp == NULL"));
 	INP_WLOCK(inp);
-	if (inp->inp_vflag & (INP_TIMEWAIT | INP_DROPPED)) {
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		error = EINVAL;
 		goto out;
 	}
@@ -557,7 +557,7 @@ tcp_usr_disconnect(struct socket *so)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("tcp_usr_disconnect: inp == NULL"));
 	INP_WLOCK(inp);
-	if (inp->inp_vflag & (INP_TIMEWAIT | INP_DROPPED)) {
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		error = ECONNRESET;
 		goto out;
 	}
@@ -594,7 +594,7 @@ tcp_usr_accept(struct socket *so, struct sockaddr **nam)
 	KASSERT(inp != NULL, ("tcp_usr_accept: inp == NULL"));
 	INP_INFO_RLOCK(&V_tcbinfo);
 	INP_WLOCK(inp);
-	if (inp->inp_vflag & (INP_TIMEWAIT | INP_DROPPED)) {
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		error = ECONNABORTED;
 		goto out;
 	}
@@ -637,7 +637,7 @@ tcp6_usr_accept(struct socket *so, struct sockaddr **nam)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("tcp6_usr_accept: inp == NULL"));
 	INP_WLOCK(inp);
-	if (inp->inp_vflag & (INP_TIMEWAIT | INP_DROPPED)) {
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		error = ECONNABORTED;
 		goto out;
 	}
@@ -687,7 +687,7 @@ tcp_usr_shutdown(struct socket *so)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("inp == NULL"));
 	INP_WLOCK(inp);
-	if (inp->inp_vflag & (INP_TIMEWAIT | INP_DROPPED)) {
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		error = ECONNRESET;
 		goto out;
 	}
@@ -695,7 +695,7 @@ tcp_usr_shutdown(struct socket *so)
 	TCPDEBUG1();
 	socantsendmore(so);
 	tcp_usrclosed(tp);
-	if (!(inp->inp_vflag & INP_DROPPED))
+	if (!(inp->inp_flags & INP_DROPPED))
 		error = tcp_output_disconnect(tp);
 
 out:
@@ -720,7 +720,7 @@ tcp_usr_rcvd(struct socket *so, int flags)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("tcp_usr_rcvd: inp == NULL"));
 	INP_WLOCK(inp);
-	if (inp->inp_vflag & (INP_TIMEWAIT | INP_DROPPED)) {
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		error = ECONNRESET;
 		goto out;
 	}
@@ -771,7 +771,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m,
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("tcp_usr_send: inp == NULL"));
 	INP_WLOCK(inp);
-	if (inp->inp_vflag & (INP_TIMEWAIT | INP_DROPPED)) {
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		if (control)
 			m_freem(control);
 		if (m)
@@ -829,7 +829,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m,
 			INP_INFO_WUNLOCK(&V_tcbinfo);
 			headlocked = 0;
 		}
-		if (!(inp->inp_vflag & INP_DROPPED)) {
+		if (!(inp->inp_flags & INP_DROPPED)) {
 			if (flags & PRUS_MORETOCOME)
 				tp->t_flags |= TF_MORETOCOME;
 			error = tcp_output_send(tp);
@@ -917,18 +917,18 @@ tcp_usr_abort(struct socket *so)
 	/*
 	 * If we still have full TCP state, and we're not dropped, drop.
 	 */
-	if (!(inp->inp_vflag & INP_TIMEWAIT) &&
-	    !(inp->inp_vflag & INP_DROPPED)) {
+	if (!(inp->inp_flags & INP_TIMEWAIT) &&
+	    !(inp->inp_flags & INP_DROPPED)) {
 		tp = intotcpcb(inp);
 		TCPDEBUG1();
 		tcp_drop(tp, ECONNABORTED);
 		TCPDEBUG2(PRU_ABORT);
 	}
-	if (!(inp->inp_vflag & INP_DROPPED)) {
+	if (!(inp->inp_flags & INP_DROPPED)) {
 		SOCK_LOCK(so);
 		so->so_state |= SS_PROTOREF;
 		SOCK_UNLOCK(so);
-		inp->inp_vflag |= INP_SOCKREF;
+		inp->inp_flags |= INP_SOCKREF;
 	}
 	INP_WUNLOCK(inp);
 	INP_INFO_WUNLOCK(&V_tcbinfo);
@@ -957,18 +957,18 @@ tcp_usr_close(struct socket *so)
 	 * If we still have full TCP state, and we're not dropped, initiate
 	 * a disconnect.
 	 */
-	if (!(inp->inp_vflag & INP_TIMEWAIT) &&
-	    !(inp->inp_vflag & INP_DROPPED)) {
+	if (!(inp->inp_flags & INP_TIMEWAIT) &&
+	    !(inp->inp_flags & INP_DROPPED)) {
 		tp = intotcpcb(inp);
 		TCPDEBUG1();
 		tcp_disconnect(tp);
 		TCPDEBUG2(PRU_CLOSE);
 	}
-	if (!(inp->inp_vflag & INP_DROPPED)) {
+	if (!(inp->inp_flags & INP_DROPPED)) {
 		SOCK_LOCK(so);
 		so->so_state |= SS_PROTOREF;
 		SOCK_UNLOCK(so);
-		inp->inp_vflag |= INP_SOCKREF;
+		inp->inp_flags |= INP_SOCKREF;
 	}
 	INP_WUNLOCK(inp);
 	INP_INFO_WUNLOCK(&V_tcbinfo);
@@ -988,7 +988,7 @@ tcp_usr_rcvoob(struct socket *so, struct mbuf *m, int flags)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("tcp_usr_rcvoob: inp == NULL"));
 	INP_WLOCK(inp);
-	if (inp->inp_vflag & (INP_TIMEWAIT | INP_DROPPED)) {
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		error = ECONNRESET;
 		goto out;
 	}
@@ -1113,7 +1113,7 @@ tcp_connect(struct tcpcb *tp, struct sockaddr *nam, struct thread *td)
 		tp->request_r_scale++;
 
 	soisconnecting(so);
-	V_tcpstat.tcps_connattempt++;
+	TCPSTAT_INC(tcps_connattempt);
 	tp->t_state = TCPS_SYN_SENT;
 	tcp_timer_activate(tp, TT_KEEP, tcp_keepinit);
 	tp->iss = tcp_new_isn(tp);
@@ -1173,11 +1173,11 @@ tcp6_connect(struct tcpcb *tp, struct sockaddr *nam, struct thread *td)
 
 	/* Compute window scaling to request.  */
 	while (tp->request_r_scale < TCP_MAX_WINSHIFT &&
-	    (TCP_MAXWIN << tp->request_r_scale) < so->so_rcv.sb_hiwat)
+	    (TCP_MAXWIN << tp->request_r_scale) < sb_max)
 		tp->request_r_scale++;
 
 	soisconnecting(so);
-	V_tcpstat.tcps_connattempt++;
+	TCPSTAT_INC(tcps_connattempt);
 	tp->t_state = TCPS_SYN_SENT;
 	tcp_timer_activate(tp, TT_KEEP, tcp_keepinit);
 	tp->iss = tcp_new_isn(tp);
@@ -1241,7 +1241,7 @@ tcp_fill_info(struct tcpcb *tp, struct tcp_info *ti)
  */
 #define INP_WLOCK_RECHECK(inp) do {					\
 	INP_WLOCK(inp);							\
-	if (inp->inp_vflag & (INP_TIMEWAIT | INP_DROPPED)) {		\
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {		\
 		INP_WUNLOCK(inp);					\
 		return (ECONNRESET);					\
 	}								\
@@ -1275,7 +1275,7 @@ tcp_ctloutput(struct socket *so, struct sockopt *sopt)
 #endif
 		return (error);
 	}
-	if (inp->inp_vflag & (INP_TIMEWAIT | INP_DROPPED)) {
+	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		INP_WUNLOCK(inp);
 		return (ECONNRESET);
 	}
@@ -1515,7 +1515,7 @@ tcp_disconnect(struct tcpcb *tp)
 		soisdisconnecting(so);
 		sbflush(&so->so_rcv);
 		tcp_usrclosed(tp);
-		if (!(inp->inp_vflag & INP_DROPPED))
+		if (!(inp->inp_flags & INP_DROPPED))
 			tcp_output_disconnect(tp);
 	}
 }
diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h
index c52506e..268db7c 100644
--- a/sys/netinet/tcp_var.h
+++ b/sys/netinet/tcp_var.h
@@ -458,6 +458,11 @@ struct	tcpstat {
 	u_long	tcps_ecn_rcwnd;		/* # times ECN reduced the cwnd */
 };
 
+#ifdef _KERNEL
+#define	TCPSTAT_ADD(name, val)	V_tcpstat.name += (val)
+#define	TCPSTAT_INC(name)	TCPSTAT_ADD(name, 1)
+#endif
+
 /*
  * TCB structure exported to user-land via sysctl(3).
  * Evil hack: declare only if in_pcb.h and sys/socketvar.h have been
diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c
index 33df73e..084b485 100644
--- a/sys/netinet/udp_usrreq.c
+++ b/sys/netinet/udp_usrreq.c
@@ -254,7 +254,7 @@ udp_append(struct inpcb *inp, struct ip *ip, struct mbuf *n, int off,
 		m_freem(n);
 		if (opts)
 			m_freem(opts);
-		V_udpstat.udps_fullsock++;
+		UDPSTAT_INC(udps_fullsock);
 	} else
 		sorwakeup_locked(so);
 }
@@ -276,7 +276,7 @@ udp_input(struct mbuf *m, int off)
 #endif
 
 	ifp = m->m_pkthdr.rcvif;
-	V_udpstat.udps_ipackets++;
+	UDPSTAT_INC(udps_ipackets);
 
 	/*
 	 * Strip IP options, if any; should skip this, make available to
@@ -294,7 +294,7 @@ udp_input(struct mbuf *m, int off)
 	ip = mtod(m, struct ip *);
 	if (m->m_len < iphlen + sizeof(struct udphdr)) {
 		if ((m = m_pullup(m, iphlen + sizeof(struct udphdr))) == 0) {
-			V_udpstat.udps_hdrops++;
+			UDPSTAT_INC(udps_hdrops);
 			return;
 		}
 		ip = mtod(m, struct ip *);
@@ -324,7 +324,7 @@ udp_input(struct mbuf *m, int off)
 	len = ntohs((u_short)uh->uh_ulen);
 	if (ip->ip_len != len) {
 		if (len > ip->ip_len || len < sizeof(struct udphdr)) {
-			V_udpstat.udps_badlen++;
+			UDPSTAT_INC(udps_badlen);
 			goto badunlocked;
 		}
 		m_adj(m, len - ip->ip_len);
@@ -364,12 +364,12 @@ udp_input(struct mbuf *m, int off)
 			bcopy(b, ((struct ipovly *)ip)->ih_x1, 9);
 		}
 		if (uh_sum) {
-			V_udpstat.udps_badsum++;
+			UDPSTAT_INC(udps_badsum);
 			m_freem(m);
 			return;
 		}
 	} else
-		V_udpstat.udps_nosum++;
+		UDPSTAT_INC(udps_nosum);
 
 #ifdef IPFIREWALL_FORWARD
 	/*
@@ -439,10 +439,10 @@ udp_input(struct mbuf *m, int off)
 					(struct sockaddr *)&udp_in);
 				if (blocked != MCAST_PASS) {
 					if (blocked == MCAST_NOTGMEMBER)
-						V_ipstat.ips_notmember++;
+						IPSTAT_INC(ips_notmember);
 					if (blocked == MCAST_NOTSMEMBER ||
 					    blocked == MCAST_MUTED)
-						V_udpstat.udps_filtermcast++;
+						UDPSTAT_INC(udps_filtermcast);
 					INP_RUNLOCK(inp);
 					continue;
 				}
@@ -494,7 +494,7 @@ udp_input(struct mbuf *m, int off)
 			 * to send an ICMP Port Unreachable for a broadcast
 			 * or multicast datgram.)
 			 */
-			V_udpstat.udps_noportbcast++;
+			UDPSTAT_INC(udps_noportbcast);
 			goto badheadlocked;
 		}
 		if (last->inp_ppcb == NULL) {
@@ -531,9 +531,9 @@ udp_input(struct mbuf *m, int off)
 			    buf, ntohs(uh->uh_dport), inet_ntoa(ip->ip_src),
 			    ntohs(uh->uh_sport));
 		}
-		V_udpstat.udps_noport++;
+		UDPSTAT_INC(udps_noport);
 		if (m->m_flags & (M_BCAST | M_MCAST)) {
-			V_udpstat.udps_noportbcast++;
+			UDPSTAT_INC(udps_noportbcast);
 			goto badheadlocked;
 		}
 		if (V_udp_blackhole)
@@ -1072,7 +1072,7 @@ udp_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr,
 	((struct ip *)ui)->ip_len = sizeof (struct udpiphdr) + len;
 	((struct ip *)ui)->ip_ttl = inp->inp_ip_ttl;	/* XXX */
 	((struct ip *)ui)->ip_tos = inp->inp_ip_tos;	/* XXX */
-	V_udpstat.udps_opackets++;
+	UDPSTAT_INC(udps_opackets);
 
 	if (unlock_udbinfo == 2)
 		INP_INFO_WUNLOCK(&V_udbinfo);
diff --git a/sys/netinet/udp_var.h b/sys/netinet/udp_var.h
index 76a31b8..d83f705 100644
--- a/sys/netinet/udp_var.h
+++ b/sys/netinet/udp_var.h
@@ -71,6 +71,11 @@ struct udpstat {
 	u_long	udps_filtermcast;	/* blocked by multicast filter */
 };
 
+#ifdef _KERNEL
+#define	UDPSTAT_ADD(name, val)	V_udpstat.name += (val)
+#define	UDPSTAT_INC(name)	UDPSTAT_ADD(name, 1)
+#endif
+
 /*
  * Names for UDP sysctl objects.
  */
diff --git a/sys/netinet/vinet.h b/sys/netinet/vinet.h
index e5b0bab..b65acc1 100644
--- a/sys/netinet/vinet.h
+++ b/sys/netinet/vinet.h
@@ -86,6 +86,11 @@ struct vnet_inet {
 	struct	tcp_hostcache _tcp_hostcache;
 	struct  callout _tcp_hc_callout;
 
+	uma_zone_t _tcp_reass_zone;
+	uma_zone_t _tcpcb_zone;
+	uma_zone_t _tcptw_zone;
+	uma_zone_t _sack_hole_zone;
+
 	struct	tcp_syncache _tcp_syncache;
 	int	_tcp_syncookies;
 	int	_tcp_syncookiesonly;
@@ -315,12 +320,15 @@ extern struct vnet_inet vnet_inet_0;
 #define	V_rtq_timeout		VNET_INET(rtq_timeout)
 #define	V_rtq_timer		VNET_INET(rtq_timer)
 #define	V_rtq_toomany		VNET_INET(rtq_toomany)
+#define	V_sack_hole_zone	VNET_INET(sack_hole_zone)
 #define	V_sameprefixcarponly	VNET_INET(sameprefixcarponly)
 #define	V_ss_fltsz		VNET_INET(ss_fltsz)
 #define	V_ss_fltsz_local	VNET_INET(ss_fltsz_local)
 #define	V_subnetsarelocal	VNET_INET(subnetsarelocal)
 #define	V_tcb			VNET_INET(tcb)
 #define	V_tcbinfo		VNET_INET(tcbinfo)
+#define	V_tcpcb_zone		VNET_INET(tcpcb_zone)
+#define	V_tcptw_zone		VNET_INET(tcptw_zone)
 #define	V_tcp_abc_l_var		VNET_INET(tcp_abc_l_var)
 #define	V_tcp_autorcvbuf_inc	VNET_INET(tcp_autorcvbuf_inc)
 #define	V_tcp_autorcvbuf_max	VNET_INET(tcp_autorcvbuf_max)
@@ -353,6 +361,7 @@ extern struct vnet_inet vnet_inet_0;
 #define	V_tcp_reass_maxseg	VNET_INET(tcp_reass_maxseg)
 #define	V_tcp_reass_overflows	VNET_INET(tcp_reass_overflows)
 #define	V_tcp_reass_qsize	VNET_INET(tcp_reass_qsize)
+#define	V_tcp_reass_zone	VNET_INET(tcp_reass_zone)
 #define	V_tcp_sack_globalholes	VNET_INET(tcp_sack_globalholes)
 #define	V_tcp_sack_globalmaxholes VNET_INET(tcp_sack_globalmaxholes)
 #define	V_tcp_sack_maxholes	VNET_INET(tcp_sack_maxholes)
diff --git a/sys/netinet6/frag6.c b/sys/netinet6/frag6.c
index e2eda6f..d0f333d 100644
--- a/sys/netinet6/frag6.c
+++ b/sys/netinet6/frag6.c
@@ -109,14 +109,16 @@ frag6_init(void)
 {
 	INIT_VNET_INET6(curvnet);
 
+	V_ip6q.ip6q_next = V_ip6q.ip6q_prev = &V_ip6q;
 	V_ip6_maxfragpackets = nmbclusters / 4;
 	V_ip6_maxfrags = nmbclusters / 4;
-	EVENTHANDLER_REGISTER(nmbclusters_change,
-	    frag6_change, NULL, EVENTHANDLER_PRI_ANY);
 
-	IP6Q_LOCK_INIT();
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
 
-	V_ip6q.ip6q_next = V_ip6q.ip6q_prev = &V_ip6q;
+	IP6Q_LOCK_INIT();
+	EVENTHANDLER_REGISTER(nmbclusters_change,
+	    frag6_change, NULL, EVENTHANDLER_PRI_ANY);
 }
 
 /*
diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c
index 27159db..277e313 100644
--- a/sys/netinet6/icmp6.c
+++ b/sys/netinet6/icmp6.c
@@ -254,14 +254,14 @@ icmp6_error(struct mbuf *m, int type, int code, int param)
 	int off;
 	int nxt;
 
-	V_icmp6stat.icp6s_error++;
+	ICMP6STAT_INC(icp6s_error);
 
 	/* count per-type-code statistics */
 	icmp6_errcount(&V_icmp6stat.icp6s_outerrhist, type, code);
 
 #ifdef M_DECRYPTED	/*not openbsd*/
 	if (m->m_flags & M_DECRYPTED) {
-		V_icmp6stat.icp6s_canterror++;
+		ICMP6STAT_INC(icp6s_canterror);
 		goto freeit;
 	}
 #endif
@@ -319,7 +319,7 @@ icmp6_error(struct mbuf *m, int type, int code, int param)
 		IP6_EXTHDR_GET(icp, struct icmp6_hdr *, m, off,
 			sizeof(*icp));
 		if (icp == NULL) {
-			V_icmp6stat.icp6s_tooshort++;
+			ICMP6STAT_INC(icp6s_tooshort);
 			return;
 		}
 #endif
@@ -330,7 +330,7 @@ icmp6_error(struct mbuf *m, int type, int code, int param)
 			 * Special case: for redirect (which is
 			 * informational) we must not send icmp6 error.
 			 */
-			V_icmp6stat.icp6s_canterror++;
+			ICMP6STAT_INC(icp6s_canterror);
 			goto freeit;
 		} else {
 			/* ICMPv6 informational - send the error */
@@ -343,7 +343,7 @@ icmp6_error(struct mbuf *m, int type, int code, int param)
 
 	/* Finally, do rate limitation check. */
 	if (icmp6_ratelimit(&oip6->ip6_src, type, code)) {
-		V_icmp6stat.icp6s_toofreq++;
+		ICMP6STAT_INC(icp6s_toofreq);
 		goto freeit;
 	}
 
@@ -384,7 +384,7 @@ icmp6_error(struct mbuf *m, int type, int code, int param)
 	 */
 	m->m_pkthdr.rcvif = NULL;
 
-	V_icmp6stat.icp6s_outhist[type]++;
+	ICMP6STAT_INC(icp6s_outhist[type]);
 	icmp6_reflect(m, sizeof(struct ip6_hdr)); /* header order: IPv6 - ICMPv6 */
 
 	return;
@@ -424,7 +424,7 @@ icmp6_input(struct mbuf **mp, int *offp, int proto)
 
 	ip6 = mtod(m, struct ip6_hdr *);
 	if (icmp6len < sizeof(struct icmp6_hdr)) {
-		V_icmp6stat.icp6s_tooshort++;
+		ICMP6STAT_INC(icp6s_tooshort);
 		goto freeit;
 	}
 
@@ -436,7 +436,7 @@ icmp6_input(struct mbuf **mp, int *offp, int proto)
 #else
 	IP6_EXTHDR_GET(icmp6, struct icmp6_hdr *, m, off, sizeof(*icmp6));
 	if (icmp6 == NULL) {
-		V_icmp6stat.icp6s_tooshort++;
+		ICMP6STAT_INC(icp6s_tooshort);
 		return IPPROTO_DONE;
 	}
 #endif
@@ -447,7 +447,7 @@ icmp6_input(struct mbuf **mp, int *offp, int proto)
 		    "ICMP6 checksum error(%d|%x) %s\n",
 		    icmp6->icmp6_type, sum,
 		    ip6_sprintf(ip6bufs, &ip6->ip6_src)));
-		V_icmp6stat.icp6s_checksum++;
+		ICMP6STAT_INC(icp6s_checksum);
 		goto freeit;
 	}
 
@@ -467,7 +467,7 @@ icmp6_input(struct mbuf **mp, int *offp, int proto)
 		}
 	}
 
-	V_icmp6stat.icp6s_inhist[icmp6->icmp6_type]++;
+	ICMP6STAT_INC(icp6s_inhist[icmp6->icmp6_type]);
 	icmp6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_msg);
 	if (icmp6->icmp6_type < ICMP6_INFOMSG_MASK)
 		icmp6_ifstat_inc(m->m_pkthdr.rcvif, ifs6_in_error);
@@ -601,8 +601,8 @@ icmp6_input(struct mbuf **mp, int *offp, int proto)
 		nicmp6->icmp6_type = ICMP6_ECHO_REPLY;
 		nicmp6->icmp6_code = 0;
 		if (n) {
-			V_icmp6stat.icp6s_reflect++;
-			V_icmp6stat.icp6s_outhist[ICMP6_ECHO_REPLY]++;
+			ICMP6STAT_INC(icp6s_reflect);
+			ICMP6STAT_INC(icp6s_outhist[ICMP6_ECHO_REPLY]);
 			icmp6_reflect(n, noff);
 		}
 		break;
@@ -734,8 +734,8 @@ icmp6_input(struct mbuf **mp, int *offp, int proto)
 		}
 #undef hostnamelen
 		if (n) {
-			V_icmp6stat.icp6s_reflect++;
-			V_icmp6stat.icp6s_outhist[ICMP6_WRUREPLY]++;
+			ICMP6STAT_INC(icp6s_reflect);
+			ICMP6STAT_INC(icp6s_outhist[ICMP6_WRUREPLY]);
 			icmp6_reflect(n, noff);
 		}
 		break;
@@ -856,11 +856,11 @@ icmp6_input(struct mbuf **mp, int *offp, int proto)
 		break;
 
 	badcode:
-		V_icmp6stat.icp6s_badcode++;
+		ICMP6STAT_INC(icp6s_badcode);
 		break;
 
 	badlen:
-		V_icmp6stat.icp6s_badlen++;
+		ICMP6STAT_INC(icp6s_badlen);
 		break;
 	}
 
@@ -885,7 +885,7 @@ icmp6_notify_error(struct mbuf **mp, int off, int icmp6len, int code)
 	struct sockaddr_in6 icmp6src, icmp6dst;
 
 	if (icmp6len < sizeof(struct icmp6_hdr) + sizeof(struct ip6_hdr)) {
-		V_icmp6stat.icp6s_tooshort++;
+		ICMP6STAT_INC(icp6s_tooshort);
 		goto freeit;
 	}
 #ifndef PULLDOWN_TEST
@@ -896,7 +896,7 @@ icmp6_notify_error(struct mbuf **mp, int off, int icmp6len, int code)
 	IP6_EXTHDR_GET(icmp6, struct icmp6_hdr *, m, off,
 	    sizeof(*icmp6) + sizeof(struct ip6_hdr));
 	if (icmp6 == NULL) {
-		V_icmp6stat.icp6s_tooshort++;
+		ICMP6STAT_INC(icp6s_tooshort);
 		return (-1);
 	}
 #endif
@@ -931,7 +931,7 @@ icmp6_notify_error(struct mbuf **mp, int off, int icmp6len, int code)
 				IP6_EXTHDR_GET(eh, struct ip6_ext *, m,
 				    eoff, sizeof(*eh));
 				if (eh == NULL) {
-					V_icmp6stat.icp6s_tooshort++;
+					ICMP6STAT_INC(icp6s_tooshort);
 					return (-1);
 				}
 #endif
@@ -959,7 +959,7 @@ icmp6_notify_error(struct mbuf **mp, int off, int icmp6len, int code)
 				IP6_EXTHDR_GET(rth, struct ip6_rthdr *, m,
 				    eoff, sizeof(*rth));
 				if (rth == NULL) {
-					V_icmp6stat.icp6s_tooshort++;
+					ICMP6STAT_INC(icp6s_tooshort);
 					return (-1);
 				}
 #endif
@@ -985,7 +985,7 @@ icmp6_notify_error(struct mbuf **mp, int off, int icmp6len, int code)
 					    struct ip6_rthdr0 *, m,
 					    eoff, rthlen);
 					if (rth0 == NULL) {
-						V_icmp6stat.icp6s_tooshort++;
+						ICMP6STAT_INC(icp6s_tooshort);
 						return (-1);
 					}
 #endif
@@ -1007,7 +1007,7 @@ icmp6_notify_error(struct mbuf **mp, int off, int icmp6len, int code)
 				IP6_EXTHDR_GET(fh, struct ip6_frag *, m,
 				    eoff, sizeof(*fh));
 				if (fh == NULL) {
-					V_icmp6stat.icp6s_tooshort++;
+					ICMP6STAT_INC(icp6s_tooshort);
 					return (-1);
 				}
 #endif
@@ -1042,7 +1042,7 @@ icmp6_notify_error(struct mbuf **mp, int off, int icmp6len, int code)
 		IP6_EXTHDR_GET(icmp6, struct icmp6_hdr *, m, off,
 		    sizeof(*icmp6) + sizeof(struct ip6_hdr));
 		if (icmp6 == NULL) {
-			V_icmp6stat.icp6s_tooshort++;
+			ICMP6STAT_INC(icp6s_tooshort);
 			return (-1);
 		}
 #endif
@@ -1156,7 +1156,7 @@ icmp6_mtudisc_update(struct ip6ctlparam *ip6cp, int validated)
 
 	if (mtu < tcp_maxmtu6(&inc, NULL)) {
 		tcp_hc_updatemtu(&inc, mtu);
-		V_icmp6stat.icp6s_pmtuchg++;
+		ICMP6STAT_INC(icp6s_pmtuchg);
 	}
 }
 
@@ -2276,7 +2276,7 @@ icmp6_redirect_input(struct mbuf *m, int off)
 #else
 	IP6_EXTHDR_GET(nd_rd, struct nd_redirect *, m, off, icmp6len);
 	if (nd_rd == NULL) {
-		V_icmp6stat.icp6s_tooshort++;
+		ICMP6STAT_INC(icp6s_tooshort);
 		return;
 	}
 #endif
@@ -2439,7 +2439,7 @@ icmp6_redirect_input(struct mbuf *m, int off)
 	return;
 
  bad:
-	V_icmp6stat.icp6s_badredirect++;
+	ICMP6STAT_INC(icp6s_badredirect);
 	m_freem(m);
 }
 
@@ -2715,7 +2715,7 @@ noredhdropt:;
 		icmp6_ifstat_inc(outif, ifs6_out_msg);
 		icmp6_ifstat_inc(outif, ifs6_out_redirect);
 	}
-	V_icmp6stat.icp6s_outhist[ND_REDIRECT]++;
+	ICMP6STAT_INC(icp6s_outhist[ND_REDIRECT]);
 
 	return;
 
diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c
index b75ffc5..d8a11d0 100644
--- a/sys/netinet6/in6.c
+++ b/sys/netinet6/in6.c
@@ -1516,9 +1516,7 @@ in6_ifinit(struct ifnet *ifp, struct in6_ifaddr *ia,
 	ia->ia_addr = *sin6;
 
 	if (ifacount <= 1 && ifp->if_ioctl) {
-		IFF_LOCKGIANT(ifp);
 		error = (*ifp->if_ioctl)(ifp, SIOCSIFADDR, (caddr_t)ia);
-		IFF_UNLOCKGIANT(ifp);
 		if (error) {
 			splx(s);
 			return (error);
diff --git a/sys/netinet6/in6_ifattach.c b/sys/netinet6/in6_ifattach.c
index 4950549..68b3542 100644
--- a/sys/netinet6/in6_ifattach.c
+++ b/sys/netinet6/in6_ifattach.c
@@ -906,10 +906,8 @@ in6_purgemaddrs(struct ifnet *ifp)
 	struct in6_multi *in6m;
 	struct in6_multi *oin6m;
 
-	IFF_LOCKGIANT(ifp);
 	LIST_FOREACH_SAFE(in6m, &in6_multihead, in6m_entry, oin6m) {
 		if (in6m->in6m_ifp == ifp)
 			in6_delmulti(in6m);
 	}
-	IFF_UNLOCKGIANT(ifp);
 }
diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c
index 79cb213..79e79cb 100644
--- a/sys/netinet6/in6_pcb.c
+++ b/sys/netinet6/in6_pcb.c
@@ -197,7 +197,7 @@ in6_pcbbind(register struct inpcb *inp, struct sockaddr *nam,
 				    &sin6->sin6_addr, lport,
 				    INPLOOKUP_WILDCARD, cred);
 				if (t &&
-				    ((t->inp_vflag & INP_TIMEWAIT) == 0) &&
+				    ((t->inp_flags & INP_TIMEWAIT) == 0) &&
 				    (so->so_type != SOCK_STREAM ||
 				     IN6_IS_ADDR_UNSPECIFIED(&t->in6p_faddr)) &&
 				    (!IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr) ||
@@ -215,7 +215,7 @@ in6_pcbbind(register struct inpcb *inp, struct sockaddr *nam,
 					    sin.sin_addr, lport,
 					    INPLOOKUP_WILDCARD, cred);
 					if (t &&
-					    ((t->inp_vflag &
+					    ((t->inp_flags &
 					      INP_TIMEWAIT) == 0) &&
 					    (so->so_type != SOCK_STREAM ||
 					     ntohl(t->inp_faddr.s_addr) ==
@@ -227,7 +227,7 @@ in6_pcbbind(register struct inpcb *inp, struct sockaddr *nam,
 			}
 			t = in6_pcblookup_local(pcbinfo, &sin6->sin6_addr,
 			    lport, wild, cred);
-			if (t && (reuseport & ((t->inp_vflag & INP_TIMEWAIT) ?
+			if (t && (reuseport & ((t->inp_flags & INP_TIMEWAIT) ?
 			    intotw(t)->tw_so_options :
 			    t->inp_socket->so_options)) == 0)
 				return (EADDRINUSE);
@@ -238,7 +238,7 @@ in6_pcbbind(register struct inpcb *inp, struct sockaddr *nam,
 				in6_sin6_2_sin(&sin, sin6);
 				t = in_pcblookup_local(pcbinfo, sin.sin_addr,
 				    lport, wild, cred);
-				if (t && t->inp_vflag & INP_TIMEWAIT) {
+				if (t && t->inp_flags & INP_TIMEWAIT) {
 					if ((reuseport &
 					    intotw(t)->tw_so_options) == 0 &&
 					    (ntohl(t->inp_laddr.s_addr) !=
diff --git a/sys/netinet6/in6_src.c b/sys/netinet6/in6_src.c
index 8030416..7ae30387 100644
--- a/sys/netinet6/in6_src.c
+++ b/sys/netinet6/in6_src.c
@@ -920,8 +920,6 @@ in6_pcbsetport(struct in6_addr *laddr, struct inpcb *inp, struct ucred *cred)
 void
 addrsel_policy_init(void)
 {
-	ADDRSEL_LOCK_INIT();
-	ADDRSEL_SXLOCK_INIT();
 	INIT_VNET_INET6(curvnet);
 
 	V_ip6_prefer_tempaddr = 0;
@@ -931,6 +929,12 @@ addrsel_policy_init(void)
 	/* initialize the "last resort" policy */
 	bzero(&V_defaultaddrpolicy, sizeof(V_defaultaddrpolicy));
 	V_defaultaddrpolicy.label = ADDR_LABEL_NOTAPP;
+
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
+
+	ADDRSEL_LOCK_INIT();
+	ADDRSEL_SXLOCK_INIT();
 }
 
 static struct in6_addrpolicy *
diff --git a/sys/netinet6/ip6_input.c b/sys/netinet6/ip6_input.c
index f094683..27c8e89 100644
--- a/sys/netinet6/ip6_input.c
+++ b/sys/netinet6/ip6_input.c
@@ -155,6 +155,25 @@ static int ip6_hopopts_input(u_int32_t *, u_int32_t *, struct mbuf **, int *);
 static struct mbuf *ip6_pullexthdr(struct mbuf *, size_t, int);
 #endif
 
+#ifndef VIMAGE_GLOBALS
+static void vnet_inet6_register(void);
+ 
+static const vnet_modinfo_t vnet_inet6_modinfo = {
+	.vmi_id		= VNET_MOD_INET6,
+	.vmi_name	= "inet6",
+	.vmi_dependson	= VNET_MOD_INET	/* XXX revisit - TCP/UDP needs this? */
+};
+ 
+static void
+vnet_inet6_register(void)
+{
+
+	vnet_mod_register(&vnet_inet6_modinfo);
+}
+ 
+SYSINIT(inet6, SI_SUB_PROTO_BEGIN, SI_ORDER_FIRST, vnet_inet6_register, 0);
+#endif
+
 /*
  * IP6 initialization: fill in IP6 protocol switch table.
  * All protocols not implemented in kernel go to raw IP6 protocol handler.
@@ -234,6 +253,17 @@ ip6_init(void)
 					/* 40 1K datagrams */
 	V_dad_init = 0;
 
+	scope6_init();
+	addrsel_policy_init();
+	nd6_init();
+	frag6_init();
+
+	V_ip6_desync_factor = arc4random() % MAX_TEMP_DESYNC_FACTOR;
+
+	/* Skip global initialization stuff for non-default instances. */
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
+
 #ifdef DIAGNOSTIC
 	if (sizeof(struct protosw) != sizeof(struct ip6protosw))
 		panic("sizeof(protosw) != sizeof(ip6protosw)");
@@ -265,18 +295,13 @@ ip6_init(void)
 		printf("%s: WARNING: unable to register pfil hook, "
 			"error %d\n", __func__, i);
 
-	ip6intrq.ifq_maxlen = V_ip6qmaxlen;
+	ip6intrq.ifq_maxlen = V_ip6qmaxlen; /* XXX */
 	mtx_init(&ip6intrq.ifq_mtx, "ip6_inq", NULL, MTX_DEF);
 	netisr_register(NETISR_IPV6, ip6_input, &ip6intrq, 0);
-	scope6_init();
-	addrsel_policy_init();
-	nd6_init();
-	frag6_init();
-	V_ip6_desync_factor = arc4random() % MAX_TEMP_DESYNC_FACTOR;
 }
 
-static void
-ip6_init2(void *dummy)
+static int
+ip6_init2_vnet(const void *unused __unused)
 {
 	INIT_VNET_INET6(curvnet);
 
@@ -290,6 +315,15 @@ ip6_init2(void *dummy)
 		      (V_ip6_temp_preferred_lifetime - V_ip6_desync_factor -
 		       V_ip6_temp_regen_advance) * hz,
 		      in6_tmpaddrtimer, NULL);
+
+	return (0);
+}
+
+static void
+ip6_init2(void *dummy)
+{
+
+	ip6_init2_vnet(NULL);
 }
 
 /* cheat */
diff --git a/sys/netinet6/ip6_mroute.c b/sys/netinet6/ip6_mroute.c
index 18e7aca..29201d6 100644
--- a/sys/netinet6/ip6_mroute.c
+++ b/sys/netinet6/ip6_mroute.c
@@ -92,6 +92,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
+#include <sys/module.h>
 #include <sys/protosw.h>
 #include <sys/signalvar.h>
 #include <sys/socket.h>
@@ -114,6 +115,7 @@ __FBSDID("$FreeBSD$");
 #include <netinet/in_var.h>
 #include <netinet/icmp6.h>
 #include <netinet/vinet.h>
+#include <netinet/ip_encap.h>
 
 #include <netinet/ip6.h>
 #include <netinet6/ip6_var.h>
@@ -130,20 +132,18 @@ static MALLOC_DEFINE(M_MRTABLE6, "mf6c", "multicast forwarding cache entry");
 /* XXX: this is a very common idiom; move to <sys/mbuf.h> ? */
 #define M_HASCL(m) ((m)->m_flags & M_EXT)
 
-static int ip6_mdq(struct mbuf *, struct ifnet *, struct mf6c *);
-static void phyint_send(struct ip6_hdr *, struct mif6 *, struct mbuf *);
-
-static void pim6_init(void);
-static int set_pim6(int *);
-static int socket_send __P((struct socket *, struct mbuf *,
-	    struct sockaddr_in6 *));
-static int register_send __P((struct ip6_hdr *, struct mif6 *,
-	    struct mbuf *));
+static int	ip6_mdq(struct mbuf *, struct ifnet *, struct mf6c *);
+static void	phyint_send(struct ip6_hdr *, struct mif6 *, struct mbuf *);
+static void	pim6_init(void);
+static int	register_send(struct ip6_hdr *, struct mif6 *, struct mbuf *);
+static int	set_pim6(int *);
+static int	socket_send(struct socket *, struct mbuf *,
+		    struct sockaddr_in6 *);
 
 extern struct domain inet6domain;
 
-/* XXX: referenced from ip_mroute.c for dynamically loading this code. */
-struct ip6protosw in6_pim_protosw = {
+static const struct encaptab *pim6_encap_cookie;
+static const struct ip6protosw in6_pim_protosw = {
 	.pr_type =		SOCK_RAW,
 	.pr_domain =		&inet6domain,
 	.pr_protocol =		IPPROTO_PIM,
@@ -154,6 +154,7 @@ struct ip6protosw in6_pim_protosw = {
 	.pr_init =		pim6_init,
 	.pr_usrreqs =		&rip6_usrreqs
 };
+static int pim6_encapcheck(const struct mbuf *, int, int, void *);
 
 #ifdef VIMAGE_GLOBALS
 static int ip6_mrouter_ver;
@@ -171,18 +172,48 @@ SYSCTL_STRUCT(_net_inet6_ip6, OID_AUTO, mrt6stat, CTLFLAG_RW,
 #define NO_RTE_FOUND	0x1
 #define RTE_FOUND	0x2
 
+static struct mtx mrouter6_mtx;
+#define	MROUTER6_LOCK()		mtx_lock(&mrouter6_mtx)
+#define	MROUTER6_UNLOCK()	mtx_unlock(&mrouter6_mtx)
+#define	MROUTER6_LOCK_ASSERT()	do {					\
+	mtx_assert(&mrouter6_mtx, MA_OWNED);				\
+	NET_ASSERT_GIANT();						\
+} while (0)
+#define	MROUTER6_LOCK_INIT()	\
+	mtx_init(&mrouter6_mtx, "IPv6 multicast forwarding", NULL, MTX_DEF)
+#define	MROUTER6_LOCK_DESTROY()	mtx_destroy(&mrouter6_mtx)
+
 static struct mf6c *mf6ctable[MF6CTBLSIZ];
 SYSCTL_OPAQUE(_net_inet6_ip6, OID_AUTO, mf6ctable, CTLFLAG_RD,
     &mf6ctable, sizeof(mf6ctable), "S,*mf6ctable[MF6CTBLSIZ]",
-    "Multicast Forwarding Table (struct *mf6ctable[MF6CTBLSIZ], "
+    "IPv6 Multicast Forwarding Table (struct *mf6ctable[MF6CTBLSIZ], "
     "netinet6/ip6_mroute.h)");
 
+static struct mtx mfc6_mtx;
+#define	MFC6_LOCK()		mtx_lock(&mfc6_mtx)
+#define	MFC6_UNLOCK()		mtx_unlock(&mfc6_mtx)
+#define	MFC6_LOCK_ASSERT()	do {					\
+	mtx_assert(&mfc6_mtx, MA_OWNED);				\
+	NET_ASSERT_GIANT();						\
+} while (0)
+#define	MFC6_LOCK_INIT()		\
+	mtx_init(&mfc6_mtx, "IPv6 multicast forwarding cache", NULL, MTX_DEF)
+#define	MFC6_LOCK_DESTROY()	mtx_destroy(&mfc6_mtx)
+
 static u_char n6expire[MF6CTBLSIZ];
 
 static struct mif6 mif6table[MAXMIFS];
 SYSCTL_OPAQUE(_net_inet6_ip6, OID_AUTO, mif6table, CTLFLAG_RD,
-    &mif6table, sizeof(mif6table), "S,vif[MAXMIFS]",
-    "Multicast Interfaces (struct mif[MAXMIFS], netinet6/ip6_mroute.h)");
+    &mif6table, sizeof(mif6table), "S,mif6[MAXMIFS]",
+    "IPv6 Multicast Interfaces (struct mif6[MAXMIFS], netinet6/ip6_mroute.h)");
+
+static struct mtx mif6_mtx;
+#define	MIF6_LOCK()		mtx_lock(&mif6_mtx)
+#define	MIF6_UNLOCK()		mtx_unlock(&mif6_mtx)
+#define	MIF6_LOCK_ASSERT()	mtx_assert(&mif6_mtx, MA_OWNED)
+#define	MIF6_LOCK_INIT()	\
+	mtx_init(&mif6_mtx, "IPv6 multicast interfaces", NULL, MTX_DEF)
+#define	MIF6_LOCK_DESTROY()	mtx_destroy(&mif6_mtx)
 
 #ifdef MRT6DEBUG
 #ifdef VIMAGE_GLOBALS
@@ -200,11 +231,9 @@ static void	expire_upcalls(void *);
 #define	EXPIRE_TIMEOUT	(hz / 4)	/* 4x / second */
 #define	UPCALL_EXPIRE	6		/* number of timeouts */
 
-#ifdef INET
-#ifdef MROUTING
-extern struct socket *ip_mrouter;
-#endif
-#endif
+/*
+ * XXX TODO: maintain a count to if_allmulti() calls in struct ifnet.
+ */
 
 /*
  * 'Interfaces' associated with decapsulator (so we can tell
@@ -298,21 +327,22 @@ static u_long upcall_data[UPCALL_MAX + 1];
 static void collate();
 #endif /* UPCALL_TIMING */
 
-static int get_sg_cnt(struct sioc_sg_req6 *);
-static int get_mif6_cnt(struct sioc_mif_req6 *);
 static int ip6_mrouter_init(struct socket *, int, int);
-static int add_m6if(struct mif6ctl *);
-static int del_m6if(mifi_t *);
 static int add_m6fc(struct mf6cctl *);
+static int add_m6if(struct mif6ctl *);
 static int del_m6fc(struct mf6cctl *);
+static int del_m6if(mifi_t *);
+static int del_m6if_locked(mifi_t *);
+static int get_mif6_cnt(struct sioc_mif_req6 *);
+static int get_sg_cnt(struct sioc_sg_req6 *);
 
 static struct callout expire_upcalls_ch;
 
-int X_ip6_mforward(struct ip6_hdr *ip6, struct ifnet *ifp, struct mbuf *m);
+int X_ip6_mforward(struct ip6_hdr *, struct ifnet *, struct mbuf *);
 int X_ip6_mrouter_done(void);
-int X_ip6_mrouter_set(struct socket *so, struct sockopt *sopt);
-int X_ip6_mrouter_get(struct socket *so, struct sockopt *sopt);
-int X_mrt6_ioctl(int cmd, caddr_t data);
+int X_ip6_mrouter_set(struct socket *, struct sockopt *);
+int X_ip6_mrouter_get(struct socket *, struct sockopt *);
+int X_mrt6_ioctl(int, caddr_t);
 
 static void
 pim6_init(void)
@@ -419,14 +449,24 @@ X_ip6_mrouter_get(struct socket *so, struct sockopt *sopt)
 int
 X_mrt6_ioctl(int cmd, caddr_t data)
 {
+	int ret;
+
+	ret = EINVAL;
+
 	switch (cmd) {
 	case SIOCGETSGCNT_IN6:
-		return (get_sg_cnt((struct sioc_sg_req6 *)data));
+		ret = get_sg_cnt((struct sioc_sg_req6 *)data);
+		break;
+
 	case SIOCGETMIFCNT_IN6:
-		return (get_mif6_cnt((struct sioc_mif_req6 *)data));
+		ret = get_mif6_cnt((struct sioc_mif_req6 *)data);
+		break;
+
 	default:
-		return (EINVAL);
+		break;
 	}
+
+	return (ret);
 }
 
 /*
@@ -436,22 +476,24 @@ static int
 get_sg_cnt(struct sioc_sg_req6 *req)
 {
 	struct mf6c *rt;
-	int s;
+	int ret;
+
+	ret = 0;
+
+	MFC6_LOCK();
 
-	s = splnet();
 	MF6CFIND(req->src.sin6_addr, req->grp.sin6_addr, rt);
-	splx(s);
-	if (rt != NULL) {
+	if (rt == NULL) {
+		ret = ESRCH;
+	} else {
 		req->pktcnt = rt->mf6c_pkt_cnt;
 		req->bytecnt = rt->mf6c_byte_cnt;
 		req->wrong_if = rt->mf6c_wrong_if;
-	} else
-		return (ESRCH);
-#if 0
-		req->pktcnt = req->bytecnt = req->wrong_if = 0xffffffff;
-#endif
+	}
 
-	return (0);
+	MFC6_UNLOCK();
+
+	return (ret);
 }
 
 /*
@@ -460,17 +502,26 @@ get_sg_cnt(struct sioc_sg_req6 *req)
 static int
 get_mif6_cnt(struct sioc_mif_req6 *req)
 {
-	mifi_t mifi = req->mifi;
+	mifi_t mifi;
+	int ret;
 
-	if (mifi >= nummifs)
-		return (EINVAL);
+	ret = 0;
+	mifi = req->mifi;
 
-	req->icount = mif6table[mifi].m6_pkt_in;
-	req->ocount = mif6table[mifi].m6_pkt_out;
-	req->ibytes = mif6table[mifi].m6_bytes_in;
-	req->obytes = mif6table[mifi].m6_bytes_out;
+	MIF6_LOCK();
 
-	return (0);
+	if (mifi >= nummifs) {
+		ret = EINVAL;
+	} else {
+		req->icount = mif6table[mifi].m6_pkt_in;
+		req->ocount = mif6table[mifi].m6_pkt_out;
+		req->ibytes = mif6table[mifi].m6_bytes_in;
+		req->obytes = mif6table[mifi].m6_bytes_out;
+	}
+
+	MIF6_UNLOCK();
+
+	return (ret);
 }
 
 static int
@@ -507,8 +558,12 @@ ip6_mrouter_init(struct socket *so, int v, int cmd)
 	if (v != 1)
 		return (ENOPROTOOPT);
 
-	if (ip6_mrouter != NULL)
+	MROUTER6_LOCK();
+
+	if (ip6_mrouter != NULL) {
+		MROUTER6_UNLOCK();
 		return (EADDRINUSE);
+	}
 
 	ip6_mrouter = so;
 	V_ip6_mrouter_ver = cmd;
@@ -522,6 +577,8 @@ ip6_mrouter_init(struct socket *so, int v, int cmd)
 	callout_reset(&expire_upcalls_ch, EXPIRE_TIMEOUT,
 	    expire_upcalls, NULL);
 
+	MROUTER6_UNLOCK();
+
 #ifdef MRT6DEBUG
 	if (V_mrt6debug)
 		log(LOG_DEBUG, "ip6_mrouter_init\n");
@@ -531,7 +588,7 @@ ip6_mrouter_init(struct socket *so, int v, int cmd)
 }
 
 /*
- * Disable multicast routing
+ * Disable IPv6 multicast forwarding.
  */
 int
 X_ip6_mrouter_done(void)
@@ -541,31 +598,22 @@ X_ip6_mrouter_done(void)
 	int i;
 	struct mf6c *rt;
 	struct rtdetq *rte;
-	int s;
 
-	s = splnet();
+	MROUTER6_LOCK();
+
+	if (ip6_mrouter == NULL) {
+		MROUTER6_UNLOCK();
+		return (EINVAL);
+	}
 
 	/*
 	 * For each phyint in use, disable promiscuous reception of all IPv6
 	 * multicasts.
 	 */
-#ifdef INET
-#ifdef MROUTING
-	/*
-	 * If there is still IPv4 multicast routing daemon,
-	 * we remain interfaces to receive all muliticasted packets.
-	 * XXX: there may be an interface in which the IPv4 multicast
-	 * daemon is not interested...
-	 */
-	if (!V_ip_mrouter)
-#endif
-#endif
-	{
-		for (mifi = 0; mifi < nummifs; mifi++) {
-			if (mif6table[mifi].m6_ifp &&
-			    !(mif6table[mifi].m6_flags & MIFF_REGISTER)) {
-				if_allmulti(mif6table[mifi].m6_ifp, 0);
-			}
+	for (mifi = 0; mifi < nummifs; mifi++) {
+		if (mif6table[mifi].m6_ifp &&
+		    !(mif6table[mifi].m6_flags & MIFF_REGISTER)) {
+			if_allmulti(mif6table[mifi].m6_ifp, 0);
 		}
 	}
 	bzero((caddr_t)mif6table, sizeof(mif6table));
@@ -578,6 +626,7 @@ X_ip6_mrouter_done(void)
 	/*
 	 * Free all multicast forwarding cache entries.
 	 */
+	MFC6_LOCK();
 	for (i = 0; i < MF6CTBLSIZ; i++) {
 		rt = mf6ctable[i];
 		while (rt) {
@@ -595,8 +644,8 @@ X_ip6_mrouter_done(void)
 			free(frt, M_MRTABLE6);
 		}
 	}
-
 	bzero((caddr_t)mf6ctable, sizeof(mf6ctable));
+	MFC6_UNLOCK();
 
 	/*
 	 * Reset register interface
@@ -611,7 +660,7 @@ X_ip6_mrouter_done(void)
 	ip6_mrouter = NULL;
 	V_ip6_mrouter_ver = 0;
 
-	splx(s);
+	MROUTER6_UNLOCK();
 
 #ifdef MRT6DEBUG
 	if (V_mrt6debug)
@@ -632,15 +681,24 @@ add_m6if(struct mif6ctl *mifcp)
 	INIT_VNET_NET(curvnet);
 	struct mif6 *mifp;
 	struct ifnet *ifp;
-	int error, s;
+	int error;
 
-	if (mifcp->mif6c_mifi >= MAXMIFS)
+	MIF6_LOCK();
+
+	if (mifcp->mif6c_mifi >= MAXMIFS) {
+		MIF6_UNLOCK();
 		return (EINVAL);
+	}
 	mifp = mif6table + mifcp->mif6c_mifi;
-	if (mifp->m6_ifp)
+	if (mifp->m6_ifp != NULL) {
+		MIF6_UNLOCK();
 		return (EADDRINUSE); /* XXX: is it appropriate? */
-	if (mifcp->mif6c_pifi == 0 || mifcp->mif6c_pifi > V_if_index)
+	}
+	if (mifcp->mif6c_pifi == 0 || mifcp->mif6c_pifi > V_if_index) {
+		MIF6_UNLOCK();
 		return (ENXIO);
+	}
+
 	ifp = ifnet_byindex(mifcp->mif6c_pifi);
 
 	if (mifcp->mif6c_flags & MIFF_REGISTER) {
@@ -661,21 +719,20 @@ add_m6if(struct mif6ctl *mifcp)
 		} else {
 			ifp = multicast_register_if6;
 		}
-
-	} /* if REGISTER */
-	else {
+	} else {
 		/* Make sure the interface supports multicast */
-		if ((ifp->if_flags & IFF_MULTICAST) == 0)
+		if ((ifp->if_flags & IFF_MULTICAST) == 0) {
+			MIF6_UNLOCK();
 			return (EOPNOTSUPP);
+		}
 
-		s = splnet();
 		error = if_allmulti(ifp, 1);
-		splx(s);
-		if (error)
+		if (error) {
+			MIF6_UNLOCK();
 			return (error);
+		}
 	}
 
-	s = splnet();
 	mifp->m6_flags     = mifcp->mif6c_flags;
 	mifp->m6_ifp       = ifp;
 
@@ -684,12 +741,14 @@ add_m6if(struct mif6ctl *mifcp)
 	mifp->m6_pkt_out   = 0;
 	mifp->m6_bytes_in  = 0;
 	mifp->m6_bytes_out = 0;
-	splx(s);
+	bzero(&mifp->m6_route, sizeof(mifp->m6_route));
 
 	/* Adjust nummifs up if the mifi is higher than nummifs */
 	if (nummifs <= mifcp->mif6c_mifi)
 		nummifs = mifcp->mif6c_mifi + 1;
 
+	MIF6_UNLOCK();
+
 #ifdef MRT6DEBUG
 	if (V_mrt6debug)
 		log(LOG_DEBUG,
@@ -705,27 +764,22 @@ add_m6if(struct mif6ctl *mifcp)
  * Delete a mif from the mif table
  */
 static int
-del_m6if(mifi_t *mifip)
+del_m6if_locked(mifi_t *mifip)
 {
 	struct mif6 *mifp = mif6table + *mifip;
 	mifi_t mifi;
 	struct ifnet *ifp;
-	int s;
+
+	MIF6_LOCK_ASSERT();
 
 	if (*mifip >= nummifs)
 		return (EINVAL);
 	if (mifp->m6_ifp == NULL)
 		return (EINVAL);
 
-	s = splnet();
-
 	if (!(mifp->m6_flags & MIFF_REGISTER)) {
-		/*
-		 * XXX: what if there is yet IPv4 multicast daemon
-		 *      using the interface?
-		 */
+		/* XXX: TODO: Maintain an ALLMULTI refcount in struct ifnet. */
 		ifp = mifp->m6_ifp;
-
 		if_allmulti(ifp, 0);
 	} else {
 		if (reg_mif_num != (mifi_t)-1 &&
@@ -745,8 +799,6 @@ del_m6if(mifi_t *mifip)
 			break;
 	nummifs = mifi;
 
-	splx(s);
-
 #ifdef MRT6DEBUG
 	if (V_mrt6debug)
 		log(LOG_DEBUG, "del_m6if %d, nummifs %d\n", *mifip, nummifs);
@@ -755,6 +807,18 @@ del_m6if(mifi_t *mifip)
 	return (0);
 }
 
+static int
+del_m6if(mifi_t *mifip)
+{
+	int cc;
+
+	MIF6_LOCK();
+	cc = del_m6if_locked(mifip);
+	MIF6_UNLOCK();
+
+	return (cc);
+}
+
 /*
  * Add an mfc entry
  */
@@ -765,9 +829,10 @@ add_m6fc(struct mf6cctl *mfccp)
 	u_long hash;
 	struct rtdetq *rte;
 	u_short nstl;
-	int s;
 	char ip6bufo[INET6_ADDRSTRLEN], ip6bufg[INET6_ADDRSTRLEN];
 
+	MFC6_LOCK();
+
 	MF6CFIND(mfccp->mf6cc_origin.sin6_addr,
 		 mfccp->mf6cc_mcastgrp.sin6_addr, rt);
 
@@ -783,17 +848,16 @@ add_m6fc(struct mf6cctl *mfccp)
 		}
 #endif
 
-		s = splnet();
 		rt->mf6c_parent = mfccp->mf6cc_parent;
 		rt->mf6c_ifset = mfccp->mf6cc_ifset;
-		splx(s);
+
+		MFC6_UNLOCK();
 		return (0);
 	}
 
 	/*
 	 * Find the entry for which the upcall was made and update
 	 */
-	s = splnet();
 	hash = MF6CHASH(mfccp->mf6cc_origin.sin6_addr,
 			mfccp->mf6cc_mcastgrp.sin6_addr);
 	for (rt = mf6ctable[hash], nstl = 0; rt; rt = rt->mf6c_next) {
@@ -891,7 +955,7 @@ add_m6fc(struct mf6cctl *mfccp)
 			rt = (struct mf6c *)malloc(sizeof(*rt), M_MRTABLE6,
 						  M_NOWAIT);
 			if (rt == NULL) {
-				splx(s);
+				MFC6_UNLOCK();
 				return (ENOBUFS);
 			}
 
@@ -912,7 +976,8 @@ add_m6fc(struct mf6cctl *mfccp)
 			mf6ctable[hash] = rt;
 		}
 	}
-	splx(s);
+
+	MFC6_UNLOCK();
 	return (0);
 }
 
@@ -953,7 +1018,6 @@ del_m6fc(struct mf6cctl *mfccp)
 	struct mf6c		*rt;
 	struct mf6c		**nptr;
 	u_long		hash;
-	int s;
 
 	origin = mfccp->mf6cc_origin;
 	mcastgrp = mfccp->mf6cc_mcastgrp;
@@ -968,7 +1032,7 @@ del_m6fc(struct mf6cctl *mfccp)
 	}
 #endif
 
-	s = splnet();
+	MFC6_LOCK();
 
 	nptr = &mf6ctable[hash];
 	while ((rt = *nptr) != NULL) {
@@ -982,14 +1046,14 @@ del_m6fc(struct mf6cctl *mfccp)
 		nptr = &rt->mf6c_next;
 	}
 	if (rt == NULL) {
-		splx(s);
+		MFC6_UNLOCK();
 		return (EADDRNOTAVAIL);
 	}
 
 	*nptr = rt->mf6c_next;
 	free(rt, M_MRTABLE6);
 
-	splx(s);
+	MFC6_UNLOCK();
 
 	return (0);
 }
@@ -1035,7 +1099,6 @@ X_ip6_mforward(struct ip6_hdr *ip6, struct ifnet *ifp, struct mbuf *m)
 	struct mf6c *rt;
 	struct mif6 *mifp;
 	struct mbuf *mm;
-	int s;
 	mifi_t mifi;
 	char ip6bufs[INET6_ADDRSTRLEN], ip6bufd[INET6_ADDRSTRLEN];
 
@@ -1078,15 +1141,16 @@ X_ip6_mforward(struct ip6_hdr *ip6, struct ifnet *ifp, struct mbuf *m)
 		return (0);
 	}
 
+	MFC6_LOCK();
+
 	/*
 	 * Determine forwarding mifs from the forwarding cache table
 	 */
-	s = splnet();
 	MF6CFIND(ip6->ip6_src, ip6->ip6_dst, rt);
 
 	/* Entry exists, so forward if necessary */
 	if (rt) {
-		splx(s);
+		MFC6_UNLOCK();
 		return (ip6_mdq(m, ifp, rt));
 	} else {
 		/*
@@ -1120,7 +1184,7 @@ X_ip6_mforward(struct ip6_hdr *ip6, struct ifnet *ifp, struct mbuf *m)
 		rte = (struct rtdetq *)malloc(sizeof(*rte), M_MRTABLE6,
 					      M_NOWAIT);
 		if (rte == NULL) {
-			splx(s);
+			MFC6_UNLOCK();
 			return (ENOBUFS);
 		}
 		mb0 = m_copy(m, 0, M_COPYALL);
@@ -1133,7 +1197,7 @@ X_ip6_mforward(struct ip6_hdr *ip6, struct ifnet *ifp, struct mbuf *m)
 			mb0 = m_pullup(mb0, sizeof(struct ip6_hdr));
 		if (mb0 == NULL) {
 			free(rte, M_MRTABLE6);
-			splx(s);
+			MFC6_UNLOCK();
 			return (ENOBUFS);
 		}
 
@@ -1160,7 +1224,7 @@ X_ip6_mforward(struct ip6_hdr *ip6, struct ifnet *ifp, struct mbuf *m)
 			if (rt == NULL) {
 				free(rte, M_MRTABLE6);
 				m_freem(mb0);
-				splx(s);
+				MFC6_UNLOCK();
 				return (ENOBUFS);
 			}
 			/*
@@ -1173,7 +1237,7 @@ X_ip6_mforward(struct ip6_hdr *ip6, struct ifnet *ifp, struct mbuf *m)
 				free(rte, M_MRTABLE6);
 				m_freem(mb0);
 				free(rt, M_MRTABLE6);
-				splx(s);
+				MFC6_UNLOCK();
 				return (ENOBUFS);
 			}
 
@@ -1203,7 +1267,7 @@ X_ip6_mforward(struct ip6_hdr *ip6, struct ifnet *ifp, struct mbuf *m)
 				free(rte, M_MRTABLE6);
 				m_freem(mb0);
 				free(rt, M_MRTABLE6);
-				splx(s);
+				MFC6_UNLOCK();
 				return (EINVAL);
 			}
 
@@ -1236,7 +1300,7 @@ X_ip6_mforward(struct ip6_hdr *ip6, struct ifnet *ifp, struct mbuf *m)
 				free(rte, M_MRTABLE6);
 				m_freem(mb0);
 				free(rt, M_MRTABLE6);
-				splx(s);
+				MFC6_UNLOCK();
 				return (ENOBUFS);
 			}
 
@@ -1269,7 +1333,7 @@ X_ip6_mforward(struct ip6_hdr *ip6, struct ifnet *ifp, struct mbuf *m)
 					mrt6stat.mrt6s_upq_ovflw++;
 					free(rte, M_MRTABLE6);
 					m_freem(mb0);
-					splx(s);
+					MFC6_UNLOCK();
 					return (0);
 				}
 
@@ -1284,7 +1348,7 @@ X_ip6_mforward(struct ip6_hdr *ip6, struct ifnet *ifp, struct mbuf *m)
 		rte->t = tp;
 #endif /* UPCALL_TIMING */
 
-		splx(s);
+		MFC6_UNLOCK();
 
 		return (0);
 	}
@@ -1300,9 +1364,8 @@ expire_upcalls(void *unused)
 	struct rtdetq *rte;
 	struct mf6c *mfc, **nptr;
 	int i;
-	int s;
 
-	s = splnet();
+	MFC6_LOCK();
 	for (i = 0; i < MF6CTBLSIZ; i++) {
 		if (n6expire[i] == 0)
 			continue;
@@ -1346,7 +1409,7 @@ expire_upcalls(void *unused)
 			}
 		}
 	}
-	splx(s);
+	MFC6_UNLOCK();
 	callout_reset(&expire_upcalls_ch, EXPIRE_TIMEOUT,
 	    expire_upcalls, NULL);
 }
@@ -1540,8 +1603,6 @@ phyint_send(struct ip6_hdr *ip6, struct mif6 *mifp, struct mbuf *m)
 	struct mbuf *mb_copy;
 	struct ifnet *ifp = mifp->m6_ifp;
 	int error = 0;
-	int s = splnet();	/* needs to protect static "ro" below. */
-	static struct route_in6 ro;
 	struct	in6_multi *in6m;
 	struct sockaddr_in6 *dst6;
 	u_long linkmtu;
@@ -1556,7 +1617,6 @@ phyint_send(struct ip6_hdr *ip6, struct mif6 *mifp, struct mbuf *m)
 	    (M_HASCL(mb_copy) || mb_copy->m_len < sizeof(struct ip6_hdr)))
 		mb_copy = m_pullup(mb_copy, sizeof(struct ip6_hdr));
 	if (mb_copy == NULL) {
-		splx(s);
 		return;
 	}
 	/* set MCAST flag to the outgoing packet */
@@ -1576,7 +1636,7 @@ phyint_send(struct ip6_hdr *ip6, struct mif6 *mifp, struct mbuf *m)
 		/* XXX: ip6_output will override ip6->ip6_hlim */
 		im6o.im6o_multicast_hlim = ip6->ip6_hlim;
 		im6o.im6o_multicast_loop = 1;
-		error = ip6_output(mb_copy, NULL, &ro,
+		error = ip6_output(mb_copy, NULL, &mifp->m6_route,
 				   IPV6_FORWARDING, &im6o, NULL, NULL);
 
 #ifdef MRT6DEBUG
@@ -1584,7 +1644,6 @@ phyint_send(struct ip6_hdr *ip6, struct mif6 *mifp, struct mbuf *m)
 			log(LOG_DEBUG, "phyint_send on mif %d err %d\n",
 			    mifp - mif6table, error);
 #endif
-		splx(s);
 		return;
 	}
 
@@ -1592,13 +1651,13 @@ phyint_send(struct ip6_hdr *ip6, struct mif6 *mifp, struct mbuf *m)
 	 * If we belong to the destination multicast group
 	 * on the outgoing interface, loop back a copy.
 	 */
-	dst6 = (struct sockaddr_in6 *)&ro.ro_dst;
+	dst6 = &mifp->m6_route.ro_dst;
 	IN6_LOOKUP_MULTI(ip6->ip6_dst, ifp, in6m);
 	if (in6m != NULL) {
 		dst6->sin6_len = sizeof(struct sockaddr_in6);
 		dst6->sin6_family = AF_INET6;
 		dst6->sin6_addr = ip6->ip6_dst;
-		ip6_mloopback(ifp, m, (struct sockaddr_in6 *)&ro.ro_dst);
+		ip6_mloopback(ifp, m, &mifp->m6_route.ro_dst);
 	}
 	/*
 	 * Put the packet into the sending queue of the outgoing interface
@@ -1614,7 +1673,7 @@ phyint_send(struct ip6_hdr *ip6, struct mif6 *mifp, struct mbuf *m)
 		 * we need no ND for a multicast forwarded packet...right?
 		 */
 		error = (*ifp->if_output)(ifp, mb_copy,
-		    (struct sockaddr *)&ro.ro_dst, NULL);
+		    (struct sockaddr *)&mifp->m6_route.ro_dst, NULL);
 #ifdef MRT6DEBUG
 		if (V_mrt6debug & DEBUG_XMIT)
 			log(LOG_DEBUG, "phyint_send on mif %d err %d\n",
@@ -1645,8 +1704,6 @@ phyint_send(struct ip6_hdr *ip6, struct mif6 *mifp, struct mbuf *m)
 			m_freem(mb_copy); /* simply discard the packet */
 		}
 	}
-
-	splx(s);
 }
 
 static int
@@ -1715,6 +1772,24 @@ register_send(struct ip6_hdr *ip6, struct mif6 *mif, struct mbuf *m)
 }
 
 /*
+ * pim6_encapcheck() is called by the encap6_input() path at runtime to
+ * determine if a packet is for PIM; allowing PIM to be dynamically loaded
+ * into the kernel.
+ */
+static int
+pim6_encapcheck(const struct mbuf *m, int off, int proto, void *arg)
+{
+
+#ifdef DIAGNOSTIC
+    KASSERT(proto == IPPROTO_PIM, ("not for IPPROTO_PIM"));
+#endif
+    if (proto != IPPROTO_PIM)
+	return 0;	/* not for us; reject the datagram. */
+
+    return 64;		/* claim the datagram. */
+}
+
+/*
  * PIM sparse mode hook
  * Receives the pim control messages, and passes them up to the listening
  * socket, using rip6_input.
@@ -1951,3 +2026,66 @@ pim6_input(struct mbuf **mp, int *offp, int proto)
 	rip6_input(&m, offp, proto);
 	return (IPPROTO_DONE);
 }
+
+static int
+ip6_mroute_modevent(module_t mod, int type, void *unused)
+{
+
+	switch (type) {
+	case MOD_LOAD:
+		MROUTER6_LOCK_INIT();
+		MFC6_LOCK_INIT();
+		MIF6_LOCK_INIT();
+
+		pim6_encap_cookie = encap_attach_func(AF_INET6, IPPROTO_PIM,
+			pim6_encapcheck,
+			(const struct protosw *)&in6_pim_protosw, NULL);
+		if (pim6_encap_cookie == NULL) {
+			printf("ip6_mroute: unable to attach pim6 encap\n");
+			MIF6_LOCK_DESTROY();
+			MFC6_LOCK_DESTROY();
+			MROUTER6_LOCK_DESTROY();
+			return (EINVAL);
+		}
+
+		ip6_mforward = X_ip6_mforward;
+		ip6_mrouter_done = X_ip6_mrouter_done;
+		ip6_mrouter_get = X_ip6_mrouter_get;
+		ip6_mrouter_set = X_ip6_mrouter_set;
+		mrt6_ioctl = X_mrt6_ioctl;
+		break;
+
+	case MOD_UNLOAD:
+		if (ip6_mrouter != NULL)
+			return EINVAL;
+
+		if (pim6_encap_cookie) {
+			encap_detach(pim6_encap_cookie);
+			pim6_encap_cookie = NULL;
+		}
+		X_ip6_mrouter_done();
+		ip6_mforward = NULL;
+		ip6_mrouter_done = NULL;
+		ip6_mrouter_get = NULL;
+		ip6_mrouter_set = NULL;
+		mrt6_ioctl = NULL;
+
+		MIF6_LOCK_DESTROY();
+		MFC6_LOCK_DESTROY();
+		MROUTER6_LOCK_DESTROY();
+		break;
+
+	default:
+		return (EOPNOTSUPP);
+	}
+
+	return (0);
+}
+
+static moduledata_t ip6_mroutemod = {
+	"ip6_mroute",
+	ip6_mroute_modevent,
+	0
+};
+
+DECLARE_MODULE(ip6_mroute, ip6_mroutemod, SI_SUB_PSEUDO, SI_ORDER_ANY);
diff --git a/sys/netinet6/ip6_mroute.h b/sys/netinet6/ip6_mroute.h
index 0e35e62..a0a5e4a 100644
--- a/sys/netinet6/ip6_mroute.h
+++ b/sys/netinet6/ip6_mroute.h
@@ -212,7 +212,7 @@ struct mif6 {
 	u_quad_t	m6_pkt_out;	/* # pkts out on interface           */
 	u_quad_t	m6_bytes_in;	/* # bytes in on interface	     */
 	u_quad_t	m6_bytes_out;	/* # bytes out on interface	     */
-	struct route_in6 m6_route;/* cached route if this is a tunnel */
+	struct route_in6 m6_route;	/* cached route */
 #ifdef notyet
 	u_int		m6_rsvp_on;	/* RSVP listening on this vif */
 	struct socket   *m6_rsvpd;	/* RSVP daemon socket */
diff --git a/sys/netinet6/mld6.c b/sys/netinet6/mld6.c
index 3e0f921..29b71be 100644
--- a/sys/netinet6/mld6.c
+++ b/sys/netinet6/mld6.c
@@ -288,7 +288,7 @@ mld6_input(struct mbuf *m, int off)
 #else
 	IP6_EXTHDR_GET(mldh, struct mld_hdr *, m, off, sizeof(*mldh));
 	if (mldh == NULL) {
-		V_icmp6stat.icp6s_tooshort++;
+		ICMP6STAT_INC(icp6s_tooshort);
 		return;
 	}
 #endif
@@ -517,7 +517,7 @@ mld6_sendpkt(struct in6_multi *in6m, int type, const struct in6_addr *dst)
 	im6o.im6o_multicast_loop = (ip6_mrouter != NULL);
 
 	/* increment output statictics */
-	V_icmp6stat.icp6s_outhist[type]++;
+	ICMP6STAT_INC(icp6s_outhist[type]);
 
 	ip6_output(mh, &V_ip6_opts, NULL, 0, &im6o, &outif, NULL);
 	if (outif) {
@@ -550,7 +550,6 @@ in6_addmulti(struct in6_addr *maddr6, struct ifnet *ifp,
 	*errorp = 0;
 	in6m = NULL;
 
-	IFF_LOCKGIANT(ifp);
 	/*IN6_MULTI_LOCK();*/
 
 	IN6_LOOKUP_MULTI(*maddr6, ifp, in6m);
@@ -622,7 +621,6 @@ in6_addmulti(struct in6_addr *maddr6, struct ifnet *ifp,
 	} while (0);
 
 	/*IN6_MULTI_UNLOCK();*/
-	IFF_UNLOCKGIANT(ifp);
 
 	return (in6m);
 }
diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c
index 535f329..857d3fc 100644
--- a/sys/netinet6/nd6.c
+++ b/sys/netinet6/nd6.c
@@ -374,7 +374,7 @@ nd6_options(union nd_opts *ndopts)
 			 * Message validation requires that all included
 			 * options have a length that is greater than zero.
 			 */
-			V_icmp6stat.icp6s_nd_badopt++;
+			ICMP6STAT_INC(icp6s_nd_badopt);
 			bzero(ndopts, sizeof(*ndopts));
 			return -1;
 		}
@@ -418,7 +418,7 @@ nd6_options(union nd_opts *ndopts)
 skip1:
 		i++;
 		if (i > V_nd6_maxndopt) {
-			V_icmp6stat.icp6s_nd_toomanyopt++;
+			ICMP6STAT_INC(icp6s_nd_toomanyopt);
 			nd6log((LOG_INFO, "too many loop in nd opt\n"));
 			break;
 		}
diff --git a/sys/netinet6/nd6_nbr.c b/sys/netinet6/nd6_nbr.c
index 1f88fb1..c83a245 100644
--- a/sys/netinet6/nd6_nbr.c
+++ b/sys/netinet6/nd6_nbr.c
@@ -128,7 +128,7 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len)
 #else
 	IP6_EXTHDR_GET(nd_ns, struct nd_neighbor_solicit *, m, off, icmp6len);
 	if (nd_ns == NULL) {
-		V_icmp6stat.icp6s_tooshort++;
+		ICMP6STAT_INC(icp6s_tooshort);
 		return;
 	}
 #endif
@@ -365,7 +365,7 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len)
 		ip6_sprintf(ip6bufs, &daddr6)));
 	nd6log((LOG_ERR, "nd6_ns_input: tgt=%s\n",
 		ip6_sprintf(ip6bufs, &taddr6)));
-	V_icmp6stat.icp6s_badns++;
+	ICMP6STAT_INC(icp6s_badns);
 	m_freem(m);
 }
 
@@ -563,7 +563,7 @@ nd6_ns_output(struct ifnet *ifp, const struct in6_addr *daddr6,
 	ip6_output(m, NULL, &ro, dad ? IPV6_UNSPECSRC : 0, &im6o, NULL, NULL);
 	icmp6_ifstat_inc(ifp, ifs6_out_msg);
 	icmp6_ifstat_inc(ifp, ifs6_out_neighborsolicit);
-	V_icmp6stat.icp6s_outhist[ND_NEIGHBOR_SOLICIT]++;
+	ICMP6STAT_INC(icp6s_outhist[ND_NEIGHBOR_SOLICIT]);
 
 	if (ro.ro_rt) {		/* we don't cache this route. */
 		RTFREE(ro.ro_rt);
@@ -625,7 +625,7 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len)
 #else
 	IP6_EXTHDR_GET(nd_na, struct nd_neighbor_advert *, m, off, icmp6len);
 	if (nd_na == NULL) {
-		V_icmp6stat.icp6s_tooshort++;
+		ICMP6STAT_INC(icp6s_tooshort);
 		return;
 	}
 #endif
@@ -897,7 +897,7 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len)
 	if (ln != NULL)
 		LLE_WUNLOCK(ln);
 
-	V_icmp6stat.icp6s_badna++;
+	ICMP6STAT_INC(icp6s_badna);
 	m_freem(m);
 }
 
@@ -1065,7 +1065,7 @@ nd6_na_output(struct ifnet *ifp, const struct in6_addr *daddr6_0,
 	ip6_output(m, NULL, &ro, 0, &im6o, NULL, NULL);
 	icmp6_ifstat_inc(ifp, ifs6_out_msg);
 	icmp6_ifstat_inc(ifp, ifs6_out_neighboradvert);
-	V_icmp6stat.icp6s_outhist[ND_NEIGHBOR_ADVERT]++;
+	ICMP6STAT_INC(icp6s_outhist[ND_NEIGHBOR_ADVERT]);
 
 	if (ro.ro_rt) {		/* we don't cache this route. */
 		RTFREE(ro.ro_rt);
diff --git a/sys/netinet6/nd6_rtr.c b/sys/netinet6/nd6_rtr.c
index e71f025..1b59db7 100644
--- a/sys/netinet6/nd6_rtr.c
+++ b/sys/netinet6/nd6_rtr.c
@@ -155,7 +155,7 @@ nd6_rs_input(struct mbuf *m, int off, int icmp6len)
 #else
 	IP6_EXTHDR_GET(nd_rs, struct nd_router_solicit *, m, off, icmp6len);
 	if (nd_rs == NULL) {
-		V_icmp6stat.icp6s_tooshort++;
+		ICMP6STAT_INC(icp6s_tooshort);
 		return;
 	}
 #endif
@@ -190,7 +190,7 @@ nd6_rs_input(struct mbuf *m, int off, int icmp6len)
 	return;
 
  bad:
-	V_icmp6stat.icp6s_badrs++;
+	ICMP6STAT_INC(icp6s_badrs);
 	m_freem(m);
 }
 
@@ -246,7 +246,7 @@ nd6_ra_input(struct mbuf *m, int off, int icmp6len)
 #else
 	IP6_EXTHDR_GET(nd_ra, struct nd_router_advert *, m, off, icmp6len);
 	if (nd_ra == NULL) {
-		V_icmp6stat.icp6s_tooshort++;
+		ICMP6STAT_INC(icp6s_tooshort);
 		return;
 	}
 #endif
@@ -422,7 +422,7 @@ nd6_ra_input(struct mbuf *m, int off, int icmp6len)
 	return;
 
  bad:
-	V_icmp6stat.icp6s_badra++;
+	ICMP6STAT_INC(icp6s_badra);
 	m_freem(m);
 }
 
diff --git a/sys/netinet6/raw_ip6.c b/sys/netinet6/raw_ip6.c
index bce1b14..2ac95e5 100644
--- a/sys/netinet6/raw_ip6.c
+++ b/sys/netinet6/raw_ip6.c
@@ -477,7 +477,7 @@ rip6_output(m, va_alist)
 	if (so->so_proto->pr_protocol == IPPROTO_ICMPV6) {
 		if (oifp)
 			icmp6_ifoutstat_inc(oifp, type, code);
-		V_icmp6stat.icp6s_outhist[type]++;
+		ICMP6STAT_INC(icp6s_outhist[type]);
 	} else
 		V_rip6stat.rip6s_opackets++;
 
diff --git a/sys/netinet6/scope6.c b/sys/netinet6/scope6.c
index 9611deb..b866cd9 100644
--- a/sys/netinet6/scope6.c
+++ b/sys/netinet6/scope6.c
@@ -83,8 +83,12 @@ scope6_init(void)
 #else
 	V_ip6_use_defzone = 0;
 #endif
-	SCOPE6_LOCK_INIT();
 	bzero(&V_sid_default, sizeof(V_sid_default));
+
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
+
+	SCOPE6_LOCK_INIT();
 }
 
 struct scope6_id *
diff --git a/sys/netinet6/udp6_usrreq.c b/sys/netinet6/udp6_usrreq.c
index 05d48f5..566cf92 100644
--- a/sys/netinet6/udp6_usrreq.c
+++ b/sys/netinet6/udp6_usrreq.c
@@ -166,7 +166,7 @@ udp6_append(struct inpcb *inp, struct mbuf *n, int off,
 		m_freem(n);
 		if (opts)
 			m_freem(opts);
-		V_udpstat.udps_fullsock++;
+		UDPSTAT_INC(udps_fullsock);
 	} else
 		sorwakeup_locked(so);
 }
@@ -202,7 +202,7 @@ udp6_input(struct mbuf **mp, int *offp, int proto)
 		return (IPPROTO_DONE);
 #endif
 
-	V_udpstat.udps_ipackets++;
+	UDPSTAT_INC(udps_ipackets);
 
 	/*
 	 * Destination port of 0 is illegal, based on RFC768.
@@ -214,7 +214,7 @@ udp6_input(struct mbuf **mp, int *offp, int proto)
 	ulen = ntohs((u_short)uh->uh_ulen);
 
 	if (plen != ulen) {
-		V_udpstat.udps_badlen++;
+		UDPSTAT_INC(udps_badlen);
 		goto badunlocked;
 	}
 
@@ -222,11 +222,11 @@ udp6_input(struct mbuf **mp, int *offp, int proto)
 	 * Checksum extended UDP header and data.
 	 */
 	if (uh->uh_sum == 0) {
-		V_udpstat.udps_nosum++;
+		UDPSTAT_INC(udps_nosum);
 		goto badunlocked;
 	}
 	if (in6_cksum(m, IPPROTO_UDP, off, ulen) != 0) {
-		V_udpstat.udps_badsum++;
+		UDPSTAT_INC(udps_badsum);
 		goto badunlocked;
 	}
 
@@ -327,8 +327,8 @@ udp6_input(struct mbuf **mp, int *offp, int proto)
 			 * to send an ICMP Port Unreachable for a broadcast
 			 * or multicast datgram.)
 			 */
-			V_udpstat.udps_noport++;
-			V_udpstat.udps_noportmcast++;
+			UDPSTAT_INC(udps_noport);
+			UDPSTAT_INC(udps_noportmcast);
 			goto badheadlocked;
 		}
 		INP_RLOCK(last);
@@ -365,10 +365,10 @@ udp6_input(struct mbuf **mp, int *offp, int proto)
 			    ip6_sprintf(ip6bufs, &ip6->ip6_src),
 			    ntohs(uh->uh_sport));
 		}
-		V_udpstat.udps_noport++;
+		UDPSTAT_INC(udps_noport);
 		if (m->m_flags & M_MCAST) {
 			printf("UDP6: M_MCAST is set in a unicast packet.\n");
-			V_udpstat.udps_noportmcast++;
+			UDPSTAT_INC(udps_noportmcast);
 			goto badheadlocked;
 		}
 		INP_INFO_RUNLOCK(&V_udbinfo);
@@ -719,7 +719,7 @@ udp6_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr6,
 
 		flags = 0;
 
-		V_udpstat.udps_opackets++;
+		UDPSTAT_INC(udps_opackets);
 		error = ip6_output(m, optp, NULL, flags, inp->in6p_moptions,
 		    NULL, inp);
 		break;
diff --git a/sys/netipsec/ipsec.c b/sys/netipsec/ipsec.c
index 00ce8de..a76afd2 100644
--- a/sys/netipsec/ipsec.c
+++ b/sys/netipsec/ipsec.c
@@ -103,6 +103,8 @@ struct vnet_ipsec vnet_ipsec_0;
 #endif
 #endif
 
+static int ipsec_iattach(const void *);
+
 #ifdef VIMAGE_GLOBALS
 /* NB: name changed so netstat doesn't use it. */
 struct ipsecstat ipsec4stat;
@@ -242,6 +244,15 @@ static void vshiftl __P((unsigned char *, int, int));
 
 MALLOC_DEFINE(M_IPSEC_INPCB, "inpcbpolicy", "inpcb-resident ipsec policy");
 
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_ipsec_modinfo = {
+	.vmi_id		= VNET_MOD_IPSEC,
+	.vmi_name	= "ipsec",
+	.vmi_dependson	= VNET_MOD_INET,	/* XXX revisit - INET6 ? */
+	.vmi_iattach	= ipsec_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
+
 void
 ipsec_init(void)
 {
@@ -1758,8 +1769,23 @@ static void
 ipsec_attach(void)
 {
 
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register(&vnet_ipsec_modinfo);
+#else
+	ipsec_iattach(NULL);
+#endif
+
+}
+
+static int
+ipsec_iattach(const void *unused __unused)
+{
+	INIT_VNET_IPSEC(curvnet);
+
 	SECPOLICY_LOCK_INIT(&V_ip4_def_policy);
 	V_ip4_def_policy.refcnt = 1;			/* NB: disallow free. */
+
+	return (0);
 }
 SYSINIT(ipsec, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST, ipsec_attach, NULL);
 
diff --git a/sys/netipsec/key.c b/sys/netipsec/key.c
index 74d31dd..70b68a8 100644
--- a/sys/netipsec/key.c
+++ b/sys/netipsec/key.c
@@ -2687,7 +2687,12 @@ key_delsah(sah)
 			if (sav->refcnt == 0) {
 				/* sanity check */
 				KEY_CHKSASTATE(state, sav->state, __func__);
-				KEY_FREESAV(&sav);
+				/* 
+				 * do NOT call KEY_FREESAV here:
+				 * it will only delete the sav if refcnt == 1,
+				 * where we already know that refcnt == 0
+				 */
+				key_delsav(sav);
 			} else {
 				/* give up to delete this sa */
 				zombie++;
@@ -4131,6 +4136,7 @@ key_flush_sad(time_t now)
 
 		/* if LARVAL entry doesn't become MATURE, delete it. */
 		LIST_FOREACH_SAFE(sav, &sah->savtree[SADB_SASTATE_LARVAL], chain, nextsav) {
+			/* Need to also check refcnt for a larval SA ??? */
 			if (now - sav->created > V_key_larval_lifetime)
 				KEY_FREESAV(&sav);
 		}
@@ -4155,11 +4161,16 @@ key_flush_sad(time_t now)
 			if (sav->lft_s->addtime != 0 &&
 			    now - sav->created > sav->lft_s->addtime) {
 				key_sa_chgstate(sav, SADB_SASTATE_DYING);
-				/* Actually, only send expire message if SA has been used, as it
-				 * was done before, but should we always send such message, and let IKE
-				 * daemon decide if it should be renegociated or not ?
-				 * XXX expire message will actually NOT be sent if SA is only used
-				 * after soft lifetime has been reached, see below (DYING state)
+				/* 
+				 * Actually, only send expire message if
+				 * SA has been used, as it was done before,
+				 * but should we always send such message,
+				 * and let IKE daemon decide if it should be
+				 * renegotiated or not ?
+				 * XXX expire message will actually NOT be
+				 * sent if SA is only used after soft
+				 * lifetime has been reached, see below
+				 * (DYING state)
 				 */
 				if (sav->lft_c->usetime != 0)
 					key_expire(sav);
@@ -7160,12 +7171,6 @@ key_init(void)
 	V_ipsec_esp_auth = 0;
 	V_ipsec_ah_keymin = 128;
 
-	SPTREE_LOCK_INIT();
-	REGTREE_LOCK_INIT();
-	SAHTREE_LOCK_INIT();
-	ACQ_LOCK_INIT();
-	SPACQ_LOCK_INIT();
-
 	for (i = 0; i < IPSEC_DIR_MAX; i++)
 		LIST_INIT(&V_sptree[i]);
 
@@ -7181,6 +7186,15 @@ key_init(void)
 	V_ip4_def_policy.policy = IPSEC_POLICY_NONE;
 	V_ip4_def_policy.refcnt++;	/*never reclaim this*/
 
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
+
+	SPTREE_LOCK_INIT();
+	REGTREE_LOCK_INIT();
+	SAHTREE_LOCK_INIT();
+	ACQ_LOCK_INIT();
+	SPACQ_LOCK_INIT();
+
 #ifndef IPSEC_DEBUG2
 	timeout((void *)key_timehandler, (void *)0, hz);
 #endif /*IPSEC_DEBUG2*/
diff --git a/sys/netipsec/xform_ah.c b/sys/netipsec/xform_ah.c
index 3a4c7dc..07d7001 100644
--- a/sys/netipsec/xform_ah.c
+++ b/sys/netipsec/xform_ah.c
@@ -73,6 +73,17 @@
 
 #include <opencrypto/cryptodev.h>
 
+static int ah_iattach(const void *);
+
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_ah_modinfo = {
+	.vmi_id		= VNET_MOD_AH,
+	.vmi_name	= "ipsec_ah",
+	.vmi_dependson	= VNET_MOD_IPSEC,
+	.vmi_iattach	= ah_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
+
 /*
  * Return header size in bytes.  The old protocol did not support
  * the replay counter; the new protocol always includes the counter.
@@ -1220,9 +1231,22 @@ static void
 ah_attach(void)
 {
 
+	xform_register(&ah_xformsw);
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register(&vnet_ah_modinfo);
+#else
+	ah_iattach(NULL);
+#endif
+}
+
+static int
+ah_iattach(const void *unused __unused)
+{
+	INIT_VNET_IPSEC(curvnet);
+
 	V_ah_enable = 1;	/* control flow of packets with AH */
 	V_ah_cleartos = 1;	/* clear ip_tos when doing AH calc */
 
-	xform_register(&ah_xformsw);
+	return (0);
 }
 SYSINIT(ah_xform_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE, ah_attach, NULL);
diff --git a/sys/netipsec/xform_esp.c b/sys/netipsec/xform_esp.c
index 98a2240..6508c14 100644
--- a/sys/netipsec/xform_esp.c
+++ b/sys/netipsec/xform_esp.c
@@ -90,6 +90,16 @@ SYSCTL_V_STRUCT(V_NET, vnet_ipsec, _net_inet_esp, IPSECCTL_STATS,
 
 static int esp_input_cb(struct cryptop *op);
 static int esp_output_cb(struct cryptop *crp);
+static int esp_iattach(const void *);
+
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_esp_modinfo = {
+	.vmi_id		= VNET_MOD_ESP,
+	.vmi_name	= "ipsec_esp",
+	.vmi_dependson	= VNET_MOD_IPSEC,
+	.vmi_iattach	= esp_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
 
 /*
  * NB: this is public for use by the PF_KEY support.
@@ -990,9 +1000,23 @@ static struct xformsw esp_xformsw = {
 static void
 esp_attach(void)
 {
+
+	xform_register(&esp_xformsw);
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register(&vnet_esp_modinfo);
+#else
+	esp_iattach(NULL);
+#endif
+}
+
+static int
+esp_iattach(const void *unused __unused)
+{
+	INIT_VNET_IPSEC(curvnet);
+
 #define	MAXIV(xform)					\
 	if (xform.blocksize > V_esp_max_ivlen)		\
-		V_esp_max_ivlen = xform.blocksize		\
+		V_esp_max_ivlen = xform.blocksize	\
 
 	V_esp_enable = 1;
 	V_esp_max_ivlen = 0;
@@ -1005,8 +1029,8 @@ esp_attach(void)
 	MAXIV(enc_xform_skipjack);	/* SADB_X_EALG_SKIPJACK */
 	MAXIV(enc_xform_null);		/* SADB_EALG_NULL */
 	MAXIV(enc_xform_camellia);	/* SADB_X_EALG_CAMELLIACBC */
-
-	xform_register(&esp_xformsw);
 #undef MAXIV
+
+	return (0);
 }
 SYSINIT(esp_xform_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE, esp_attach, NULL);
diff --git a/sys/netipsec/xform_ipcomp.c b/sys/netipsec/xform_ipcomp.c
index d64abf0f..8e2f1c4 100644
--- a/sys/netipsec/xform_ipcomp.c
+++ b/sys/netipsec/xform_ipcomp.c
@@ -80,6 +80,16 @@ SYSCTL_V_STRUCT(V_NET, vnet_ipsec, _net_inet_ipcomp, IPSECCTL_STATS,
 
 static int ipcomp_input_cb(struct cryptop *crp);
 static int ipcomp_output_cb(struct cryptop *crp);
+static int ipcomp_iattach(const void *);
+
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_ipcomp_modinfo = {
+	.vmi_id		= VNET_MOD_IPCOMP,
+	.vmi_name	= "ipsec_ipcomp",
+	.vmi_dependson	= VNET_MOD_IPSEC,
+	.vmi_iattach	= ipcomp_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
 
 struct comp_algo *
 ipcomp_algorithm_lookup(int alg)
@@ -600,7 +610,20 @@ static void
 ipcomp_attach(void)
 {
 
-	V_ipcomp_enable = 0;
 	xform_register(&ipcomp_xformsw);
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register(&vnet_ipcomp_modinfo);
+#else
+	ipcomp_iattach(NULL);
+#endif
+}
+
+static int
+ipcomp_iattach(const void *unused __unused)
+{
+	INIT_VNET_IPSEC(curvnet);
+
+	V_ipcomp_enable = 0;
+	return (0);
 }
 SYSINIT(ipcomp_xform_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE, ipcomp_attach, NULL);
diff --git a/sys/netipsec/xform_ipip.c b/sys/netipsec/xform_ipip.c
index cbc447c..4d97168 100644
--- a/sys/netipsec/xform_ipip.c
+++ b/sys/netipsec/xform_ipip.c
@@ -108,6 +108,16 @@ SYSCTL_V_STRUCT(V_NET, vnet_ipsec, _net_inet_ipip, IPSECCTL_STATS,
 #define	M_IPSEC	(M_AUTHIPHDR|M_AUTHIPDGM|M_DECRYPTED)
 
 static void _ipip_input(struct mbuf *m, int iphlen, struct ifnet *gifp);
+static int ipe4_iattach(const void *);
+
+#ifndef VIMAGE_GLOBALS
+static const vnet_modinfo_t vnet_ipip_modinfo = {
+	.vmi_id		= VNET_MOD_IPIP,
+	.vmi_name	= "ipsec_ipip",
+	.vmi_dependson	= VNET_MOD_IPSEC,
+	.vmi_iattach	= ipe4_iattach
+};
+#endif /* !VIMAGE_GLOBALS */
 
 #ifdef INET6
 /*
@@ -697,11 +707,18 @@ ipe4_encapcheck(const struct mbuf *m, int off, int proto, void *arg)
 	return ((m->m_flags & M_IPSEC) != 0 ? 1 : 0);
 }
 
-static void
-ipe4_attach(void)
+static int
+ipe4_iattach(const void *unused __unused)
 {
+	INIT_VNET_IPSEC(curvnet);
 
 	V_ipip_allow = 0;
+	return (0);
+}
+
+static void
+ipe4_attach(void)
+{
 
 	xform_register(&ipe4_xformsw);
 	/* attach to encapsulation framework */
@@ -712,6 +729,11 @@ ipe4_attach(void)
 	(void) encap_attach_func(AF_INET6, -1,
 		ipe4_encapcheck, (struct protosw *)&ipe6_protosw, NULL);
 #endif
+#ifndef VIMAGE_GLOBALS
+	vnet_mod_register(&vnet_ipip_modinfo);
+#else
+	ipe4_iattach(NULL);
+#endif
 }
 SYSINIT(ipe4_xform_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE, ipe4_attach, NULL);
 #endif	/* IPSEC */
diff --git a/sys/netnatm/natm.c b/sys/netnatm/natm.c
index b9705db..baa596e 100644
--- a/sys/netnatm/natm.c
+++ b/sys/netnatm/natm.c
@@ -221,13 +221,9 @@ natm_usr_connect(struct socket *so, struct sockaddr *nam, d_thread_t *p)
 	op.param.traffic = ATMIO_TRAFFIC_UBR;
 	NATM_UNLOCK();
 
-	IFF_LOCKGIANT(ifp);
 	if (ifp->if_ioctl == NULL || 
-	    ifp->if_ioctl(ifp, SIOCATMOPENVCC, (caddr_t)&op) != 0) {
-		IFF_UNLOCKGIANT(ifp);
+	    ifp->if_ioctl(ifp, SIOCATMOPENVCC, (caddr_t)&op) != 0)
 		return (EIO);
-	}
-	IFF_UNLOCKGIANT(ifp);
 	soisconnected(so);
 	return (error);
 }
@@ -259,11 +255,8 @@ natm_usr_disconnect(struct socket *so)
 	cl.vpi = npcb->npcb_vpi;
 	cl.vci = npcb->npcb_vci;
 	NATM_UNLOCK();
-	if (ifp->if_ioctl != NULL) {
-		IFF_LOCKGIANT(ifp);
+	if (ifp->if_ioctl != NULL)
 		ifp->if_ioctl(ifp, SIOCATMCLOSEVCC, (caddr_t)&cl);
-		IFF_UNLOCKGIANT(ifp);
-	}
 	soisdisconnected(so);
 	return (error);
 }
@@ -342,17 +335,13 @@ natm_usr_control(struct socket *so, u_long cmd, caddr_t arg,
 	struct ifnet *ifp, d_thread_t *p)
 {
 	struct natmpcb *npcb;
-	int error;
 
 	npcb = (struct natmpcb *)so->so_pcb;
 	KASSERT(npcb != NULL, ("natm_usr_control: npcb == NULL"));
 
 	if (ifp == NULL || ifp->if_ioctl == NULL)
 		return (EOPNOTSUPP);
-	IFF_LOCKGIANT(ifp);
-	error = ((*ifp->if_ioctl)(ifp, cmd, arg));
-	IFF_UNLOCKGIANT(ifp);
-	return (error);
+	return ((*ifp->if_ioctl)(ifp, cmd, arg));
 }
 
 static void
diff --git a/sys/nfs/nfs_nfssvc.c b/sys/nfs/nfs_nfssvc.c
new file mode 100644
index 0000000..9379fbe
--- /dev/null
+++ b/sys/nfs/nfs_nfssvc.c
@@ -0,0 +1,153 @@
+/*-
+ * Copyright (c) 1989, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * Rick Macklem at The University of Guelph.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_nfs.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/sysproto.h>
+#include <sys/kernel.h>
+#include <sys/sysctl.h>
+#include <sys/priv.h>
+#include <sys/proc.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/module.h>
+#include <sys/sysent.h>
+#include <sys/syscall.h>
+#include <sys/sysproto.h>
+
+#include <security/audit/audit.h>
+
+#include <nfs/nfssvc.h>
+
+static int nfssvc_offset = SYS_nfssvc;
+static struct sysent nfssvc_prev_sysent;
+MAKE_SYSENT(nfssvc);
+
+/*
+ * This tiny module simply handles the nfssvc() system call. The other
+ * nfs modules that use the system call register themselves by setting
+ * the nfsd_call_xxx function pointers non-NULL.
+ */
+
+int (*nfsd_call_nfsserver)(struct thread *, struct nfssvc_args *) = NULL;
+int (*nfsd_call_nfscommon)(struct thread *, struct nfssvc_args *) = NULL;
+int (*nfsd_call_nfscl)(struct thread *, struct nfssvc_args *) = NULL;
+int (*nfsd_call_nfsd)(struct thread *, struct nfssvc_args *) = NULL;
+
+/*
+ * Nfs server psuedo system call for the nfsd's
+ */
+int
+nfssvc(struct thread *td, struct nfssvc_args *uap)
+{
+	int error;
+
+	KASSERT(!mtx_owned(&Giant), ("nfssvc(): called with Giant"));
+
+	AUDIT_ARG(cmd, uap->flag);
+
+	error = priv_check(td, PRIV_NFS_DAEMON);
+	if (error)
+		return (error);
+	error = EINVAL;
+	if ((uap->flag & (NFSSVC_ADDSOCK | NFSSVC_OLDNFSD | NFSSVC_NFSD)) &&
+	    nfsd_call_nfsserver != NULL)
+		error = (*nfsd_call_nfsserver)(td, uap);
+	else if ((uap->flag & (NFSSVC_CBADDSOCK | NFSSVC_NFSCBD)) &&
+	    nfsd_call_nfscl != NULL)
+		error = (*nfsd_call_nfscl)(td, uap);
+	else if ((uap->flag & (NFSSVC_IDNAME | NFSSVC_GETSTATS |
+	    NFSSVC_GSSDADDPORT | NFSSVC_GSSDADDFIRST | NFSSVC_GSSDDELETEALL |
+	    NFSSVC_NFSUSERDPORT | NFSSVC_NFSUSERDDELPORT)) &&
+	    nfsd_call_nfscommon != NULL)
+		error = (*nfsd_call_nfscommon)(td, uap);
+	else if ((uap->flag & (NFSSVC_NFSDNFSD | NFSSVC_NFSDADDSOCK |
+	    NFSSVC_PUBLICFH | NFSSVC_V4ROOTEXPORT | NFSSVC_NOPUBLICFH |
+	    NFSSVC_STABLERESTART | NFSSVC_ADMINREVOKE |
+	    NFSSVC_DUMPCLIENTS | NFSSVC_DUMPLOCKS)) &&
+	    nfsd_call_nfsd != NULL)
+		error = (*nfsd_call_nfsd)(td, uap);
+	if (error == EINTR || error == ERESTART)
+		error = 0;
+	return (error);
+}
+
+/*
+ * Called once to initialize data structures...
+ */
+static int
+nfssvc_modevent(module_t mod, int type, void *data)
+{
+	static int registered;
+	int error = 0;
+
+	switch (type) {
+	case MOD_LOAD:
+		error = syscall_register(&nfssvc_offset, &nfssvc_sysent,
+		    &nfssvc_prev_sysent);
+		if (error)
+			break;
+		registered = 1;
+		break;
+
+	case MOD_UNLOAD:
+		if (nfsd_call_nfsserver != NULL || nfsd_call_nfscommon != NULL
+		    || nfsd_call_nfscl != NULL || nfsd_call_nfsd != NULL) {
+			error = EBUSY;
+			break;
+		}
+		if (registered)
+			syscall_deregister(&nfssvc_offset, &nfssvc_prev_sysent);
+		registered = 0;
+		break;
+	default:
+		error = EOPNOTSUPP;
+		break;
+	}
+	return error;
+}
+static moduledata_t nfssvc_mod = {
+	"nfssvc",
+	nfssvc_modevent,
+	NULL,
+};
+DECLARE_MODULE(nfssvc, nfssvc_mod, SI_SUB_VFS, SI_ORDER_ANY);
+
+/* So that loader and kldload(2) can find us, wherever we are.. */
+MODULE_VERSION(nfssvc, 1);
+
diff --git a/sys/net/slip.h b/sys/nfs/nfssvc.h
index 7b4164e..5578240 100644
--- a/sys/net/slip.h
+++ b/sys/nfs/nfssvc.h
@@ -1,7 +1,10 @@
 /*-
- * Copyright (c) 1994
+ * Copyright (c) 1989, 1993, 1995
  *	The Regents of the University of California.  All rights reserved.
  *
+ * This code is derived from software contributed to Berkeley by
+ * Rick Macklem at The University of Guelph.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -26,33 +29,39 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	@(#)slip.h	8.1 (Berkeley) 2/12/94
  * $FreeBSD$
  */
 
-#ifndef _NET_SLIP_H_
-#define	_NET_SLIP_H_
-
-/* Ioctls operating on SLIP ttys. */
-#define	SLIOCGUNIT	_IOR('t', 88, int)	/* get slip unit number */
-#define	SLIOCSKEEPAL	_IOW('t', 84, int)	/* set keepalive */
-#define	SLIOCSOUTFILL	_IOW('t', 83, int)	/* set out fill time */
-#define	SLIOCGKEEPAL	_IOR('t', 82, int)	/* get keepalive time */
-#define	SLIOCGOUTFILL	_IOR('t', 81, int)	/* get out fill time */
-#define	SLIOCSUNIT	_IOW('t', 80, int)	/* set slip unit number */
+#ifndef _NFS_NFSSVC_H_
+#define _NFS_NFSSVC_H_
 
 /*
- * Definitions of the pseudo-link-level header attached to slip
- * packets grabbed by the packet filter (bpf) traffic monitor.
+ * Flags for nfssvc() system call.
  */
-#define	SLIP_HDRLEN	16		/* BPF SLIP header length */
-
-/* Offsets into BPF SLIP header. */
-#define	SLX_DIR		0		/* direction; see below */
-#define	SLX_CHDR	1		/* compressed header data */
-#define	CHDR_LEN	15		/* length of compressed header data */
+#define	NFSSVC_OLDNFSD	0x004
+#define	NFSSVC_ADDSOCK	0x008
+#define	NFSSVC_NFSD	0x010
 
-#define	SLIPDIR_IN	0		/* incoming */
-#define	SLIPDIR_OUT	1		/* outgoing */
+/*
+ * and ones for nfsv4.
+ */
+#define	NFSSVC_NOPUBLICFH	0x00000020
+#define	NFSSVC_STABLERESTART	0x00000040
+#define	NFSSVC_NFSDNFSD		0x00000080
+#define	NFSSVC_NFSDADDSOCK	0x00000100
+#define	NFSSVC_IDNAME		0x00000200
+#define	NFSSVC_GSSDDELETEALL	0x00000400
+#define	NFSSVC_GSSDADDPORT	0x00000800
+#define	NFSSVC_NFSUSERDPORT	0x00001000
+#define	NFSSVC_NFSUSERDDELPORT	0x00002000
+#define	NFSSVC_V4ROOTEXPORT	0x00004000
+#define	NFSSVC_ADMINREVOKE	0x00008000
+#define	NFSSVC_DUMPCLIENTS	0x00010000
+#define	NFSSVC_DUMPLOCKS	0x00020000
+#define	NFSSVC_GSSDADDFIRST	0x00040000
+#define	NFSSVC_PUBLICFH		0x00080000
+#define	NFSSVC_NFSCBD		0x00100000
+#define	NFSSVC_CBADDSOCK	0x00200000
+#define	NFSSVC_GETSTATS		0x00400000
 
-#endif /* !_NET_SLIP_H */
+#endif /* _NFS_NFSSVC_H */
diff --git a/sys/nfs4client/nfs4_socket.c b/sys/nfs4client/nfs4_socket.c
index 754ba47..c1fe154 100644
--- a/sys/nfs4client/nfs4_socket.c
+++ b/sys/nfs4client/nfs4_socket.c
@@ -259,7 +259,7 @@ nfs4_request(struct vnode *vp, struct mbuf *mrest, int procnum,
 	 ** lookup cache, just in case.
 	 **/
 	if (error == ESTALE)
-		cache_purge(vp);
+		nfs_purgecache(vp);
 
 	return (error);
 }
diff --git a/sys/nfs4client/nfs4_vnops.c b/sys/nfs4client/nfs4_vnops.c
index 4dd7316..af85494 100644
--- a/sys/nfs4client/nfs4_vnops.c
+++ b/sys/nfs4client/nfs4_vnops.c
@@ -173,7 +173,6 @@ struct vop_vector nfs4_vnodeops = {
 	.vop_getpages =		nfs_getpages,
 	.vop_putpages =		nfs_putpages,
 	.vop_inactive =		nfs_inactive,
-	.vop_lease =		VOP_NULL,
 	.vop_link =		nfs4_link,
 	.vop_lookup =		nfs4_lookup,
 	.vop_mkdir =		nfs4_mkdir,
@@ -241,11 +240,11 @@ SYSCTL_INT(_vfs_nfs4, OID_AUTO, access_cache_misses, CTLFLAG_RD,
 			 | NFSV3ACCESS_DELETE | NFSV3ACCESS_LOOKUP)
 static int
 nfs4_v3_access_otw(struct vnode *vp, int wmode, struct thread *td,
-    struct ucred *cred)
+    struct ucred *cred, uint32_t *retmode)
 {
 	const int v3 = 1;
 	u_int32_t *tl;
-	int error = 0, attrflag;
+	int error = 0, attrflag, i, lrupos;
 
 	return (0);
 
@@ -264,11 +263,26 @@ nfs4_v3_access_otw(struct vnode *vp, int wmode, struct thread *td,
 	nfsm_request(vp, NFSPROC_ACCESS, td, cred);
 	nfsm_postop_attr(vp, attrflag);
 	if (!error) {
+		lrupos = 0;
 		tl = nfsm_dissect(u_int32_t *, NFSX_UNSIGNED);
 		rmode = fxdr_unsigned(u_int32_t, *tl);
-		np->n_mode = rmode;
-		np->n_modeuid = cred->cr_uid;
-		np->n_modestamp = time_second;
+		for (i = 0; i < NFS_ACCESSCACHESIZE; i++) {
+			if (np->n_accesscache[i].uid == cred->cr_uid) {
+				np->n_accesscache[i].mode = rmode;
+				np->n_accesscache[i].stamp = time_second;
+				break;
+			}
+			if (i > 0 && np->n_accesscache[i].stamp <
+			    np->n_accesscache[lrupos].stamp)
+				lrupos = i;
+		}
+		if (i == NFS_ACCESSCACHESIZE) {
+			np->n_accesscache[lrupos].uid = cred->cr_uid;
+			np->n_accesscache[lrupos].mode = rmode;
+			np->n_accesscache[lrupos].stamp = time_second;
+		}
+		if (retmode != NULL)
+			*retmode = rmode;
 	}
 	m_freem(mrep);
 nfsmout:
@@ -285,8 +299,8 @@ static int
 nfs4_access(struct vop_access_args *ap)
 {
 	struct vnode *vp = ap->a_vp;
-	int error = 0;
-	u_int32_t mode, wmode;
+	int error = 0, i, gotahit;
+	u_int32_t mode, rmode, wmode;
 	int v3 = NFS_ISV3(vp);	/* v3 \in v4 */
 	struct nfsnode *np = VTONFS(vp);
 	caddr_t bpos, dpos;
@@ -350,19 +364,27 @@ nfs4_access(struct vop_access_args *ap)
 		 * Does our cached result allow us to give a definite yes to
 		 * this request?
 		 */
-		if (time_second < np->n_modestamp + nfs4_access_cache_timeout &&
-		    ap->a_cred->cr_uid == np->n_modeuid &&
-		    (np->n_mode & mode) == mode) {
-			nfsstats.accesscache_hits++;
-		} else {
+		gotahit = 0;
+		for (i = 0; i < NFS_ACCESSCACHESIZE; i++) {
+			if (ap->a_cred->cr_uid == np->n_accesscache[i].uid) {
+				if (time_second < (np->n_accesscache[i].stamp +
+				    nfs4_access_cache_timeout) &&
+				    (np->n_accesscache[i].mode & mode) == mode) {
+					nfsstats.accesscache_hits++;
+					gotahit = 1;
+				}
+				break;
+			}
+		}
+		if (gotahit == 0) {
 			/*
 			 * Either a no, or a don't know.  Go to the wire.
 			 */
 			nfsstats.accesscache_misses++;
 		        error = nfs4_v3_access_otw(vp, wmode, ap->a_td,
-			    ap->a_cred);
+			    ap->a_cred, &rmode);
 			if (error == 0) {
-				if ((np->n_mode & mode) != mode)
+				if ((rmode & mode) != mode)
 					error = EACCES;
 			}
 		}
diff --git a/sys/nfsclient/nfs.h b/sys/nfsclient/nfs.h
index c5b1c51..4dce2fe 100644
--- a/sys/nfsclient/nfs.h
+++ b/sys/nfsclient/nfs.h
@@ -63,11 +63,14 @@
 #define	NFS_MAXATTRTIMO 60
 #endif
 #ifndef NFS_MINDIRATTRTIMO
-#define	NFS_MINDIRATTRTIMO 30		/* VDIR attrib cache timeout in sec */
+#define	NFS_MINDIRATTRTIMO 3		/* VDIR attrib cache timeout in sec */
 #endif
 #ifndef NFS_MAXDIRATTRTIMO
 #define	NFS_MAXDIRATTRTIMO 60
 #endif
+#ifndef	NFS_ACCESSCACHESIZE
+#define	NFS_ACCESSCACHESIZE 8		/* Per-node access cache entries */
+#endif
 #define	NFS_WSIZE	8192		/* Def. write data size <= 8192 */
 #define	NFS_RSIZE	8192		/* Def. read data size <= 8192 */
 #define NFS_READDIRSIZE	8192		/* Def. readdir size */
@@ -319,6 +322,7 @@ void	nfs_down(struct nfsreq *, struct nfsmount *, struct thread *,
 #endif /* ! NFS4_USE_RPCCLNT */
 #endif
 
+void	nfs_purgecache(struct vnode *);
 int	nfs_vinvalbuf(struct vnode *, int, struct thread *, int);
 int	nfs_readrpc(struct vnode *, struct uio *, struct ucred *);
 int	nfs_writerpc(struct vnode *, struct uio *, struct ucred *, int *,
diff --git a/sys/nfsclient/nfs_bio.c b/sys/nfsclient/nfs_bio.c
index ef863e5..97820f0 100644
--- a/sys/nfsclient/nfs_bio.c
+++ b/sys/nfsclient/nfs_bio.c
@@ -35,6 +35,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_kdtrace.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bio.h>
@@ -61,6 +63,7 @@ __FBSDID("$FreeBSD$");
 #include <nfsclient/nfs.h>
 #include <nfsclient/nfsmount.h>
 #include <nfsclient/nfsnode.h>
+#include <nfsclient/nfs_kdtrace.h>
 
 #include <nfs4client/nfs4.h>
 
@@ -403,6 +406,7 @@ nfs_bioread_check_cons(struct vnode *vp, struct thread *td, struct ucred *cred)
 				goto out;
 		}
 		np->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 		error = VOP_GETATTR(vp, &vattr, cred);
 		if (error)
 			goto out;
@@ -915,6 +919,7 @@ nfs_write(struct vop_write_args *ap)
 #endif
 flush_and_restart:
 			np->n_attrstamp = 0;
+			KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 			error = nfs_vinvalbuf(vp, V_SAVE, td, 1);
 			if (error)
 				return (error);
@@ -928,6 +933,7 @@ flush_and_restart:
 	 */
 	if (ioflag & IO_APPEND) {
 		np->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 		error = VOP_GETATTR(vp, &vattr, cred);
 		if (error)
 			return (error);
@@ -1756,6 +1762,7 @@ nfs_doio(struct vnode *vp, struct buf *bp, struct ucred *cr, struct thread *td)
 			mtx_lock(&np->n_mtx);
 			np->n_flag |= NWRITEERR;
 			np->n_attrstamp = 0;
+			KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 			mtx_unlock(&np->n_mtx);
 		    }
 		    bp->b_dirtyoff = bp->b_dirtyend = 0;
diff --git a/sys/nfsclient/nfs_kdtrace.c b/sys/nfsclient/nfs_kdtrace.c
new file mode 100644
index 0000000..8444ae2
--- /dev/null
+++ b/sys/nfsclient/nfs_kdtrace.c
@@ -0,0 +1,545 @@
+/*-
+ * Copyright (c) 2009 Robert N. M. Watson
+ * All rights reserved.
+ *
+ * This software was developed at the University of Cambridge Computer
+ * Laboratory with support from a grant from Google, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/conf.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+
+#include <sys/dtrace.h>
+#include <sys/dtrace_bsd.h>
+
+#include <nfs/nfsproto.h>
+
+/*
+ * dtnfsclient is a DTrace provider that tracks the intent to perform RPCs
+ * in the NFS client, as well as acess to and maintenance of the access and
+ * attribute caches.  This is not quite the same as RPCs, because NFS may
+ * issue multiple RPC transactions in the event that authentication fails,
+ * there's a jukebox error, or none at all if the access or attribute cache
+ * hits.  However, it cleanly represents the logical layer between RPC
+ * transmission and vnode/vfs operations, providing access to state linking
+ * the two.
+ */
+
+static int	dtnfsclient_unload(void);
+static void	dtnfsclient_getargdesc(void *, dtrace_id_t, void *,
+		    dtrace_argdesc_t *);
+static void	dtnfsclient_provide(void *, dtrace_probedesc_t *);
+static void	dtnfsclient_destroy(void *, dtrace_id_t, void *);
+static void	dtnfsclient_enable(void *, dtrace_id_t, void *);
+static void	dtnfsclient_disable(void *, dtrace_id_t, void *);
+static void	dtnfsclient_load(void *);
+
+static dtrace_pattr_t dtnfsclient_attr = {
+{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
+{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
+{ DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
+};
+
+/*
+ * Descrition of NFSv3 and (optional) NFSv2 probes for a procedure.
+ */
+struct dtnfsclient_rpc {
+	char		*nr_v3_name;
+	char		*nr_v2_name;	/* Or NULL if none. */
+
+	/*
+	 * IDs for the start and done cases, for both NFSv2 and NFSv3.
+	 */
+	uint32_t	 nr_v2_id_start, nr_v2_id_done;
+	uint32_t	 nr_v3_id_start, nr_v3_id_done;
+};
+
+/*
+ * This table is indexed by NFSv3 procedure number, but also used for NFSv2
+ * procedure names.
+ */
+static struct dtnfsclient_rpc	dtnfsclient_rpcs[NFS_NPROCS] = {
+	{ "null", "null" },
+	{ "getattr", "getattr" },
+	{ "setattr", "setattr" },
+	{ "lookup", "lookup" },
+	{ "access" },
+	{ "readlink", "readlink" },
+	{ "read", "read" },
+	{ "write", "write" },
+	{ "create", "create" },
+	{ "mkdir", "mkdir" },
+	{ "symlink", "symlink" },
+	{ "mknod" },
+	{ "remove", "remove" },
+	{ "rmdir", "rmdir" },
+	{ "rename", "rename" },
+	{ "link", "link" },
+	{ "readdir", "readdir" },
+	{ "readdirplus" },
+	{ "fsstat", "statfs" },
+	{ "fsinfo" },
+	{ "pathconf" },
+	{ "commit" },
+	{ "noop" },
+};
+
+/*
+ * Module name strings.
+ */
+static char	*dtnfsclient_accesscache_str = "accesscache";
+static char	*dtnfsclient_attrcache_str = "attrcache";
+static char	*dtnfsclient_nfs2_str = "nfs2";
+static char	*dtnfsclient_nfs3_str = "nfs3";
+
+/*
+ * Function name strings.
+ */
+static char	*dtnfsclient_flush_str = "flush";
+static char	*dtnfsclient_load_str = "load";
+static char	*dtnfsclient_get_str = "get";
+
+/*
+ * Name strings.
+ */
+static char	*dtnfsclient_done_str = "done";
+static char	*dtnfsclient_hit_str = "hit";
+static char	*dtnfsclient_miss_str = "miss";
+static char	*dtnfsclient_start_str = "start";
+
+static dtrace_pops_t dtnfsclient_pops = {
+	dtnfsclient_provide,
+	NULL,
+	dtnfsclient_enable,
+	dtnfsclient_disable,
+	NULL,
+	NULL,
+	dtnfsclient_getargdesc,
+	NULL,
+	NULL,
+	dtnfsclient_destroy
+};
+
+static dtrace_provider_id_t	dtnfsclient_id;
+
+/*
+ * Most probes are generated from the above RPC table, but for access and
+ * attribute caches, we have specific IDs we recognize and handle specially
+ * in various spots.
+ */
+extern uint32_t	nfsclient_accesscache_flush_done_id;
+extern uint32_t	nfsclient_accesscache_get_hit_id;
+extern uint32_t	nfsclient_accesscache_get_miss_id;
+extern uint32_t	nfsclient_accesscache_load_done_id;
+
+extern uint32_t	nfsclient_attrcache_flush_done_id;
+extern uint32_t	nfsclient_attrcache_get_hit_id;
+extern uint32_t	nfsclient_attrcache_get_miss_id;
+extern uint32_t	nfsclient_attrcache_load_done_id;
+
+/*
+ * When tracing on a procedure is enabled, the DTrace ID for an RPC event is
+ * stored in one of these two NFS client-allocated arrays; 0 indicates that
+ * the event is not being traced so probes should not be called.
+ *
+ * For simplicity, we allocate both v2 and v3 arrays as NFS_NPROCS, and the
+ * v2 array is simply sparse.
+ */
+extern uint32_t			nfsclient_nfs2_start_probes[NFS_NPROCS];
+extern uint32_t			nfsclient_nfs2_done_probes[NFS_NPROCS];
+
+extern uint32_t			nfsclient_nfs3_start_probes[NFS_NPROCS];
+extern uint32_t			nfsclient_nfs3_done_probes[NFS_NPROCS];
+
+/*
+ * Look up a DTrace probe ID to see if it's associated with a "done" event --
+ * if so, we will return a fourth argument type of "int".
+ */
+static int
+dtnfs23_isdoneprobe(dtrace_id_t id)
+{
+	int i;
+
+	for (i = 0; i < NFS_NPROCS; i++) {
+		if (dtnfsclient_rpcs[i].nr_v3_id_done == id ||
+		    dtnfsclient_rpcs[i].nr_v2_id_done == id)
+			return (1);
+	}
+	return (0);
+}
+
+static void
+dtnfsclient_getargdesc(void *arg, dtrace_id_t id, void *parg,
+    dtrace_argdesc_t *desc)
+{
+	const char *p = NULL;
+
+	if (id == nfsclient_accesscache_flush_done_id ||
+	    id == nfsclient_attrcache_flush_done_id ||
+	    id == nfsclient_attrcache_get_miss_id) {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	} else if (id == nfsclient_accesscache_get_hit_id ||
+	    id == nfsclient_accesscache_get_miss_id) {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		case 1:
+			p = "uid_t";
+			break;
+		case 2:
+			p = "uint32_t";
+			break;
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	} else if (id == nfsclient_accesscache_load_done_id) {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		case 1:
+			p = "uid_t";
+			break;
+		case 2:
+			p = "uint32_t";
+			break;
+		case 3:
+			p = "int";
+			break;
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	} else if (id == nfsclient_attrcache_get_hit_id) {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		case 1:
+			p = "struct vattr *";
+			break;
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	} else if (id == nfsclient_attrcache_load_done_id) {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		case 1:
+			p = "struct vattr *";
+			break;
+		case 2:
+			p = "int";
+			break;
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	} else {
+		switch (desc->dtargd_ndx) {
+		case 0:
+			p = "struct vnode *";
+			break;
+		case 1:
+			p = "struct mbuf *";
+			break;
+		case 2:
+			p = "struct ucred *";
+			break;
+		case 3:
+			p = "int";
+			break;
+		case 4:
+			if (dtnfs23_isdoneprobe(id)) {
+				p = "int";
+				break;
+			}
+			/* FALLSTHROUGH */
+		default:
+			desc->dtargd_ndx = DTRACE_ARGNONE;
+			break;
+		}
+	}
+	if (p != NULL)
+		strlcpy(desc->dtargd_native, p, sizeof(desc->dtargd_native));
+}
+
+static void
+dtnfsclient_provide(void *arg, dtrace_probedesc_t *desc)
+{
+	int i;
+
+	if (desc != NULL)
+		return;
+
+	/*
+	 * Register access cache probes.
+	 */
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_accesscache_str,
+	    dtnfsclient_flush_str, dtnfsclient_done_str) == 0) {
+		nfsclient_accesscache_flush_done_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_accesscache_str,
+		    dtnfsclient_flush_str, dtnfsclient_done_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_accesscache_str,
+	    dtnfsclient_get_str, dtnfsclient_hit_str) == 0) {
+		nfsclient_accesscache_get_hit_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_accesscache_str,
+		    dtnfsclient_get_str, dtnfsclient_hit_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_accesscache_str,
+	    dtnfsclient_get_str, dtnfsclient_miss_str) == 0) {
+		nfsclient_accesscache_get_miss_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_accesscache_str,
+		    dtnfsclient_get_str, dtnfsclient_miss_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_accesscache_str,
+	    dtnfsclient_load_str, dtnfsclient_done_str) == 0) {
+		nfsclient_accesscache_load_done_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_accesscache_str,
+		    dtnfsclient_load_str, dtnfsclient_done_str, 0, NULL);
+	}
+
+	/*
+	 * Register attribute cache probes.
+	 */
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_attrcache_str,
+	    dtnfsclient_flush_str, dtnfsclient_done_str) == 0) {
+		nfsclient_attrcache_flush_done_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_attrcache_str,
+		    dtnfsclient_flush_str, dtnfsclient_done_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_attrcache_str,
+	    dtnfsclient_get_str, dtnfsclient_hit_str) == 0) {
+		nfsclient_attrcache_get_hit_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_attrcache_str,
+		    dtnfsclient_get_str, dtnfsclient_hit_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_attrcache_str,
+	    dtnfsclient_get_str, dtnfsclient_miss_str) == 0) {
+		nfsclient_attrcache_get_miss_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_attrcache_str,
+		    dtnfsclient_get_str, dtnfsclient_miss_str, 0, NULL);
+	}
+	if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_attrcache_str,
+	    dtnfsclient_load_str, dtnfsclient_done_str) == 0) {
+		nfsclient_attrcache_load_done_id = dtrace_probe_create(
+		    dtnfsclient_id, dtnfsclient_attrcache_str,
+		    dtnfsclient_load_str, dtnfsclient_done_str, 0, NULL);
+	}
+
+	/*
+	 * Register NFSv2 RPC procedures; note sparseness check for each slot
+	 * in the NFSv3 procnum-indexed array.
+	 */
+	for (i = 0; i < NFS_NPROCS; i++) {
+		if (dtnfsclient_rpcs[i].nr_v2_name != NULL &&
+		    dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_nfs2_str,
+		    dtnfsclient_rpcs[i].nr_v2_name, dtnfsclient_start_str) ==
+		    0) {
+			dtnfsclient_rpcs[i].nr_v2_id_start =
+			    dtrace_probe_create(dtnfsclient_id,
+			    dtnfsclient_nfs2_str,
+			    dtnfsclient_rpcs[i].nr_v2_name,
+			    dtnfsclient_start_str, 0,
+			    &nfsclient_nfs2_start_probes[i]);
+		}
+		if (dtnfsclient_rpcs[i].nr_v2_name != NULL &&
+		    dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_nfs2_str,
+		    dtnfsclient_rpcs[i].nr_v2_name, dtnfsclient_done_str) ==
+		    0) {
+			dtnfsclient_rpcs[i].nr_v2_id_done = 
+			    dtrace_probe_create(dtnfsclient_id,
+			    dtnfsclient_nfs2_str,
+			    dtnfsclient_rpcs[i].nr_v2_name,
+			    dtnfsclient_done_str, 0,
+			    &nfsclient_nfs2_done_probes[i]);
+		}
+	}
+
+	/*
+	 * Register NFSv3 RPC procedures.
+	 */
+	for (i = 0; i < NFS_NPROCS; i++) {
+		if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_nfs3_str,
+		    dtnfsclient_rpcs[i].nr_v3_name, dtnfsclient_start_str) ==
+		    0) {
+			dtnfsclient_rpcs[i].nr_v3_id_start =
+			    dtrace_probe_create(dtnfsclient_id,
+			    dtnfsclient_nfs3_str,
+			    dtnfsclient_rpcs[i].nr_v3_name,
+			    dtnfsclient_start_str, 0,
+			    &nfsclient_nfs3_start_probes[i]);
+		}
+		if (dtrace_probe_lookup(dtnfsclient_id, dtnfsclient_nfs3_str,
+		    dtnfsclient_rpcs[i].nr_v3_name, dtnfsclient_done_str) ==
+		    0) {
+			dtnfsclient_rpcs[i].nr_v3_id_done = 
+			    dtrace_probe_create(dtnfsclient_id,
+			    dtnfsclient_nfs3_str,
+			    dtnfsclient_rpcs[i].nr_v3_name,
+			    dtnfsclient_done_str, 0,
+			    &nfsclient_nfs3_done_probes[i]);
+		}
+	}
+}
+
+static void
+dtnfsclient_destroy(void *arg, dtrace_id_t id, void *parg)
+{
+}
+
+static void
+dtnfsclient_enable(void *arg, dtrace_id_t id, void *parg)
+{
+	uint32_t *p = parg;
+	void *f = dtrace_probe;
+
+	if (id == nfsclient_accesscache_flush_done_id)
+		dtrace_nfsclient_accesscache_flush_done_probe = f;
+	else if (id == nfsclient_accesscache_get_hit_id)
+		dtrace_nfsclient_accesscache_get_hit_probe = f;
+	else if (id == nfsclient_accesscache_get_miss_id)
+		dtrace_nfsclient_accesscache_get_miss_probe = f;
+	else if (id == nfsclient_accesscache_load_done_id)
+		dtrace_nfsclient_accesscache_load_done_probe = f;
+	else if (id == nfsclient_attrcache_flush_done_id)
+		dtrace_nfsclient_attrcache_flush_done_probe = f;
+	else if (id == nfsclient_attrcache_get_hit_id)
+		dtrace_nfsclient_attrcache_get_hit_probe = f;
+	else if (id == nfsclient_attrcache_get_miss_id)
+		dtrace_nfsclient_attrcache_get_miss_probe = f;
+	else if (id == nfsclient_attrcache_load_done_id)
+		dtrace_nfsclient_attrcache_load_done_probe = f;
+	else
+		*p = id;
+}
+
+static void
+dtnfsclient_disable(void *arg, dtrace_id_t id, void *parg)
+{
+	uint32_t *p = parg;
+
+	if (id == nfsclient_accesscache_flush_done_id)
+		dtrace_nfsclient_accesscache_flush_done_probe = NULL;
+	else if (id == nfsclient_accesscache_get_hit_id)
+		dtrace_nfsclient_accesscache_get_hit_probe = NULL;
+	else if (id == nfsclient_accesscache_get_miss_id)
+		dtrace_nfsclient_accesscache_get_miss_probe = NULL;
+	else if (id == nfsclient_accesscache_load_done_id)
+		dtrace_nfsclient_accesscache_load_done_probe = NULL;
+	else if (id == nfsclient_attrcache_flush_done_id)
+		dtrace_nfsclient_attrcache_flush_done_probe = NULL;
+	else if (id == nfsclient_attrcache_get_hit_id)
+		dtrace_nfsclient_attrcache_get_hit_probe = NULL;
+	else if (id == nfsclient_attrcache_get_miss_id)
+		dtrace_nfsclient_attrcache_get_miss_probe = NULL;
+	else if (id == nfsclient_attrcache_load_done_id)
+		dtrace_nfsclient_attrcache_load_done_probe = NULL;
+	else
+		*p = 0;
+}
+
+static void
+dtnfsclient_load(void *dummy)
+{
+
+	if (dtrace_register("nfsclient", &dtnfsclient_attr,
+	    DTRACE_PRIV_USER, NULL, &dtnfsclient_pops, NULL,
+	    &dtnfsclient_id) != 0)
+		return;
+
+	dtrace_nfsclient_nfs23_start_probe =
+	    (dtrace_nfsclient_nfs23_start_probe_func_t)dtrace_probe;
+	dtrace_nfsclient_nfs23_done_probe =
+	    (dtrace_nfsclient_nfs23_done_probe_func_t)dtrace_probe;
+}
+
+
+static int
+dtnfsclient_unload()
+{
+	int error = 0;
+
+	dtrace_nfsclient_nfs23_start_probe = NULL;
+	dtrace_nfsclient_nfs23_done_probe = NULL;
+
+	if ((error = dtrace_unregister(dtnfsclient_id)) != 0)
+		return (error);
+
+	return (error);
+}
+
+static int
+dtnfsclient_modevent(module_t mod __unused, int type, void *data __unused)
+{
+	int error = 0;
+
+	switch (type) {
+	case MOD_LOAD:
+		break;
+
+	case MOD_UNLOAD:
+		break;
+
+	case MOD_SHUTDOWN:
+		break;
+
+	default:
+		error = EOPNOTSUPP;
+		break;
+	}
+
+	return (error);
+}
+
+SYSINIT(dtnfsclient_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY,
+    dtnfsclient_load, NULL);
+SYSUNINIT(dtnfsclient_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY,
+    dtnfsclient_unload, NULL);
+
+DEV_MODULE(dtnfsclient, dtnfsclient_modevent, NULL);
+MODULE_VERSION(dtnfsclient, 1);
+MODULE_DEPEND(dtnfsclient, dtrace, 1, 1, 1);
+MODULE_DEPEND(dtnfsclient, opensolaris, 1, 1, 1);
diff --git a/sys/nfsclient/nfs_kdtrace.h b/sys/nfsclient/nfs_kdtrace.h
new file mode 100644
index 0000000..d29aa68
--- /dev/null
+++ b/sys/nfsclient/nfs_kdtrace.h
@@ -0,0 +1,120 @@
+/*-
+ * Copyright (c) 2009 Robert N. M. Watson
+ * All rights reserved.
+ *
+ * This software was developed at the University of Cambridge Computer
+ * Laboratory with support from a grant from Google, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _NFSCLIENT_NFS_KDTRACE_H_
+#define	_NFSCLIENT_NFS_KDTRACE_H_
+
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+
+/*
+ * Definitions for NFS access cache probes.
+ */
+extern uint32_t nfsclient_accesscache_flush_done_id;
+extern uint32_t nfsclient_accesscache_get_hit_id;
+extern uint32_t nfsclient_accesscache_get_miss_id;
+extern uint32_t nfsclient_accesscache_load_done_id;
+
+#define	KDTRACE_NFS_ACCESSCACHE_FLUSH_DONE(vp)	do {			\
+	if (dtrace_nfsclient_accesscache_flush_done_probe != NULL)	\
+		(dtrace_nfsclient_accesscache_flush_done_probe)(	\
+		    nfsclient_accesscache_flush_done_id, (vp));		\
+} while (0)
+
+#define	KDTRACE_NFS_ACCESSCACHE_GET_HIT(vp, uid, mode)	do {		\
+	if (dtrace_nfsclient_accesscache_get_hit_probe != NULL)		\
+		(dtrace_nfsclient_accesscache_get_hit_probe)(		\
+		    nfsclient_accesscache_get_hit_id, (vp), (uid),	\
+		    (mode));						\
+} while (0)
+	
+#define	KDTRACE_NFS_ACCESSCACHE_GET_MISS(vp, uid, mode)	do {		\
+	if (dtrace_nfsclient_accesscache_get_miss_probe != NULL)	\
+		(dtrace_nfsclient_accesscache_get_miss_probe)(		\
+		    nfsclient_accesscache_get_miss_id, (vp), (uid),	\
+		    (mode));						\
+} while (0)
+
+#define	KDTRACE_NFS_ACCESSCACHE_LOAD_DONE(vp, uid, rmode, error) do {	\
+	if (dtrace_nfsclient_accesscache_load_done_probe != NULL)	\
+		(dtrace_nfsclient_accesscache_load_done_probe)(		\
+		    nfsclient_accesscache_load_done_id, (vp), (uid),	\
+		    (rmode), (error));					\
+} while (0)
+
+/*
+ * Definitions for NFS attribute cache probes.
+ */
+extern uint32_t nfsclient_attrcache_flush_done_id;
+extern uint32_t nfsclient_attrcache_get_hit_id;
+extern uint32_t nfsclient_attrcache_get_miss_id;
+extern uint32_t nfsclient_attrcache_load_done_id;
+
+#define	KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp)	do {			\
+	if (dtrace_nfsclient_attrcache_flush_done_probe != NULL)	\
+		(dtrace_nfsclient_attrcache_flush_done_probe)(		\
+		    nfsclient_attrcache_flush_done_id, (vp));		\
+} while (0)
+
+#define	KDTRACE_NFS_ATTRCACHE_GET_HIT(vp, vap)	do {			\
+	if (dtrace_nfsclient_attrcache_get_hit_probe != NULL)		\
+		(dtrace_nfsclient_attrcache_get_hit_probe)(		\
+		    nfsclient_attrcache_get_hit_id, (vp), (vap));	\
+} while (0)
+
+#define	KDTRACE_NFS_ATTRCACHE_GET_MISS(vp)	do {			\
+	if (dtrace_nfsclient_attrcache_get_miss_probe != NULL)		\
+		(dtrace_nfsclient_attrcache_get_miss_probe)(		\
+			    nfsclient_attrcache_get_miss_id, (vp));	\
+} while (0)
+
+#define	KDTRACE_NFS_ATTRCACHE_LOAD_DONE(vp, vap, error)	do {		\
+	if (dtrace_nfsclient_attrcache_load_done_probe != NULL)		\
+		(dtrace_nfsclient_attrcache_load_done_probe)(		\
+		    nfsclient_attrcache_load_done_id, (vp), (vap),	\
+		    (error));						\
+} while (0)
+
+#else /* !KDTRACE_HOOKS */
+
+#define	KDTRACE_NFS_ACCESSCACHE_FLUSH_DONE(vp)
+#define	KDTRACE_NFS_ACCESSCACHE_GET_HIT(vp, uid, mode)
+#define	KDTRACE_NFS_ACCESSCACHE_GET_MISS(vp, uid, mode)
+#define	KDTRACE_NFS_ACCESSCACHE_LOAD_DONE(vp, uid, rmode, error)
+
+#define	KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp)
+#define	KDTRACE_NFS_ATTRCACHE_GET_HIT(vp, vap)
+#define	KDTRACE_NFS_ATTRCACHE_GET_MISS(vp)
+#define	KDTRACE_NFS_ATTRCACHE_LOAD_DONE(vp, vap, error)
+
+#endif /* KDTRACE_HOOKS */
+
+#endif /* !_NFSCLIENT_NFS_KDTRACE_H_ */
diff --git a/sys/nfsclient/nfs_krpc.c b/sys/nfsclient/nfs_krpc.c
index 8a5f805..767dc0f 100644
--- a/sys/nfsclient/nfs_krpc.c
+++ b/sys/nfsclient/nfs_krpc.c
@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
  */
 
 #include "opt_inet6.h"
+#include "opt_kdtrace.h"
 #include "opt_kgssapi.h"
 
 #include <sys/param.h>
@@ -73,6 +74,25 @@ __FBSDID("$FreeBSD$");
 
 #ifndef NFS_LEGACYRPC
 
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+
+dtrace_nfsclient_nfs23_start_probe_func_t
+    dtrace_nfsclient_nfs23_start_probe;
+
+dtrace_nfsclient_nfs23_done_probe_func_t
+    dtrace_nfsclient_nfs23_done_probe;
+
+/*
+ * Registered probes by RPC type.
+ */
+uint32_t	nfsclient_nfs2_start_probes[NFS_NPROCS];
+uint32_t	nfsclient_nfs2_done_probes[NFS_NPROCS];
+
+uint32_t	nfsclient_nfs3_start_probes[NFS_NPROCS];
+uint32_t	nfsclient_nfs3_done_probes[NFS_NPROCS];
+#endif
+
 static int	nfs_realign_test;
 static int	nfs_realign_count;
 static int	nfs_bufpackets = 4;
@@ -468,6 +488,25 @@ nfs_request(struct vnode *vp, struct mbuf *mreq, int procnum,
 		ext.rc_timers = NULL;
 	}
 
+#ifdef KDTRACE_HOOKS
+	if (dtrace_nfsclient_nfs23_start_probe != NULL) {
+		uint32_t probe_id;
+		int probe_procnum;
+
+		if (nmp->nm_flag & NFSMNT_NFSV3) {
+			probe_id = nfsclient_nfs3_start_probes[procnum];
+			probe_procnum = procnum;
+		} else {
+			probe_id = nfsclient_nfs2_start_probes[procnum];
+			probe_procnum = (nmp->nm_flag & NFSMNT_NFSV3) ?
+			    procnum : nfsv2_procid[procnum];
+		}
+		if (probe_id != 0)
+			(dtrace_nfsclient_nfs23_start_probe)(probe_id, vp,
+			    mreq, cred, probe_procnum);
+	}
+#endif
+
 	nfsstats.rpcrequests++;
 tryagain:
 	timo.tv_sec = nmp->nm_timeo / NFS_HZ;
@@ -493,11 +532,8 @@ tryagain:
 		error = EACCES;
 	}
 	md = mrep;
-	if (error) {
-		m_freem(mreq);
-		AUTH_DESTROY(auth);
-		return (error);
-	}
+	if (error)
+		goto nfsmout;
 
 	KASSERT(mrep != NULL, ("mrep shouldn't be NULL if no error\n"));
 
@@ -521,7 +557,7 @@ tryagain:
 		 * cache, just in case.
 		 */
 		if (error == ESTALE)
-			cache_purge(vp);
+			nfs_purgecache(vp);
 		/*
 		 * Skip wcc data on NFS errors for now. NetApp filers
 		 * return corrupt postop attrs in the wcc data for NFS
@@ -535,11 +571,27 @@ tryagain:
 			error |= NFSERR_RETERR;
 		} else
 			m_freem(mrep);
-		m_freem(mreq);
-		AUTH_DESTROY(auth);
-		return (error);
+		goto nfsmout;
 	}
 
+#ifdef KDTRACE_HOOKS
+	if (dtrace_nfsclient_nfs23_done_probe != NULL) {
+		uint32_t probe_id;
+		int probe_procnum;
+
+		if (nmp->nm_flag & NFSMNT_NFSV3) {
+			probe_id = nfsclient_nfs3_done_probes[procnum];
+			probe_procnum = procnum;
+		} else {
+			probe_id = nfsclient_nfs2_done_probes[procnum];
+			probe_procnum = (nmp->nm_flag & NFSMNT_NFSV3) ?
+			    procnum : nfsv2_procid[procnum];
+		}
+		if (probe_id != 0)
+			(dtrace_nfsclient_nfs23_done_probe)(probe_id, vp,
+			    mreq, cred, probe_procnum, 0);
+	}
+#endif
 	m_freem(mreq);
 	*mrp = mrep;
 	*mdp = md;
@@ -548,6 +600,24 @@ tryagain:
 	return (0);
 
 nfsmout:
+#ifdef KDTRACE_HOOKS
+	if (dtrace_nfsclient_nfs23_done_probe != NULL) {
+		uint32_t probe_id;
+		int probe_procnum;
+
+		if (nmp->nm_flag & NFSMNT_NFSV3) {
+			probe_id = nfsclient_nfs3_done_probes[procnum];
+			probe_procnum = procnum;
+		} else {
+			probe_id = nfsclient_nfs2_done_probes[procnum];
+			probe_procnum = (nmp->nm_flag & NFSMNT_NFSV3) ?
+			    procnum : nfsv2_procid[procnum];
+		}
+		if (probe_id != 0)
+			(dtrace_nfsclient_nfs23_done_probe)(probe_id, vp,
+			    mreq, cred, probe_procnum, error);
+	}
+#endif
 	m_freem(mreq);
 	if (auth)
 		AUTH_DESTROY(auth);
diff --git a/sys/nfsclient/nfs_socket.c b/sys/nfsclient/nfs_socket.c
index a61962f..8057604 100644
--- a/sys/nfsclient/nfs_socket.c
+++ b/sys/nfsclient/nfs_socket.c
@@ -1364,7 +1364,7 @@ wait_for_pinned_req:
 			 * lookup cache, just in case.
 			 */
 			if (error == ESTALE)
-				cache_purge(vp);
+				nfs_purgecache(vp);
 			/*
 			 * Skip wcc data on NFS errors for now. NetApp filers return corrupt
 			 * postop attrs in the wcc data for NFS err EROFS. Not sure if they 
diff --git a/sys/nfsclient/nfs_subs.c b/sys/nfsclient/nfs_subs.c
index be6ab92..1453dbf 100644
--- a/sys/nfsclient/nfs_subs.c
+++ b/sys/nfsclient/nfs_subs.c
@@ -41,6 +41,8 @@ __FBSDID("$FreeBSD$");
  * copy data between mbuf chains and uio lists.
  */
 
+#include "opt_kdtrace.h"
+
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
@@ -69,6 +71,7 @@ __FBSDID("$FreeBSD$");
 #include <nfs/nfsproto.h>
 #include <nfsclient/nfs.h>
 #include <nfsclient/nfsnode.h>
+#include <nfsclient/nfs_kdtrace.h>
 #include <nfs/xdr_subs.h>
 #include <nfsclient/nfsm_subs.h>
 #include <nfsclient/nfsmount.h>
@@ -81,6 +84,24 @@ __FBSDID("$FreeBSD$");
  */
 #include <machine/stdarg.h>
 
+#ifdef KDTRACE_HOOKS
+dtrace_nfsclient_attrcache_flush_probe_func_t
+    dtrace_nfsclient_attrcache_flush_done_probe;
+uint32_t nfsclient_attrcache_flush_done_id;
+
+dtrace_nfsclient_attrcache_get_hit_probe_func_t
+    dtrace_nfsclient_attrcache_get_hit_probe;
+uint32_t nfsclient_attrcache_get_hit_id;
+
+dtrace_nfsclient_attrcache_get_miss_probe_func_t
+    dtrace_nfsclient_attrcache_get_miss_probe;
+uint32_t nfsclient_attrcache_get_miss_id;
+
+dtrace_nfsclient_attrcache_load_probe_func_t
+    dtrace_nfsclient_attrcache_load_done_probe;
+uint32_t nfsclient_attrcache_load_done_id;
+#endif /* !KDTRACE_HOOKS */
+
 /*
  * Data items converted to xdr at startup, since they are constant
  * This is kinda hokey, but may save a little time doing byte swaps
@@ -556,7 +577,7 @@ nfs_loadattrcache(struct vnode **vpp, struct mbuf **mdp, caddr_t *dposp,
 	struct vnode *vp = *vpp;
 	struct vattr *vap;
 	struct nfs_fattr *fp;
-	struct nfsnode *np;
+	struct nfsnode *np = NULL;
 	int32_t t1;
 	caddr_t cp2;
 	int rdev;
@@ -566,12 +587,15 @@ nfs_loadattrcache(struct vnode **vpp, struct mbuf **mdp, caddr_t *dposp,
 	struct timespec mtime, mtime_save;
 	int v3 = NFS_ISV3(vp);
 	struct thread *td = curthread;
+	int error = 0;
 
 	md = *mdp;
 	t1 = (mtod(md, caddr_t) + md->m_len) - *dposp;
 	cp2 = nfsm_disct(mdp, dposp, NFSX_FATTR(v3), t1, M_WAIT);
-	if (cp2 == NULL)
-		return EBADRPC;
+	if (cp2 == NULL) {
+		error = EBADRPC;
+		goto out;
+	}
 	fp = (struct nfs_fattr *)cp2;
 	if (v3) {
 		vtyp = nfsv3tov_type(fp->fa_type);
@@ -684,6 +708,7 @@ nfs_loadattrcache(struct vnode **vpp, struct mbuf **mdp, caddr_t *dposp,
 				 */
 				vap->va_size = np->n_size;
 				np->n_attrstamp = 0;
+				KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 			} else if (np->n_flag & NMODIFIED) {
 				/*
 				 * We've modified the file: Use the larger
@@ -716,9 +741,11 @@ nfs_loadattrcache(struct vnode **vpp, struct mbuf **mdp, caddr_t *dposp,
 	 * We detect this by for the mtime moving back. We invalidate the 
 	 * attrcache when this happens.
 	 */
-	if (timespeccmp(&mtime_save, &vap->va_mtime, >))
+	if (timespeccmp(&mtime_save, &vap->va_mtime, >)) {
 		/* Size changed or mtime went backwards */
 		np->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
+	}
 	if (vaper != NULL) {
 		bcopy((caddr_t)vap, (caddr_t)vaper, sizeof(*vap));
 		if (np->n_flag & NCHG) {
@@ -728,8 +755,18 @@ nfs_loadattrcache(struct vnode **vpp, struct mbuf **mdp, caddr_t *dposp,
 				vaper->va_mtime = np->n_mtim;
 		}
 	}
+
+#ifdef KDTRACE_HOOKS
+	if (np->n_attrstamp != 0)
+		KDTRACE_NFS_ATTRCACHE_LOAD_DONE(vp, &np->n_vattr, 0);
+#endif
 	mtx_unlock(&np->n_mtx);
-	return (0);
+out:
+#ifdef KDTRACE_HOOKS
+	if (error)
+		KDTRACE_NFS_ATTRCACHE_LOAD_DONE(vp, NULL, error);
+#endif
+	return (error);
 }
 
 #ifdef NFS_ACDEBUG
@@ -794,7 +831,8 @@ nfs_getattrcache(struct vnode *vp, struct vattr *vaper)
 	if ((time_second - np->n_attrstamp) >= timeo) {
 		nfsstats.attrcache_misses++;
 		mtx_unlock(&np->n_mtx);
-		return( ENOENT);
+		KDTRACE_NFS_ATTRCACHE_GET_MISS(vp);
+		return (ENOENT);
 	}
 	nfsstats.attrcache_hits++;
 	if (vap->va_size != np->n_size) {
@@ -823,9 +861,33 @@ nfs_getattrcache(struct vnode *vp, struct vattr *vaper)
 #ifdef NFS_ACDEBUG
 	mtx_unlock(&Giant);	/* nfs_printf() */
 #endif
+	KDTRACE_NFS_ATTRCACHE_GET_HIT(vp, vap);
 	return (0);
 }
 
+/*
+ * Purge all cached information about an NFS vnode including name
+ * cache entries, the attribute cache, and the access cache.  This is
+ * called when an NFS request for a node fails with a stale
+ * filehandle.
+ */
+void
+nfs_purgecache(struct vnode *vp)
+{
+	struct nfsnode *np;
+	int i;
+
+	np = VTONFS(vp);
+	cache_purge(vp);
+	mtx_lock(&np->n_mtx);
+	np->n_attrstamp = 0;
+	KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
+	for (i = 0; i < NFS_ACCESSCACHESIZE; i++)
+		np->n_accesscache[i].stamp = 0;
+	KDTRACE_NFS_ACCESSCACHE_FLUSH_DONE(vp);
+	mtx_unlock(&np->n_mtx);
+}
+
 static nfsuint64 nfs_nullcookie = { { 0, 0 } };
 /*
  * This function finds the directory cookie that corresponds to the
diff --git a/sys/nfsclient/nfs_vnops.c b/sys/nfsclient/nfs_vnops.c
index fd44c1a..2093b3f 100644
--- a/sys/nfsclient/nfs_vnops.c
+++ b/sys/nfsclient/nfs_vnops.c
@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
  */
 
 #include "opt_inet.h"
+#include "opt_kdtrace.h"
 
 #include <sys/param.h>
 #include <sys/kernel.h>
@@ -76,6 +77,7 @@ __FBSDID("$FreeBSD$");
 #include <nfsclient/nfs.h>
 #include <nfsclient/nfsnode.h>
 #include <nfsclient/nfsmount.h>
+#include <nfsclient/nfs_kdtrace.h>
 #include <nfsclient/nfs_lock.h>
 #include <nfs/xdr_subs.h>
 #include <nfsclient/nfsm_subs.h>
@@ -85,6 +87,26 @@ __FBSDID("$FreeBSD$");
 #include <netinet/in_var.h>
 #include <netinet/vinet.h>
 
+#include <machine/stdarg.h>
+
+#ifdef KDTRACE_HOOKS
+#include <sys/dtrace_bsd.h>
+
+dtrace_nfsclient_accesscache_flush_probe_func_t
+    dtrace_nfsclient_accesscache_flush_done_probe;
+uint32_t nfsclient_accesscache_flush_done_id;
+
+dtrace_nfsclient_accesscache_get_probe_func_t
+    dtrace_nfsclient_accesscache_get_hit_probe,
+    dtrace_nfsclient_accesscache_get_miss_probe;
+uint32_t nfsclient_accesscache_get_hit_id;
+uint32_t nfsclient_accesscache_get_miss_id;
+
+dtrace_nfsclient_accesscache_load_probe_func_t
+    dtrace_nfsclient_accesscache_load_done_probe;
+uint32_t nfsclient_accesscache_load_done_id;
+#endif /* !KDTRACE_HOOKS */
+
 /* Defs */
 #define	TRUE	1
 #define	FALSE	0
@@ -146,7 +168,6 @@ struct vop_vector nfs_vnodeops = {
 	.vop_getpages =		nfs_getpages,
 	.vop_putpages =		nfs_putpages,
 	.vop_inactive =		nfs_inactive,
-	.vop_lease =		VOP_NULL,
 	.vop_link =		nfs_link,
 	.vop_lookup =		nfs_lookup,
 	.vop_mkdir =		nfs_mkdir,
@@ -270,11 +291,11 @@ SYSCTL_INT(_vfs_nfs, OID_AUTO, access_cache_misses, CTLFLAG_RD,
 
 static int
 nfs3_access_otw(struct vnode *vp, int wmode, struct thread *td,
-    struct ucred *cred)
+    struct ucred *cred, uint32_t *retmode)
 {
 	const int v3 = 1;
 	u_int32_t *tl;
-	int error = 0, attrflag;
+	int error = 0, attrflag, i, lrupos;
 
 	struct mbuf *mreq, *mrep, *md, *mb;
 	caddr_t bpos, dpos;
@@ -291,16 +312,38 @@ nfs3_access_otw(struct vnode *vp, int wmode, struct thread *td,
 	nfsm_request(vp, NFSPROC_ACCESS, td, cred);
 	nfsm_postop_attr(vp, attrflag);
 	if (!error) {
+		lrupos = 0;
 		tl = nfsm_dissect(u_int32_t *, NFSX_UNSIGNED);
 		rmode = fxdr_unsigned(u_int32_t, *tl);
 		mtx_lock(&np->n_mtx);
-		np->n_mode = rmode;
-		np->n_modeuid = cred->cr_uid;
-		np->n_modestamp = time_second;
+		for (i = 0; i < NFS_ACCESSCACHESIZE; i++) {
+			if (np->n_accesscache[i].uid == cred->cr_uid) {
+				np->n_accesscache[i].mode = rmode;
+				np->n_accesscache[i].stamp = time_second;
+				break;
+			}
+			if (i > 0 && np->n_accesscache[i].stamp <
+			    np->n_accesscache[lrupos].stamp)
+				lrupos = i;
+		}
+		if (i == NFS_ACCESSCACHESIZE) {
+			np->n_accesscache[lrupos].uid = cred->cr_uid;
+			np->n_accesscache[lrupos].mode = rmode;
+			np->n_accesscache[lrupos].stamp = time_second;
+		}
 		mtx_unlock(&np->n_mtx);
+		if (retmode != NULL)
+			*retmode = rmode;
+		KDTRACE_NFS_ACCESSCACHE_LOAD_DONE(vp, cred->cr_uid, rmode, 0);
 	}
 	m_freem(mrep);
 nfsmout:
+#ifdef KDTRACE_HOOKS
+	if (error) {
+		KDTRACE_NFS_ACCESSCACHE_LOAD_DONE(vp, cred->cr_uid, 0,
+		    error);
+	}
+#endif
 	return (error);
 }
 
@@ -314,8 +357,8 @@ static int
 nfs_access(struct vop_access_args *ap)
 {
 	struct vnode *vp = ap->a_vp;
-	int error = 0;
-	u_int32_t mode, wmode;
+	int error = 0, i, gotahit;
+	u_int32_t mode, rmode, wmode;
 	int v3 = NFS_ISV3(vp);
 	struct nfsnode *np = VTONFS(vp);
 
@@ -372,26 +415,40 @@ nfs_access(struct vop_access_args *ap)
 		 * Does our cached result allow us to give a definite yes to
 		 * this request?
 		 */
+		gotahit = 0;
 		mtx_lock(&np->n_mtx);
-		if ((time_second < (np->n_modestamp + nfsaccess_cache_timeout)) &&
-		    (ap->a_cred->cr_uid == np->n_modeuid) &&
-		    ((np->n_mode & mode) == mode)) {
-			nfsstats.accesscache_hits++;
-		} else {
+		for (i = 0; i < NFS_ACCESSCACHESIZE; i++) {
+			if (ap->a_cred->cr_uid == np->n_accesscache[i].uid) {
+				if (time_second < (np->n_accesscache[i].stamp +
+				    nfsaccess_cache_timeout) &&
+				    (np->n_accesscache[i].mode & mode) == mode) {
+					nfsstats.accesscache_hits++;
+					gotahit = 1;
+				}
+				break;
+			}
+		}
+		mtx_unlock(&np->n_mtx);
+#ifdef KDTRACE_HOOKS
+		if (gotahit)
+			KDTRACE_NFS_ACCESSCACHE_GET_HIT(vp,
+			    ap->a_cred->cr_uid, mode);
+		else
+			KDTRACE_NFS_ACCESSCACHE_GET_MISS(vp,
+			    ap->a_cred->cr_uid, mode);
+#endif
+		if (gotahit == 0) {
 			/*
 			 * Either a no, or a don't know.  Go to the wire.
 			 */
 			nfsstats.accesscache_misses++;
-			mtx_unlock(&np->n_mtx);
-		        error = nfs3_access_otw(vp, wmode, ap->a_td,ap->a_cred);
-			mtx_lock(&np->n_mtx);
+		        error = nfs3_access_otw(vp, wmode, ap->a_td, ap->a_cred,
+			    &rmode);
 			if (!error) {
-				if ((np->n_mode & mode) != mode) {
+				if ((rmode & mode) != mode)
 					error = EACCES;
-				}
 			}
 		}
-		mtx_unlock(&np->n_mtx);
 		return (error);
 	} else {
 		if ((error = nfsspec_access(ap)) != 0) {
@@ -473,6 +530,7 @@ nfs_open(struct vop_open_args *ap)
 		if (error == EINTR || error == EIO)
 			return (error);
 		np->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 		if (vp->v_type == VDIR)
 			np->n_direofoffset = 0;
 		error = VOP_GETATTR(vp, &vattr, ap->a_cred);
@@ -489,6 +547,7 @@ nfs_open(struct vop_open_args *ap)
 		    td->td_proc == NULL ||
 		    np->n_ac_ts_pid != td->td_proc->p_pid) {
 			np->n_attrstamp = 0;
+			KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 		}
 		mtx_unlock(&np->n_mtx);						
 		error = VOP_GETATTR(vp, &vattr, ap->a_cred);
@@ -651,7 +710,7 @@ nfs_getattr(struct vop_getattr_args *ap)
 		goto nfsmout;
 	if (v3 && nfs_prime_access_cache && nfsaccess_cache_timeout > 0) {
 		nfsstats.accesscache_misses++;
-		nfs3_access_otw(vp, NFSV3ACCESS_ALL, td, ap->a_cred);
+		nfs3_access_otw(vp, NFSV3ACCESS_ALL, td, ap->a_cred, NULL);
 		if (nfs_getattrcache(vp, &vattr) == 0)
 			goto nfsmout;
 	}
@@ -810,7 +869,7 @@ nfs_setattrrpc(struct vnode *vp, struct vattr *vap, struct ucred *cred)
 	struct nfsnode *np = VTONFS(vp);
 	caddr_t bpos, dpos;
 	u_int32_t *tl;
-	int error = 0, wccflag = NFSV3_WCCRATTR;
+	int error = 0, i, wccflag = NFSV3_WCCRATTR;
 	struct mbuf *mreq, *mrep, *md, *mb;
 	int v3 = NFS_ISV3(vp);
 
@@ -843,7 +902,11 @@ nfs_setattrrpc(struct vnode *vp, struct vattr *vap, struct ucred *cred)
 	}
 	nfsm_request(vp, NFSPROC_SETATTR, curthread, cred);
 	if (v3) {
-		np->n_modestamp = 0;
+		mtx_lock(&np->n_mtx);
+		for (i = 0; i < NFS_ACCESSCACHESIZE; i++)
+			np->n_accesscache[i].stamp = 0;
+		mtx_unlock(&np->n_mtx);
+		KDTRACE_NFS_ACCESSCACHE_FLUSH_DONE(vp);
 		nfsm_wcc_data(vp, wccflag);
 	} else
 		nfsm_loadattr(vp, NULL);
@@ -914,6 +977,8 @@ nfs_lookup(struct vop_lookup_args *ap)
 			vrele(newvp);
 		*vpp = NULLVP;
 	} else if (error == ENOENT) {
+		if (dvp->v_iflag & VI_DOOMED)
+			return (ENOENT);
 		/*
 		 * We only accept a negative hit in the cache if the
 		 * modification time of the parent directory matches
@@ -1397,8 +1462,10 @@ nfsmout:
 	}
 	mtx_lock(&(VTONFS(dvp))->n_mtx);
 	VTONFS(dvp)->n_flag |= NMODIFIED;
-	if (!wccflag)
+	if (!wccflag) {
 		VTONFS(dvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
+	}
 	mtx_unlock(&(VTONFS(dvp))->n_mtx);
 	return (error);
 }
@@ -1529,8 +1596,10 @@ nfsmout:
 	}
 	mtx_lock(&(VTONFS(dvp))->n_mtx);
 	VTONFS(dvp)->n_flag |= NMODIFIED;
-	if (!wccflag)
+	if (!wccflag) {
 		VTONFS(dvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
+	}
 	mtx_unlock(&(VTONFS(dvp))->n_mtx);
 	return (error);
 }
@@ -1594,6 +1663,7 @@ nfs_remove(struct vop_remove_args *ap)
 	} else if (!np->n_sillyrename)
 		error = nfs_sillyrename(dvp, vp, cnp);
 	np->n_attrstamp = 0;
+	KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
 	return (error);
 }
 
@@ -1639,8 +1709,10 @@ nfs_removerpc(struct vnode *dvp, const char *name, int namelen,
 nfsmout:
 	mtx_lock(&(VTONFS(dvp))->n_mtx);
 	VTONFS(dvp)->n_flag |= NMODIFIED;
-	if (!wccflag)
+	if (!wccflag) {
 		VTONFS(dvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
+	}
 	mtx_unlock(&(VTONFS(dvp))->n_mtx);
 	return (error);
 }
@@ -1785,10 +1857,14 @@ nfsmout:
 	mtx_lock(&(VTONFS(tdvp))->n_mtx);
 	VTONFS(tdvp)->n_flag |= NMODIFIED;
 	mtx_unlock(&(VTONFS(tdvp))->n_mtx);
-	if (!fwccflag)
+	if (!fwccflag) {
 		VTONFS(fdvp)->n_attrstamp = 0;
-	if (!twccflag)
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(fdvp);
+	}
+	if (!twccflag) {
 		VTONFS(tdvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(tdvp);
+	}
 	return (error);
 }
 
@@ -1836,10 +1912,14 @@ nfsmout:
 	mtx_lock(&(VTONFS(tdvp))->n_mtx);
 	VTONFS(tdvp)->n_flag |= NMODIFIED;
 	mtx_unlock(&(VTONFS(tdvp))->n_mtx);
-	if (!attrflag)
+	if (!attrflag) {
 		VTONFS(vp)->n_attrstamp = 0;
-	if (!wccflag)
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(vp);
+	}
+	if (!wccflag) {
 		VTONFS(tdvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(tdvp);
+	}
 	return (error);
 }
 
@@ -1924,8 +2004,10 @@ nfsmout:
 	mtx_lock(&(VTONFS(dvp))->n_mtx);
 	VTONFS(dvp)->n_flag |= NMODIFIED;
 	mtx_unlock(&(VTONFS(dvp))->n_mtx);
-	if (!wccflag)
+	if (!wccflag) {
 		VTONFS(dvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
+	}
 	return (error);
 }
 
@@ -1980,8 +2062,10 @@ nfsmout:
 	mtx_lock(&(VTONFS(dvp))->n_mtx);
 	VTONFS(dvp)->n_flag |= NMODIFIED;
 	mtx_unlock(&(VTONFS(dvp))->n_mtx);
-	if (!wccflag)
+	if (!wccflag) {
 		VTONFS(dvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
+	}
 	if (error == 0 && newvp == NULL) {
 		error = nfs_lookitup(dvp, cnp->cn_nameptr, len, cnp->cn_cred,
 			cnp->cn_thread, &np);
@@ -2030,8 +2114,10 @@ nfsmout:
 	mtx_lock(&(VTONFS(dvp))->n_mtx);
 	VTONFS(dvp)->n_flag |= NMODIFIED;
 	mtx_unlock(&(VTONFS(dvp))->n_mtx);
-	if (!wccflag)
+	if (!wccflag) {
 		VTONFS(dvp)->n_attrstamp = 0;
+		KDTRACE_NFS_ATTRCACHE_FLUSH_DONE(dvp);
+	}
 	cache_purge(dvp);
 	cache_purge(vp);
 	/*
diff --git a/sys/nfsclient/nfsnode.h b/sys/nfsclient/nfsnode.h
index 7781ae6..6f10d79 100644
--- a/sys/nfsclient/nfsnode.h
+++ b/sys/nfsclient/nfsnode.h
@@ -84,6 +84,12 @@ struct nfs_attrcache_timestamp {
 	unsigned long	nfs_ac_ts_syscalls;	
 };
 
+struct nfs_accesscache {
+	u_int32_t		mode;		/* ACCESS mode cache */
+	uid_t			uid;		/* credentials having mode */
+	time_t			stamp;		/* mode cache timestamp */
+};
+	
 /*
  * The nfsnode is the nfs equivalent to ufs's inode. Any similarity
  * is purely coincidental.
@@ -104,9 +110,7 @@ struct nfsnode {
 	u_quad_t		n_lrev;		/* Modify rev for lease */
 	struct vattr		n_vattr;	/* Vnode attribute cache */
 	time_t			n_attrstamp;	/* Attr. cache timestamp */
-	u_int32_t		n_mode;		/* ACCESS mode cache */
-	uid_t			n_modeuid;	/* credentials having mode */
-	time_t			n_modestamp;	/* mode cache timestamp */
+	struct nfs_accesscache	n_accesscache[NFS_ACCESSCACHESIZE];
 	struct timespec		n_mtime;	/* Prev modify time. */
 	time_t			n_ctime;	/* Prev create time. */
 	time_t			n_dmtime;	/* Prev dir modify time. */
diff --git a/sys/nfsserver/nfs.h b/sys/nfsserver/nfs.h
index 2709377..b47ffea 100644
--- a/sys/nfsserver/nfs.h
+++ b/sys/nfsserver/nfs.h
@@ -40,6 +40,8 @@
 #include "opt_nfs.h"
 #endif
 
+#include <nfs/nfssvc.h>
+
 /*
  * Tunable constants for nfs
  */
@@ -116,13 +118,6 @@ struct nfsd_nfsd_args {
 #endif
 
 /*
- * Flags for nfssvc() system call.
- */
-#define	NFSSVC_OLDNFSD	0x004
-#define	NFSSVC_ADDSOCK	0x008
-#define	NFSSVC_NFSD	0x010
-
-/*
  * vfs.nfsrv sysctl(3) identifiers
  */
 #define NFS_NFSRVSTATS	1		/* struct: struct nfsrvstats */
@@ -447,6 +442,13 @@ int	nfsrv_symlink(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	    struct mbuf **mrq);
 int	nfsrv_write(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 	    struct mbuf **mrq);
+/*
+ * #ifdef _SYS_SYSPROTO_H_ so that it is only defined when sysproto.h
+ * has been included, so that "struct nfssvc_args" is defined.
+ */
+#ifdef _SYS_SYSPROTO_H_
+int nfssvc_nfsserver(struct thread *, struct nfssvc_args *);
+#endif
 #endif	/* _KERNEL */
 
 #endif
diff --git a/sys/nfsserver/nfs_srvkrpc.c b/sys/nfsserver/nfs_srvkrpc.c
index e52ac03..77bdec8 100644
--- a/sys/nfsserver/nfs_srvkrpc.c
+++ b/sys/nfsserver/nfs_srvkrpc.c
@@ -151,6 +151,9 @@ int32_t (*nfsrv3_procs[NFS_NPROCS])(struct nfsrv_descript *nd,
 /*
  * NFS server system calls
  */
+/*
+ * This is now called from nfssvc() in nfs/nfs_nfssvc.c.
+ */
 
 /*
  * Nfs server psuedo system call for the nfsd's
@@ -163,25 +166,14 @@ int32_t (*nfsrv3_procs[NFS_NPROCS])(struct nfsrv_descript *nd,
  *  - sockaddr with no IPv4-mapped addresses
  *  - mask for both INET and INET6 families if there is IPv4-mapped overlap
  */
-#ifndef _SYS_SYSPROTO_H_
-struct nfssvc_args {
-	int flag;
-	caddr_t argp;
-};
-#endif
 int
-nfssvc(struct thread *td, struct nfssvc_args *uap)
+nfssvc_nfsserver(struct thread *td, struct nfssvc_args *uap)
 {
 	struct file *fp;
 	struct nfsd_addsock_args addsockarg;
 	struct nfsd_nfsd_args nfsdarg;
 	int error;
 
-	KASSERT(!mtx_owned(&Giant), ("nfssvc(): called with Giant"));
-
-	error = priv_check(td, PRIV_NFS_DAEMON);
-	if (error)
-		return (error);
 	if (uap->flag & NFSSVC_ADDSOCK) {
 		error = copyin(uap->argp, (caddr_t)&addsockarg,
 		    sizeof(addsockarg));
@@ -208,8 +200,6 @@ nfssvc(struct thread *td, struct nfssvc_args *uap)
 	} else {
 		error = ENXIO;
 	}
-	if (error == EINTR || error == ERESTART)
-		error = 0;
 	return (error);
 }
 
@@ -397,6 +387,7 @@ nfssvc_program(struct svc_req *rqst, SVCXPRT *xprt)
 #endif
 #endif
 			    inet_ntoa(sin->sin_addr), port);
+			m_freem(mreq);
 			svcerr_weakauth(rqst);
 			svc_freereq(rqst);
 			return;
@@ -405,6 +396,7 @@ nfssvc_program(struct svc_req *rqst, SVCXPRT *xprt)
 
 	if (proc != nfsrv_null) {
 		if (!svc_getcred(rqst, &nd.nd_cr, &nd.nd_credflavor)) {
+			m_freem(mreq);
 			svcerr_weakauth(rqst);
 			svc_freereq(rqst);
 			return;
diff --git a/sys/nfsserver/nfs_srvsubs.c b/sys/nfsserver/nfs_srvsubs.c
index 00e5216..d154463 100644
--- a/sys/nfsserver/nfs_srvsubs.c
+++ b/sys/nfsserver/nfs_srvsubs.c
@@ -100,10 +100,6 @@ struct nfsd_head nfsd_head;
 int nfsd_head_flag;
 #endif
 
-static int nfssvc_offset = SYS_nfssvc;
-static struct sysent nfssvc_prev_sysent;
-MAKE_SYSENT(nfssvc);
-
 struct mtx nfsd_mtx;
 
 /*
@@ -519,13 +515,14 @@ static const short *nfsrv_v3errmap[] = {
 	nfsv3err_commit,
 };
 
+extern int (*nfsd_call_nfsserver)(struct thread *, struct nfssvc_args *);
+
 /*
  * Called once to initialize data structures...
  */
 static int
 nfsrv_modevent(module_t mod, int type, void *data)
 {
-	static int registered;
 	int error = 0;
 
 	switch (type) {
@@ -560,11 +557,7 @@ nfsrv_modevent(module_t mod, int type, void *data)
 		NFSD_UNLOCK();
 #endif
 
-		error = syscall_register(&nfssvc_offset, &nfssvc_sysent,
-		    &nfssvc_prev_sysent);
-		if (error)
-			break;
-		registered = 1;
+		nfsd_call_nfsserver = nfssvc_nfsserver;
 		break;
 
 	case MOD_UNLOAD:
@@ -573,8 +566,7 @@ nfsrv_modevent(module_t mod, int type, void *data)
 			break;
 		}
 
-		if (registered)
-			syscall_deregister(&nfssvc_offset, &nfssvc_prev_sysent);
+		nfsd_call_nfsserver = NULL;
 		callout_drain(&nfsrv_callout);
 #ifdef NFS_LEGACYRPC
 		nfsrv_destroycache();	/* Free the server request cache */
@@ -596,6 +588,7 @@ DECLARE_MODULE(nfsserver, nfsserver_mod, SI_SUB_VFS, SI_ORDER_ANY);
 
 /* So that loader and kldload(2) can find us, wherever we are.. */
 MODULE_VERSION(nfsserver, 1);
+MODULE_DEPEND(nfsserver, nfssvc, 1, 1, 1);
 #ifndef NFS_LEGACYRPC
 MODULE_DEPEND(nfsserver, krpc, 1, 1, 1);
 #endif
diff --git a/sys/nfsserver/nfs_syscalls.c b/sys/nfsserver/nfs_syscalls.c
index 326d623..59a30a3 100644
--- a/sys/nfsserver/nfs_syscalls.c
+++ b/sys/nfsserver/nfs_syscalls.c
@@ -113,6 +113,9 @@ extern u_long sb_max_adj;
  */
 
 /*
+ * This is now called from nfssvc() in nfs/nfs_nfssvc.c.
+ */
+/*
  * Nfs server psuedo system call for the nfsd's
  * Based on the flag value it either:
  * - adds a socket to the selection list
@@ -123,27 +126,14 @@ extern u_long sb_max_adj;
  *  - sockaddr with no IPv4-mapped addresses
  *  - mask for both INET and INET6 families if there is IPv4-mapped overlap
  */
-#ifndef _SYS_SYSPROTO_H_
-struct nfssvc_args {
-	int flag;
-	caddr_t argp;
-};
-#endif
 int
-nfssvc(struct thread *td, struct nfssvc_args *uap)
+nfssvc_nfsserver(struct thread *td, struct nfssvc_args *uap)
 {
 	struct file *fp;
 	struct sockaddr *nam;
 	struct nfsd_addsock_args nfsdarg;
 	int error;
 
-	KASSERT(!mtx_owned(&Giant), ("nfssvc(): called with Giant"));
-
-	AUDIT_ARG(cmd, uap->flag);
-
-	error = priv_check(td, PRIV_NFS_DAEMON);
-	if (error)
-		return (error);
 	NFSD_LOCK();
 	while (nfssvc_sockhead_flag & SLP_INIT) {
 		 nfssvc_sockhead_flag |= SLP_WANTINIT;
@@ -181,8 +171,6 @@ nfssvc(struct thread *td, struct nfssvc_args *uap)
 	} else {
 		error = ENXIO;
 	}
-	if (error == EINTR || error == ERESTART)
-		error = 0;
 	return (error);
 }
 
diff --git a/sys/pc98/conf/GENERIC b/sys/pc98/conf/GENERIC
index a393748..b6adce6 100644
--- a/sys/pc98/conf/GENERIC
+++ b/sys/pc98/conf/GENERIC
@@ -265,7 +265,6 @@ device		bpf		# Berkeley packet filter
 #device		rum		# Ralink Technology RT2501USB wireless NICs
 #device		zyd		# ZyDAS zb1211/zb1211b wireless NICs
 #device		urio		# Diamond Rio 500 MP3 player
-#device		uscanner	# Scanners
 # USB Serial devices
 #device		uark		# Technologies ARK3116 based serial adapters
 #device		ubsa		# Belkin F5U103 and compatible serial adapters
diff --git a/sys/pc98/conf/NOTES b/sys/pc98/conf/NOTES
index e9d5af5..34823da 100644
--- a/sys/pc98/conf/NOTES
+++ b/sys/pc98/conf/NOTES
@@ -372,8 +372,6 @@ hint.mse.0.irq="13"
 # Network interfaces:
 #
 
-# ar:   Arnet SYNC/570i hdlc sync 2/4 port V.35/X.21 serial driver
-#       (requires sppp)
 # ce:   Cronyx Tau-PCI/32 sync single/dual port G.703/E1 serial adaptor
 #       with 32 HDLC subchannels (requires sppp (default), or NETGRAPH if
 #       NETGRAPH_CRONYX is configured)
@@ -391,13 +389,11 @@ hint.mse.0.irq="13"
 # ral:	Ralink Technology IEEE 802.11 wireless adapter
 # sbni: Granch SBNI12-xx ISA and PCI adapters
 # snc:	National Semiconductor DP8393X SONIC Ethernet adapter driver
-# sr:   RISCom/N2 hdlc sync 1/2 port V.35/X.21 serial driver (requires sppp)
 # ural:	Ralink Technology RT2500USB IEEE 802.11 wireless adapter
 # ath:  Atheros a/b/g WiFi adapters (requires ath_hal and wlan)
 
 # Order for ISA/EISA devices is important here
 
-device		ar
 device		ce
 device		cp
 device		cs
@@ -427,7 +423,6 @@ hint.snc.0.at="isa"
 hint.snc.0.port="0x888"
 hint.snc.0.irq="6"
 hint.snc.0.maddr="0xc0000"
-device		sr
 device		ural
 
 device		ath		# Atheros pci/cardbus NIC's
diff --git a/sys/pc98/pc98/machdep.c b/sys/pc98/pc98/machdep.c
index 3623519..a42b903 100644
--- a/sys/pc98/pc98/machdep.c
+++ b/sys/pc98/pc98/machdep.c
@@ -525,6 +525,7 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	struct sigacts *psp;
 	char *sp;
 	struct trapframe *regs;
+	struct segment_descriptor *sdp;
 	int sig;
 	int oonstack;
 
@@ -561,6 +562,15 @@ sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
 	sf.sf_uc.uc_mcontext.mc_len = sizeof(sf.sf_uc.uc_mcontext); /* magic */
 	get_fpcontext(td, &sf.sf_uc.uc_mcontext);
 	fpstate_drop(td);
+	/*
+	 * Unconditionally fill the fsbase and gsbase into the mcontext.
+	 */
+	sdp = &td->td_pcb->pcb_gsd;
+	sf.sf_uc.uc_mcontext.mc_fsbase = sdp->sd_hibase << 24 |
+	    sdp->sd_lobase;
+	sdp = &td->td_pcb->pcb_fsd;
+	sf.sf_uc.uc_mcontext.mc_gsbase = sdp->sd_hibase << 24 |
+	    sdp->sd_lobase;
 
 	/* Allocate space for the signal handler context. */
 	if ((td->td_pflags & TDP_ALTSTACK) != 0 && !oonstack &&
@@ -1308,234 +1318,234 @@ extern  vm_offset_t	proc0kstack;
  */
 struct soft_segment_descriptor gdt_segs[] = {
 /* GNULL_SEL	0 Null Descriptor */
-{	0x0,			/* segment base address  */
-	0x0,			/* length */
-	0,			/* segment type */
-	0,			/* segment descriptor priority level */
-	0,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 /* GPRIV_SEL	1 SMP Per-Processor Private Data Descriptor */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMRWA,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GUFS_SEL	2 %fs Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMRWA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GUGS_SEL	3 %gs Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMRWA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GCODE_SEL	4 Code Descriptor for kernel */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMERA,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMERA,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GDATA_SEL	5 Data Descriptor for kernel */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMRWA,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GUCODE_SEL	6 Code Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMERA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMERA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GUDATA_SEL	7 Data Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMRWA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GBIOSLOWMEM_SEL 8 BIOS access to realmode segment 0x40, must be #8 in GDT */
-{	0x400,			/* segment base address */
-	0xfffff,		/* length */
-	SDT_MEMRWA,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x400,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GPROC0_SEL	9 Proc 0 Tss Descriptor */
 {
-	0x0,			/* segment base address */
-	sizeof(struct i386tss)-1,/* length  */
-	SDT_SYS386TSS,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* unused - default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+	.ssd_base = 0x0,
+	.ssd_limit = sizeof(struct i386tss)-1,
+	.ssd_type = SDT_SYS386TSS,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 /* GLDT_SEL	10 LDT Descriptor */
-{	(int) ldt,		/* segment base address  */
-	sizeof(ldt)-1,		/* length - all address space */
-	SDT_SYSLDT,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* unused - default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = (int) ldt,
+	.ssd_limit = sizeof(ldt)-1,
+	.ssd_type = SDT_SYSLDT,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 /* GUSERLDT_SEL	11 User LDT Descriptor per process */
-{	(int) ldt,		/* segment base address  */
-	(512 * sizeof(union descriptor)-1),		/* length */
-	SDT_SYSLDT,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* unused - default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = (int) ldt,
+	.ssd_limit = (512 * sizeof(union descriptor)-1),
+	.ssd_type = SDT_SYSLDT,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 /* GPANIC_SEL	12 Panic Tss Descriptor */
-{	(int) &dblfault_tss,	/* segment base address  */
-	sizeof(struct i386tss)-1,/* length - all address space */
-	SDT_SYS386TSS,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* unused - default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = (int) &dblfault_tss,
+	.ssd_limit = sizeof(struct i386tss)-1,
+	.ssd_type = SDT_SYS386TSS,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 /* GBIOSCODE32_SEL 13 BIOS 32-bit interface (32bit Code) */
-{	0,			/* segment base address (overwritten)  */
-	0xfffff,		/* length */
-	SDT_MEMERA,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMERA,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 1		},
 /* GBIOSCODE16_SEL 14 BIOS 32-bit interface (16bit Code) */
-{	0,			/* segment base address (overwritten)  */
-	0xfffff,		/* length */
-	SDT_MEMERA,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMERA,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 1		},
 /* GBIOSDATA_SEL 15 BIOS 32-bit interface (Data) */
-{	0,			/* segment base address (overwritten) */
-	0xfffff,		/* length */
-	SDT_MEMRWA,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 /* GBIOSUTIL_SEL 16 BIOS 16-bit interface (Utility) */
-{	0,			/* segment base address (overwritten) */
-	0xfffff,		/* length */
-	SDT_MEMRWA,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 1		},
 /* GBIOSARGS_SEL 17 BIOS 16-bit interface (Arguments) */
-{	0,			/* segment base address (overwritten) */
-	0xfffff,		/* length */
-	SDT_MEMRWA,		/* segment type */
-	0,			/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = 0,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 1		},
 /* GNDIS_SEL	18 NDIS Descriptor */
-{	0x0,			/* segment base address  */
-	0x0,			/* length */
-	0,			/* segment type */
-	0,			/* segment descriptor priority level */
-	0,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 };
 
 static struct soft_segment_descriptor ldt_segs[] = {
 	/* Null Descriptor - overwritten by call gate */
-{	0x0,			/* segment base address  */
-	0x0,			/* length - all address space */
-	0,			/* segment type */
-	0,			/* segment descriptor priority level */
-	0,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 	/* Null Descriptor - overwritten by call gate */
-{	0x0,			/* segment base address  */
-	0x0,			/* length - all address space */
-	0,			/* segment type */
-	0,			/* segment descriptor priority level */
-	0,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 	/* Null Descriptor - overwritten by call gate */
-{	0x0,			/* segment base address  */
-	0x0,			/* length - all address space */
-	0,			/* segment type */
-	0,			/* segment descriptor priority level */
-	0,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 	/* Code Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMERA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMERA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 	/* Null Descriptor - overwritten by call gate */
-{	0x0,			/* segment base address  */
-	0x0,			/* length - all address space */
-	0,			/* segment type */
-	0,			/* segment descriptor priority level */
-	0,			/* segment descriptor present */
-	0, 0,
-	0,			/* default 32 vs 16 bit size */
-	0  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0x0,
+	.ssd_type = 0,
+	.ssd_dpl = 0,
+	.ssd_p = 0,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 0,
+	.ssd_gran = 0		},
 	/* Data Descriptor for user */
-{	0x0,			/* segment base address  */
-	0xfffff,		/* length - all address space */
-	SDT_MEMRWA,		/* segment type */
-	SEL_UPL,		/* segment descriptor priority level */
-	1,			/* segment descriptor present */
-	0, 0,
-	1,			/* default 32 vs 16 bit size */
-	1  			/* limit granularity (byte/page units)*/ },
+{	.ssd_base = 0x0,
+	.ssd_limit = 0xfffff,
+	.ssd_type = SDT_MEMRWA,
+	.ssd_dpl = SEL_UPL,
+	.ssd_p = 1,
+	.ssd_xx = 0, .ssd_xx1 = 0,
+	.ssd_def32 = 1,
+	.ssd_gran = 1		},
 };
 
 void
@@ -2410,6 +2420,7 @@ int
 get_mcontext(struct thread *td, mcontext_t *mcp, int flags)
 {
 	struct trapframe *tp;
+	struct segment_descriptor *sdp;
 
 	tp = td->td_frame;
 
@@ -2441,6 +2452,11 @@ get_mcontext(struct thread *td, mcontext_t *mcp, int flags)
 	mcp->mc_ss = tp->tf_ss;
 	mcp->mc_len = sizeof(*mcp);
 	get_fpcontext(td, mcp);
+	sdp = &td->td_pcb->pcb_gsd;
+	mcp->mc_fsbase = sdp->sd_hibase << 24 | sdp->sd_lobase;
+	sdp = &td->td_pcb->pcb_fsd;
+	mcp->mc_gsbase = sdp->sd_hibase << 24 | sdp->sd_lobase;
+
 	return (0);
 }
 
@@ -2775,45 +2791,24 @@ user_dbreg_trap(void)
 #ifdef KDB
 
 /*
- * Provide inb() and outb() as functions.  They are normally only
- * available as macros calling inlined functions, thus cannot be
- * called from the debugger.
- *
- * The actual code is stolen from <machine/cpufunc.h>, and de-inlined.
+ * Provide inb() and outb() as functions.  They are normally only available as
+ * inline functions, thus cannot be called from the debugger.
  */
 
-#undef inb
-#undef outb
-
 /* silence compiler warnings */
-u_char inb(u_int);
-void outb(u_int, u_char);
+u_char inb_(u_short);
+void outb_(u_short, u_char);
 
 u_char
-inb(u_int port)
+inb_(u_short port)
 {
-	u_char	data;
-	/*
-	 * We use %%dx and not %1 here because i/o is done at %dx and not at
-	 * %edx, while gcc generates inferior code (movw instead of movl)
-	 * if we tell it to load (u_short) port.
-	 */
-	__asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port));
-	return (data);
+	return inb(port);
 }
 
 void
-outb(u_int port, u_char data)
+outb_(u_short port, u_char data)
 {
-	u_char	al;
-	/*
-	 * Use an unnecessary assignment to help gcc's register allocator.
-	 * This make a large difference for gcc-1.40 and a tiny difference
-	 * for gcc-2.6.0.  For gcc-1.40, al had to be ``asm("ax")'' for
-	 * best results.  gcc-2.6.0 can't handle this.
-	 */
-	al = data;
-	__asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port));
+	outb(port, data);
 }
 
 #endif /* KDB */
diff --git a/sys/pci/intpm.c b/sys/pci/intpm.c
index 6b71183..63eb4c4 100644
--- a/sys/pci/intpm.c
+++ b/sys/pci/intpm.c
@@ -167,10 +167,6 @@ intsmb_attach(device_t dev)
 		goto fail;
 	}
 
-	value = pci_read_config(dev, PCI_BASE_ADDR_PM, 4);
-	device_printf(dev, "PM %s %x\n", (value & 1) ? "I/O mapped" : "Memory",
-	    value & 0xfffe);
-
 	sc->isbusy = 0;
 	sc->smbus = device_add_child(dev, "smbus", -1);
 	if (sc->smbus == NULL) {
@@ -365,13 +361,13 @@ intsmb_start(struct intsmb_softc *sc, unsigned char cmd, int nointr)
 	tmp |= PIIX4_SMBHSTCNT_START;
 
 	/* While not in autoconfiguration enable interrupts. */
-	if (!cold || !nointr)
+	if (!cold && !nointr)
 		tmp |= PIIX4_SMBHSTCNT_INTREN;
 	bus_write_1(sc->io_res, PIIX4_SMBHSTCNT, tmp);
 }
 
 static int
-intsmb_error(int status)
+intsmb_error(device_t dev, int status)
 {
 	int error = 0;
 
@@ -381,6 +377,10 @@ intsmb_error(int status)
 		error |= SMB_ECOLLI;
 	if (status & PIIX4_SMBHSTSTAT_FAIL)
 		error |= SMB_ENOACK;
+
+	if (error != 0 && bootverbose)
+		device_printf(dev, "error = %d, status = %#x\n", error, status);
+
 	return (error);
 }
 
@@ -410,7 +410,7 @@ intsmb_stop_poll(struct intsmb_softc *sc)
 		status = bus_read_1(sc->io_res, PIIX4_SMBHSTSTS);
 		if (!(status & PIIX4_SMBHSTSTAT_BUSY)) {
 			sc->isbusy = 0;
-			error = intsmb_error(status);
+			error = intsmb_error(sc->dev, status);
 			if (error == 0 && !(status & PIIX4_SMBHSTSTAT_INTR))
 				device_printf(sc->dev, "unknown cause why?\n");
 			return (error);
@@ -442,7 +442,7 @@ intsmb_stop(struct intsmb_softc *sc)
 	if (error == 0) {
 		status = bus_read_1(sc->io_res, PIIX4_SMBHSTSTS);
 		if (!(status & PIIX4_SMBHSTSTAT_BUSY)) {
-			error = intsmb_error(status);
+			error = intsmb_error(sc->dev, status);
 			if (error == 0 && !(status & PIIX4_SMBHSTSTAT_INTR))
 				device_printf(sc->dev, "unknown cause why?\n");
 #ifdef ENABLE_ALART
diff --git a/sys/powerpc/aim/machdep.c b/sys/powerpc/aim/machdep.c
index da8f35f..fb83e3a 100644
--- a/sys/powerpc/aim/machdep.c
+++ b/sys/powerpc/aim/machdep.c
@@ -117,6 +117,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/powerpc.h>
 #include <machine/reg.h>
 #include <machine/sigframe.h>
+#include <machine/spr.h>
 #include <machine/trap.h>
 #include <machine/vmparam.h>
 
@@ -130,6 +131,8 @@ extern vm_offset_t ksym_start, ksym_end;
 
 int cold = 1;
 int cacheline_size = 32;
+int ppc64 = 0;
+int hw_direct_map = 1;
 
 struct pcpu __pcpu[MAXCPU];
 
@@ -230,10 +233,13 @@ cpu_startup(void *dummy)
 
 extern char	kernel_text[], _end[];
 
+extern void	*testppc64, *testppc64size;
+extern void	*restorebridge, *restorebridgesize;
+extern void	*rfid_patch, *rfi_patch1, *rfi_patch2;
 #ifdef SMP
 extern void	*rstcode, *rstsize;
 #endif
-extern void	*trapcode, *trapsize;
+extern void	*trapcode, *trapcode64, *trapsize;
 extern void	*alitrap, *alisize;
 extern void	*dsitrap, *dsisize;
 extern void	*decrint, *decrsize;
@@ -245,11 +251,16 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp)
 {
 	struct		pcpu *pc;
 	vm_offset_t	end;
+	void		*generictrap;
+	size_t		trap_offset;
 	void		*kmdp;
         char		*env;
+	uint32_t	msr, scratch;
+	uint8_t		*cache_check;
 
 	end = 0;
 	kmdp = NULL;
+	trap_offset = 0;
 
 	/*
 	 * Parse metadata if present and fetch parameters.  Must be done
@@ -315,54 +326,164 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp)
 		printf("powerpc_init: no loader metadata.\n");
 	}
 
+	/*
+	 * Init KDB
+	 */
+
 	kdb_init();
 
 	/*
-	 * XXX: Initialize the interrupt tables.
-	 *      Disable translation in case the vector area
-	 *      hasn't been mapped (G5)
+	 * PowerPC 970 CPUs have a misfeature requested by Apple that makes
+	 * them pretend they have a 32-byte cacheline. Turn this off
+	 * before we measure the cacheline size.
 	 */
-	mtmsr(mfmsr() & ~(PSL_IR | PSL_DR));
+
+	switch (mfpvr() >> 16) {
+		case IBM970:
+		case IBM970FX:
+		case IBM970MP:
+		case IBM970GX:
+			scratch = mfspr64upper(SPR_HID5,msr);
+			scratch &= ~HID5_970_DCBZ_SIZE_HI;
+			mtspr64(SPR_HID5, scratch, mfspr(SPR_HID5), msr);
+			break;
+	}
+
+	/*
+	 * Initialize the interrupt tables and figure out our cache line
+	 * size and whether or not we need the 64-bit bridge code.
+	 */
+
+	/*
+	 * Disable translation in case the vector area hasn't been
+	 * mapped (G5).
+	 */
+
+	msr = mfmsr();
+	mtmsr(msr & ~(PSL_IR | PSL_DR));
 	isync();
+
+	/*
+	 * Measure the cacheline size using dcbz
+	 *
+	 * Use EXC_PGM as a playground. We are about to overwrite it
+	 * anyway, we know it exists, and we know it is cache-aligned.
+	 */
+
+	cache_check = (void *)EXC_PGM;
+
+	for (cacheline_size = 0; cacheline_size < 0x100; cacheline_size++)
+		cache_check[cacheline_size] = 0xff;
+
+	__asm __volatile("dcbz %0,0":: "r" (cache_check) : "memory");
+
+	/* Find the first byte dcbz did not zero to get the cache line size */
+	for (cacheline_size = 0; cacheline_size < 0x100 &&
+	    cache_check[cacheline_size] == 0; cacheline_size++);
+
+	/*
+	 * Figure out whether we need to use the 64 bit PMAP. This works by
+	 * executing an instruction that is only legal on 64-bit PPC (mtmsrd),
+	 * and setting ppc64 = 0 if that causes a trap.
+	 */
+
+	ppc64 = 1;
+
+	bcopy(&testppc64, (void *)EXC_PGM,  (size_t)&testppc64size);
+	__syncicache((void *)EXC_PGM, (size_t)&testppc64size);
+
+	__asm __volatile("\
+		mfmsr %0;	\
+		mtsprg2 %1;	\
+				\
+		mtmsrd %0;	\
+		mfsprg2 %1;"
+	    : "=r"(scratch), "=r"(ppc64));
+
+	/*
+	 * Now copy restorebridge into all the handlers, if necessary,
+	 * and set up the trap tables.
+	 */
+
+	if (ppc64) {
+		/* Patch the two instances of rfi -> rfid */
+		bcopy(&rfid_patch,&rfi_patch1,4);
+	#ifdef KDB
+		/* rfi_patch2 is at the end of dbleave */
+		bcopy(&rfid_patch,&rfi_patch2,4);
+	#endif
+
+		/*
+		 * Copy a code snippet to restore 32-bit bridge mode
+		 * to the top of every non-generic trap handler
+		 */
+
+		trap_offset += (size_t)&restorebridgesize;
+		bcopy(&restorebridge, (void *)EXC_RST, trap_offset); 
+		bcopy(&restorebridge, (void *)EXC_DSI, trap_offset); 
+		bcopy(&restorebridge, (void *)EXC_ALI, trap_offset); 
+		bcopy(&restorebridge, (void *)EXC_PGM, trap_offset); 
+		bcopy(&restorebridge, (void *)EXC_MCHK, trap_offset); 
+		bcopy(&restorebridge, (void *)EXC_TRC, trap_offset); 
+		bcopy(&restorebridge, (void *)EXC_BPT, trap_offset); 
+
+		/*
+		 * Set the common trap entry point to the one that
+		 * knows to restore 32-bit operation on execution.
+		 */
+
+		generictrap = &trapcode64;
+	} else {
+		generictrap = &trapcode;
+	}
+
 #ifdef SMP
-	bcopy(&rstcode,  (void *)EXC_RST,  (size_t)&rstsize);
+	bcopy(&rstcode, (void *)(EXC_RST + trap_offset),  (size_t)&rstsize);
 #else
-	bcopy(&trapcode, (void *)EXC_RST,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_RST,  (size_t)&trapsize);
 #endif
-	bcopy(&trapcode, (void *)EXC_MCHK, (size_t)&trapsize);
-	bcopy(&dsitrap,  (void *)EXC_DSI,  (size_t)&dsisize);
-	bcopy(&trapcode, (void *)EXC_ISI,  (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_EXI,  (size_t)&trapsize);
-	bcopy(&alitrap,  (void *)EXC_ALI,  (size_t)&alisize);
-	bcopy(&trapcode, (void *)EXC_PGM,  (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_FPU,  (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_DECR, (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_SC,   (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_TRC,  (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_FPA,  (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_VEC,  (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_VECAST, (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_THRM, (size_t)&trapsize);
-	bcopy(&trapcode, (void *)EXC_BPT,  (size_t)&trapsize);
+
 #ifdef KDB
-	bcopy(&dblow,   (void *)EXC_MCHK, (size_t)&dbsize);
-	bcopy(&dblow,   (void *)EXC_PGM,  (size_t)&dbsize);
-	bcopy(&dblow,   (void *)EXC_TRC,  (size_t)&dbsize);
-	bcopy(&dblow,   (void *)EXC_BPT,  (size_t)&dbsize);
+	bcopy(&dblow,	(void *)(EXC_MCHK + trap_offset), (size_t)&dbsize);
+	bcopy(&dblow,   (void *)(EXC_PGM + trap_offset),  (size_t)&dbsize);
+	bcopy(&dblow,   (void *)(EXC_TRC + trap_offset),  (size_t)&dbsize);
+	bcopy(&dblow,   (void *)(EXC_BPT + trap_offset),  (size_t)&dbsize);
+#else
+	bcopy(generictrap, (void *)EXC_MCHK, (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_PGM,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_TRC,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_BPT,  (size_t)&trapsize);
 #endif
+	bcopy(&dsitrap,  (void *)(EXC_DSI + trap_offset),  (size_t)&dsisize);
+	bcopy(&alitrap,  (void *)(EXC_ALI + trap_offset),  (size_t)&alisize);
+	bcopy(generictrap, (void *)EXC_ISI,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_EXI,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_FPU,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_DECR, (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_SC,   (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_FPA,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_VEC,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_VECAST, (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_THRM, (size_t)&trapsize);
 	__syncicache(EXC_RSVD, EXC_LAST - EXC_RSVD);
 
 	/*
-	 * Make sure translation has been enabled
+	 * Restore MSR
 	 */
-	mtmsr(mfmsr() | PSL_IR|PSL_DR|PSL_ME|PSL_RI);
+	mtmsr(msr);
 	isync();
 
 	/*
 	 * Initialise virtual memory.
 	 */
-	pmap_mmu_install(MMU_TYPE_OEA, 0);		/* XXX temporary */
+	if (ppc64)
+		pmap_mmu_install(MMU_TYPE_G5, 0);
+	else
+		pmap_mmu_install(MMU_TYPE_OEA, 0);
+
 	pmap_bootstrap(startkernel, endkernel);
+	mtmsr(mfmsr() | PSL_IR|PSL_DR|PSL_ME|PSL_RI);
+	isync();
 
 	/*
 	 * Initialize params/tunables that are derived from memsize
diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c
index 158b2ab..71947fa 100644
--- a/sys/powerpc/aim/mmu_oea.c
+++ b/sys/powerpc/aim/mmu_oea.c
@@ -323,6 +323,7 @@ void moea_zero_page_area(mmu_t, vm_page_t, int, int);
 void moea_zero_page_idle(mmu_t, vm_page_t);
 void moea_activate(mmu_t, struct thread *);
 void moea_deactivate(mmu_t, struct thread *);
+void moea_cpu_bootstrap(mmu_t, int);
 void moea_bootstrap(mmu_t, vm_offset_t, vm_offset_t);
 void *moea_mapdev(mmu_t, vm_offset_t, vm_size_t);
 void moea_unmapdev(mmu_t, vm_offset_t, vm_size_t);
@@ -364,6 +365,7 @@ static mmu_method_t moea_methods[] = {
 
 	/* Internal interfaces */
 	MMUMETHOD(mmu_bootstrap,       	moea_bootstrap),
+	MMUMETHOD(mmu_cpu_bootstrap,   	moea_cpu_bootstrap),
 	MMUMETHOD(mmu_mapdev,		moea_mapdev),
 	MMUMETHOD(mmu_unmapdev,		moea_unmapdev),
 	MMUMETHOD(mmu_kextract,		moea_kextract),
@@ -617,7 +619,7 @@ om_cmp(const void *a, const void *b)
 }
 
 void
-pmap_cpu_bootstrap(int ap)
+moea_cpu_bootstrap(mmu_t mmup, int ap)
 {
 	u_int sdr;
 	int i;
@@ -709,6 +711,9 @@ moea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
 	__asm __volatile("mtdbatl 1,%0" :: "r"(battable[8].batl));
 	isync();
 
+	/* set global direct map flag */
+	hw_direct_map = 1;
+
 	mem_regions(&pregions, &pregions_sz, &regions, &regions_sz);
 	CTR0(KTR_PMAP, "moea_bootstrap: physical memory");
 
@@ -895,7 +900,7 @@ moea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
 	kernel_pmap->pm_sr[KERNEL2_SR] = KERNEL2_SEGMENT;
 	kernel_pmap->pm_active = ~0;
 
-	pmap_cpu_bootstrap(0);
+	moea_cpu_bootstrap(mmup,0);
 
 	pmap_bootstrapped++;
 
diff --git a/sys/powerpc/aim/mmu_oea64.c b/sys/powerpc/aim/mmu_oea64.c
new file mode 100644
index 0000000..64e8c75
--- /dev/null
+++ b/sys/powerpc/aim/mmu_oea64.c
@@ -0,0 +1,2443 @@
+/*-
+ * Copyright (c) 2001 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Matt Thomas <matt@3am-software.com> of Allegro Networks, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *        This product includes software developed by the NetBSD
+ *        Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+/*-
+ * Copyright (C) 1995, 1996 Wolfgang Solfrank.
+ * Copyright (C) 1995, 1996 TooLs GmbH.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by TooLs GmbH.
+ * 4. The name of TooLs GmbH may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $NetBSD: pmap.c,v 1.28 2000/03/26 20:42:36 kleink Exp $
+ */
+/*-
+ * Copyright (C) 2001 Benno Rice.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * Manages physical address maps.
+ *
+ * In addition to hardware address maps, this module is called upon to
+ * provide software-use-only maps which may or may not be stored in the
+ * same form as hardware maps.  These pseudo-maps are used to store
+ * intermediate results from copy operations to and from address spaces.
+ *
+ * Since the information managed by this module is also stored by the
+ * logical address mapping module, this module may throw away valid virtual
+ * to physical mappings at almost any time.  However, invalidations of
+ * mappings must be done as requested.
+ *
+ * In order to cope with hardware architectures which make virtual to
+ * physical map invalidates expensive, this module may delay invalidate
+ * reduced protection operations until such time as they are actually
+ * necessary.  This module is given full information as to which processors
+ * are currently using which maps, and to when physical maps must be made
+ * correct.
+ */
+
+#include "opt_kstack_pages.h"
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/lock.h>
+#include <sys/msgbuf.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+#include <sys/vmmeter.h>
+
+#include <sys/kdb.h>
+
+#include <dev/ofw/openfirm.h>
+
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+#include <vm/vm_object.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_pageout.h>
+#include <vm/vm_pager.h>
+#include <vm/uma.h>
+
+#include <machine/cpu.h>
+#include <machine/powerpc.h>
+#include <machine/frame.h>
+#include <machine/md_var.h>
+#include <machine/psl.h>
+#include <machine/bat.h>
+#include <machine/pte.h>
+#include <machine/sr.h>
+#include <machine/trap.h>
+#include <machine/mmuvar.h>
+
+#include "mmu_if.h"
+
+#define	MOEA_DEBUG
+
+#define TODO	panic("%s: not implemented", __func__);
+
+static __inline u_int32_t
+cntlzw(volatile u_int32_t a) {
+	u_int32_t b;
+	__asm ("cntlzw %0, %1" : "=r"(b) : "r"(a));
+	return b;
+}
+
+static __inline uint64_t
+va_to_vsid(pmap_t pm, vm_offset_t va)
+{
+	return ((pm->pm_sr[(uintptr_t)va >> ADDR_SR_SHFT]) & SR_VSID_MASK);
+}
+
+#define	TLBSYNC()	__asm __volatile("tlbsync; ptesync");
+#define	SYNC()		__asm __volatile("sync");
+#define	EIEIO()		__asm __volatile("eieio");
+
+/*
+ * The tlbie instruction must be executed in 64-bit mode
+ * so we have to twiddle MSR[SF] around every invocation.
+ * Just to add to the fun, exceptions must be off as well
+ * so that we can't trap in 64-bit mode. What a pain.
+ */
+
+static __inline void
+TLBIE(pmap_t pmap, vm_offset_t va) {
+	register_t msr;
+	register_t scratch;
+
+	uint64_t vpn;
+	register_t vpn_hi, vpn_lo;
+
+#if 1
+	/*
+	 * CPU documentation says that tlbie takes the VPN, not the
+	 * VA. I think the code below does this correctly. We will see.
+	 */
+
+	vpn = (uint64_t)(va & ADDR_PIDX);
+	if (pmap != NULL)
+		vpn |= (va_to_vsid(pmap,va) << 28);
+#else
+	vpn = va;
+#endif
+
+	vpn_hi = (uint32_t)(vpn >> 32);
+	vpn_lo = (uint32_t)vpn;
+
+	__asm __volatile("\
+	    mfmsr %0; \
+	    clrldi %1,%0,49; \
+	    insrdi %1,1,1,0; \
+	    mtmsrd %1; \
+	    ptesync; \
+	    \
+	    sld %1,%2,%4; \
+	    or %1,%1,%3; \
+	    tlbie %1; \
+	    \
+	    mtmsrd %0; \
+	    eieio; \
+	    tlbsync; \
+	    ptesync;" 
+	: "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32));
+}
+
+#define DISABLE_TRANS(msr)	msr = mfmsr(); mtmsr(msr & ~PSL_DR); isync()
+#define ENABLE_TRANS(msr)	mtmsr(msr); isync()
+
+#define	VSID_MAKE(sr, hash)	((sr) | (((hash) & 0xfffff) << 4))
+#define	VSID_TO_SR(vsid)	((vsid) & 0xf)
+#define	VSID_TO_HASH(vsid)	(((vsid) >> 4) & 0xfffff)
+
+#define	PVO_PTEGIDX_MASK	0x007		/* which PTEG slot */
+#define	PVO_PTEGIDX_VALID	0x008		/* slot is valid */
+#define	PVO_WIRED		0x010		/* PVO entry is wired */
+#define	PVO_MANAGED		0x020		/* PVO entry is managed */
+#define	PVO_BOOTSTRAP		0x080		/* PVO entry allocated during
+						   bootstrap */
+#define PVO_FAKE		0x100		/* fictitious phys page */
+#define	PVO_VADDR(pvo)		((pvo)->pvo_vaddr & ~ADDR_POFF)
+#define PVO_ISFAKE(pvo)		((pvo)->pvo_vaddr & PVO_FAKE)
+#define	PVO_PTEGIDX_GET(pvo)	((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK)
+#define	PVO_PTEGIDX_ISSET(pvo)	((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID)
+#define	PVO_PTEGIDX_CLR(pvo)	\
+	((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK)))
+#define	PVO_PTEGIDX_SET(pvo, i)	\
+	((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID))
+
+#define	MOEA_PVO_CHECK(pvo)
+
+#define LOCK_TABLE() mtx_lock(&moea64_table_mutex)
+#define UNLOCK_TABLE() mtx_unlock(&moea64_table_mutex);
+#define ASSERT_TABLE_LOCK() mtx_assert(&moea64_table_mutex, MA_OWNED)
+
+struct ofw_map {
+	vm_offset_t	om_va;
+	vm_size_t	om_len;
+	vm_offset_t	om_pa_hi;
+	vm_offset_t	om_pa_lo;
+	u_int		om_mode;
+};
+
+/*
+ * Map of physical memory regions.
+ */
+static struct	mem_region *regions;
+static struct	mem_region *pregions;
+extern u_int	phys_avail_count;
+extern int	regions_sz, pregions_sz;
+extern int	ofw_real_mode;
+static struct	ofw_map translations[64];
+
+extern struct pmap ofw_pmap;
+
+extern void bs_remap_earlyboot(void);
+
+
+/*
+ * Lock for the pteg and pvo tables.
+ */
+struct mtx	moea64_table_mutex;
+
+/*
+ * PTEG data.
+ */
+static struct	lpteg *moea64_pteg_table;
+u_int		moea64_pteg_count;
+u_int		moea64_pteg_mask;
+
+/*
+ * PVO data.
+ */
+struct	pvo_head *moea64_pvo_table;		/* pvo entries by pteg index */
+/* lists of unmanaged pages */
+struct	pvo_head moea64_pvo_kunmanaged =
+    LIST_HEAD_INITIALIZER(moea64_pvo_kunmanaged);
+struct	pvo_head moea64_pvo_unmanaged =
+    LIST_HEAD_INITIALIZER(moea64_pvo_unmanaged);
+
+uma_zone_t	moea64_upvo_zone; /* zone for pvo entries for unmanaged pages */
+uma_zone_t	moea64_mpvo_zone; /* zone for pvo entries for managed pages */
+
+vm_offset_t	pvo_allocator_start;
+vm_offset_t	pvo_allocator_end;
+
+#define	BPVO_POOL_SIZE	327680
+static struct	pvo_entry *moea64_bpvo_pool;
+static int	moea64_bpvo_pool_index = 0;
+
+#define	VSID_NBPW	(sizeof(u_int32_t) * 8)
+static u_int	moea64_vsid_bitmap[NPMAPS / VSID_NBPW];
+
+static boolean_t moea64_initialized = FALSE;
+
+/*
+ * Statistics.
+ */
+u_int	moea64_pte_valid = 0;
+u_int	moea64_pte_overflow = 0;
+u_int	moea64_pvo_entries = 0;
+u_int	moea64_pvo_enter_calls = 0;
+u_int	moea64_pvo_remove_calls = 0;
+SYSCTL_INT(_machdep, OID_AUTO, moea64_pte_valid, CTLFLAG_RD, 
+    &moea64_pte_valid, 0, "");
+SYSCTL_INT(_machdep, OID_AUTO, moea64_pte_overflow, CTLFLAG_RD,
+    &moea64_pte_overflow, 0, "");
+SYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_entries, CTLFLAG_RD, 
+    &moea64_pvo_entries, 0, "");
+SYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_enter_calls, CTLFLAG_RD,
+    &moea64_pvo_enter_calls, 0, "");
+SYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_remove_calls, CTLFLAG_RD,
+    &moea64_pvo_remove_calls, 0, "");
+
+vm_offset_t	moea64_scratchpage_va[2];
+struct	pvo_entry *moea64_scratchpage_pvo[2];
+struct	lpte 	*moea64_scratchpage_pte[2];
+struct	mtx	moea64_scratchpage_mtx;
+
+/*
+ * Allocate physical memory for use in moea64_bootstrap.
+ */
+static vm_offset_t	moea64_bootstrap_alloc(vm_size_t, u_int);
+
+/*
+ * PTE calls.
+ */
+static int		moea64_pte_insert(u_int, struct lpte *);
+
+/*
+ * PVO calls.
+ */
+static int	moea64_pvo_enter(pmap_t, uma_zone_t, struct pvo_head *,
+		    vm_offset_t, vm_offset_t, uint64_t, int, int);
+static void	moea64_pvo_remove(struct pvo_entry *, int);
+static struct	pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t, int *);
+static struct	lpte *moea64_pvo_to_pte(const struct pvo_entry *, int);
+
+/*
+ * Utility routines.
+ */
+static void		moea64_bridge_bootstrap(mmu_t mmup, 
+			    vm_offset_t kernelstart, vm_offset_t kernelend);
+static void		moea64_bridge_cpu_bootstrap(mmu_t, int ap);
+static void		moea64_enter_locked(pmap_t, vm_offset_t, vm_page_t,
+			    vm_prot_t, boolean_t);
+static boolean_t	moea64_query_bit(vm_page_t, u_int64_t);
+static u_int		moea64_clear_bit(vm_page_t, u_int64_t, u_int64_t *);
+static void		moea64_kremove(mmu_t, vm_offset_t);
+static void		moea64_syncicache(pmap_t pmap, vm_offset_t va, 
+			    vm_offset_t pa);
+static void		tlbia(void);
+
+/*
+ * Kernel MMU interface
+ */
+void moea64_change_wiring(mmu_t, pmap_t, vm_offset_t, boolean_t);
+void moea64_clear_modify(mmu_t, vm_page_t);
+void moea64_clear_reference(mmu_t, vm_page_t);
+void moea64_copy_page(mmu_t, vm_page_t, vm_page_t);
+void moea64_enter(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t);
+void moea64_enter_object(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_page_t,
+    vm_prot_t);
+void moea64_enter_quick(mmu_t, pmap_t, vm_offset_t, vm_page_t, vm_prot_t);
+vm_paddr_t moea64_extract(mmu_t, pmap_t, vm_offset_t);
+vm_page_t moea64_extract_and_hold(mmu_t, pmap_t, vm_offset_t, vm_prot_t);
+void moea64_init(mmu_t);
+boolean_t moea64_is_modified(mmu_t, vm_page_t);
+boolean_t moea64_ts_referenced(mmu_t, vm_page_t);
+vm_offset_t moea64_map(mmu_t, vm_offset_t *, vm_offset_t, vm_offset_t, int);
+boolean_t moea64_page_exists_quick(mmu_t, pmap_t, vm_page_t);
+int moea64_page_wired_mappings(mmu_t, vm_page_t);
+void moea64_pinit(mmu_t, pmap_t);
+void moea64_pinit0(mmu_t, pmap_t);
+void moea64_protect(mmu_t, pmap_t, vm_offset_t, vm_offset_t, vm_prot_t);
+void moea64_qenter(mmu_t, vm_offset_t, vm_page_t *, int);
+void moea64_qremove(mmu_t, vm_offset_t, int);
+void moea64_release(mmu_t, pmap_t);
+void moea64_remove(mmu_t, pmap_t, vm_offset_t, vm_offset_t);
+void moea64_remove_all(mmu_t, vm_page_t);
+void moea64_remove_write(mmu_t, vm_page_t);
+void moea64_zero_page(mmu_t, vm_page_t);
+void moea64_zero_page_area(mmu_t, vm_page_t, int, int);
+void moea64_zero_page_idle(mmu_t, vm_page_t);
+void moea64_activate(mmu_t, struct thread *);
+void moea64_deactivate(mmu_t, struct thread *);
+void *moea64_mapdev(mmu_t, vm_offset_t, vm_size_t);
+void moea64_unmapdev(mmu_t, vm_offset_t, vm_size_t);
+vm_offset_t moea64_kextract(mmu_t, vm_offset_t);
+void moea64_kenter(mmu_t, vm_offset_t, vm_offset_t);
+boolean_t moea64_dev_direct_mapped(mmu_t, vm_offset_t, vm_size_t);
+boolean_t moea64_page_executable(mmu_t, vm_page_t);
+
+static mmu_method_t moea64_bridge_methods[] = {
+	MMUMETHOD(mmu_change_wiring,	moea64_change_wiring),
+	MMUMETHOD(mmu_clear_modify,	moea64_clear_modify),
+	MMUMETHOD(mmu_clear_reference,	moea64_clear_reference),
+	MMUMETHOD(mmu_copy_page,	moea64_copy_page),
+	MMUMETHOD(mmu_enter,		moea64_enter),
+	MMUMETHOD(mmu_enter_object,	moea64_enter_object),
+	MMUMETHOD(mmu_enter_quick,	moea64_enter_quick),
+	MMUMETHOD(mmu_extract,		moea64_extract),
+	MMUMETHOD(mmu_extract_and_hold,	moea64_extract_and_hold),
+	MMUMETHOD(mmu_init,		moea64_init),
+	MMUMETHOD(mmu_is_modified,	moea64_is_modified),
+	MMUMETHOD(mmu_ts_referenced,	moea64_ts_referenced),
+	MMUMETHOD(mmu_map,     		moea64_map),
+	MMUMETHOD(mmu_page_exists_quick,moea64_page_exists_quick),
+	MMUMETHOD(mmu_page_wired_mappings,moea64_page_wired_mappings),
+	MMUMETHOD(mmu_pinit,		moea64_pinit),
+	MMUMETHOD(mmu_pinit0,		moea64_pinit0),
+	MMUMETHOD(mmu_protect,		moea64_protect),
+	MMUMETHOD(mmu_qenter,		moea64_qenter),
+	MMUMETHOD(mmu_qremove,		moea64_qremove),
+	MMUMETHOD(mmu_release,		moea64_release),
+	MMUMETHOD(mmu_remove,		moea64_remove),
+	MMUMETHOD(mmu_remove_all,      	moea64_remove_all),
+	MMUMETHOD(mmu_remove_write,	moea64_remove_write),
+	MMUMETHOD(mmu_zero_page,       	moea64_zero_page),
+	MMUMETHOD(mmu_zero_page_area,	moea64_zero_page_area),
+	MMUMETHOD(mmu_zero_page_idle,	moea64_zero_page_idle),
+	MMUMETHOD(mmu_activate,		moea64_activate),
+	MMUMETHOD(mmu_deactivate,      	moea64_deactivate),
+
+	/* Internal interfaces */
+	MMUMETHOD(mmu_bootstrap,       	moea64_bridge_bootstrap),
+	MMUMETHOD(mmu_cpu_bootstrap,   	moea64_bridge_cpu_bootstrap),
+	MMUMETHOD(mmu_mapdev,		moea64_mapdev),
+	MMUMETHOD(mmu_unmapdev,		moea64_unmapdev),
+	MMUMETHOD(mmu_kextract,		moea64_kextract),
+	MMUMETHOD(mmu_kenter,		moea64_kenter),
+	MMUMETHOD(mmu_dev_direct_mapped,moea64_dev_direct_mapped),
+	MMUMETHOD(mmu_page_executable,	moea64_page_executable),
+
+	{ 0, 0 }
+};
+
+static mmu_def_t oea64_bridge_mmu = {
+	MMU_TYPE_G5,
+	moea64_bridge_methods,
+	0
+};
+MMU_DEF(oea64_bridge_mmu);
+
+static __inline u_int
+va_to_pteg(uint64_t vsid, vm_offset_t addr)
+{
+	u_int hash;
+
+	hash = vsid ^ (((uint64_t)addr & ADDR_PIDX) >>
+	    ADDR_PIDX_SHFT);
+	return (hash & moea64_pteg_mask);
+}
+
+static __inline struct pvo_head *
+pa_to_pvoh(vm_offset_t pa, vm_page_t *pg_p)
+{
+	struct	vm_page *pg;
+
+	pg = PHYS_TO_VM_PAGE(pa);
+
+	if (pg_p != NULL)
+		*pg_p = pg;
+
+	if (pg == NULL)
+		return (&moea64_pvo_unmanaged);
+
+	return (&pg->md.mdpg_pvoh);
+}
+
+static __inline struct pvo_head *
+vm_page_to_pvoh(vm_page_t m)
+{
+
+	return (&m->md.mdpg_pvoh);
+}
+
+static __inline void
+moea64_attr_clear(vm_page_t m, u_int64_t ptebit)
+{
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	m->md.mdpg_attrs &= ~ptebit;
+}
+
+static __inline u_int64_t
+moea64_attr_fetch(vm_page_t m)
+{
+
+	return (m->md.mdpg_attrs);
+}
+
+static __inline void
+moea64_attr_save(vm_page_t m, u_int64_t ptebit)
+{
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	m->md.mdpg_attrs |= ptebit;
+}
+
+static __inline int
+moea64_pte_compare(const struct lpte *pt, const struct lpte *pvo_pt)
+{
+	if (pt->pte_hi == pvo_pt->pte_hi)
+		return (1);
+
+	return (0);
+}
+
+static __inline int
+moea64_pte_match(struct lpte *pt, uint64_t vsid, vm_offset_t va, int which)
+{
+	return (pt->pte_hi & ~LPTE_VALID) ==
+	    ((vsid << LPTE_VSID_SHIFT) |
+	    ((uint64_t)(va >> ADDR_API_SHFT64) & LPTE_API) | which);
+}
+
+static __inline void
+moea64_pte_create(struct lpte *pt, uint64_t vsid, vm_offset_t va, 
+    uint64_t pte_lo)
+{
+	ASSERT_TABLE_LOCK();
+
+	/*
+	 * Construct a PTE.  Default to IMB initially.  Valid bit only gets
+	 * set when the real pte is set in memory.
+	 *
+	 * Note: Don't set the valid bit for correct operation of tlb update.
+	 */
+	pt->pte_hi = (vsid << LPTE_VSID_SHIFT) |
+	    (((uint64_t)(va & ADDR_PIDX) >> ADDR_API_SHFT64) & LPTE_API);
+
+	pt->pte_lo = pte_lo;
+}
+
+static __inline void
+moea64_pte_synch(struct lpte *pt, struct lpte *pvo_pt)
+{
+
+	ASSERT_TABLE_LOCK();
+
+	pvo_pt->pte_lo |= pt->pte_lo & (LPTE_REF | LPTE_CHG);
+}
+
+static __inline void
+moea64_pte_clear(struct lpte *pt, pmap_t pmap, vm_offset_t va, u_int64_t ptebit)
+{
+	ASSERT_TABLE_LOCK();
+
+	/*
+	 * As shown in Section 7.6.3.2.3
+	 */
+	pt->pte_lo &= ~ptebit;
+	TLBIE(pmap,va);
+}
+
+static __inline void
+moea64_pte_set(struct lpte *pt, struct lpte *pvo_pt)
+{
+
+	ASSERT_TABLE_LOCK();
+	pvo_pt->pte_hi |= LPTE_VALID;
+
+	/*
+	 * Update the PTE as defined in section 7.6.3.1.
+	 * Note that the REF/CHG bits are from pvo_pt and thus should have
+	 * been saved so this routine can restore them (if desired).
+	 */
+	pt->pte_lo = pvo_pt->pte_lo;
+	EIEIO();
+	pt->pte_hi = pvo_pt->pte_hi;
+	SYNC();
+	moea64_pte_valid++;
+}
+
+static __inline void
+moea64_pte_unset(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t va)
+{
+	ASSERT_TABLE_LOCK();
+	pvo_pt->pte_hi &= ~LPTE_VALID;
+
+	/*
+	 * Force the reg & chg bits back into the PTEs.
+	 */
+	SYNC();
+
+	/*
+	 * Invalidate the pte.
+	 */
+	pt->pte_hi &= ~LPTE_VALID;
+
+	TLBIE(pmap,va);
+
+	/*
+	 * Save the reg & chg bits.
+	 */
+	moea64_pte_synch(pt, pvo_pt);
+	moea64_pte_valid--;
+}
+
+static __inline void
+moea64_pte_change(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t va)
+{
+
+	/*
+	 * Invalidate the PTE
+	 */
+	moea64_pte_unset(pt, pvo_pt, pmap, va);
+	moea64_pte_set(pt, pvo_pt);
+}
+
+static __inline uint64_t
+moea64_calc_wimg(vm_offset_t pa)
+{
+	uint64_t pte_lo;
+	int i;
+
+	/*
+	 * Assume the page is cache inhibited and access is guarded unless
+	 * it's in our available memory array.
+	 */
+	pte_lo = LPTE_I | LPTE_G;
+	for (i = 0; i < pregions_sz; i++) {
+		if ((pa >= pregions[i].mr_start) &&
+		    (pa < (pregions[i].mr_start + pregions[i].mr_size))) {
+			pte_lo &= ~(LPTE_I | LPTE_G);
+			pte_lo |= LPTE_M;
+			break;
+		}
+	}
+
+	return pte_lo;
+}
+
+/*
+ * Quick sort callout for comparing memory regions.
+ */
+static int	mr_cmp(const void *a, const void *b);
+static int	om_cmp(const void *a, const void *b);
+
+static int
+mr_cmp(const void *a, const void *b)
+{
+	const struct	mem_region *regiona;
+	const struct	mem_region *regionb;
+
+	regiona = a;
+	regionb = b;
+	if (regiona->mr_start < regionb->mr_start)
+		return (-1);
+	else if (regiona->mr_start > regionb->mr_start)
+		return (1);
+	else
+		return (0);
+}
+
+static int
+om_cmp(const void *a, const void *b)
+{
+	const struct	ofw_map *mapa;
+	const struct	ofw_map *mapb;
+
+	mapa = a;
+	mapb = b;
+	if (mapa->om_pa_hi < mapb->om_pa_hi)
+		return (-1);
+	else if (mapa->om_pa_hi > mapb->om_pa_hi)
+		return (1);
+	else if (mapa->om_pa_lo < mapb->om_pa_lo)
+		return (-1);
+	else if (mapa->om_pa_lo > mapb->om_pa_lo)
+		return (1);
+	else
+		return (0);
+}
+
+static void
+moea64_bridge_cpu_bootstrap(mmu_t mmup, int ap)
+{
+	int i = 0;
+
+	/*
+	 * Initialize segment registers and MMU
+	 */
+
+	mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR); isync();
+	for (i = 0; i < 16; i++) {
+		mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]);
+	}
+	__asm __volatile ("sync; mtsdr1 %0; isync"
+	    :: "r"((u_int)moea64_pteg_table 
+		     | (32 - cntlzw(moea64_pteg_mask >> 11))));
+	tlbia();
+}
+
+static void
+moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
+{
+	ihandle_t	mmui;
+	phandle_t	chosen;
+	phandle_t	mmu;
+	int		sz;
+	int		i, j;
+	int		ofw_mappings;
+	vm_size_t	size, physsz, hwphyssz;
+	vm_offset_t	pa, va, off;
+	uint32_t	msr;
+
+	/* We don't have a direct map since there is no BAT */
+	hw_direct_map = 0;
+
+	/* Make sure battable is zero, since we have no BAT */
+	for (i = 0; i < 16; i++) {
+		battable[i].batu = 0;
+		battable[i].batl = 0;
+	}
+
+	/* Get physical memory regions from firmware */
+	mem_regions(&pregions, &pregions_sz, &regions, &regions_sz);
+	CTR0(KTR_PMAP, "moea64_bootstrap: physical memory");
+
+	qsort(pregions, pregions_sz, sizeof(*pregions), mr_cmp);
+	if (sizeof(phys_avail)/sizeof(phys_avail[0]) < regions_sz)
+		panic("moea64_bootstrap: phys_avail too small");
+	qsort(regions, regions_sz, sizeof(*regions), mr_cmp);
+	phys_avail_count = 0;
+	physsz = 0;
+	hwphyssz = 0;
+	TUNABLE_ULONG_FETCH("hw.physmem", (u_long *) &hwphyssz);
+	for (i = 0, j = 0; i < regions_sz; i++, j += 2) {
+		CTR3(KTR_PMAP, "region: %#x - %#x (%#x)", regions[i].mr_start,
+		    regions[i].mr_start + regions[i].mr_size,
+		    regions[i].mr_size);
+		if (hwphyssz != 0 &&
+		    (physsz + regions[i].mr_size) >= hwphyssz) {
+			if (physsz < hwphyssz) {
+				phys_avail[j] = regions[i].mr_start;
+				phys_avail[j + 1] = regions[i].mr_start +
+				    hwphyssz - physsz;
+				physsz = hwphyssz;
+				phys_avail_count++;
+			}
+			break;
+		}
+		phys_avail[j] = regions[i].mr_start;
+		phys_avail[j + 1] = regions[i].mr_start + regions[i].mr_size;
+		phys_avail_count++;
+		physsz += regions[i].mr_size;
+	}
+	physmem = btoc(physsz);
+
+	/*
+	 * Allocate PTEG table.
+	 */
+#ifdef PTEGCOUNT
+	moea64_pteg_count = PTEGCOUNT;
+#else
+	moea64_pteg_count = 0x1000;
+
+	while (moea64_pteg_count < physmem)
+		moea64_pteg_count <<= 1;
+
+	moea64_pteg_count >>= 1;
+#endif /* PTEGCOUNT */
+
+	size = moea64_pteg_count * sizeof(struct lpteg);
+	CTR2(KTR_PMAP, "moea64_bootstrap: %d PTEGs, %d bytes", 
+	    moea64_pteg_count, size);
+
+	/*
+	 * We now need to allocate memory. This memory, to be allocated,
+	 * has to reside in a page table. The page table we are about to
+	 * allocate. We don't have BAT. So drop to data real mode for a minute
+	 * as a measure of last resort. We do this a couple times.
+	 */
+
+	moea64_pteg_table = (struct lpteg *)moea64_bootstrap_alloc(size, size);
+	DISABLE_TRANS(msr);
+	bzero((void *)moea64_pteg_table, moea64_pteg_count * sizeof(struct lpteg));
+	ENABLE_TRANS(msr);
+
+	moea64_pteg_mask = moea64_pteg_count - 1;
+
+	CTR1(KTR_PMAP, "moea64_bootstrap: PTEG table at %p", moea64_pteg_table);
+
+	/*
+	 * Allocate pv/overflow lists.
+	 */
+	size = sizeof(struct pvo_head) * moea64_pteg_count;
+
+	moea64_pvo_table = (struct pvo_head *)moea64_bootstrap_alloc(size,
+	    PAGE_SIZE);
+	CTR1(KTR_PMAP, "moea64_bootstrap: PVO table at %p", moea64_pvo_table);
+
+	DISABLE_TRANS(msr);
+	for (i = 0; i < moea64_pteg_count; i++)
+		LIST_INIT(&moea64_pvo_table[i]);
+	ENABLE_TRANS(msr);
+
+	/*
+	 * Initialize the lock that synchronizes access to the pteg and pvo
+	 * tables.
+	 */
+	mtx_init(&moea64_table_mutex, "pmap table", NULL, MTX_DEF |
+	    MTX_RECURSE);
+
+	/*
+	 * Initialise the unmanaged pvo pool.
+	 */
+	moea64_bpvo_pool = (struct pvo_entry *)moea64_bootstrap_alloc(
+		BPVO_POOL_SIZE*sizeof(struct pvo_entry), 0);
+	moea64_bpvo_pool_index = 0;
+
+	/*
+	 * Make sure kernel vsid is allocated as well as VSID 0.
+	 */
+	moea64_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS - 1)) / VSID_NBPW]
+		|= 1 << (KERNEL_VSIDBITS % VSID_NBPW);
+	moea64_vsid_bitmap[0] |= 1;
+
+	/*
+	 * Initialize the kernel pmap (which is statically allocated).
+	 */
+	for (i = 0; i < 16; i++) 
+		kernel_pmap->pm_sr[i] = EMPTY_SEGMENT + i;
+
+	kernel_pmap->pmap_phys = kernel_pmap;
+	kernel_pmap->pm_active = ~0;
+
+	PMAP_LOCK_INIT(kernel_pmap);
+
+	/*
+	 * Now map in all the other buffers we allocated earlier
+	 */
+
+	DISABLE_TRANS(msr);
+	size = moea64_pteg_count * sizeof(struct lpteg);
+	off = (vm_offset_t)(moea64_pteg_table);
+	for (pa = off; pa < off + size; pa += PAGE_SIZE) 
+		moea64_kenter(mmup, pa, pa);
+	size = sizeof(struct pvo_head) * moea64_pteg_count;
+	off = (vm_offset_t)(moea64_pvo_table);
+	for (pa = off; pa < off + size; pa += PAGE_SIZE) 
+		moea64_kenter(mmup, pa, pa);
+	size = BPVO_POOL_SIZE*sizeof(struct pvo_entry);
+	off = (vm_offset_t)(moea64_bpvo_pool);
+	for (pa = off; pa < off + size; pa += PAGE_SIZE) 
+		moea64_kenter(mmup, pa, pa);
+	ENABLE_TRANS(msr);
+
+	/*
+	 * Map certain important things, like ourselves and the exception
+	 * vectors
+	 */
+
+	DISABLE_TRANS(msr);
+	for (pa = kernelstart & ~PAGE_MASK; pa < kernelend; pa += PAGE_SIZE) 
+		moea64_kenter(mmup, pa, pa);
+	for (pa = EXC_RSVD; pa < EXC_LAST; pa += PAGE_SIZE) 
+		moea64_kenter(mmup, pa, pa);
+	ENABLE_TRANS(msr);
+
+	if (!ofw_real_mode) {
+	    /*
+	     * Set up the Open Firmware pmap and add its mappings.
+	     */
+
+	    moea64_pinit(mmup, &ofw_pmap);
+	    ofw_pmap.pm_sr[KERNEL_SR] = kernel_pmap->pm_sr[KERNEL_SR];
+	    ofw_pmap.pm_sr[KERNEL2_SR] = kernel_pmap->pm_sr[KERNEL2_SR];
+
+	    if ((chosen = OF_finddevice("/chosen")) == -1)
+		panic("moea64_bootstrap: can't find /chosen");
+	    OF_getprop(chosen, "mmu", &mmui, 4);
+	    if ((mmu = OF_instance_to_package(mmui)) == -1)
+		panic("moea64_bootstrap: can't get mmu package");
+	    if ((sz = OF_getproplen(mmu, "translations")) == -1)
+		panic("moea64_bootstrap: can't get ofw translation count");
+
+	    bzero(translations, sz);
+	    if (OF_getprop(mmu, "translations", translations, sz) == -1)
+		panic("moea64_bootstrap: can't get ofw translations");
+
+	    CTR0(KTR_PMAP, "moea64_bootstrap: translations");
+	    sz /= sizeof(*translations);
+	    qsort(translations, sz, sizeof (*translations), om_cmp);
+
+	    for (i = 0, ofw_mappings = 0; i < sz; i++) {
+		CTR3(KTR_PMAP, "translation: pa=%#x va=%#x len=%#x",
+		    (uint32_t)(translations[i].om_pa_lo), translations[i].om_va,
+		    translations[i].om_len);
+
+		if (translations[i].om_pa_lo % PAGE_SIZE)
+			panic("OFW translation not page-aligned!");
+
+		if (translations[i].om_pa_hi)
+			panic("OFW translations above 32-bit boundary!");
+
+		/* Now enter the pages for this mapping */
+
+		/*
+		 * Lock the ofw pmap. pmap_kenter(), which we use for the
+		 * pages the kernel also needs, does its own locking.
+		 */
+		PMAP_LOCK(&ofw_pmap); 
+		DISABLE_TRANS(msr);
+		for (off = 0; off < translations[i].om_len; off += PAGE_SIZE) {
+			struct vm_page m;
+
+			/* Map low memory mappings into the kernel pmap, too.
+			 * These are typically mappings made by the loader,
+			 * so we need them if we want to keep executing. */
+
+			if (translations[i].om_va + off < SEGMENT_LENGTH)
+				moea64_kenter(mmup, translations[i].om_va + off,
+				    translations[i].om_va + off);
+
+			m.phys_addr = translations[i].om_pa_lo + off;
+			moea64_enter_locked(&ofw_pmap,
+			    translations[i].om_va + off, &m, VM_PROT_ALL, 1);
+
+			ofw_mappings++;
+		}
+		ENABLE_TRANS(msr);
+		PMAP_UNLOCK(&ofw_pmap);
+	    }
+	}
+
+#ifdef SMP
+	TLBSYNC();
+#endif
+
+	/*
+	 * Calculate the last available physical address.
+	 */
+	for (i = 0; phys_avail[i + 2] != 0; i += 2)
+		;
+	Maxmem = powerpc_btop(phys_avail[i + 1]);
+
+	/*
+	 * Initialize MMU and remap early physical mappings
+	 */
+	moea64_bridge_cpu_bootstrap(mmup,0);
+	mtmsr(mfmsr() | PSL_DR | PSL_IR); isync();
+	pmap_bootstrapped++;
+	bs_remap_earlyboot();
+
+	/*
+	 * Set the start and end of kva.
+	 */
+	virtual_avail = VM_MIN_KERNEL_ADDRESS;
+	virtual_end = VM_MAX_KERNEL_ADDRESS;
+
+	/*
+	 * Allocate some stupid buffer regions.
+	 */
+
+	pvo_allocator_start = virtual_avail;
+	virtual_avail += SEGMENT_LENGTH/4;
+	pvo_allocator_end = virtual_avail;
+
+	/*
+	 * Allocate some things for page zeroing
+	 */
+
+	mtx_init(&moea64_scratchpage_mtx, "pvo zero page", NULL, MTX_DEF);
+	for (i = 0; i < 2; i++) {
+		moea64_scratchpage_va[i] = virtual_avail;
+		virtual_avail += PAGE_SIZE;
+
+		moea64_kenter(mmup,moea64_scratchpage_va[i],kernelstart);
+
+		LOCK_TABLE();
+		moea64_scratchpage_pvo[i] = moea64_pvo_find_va(kernel_pmap,
+		    moea64_scratchpage_va[i],&j);
+		moea64_scratchpage_pte[i] = moea64_pvo_to_pte(
+		    moea64_scratchpage_pvo[i],j);
+		UNLOCK_TABLE();
+	}
+
+	/*
+	 * Allocate a kernel stack with a guard page for thread0 and map it
+	 * into the kernel page map.
+	 */
+	pa = moea64_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE, PAGE_SIZE);
+	va = virtual_avail + KSTACK_GUARD_PAGES * PAGE_SIZE;
+	virtual_avail = va + KSTACK_PAGES * PAGE_SIZE;
+	CTR2(KTR_PMAP, "moea_bootstrap: kstack0 at %#x (%#x)", pa, va);
+	thread0.td_kstack = va;
+	thread0.td_kstack_pages = KSTACK_PAGES;
+	for (i = 0; i < KSTACK_PAGES; i++) {
+		moea64_kenter(mmup, va, pa);;
+		pa += PAGE_SIZE;
+		va += PAGE_SIZE;
+	}
+
+	/*
+	 * Allocate virtual address space for the message buffer.
+	 */
+	pa = msgbuf_phys = moea64_bootstrap_alloc(MSGBUF_SIZE, PAGE_SIZE);
+	msgbufp = (struct msgbuf *)virtual_avail;
+	va = virtual_avail;
+	virtual_avail += round_page(MSGBUF_SIZE);
+	while (va < virtual_avail) {
+		moea64_kenter(mmup, va, pa);;
+		pa += PAGE_SIZE;
+		va += PAGE_SIZE;
+	}
+}
+
+/*
+ * Activate a user pmap.  The pmap must be activated before it's address
+ * space can be accessed in any way.
+ */
+void
+moea64_activate(mmu_t mmu, struct thread *td)
+{
+	pmap_t	pm, pmr;
+
+	/*
+	 * Load all the data we need up front to encourage the compiler to
+	 * not issue any loads while we have interrupts disabled below.
+	 */
+	pm = &td->td_proc->p_vmspace->vm_pmap;
+	pmr = pm->pmap_phys;
+
+	pm->pm_active |= PCPU_GET(cpumask);
+	PCPU_SET(curpmap, pmr);
+}
+
+void
+moea64_deactivate(mmu_t mmu, struct thread *td)
+{
+	pmap_t	pm;
+
+	pm = &td->td_proc->p_vmspace->vm_pmap;
+	pm->pm_active &= ~(PCPU_GET(cpumask));
+	PCPU_SET(curpmap, NULL);
+}
+
+void
+moea64_change_wiring(mmu_t mmu, pmap_t pm, vm_offset_t va, boolean_t wired)
+{
+	struct	pvo_entry *pvo;
+
+	PMAP_LOCK(pm);
+	pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF, NULL);
+
+	if (pvo != NULL) {
+		if (wired) {
+			if ((pvo->pvo_vaddr & PVO_WIRED) == 0)
+				pm->pm_stats.wired_count++;
+			pvo->pvo_vaddr |= PVO_WIRED;
+		} else {
+			if ((pvo->pvo_vaddr & PVO_WIRED) != 0)
+				pm->pm_stats.wired_count--;
+			pvo->pvo_vaddr &= ~PVO_WIRED;
+		}
+	}
+	PMAP_UNLOCK(pm);
+}
+
+/*
+ * Zero a page of physical memory by temporarily mapping it into the tlb.
+ */
+void
+moea64_zero_page(mmu_t mmu, vm_page_t m)
+{
+	moea64_zero_page_area(mmu,m,0,PAGE_SIZE);
+}
+
+/*
+ * This goes through and sets the physical address of our
+ * special scratch PTE to the PA we want to zero or copy. Because
+ * of locking issues (this can get called in pvo_enter() by
+ * the UMA allocator), we can't use most other utility functions here
+ */
+
+static __inline
+void moea64_set_scratchpage_pa(int which, vm_offset_t pa) {
+	moea64_scratchpage_pvo[which]->pvo_pte.lpte.pte_lo &= 
+	    (~LPTE_WIMG & ~LPTE_RPGN);
+	moea64_scratchpage_pvo[which]->pvo_pte.lpte.pte_lo |= 
+	    moea64_calc_wimg(pa) | (uint64_t)pa;
+
+	moea64_scratchpage_pte[which]->pte_hi &= ~LPTE_VALID;
+	TLBIE(kernel_pmap, moea64_scratchpage_va[which]);
+	
+	moea64_scratchpage_pte[which]->pte_lo = 
+	    moea64_scratchpage_pvo[which]->pvo_pte.lpte.pte_lo;
+	EIEIO();
+
+	moea64_scratchpage_pte[which]->pte_hi |= LPTE_VALID;
+	TLBIE(kernel_pmap, moea64_scratchpage_va[which]);
+}
+
+void
+moea64_copy_page(mmu_t mmu, vm_page_t msrc, vm_page_t mdst)
+{
+	vm_offset_t	dst;
+	vm_offset_t	src;
+
+	dst = VM_PAGE_TO_PHYS(mdst);
+	src = VM_PAGE_TO_PHYS(msrc);
+
+	mtx_lock(&moea64_scratchpage_mtx);
+
+	moea64_set_scratchpage_pa(0,src);
+	moea64_set_scratchpage_pa(1,dst);
+
+	kcopy((void *)moea64_scratchpage_va[0], 
+	    (void *)moea64_scratchpage_va[1], PAGE_SIZE);
+
+	__syncicache((void *)moea64_scratchpage_va[1],PAGE_SIZE);
+
+	mtx_unlock(&moea64_scratchpage_mtx);
+}
+
+void
+moea64_zero_page_area(mmu_t mmu, vm_page_t m, int off, int size)
+{
+	vm_offset_t pa = VM_PAGE_TO_PHYS(m);
+
+	if (!moea64_initialized)
+		panic("moea64_zero_page: can't zero pa %#x", pa);
+	if (size + off > PAGE_SIZE)
+		panic("moea64_zero_page: size + off > PAGE_SIZE");
+
+	mtx_lock(&moea64_scratchpage_mtx);
+
+	moea64_set_scratchpage_pa(0,pa);
+	bzero((caddr_t)moea64_scratchpage_va[0] + off, size);
+	__syncicache((void *)moea64_scratchpage_va[0],PAGE_SIZE);
+
+	mtx_unlock(&moea64_scratchpage_mtx);
+}
+
+void
+moea64_zero_page_idle(mmu_t mmu, vm_page_t m)
+{
+
+	moea64_zero_page(mmu, m);
+}
+
+/*
+ * Map the given physical page at the specified virtual address in the
+ * target pmap with the protection requested.  If specified the page
+ * will be wired down.
+ */
+void
+moea64_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, 
+    vm_prot_t prot, boolean_t wired)
+{
+
+	vm_page_lock_queues();
+	PMAP_LOCK(pmap);
+	moea64_enter_locked(pmap, va, m, prot, wired);
+	vm_page_unlock_queues();
+	PMAP_UNLOCK(pmap);
+}
+
+/*
+ * Map the given physical page at the specified virtual address in the
+ * target pmap with the protection requested.  If specified the page
+ * will be wired down.
+ *
+ * The page queues and pmap must be locked.
+ */
+
+static void
+moea64_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
+    boolean_t wired)
+{
+	struct		pvo_head *pvo_head;
+	uma_zone_t	zone;
+	vm_page_t	pg;
+	uint64_t	pte_lo;
+	u_int		pvo_flags;
+	int		error;
+
+	if (!moea64_initialized) {
+		pvo_head = &moea64_pvo_kunmanaged;
+		pg = NULL;
+		zone = moea64_upvo_zone;
+		pvo_flags = 0;
+	} else {
+		pvo_head = vm_page_to_pvoh(m);
+		pg = m;
+		zone = moea64_mpvo_zone;
+		pvo_flags = PVO_MANAGED;
+	}
+
+	if (pmap_bootstrapped)
+		mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+
+	/* XXX change the pvo head for fake pages */
+	if ((m->flags & PG_FICTITIOUS) == PG_FICTITIOUS) {
+		pvo_flags &= ~PVO_MANAGED;
+		pvo_head = &moea64_pvo_kunmanaged;
+		zone = moea64_upvo_zone;
+	}
+
+	pte_lo = moea64_calc_wimg(VM_PAGE_TO_PHYS(m));
+
+	if (prot & VM_PROT_WRITE) {
+		pte_lo |= LPTE_BW;
+		if (pmap_bootstrapped)
+			vm_page_flag_set(m, PG_WRITEABLE);
+	} else
+		pte_lo |= LPTE_BR;
+
+	if (prot & VM_PROT_EXECUTE)
+		pvo_flags |= VM_PROT_EXECUTE;
+
+	if (wired)
+		pvo_flags |= PVO_WIRED;
+
+	if ((m->flags & PG_FICTITIOUS) != 0)
+		pvo_flags |= PVO_FAKE;
+
+	error = moea64_pvo_enter(pmap, zone, pvo_head, va, VM_PAGE_TO_PHYS(m),
+	    pte_lo, pvo_flags, 0);
+
+	if (pmap == kernel_pmap)
+		TLBIE(pmap, va);
+
+	/*
+	 * Flush the page from the instruction cache if this page is
+	 * mapped executable and cacheable.
+	 */
+	if ((pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
+		moea64_syncicache(pmap, va, VM_PAGE_TO_PHYS(m));
+	}
+}
+
+static void
+moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_offset_t pa)
+{
+	/*
+	 * This is much trickier than on older systems because
+	 * we can't sync the icache on physical addresses directly
+	 * without a direct map. Instead we check a couple of cases
+	 * where the memory is already mapped in and, failing that,
+	 * use the same trick we use for page zeroing to create
+	 * a temporary mapping for this physical address.
+	 */
+
+	if (!pmap_bootstrapped) {
+		/*
+		 * If PMAP is not bootstrapped, we are likely to be
+		 * in real mode.
+		 */
+		__syncicache((void *)pa,PAGE_SIZE);
+	} else if (pmap == kernel_pmap) {
+		__syncicache((void *)va,PAGE_SIZE);
+	} else {
+		/* Use the scratch page to set up a temp mapping */
+
+		mtx_lock(&moea64_scratchpage_mtx);
+
+		moea64_set_scratchpage_pa(1,pa);
+		__syncicache((void *)moea64_scratchpage_va[1],PAGE_SIZE);
+
+		mtx_unlock(&moea64_scratchpage_mtx);
+	}
+}
+
+/*
+ * Maps a sequence of resident pages belonging to the same object.
+ * The sequence begins with the given page m_start.  This page is
+ * mapped at the given virtual address start.  Each subsequent page is
+ * mapped at a virtual address that is offset from start by the same
+ * amount as the page is offset from m_start within the object.  The
+ * last page in the sequence is the page with the largest offset from
+ * m_start that can be mapped at a virtual address less than the given
+ * virtual address end.  Not every virtual page between start and end
+ * is mapped; only those for which a resident page exists with the
+ * corresponding offset from m_start are mapped.
+ */
+void
+moea64_enter_object(mmu_t mmu, pmap_t pm, vm_offset_t start, vm_offset_t end,
+    vm_page_t m_start, vm_prot_t prot)
+{
+	vm_page_t m;
+	vm_pindex_t diff, psize;
+
+	psize = atop(end - start);
+	m = m_start;
+	PMAP_LOCK(pm);
+	while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
+		moea64_enter_locked(pm, start + ptoa(diff), m, prot &
+		    (VM_PROT_READ | VM_PROT_EXECUTE), FALSE);
+		m = TAILQ_NEXT(m, listq);
+	}
+	PMAP_UNLOCK(pm);
+}
+
+void
+moea64_enter_quick(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_page_t m,
+    vm_prot_t prot)
+{
+	PMAP_LOCK(pm);
+	moea64_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE),
+	    FALSE);
+	PMAP_UNLOCK(pm);
+
+}
+
+vm_paddr_t
+moea64_extract(mmu_t mmu, pmap_t pm, vm_offset_t va)
+{
+	struct	pvo_entry *pvo;
+	vm_paddr_t pa;
+
+	PMAP_LOCK(pm);
+	pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF, NULL);
+	if (pvo == NULL)
+		pa = 0;
+	else
+		pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) | (va & ADDR_POFF);
+	PMAP_UNLOCK(pm);
+	return (pa);
+}
+
+/*
+ * Atomically extract and hold the physical page with the given
+ * pmap and virtual address pair if that mapping permits the given
+ * protection.
+ */
+vm_page_t
+moea64_extract_and_hold(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_prot_t prot)
+{
+	struct	pvo_entry *pvo;
+	vm_page_t m;
+        
+	m = NULL;
+	vm_page_lock_queues();
+	PMAP_LOCK(pmap);
+	pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF, NULL);
+	if (pvo != NULL && (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) &&
+	    ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) == LPTE_RW ||
+	     (prot & VM_PROT_WRITE) == 0)) {
+		m = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
+		vm_page_hold(m);
+	}
+	vm_page_unlock_queues();
+	PMAP_UNLOCK(pmap);
+	return (m);
+}
+
+static void *
+moea64_uma_page_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 
+{
+	/*
+	 * This entire routine is a horrible hack to avoid bothering kmem
+	 * for new KVA addresses. Because this can get called from inside
+	 * kmem allocation routines, calling kmem for a new address here
+	 * can lead to multiply locking non-recursive mutexes.
+	 */
+	static vm_pindex_t color;
+        vm_offset_t va;
+
+        vm_page_t m;
+        int pflags, needed_lock;
+
+	*flags = UMA_SLAB_PRIV;
+	needed_lock = !PMAP_LOCKED(kernel_pmap);
+
+	if (needed_lock)
+		PMAP_LOCK(kernel_pmap);
+
+        if ((wait & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT)
+                pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED;
+        else
+                pflags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED;
+        if (wait & M_ZERO)
+                pflags |= VM_ALLOC_ZERO;
+
+        for (;;) {
+                m = vm_page_alloc(NULL, color++, pflags | VM_ALLOC_NOOBJ);
+                if (m == NULL) {
+                        if (wait & M_NOWAIT)
+                                return (NULL);
+                        VM_WAIT;
+                } else
+                        break;
+        }
+
+	va = pvo_allocator_start;
+	pvo_allocator_start += PAGE_SIZE;
+
+	if (pvo_allocator_start >= pvo_allocator_end)
+		panic("Ran out of PVO allocator buffer space!");
+
+	/* Now call pvo_enter in recursive mode */
+	moea64_pvo_enter(kernel_pmap, moea64_upvo_zone,
+	    &moea64_pvo_kunmanaged, va,  VM_PAGE_TO_PHYS(m), LPTE_M, 
+	    PVO_WIRED | PVO_BOOTSTRAP, 1);
+
+	TLBIE(kernel_pmap, va);
+	
+	if (needed_lock)
+		PMAP_UNLOCK(kernel_pmap);
+
+	if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0)
+                bzero((void *)va, PAGE_SIZE);
+
+	return (void *)va;
+}
+
+void
+moea64_init(mmu_t mmu)
+{
+
+	CTR0(KTR_PMAP, "moea64_init");
+
+	moea64_upvo_zone = uma_zcreate("UPVO entry", sizeof (struct pvo_entry),
+	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
+	    UMA_ZONE_VM | UMA_ZONE_NOFREE);
+	moea64_mpvo_zone = uma_zcreate("MPVO entry", sizeof(struct pvo_entry),
+	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
+	    UMA_ZONE_VM | UMA_ZONE_NOFREE);
+
+	if (!hw_direct_map) {
+		uma_zone_set_allocf(moea64_upvo_zone,moea64_uma_page_alloc);
+		uma_zone_set_allocf(moea64_mpvo_zone,moea64_uma_page_alloc);
+	}
+
+	moea64_initialized = TRUE;
+}
+
+boolean_t
+moea64_is_modified(mmu_t mmu, vm_page_t m)
+{
+
+	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
+		return (FALSE);
+
+	return (moea64_query_bit(m, LPTE_CHG));
+}
+
+void
+moea64_clear_reference(mmu_t mmu, vm_page_t m)
+{
+
+	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
+		return;
+	moea64_clear_bit(m, LPTE_REF, NULL);
+}
+
+void
+moea64_clear_modify(mmu_t mmu, vm_page_t m)
+{
+
+	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
+		return;
+	moea64_clear_bit(m, LPTE_CHG, NULL);
+}
+
+/*
+ * Clear the write and modified bits in each of the given page's mappings.
+ */
+void
+moea64_remove_write(mmu_t mmu, vm_page_t m)
+{
+	struct	pvo_entry *pvo;
+	struct	lpte *pt;
+	pmap_t	pmap;
+	uint64_t lo;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 ||
+	    (m->flags & PG_WRITEABLE) == 0)
+		return;
+	lo = moea64_attr_fetch(m);
+	SYNC();
+	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
+		pmap = pvo->pvo_pmap;
+		PMAP_LOCK(pmap);
+		if ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) {
+			LOCK_TABLE();
+			pt = moea64_pvo_to_pte(pvo, -1);
+			pvo->pvo_pte.lpte.pte_lo &= ~LPTE_PP;
+			pvo->pvo_pte.lpte.pte_lo |= LPTE_BR;
+			if (pt != NULL) {
+				moea64_pte_synch(pt, &pvo->pvo_pte.lpte);
+				lo |= pvo->pvo_pte.lpte.pte_lo;
+				pvo->pvo_pte.lpte.pte_lo &= ~LPTE_CHG;
+				moea64_pte_change(pt, &pvo->pvo_pte.lpte,
+				    pvo->pvo_pmap, pvo->pvo_vaddr);
+			}
+			UNLOCK_TABLE();
+		}
+		PMAP_UNLOCK(pmap);
+	}
+	if ((lo & LPTE_CHG) != 0) {
+		moea64_attr_clear(m, LPTE_CHG);
+		vm_page_dirty(m);
+	}
+	vm_page_flag_clear(m, PG_WRITEABLE);
+}
+
+/*
+ *	moea64_ts_referenced:
+ *
+ *	Return a count of reference bits for a page, clearing those bits.
+ *	It is not necessary for every reference bit to be cleared, but it
+ *	is necessary that 0 only be returned when there are truly no
+ *	reference bits set.
+ *
+ *	XXX: The exact number of bits to check and clear is a matter that
+ *	should be tested and standardized at some point in the future for
+ *	optimal aging of shared pages.
+ */
+boolean_t
+moea64_ts_referenced(mmu_t mmu, vm_page_t m)
+{
+	int count;
+
+	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0)
+		return (0);
+
+	count = moea64_clear_bit(m, LPTE_REF, NULL);
+
+	return (count);
+}
+
+/*
+ * Map a wired page into kernel virtual address space.
+ */
+void
+moea64_kenter(mmu_t mmu, vm_offset_t va, vm_offset_t pa)
+{
+	uint64_t	pte_lo;
+	int		error;	
+
+	if (!pmap_bootstrapped) {
+		if (va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS)
+			panic("Trying to enter an address in KVA -- %#x!\n",pa);
+	}
+
+	pte_lo = moea64_calc_wimg(pa);
+
+	PMAP_LOCK(kernel_pmap);
+	error = moea64_pvo_enter(kernel_pmap, moea64_upvo_zone,
+	    &moea64_pvo_kunmanaged, va, pa, pte_lo, 
+	    PVO_WIRED | VM_PROT_EXECUTE, 0);
+
+	TLBIE(kernel_pmap, va);
+
+	if (error != 0 && error != ENOENT)
+		panic("moea64_kenter: failed to enter va %#x pa %#x: %d", va,
+		    pa, error);
+
+	/*
+	 * Flush the memory from the instruction cache.
+	 */
+	if ((pte_lo & (LPTE_I | LPTE_G)) == 0) {
+		__syncicache((void *)va, PAGE_SIZE);
+	}
+	PMAP_UNLOCK(kernel_pmap);
+}
+
+/*
+ * Extract the physical page address associated with the given kernel virtual
+ * address.
+ */
+vm_offset_t
+moea64_kextract(mmu_t mmu, vm_offset_t va)
+{
+	struct		pvo_entry *pvo;
+	vm_paddr_t pa;
+
+	PMAP_LOCK(kernel_pmap);
+	pvo = moea64_pvo_find_va(kernel_pmap, va & ~ADDR_POFF, NULL);
+	KASSERT(pvo != NULL, ("moea64_kextract: no addr found"));
+	pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) | (va & ADDR_POFF);
+	PMAP_UNLOCK(kernel_pmap);
+	return (pa);
+}
+
+/*
+ * Remove a wired page from kernel virtual address space.
+ */
+void
+moea64_kremove(mmu_t mmu, vm_offset_t va)
+{
+	moea64_remove(mmu, kernel_pmap, va, va + PAGE_SIZE);
+}
+
+/*
+ * Map a range of physical addresses into kernel virtual address space.
+ *
+ * The value passed in *virt is a suggested virtual address for the mapping.
+ * Architectures which can support a direct-mapped physical to virtual region
+ * can return the appropriate address within that region, leaving '*virt'
+ * unchanged.  We cannot and therefore do not; *virt is updated with the
+ * first usable address after the mapped region.
+ */
+vm_offset_t
+moea64_map(mmu_t mmu, vm_offset_t *virt, vm_offset_t pa_start,
+    vm_offset_t pa_end, int prot)
+{
+	vm_offset_t	sva, va;
+
+	sva = *virt;
+	va = sva;
+	for (; pa_start < pa_end; pa_start += PAGE_SIZE, va += PAGE_SIZE)
+		moea64_kenter(mmu, va, pa_start);
+	*virt = va;
+
+	return (sva);
+}
+
+/*
+ * Returns true if the pmap's pv is one of the first
+ * 16 pvs linked to from this page.  This count may
+ * be changed upwards or downwards in the future; it
+ * is only necessary that true be returned for a small
+ * subset of pmaps for proper page aging.
+ */
+boolean_t
+moea64_page_exists_quick(mmu_t mmu, pmap_t pmap, vm_page_t m)
+{
+        int loops;
+	struct pvo_entry *pvo;
+
+        if (!moea64_initialized || (m->flags & PG_FICTITIOUS))
+                return FALSE;
+
+	loops = 0;
+	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
+		if (pvo->pvo_pmap == pmap)
+			return (TRUE);
+		if (++loops >= 16)
+			break;
+	}
+
+	return (FALSE);
+}
+
+/*
+ * Return the number of managed mappings to the given physical page
+ * that are wired.
+ */
+int
+moea64_page_wired_mappings(mmu_t mmu, vm_page_t m)
+{
+	struct pvo_entry *pvo;
+	int count;
+
+	count = 0;
+	if (!moea64_initialized || (m->flags & PG_FICTITIOUS) != 0)
+		return (count);
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink)
+		if ((pvo->pvo_vaddr & PVO_WIRED) != 0)
+			count++;
+	return (count);
+}
+
+static u_int	moea64_vsidcontext;
+
+void
+moea64_pinit(mmu_t mmu, pmap_t pmap)
+{
+	int	i, mask;
+	u_int	entropy;
+
+	PMAP_LOCK_INIT(pmap);
+
+	entropy = 0;
+	__asm __volatile("mftb %0" : "=r"(entropy));
+
+	if (pmap_bootstrapped)
+		pmap->pmap_phys = (pmap_t)moea64_kextract(mmu, (vm_offset_t)pmap);
+	else
+		pmap->pmap_phys = pmap;
+
+	/*
+	 * Allocate some segment registers for this pmap.
+	 */
+	for (i = 0; i < NPMAPS; i += VSID_NBPW) {
+		u_int	hash, n;
+
+		/*
+		 * Create a new value by mutiplying by a prime and adding in
+		 * entropy from the timebase register.  This is to make the
+		 * VSID more random so that the PT hash function collides
+		 * less often.  (Note that the prime casues gcc to do shifts
+		 * instead of a multiply.)
+		 */
+		moea64_vsidcontext = (moea64_vsidcontext * 0x1105) + entropy;
+		hash = moea64_vsidcontext & (NPMAPS - 1);
+		if (hash == 0)		/* 0 is special, avoid it */
+			continue;
+		n = hash >> 5;
+		mask = 1 << (hash & (VSID_NBPW - 1));
+		hash = (moea64_vsidcontext & 0xfffff);
+		if (moea64_vsid_bitmap[n] & mask) {	/* collision? */
+			/* anything free in this bucket? */
+			if (moea64_vsid_bitmap[n] == 0xffffffff) {
+				entropy = (moea64_vsidcontext >> 20);
+				continue;
+			}
+			i = ffs(~moea64_vsid_bitmap[i]) - 1;
+			mask = 1 << i;
+			hash &= 0xfffff & ~(VSID_NBPW - 1);
+			hash |= i;
+		}
+		moea64_vsid_bitmap[n] |= mask;
+		for (i = 0; i < 16; i++) {
+			pmap->pm_sr[i] = VSID_MAKE(i, hash);
+		}
+		return;
+	}
+
+	panic("moea64_pinit: out of segments");
+}
+
+/*
+ * Initialize the pmap associated with process 0.
+ */
+void
+moea64_pinit0(mmu_t mmu, pmap_t pm)
+{
+	moea64_pinit(mmu, pm);
+	bzero(&pm->pm_stats, sizeof(pm->pm_stats));
+}
+
+/*
+ * Set the physical protection on the specified range of this map as requested.
+ */
+void
+moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
+    vm_prot_t prot)
+{
+	struct	pvo_entry *pvo;
+	struct	lpte *pt;
+	int	pteidx;
+
+	CTR4(KTR_PMAP, "moea64_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm, sva,
+	    eva, prot);
+
+
+	KASSERT(pm == &curproc->p_vmspace->vm_pmap || pm == kernel_pmap,
+	    ("moea64_protect: non current pmap"));
+
+	if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
+		moea64_remove(mmu, pm, sva, eva);
+		return;
+	}
+
+	vm_page_lock_queues();
+	PMAP_LOCK(pm);
+	for (; sva < eva; sva += PAGE_SIZE) {
+		pvo = moea64_pvo_find_va(pm, sva, &pteidx);
+		if (pvo == NULL)
+			continue;
+
+		/*
+		 * Grab the PTE pointer before we diddle with the cached PTE
+		 * copy.
+		 */
+		LOCK_TABLE();
+		pt = moea64_pvo_to_pte(pvo, pteidx);
+
+		/*
+		 * Change the protection of the page.
+		 */
+		pvo->pvo_pte.lpte.pte_lo &= ~LPTE_PP;
+		pvo->pvo_pte.lpte.pte_lo |= LPTE_BR;
+		pvo->pvo_pte.lpte.pte_lo &= ~LPTE_NOEXEC;
+		if ((prot & VM_PROT_EXECUTE) == 0) 
+			pvo->pvo_pte.lpte.pte_lo |= LPTE_NOEXEC;
+
+		/*
+		 * If the PVO is in the page table, update that pte as well.
+		 */
+		if (pt != NULL) {
+			moea64_pte_change(pt, &pvo->pvo_pte.lpte, 
+			    pvo->pvo_pmap, pvo->pvo_vaddr);
+			if ((pvo->pvo_pte.lpte.pte_lo & 
+			    (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
+				moea64_syncicache(pm, sva, 
+				     pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
+			}
+		}
+		UNLOCK_TABLE();
+	}
+	vm_page_unlock_queues();
+	PMAP_UNLOCK(pm);
+}
+
+/*
+ * Map a list of wired pages into kernel virtual address space.  This is
+ * intended for temporary mappings which do not need page modification or
+ * references recorded.  Existing mappings in the region are overwritten.
+ */
+void
+moea64_qenter(mmu_t mmu, vm_offset_t va, vm_page_t *m, int count)
+{
+	while (count-- > 0) {
+		moea64_kenter(mmu, va, VM_PAGE_TO_PHYS(*m));
+		va += PAGE_SIZE;
+		m++;
+	}
+}
+
+/*
+ * Remove page mappings from kernel virtual address space.  Intended for
+ * temporary mappings entered by moea64_qenter.
+ */
+void
+moea64_qremove(mmu_t mmu, vm_offset_t va, int count)
+{
+	while (count-- > 0) {
+		moea64_kremove(mmu, va);
+		va += PAGE_SIZE;
+	}
+}
+
+void
+moea64_release(mmu_t mmu, pmap_t pmap)
+{
+        int idx, mask;
+        
+	/*
+	 * Free segment register's VSID
+	 */
+        if (pmap->pm_sr[0] == 0)
+                panic("moea64_release");
+
+        idx = VSID_TO_HASH(pmap->pm_sr[0]) & (NPMAPS-1);
+        mask = 1 << (idx % VSID_NBPW);
+        idx /= VSID_NBPW;
+        moea64_vsid_bitmap[idx] &= ~mask;
+	PMAP_LOCK_DESTROY(pmap);
+}
+
+/*
+ * Remove the given range of addresses from the specified map.
+ */
+void
+moea64_remove(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva)
+{
+	struct	pvo_entry *pvo;
+	int	pteidx;
+
+	vm_page_lock_queues();
+	PMAP_LOCK(pm);
+	for (; sva < eva; sva += PAGE_SIZE) {
+		pvo = moea64_pvo_find_va(pm, sva, &pteidx);
+		if (pvo != NULL) {
+			moea64_pvo_remove(pvo, pteidx);
+		}
+	}
+	vm_page_unlock_queues();
+	PMAP_UNLOCK(pm);
+}
+
+/*
+ * Remove physical page from all pmaps in which it resides. moea64_pvo_remove()
+ * will reflect changes in pte's back to the vm_page.
+ */
+void
+moea64_remove_all(mmu_t mmu, vm_page_t m)
+{
+	struct  pvo_head *pvo_head;
+	struct	pvo_entry *pvo, *next_pvo;
+	pmap_t	pmap;
+
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+
+	pvo_head = vm_page_to_pvoh(m);
+	for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) {
+		next_pvo = LIST_NEXT(pvo, pvo_vlink);
+
+		MOEA_PVO_CHECK(pvo);	/* sanity check */
+		pmap = pvo->pvo_pmap;
+		PMAP_LOCK(pmap);
+		moea64_pvo_remove(pvo, -1);
+		PMAP_UNLOCK(pmap);
+	}
+	vm_page_flag_clear(m, PG_WRITEABLE);
+}
+
+/*
+ * Allocate a physical page of memory directly from the phys_avail map.
+ * Can only be called from moea64_bootstrap before avail start and end are
+ * calculated.
+ */
+static vm_offset_t
+moea64_bootstrap_alloc(vm_size_t size, u_int align)
+{
+	vm_offset_t	s, e;
+	int		i, j;
+
+	size = round_page(size);
+	for (i = 0; phys_avail[i + 1] != 0; i += 2) {
+		if (align != 0)
+			s = (phys_avail[i] + align - 1) & ~(align - 1);
+		else
+			s = phys_avail[i];
+		e = s + size;
+
+		if (s < phys_avail[i] || e > phys_avail[i + 1])
+			continue;
+
+		if (s == phys_avail[i]) {
+			phys_avail[i] += size;
+		} else if (e == phys_avail[i + 1]) {
+			phys_avail[i + 1] -= size;
+		} else {
+			for (j = phys_avail_count * 2; j > i; j -= 2) {
+				phys_avail[j] = phys_avail[j - 2];
+				phys_avail[j + 1] = phys_avail[j - 1];
+			}
+
+			phys_avail[i + 3] = phys_avail[i + 1];
+			phys_avail[i + 1] = s;
+			phys_avail[i + 2] = e;
+			phys_avail_count++;
+		}
+
+		return (s);
+	}
+	panic("moea64_bootstrap_alloc: could not allocate memory");
+}
+
+static void
+tlbia(void)
+{
+	vm_offset_t i;
+
+	for (i = 0; i < 0xFF000; i += 0x00001000) 
+		TLBIE(NULL,i);
+}
+
+static int
+moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
+    vm_offset_t va, vm_offset_t pa, uint64_t pte_lo, int flags, int recurse)
+{
+	struct	 pvo_entry *pvo;
+	uint64_t vsid;
+	int	 first;
+	u_int	 ptegidx;
+	int	 i;
+	int      bootstrap;
+
+	/*
+	 * One nasty thing that can happen here is that the UMA calls to
+	 * allocate new PVOs need to map more memory, which calls pvo_enter(),
+	 * which calls UMA...
+	 *
+	 * We break the loop by detecting recursion and allocating out of
+	 * the bootstrap pool.
+	 */
+
+	moea64_pvo_enter_calls++;
+	first = 0;
+	bootstrap = (flags & PVO_BOOTSTRAP);
+
+	if (!moea64_initialized)
+		bootstrap = 1;
+
+	/*
+	 * Compute the PTE Group index.
+	 */
+	va &= ~ADDR_POFF;
+	vsid = va_to_vsid(pm, va);
+	ptegidx = va_to_pteg(vsid, va);
+
+	/*
+	 * Remove any existing mapping for this page.  Reuse the pvo entry if
+	 * there is a mapping.
+	 */
+	if (!recurse)
+		LOCK_TABLE();
+
+	LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) {
+		if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
+			if ((pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) == pa &&
+			    (pvo->pvo_pte.lpte.pte_lo & LPTE_PP) ==
+			    (pte_lo & LPTE_PP)) {
+				if (!recurse)
+					UNLOCK_TABLE();
+				return (0);
+			}
+			moea64_pvo_remove(pvo, -1);
+			break;
+		}
+	}
+
+	/*
+	 * If we aren't overwriting a mapping, try to allocate.
+	 */
+	if (bootstrap) {
+		if (moea64_bpvo_pool_index >= BPVO_POOL_SIZE) {
+			panic("moea64_enter: bpvo pool exhausted, %d, %d, %d",
+			      moea64_bpvo_pool_index, BPVO_POOL_SIZE, 
+			      BPVO_POOL_SIZE * sizeof(struct pvo_entry));
+		}
+		pvo = &moea64_bpvo_pool[moea64_bpvo_pool_index];
+		moea64_bpvo_pool_index++;
+		bootstrap = 1;
+	} else {
+		pvo = uma_zalloc(zone, M_NOWAIT);
+	}
+
+	if (pvo == NULL) {
+		if (!recurse)
+			UNLOCK_TABLE();
+		return (ENOMEM);
+	}
+
+	moea64_pvo_entries++;
+	pvo->pvo_vaddr = va;
+	pvo->pvo_pmap = pm;
+	LIST_INSERT_HEAD(&moea64_pvo_table[ptegidx], pvo, pvo_olink);
+	pvo->pvo_vaddr &= ~ADDR_POFF;
+
+	if (!(flags & VM_PROT_EXECUTE))
+		pte_lo |= LPTE_NOEXEC;
+	if (flags & PVO_WIRED)
+		pvo->pvo_vaddr |= PVO_WIRED;
+	if (pvo_head != &moea64_pvo_kunmanaged)
+		pvo->pvo_vaddr |= PVO_MANAGED;
+	if (bootstrap)
+		pvo->pvo_vaddr |= PVO_BOOTSTRAP;
+	if (flags & PVO_FAKE)
+		pvo->pvo_vaddr |= PVO_FAKE;
+
+	moea64_pte_create(&pvo->pvo_pte.lpte, vsid, va, 
+	    (uint64_t)(pa) | pte_lo);
+
+	/*
+	 * Remember if the list was empty and therefore will be the first
+	 * item.
+	 */
+	if (LIST_FIRST(pvo_head) == NULL)
+		first = 1;
+	LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink);
+
+	if (pvo->pvo_pte.lpte.pte_lo & PVO_WIRED)
+		pm->pm_stats.wired_count++;
+	pm->pm_stats.resident_count++;
+
+	/*
+	 * We hope this succeeds but it isn't required.
+	 */
+	i = moea64_pte_insert(ptegidx, &pvo->pvo_pte.lpte);
+	if (i >= 0) {
+		PVO_PTEGIDX_SET(pvo, i);
+	} else {
+		panic("moea64_pvo_enter: overflow");
+		moea64_pte_overflow++;
+	}
+
+	if (!recurse)
+		UNLOCK_TABLE();
+
+	return (first ? ENOENT : 0);
+}
+
+static void
+moea64_pvo_remove(struct pvo_entry *pvo, int pteidx)
+{
+	struct	lpte *pt;
+
+	/*
+	 * If there is an active pte entry, we need to deactivate it (and
+	 * save the ref & cfg bits).
+	 */
+	LOCK_TABLE();
+	pt = moea64_pvo_to_pte(pvo, pteidx);
+	if (pt != NULL) {
+		moea64_pte_unset(pt, &pvo->pvo_pte.lpte, pvo->pvo_pmap,
+		    pvo->pvo_vaddr);
+		PVO_PTEGIDX_CLR(pvo);
+	} else {
+		moea64_pte_overflow--;
+	}
+	UNLOCK_TABLE();
+
+	/*
+	 * Update our statistics.
+	 */
+	pvo->pvo_pmap->pm_stats.resident_count--;
+	if (pvo->pvo_pte.lpte.pte_lo & PVO_WIRED)
+		pvo->pvo_pmap->pm_stats.wired_count--;
+
+	/*
+	 * Save the REF/CHG bits into their cache if the page is managed.
+	 */
+	if ((pvo->pvo_vaddr & (PVO_MANAGED|PVO_FAKE)) == PVO_MANAGED) {
+		struct	vm_page *pg;
+
+		pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
+		if (pg != NULL) {
+			moea64_attr_save(pg, pvo->pvo_pte.lpte.pte_lo &
+			    (LPTE_REF | LPTE_CHG));
+		}
+	}
+
+	/*
+	 * Remove this PVO from the PV list.
+	 */
+	LIST_REMOVE(pvo, pvo_vlink);
+
+	/*
+	 * Remove this from the overflow list and return it to the pool
+	 * if we aren't going to reuse it.
+	 */
+	LIST_REMOVE(pvo, pvo_olink);
+	if (!(pvo->pvo_vaddr & PVO_BOOTSTRAP))
+		uma_zfree(pvo->pvo_vaddr & PVO_MANAGED ? moea64_mpvo_zone :
+		    moea64_upvo_zone, pvo);
+	moea64_pvo_entries--;
+	moea64_pvo_remove_calls++;
+}
+
+static __inline int
+moea64_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx)
+{
+	int	pteidx;
+
+	/*
+	 * We can find the actual pte entry without searching by grabbing
+	 * the PTEG index from 3 unused bits in pte_lo[11:9] and by
+	 * noticing the HID bit.
+	 */
+	pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo);
+	if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID)
+		pteidx ^= moea64_pteg_mask * 8;
+
+	return (pteidx);
+}
+
+static struct pvo_entry *
+moea64_pvo_find_va(pmap_t pm, vm_offset_t va, int *pteidx_p)
+{
+	struct		pvo_entry *pvo;
+	int		ptegidx;
+	uint64_t	vsid;
+
+	va &= ~ADDR_POFF;
+	vsid = va_to_vsid(pm, va);
+	ptegidx = va_to_pteg(vsid, va);
+
+	LOCK_TABLE();
+	LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) {
+		if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
+			if (pteidx_p)
+				*pteidx_p = moea64_pvo_pte_index(pvo, ptegidx);
+			break;
+		}
+	}
+	UNLOCK_TABLE();
+
+	return (pvo);
+}
+
+static struct lpte *
+moea64_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
+{
+	struct lpte *pt;
+
+	/*
+	 * If we haven't been supplied the ptegidx, calculate it.
+	 */
+	if (pteidx == -1) {
+		int		ptegidx;
+		uint64_t	vsid;
+
+		vsid = va_to_vsid(pvo->pvo_pmap, pvo->pvo_vaddr);
+		ptegidx = va_to_pteg(vsid, pvo->pvo_vaddr);
+		pteidx = moea64_pvo_pte_index(pvo, ptegidx);
+	}
+
+	pt = &moea64_pteg_table[pteidx >> 3].pt[pteidx & 7];
+
+	if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) && 
+	    !PVO_PTEGIDX_ISSET(pvo)) {
+		panic("moea64_pvo_to_pte: pvo %p has valid pte in pvo but no "
+		    "valid pte index", pvo);
+	}
+
+	if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0 && 
+	    PVO_PTEGIDX_ISSET(pvo)) {
+		panic("moea64_pvo_to_pte: pvo %p has valid pte index in pvo "
+		    "pvo but no valid pte", pvo);
+	}
+
+	if ((pt->pte_hi ^ (pvo->pvo_pte.lpte.pte_hi & ~LPTE_VALID)) == 
+	    LPTE_VALID) {
+		if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0) {
+			panic("moea64_pvo_to_pte: pvo %p has valid pte in "
+			    "moea64_pteg_table %p but invalid in pvo", pvo, pt);
+		}
+
+		if (((pt->pte_lo ^ pvo->pvo_pte.lpte.pte_lo) & 
+		    ~(LPTE_CHG|LPTE_REF)) != 0) {
+			panic("moea64_pvo_to_pte: pvo %p pte does not match "
+			    "pte %p in moea64_pteg_table difference is %#x", 
+			    pvo, pt,
+			    (uint32_t)(pt->pte_lo ^ pvo->pvo_pte.lpte.pte_lo));
+		}
+
+		ASSERT_TABLE_LOCK();
+		return (pt);
+	}
+
+	if (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) {
+		panic("moea64_pvo_to_pte: pvo %p has invalid pte %p in "
+		    "moea64_pteg_table but valid in pvo", pvo, pt);
+	}
+
+	return (NULL);
+}
+
+static int
+moea64_pte_insert(u_int ptegidx, struct lpte *pvo_pt)
+{
+	struct	lpte *pt;
+	int	i;
+
+	ASSERT_TABLE_LOCK();
+
+	/*
+	 * First try primary hash.
+	 */
+	for (pt = moea64_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
+		if ((pt->pte_hi & LPTE_VALID) == 0) {
+			pvo_pt->pte_hi &= ~LPTE_HID;
+			moea64_pte_set(pt, pvo_pt);
+			return (i);
+		}
+	}
+
+	/*
+	 * Now try secondary hash.
+	 */
+	ptegidx ^= moea64_pteg_mask;
+
+	for (pt = moea64_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
+		if ((pt->pte_hi & LPTE_VALID) == 0) {
+			pvo_pt->pte_hi |= LPTE_HID;
+			moea64_pte_set(pt, pvo_pt);
+			return (i);
+		}
+	}
+
+	panic("moea64_pte_insert: overflow");
+	return (-1);
+}
+
+static boolean_t
+moea64_query_bit(vm_page_t m, u_int64_t ptebit)
+{
+	struct	pvo_entry *pvo;
+	struct	lpte *pt;
+
+#if 0
+	if (moea64_attr_fetch(m) & ptebit)
+		return (TRUE);
+#endif
+
+	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
+		MOEA_PVO_CHECK(pvo);	/* sanity check */
+
+		/*
+		 * See if we saved the bit off.  If so, cache it and return
+		 * success.
+		 */
+		if (pvo->pvo_pte.lpte.pte_lo & ptebit) {
+			moea64_attr_save(m, ptebit);
+			MOEA_PVO_CHECK(pvo);	/* sanity check */
+			return (TRUE);
+		}
+	}
+
+	/*
+	 * No luck, now go through the hard part of looking at the PTEs
+	 * themselves.  Sync so that any pending REF/CHG bits are flushed to
+	 * the PTEs.
+	 */
+	SYNC();
+	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
+		MOEA_PVO_CHECK(pvo);	/* sanity check */
+
+		/*
+		 * See if this pvo has a valid PTE.  if so, fetch the
+		 * REF/CHG bits from the valid PTE.  If the appropriate
+		 * ptebit is set, cache it and return success.
+		 */
+		LOCK_TABLE();
+		pt = moea64_pvo_to_pte(pvo, -1);
+		if (pt != NULL) {
+			moea64_pte_synch(pt, &pvo->pvo_pte.lpte);
+			if (pvo->pvo_pte.lpte.pte_lo & ptebit) {
+				UNLOCK_TABLE();
+
+				moea64_attr_save(m, ptebit);
+				MOEA_PVO_CHECK(pvo);	/* sanity check */
+				return (TRUE);
+			}
+		}
+		UNLOCK_TABLE();
+	}
+
+	return (FALSE);
+}
+
+static u_int
+moea64_clear_bit(vm_page_t m, u_int64_t ptebit, u_int64_t *origbit)
+{
+	u_int	count;
+	struct	pvo_entry *pvo;
+	struct	lpte *pt;
+	uint64_t rv;
+
+	/*
+	 * Clear the cached value.
+	 */
+	rv = moea64_attr_fetch(m);
+	moea64_attr_clear(m, ptebit);
+
+	/*
+	 * Sync so that any pending REF/CHG bits are flushed to the PTEs (so
+	 * we can reset the right ones).  note that since the pvo entries and
+	 * list heads are accessed via BAT0 and are never placed in the page
+	 * table, we don't have to worry about further accesses setting the
+	 * REF/CHG bits.
+	 */
+	SYNC();
+
+	/*
+	 * For each pvo entry, clear the pvo's ptebit.  If this pvo has a
+	 * valid pte clear the ptebit from the valid pte.
+	 */
+	count = 0;
+	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
+		MOEA_PVO_CHECK(pvo);	/* sanity check */
+
+		LOCK_TABLE();
+		pt = moea64_pvo_to_pte(pvo, -1);
+		if (pt != NULL) {
+			moea64_pte_synch(pt, &pvo->pvo_pte.lpte);
+			if (pvo->pvo_pte.lpte.pte_lo & ptebit) {
+				count++;
+				moea64_pte_clear(pt, pvo->pvo_pmap, PVO_VADDR(pvo), ptebit);
+			}
+		}
+		UNLOCK_TABLE();
+		rv |= pvo->pvo_pte.lpte.pte_lo;
+		pvo->pvo_pte.lpte.pte_lo &= ~ptebit;
+		MOEA_PVO_CHECK(pvo);	/* sanity check */
+	}
+
+	if (origbit != NULL) {
+		*origbit = rv;
+	}
+
+	return (count);
+}
+
+boolean_t
+moea64_dev_direct_mapped(mmu_t mmu, vm_offset_t pa, vm_size_t size)
+{
+	return (EFAULT);
+}
+
+boolean_t
+moea64_page_executable(mmu_t mmu, vm_page_t pg)
+{
+	return (!moea64_query_bit(pg, LPTE_NOEXEC));
+}
+
+/*
+ * Map a set of physical memory pages into the kernel virtual
+ * address space. Return a pointer to where it is mapped. This
+ * routine is intended to be used for mapping device memory,
+ * NOT real memory.
+ */
+void *
+moea64_mapdev(mmu_t mmu, vm_offset_t pa, vm_size_t size)
+{
+	vm_offset_t va, tmpva, ppa, offset;
+
+	ppa = trunc_page(pa);
+	offset = pa & PAGE_MASK;
+	size = roundup(offset + size, PAGE_SIZE);
+
+	va = kmem_alloc_nofault(kernel_map, size);
+
+	if (!va)
+		panic("moea64_mapdev: Couldn't alloc kernel virtual memory");
+
+	for (tmpva = va; size > 0;) {
+		moea64_kenter(mmu, tmpva, ppa);
+		size -= PAGE_SIZE;
+		tmpva += PAGE_SIZE;
+		ppa += PAGE_SIZE;
+	}
+
+	return ((void *)(va + offset));
+}
+
+void
+moea64_unmapdev(mmu_t mmu, vm_offset_t va, vm_size_t size)
+{
+	vm_offset_t base, offset;
+
+	base = trunc_page(va);
+	offset = va & PAGE_MASK;
+	size = roundup(offset + size, PAGE_SIZE);
+
+	kmem_free(kernel_map, base, size);
+}
+
diff --git a/sys/powerpc/aim/mp_cpudep.c b/sys/powerpc/aim/mp_cpudep.c
index cfdb53d..e226d3b 100644
--- a/sys/powerpc/aim/mp_cpudep.c
+++ b/sys/powerpc/aim/mp_cpudep.c
@@ -35,7 +35,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/smp.h>
 
-#include <machine/bat.h>
 #include <machine/bus.h>
 #include <machine/cpu.h>
 #include <machine/hid.h>
@@ -250,8 +249,10 @@ cpudep_ap_bootstrap(void)
 	mtmsr(msr);
 	isync();
 
-	reg = l3_enable();
-	reg = l2_enable();
+	if (l3cr_config != 0)
+		reg = l3_enable();
+	if (l2cr_config != 0)
+		reg = l2_enable();
 	reg = l1d_enable();
 	reg = l1i_enable();
 
diff --git a/sys/powerpc/aim/ofw_machdep.c b/sys/powerpc/aim/ofw_machdep.c
index 0d5f03b..977f81a 100644
--- a/sys/powerpc/aim/ofw_machdep.c
+++ b/sys/powerpc/aim/ofw_machdep.c
@@ -62,6 +62,12 @@ __FBSDID("$FreeBSD$");
 static struct mem_region OFmem[OFMEM_REGIONS + 1], OFavail[OFMEM_REGIONS + 3];
 static struct mem_region OFfree[OFMEM_REGIONS + 3];
 
+struct mem_region64 {
+        vm_offset_t     mr_start_hi;
+        vm_offset_t     mr_start_lo;
+        vm_size_t       mr_size;
+};	
+
 extern register_t ofmsr[5];
 extern struct	pmap ofw_pmap;
 static int	(*ofwcall)(void *);
@@ -141,24 +147,86 @@ void
 mem_regions(struct mem_region **memp, int *memsz,
 		struct mem_region **availp, int *availsz)
 {
-	int phandle;
+	phandle_t phandle;
 	int asz, msz, fsz;
 	int i, j;
 	int still_merging;
+	cell_t address_cells;
+
+	asz = msz = 0;
+
+	/*
+	 * Get #address-cells from root node, defaulting to 1 if it cannot
+	 * be found.
+	 */
+	phandle = OF_finddevice("/");
+	if (OF_getprop(phandle, "#address-cells", &address_cells, 
+	    sizeof(address_cells)) < sizeof(address_cells))
+		address_cells = 1;
 	
 	/*
 	 * Get memory.
 	 */
 	if ((phandle = OF_finddevice("/memory")) == -1
-	    || (msz = OF_getprop(phandle, "reg",
-			  OFmem, sizeof OFmem[0] * OFMEM_REGIONS))
-	       <= 0
 	    || (asz = OF_getprop(phandle, "available",
-			  OFavail, sizeof OFavail[0] * OFMEM_REGIONS))
-	       <= 0)
-		panic("no memory?");
+		  OFavail, sizeof OFavail[0] * OFMEM_REGIONS)) <= 0)
+	{
+		if (ofw_real_mode) {
+			/* XXX MAMBO */
+			printf("Physical memory unknown -- guessing 128 MB\n");
+
+			/* Leave the first 0xA000000 bytes for the kernel */
+			OFavail[0].mr_start = 0xA00000;
+			OFavail[0].mr_size = 0x75FFFFF;
+			asz = sizeof(OFavail[0]);
+		} else {
+			panic("no memory?");
+		}
+	}
+
+	if (address_cells == 2) {
+	    struct mem_region64 OFmem64[OFMEM_REGIONS + 1];
+	    if ((phandle == -1) || (msz = OF_getprop(phandle, "reg",
+			  OFmem64, sizeof OFmem64[0] * OFMEM_REGIONS)) <= 0) {
+		if (ofw_real_mode) {
+			/* XXX MAMBO */
+			OFmem64[0].mr_start_hi = 0;
+			OFmem64[0].mr_start_lo = 0x0;
+			OFmem64[0].mr_size = 0x7FFFFFF;
+			msz = sizeof(OFmem64[0]);
+		} else {
+			panic("Physical memory map not found");
+		}
+	    }
+
+	    for (i = 0, j = 0; i < msz/sizeof(OFmem64[0]); i++) {
+		if (OFmem64[i].mr_start_hi == 0) {
+			OFmem[i].mr_start = OFmem64[i].mr_start_lo;
+			OFmem[i].mr_size = OFmem64[i].mr_size;
+
+			/*
+			 * Check for memory regions extending above 32-bit
+			 * memory space, and restrict them to stay there.
+			 */
+			if (((uint64_t)OFmem[i].mr_start +
+			    (uint64_t)OFmem[i].mr_size) >
+			    BUS_SPACE_MAXADDR_32BIT) {
+				OFmem[i].mr_size = BUS_SPACE_MAXADDR_32BIT -
+				    OFmem[i].mr_start;
+			}
+			j++;
+		}
+	    }
+	    msz = j*sizeof(OFmem[0]);
+	} else {
+	    if ((msz = OF_getprop(phandle, "reg",
+			  OFmem, sizeof OFmem[0] * OFMEM_REGIONS)) <= 0)
+		panic("Physical memory map not found");
+	}
+
 	*memp = OFmem;
 	*memsz = msz / sizeof(struct mem_region);
+	
 
 	/*
 	 * OFavail may have overlapping regions - collapse these
@@ -268,8 +336,10 @@ openfirmware(void *args)
 		/*
 		 * Clear battable[] translations
 		 */
-		__asm __volatile("mtdbatu 2, %0\n"
-				 "mtdbatu 3, %0" : : "r" (0));
+		if (!ppc64) {
+			__asm __volatile("mtdbatu 2, %0\n"
+					 "mtdbatu 3, %0" : : "r" (0));
+		}
 		isync();
 	}
 
@@ -469,3 +539,4 @@ mem_valid(vm_offset_t addr, int len)
 
 	return (EFAULT);
 }
+
diff --git a/sys/powerpc/aim/swtch.S b/sys/powerpc/aim/swtch.S
index 672fe6b..619bf89 100644
--- a/sys/powerpc/aim/swtch.S
+++ b/sys/powerpc/aim/swtch.S
@@ -155,6 +155,12 @@ cpu_switchin:
 	mtsr	USER_SR,%r5
 	isync
 	lwz	%r1,PCB_SP(%r3)		/* Load the stack pointer */
+	/*
+	 * Perform a dummy stwcx. to clear any reservations we may have
+	 * inherited from the previous thread. It doesn't matter if the
+	 * stwcx succeeds or not. pcb_context[0] can be clobbered.
+	 */
+	stwcx.	%r1, 0, %r3
 	blr
 
 /*
diff --git a/sys/powerpc/aim/trap_subr.S b/sys/powerpc/aim/trap_subr.S
index 58e0663..0c62d83 100644
--- a/sys/powerpc/aim/trap_subr.S
+++ b/sys/powerpc/aim/trap_subr.S
@@ -223,10 +223,54 @@
 	lwz	%r3,(savearea+CPUSAVE_SRR0)(%r2); /* restore srr0 */	\
 	mtsrr0	%r3;							\
 	lwz	%r3,(savearea+CPUSAVE_SRR1)(%r2); /* restore srr1 */	\
+									\
+	/* Make sure HV bit of MSR propagated to SRR1 */		\
+	mfmsr	%r2;							\
+	or	%r3,%r2,%r3;						\
+									\
 	mtsrr1	%r3;							\
 	mfsprg2	%r2;			/* restore r2 & r3 */		\
 	mfsprg3	%r3
 
+/*
+ * The next two routines are 64-bit glue code. The first is used to test if
+ * we are on a 64-bit system. By copying it to the illegal instruction
+ * handler, we can test for 64-bit mode by trying to execute a 64-bit
+ * instruction and seeing what happens. The second gets copied in front
+ * of all the other handlers to restore 32-bit bridge mode when traps
+ * are taken.
+ */
+
+/* 64-bit test code. Sets SPRG2 to 0 if an illegal instruction is executed */
+
+	.globl	CNAME(testppc64),CNAME(testppc64size)
+CNAME(testppc64):
+	mtsprg1 %r31
+	mfsrr0  %r31
+	addi	%r31, %r31, 4
+	mtsrr0  %r31
+
+	li	%r31, 0
+	mtsprg2 %r31
+	mfsprg1 %r31
+
+	rfi
+CNAME(testppc64size) = .-CNAME(testppc64)
+
+
+/* 64-bit bridge mode restore snippet. Gets copied in front of everything else
+ * on 64-bit systems. */
+
+	.globl	CNAME(restorebridge),CNAME(restorebridgesize)
+CNAME(restorebridge):
+	mtsprg1	%r31
+	mfmsr	%r31
+	clrldi	%r31,%r31,1
+	mtmsrd	%r31
+	mfsprg1	%r31
+	isync
+CNAME(restorebridgesize) = .-CNAME(restorebridge)
+
 #ifdef SMP
 /*
  * Processor reset exception handler. These are typically
@@ -270,6 +314,17 @@ CNAME(trapcode):
 CNAME(trapsize) = .-CNAME(trapcode)
 
 /*
+ * 64-bit version of trapcode. Identical, except it calls generictrap64.
+ */
+	.globl	CNAME(trapcode64)
+CNAME(trapcode64):
+	mtsprg1	%r1			/* save SP */
+	mflr	%r1			/* Save the old LR in r1 */
+	mtsprg2 %r1			/* And then in SPRG2 */
+	li	%r1, 0x20		/* How to get the vector from LR */
+	bla	generictrap64		/* LR & SPRG3 is exception # */
+
+/*
  * For ALI: has to save DSISR and DAR
  */
 	.globl	CNAME(alitrap),CNAME(alisize)
@@ -405,9 +460,7 @@ disitrap:
 	stw	%r30,(PC_DBSAVE  +CPUSAVE_R30)(%r1) /* save r30 */
 	lwz	%r31,(PC_DISISAVE+CPUSAVE_R31)(%r1) /* get  r31 */
 	stw	%r31,(PC_DBSAVE  +CPUSAVE_R31)(%r1) /* save r31 */
-	lis	%r1,(tmpstk+TMPSTKSZ-16)@ha	/* get new SP */
-	addi	%r1,%r1,(tmpstk+TMPSTKSZ-16)@l
-	b	dbtrap
+	bla	dbtrap
 #endif
 
 	/* XXX need stack probe here */
@@ -433,6 +486,14 @@ realtrap:
  * SPRG2 - Original LR
  */
 
+generictrap64:
+	mtsprg3	%r31
+	mfmsr	%r31
+	clrldi	%r31,%r31,1
+	mtmsrd	%r31
+	mfsprg3	%r31
+	isync
+
 generictrap:
 	/* Save R1 for computing the exception vector */
 	mtsprg3 %r1
@@ -504,8 +565,15 @@ CNAME(asttrapexit):
 	b	trapexit		/* test ast ret value ? */
 1:
 	FRAME_LEAVE(PC_TEMPSAVE)
+
+	.globl	CNAME(rfi_patch1)	/* replace rfi with rfid on ppc64 */
+CNAME(rfi_patch1):
 	rfi
 
+	.globl	CNAME(rfid_patch)
+CNAME(rfid_patch):
+	rfid
+
 #if defined(KDB)
 /*
  * Deliberate entry to dbtrap
@@ -561,11 +629,13 @@ dbtrap:
 	mflr	%r28
 	mfcr	%r29
 	lwz	%r31,(PC_DBSAVE+CPUSAVE_R31)(%r1)
-	mtlr	%r31
+	mtsprg3	%r31			/* SPRG3 was clobbered by FRAME_LEAVE */
 	mfsprg1	%r1
 	b	realtrap
 dbleave:
 	FRAME_LEAVE(PC_DBSAVE)
+	.globl	CNAME(rfi_patch2)	/* replace rfi with rfid on ppc64 */
+CNAME(rfi_patch2):
 	rfi
 
 /*
diff --git a/sys/powerpc/aim/uio_machdep.c b/sys/powerpc/aim/uio_machdep.c
deleted file mode 100644
index 070ad98..0000000
--- a/sys/powerpc/aim/uio_machdep.c
+++ /dev/null
@@ -1,124 +0,0 @@
-/*-
- * Copyright (c) 2004 Alan L. Cox <alc@cs.rice.edu>
- * Copyright (c) 1982, 1986, 1991, 1993
- *	The Regents of the University of California.  All rights reserved.
- * (c) UNIX System Laboratories, Inc.
- * All or some portions of this file are derived from material licensed
- * to the University of California by American Telephone and Telegraph
- * Co. or Unix System Laboratories, Inc. and are reproduced herein with
- * the permission of UNIX System Laboratories, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)kern_subr.c	8.3 (Berkeley) 1/21/94
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/kernel.h>
-#include <sys/lock.h>
-#include <sys/mutex.h>
-#include <sys/proc.h>
-#include <sys/systm.h>
-#include <sys/uio.h>
-
-#include <vm/vm.h>
-#include <vm/vm_page.h>
-
-#include <machine/md_var.h>
-#include <machine/vmparam.h>
-
-/*
- * Implement uiomove(9) from physical memory using the direct map to
- * avoid the creation and destruction of ephemeral mappings.
- */
-int
-uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
-{
-	struct thread *td = curthread;
-	struct iovec *iov;
-	void *cp;
-	vm_offset_t page_offset;
-	size_t cnt;
-	int error = 0;
-	int save = 0;
-
-	KASSERT(uio->uio_rw == UIO_READ || uio->uio_rw == UIO_WRITE,
-	    ("uiomove_fromphys: mode"));
-	KASSERT(uio->uio_segflg != UIO_USERSPACE || uio->uio_td == curthread,
-	    ("uiomove_fromphys proc"));
-	save = td->td_pflags & TDP_DEADLKTREAT;
-	td->td_pflags |= TDP_DEADLKTREAT;
-	while (n > 0 && uio->uio_resid) {
-		iov = uio->uio_iov;
-		cnt = iov->iov_len;
-		if (cnt == 0) {
-			uio->uio_iov++;
-			uio->uio_iovcnt--;
-			continue;
-		}
-		if (cnt > n)
-			cnt = n;
-		page_offset = offset & PAGE_MASK;
-		cnt = min(cnt, PAGE_SIZE - page_offset);
-		cp = (char *)VM_PAGE_TO_PHYS(ma[offset >> PAGE_SHIFT]) +
-		    page_offset;
-		switch (uio->uio_segflg) {
-		case UIO_USERSPACE:
-			if (ticks - PCPU_GET(switchticks) >= hogticks)
-				uio_yield();
-			if (uio->uio_rw == UIO_READ)
-				error = copyout(cp, iov->iov_base, cnt);
-			else 
-				error = copyin(iov->iov_base, cp, cnt);
-			if (error)
-				goto out;
-			if (uio->uio_rw == UIO_WRITE &&
-			    pmap_page_executable(ma[offset >> PAGE_SHIFT]))
-				__syncicache(cp, cnt);
-			break;
-		case UIO_SYSSPACE:
-			if (uio->uio_rw == UIO_READ)
-				bcopy(cp, iov->iov_base, cnt);
-			else
-				bcopy(iov->iov_base, cp, cnt);
-			break;
-		case UIO_NOCOPY:
-			break;
-		}
-		iov->iov_base = (char *)iov->iov_base + cnt;
-		iov->iov_len -= cnt;
-		uio->uio_resid -= cnt;
-		uio->uio_offset += cnt;
-		offset += cnt;
-		n -= cnt;
-	}
-out:
-	if (save == 0)
-		td->td_pflags &= ~TDP_DEADLKTREAT;
-	return (error);
-}
diff --git a/sys/powerpc/aim/uma_machdep.c b/sys/powerpc/aim/uma_machdep.c
index 89d092a..dc03a26 100644
--- a/sys/powerpc/aim/uma_machdep.c
+++ b/sys/powerpc/aim/uma_machdep.c
@@ -35,9 +35,13 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <vm/vm.h>
 #include <vm/vm_page.h>
+#include <vm/vm_kern.h>
 #include <vm/vm_pageout.h>
+#include <vm/vm_extern.h>
+#include <vm/uma.h>
 #include <vm/uma.h>
 #include <vm/uma_int.h>
+#include <machine/md_var.h>
 #include <machine/vmparam.h>
 
 static int hw_uma_mdpages;
@@ -51,6 +55,13 @@ uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
 	void *va;
 	vm_page_t m;
 	int pflags;
+	
+	if (!hw_direct_map) {
+		*flags = UMA_SLAB_KMEM;
+		va = (void *)kmem_malloc(kmem_map, bytes, wait);
+	
+		return va;
+	}
 
 	*flags = UMA_SLAB_PRIV;
 	if ((wait & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT)
@@ -83,6 +94,12 @@ uma_small_free(void *mem, int size, u_int8_t flags)
 {
 	vm_page_t m;
 
+	if (!hw_direct_map) {
+		kmem_free(kmem_map, (vm_offset_t)mem, size);
+
+		return;
+	}
+
 	m = PHYS_TO_VM_PAGE((u_int32_t)mem);
 	m->wire_count--;
 	vm_page_free(m);
diff --git a/sys/powerpc/aim/vm_machdep.c b/sys/powerpc/aim/vm_machdep.c
index 9b0faf3..e20ba7b 100644
--- a/sys/powerpc/aim/vm_machdep.c
+++ b/sys/powerpc/aim/vm_machdep.c
@@ -101,6 +101,37 @@
 #include <vm/vm_extern.h>
 
 /*
+ * On systems without a direct mapped region (e.g. PPC64),
+ * we use the same code as the Book E implementation. Since
+ * we need to have runtime detection of this, define some machinery
+ * for sf_bufs in this case, and ignore it on systems with direct maps.
+ */
+
+#ifndef NSFBUFS
+#define NSFBUFS         (512 + maxusers * 16)
+#endif
+
+static void sf_buf_init(void *arg);
+SYSINIT(sock_sf, SI_SUB_MBUF, SI_ORDER_ANY, sf_buf_init, NULL);
+ 
+LIST_HEAD(sf_head, sf_buf);
+ 
+/* A hash table of active sendfile(2) buffers */
+static struct sf_head *sf_buf_active;
+static u_long sf_buf_hashmask;
+
+#define SF_BUF_HASH(m)  (((m) - vm_page_array) & sf_buf_hashmask)
+
+static TAILQ_HEAD(, sf_buf) sf_buf_freelist;
+static u_int sf_buf_alloc_want;
+
+/*
+ * A lock used to synchronize access to the hash table and free list
+ */
+static struct mtx sf_buf_lock;
+
+
+/*
  * Finish a fork operation, with process p2 nearly set up.
  * Copy and update the pcb, set up the stack so that the child
  * ready to run and return to user mode.
@@ -202,24 +233,122 @@ cpu_reset()
 }
 
 /*
- * Allocate an sf_buf for the given vm_page.  On this machine, however, there
- * is no sf_buf object.  Instead, an opaque pointer to the given vm_page is
- * returned.
+ * Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-))
  */
-struct sf_buf *
-sf_buf_alloc(struct vm_page *m, int pri)
+static void
+sf_buf_init(void *arg)
 {
+        struct sf_buf *sf_bufs;
+        vm_offset_t sf_base;
+        int i;
+
+	/* Don't bother on systems with a direct map */
 
-	return ((struct sf_buf *)m);
+	if (hw_direct_map)
+		return;
+
+        nsfbufs = NSFBUFS;
+        TUNABLE_INT_FETCH("kern.ipc.nsfbufs", &nsfbufs);
+
+        sf_buf_active = hashinit(nsfbufs, M_TEMP, &sf_buf_hashmask);
+        TAILQ_INIT(&sf_buf_freelist);
+        sf_base = kmem_alloc_nofault(kernel_map, nsfbufs * PAGE_SIZE);
+        sf_bufs = malloc(nsfbufs * sizeof(struct sf_buf), M_TEMP, M_NOWAIT | M_ZERO);
+
+        for (i = 0; i < nsfbufs; i++) {
+                sf_bufs[i].kva = sf_base + i * PAGE_SIZE;
+                TAILQ_INSERT_TAIL(&sf_buf_freelist, &sf_bufs[i], free_entry);
+        }
+        sf_buf_alloc_want = 0;
+        mtx_init(&sf_buf_lock, "sf_buf", NULL, MTX_DEF);
+}
+
+/*
+ * Get an sf_buf from the freelist. Will block if none are available.
+ */
+struct sf_buf *
+sf_buf_alloc(struct vm_page *m, int flags)
+{
+        struct sf_head *hash_list;
+        struct sf_buf *sf;
+        int error;
+
+	if (hw_direct_map) {
+		/* Shortcut the direct mapped case */
+
+		return ((struct sf_buf *)m);
+	}
+
+        hash_list = &sf_buf_active[SF_BUF_HASH(m)];
+        mtx_lock(&sf_buf_lock);
+        LIST_FOREACH(sf, hash_list, list_entry) {
+                if (sf->m == m) {
+                        sf->ref_count++;
+                        if (sf->ref_count == 1) {
+                                TAILQ_REMOVE(&sf_buf_freelist, sf, free_entry);
+                                nsfbufsused++;
+                                nsfbufspeak = imax(nsfbufspeak, nsfbufsused);
+                        }
+                        goto done;
+                }
+        }
+
+        while ((sf = TAILQ_FIRST(&sf_buf_freelist)) == NULL) {
+                if (flags & SFB_NOWAIT)
+                        goto done;
+
+                sf_buf_alloc_want++;
+                mbstat.sf_allocwait++;
+                error = msleep(&sf_buf_freelist, &sf_buf_lock,
+                    (flags & SFB_CATCH) ? PCATCH | PVM : PVM, "sfbufa", 0);
+                sf_buf_alloc_want--;
+
+                /*
+                 * If we got a signal, don't risk going back to sleep.
+                 */
+                if (error)
+                        goto done;
+        }
+
+        TAILQ_REMOVE(&sf_buf_freelist, sf, free_entry);
+        if (sf->m != NULL)
+                LIST_REMOVE(sf, list_entry);
+
+        LIST_INSERT_HEAD(hash_list, sf, list_entry);
+        sf->ref_count = 1;
+        sf->m = m;
+        nsfbufsused++;
+        nsfbufspeak = imax(nsfbufspeak, nsfbufsused);
+        pmap_qenter(sf->kva, &sf->m, 1);
+done:
+        mtx_unlock(&sf_buf_lock);
+        return (sf);
 }
 
 /*
- * Free the sf_buf.  In fact, do nothing because there are no resources
- * associated with the sf_buf.
+ * Detatch mapped page and release resources back to the system.
+ *
+ * Remove a reference from the given sf_buf, adding it to the free
+ * list when its reference count reaches zero. A freed sf_buf still,
+ * however, retains its virtual-to-physical mapping until it is
+ * recycled or reactivated by sf_buf_alloc(9).
  */
 void
 sf_buf_free(struct sf_buf *sf)
 {
+	if (hw_direct_map)
+		return;
+
+        mtx_lock(&sf_buf_lock);
+        sf->ref_count--;
+        if (sf->ref_count == 0) {
+                TAILQ_INSERT_TAIL(&sf_buf_freelist, sf, free_entry);
+                nsfbufsused--;
+
+                if (sf_buf_alloc_want > 0)
+                        wakeup_one(&sf_buf_freelist);
+        }
+        mtx_unlock(&sf_buf_lock);
 }
 
 /*
diff --git a/sys/powerpc/booke/machdep.c b/sys/powerpc/booke/machdep.c
index e42aa8c..bd72603 100644
--- a/sys/powerpc/booke/machdep.c
+++ b/sys/powerpc/booke/machdep.c
@@ -176,6 +176,9 @@ int cacheline_size = 32;
 SYSCTL_INT(_machdep, CPU_CACHELINE, cacheline_size,
 	   CTLFLAG_RD, &cacheline_size, 0, "");
 
+int hw_direct_map = 0;
+int ppc64 = 0;
+
 static void cpu_e500_startup(void *);
 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_e500_startup, NULL);
 
diff --git a/sys/powerpc/booke/pmap.c b/sys/powerpc/booke/pmap.c
index 136279d..223c42c 100644
--- a/sys/powerpc/booke/pmap.c
+++ b/sys/powerpc/booke/pmap.c
@@ -39,7 +39,7 @@
   * 0x0000_0000 - 0xafff_ffff	: user process
   * 0xb000_0000 - 0xbfff_ffff	: pmap_mapdev()-ed area (PCI/PCIE etc.)
   * 0xc000_0000 - 0xc0ff_ffff	: kernel reserved
-  *   0xc000_0000 - kernelend	: kernel code+data, env, metadata etc.
+  *   0xc000_0000 - data_end	: kernel code+data, env, metadata etc.
   * 0xc100_0000 - 0xfeef_ffff	: KVA
   *   0xc100_0000 - 0xc100_3fff : reserved for page zero/copy
   *   0xc100_4000 - 0xc200_3fff : reserved for ptbl bufs
@@ -76,6 +76,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_pager.h>
 #include <vm/uma.h>
 
+#include <machine/bootinfo.h>
 #include <machine/cpu.h>
 #include <machine/pcb.h>
 #include <machine/powerpc.h>
@@ -107,8 +108,19 @@ __FBSDID("$FreeBSD$");
 #endif
 extern struct mtx sched_lock;
 
+extern int dumpsys_minidump;
+
+extern unsigned char _etext[];
+extern unsigned char _end[];
+
 /* Kernel physical load address. */
 extern uint32_t kernload;
+vm_offset_t kernstart;
+vm_size_t kernsize;
+
+/* Message buffer and tables. */
+static vm_offset_t data_start;
+static vm_size_t data_end;
 
 struct mem_region availmem_regions[MEM_REGIONS];
 int availmem_regions_sz;
@@ -304,6 +316,11 @@ static void		mmu_booke_kenter(mmu_t, vm_offset_t, vm_offset_t);
 static void		mmu_booke_kremove(mmu_t, vm_offset_t);
 static boolean_t	mmu_booke_dev_direct_mapped(mmu_t, vm_offset_t, vm_size_t);
 static boolean_t	mmu_booke_page_executable(mmu_t, vm_page_t);
+static vm_offset_t	mmu_booke_dumpsys_map(mmu_t, struct pmap_md *,
+    vm_size_t, vm_size_t *);
+static void		mmu_booke_dumpsys_unmap(mmu_t, struct pmap_md *,
+    vm_size_t, vm_offset_t);
+static struct pmap_md	*mmu_booke_scan_md(mmu_t, struct pmap_md *);
 
 static mmu_method_t mmu_booke_methods[] = {
 	/* pmap dispatcher interface */
@@ -352,6 +369,11 @@ static mmu_method_t mmu_booke_methods[] = {
 	MMUMETHOD(mmu_page_executable,	mmu_booke_page_executable),
 	MMUMETHOD(mmu_unmapdev,		mmu_booke_unmapdev),
 
+	/* dumpsys() support */
+	MMUMETHOD(mmu_dumpsys_map,	mmu_booke_dumpsys_map),
+	MMUMETHOD(mmu_dumpsys_unmap,	mmu_booke_dumpsys_unmap),
+	MMUMETHOD(mmu_scan_md,		mmu_booke_scan_md),
+
 	{ 0, 0 }
 };
 
@@ -884,7 +906,7 @@ pte_find(mmu_t mmu, pmap_t pmap, vm_offset_t va)
  * This is called during e500_init, before the system is really initialized.
  */
 static void
-mmu_booke_bootstrap(mmu_t mmu, vm_offset_t kernelstart, vm_offset_t kernelend)
+mmu_booke_bootstrap(mmu_t mmu, vm_offset_t start, vm_offset_t kernelend)
 {
 	vm_offset_t phys_kernelend;
 	struct mem_region *mp, *mp1;
@@ -904,47 +926,52 @@ mmu_booke_bootstrap(mmu_t mmu, vm_offset_t kernelstart, vm_offset_t kernelend)
 	tlb0_get_tlbconf();
 
 	/* Align kernel start and end address (kernel image). */
-	kernelstart = trunc_page(kernelstart);
-	kernelend = round_page(kernelend);
+	kernstart = trunc_page(start);
+	data_start = round_page(kernelend);
+	kernsize = data_start - kernstart;
+
+	data_end = data_start;
 
 	/* Allocate space for the message buffer. */
-	msgbufp = (struct msgbuf *)kernelend;
-	kernelend += MSGBUF_SIZE;
+	msgbufp = (struct msgbuf *)data_end;
+	data_end += MSGBUF_SIZE;
 	debugf(" msgbufp at 0x%08x end = 0x%08x\n", (uint32_t)msgbufp,
-	    kernelend);
+	    data_end);
 
-	kernelend = round_page(kernelend);
+	data_end = round_page(data_end);
 
 	/* Allocate space for ptbl_bufs. */
-	ptbl_bufs = (struct ptbl_buf *)kernelend;
-	kernelend += sizeof(struct ptbl_buf) * PTBL_BUFS;
+	ptbl_bufs = (struct ptbl_buf *)data_end;
+	data_end += sizeof(struct ptbl_buf) * PTBL_BUFS;
 	debugf(" ptbl_bufs at 0x%08x end = 0x%08x\n", (uint32_t)ptbl_bufs,
-	    kernelend);
+	    data_end);
 
-	kernelend = round_page(kernelend);
+	data_end = round_page(data_end);
 
 	/* Allocate PTE tables for kernel KVA. */
-	kernel_pdir = kernelend;
+	kernel_pdir = data_end;
 	kernel_ptbls = (VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS +
 	    PDIR_SIZE - 1) / PDIR_SIZE;
-	kernelend += kernel_ptbls * PTBL_PAGES * PAGE_SIZE;
+	data_end += kernel_ptbls * PTBL_PAGES * PAGE_SIZE;
 	debugf(" kernel ptbls: %d\n", kernel_ptbls);
-	debugf(" kernel pdir at 0x%08x end = 0x%08x\n", kernel_pdir, kernelend);
+	debugf(" kernel pdir at 0x%08x end = 0x%08x\n", kernel_pdir, data_end);
 
-	debugf(" kernelend: 0x%08x\n", kernelend);
-	if (kernelend - kernelstart > 0x1000000) {
-		kernelend = (kernelend + 0x3fffff) & ~0x3fffff;
-		tlb1_mapin_region(kernelstart + 0x1000000,
-		    kernload + 0x1000000, kernelend - kernelstart - 0x1000000);
+	debugf(" data_end: 0x%08x\n", data_end);
+	if (data_end - kernstart > 0x1000000) {
+		data_end = (data_end + 0x3fffff) & ~0x3fffff;
+		tlb1_mapin_region(kernstart + 0x1000000,
+		    kernload + 0x1000000, data_end - kernstart - 0x1000000);
 	} else
-		kernelend = (kernelend + 0xffffff) & ~0xffffff;
+		data_end = (data_end + 0xffffff) & ~0xffffff;
 
-	debugf(" updated kernelend: 0x%08x\n", kernelend);
+	debugf(" updated data_end: 0x%08x\n", data_end);
+
+	kernsize += data_end - data_start;
 
 	/*
 	 * Clear the structures - note we can only do it safely after the
 	 * possible additional TLB1 translations are in place (above) so that
-	 * all range up to the currently calculated 'kernelend' is covered.
+	 * all range up to the currently calculated 'data_end' is covered.
 	 */
 	memset((void *)ptbl_bufs, 0, sizeof(struct ptbl_buf) * PTBL_SIZE);
 	memset((void *)kernel_pdir, 0, kernel_ptbls * PTBL_PAGES * PAGE_SIZE);
@@ -952,7 +979,7 @@ mmu_booke_bootstrap(mmu_t mmu, vm_offset_t kernelstart, vm_offset_t kernelend)
 	/*******************************************************/
 	/* Set the start and end of kva. */
 	/*******************************************************/
-	virtual_avail = kernelend;
+	virtual_avail = round_page(data_end);
 	virtual_end = VM_MAX_KERNEL_ADDRESS;
 
 	/* Allocate KVA space for page zero/copy operations. */
@@ -980,12 +1007,11 @@ mmu_booke_bootstrap(mmu_t mmu, vm_offset_t kernelstart, vm_offset_t kernelend)
 	    ptbl_buf_pool_vabase, virtual_avail);
 
 	/* Calculate corresponding physical addresses for the kernel region. */
-	phys_kernelend = kernload + (kernelend - kernelstart);
+	phys_kernelend = kernload + kernsize;
 	debugf("kernel image and allocated data:\n");
 	debugf(" kernload    = 0x%08x\n", kernload);
-	debugf(" kernelstart = 0x%08x\n", kernelstart);
-	debugf(" kernelend   = 0x%08x\n", kernelend);
-	debugf(" kernel size = 0x%08x\n", kernelend - kernelstart);
+	debugf(" kernstart   = 0x%08x\n", kernstart);
+	debugf(" kernsize    = 0x%08x\n", kernsize);
 
 	if (sizeof(phys_avail) / sizeof(phys_avail[0]) < availmem_regions_sz)
 		panic("mmu_booke_bootstrap: phys_avail too small");
@@ -2287,6 +2313,140 @@ mmu_booke_dev_direct_mapped(mmu_t mmu, vm_offset_t pa, vm_size_t size)
 	return (EFAULT);
 }
 
+vm_offset_t
+mmu_booke_dumpsys_map(mmu_t mmu, struct pmap_md *md, vm_size_t ofs,
+    vm_size_t *sz)
+{
+	vm_paddr_t pa, ppa;
+	vm_offset_t va;
+	vm_size_t gran;
+
+	/* Raw physical memory dumps don't have a virtual address. */
+	if (md->md_vaddr == ~0UL) {
+		/* We always map a 256MB page at 256M. */
+		gran = 256 * 1024 * 1024;
+		pa = md->md_paddr + ofs;
+		ppa = pa & ~(gran - 1);
+		ofs = pa - ppa;
+		va = gran;
+		tlb1_set_entry(va, ppa, gran, _TLB_ENTRY_IO);
+		if (*sz > (gran - ofs))
+			*sz = gran - ofs;
+		return (va + ofs);
+	}
+
+	/* Minidumps are based on virtual memory addresses. */
+	va = md->md_vaddr + ofs;
+	if (va >= kernstart + kernsize) {
+		gran = PAGE_SIZE - (va & PAGE_MASK);
+		if (*sz > gran)
+			*sz = gran;
+	}
+	return (va);
+}
+
+void
+mmu_booke_dumpsys_unmap(mmu_t mmu, struct pmap_md *md, vm_size_t ofs,
+    vm_offset_t va)
+{
+
+	/* Raw physical memory dumps don't have a virtual address. */
+	if (md->md_vaddr == ~0UL) {
+		tlb1_idx--;
+		tlb1[tlb1_idx].mas1 = 0;
+		tlb1[tlb1_idx].mas2 = 0;
+		tlb1[tlb1_idx].mas3 = 0;
+		tlb1_write_entry(tlb1_idx);
+		return;
+	}
+ 
+	/* Minidumps are based on virtual memory addresses. */
+	/* Nothing to do... */
+}
+
+struct pmap_md *
+mmu_booke_scan_md(mmu_t mmu, struct pmap_md *prev)
+{
+	static struct pmap_md md;
+	struct bi_mem_region *mr;
+	pte_t *pte;
+	vm_offset_t va;
+ 
+	if (dumpsys_minidump) {
+		md.md_paddr = ~0UL;	/* Minidumps use virtual addresses. */
+		if (prev == NULL) {
+			/* 1st: kernel .data and .bss. */
+			md.md_index = 1;
+			md.md_vaddr = trunc_page((uintptr_t)_etext);
+			md.md_size = round_page((uintptr_t)_end) - md.md_vaddr;
+			return (&md);
+		}
+		switch (prev->md_index) {
+		case 1:
+			/* 2nd: msgbuf and tables (see pmap_bootstrap()). */
+			md.md_index = 2;
+			md.md_vaddr = data_start;
+			md.md_size = data_end - data_start;
+			break;
+		case 2:
+			/* 3rd: kernel VM. */
+			va = prev->md_vaddr + prev->md_size;
+			/* Find start of next chunk (from va). */
+			while (va < virtual_end) {
+				/* Don't dump the buffer cache. */
+				if (va >= kmi.buffer_sva &&
+				    va < kmi.buffer_eva) {
+					va = kmi.buffer_eva;
+					continue;
+				}
+				pte = pte_find(mmu, kernel_pmap, va);
+				if (pte != NULL && PTE_ISVALID(pte))
+					break;
+				va += PAGE_SIZE;
+			}
+			if (va < virtual_end) {
+				md.md_vaddr = va;
+				va += PAGE_SIZE;
+				/* Find last page in chunk. */
+				while (va < virtual_end) {
+					/* Don't run into the buffer cache. */
+					if (va == kmi.buffer_sva)
+						break;
+					pte = pte_find(mmu, kernel_pmap, va);
+					if (pte == NULL || !PTE_ISVALID(pte))
+						break;
+					va += PAGE_SIZE;
+				}
+				md.md_size = va - md.md_vaddr;
+				break;
+			}
+			md.md_index = 3;
+			/* FALLTHROUGH */
+		default:
+			return (NULL);
+		}
+	} else { /* minidumps */
+		mr = bootinfo_mr();
+		if (prev == NULL) {
+			/* first physical chunk. */
+			md.md_paddr = mr->mem_base;
+			md.md_size = mr->mem_size;
+			md.md_vaddr = ~0UL;
+			md.md_index = 1;
+		} else if (md.md_index < bootinfo->bi_mem_reg_no) {
+			md.md_paddr = mr[md.md_index].mem_base;
+			md.md_size = mr[md.md_index].mem_size;
+			md.md_vaddr = ~0UL;
+			md.md_index++;
+		} else {
+			/* There's no next physical chunk. */
+			return (NULL);
+		}
+	}
+
+	return (&md);
+}
+
 /*
  * Map a set of physical memory pages into the kernel virtual address space.
  * Return a pointer to where it is mapped. This routine is intended to be used
diff --git a/sys/powerpc/conf/GENERIC b/sys/powerpc/conf/GENERIC
index b87f899..a85ded0 100644
--- a/sys/powerpc/conf/GENERIC
+++ b/sys/powerpc/conf/GENERIC
@@ -140,7 +140,6 @@ device		ulpt		# Printer
 device		umass		# Disks/Mass storage - Requires scbus and da0
 device		ums		# Mouse
 device		urio		# Diamond Rio 500 MP3 player
-device		uscanner	# Scanners
 # USB Ethernet
 device		aue		# ADMtek USB Ethernet
 device		axe		# ASIX Electronics USB Ethernet
diff --git a/sys/powerpc/include/elf.h b/sys/powerpc/include/elf.h
index 422a86a..1224fbb 100644
--- a/sys/powerpc/include/elf.h
+++ b/sys/powerpc/include/elf.h
@@ -77,8 +77,9 @@ __ElfType(Auxinfo);
 #define	AT_DCACHEBSIZE	10	/* Data cache block size for the processor. */
 #define	AT_ICACHEBSIZE	11	/* Instruction cache block size for the uP. */
 #define	AT_UCACHEBSIZE	12	/* Cache block size, or `0' if cache not unified. */
+#define	AT_EXECPATH	13	/* Path to the executable. */
 
-#define	AT_COUNT	13	/* Count of defined aux entry types. */
+#define	AT_COUNT	14	/* Count of defined aux entry types. */
 
 /*
  * Relocation types.
diff --git a/sys/powerpc/include/hid.h b/sys/powerpc/include/hid.h
index d9fd6fe..41602fd 100644
--- a/sys/powerpc/include/hid.h
+++ b/sys/powerpc/include/hid.h
@@ -47,6 +47,7 @@
 #define HID0_SLEEP	0x00200000  /* Enable sleep mode */
 #define HID0_DPM	0x00100000  /* Enable Dynamic power management */
 #define HID0_RISEG	0x00080000  /* Read I-SEG */
+#define HID0_TG		0x00040000  /* Timebase Granularity (OEA64) */
 #define HID0_BHTCLR	0x00040000  /* Clear branch history table (7450) */
 #define HID0_EIEC	0x00040000  /* Enable internal error checking */
 #define HID0_XAEN	0x00020000  /* Enable eXtended Addressing (7450) */
@@ -146,4 +147,6 @@
  * 7457: XBSEN = Extended BAT Block Size Enable
  */
 
+#define HID5_970_DCBZ_SIZE_HI	0x01000000	/* dcbz does a 32-byte store */
+
 #endif /* _POWERPC_HID_H_ */
diff --git a/sys/powerpc/include/intr.h b/sys/powerpc/include/intr.h
index 8bc417c..72c8542 100644
--- a/sys/powerpc/include/intr.h
+++ b/sys/powerpc/include/intr.h
@@ -63,48 +63,11 @@
 
 #ifndef LOCORE
 
-#if 0
-/*
- * Interrupt handler chains.  intr_establish() inserts a handler into
- * the list.  The handler is called with its (single) argument.
- */
-struct intrhand {
-	int	(*ih_fun)(void *);
-	void	*ih_arg;
-	u_long	ih_count;
-	struct	intrhand *ih_next;
-	int	ih_level;
-	int	ih_irq;
-};
-#endif
-
-void	setsoftclock(void);
-void	clearsoftclock(void);
-void	setsoftnet(void);
-void	clearsoftnet(void);
-
 void	do_pending_int(void);
 
-static __inline void softintr(int);
-
 extern u_int cpl, ipending, tickspending;
 extern int imask[];
 
-/* Following code should be implemented with lwarx/stwcx to avoid
- * the disable/enable. i need to read the manual once more.... */
-static __inline void
-softintr(int ipl)
-{
-	unsigned int	msrsave;
-
-	msrsave = mfmsr();
-	mtmsr(msrsave & ~PSL_EE);
-
-	ipending |= 1 << ipl;
-
-	mtmsr(msrsave);
-}
-
 #define	ICU_LEN		64
 
 /* Soft interrupt masks. */
@@ -113,46 +76,6 @@ softintr(int ipl)
 #define	SIR_SERIAL	30
 #define	SPL_CLOCK	31
 
-#if 0
-
-/*
- * Hardware interrupt masks
- */
-
-#define	splbio()	splraise(imask[IPL_BIO])
-#define	splnet()	splraise(imask[IPL_NET])
-#define	spltty()	splraise(imask[IPL_TTY])
-#define	splaudio()	splraise(imask[IPL_AUDIO])
-#define	splclock()	splraise(imask[IPL_CLOCK])
-#define	splstatclock()	splclock()
-#define	splserial()	splraise(imask[IPL_SERIAL])
-
-#define	spllpt()	spltty()
-
-/*
- * Software interrupt masks
- *
- * NOTE: splsoftclock() is used by hardclock() to lower the priority from
- * clock to softclock before it calls softclock().
- */
-#define	spllowersoftclock() spllower(imask[IPL_SOFTCLOCK])
-#define	splsoftclock()	splraise(imask[IPL_SOFTCLOCK])
-#define	splsoftnet()	splraise(imask[IPL_SOFTNET])
-#define	splsoftserial()	splraise(imask[IPL_SOFTSERIAL])
-
-/*
- * Miscellaneous
- */
-#define	splimp()	splraise(imask[IPL_IMP])
-#define	splhigh()	splraise(imask[IPL_HIGH])
-#define	spl0()		spllower(0)
-
-#endif /* 0 */
-
-#define	setsoftclock()	softintr(SIR_CLOCK)
-#define	setsoftnet()	softintr(SIR_NET)
-#define	setsoftserial()	softintr(SIR_SERIAL)
-
 #define	CNT_IRQ0	0
 #define	CNT_CLOCK	64
 #define	CNT_SOFTCLOCK	65
diff --git a/sys/powerpc/include/md_var.h b/sys/powerpc/include/md_var.h
index c554f5f..709e4a9 100644
--- a/sys/powerpc/include/md_var.h
+++ b/sys/powerpc/include/md_var.h
@@ -46,6 +46,8 @@ extern	u_long	ns_per_tick;
 
 extern	int powerpc_pow_enabled;
 extern	int cacheline_size;
+extern  int ppc64;
+extern  int hw_direct_map;
 
 void	__syncicache(void *, int);
 
diff --git a/sys/powerpc/include/pmap.h b/sys/powerpc/include/pmap.h
index 2b2d6af..5e6d9c6 100644
--- a/sys/powerpc/include/pmap.h
+++ b/sys/powerpc/include/pmap.h
@@ -71,6 +71,13 @@
 #include <machine/pte.h>
 #include <machine/tlb.h>
 
+struct pmap_md {
+	u_int		md_index;
+	vm_paddr_t      md_paddr;
+	vm_offset_t     md_vaddr;
+	vm_size_t       md_size;
+};
+
 #if defined(AIM)
 
 #if !defined(NPMAPS)
@@ -179,6 +186,11 @@ extern	vm_offset_t msgbuf_phys;
 
 extern	int pmap_bootstrapped;
 
+extern vm_offset_t pmap_dumpsys_map(struct pmap_md *, vm_size_t, vm_size_t *);
+extern void pmap_dumpsys_unmap(struct pmap_md *, vm_size_t, vm_offset_t);
+
+extern struct pmap_md *pmap_scan_md(struct pmap_md *);
+
 #endif
 
 #endif /* !_MACHINE_PMAP_H_ */
diff --git a/sys/powerpc/include/sf_buf.h b/sys/powerpc/include/sf_buf.h
index 162f5a6..7ddb981 100644
--- a/sys/powerpc/include/sf_buf.h
+++ b/sys/powerpc/include/sf_buf.h
@@ -32,33 +32,9 @@
 #include <vm/vm.h>
 #include <vm/vm_param.h>
 #include <vm/vm_page.h>
+#include <machine/md_var.h>
 #include <sys/queue.h>
 
-#if defined(AIM)
-/*
- * On this machine, the only purpose for which sf_buf is used is to implement
- * an opaque pointer required by the machine-independent parts of the kernel.
- * That pointer references the vm_page that is "mapped" by the sf_buf.  The
- * actual mapping is provided by the direct virtual-to-physical mapping.  
- */
-struct sf_buf;
-
-static __inline vm_offset_t
-sf_buf_kva(struct sf_buf *sf)
-{
-
-	return (VM_PAGE_TO_PHYS((vm_page_t)sf));
-}
-
-static __inline vm_page_t
-sf_buf_page(struct sf_buf *sf)
-{
-
-	return ((vm_page_t)sf);
-}
-
-#elif defined(E500)
-
 struct vm_page;
 
 struct sf_buf {
@@ -69,9 +45,22 @@ struct sf_buf {
 	int		ref_count;	/* usage of this mapping */
 };
 
+/*
+ * On 32-bit OEA, the only purpose for which sf_buf is used is to implement
+ * an opaque pointer required by the machine-independent parts of the kernel.
+ * That pointer references the vm_page that is "mapped" by the sf_buf.  The
+ * actual mapping is provided by the direct virtual-to-physical mapping.  
+ *
+ * On OEA64 and Book-E, we need to do something a little more complicated. Use
+ * the runtime-detected hw_direct_map to pick between the two cases. Our
+ * friends in vm_machdep.c will do the same to ensure nothing gets confused.
+ */
+
 static __inline vm_offset_t
 sf_buf_kva(struct sf_buf *sf)
 {
+	if (hw_direct_map)
+		return (VM_PAGE_TO_PHYS((vm_page_t)sf));
 
 	return (sf->kva);
 }
@@ -79,10 +68,10 @@ sf_buf_kva(struct sf_buf *sf)
 static __inline struct vm_page *
 sf_buf_page(struct sf_buf *sf)
 {
+	if (hw_direct_map)
+		return ((vm_page_t)sf);
 
 	return (sf->m);
 }
 
-#endif
-
 #endif /* !_MACHINE_SF_BUF_H_ */
diff --git a/sys/powerpc/include/spr.h b/sys/powerpc/include/spr.h
index 70a0012..84fab0d 100644
--- a/sys/powerpc/include/spr.h
+++ b/sys/powerpc/include/spr.h
@@ -43,6 +43,44 @@
 	( { register_t val;						\
 	  __asm __volatile("mfspr %0,%1" : "=r"(val) : "K"(reg));	\
 	  val; } )
+
+/* The following routines allow manipulation of the full 64-bit width 
+ * of SPRs on 64 bit CPUs in bridge mode */
+
+#define mtspr64(reg,valhi,vallo,scratch)				\
+	__asm __volatile("						\
+		mfmsr %0; 						\
+		insrdi %0,1,1,0; 					\
+		mtmsrd %0; 						\
+		isync; 							\
+									\
+		sld %1,%1,%4;						\
+		or %1,%1,%2;						\
+		mtspr %3,%1;						\
+		srd %1,%1,%4;						\
+									\
+		clrldi %0,%0,1; 					\
+		mtmsrd %0; 						\
+		isync;"							\
+	: "=r"(scratch), "=r"(valhi) : "r"(vallo), "K"(reg), "r"(32))
+
+#define mfspr64upper(reg,scratch)					\
+	( { register_t val;						\
+	    __asm __volatile("						\
+		mfmsr %0; 						\
+		insrdi %0,1,1,0; 					\
+		mtmsrd %0; 						\
+		isync; 							\
+									\
+		mfspr %1,%2;						\
+		srd %1,%1,%3;						\
+									\
+		clrldi %0,%0,1; 					\
+		mtmsrd %0; 						\
+		isync;" 						\
+	    : "=r"(scratch), "=r"(val) : "K"(reg), "r"(32)); 			\
+	    val; } )
+
 #endif /* _LOCORE */
 
 /*
@@ -112,7 +150,11 @@
 #define	  IBM401E2		  0x0025
 #define	  IBM401F2		  0x0026
 #define	  IBM401G2		  0x0027
+#define	  IBM970		  0x0039
+#define	  IBM970FX		  0x003c
 #define	  IBMPOWER3		  0x0041
+#define	  IBM970MP		  0x0044
+#define	  IBM970GX		  0x0045
 #define	  MPC860		  0x0050
 #define	  MPC8240		  0x0081
 #define	  IBM405GP		  0x4011
@@ -349,6 +391,8 @@
 #define	SPR_SRR3		0x3df	/* 4.. Save/Restore Register 3 */
 #define	SPR_HID0		0x3f0	/* ..8 Hardware Implementation Register 0 */
 #define	SPR_HID1		0x3f1	/* ..8 Hardware Implementation Register 1 */
+#define	SPR_HID4		0x3f4	/* ..8 Hardware Implementation Register 4 */
+#define	SPR_HID5		0x3f6	/* ..8 Hardware Implementation Register 5 */
 
 #if defined(AIM)
 #define	SPR_DBSR		0x3f0	/* 4.. Debug Status Register */
@@ -564,6 +608,8 @@
 #define	  SVR_MPC8533E		  0x8034
 #define	  SVR_MPC8541		  0x8072
 #define	  SVR_MPC8541E		  0x807a
+#define	  SVR_MPC8548		  0x8031
+#define	  SVR_MPC8548E		  0x8039
 #define	  SVR_MPC8555		  0x8071
 #define	  SVR_MPC8555E		  0x8079
 #define	  SVR_MPC8572		  0x80e0
diff --git a/sys/powerpc/include/sysarch.h b/sys/powerpc/include/sysarch.h
new file mode 100644
index 0000000..c46d100
--- /dev/null
+++ b/sys/powerpc/include/sysarch.h
@@ -0,0 +1,43 @@
+/*-
+ * Copyright (c) 1993 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_SYSARCH_H_
+#define	_MACHINE_SYSARCH_H_
+
+#ifndef _KERNEL
+#include <sys/cdefs.h>
+
+__BEGIN_DECLS
+int	sysarch(int, void *);
+__END_DECLS
+#endif
+
+#endif /* !_MACHINE_SYSARCH_H_ */
diff --git a/sys/powerpc/include/vmparam.h b/sys/powerpc/include/vmparam.h
index 489159d..5af62eb 100644
--- a/sys/powerpc/include/vmparam.h
+++ b/sys/powerpc/include/vmparam.h
@@ -106,6 +106,13 @@
  */
 #define UMA_MD_SMALL_ALLOC
 
+/*
+ * On 64-bit systems in bridge mode, we have no direct map, so we fake
+ * the small_alloc() calls. But we need the VM to be in a reasonable
+ * state first.
+ */
+#define UMA_MD_SMALL_ALLOC_NEEDS_VM
+
 #else
 
 /*
diff --git a/sys/powerpc/mpc85xx/mpc85xx.c b/sys/powerpc/mpc85xx/mpc85xx.c
index de9c771..b6de50d 100644
--- a/sys/powerpc/mpc85xx/mpc85xx.c
+++ b/sys/powerpc/mpc85xx/mpc85xx.c
@@ -61,7 +61,7 @@ ccsr_write4(uintptr_t addr, uint32_t val)
 	__asm __volatile("eieio; sync");
 }
 
-static __inline int
+int
 law_getmax(void)
 {
 	uint32_t ver;
@@ -69,6 +69,8 @@ law_getmax(void)
 	ver = SVR_VER(mfspr(SPR_SVR));
 	if (ver == SVR_MPC8572E || ver == SVR_MPC8572)
 		return (12);
+	else if (ver == SVR_MPC8548E || ver == SVR_MPC8548)
+		return (10);
 	else
 		return (8);
 }
@@ -132,7 +134,8 @@ cpu_reset(void)
 {
 	uint32_t ver = SVR_VER(mfspr(SPR_SVR));
 
-	if (ver == SVR_MPC8572E || ver == SVR_MPC8572)
+	if (ver == SVR_MPC8572E || ver == SVR_MPC8572 ||
+	    ver == SVR_MPC8548E || ver == SVR_MPC8548)
 		/* Systems with dedicated reset register */
 		ccsr_write4(OCP85XX_RSTCR, 2);
 	else {
diff --git a/sys/powerpc/mpc85xx/mpc85xx.h b/sys/powerpc/mpc85xx/mpc85xx.h
index 4d31b58..e558489 100644
--- a/sys/powerpc/mpc85xx/mpc85xx.h
+++ b/sys/powerpc/mpc85xx/mpc85xx.h
@@ -33,5 +33,6 @@ uint32_t ccsr_read4(uintptr_t addr);
 void ccsr_write4(uintptr_t addr, uint32_t val);
 int law_enable(int trgt, u_long addr, u_long size);
 int law_disable(int trgt, u_long addr, u_long size);
+int law_getmax(void);
 
 #endif /* _MPC85XX_H_ */
diff --git a/sys/powerpc/mpc85xx/ocpbus.c b/sys/powerpc/mpc85xx/ocpbus.c
index ae56fb0..ddc2e74 100644
--- a/sys/powerpc/mpc85xx/ocpbus.c
+++ b/sys/powerpc/mpc85xx/ocpbus.c
@@ -114,8 +114,6 @@ devclass_t ocpbus_devclass;
 
 DRIVER_MODULE(ocpbus, nexus, ocpbus_driver, ocpbus_devclass, 0, 0);
 
-static int law_max = 0;
-
 static device_t
 ocpbus_mk_child(device_t dev, int type, int unit)
 {
@@ -189,16 +187,6 @@ ocpbus_write_law(int trgt, int type, u_long *startp, u_long *countp)
 static int
 ocpbus_probe(device_t dev)
 {
-	struct ocpbus_softc *sc;
-	uint32_t ver;
-
-	sc = device_get_softc(dev);
-
-	ver = SVR_VER(mfspr(SPR_SVR));
-	if (ver == SVR_MPC8572E || ver == SVR_MPC8572)
-		law_max = 12;
-	else
-		law_max = 8;
 
 	device_set_desc(dev, "On-Chip Peripherals bus");
 	return (BUS_PROBE_DEFAULT);
@@ -208,7 +196,7 @@ static int
 ocpbus_attach(device_t dev)
 {
 	struct ocpbus_softc *sc;
-	int error, i, tgt;
+	int error, i, tgt, law_max;
 	uint32_t sr;
 	u_long start, end;
 
@@ -261,6 +249,7 @@ ocpbus_attach(device_t dev)
 	 * Clear local access windows. Skip DRAM entries, so we don't shoot
 	 * ourselves in the foot.
 	 */
+	law_max = law_getmax();
 	for (i = 0; i < law_max; i++) {
 		sr = ccsr_read4(OCP85XX_LAWSR(i));
 		if ((sr & 0x80000000) == 0)
diff --git a/sys/powerpc/ofw/ofw_real.c b/sys/powerpc/ofw/ofw_real.c
new file mode 100644
index 0000000..c5f11b0
--- /dev/null
+++ b/sys/powerpc/ofw/ofw_real.c
@@ -0,0 +1,922 @@
+/*	$NetBSD: Locore.c,v 1.7 2000/08/20 07:04:59 tsubai Exp $	*/
+
+/*-
+ * Copyright (C) 1995, 1996 Wolfgang Solfrank.
+ * Copyright (C) 1995, 1996 TooLs GmbH.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by TooLs GmbH.
+ * 4. The name of TooLs GmbH may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/*-
+ * Copyright (C) 2000 Benno Rice.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/systm.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/stdarg.h>
+#include <machine/bus.h>
+#include <machine/pmap.h>
+#include <machine/ofw_machdep.h>
+
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofwvar.h>
+#include "ofw_if.h"
+
+static void ofw_real_init(ofw_t, void *openfirm);
+static int ofw_real_test(ofw_t, const char *name);
+static phandle_t ofw_real_peer(ofw_t, phandle_t node);
+static phandle_t ofw_real_child(ofw_t, phandle_t node);
+static phandle_t ofw_real_parent(ofw_t, phandle_t node);
+static phandle_t ofw_real_instance_to_package(ofw_t, ihandle_t instance);
+static ssize_t ofw_real_getproplen(ofw_t, phandle_t package, 
+    const char *propname);
+static ssize_t ofw_real_getprop(ofw_t, phandle_t package, const char *propname, 
+    void *buf, size_t buflen);
+static int ofw_real_nextprop(ofw_t, phandle_t package, const char *previous, 
+    char *buf, size_t);
+static int ofw_real_setprop(ofw_t, phandle_t package, char *propname, 
+    void *buf, size_t len);
+static ssize_t ofw_real_canon(ofw_t, const char *device, char *buf, size_t len);
+static phandle_t ofw_real_finddevice(ofw_t, const char *device);
+static ssize_t ofw_real_instance_to_path(ofw_t, ihandle_t instance, char *buf, 
+    size_t len);
+static ssize_t ofw_real_package_to_path(ofw_t, phandle_t package, char *buf, 
+    size_t len);
+static int ofw_real_call_method(ofw_t, ihandle_t instance, const char *method, 
+    int nargs, int nreturns, unsigned long *args_and_returns);
+static ihandle_t ofw_real_open(ofw_t, const char *device);
+static void ofw_real_close(ofw_t, ihandle_t instance);
+static ssize_t ofw_real_read(ofw_t, ihandle_t instance, void *addr, size_t len);
+static ssize_t ofw_real_write(ofw_t, ihandle_t instance, const void *addr, 
+    size_t len);
+static int ofw_real_seek(ofw_t, ihandle_t instance, u_int64_t pos);
+static caddr_t ofw_real_claim(ofw_t, void *virt, size_t size, u_int align);
+static void ofw_real_release(ofw_t, void *virt, size_t size);
+static void ofw_real_enter(ofw_t);
+static void ofw_real_exit(ofw_t);
+
+static ofw_method_t ofw_real_methods[] = {
+	OFWMETHOD(ofw_init,			ofw_real_init),
+	OFWMETHOD(ofw_peer,			ofw_real_peer),
+	OFWMETHOD(ofw_child,			ofw_real_child),
+	OFWMETHOD(ofw_parent,			ofw_real_parent),
+	OFWMETHOD(ofw_instance_to_package,	ofw_real_instance_to_package),
+	OFWMETHOD(ofw_getproplen,		ofw_real_getproplen),
+	OFWMETHOD(ofw_getprop,			ofw_real_getprop),
+	OFWMETHOD(ofw_nextprop,			ofw_real_nextprop),
+	OFWMETHOD(ofw_setprop,			ofw_real_setprop),
+	OFWMETHOD(ofw_canon,			ofw_real_canon),
+	OFWMETHOD(ofw_finddevice,		ofw_real_finddevice),
+	OFWMETHOD(ofw_instance_to_path,		ofw_real_instance_to_path),
+	OFWMETHOD(ofw_package_to_path,		ofw_real_package_to_path),
+
+	OFWMETHOD(ofw_test,			ofw_real_test),
+	OFWMETHOD(ofw_call_method,		ofw_real_call_method),
+	OFWMETHOD(ofw_open,			ofw_real_open),
+	OFWMETHOD(ofw_close,			ofw_real_close),
+	OFWMETHOD(ofw_read,			ofw_real_read),
+	OFWMETHOD(ofw_write,			ofw_real_write),
+	OFWMETHOD(ofw_seek,			ofw_real_seek),
+	OFWMETHOD(ofw_claim,			ofw_real_claim),
+	OFWMETHOD(ofw_release,			ofw_real_release),
+	OFWMETHOD(ofw_enter,			ofw_real_enter),
+	OFWMETHOD(ofw_exit,			ofw_real_exit),
+
+	{ 0, 0 }
+};
+
+static ofw_def_t ofw_real = {
+	OFW_STD_REAL,
+	ofw_real_methods,
+	0
+};
+OFW_DEF(ofw_real);
+
+MALLOC_DEFINE(M_OFWREAL, "ofwreal", "Open Firmware Real Mode Bounce Page");
+
+static int (*openfirmware)(void *);
+
+static vm_offset_t	of_bounce_phys;
+static caddr_t		of_bounce_virt;
+static off_t		of_bounce_offset;
+static size_t		of_bounce_size;
+static struct mtx	of_bounce_mtx;
+
+/*
+ * After the VM is up, allocate a wired, low memory bounce page.
+ */
+
+static void ofw_real_bounce_alloc(void *);
+
+SYSINIT(ofw_real_bounce_alloc, SI_SUB_VM, SI_ORDER_ANY, 
+    ofw_real_bounce_alloc, NULL);
+
+static void
+ofw_real_start(void)
+{
+	mtx_lock(&of_bounce_mtx);
+	of_bounce_offset = 0;
+}
+	
+static void
+ofw_real_stop(void)
+{
+	mtx_unlock(&of_bounce_mtx);
+}
+
+static void
+ofw_real_bounce_alloc(void *junk)
+{
+	/*
+	 * Check that ofw_real is actually in use before allocating wads 
+	 * of memory. Do this by checking if our mutex has been set up.
+	 */
+	if (!mtx_initialized(&of_bounce_mtx))
+		return;
+
+	/*
+	 * Allocate a page of contiguous, wired physical memory that can
+	 * fit into a 32-bit address space.
+	 */
+
+	mtx_lock(&of_bounce_mtx);
+
+	of_bounce_virt = contigmalloc(PAGE_SIZE, M_OFWREAL, 0,
+			     0, BUS_SPACE_MAXADDR_32BIT, PAGE_SIZE, PAGE_SIZE);
+	of_bounce_phys = vtophys(of_bounce_virt);
+	of_bounce_size = PAGE_SIZE;
+
+	mtx_unlock(&of_bounce_mtx);
+}
+
+static cell_t
+ofw_real_map(const void *buf, size_t len)
+{
+	cell_t phys;
+
+	mtx_assert(&of_bounce_mtx, MA_OWNED);
+
+	if (of_bounce_virt == NULL) {
+		if (!pmap_bootstrapped)
+			return (cell_t)buf;
+
+		/*
+		 * XXX: It is possible for us to get called before the VM has
+		 * come online, but after the MMU is up. We don't have the
+		 * bounce buffer yet, but can no longer presume a 1:1 mapping.
+		 * Grab the physical address of the buffer, and hope it is
+		 * in range if this happens.
+		 */
+		return (cell_t)vtophys(buf);
+	}
+
+	/*
+	 * Make sure the bounce page offset satisfies any reasonable
+	 * alignment constraint.
+	 */
+	of_bounce_offset += of_bounce_offset % sizeof(register_t);
+
+	if (of_bounce_offset + len > of_bounce_size) {
+		panic("Oversize Open Firmware call!");
+		return 0;
+	}
+
+	memcpy(of_bounce_virt + of_bounce_offset, buf, len);
+	phys = of_bounce_phys + of_bounce_offset;
+
+	of_bounce_offset += len;
+
+	return phys;
+}
+
+static void
+ofw_real_unmap(cell_t physaddr, void *buf, size_t len)
+{
+	mtx_assert(&of_bounce_mtx, MA_OWNED);
+
+	if (of_bounce_virt == NULL)
+		return;
+
+	memcpy(buf,of_bounce_virt + (physaddr - of_bounce_phys),len);
+}
+
+/* Initialiser */
+
+static void
+ofw_real_init(ofw_t ofw, void *openfirm)
+{
+	openfirmware = (int (*)(void *))openfirm;
+
+	mtx_init(&of_bounce_mtx, "OF Bounce Page", MTX_DEF, 0);
+	of_bounce_virt = NULL;
+}
+
+/*
+ * Generic functions
+ */
+
+/* Test to see if a service exists. */
+static int
+ofw_real_test(ofw_t ofw, const char *name)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t service;
+		cell_t missing;
+	} args = {
+		(cell_t)"test",
+		1,
+		1,
+	};
+
+	ofw_real_start();
+
+	args.service = ofw_real_map(name, strlen(name) + 1);
+	if (args.service == 0 || openfirmware(&args) == -1) {
+		ofw_real_stop();
+		return (-1);
+	}
+	ofw_real_stop();
+	return (args.missing);
+}
+
+/*
+ * Device tree functions
+ */
+
+/* Return the next sibling of this node or 0. */
+static phandle_t
+ofw_real_peer(ofw_t ofw, phandle_t node)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t node;
+		cell_t next;
+	} args = {
+		(cell_t)"peer",
+		1,
+		1,
+	};
+
+	args.node = node;
+	if (openfirmware(&args) == -1)
+		return (-1);
+	return (args.next);
+}
+
+/* Return the first child of this node or 0. */
+static phandle_t
+ofw_real_child(ofw_t ofw, phandle_t node)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t node;
+		cell_t child;
+	} args = {
+		(cell_t)"child",
+		1,
+		1,
+	};
+
+	args.node = node;
+	if (openfirmware(&args) == -1)
+		return (-1);
+	return (args.child);
+}
+
+/* Return the parent of this node or 0. */
+static phandle_t
+ofw_real_parent(ofw_t ofw, phandle_t node)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t node;
+		cell_t parent;
+	} args = {
+		(cell_t)"parent",
+		1,
+		1,
+	};
+
+	args.node = node;
+	if (openfirmware(&args) == -1)
+		return (-1);
+	return (args.parent);
+}
+
+/* Return the package handle that corresponds to an instance handle. */
+static phandle_t
+ofw_real_instance_to_package(ofw_t ofw, ihandle_t instance)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t instance;
+		cell_t package;
+	} args = {
+		(cell_t)"instance-to-package",
+		1,
+		1,
+	};
+
+	args.instance = instance;
+	if (openfirmware(&args) == -1)
+		return (-1);
+	return (args.package);
+}
+
+/* Get the length of a property of a package. */
+static ssize_t
+ofw_real_getproplen(ofw_t ofw, phandle_t package, const char *propname)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t package;
+		cell_t propname;
+		cell_t proplen;
+	} args = {
+		(cell_t)"getproplen",
+		2,
+		1,
+	};
+
+	ofw_real_start();
+
+	args.package = package;
+	args.propname = ofw_real_map(propname, strlen(propname) + 1);
+	if (args.propname == 0 || openfirmware(&args) == -1) {
+		ofw_real_stop();
+		return (-1);
+	}
+	ofw_real_stop();
+	return (args.proplen);
+}
+
+/* Get the value of a property of a package. */
+static ssize_t
+ofw_real_getprop(ofw_t ofw, phandle_t package, const char *propname, void *buf, 
+    size_t buflen)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t package;
+		cell_t propname;
+		cell_t buf;
+		cell_t buflen;
+		cell_t size;
+	} args = {
+		(cell_t)"getprop",
+		4,
+		1,
+	};
+
+	ofw_real_start();
+
+	args.package = package;
+	args.propname = ofw_real_map(propname, strlen(propname) + 1);
+	args.buf = ofw_real_map(buf, buflen);
+	args.buflen = buflen;
+	if (args.propname == 0 || args.buf == 0 || openfirmware(&args) == -1) {
+		ofw_real_stop();
+		return (-1);
+	}
+	ofw_real_unmap(args.buf, buf, buflen);
+
+	ofw_real_stop();
+	return (args.size);
+}
+
+/* Get the next property of a package. */
+static int
+ofw_real_nextprop(ofw_t ofw, phandle_t package, const char *previous, 
+    char *buf, size_t size)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t package;
+		cell_t previous;
+		cell_t buf;
+		cell_t flag;
+	} args = {
+		(cell_t)"nextprop",
+		3,
+		1,
+	};
+
+	ofw_real_start();
+
+	args.package = package;
+	args.previous = ofw_real_map(previous, strlen(previous) + 1);
+	args.buf = ofw_real_map(buf, size);
+	if (args.previous == 0 || args.buf == 0 || openfirmware(&args) == -1) {
+		ofw_real_stop();
+		return (-1);
+	}
+	ofw_real_unmap(args.buf, buf, size);
+
+	ofw_real_stop();
+	return (args.flag);
+}
+
+/* Set the value of a property of a package. */
+/* XXX Has a bug on FirePower */
+static int
+ofw_real_setprop(ofw_t ofw, phandle_t package, char *propname, void *buf, 
+    size_t len)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t package;
+		cell_t propname;
+		cell_t buf;
+		cell_t len;
+		cell_t size;
+	} args = {
+		(cell_t)"setprop",
+		4,
+		1,
+	};
+
+	ofw_real_start();
+
+	args.package = package;
+	args.propname = ofw_real_map(propname, strlen(propname) + 1);
+	args.buf = ofw_real_map(buf, len);
+	args.len = len;
+	if (args.propname == 0 || args.buf == 0 || openfirmware(&args) == -1) {
+		ofw_real_stop();
+		return (-1);
+	}
+	ofw_real_stop();
+	return (args.size);
+}
+
+/* Convert a device specifier to a fully qualified pathname. */
+static ssize_t
+ofw_real_canon(ofw_t ofw, const char *device, char *buf, size_t len)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t device;
+		cell_t buf;
+		cell_t len;
+		cell_t size;
+	} args = {
+		(cell_t)"canon",
+		3,
+		1,
+	};
+
+	ofw_real_start();
+
+	args.device = ofw_real_map(device, strlen(device) + 1);
+	args.buf = ofw_real_map(buf, len);
+	args.len = len;
+	if (args.device == 0 || args.buf == 0 || openfirmware(&args) == -1) {
+		ofw_real_stop();
+		return (-1);
+	}
+	ofw_real_unmap(args.buf, buf, len);
+
+	ofw_real_stop();
+	return (args.size);
+}
+
+/* Return a package handle for the specified device. */
+static phandle_t
+ofw_real_finddevice(ofw_t ofw, const char *device)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t device;
+		cell_t package;
+	} args = {
+		(cell_t)"finddevice",
+		1,
+		1,
+	};
+
+	ofw_real_start();
+
+	args.device = ofw_real_map(device, strlen(device) + 1);
+	if (args.device == 0 || openfirmware(&args) == -1) {
+		ofw_real_stop();
+		return (-1);
+	}
+	ofw_real_stop();
+	return (args.package);
+}
+
+/* Return the fully qualified pathname corresponding to an instance. */
+static ssize_t
+ofw_real_instance_to_path(ofw_t ofw, ihandle_t instance, char *buf, size_t len)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t instance;
+		cell_t buf;
+		cell_t len;
+		cell_t size;
+	} args = {
+		(cell_t)"instance-to-path",
+		3,
+		1,
+	};
+
+	ofw_real_start();
+
+	args.instance = instance;
+	args.buf = ofw_real_map(buf, len);
+	args.len = len;
+	if (args.buf == 0 || openfirmware(&args) == -1) {
+		ofw_real_stop();
+		return (-1);
+	}
+	ofw_real_unmap(args.buf, buf, len);
+
+	ofw_real_stop();
+	return (args.size);
+}
+
+/* Return the fully qualified pathname corresponding to a package. */
+static ssize_t
+ofw_real_package_to_path(ofw_t ofw, phandle_t package, char *buf, size_t len)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t package;
+		cell_t buf;
+		cell_t len;
+		cell_t size;
+	} args = {
+		(cell_t)"package-to-path",
+		3,
+		1,
+	};
+
+	ofw_real_start();
+
+	args.package = package;
+	args.buf = ofw_real_map(buf, len);
+	args.len = len;
+	if (args.buf == 0 || openfirmware(&args) == -1) {
+		ofw_real_stop();
+		return (-1);
+	}
+	ofw_real_unmap(args.buf, buf, len);
+
+	ofw_real_stop();
+	return (args.size);
+}
+
+/*  Call the method in the scope of a given instance. */
+static int
+ofw_real_call_method(ofw_t ofw, ihandle_t instance, const char *method, 
+    int nargs, int nreturns, unsigned long *args_and_returns)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t method;
+		cell_t instance;
+		cell_t args_n_results[12];
+	} args = {
+		(cell_t)"call-method",
+		2,
+		1,
+	};
+	cell_t *cp;
+	unsigned long *ap;
+	int n;
+
+	if (nargs > 6)
+		return (-1);
+
+	ofw_real_start();
+	args.nargs = nargs + 2;
+	args.nreturns = nreturns + 1;
+	args.method = ofw_real_map(method, strlen(method) + 1);
+	args.instance = instance;
+
+	ap = args_and_returns;
+	for (cp = args.args_n_results + (n = nargs); --n >= 0;)
+		*--cp = *(ap++);
+	if (args.method == 0 || openfirmware(&args) == -1) {
+		ofw_real_stop();
+		return (-1);
+	}
+	ofw_real_stop();
+	if (args.args_n_results[nargs])
+		return (args.args_n_results[nargs]);
+	for (cp = args.args_n_results + nargs + (n = args.nreturns); --n > 0;)
+		*(ap++) = *--cp;
+	return (0);
+}
+
+/*
+ * Device I/O functions
+ */
+
+/* Open an instance for a device. */
+static ihandle_t
+ofw_real_open(ofw_t ofw, const char *device)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t device;
+		cell_t instance;
+	} args = {
+		(cell_t)"open",
+		1,
+		1,
+	};
+
+	ofw_real_start();
+
+	args.device = ofw_real_map(device, strlen(device) + 1);
+	if (args.device == 0 || openfirmware(&args) == -1 
+	    || args.instance == 0) {
+		ofw_real_stop();
+		return (-1);
+	}
+	ofw_real_stop();
+	return (args.instance);
+}
+
+/* Close an instance. */
+static void
+ofw_real_close(ofw_t ofw, ihandle_t instance)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t instance;
+	} args = {
+		(cell_t)"close",
+		1,
+	};
+
+	args.instance = instance;
+	openfirmware(&args);
+}
+
+/* Read from an instance. */
+static ssize_t
+ofw_real_read(ofw_t ofw, ihandle_t instance, void *addr, size_t len)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t instance;
+		cell_t addr;
+		cell_t len;
+		cell_t actual;
+	} args = {
+		(cell_t)"read",
+		3,
+		1,
+	};
+
+	ofw_real_start();
+
+	args.instance = instance;
+	args.addr = ofw_real_map(addr, len);
+	args.len = len;
+	if (args.addr == 0 || openfirmware(&args) == -1) {
+		ofw_real_stop();
+		return (-1);
+	}
+	ofw_real_unmap(args.addr, addr, len);
+
+	ofw_real_stop();
+	return (args.actual);
+}
+
+/* Write to an instance. */
+static ssize_t
+ofw_real_write(ofw_t ofw, ihandle_t instance, const void *addr, size_t len)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t instance;
+		cell_t addr;
+		cell_t len;
+		cell_t actual;
+	} args = {
+		(cell_t)"write",
+		3,
+		1,
+	};
+
+	ofw_real_start();
+
+	args.instance = instance;
+	args.addr = ofw_real_map(addr, len);
+	args.len = len;
+	if (args.addr == 0 || openfirmware(&args) == -1) {
+		ofw_real_stop();
+		return (-1);
+	}
+	ofw_real_stop();
+	return (args.actual);
+}
+
+/* Seek to a position. */
+static int
+ofw_real_seek(ofw_t ofw, ihandle_t instance, u_int64_t pos)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t instance;
+		cell_t poshi;
+		cell_t poslo;
+		cell_t status;
+	} args = {
+		(cell_t)"seek",
+		3,
+		1,
+	};
+
+	args.instance = instance;
+	args.poshi = pos >> 32;
+	args.poslo = pos;
+	if (openfirmware(&args) == -1)
+		return (-1);
+	return (args.status);
+}
+
+/*
+ * Memory functions
+ */
+
+/* Claim an area of memory. */
+static caddr_t
+ofw_real_claim(ofw_t ofw, void *virt, size_t size, u_int align)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t virt;
+		cell_t size;
+		cell_t align;
+		cell_t baseaddr;
+	} args = {
+		(cell_t)"claim",
+		3,
+		1,
+	};
+
+	args.virt = (cell_t)virt;
+	args.size = size;
+	args.align = align;
+	if (openfirmware(&args) == -1)
+		return ((void *)-1);
+	return ((void *)args.baseaddr);
+}
+
+/* Release an area of memory. */
+static void
+ofw_real_release(ofw_t ofw, void *virt, size_t size)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+		cell_t virt;
+		cell_t size;
+	} args = {
+		(cell_t)"release",
+		2,
+	};
+
+	args.virt = (cell_t)virt;
+	args.size = size;
+	openfirmware(&args);
+}
+
+/*
+ * Control transfer functions
+ */
+
+/* Suspend and drop back to the Open Firmware interface. */
+static void
+ofw_real_enter(ofw_t ofw)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+	} args = {
+		(cell_t)"enter",
+	};
+
+	openfirmware(&args);
+	/* We may come back. */
+}
+
+/* Shut down and drop back to the Open Firmware interface. */
+static void
+ofw_real_exit(ofw_t ofw)
+{
+	static struct {
+		cell_t name;
+		cell_t nargs;
+		cell_t nreturns;
+	} args = {
+		(cell_t)"exit",
+	};
+
+	openfirmware(&args);
+	for (;;)			/* just in case */
+		;
+}
+
diff --git a/sys/powerpc/ofw/ofw_syscons.c b/sys/powerpc/ofw/ofw_syscons.c
index 1080dad..5ab5939 100644
--- a/sys/powerpc/ofw/ofw_syscons.c
+++ b/sys/powerpc/ofw/ofw_syscons.c
@@ -216,6 +216,7 @@ ofwfb_configure(int flags)
         phandle_t chosen;
         ihandle_t stdout;
 	phandle_t node;
+	bus_addr_t fb_phys;
 	int depth;
 	int disable;
 	int len;
@@ -270,10 +271,16 @@ ofwfb_configure(int flags)
 	OF_getprop(node, "linebytes", &sc->sc_stride, sizeof(sc->sc_stride));
 
 	/*
-	 * XXX the physical address of the frame buffer is assumed to be
-	 * BAT-mapped so it can be accessed directly
+	 * Grab the physical address of the framebuffer, and then map it
+	 * into our memory space. If the MMU is not yet up, it will be
+	 * remapped for us when relocation turns on.
+	 *
+	 * XXX We assume #address-cells is 1 at this point.
 	 */
-	OF_getprop(node, "address", &sc->sc_addr, sizeof(sc->sc_addr));
+	OF_getprop(node, "address", &fb_phys, sizeof(fb_phys));
+
+	bus_space_map(&bs_be_tag, fb_phys, sc->sc_height * sc->sc_stride,
+	    0, &sc->sc_addr);
 
 	/*
 	 * Get the PCI addresses of the adapter. The node may be the
@@ -283,8 +290,8 @@ ofwfb_configure(int flags)
 	len = OF_getprop(node, "assigned-addresses", sc->sc_pciaddrs,
 	          sizeof(sc->sc_pciaddrs));
 	if (len == -1) {
-		len = OF_getprop(OF_parent(node), "assigned-addresses", sc->sc_pciaddrs,
-		          sizeof(sc->sc_pciaddrs));
+		len = OF_getprop(OF_parent(node), "assigned-addresses",
+		    sc->sc_pciaddrs, sizeof(sc->sc_pciaddrs));
 	}
 
 	if (len != -1) {
@@ -941,13 +948,17 @@ ofwfb_scidentify(driver_t *driver, device_t parent)
 static int
 ofwfb_scprobe(device_t dev)
 {
-	/* This is a fake device, so make sure there is no OF node for it */
-	if (ofw_bus_get_node(dev) != -1)
-		return ENXIO;
-	
+	int error;
+
 	device_set_desc(dev, "System console");
-	return (sc_probe_unit(device_get_unit(dev), 
-	    device_get_flags(dev) | SC_AUTODETECT_KBD));
+
+	error = sc_probe_unit(device_get_unit(dev), 
+	    device_get_flags(dev) | SC_AUTODETECT_KBD);
+	if (error != 0)
+		return (error);
+
+	/* This is a fake device, so make sure we added it ourselves */
+	return (BUS_PROBE_NOWILDCARD);
 }
 
 static int
diff --git a/sys/powerpc/powermac/ata_macio.c b/sys/powerpc/powermac/ata_macio.c
index 482f76e..320e86e 100644
--- a/sys/powerpc/powermac/ata_macio.c
+++ b/sys/powerpc/powermac/ata_macio.c
@@ -259,6 +259,10 @@ ata_macio_setmode(device_t parent, device_t dev)
 
 	mode = ata_limit_mode(dev, mode, sc->max_mode);
 
+	/* XXX Some controllers don't work correctly with ATAPI DMA */
+	if (atadev->param.config & ATA_PROTO_ATAPI)
+		mode = ata_limit_mode(dev, mode, ATA_PIO_MAX);
+
 	if (ata_controlcmd(dev, ATA_SETFEATURES, ATA_SF_SETXFER, 0, mode))
 		return;
 
diff --git a/sys/powerpc/powermac/cpcht.c b/sys/powerpc/powermac/cpcht.c
new file mode 100644
index 0000000..b8ab22f
--- /dev/null
+++ b/sys/powerpc/powermac/cpcht.c
@@ -0,0 +1,625 @@
+/*-
+ * Copyright (C) 2008 Nathan Whitehorn
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/module.h>
+#include <sys/bus.h>
+#include <sys/conf.h>
+#include <sys/kernel.h>
+
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_pci.h>
+
+#include <dev/pci/pcivar.h>
+#include <dev/pci/pcireg.h>
+
+#include <machine/bus.h>
+#include <machine/md_var.h>
+#include <machine/pio.h>
+#include <machine/resource.h>
+
+#include <sys/rman.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+#include <powerpc/powermac/cpchtvar.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include "pcib_if.h"
+
+#include "opt_isa.h"
+
+#ifdef DEV_ISA
+#include <isa/isavar.h>
+#endif
+
+static MALLOC_DEFINE(M_CPCHT, "cpcht", "CPC HT device information");
+
+/*
+ * HT Driver methods.
+ */
+static int		cpcht_probe(device_t);
+static int		cpcht_attach(device_t);
+static ofw_bus_get_devinfo_t cpcht_get_devinfo;
+
+
+static device_method_t	cpcht_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		cpcht_probe),
+	DEVMETHOD(device_attach,	cpcht_attach),
+
+	/* Bus interface */
+	DEVMETHOD(bus_print_child,	bus_generic_print_child),
+	DEVMETHOD(bus_read_ivar,	bus_generic_read_ivar),
+	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
+	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
+	DEVMETHOD(bus_alloc_resource,	bus_generic_alloc_resource),
+	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
+	DEVMETHOD(bus_activate_resource,bus_generic_activate_resource),
+
+	/* ofw_bus interface */
+	DEVMETHOD(ofw_bus_get_devinfo,  cpcht_get_devinfo),
+	DEVMETHOD(ofw_bus_get_compat,   ofw_bus_gen_get_compat),
+	DEVMETHOD(ofw_bus_get_model,    ofw_bus_gen_get_model),
+	DEVMETHOD(ofw_bus_get_name,     ofw_bus_gen_get_name),
+	DEVMETHOD(ofw_bus_get_node,     ofw_bus_gen_get_node),
+	DEVMETHOD(ofw_bus_get_type,     ofw_bus_gen_get_type),
+
+	{ 0, 0 }
+};
+
+static driver_t	cpcht_driver = {
+	"cpcht",
+	cpcht_methods,
+	0
+};
+
+static devclass_t	cpcht_devclass;
+
+DRIVER_MODULE(cpcht, nexus, cpcht_driver, cpcht_devclass, 0, 0);
+
+static int 
+cpcht_probe(device_t dev) 
+{
+	const char	*type, *compatible;
+
+	type = ofw_bus_get_type(dev);
+	compatible = ofw_bus_get_compat(dev);
+
+	if (type == NULL || compatible == NULL)
+		return (ENXIO);
+
+	if (strcmp(type, "ht") != 0)
+		return (ENXIO);
+
+	if (strcmp(compatible, "u3-ht") == 0) {
+		device_set_desc(dev, "IBM CPC925 HyperTransport Tunnel");
+		return (0);
+	} else if (strcmp(compatible,"u4-ht") == 0) {
+		device_set_desc(dev, "IBM CPC945 HyperTransport Tunnel");
+		return (0);
+	}
+
+	return (ENXIO);
+}
+
+static int 
+cpcht_attach(device_t dev) 
+{
+	phandle_t root, child;
+	device_t cdev;
+	struct ofw_bus_devinfo *dinfo;
+	u_int32_t reg[6];
+
+	root = ofw_bus_get_node(dev);
+
+	if (OF_getprop(root, "reg", reg, sizeof(reg)) < 8)
+		return (ENXIO);
+
+	for (child = OF_child(root); child != 0; child = OF_peer(child)) {
+		dinfo = malloc(sizeof(*dinfo), M_CPCHT, M_WAITOK | M_ZERO);
+
+                if (ofw_bus_gen_setup_devinfo(dinfo, child) != 0) {
+                        free(dinfo, M_CPCHT);
+                        continue;
+                }
+                cdev = device_add_child(dev, NULL, -1);
+                if (cdev == NULL) {
+                        device_printf(dev, "<%s>: device_add_child failed\n",
+                            dinfo->obd_name);
+                        ofw_bus_gen_destroy_devinfo(dinfo);
+                        free(dinfo, M_CPCHT);
+                        continue;
+                }
+		device_set_ivars(cdev, dinfo);
+	}
+
+	return (bus_generic_attach(dev));
+}
+
+static const struct ofw_bus_devinfo *
+cpcht_get_devinfo(device_t dev, device_t child) 
+{
+	return (device_get_ivars(child));	
+}
+
+#ifdef DEV_ISA
+
+/*
+ * CPC ISA Device interface.
+ */
+static int		cpcisa_probe(device_t);
+
+/*
+ * Driver methods.
+ */
+static device_method_t	cpcisa_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		cpcisa_probe),
+	DEVMETHOD(device_attach,	isab_attach),
+
+	/* Bus interface */
+	DEVMETHOD(bus_print_child,	bus_generic_print_child),
+	DEVMETHOD(bus_read_ivar,	bus_generic_read_ivar),
+	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
+	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
+	DEVMETHOD(bus_alloc_resource,	bus_generic_alloc_resource),
+	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
+	DEVMETHOD(bus_activate_resource,bus_generic_activate_resource),
+
+	{0,0}
+};
+
+static driver_t	cpcisa_driver = {
+	"isab",
+	cpcisa_methods,
+	0
+};
+
+DRIVER_MODULE(cpcisa, cpcht, cpcisa_driver, isab_devclass, 0, 0);
+
+static int
+cpcisa_probe(device_t dev)
+{
+	const char	*type;
+
+	type = ofw_bus_get_type(dev);
+
+	if (type == NULL)
+		return (ENXIO);
+
+	if (strcmp(type, "isa") != 0)
+		return (ENXIO);
+
+	device_set_desc(dev, "HyperTransport-ISA bridge");
+	
+	return (0);
+}
+
+#endif /* DEV_ISA */
+
+/*
+ * CPC PCI Device interface.
+ */
+static int		cpcpci_probe(device_t);
+static int		cpcpci_attach(device_t);
+
+/*
+ * Bus interface.
+ */
+static int		cpcpci_read_ivar(device_t, device_t, int,
+			    uintptr_t *);
+static struct		resource * cpcpci_alloc_resource(device_t bus,
+			    device_t child, int type, int *rid, u_long start,
+			    u_long end, u_long count, u_int flags);
+static int		cpcpci_activate_resource(device_t bus, device_t child,
+			    int type, int rid, struct resource *res);
+
+/*
+ * pcib interface.
+ */
+static int		cpcpci_maxslots(device_t);
+static u_int32_t	cpcpci_read_config(device_t, u_int, u_int, u_int,
+			    u_int, int);
+static void		cpcpci_write_config(device_t, u_int, u_int, u_int,
+			    u_int, u_int32_t, int);
+static int		cpcpci_route_interrupt(device_t, device_t, int);
+
+/*
+ * ofw_bus interface
+ */
+
+static phandle_t	cpcpci_get_node(device_t bus, device_t child);
+
+/*
+ * Driver methods.
+ */
+static device_method_t	cpcpci_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		cpcpci_probe),
+	DEVMETHOD(device_attach,	cpcpci_attach),
+
+	/* Bus interface */
+	DEVMETHOD(bus_print_child,	bus_generic_print_child),
+	DEVMETHOD(bus_read_ivar,	cpcpci_read_ivar),
+	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
+	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
+	DEVMETHOD(bus_alloc_resource,	cpcpci_alloc_resource),
+	DEVMETHOD(bus_activate_resource,	cpcpci_activate_resource),
+
+	/* pcib interface */
+	DEVMETHOD(pcib_maxslots,	cpcpci_maxslots),
+	DEVMETHOD(pcib_read_config,	cpcpci_read_config),
+	DEVMETHOD(pcib_write_config,	cpcpci_write_config),
+	DEVMETHOD(pcib_route_interrupt,	cpcpci_route_interrupt),
+
+	/* ofw_bus interface */
+	DEVMETHOD(ofw_bus_get_node,     cpcpci_get_node),
+	{ 0, 0 }
+};
+
+static driver_t	cpcpci_driver = {
+	"pcib",
+	cpcpci_methods,
+	sizeof(struct cpcpci_softc)
+};
+
+static devclass_t	cpcpci_devclass;
+
+DRIVER_MODULE(cpcpci, cpcht, cpcpci_driver, cpcpci_devclass, 0, 0);
+
+static int
+cpcpci_probe(device_t dev)
+{
+	const char	*type;
+
+	type = ofw_bus_get_type(dev);
+
+	if (type == NULL)
+		return (ENXIO);
+
+	if (strcmp(type, "pci") != 0)
+		return (ENXIO);
+
+	device_set_desc(dev, "HyperTransport-PCI bridge");
+	
+	return (0);
+}
+
+static int
+cpcpci_attach(device_t dev)
+{
+	struct		cpcpci_softc *sc;
+	phandle_t	node;
+	u_int32_t	reg[2], busrange[2], config_base;
+	struct		cpcpci_range *rp, *io, *mem[2];
+	struct		cpcpci_range fakeio;
+	int		nmem, i;
+
+	node = ofw_bus_get_node(dev);
+	sc = device_get_softc(dev);
+
+	if (OF_getprop(OF_parent(node), "reg", reg, sizeof(reg)) < 8)
+		return (ENXIO);
+
+	if (OF_getprop(node, "bus-range", busrange, sizeof(busrange)) != 8)
+		return (ENXIO);
+
+	sc->sc_dev = dev;
+	sc->sc_node = node;
+	sc->sc_bus = busrange[0];
+	config_base = reg[1];
+	if (sc->sc_bus)
+		config_base += 0x01000000UL + (sc->sc_bus << 16);
+	sc->sc_data = (vm_offset_t)pmap_mapdev(config_base, PAGE_SIZE << 4);
+
+	bzero(sc->sc_range, sizeof(sc->sc_range));
+	sc->sc_nrange = OF_getprop(node, "ranges", sc->sc_range,
+	    sizeof(sc->sc_range));
+
+	if (sc->sc_nrange == -1) {
+		device_printf(dev, "could not get ranges\n");
+		return (ENXIO);
+	}
+
+	sc->sc_range[6].pci_hi = 0;
+	io = NULL;
+	nmem = 0;
+
+	for (rp = sc->sc_range; rp->pci_hi != 0; rp++) {
+		switch (rp->pci_hi & OFW_PCI_PHYS_HI_SPACEMASK) {
+		case OFW_PCI_PHYS_HI_SPACE_CONFIG:
+			break;
+		case OFW_PCI_PHYS_HI_SPACE_IO:
+			io = rp;
+			break;
+		case OFW_PCI_PHYS_HI_SPACE_MEM32:
+			mem[nmem] = rp;
+			nmem++;
+			break;
+		case OFW_PCI_PHYS_HI_SPACE_MEM64:
+			break;
+		}
+	}
+
+	if (io == NULL) {
+		/*
+		 * On at least some machines, the I/O port range is
+		 * not exported in the OF device tree. So hardcode it.
+		 */
+
+		fakeio.host_lo = 0;
+		fakeio.pci_lo = reg[1];
+		fakeio.size_lo = 0x00400000;
+		if (sc->sc_bus)
+			fakeio.pci_lo += 0x02000000UL + (sc->sc_bus << 14);
+		io = &fakeio;
+	}
+	sc->sc_io_rman.rm_type = RMAN_ARRAY;
+	sc->sc_io_rman.rm_descr = "CPC 9xx PCI I/O Ports";
+	sc->sc_iostart = io->host_lo;
+	if (rman_init(&sc->sc_io_rman) != 0 ||
+	    rman_manage_region(&sc->sc_io_rman, io->pci_lo,
+	    io->pci_lo + io->size_lo - 1) != 0) {
+		device_printf(dev, "failed to set up io range management\n");
+		return (ENXIO);
+	}
+
+	if (nmem == 0) {
+		device_printf(dev, "can't find mem ranges\n");
+		return (ENXIO);
+	}
+	sc->sc_mem_rman.rm_type = RMAN_ARRAY;
+	sc->sc_mem_rman.rm_descr = "CPC 9xx PCI Memory";
+	if (rman_init(&sc->sc_mem_rman) != 0) {
+		device_printf(dev,
+		    "failed to init mem range resources\n");
+		return (ENXIO);
+	}
+	for (i = 0; i < nmem; i++) {
+		if (rman_manage_region(&sc->sc_mem_rman, mem[i]->pci_lo,
+		    mem[i]->pci_lo + mem[i]->size_lo - 1) != 0) {
+			device_printf(dev,
+			    "failed to set up memory range management\n");
+			return (ENXIO);
+		}
+	}
+
+	ofw_bus_setup_iinfo(node, &sc->sc_pci_iinfo, sizeof(cell_t));
+
+	device_add_child(dev, "pci", device_get_unit(dev));
+
+	return (bus_generic_attach(dev));
+}
+
+static int
+cpcpci_maxslots(device_t dev)
+{
+
+	return (PCI_SLOTMAX);
+}
+
+static u_int32_t
+cpcpci_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
+    int width)
+{
+	struct		cpcpci_softc *sc;
+	vm_offset_t	caoff;
+
+	sc = device_get_softc(dev);
+	caoff = sc->sc_data + 
+		(((((slot & 0x1f) << 3) | (func & 0x07)) << 8) | reg);
+
+	switch (width) {
+	case 1:
+		return (in8rb(caoff));
+		break;
+	case 2:
+		return (in16rb(caoff));
+		break;
+	case 4:
+		return (in32rb(caoff));
+		break;
+	}
+
+	return (0xffffffff);
+}
+
+static void
+cpcpci_write_config(device_t dev, u_int bus, u_int slot, u_int func,
+    u_int reg, u_int32_t val, int width)
+{
+	struct		cpcpci_softc *sc;
+	vm_offset_t	caoff;
+
+	sc = device_get_softc(dev);
+	caoff = sc->sc_data + 
+		(((((slot & 0x1f) << 3) | (func & 0x07)) << 8) | reg);
+
+	switch (width) {
+	case 1:
+		out8rb(caoff, val);
+		break;
+	case 2:
+		out16rb(caoff, val);
+		break;
+	case 4:
+		out32rb(caoff, val);
+		break;
+	}
+}
+
+static int
+cpcpci_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
+{
+	struct	cpcpci_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	switch (which) {
+	case PCIB_IVAR_DOMAIN:
+		*result = device_get_unit(dev);
+		return (0);
+	case PCIB_IVAR_BUS:
+		*result = sc->sc_bus;
+		return (0);
+	}
+
+	return (ENOENT);
+}
+
+static struct resource *
+cpcpci_alloc_resource(device_t bus, device_t child, int type, int *rid,
+    u_long start, u_long end, u_long count, u_int flags)
+{
+	struct			cpcpci_softc *sc;
+	struct			resource *rv;
+	struct			rman *rm;
+	int			needactivate;
+
+	needactivate = flags & RF_ACTIVE;
+	flags &= ~RF_ACTIVE;
+
+	sc = device_get_softc(bus);
+
+	switch (type) {
+	case SYS_RES_MEMORY:
+		rm = &sc->sc_mem_rman;
+		break;
+
+	case SYS_RES_IOPORT:
+		rm = &sc->sc_io_rman;
+		if (rm == NULL)
+			return (NULL);
+		break;
+
+	case SYS_RES_IRQ:
+		return (bus_alloc_resource(bus, type, rid, start, end, count,
+		    flags));
+
+	default:
+		device_printf(bus, "unknown resource request from %s\n",
+		    device_get_nameunit(child));
+		return (NULL);
+	}
+
+	rv = rman_reserve_resource(rm, start, end, count, flags, child);
+	if (rv == NULL) {
+		device_printf(bus, "failed to reserve resource for %s\n",
+		    device_get_nameunit(child));
+		return (NULL);
+	}
+
+	rman_set_rid(rv, *rid);
+
+	if (needactivate) {
+		if (bus_activate_resource(child, type, *rid, rv) != 0) {
+			device_printf(bus,
+			    "failed to activate resource for %s\n",
+			    device_get_nameunit(child));
+			rman_release_resource(rv);
+			return (NULL);
+		}
+	}
+
+	return (rv);
+}
+
+static int
+cpcpci_activate_resource(device_t bus, device_t child, int type, int rid,
+    struct resource *res)
+{
+	void	*p;
+	struct	cpcpci_softc *sc;
+
+	sc = device_get_softc(bus);
+
+	if (type == SYS_RES_IRQ)
+		return (bus_activate_resource(bus, type, rid, res));
+
+	if (type == SYS_RES_MEMORY || type == SYS_RES_IOPORT) {
+		vm_offset_t start;
+
+		start = (vm_offset_t)rman_get_start(res);
+		/*
+		 * For i/o-ports, convert the start address to the
+		 * CPC PCI i/o window
+		 */
+		if (type == SYS_RES_IOPORT)
+			start += sc->sc_iostart;
+
+		if (bootverbose)
+			printf("cpcpci mapdev: start %x, len %ld\n", start,
+			    rman_get_size(res));
+
+		p = pmap_mapdev(start, (vm_size_t)rman_get_size(res));
+		if (p == NULL)
+			return (ENOMEM);
+		rman_set_virtual(res, p);
+		rman_set_bustag(res, &bs_le_tag);
+		rman_set_bushandle(res, (u_long)p);
+	}
+
+	return (rman_activate_resource(res));
+}
+
+static phandle_t
+cpcpci_get_node(device_t bus, device_t dev)
+{
+	struct cpcpci_softc *sc;
+
+	sc = device_get_softc(bus);
+	/* We only have one child, the PCI bus, which needs our own node. */
+	return (sc->sc_node);
+}
+
+static int
+cpcpci_route_interrupt(device_t bus, device_t dev, int pin)
+{
+	struct cpcpci_softc *sc;
+	struct ofw_pci_register reg;
+	uint32_t pintr, mintr;
+	uint8_t maskbuf[sizeof(reg) + sizeof(pintr)];
+
+	sc = device_get_softc(bus);
+	pintr = pin;
+	if (ofw_bus_lookup_imap(ofw_bus_get_node(dev), &sc->sc_pci_iinfo, &reg,
+	    sizeof(reg), &pintr, sizeof(pintr), &mintr, sizeof(mintr), maskbuf))
+		return (mintr);
+
+	/* Maybe it's a real interrupt, not an intpin */
+	if (pin > 4)
+		return (pin);
+
+	device_printf(bus, "could not route pin %d for device %d.%d\n",
+	    pin, pci_get_slot(dev), pci_get_function(dev));
+	return (PCI_INVALID_IRQ);
+}
+
diff --git a/sys/powerpc/powermac/cpchtvar.h b/sys/powerpc/powermac/cpchtvar.h
new file mode 100644
index 0000000..270345f
--- /dev/null
+++ b/sys/powerpc/powermac/cpchtvar.h
@@ -0,0 +1,58 @@
+/*-
+ * Copyright (C) 2008 Nathan Whitehorn
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_POWERPC_POWERMAC_CPCHTVAR_H_
+#define	_POWERPC_POWERMAC_CPCHTVAR_H_
+
+struct cpcpci_range {
+	u_int32_t	pci_hi;
+	u_int32_t	pci_mid;
+	u_int32_t	pci_lo;
+	u_int32_t	junk;
+	u_int32_t	host_hi;
+	u_int32_t	host_lo;
+	u_int32_t	size_hi;
+	u_int32_t	size_lo;
+};
+
+struct cpcpci_softc {
+	device_t		sc_dev;
+	phandle_t		sc_node;
+	vm_offset_t		sc_data;
+	int			sc_bus;
+	struct			cpcpci_range sc_range[6];
+	int			sc_nrange;
+	int			sc_iostart;
+	struct			rman sc_io_rman;
+	struct			rman sc_mem_rman;
+	bus_space_tag_t		sc_iot;
+	bus_space_tag_t		sc_memt;
+	bus_dma_tag_t		sc_dmat;
+	struct ofw_bus_iinfo	sc_pci_iinfo;
+};
+
+#endif  /* _POWERPC_POWERMAC_CPCHTVAR_H_ */
diff --git a/sys/powerpc/powerpc/bus_machdep.c b/sys/powerpc/powerpc/bus_machdep.c
index 42e5baa..742454b 100644
--- a/sys/powerpc/powerpc/bus_machdep.c
+++ b/sys/powerpc/powerpc/bus_machdep.c
@@ -42,12 +42,25 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
 
 #include <machine/bus.h>
 #include <machine/pio.h>
+#include <machine/md_var.h>
 
 #define TODO panic("%s: not implemented", __func__)
 
+#define	MAX_EARLYBOOT_MAPPINGS	6
+
+static struct {
+	bus_addr_t addr;
+	bus_size_t size;
+} earlyboot_mappings[MAX_EARLYBOOT_MAPPINGS];
+static int earlyboot_map_idx = 0;
+
+void bs_remap_earlyboot(void);
+
 static __inline void *
 __ppc_ba(bus_space_handle_t bsh, bus_size_t ofs)
 {
@@ -58,10 +71,44 @@ static int
 bs_gen_map(bus_addr_t addr, bus_size_t size __unused, int flags __unused,
     bus_space_handle_t *bshp)
 {
-	*bshp = addr;
+	/*
+	 * Record what we did if we haven't enabled the MMU yet. We
+	 * will need to remap it as soon as the MMU comes up.
+	 */
+	if (!pmap_bootstrapped) {
+		KASSERT(earlyboot_map_idx < MAX_EARLYBOOT_MAPPINGS,
+		    ("%s: too many early boot mapping requests", __func__));
+		earlyboot_mappings[earlyboot_map_idx].addr = addr;
+		earlyboot_mappings[earlyboot_map_idx].size = size;
+		earlyboot_map_idx++;
+		*bshp = addr;
+	} else {
+		*bshp = (bus_space_handle_t)pmap_mapdev(addr,size);
+	}
+
 	return (0);
 }
 
+void
+bs_remap_earlyboot(void)
+{
+	int i;
+	vm_offset_t pa, spa;
+
+	if (hw_direct_map)
+		return;
+
+	for (i = 0; i < earlyboot_map_idx; i++) {
+		spa = earlyboot_mappings[i].addr;
+
+		pa = trunc_page(spa);
+		while (pa < spa + earlyboot_mappings[i].size) {
+			pmap_kenter(pa,pa);
+			pa += PAGE_SIZE;
+		}
+	}
+}
+
 static void
 bs_gen_unmap(bus_size_t size __unused)
 {
diff --git a/sys/powerpc/powerpc/cpu.c b/sys/powerpc/powerpc/cpu.c
index 4724313..a15ef58 100644
--- a/sys/powerpc/powerpc/cpu.c
+++ b/sys/powerpc/powerpc/cpu.c
@@ -91,6 +91,10 @@ static const struct cputab models[] = {
         { "Motorola PowerPC 620",	MPC620,		REVFMT_HEX },
         { "Motorola PowerPC 750",	MPC750,		REVFMT_MAJMIN },
         { "IBM PowerPC 750FX",		IBM750FX,	REVFMT_MAJMIN },
+        { "IBM PowerPC 970",		IBM970,		REVFMT_MAJMIN },
+        { "IBM PowerPC 970FX",		IBM970FX,	REVFMT_MAJMIN },
+        { "IBM PowerPC 970GX",		IBM970GX,	REVFMT_MAJMIN },
+        { "IBM PowerPC 970MP",		IBM970MP,	REVFMT_MAJMIN },
         { "Motorola PowerPC 7400",	MPC7400,	REVFMT_MAJMIN },
         { "Motorola PowerPC 7410",	MPC7410,	REVFMT_MAJMIN },
         { "Motorola PowerPC 7450",	MPC7450,	REVFMT_MAJMIN },
diff --git a/sys/powerpc/powerpc/dump_machdep.c b/sys/powerpc/powerpc/dump_machdep.c
new file mode 100644
index 0000000..acda560
--- /dev/null
+++ b/sys/powerpc/powerpc/dump_machdep.c
@@ -0,0 +1,303 @@
+/*-
+ * Copyright (c) 2002 Marcel Moolenaar
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/conf.h>
+#include <sys/cons.h>
+#include <sys/kernel.h>
+#include <sys/kerneldump.h>
+#include <sys/sysctl.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <machine/elf.h>
+#include <machine/md_var.h>
+
+CTASSERT(sizeof(struct kerneldumpheader) == 512);
+
+/*
+ * Don't touch the first SIZEOF_METADATA bytes on the dump device. This
+ * is to protect us from metadata and to protect metadata from us.
+ */
+#define	SIZEOF_METADATA		(64*1024)
+
+#define	MD_ALIGN(x)	(((off_t)(x) + PAGE_MASK) & ~PAGE_MASK)
+#define	DEV_ALIGN(x)	(((off_t)(x) + (DEV_BSIZE-1)) & ~(DEV_BSIZE-1))
+
+typedef int callback_t(struct pmap_md *, int, void *);
+
+static struct kerneldumpheader kdh;
+static off_t dumplo, fileofs;
+
+/* Handle buffered writes. */
+static char buffer[DEV_BSIZE];
+static size_t fragsz;
+
+int dumpsys_minidump = 1;
+SYSCTL_INT(_debug, OID_AUTO, minidump, CTLFLAG_RD, &dumpsys_minidump, 0,
+    "Kernel makes compressed crash dumps");
+
+static int
+buf_write(struct dumperinfo *di, char *ptr, size_t sz)
+{
+	size_t len;
+	int error;
+
+	while (sz) {
+		len = DEV_BSIZE - fragsz;
+		if (len > sz)
+			len = sz;
+		bcopy(ptr, buffer + fragsz, len);
+		fragsz += len;
+		ptr += len;
+		sz -= len;
+		if (fragsz == DEV_BSIZE) {
+			error = di->dumper(di->priv, buffer, 0, dumplo,
+			    DEV_BSIZE);
+			if (error)
+				return error;
+			dumplo += DEV_BSIZE;
+			fragsz = 0;
+		}
+	}
+
+	return (0);
+}
+
+static int
+buf_flush(struct dumperinfo *di)
+{
+	int error;
+
+	if (fragsz == 0)
+		return (0);
+
+	error = di->dumper(di->priv, buffer, 0, dumplo, DEV_BSIZE);
+	dumplo += DEV_BSIZE;
+	fragsz = 0;
+	return (error);
+}
+
+static int
+cb_dumpdata(struct pmap_md *md, int seqnr, void *arg)
+{
+	struct dumperinfo *di = (struct dumperinfo*)arg;
+	vm_offset_t va;
+	size_t counter, ofs, resid, sz;
+	int c, error, twiddle;
+
+	error = 0;
+	counter = 0;	/* Update twiddle every 16MB */
+	twiddle = 0;
+
+	ofs = 0;	/* Logical offset within the chunk */
+	resid = md->md_size;
+
+	printf("  chunk %d: %lu bytes ", seqnr, (u_long)resid);
+
+	while (resid) {
+		sz = (resid > DFLTPHYS) ? DFLTPHYS : resid;
+		va = pmap_dumpsys_map(md, ofs, &sz);
+		counter += sz;
+		if (counter >> 24) {
+			printf("%c\b", "|/-\\"[twiddle++ & 3]);
+			counter &= (1<<24) - 1;
+		}
+		error = di->dumper(di->priv, (void*)va, 0, dumplo, sz);
+		pmap_dumpsys_unmap(md, ofs, va);
+		if (error)
+			break;
+		dumplo += sz;
+		resid -= sz;
+		ofs += sz;
+
+		/* Check for user abort. */
+		c = cncheckc();
+		if (c == 0x03)
+			return (ECANCELED);
+		if (c != -1)
+			printf("(CTRL-C to abort)  ");
+	}
+	printf("... %s\n", (error) ? "fail" : "ok");
+	return (error);
+}
+
+static int
+cb_dumphdr(struct pmap_md *md, int seqnr, void *arg)
+{
+	struct dumperinfo *di = (struct dumperinfo*)arg;
+	Elf32_Phdr phdr;
+	int error;
+
+	bzero(&phdr, sizeof(phdr));
+	phdr.p_type = PT_LOAD;
+	phdr.p_flags = PF_R;			/* XXX */
+	phdr.p_offset = fileofs;
+	phdr.p_vaddr = md->md_vaddr;
+	phdr.p_paddr = md->md_paddr;
+	phdr.p_filesz = md->md_size;
+	phdr.p_memsz = md->md_size;
+	phdr.p_align = PAGE_SIZE;
+
+	error = buf_write(di, (char*)&phdr, sizeof(phdr));
+	fileofs += phdr.p_filesz;
+	return (error);
+}
+
+static int
+cb_size(struct pmap_md *md, int seqnr, void *arg)
+{
+	uint32_t *sz = (uint32_t*)arg;
+
+	*sz += md->md_size;
+	return (0);
+}
+
+static int
+foreach_chunk(callback_t cb, void *arg)
+{
+	struct pmap_md *md;
+	int error, seqnr;
+
+	seqnr = 0;
+	md = pmap_scan_md(NULL);
+	while (md != NULL) {
+		error = (*cb)(md, seqnr++, arg);
+		if (error)
+			return (-error);
+		md = pmap_scan_md(md);
+	}
+	return (seqnr);
+}
+
+void
+dumpsys(struct dumperinfo *di)
+{
+	Elf32_Ehdr ehdr;
+	uint32_t dumpsize;
+	off_t hdrgap;
+	size_t hdrsz;
+	int error;
+
+	bzero(&ehdr, sizeof(ehdr));
+	ehdr.e_ident[EI_MAG0] = ELFMAG0;
+	ehdr.e_ident[EI_MAG1] = ELFMAG1;
+	ehdr.e_ident[EI_MAG2] = ELFMAG2;
+	ehdr.e_ident[EI_MAG3] = ELFMAG3;
+	ehdr.e_ident[EI_CLASS] = ELFCLASS32;
+#if BYTE_ORDER == LITTLE_ENDIAN
+	ehdr.e_ident[EI_DATA] = ELFDATA2LSB;
+#else
+	ehdr.e_ident[EI_DATA] = ELFDATA2MSB;
+#endif
+	ehdr.e_ident[EI_VERSION] = EV_CURRENT;
+	ehdr.e_ident[EI_OSABI] = ELFOSABI_STANDALONE;	/* XXX big picture? */
+	ehdr.e_type = ET_CORE;
+	ehdr.e_machine = EM_PPC;
+	ehdr.e_phoff = sizeof(ehdr);
+	ehdr.e_ehsize = sizeof(ehdr);
+	ehdr.e_phentsize = sizeof(Elf32_Phdr);
+	ehdr.e_shentsize = sizeof(Elf32_Shdr);
+
+	/* Calculate dump size. */
+	dumpsize = 0L;
+	ehdr.e_phnum = foreach_chunk(cb_size, &dumpsize);
+	hdrsz = ehdr.e_phoff + ehdr.e_phnum * ehdr.e_phentsize;
+	fileofs = MD_ALIGN(hdrsz);
+	dumpsize += fileofs;
+	hdrgap = fileofs - DEV_ALIGN(hdrsz);
+
+	/* For block devices, determine the dump offset on the device. */
+	if (di->mediasize > 0) {
+		if (di->mediasize <
+		    SIZEOF_METADATA + dumpsize + sizeof(kdh) * 2) {
+			error = ENOSPC;
+			goto fail;
+		}
+		dumplo = di->mediaoffset + di->mediasize - dumpsize;
+		dumplo -= sizeof(kdh) * 2;
+	} else
+		dumplo = 0;
+
+	mkdumpheader(&kdh, KERNELDUMPMAGIC, KERNELDUMP_POWERPC_VERSION, dumpsize,
+	    di->blocksize);
+
+	printf("Dumping %u MB (%d chunks)\n", dumpsize >> 20,
+	    ehdr.e_phnum);
+
+	/* Dump leader */
+	error = di->dumper(di->priv, &kdh, 0, dumplo, sizeof(kdh));
+	if (error)
+		goto fail;
+	dumplo += sizeof(kdh);
+
+	/* Dump ELF header */
+	error = buf_write(di, (char*)&ehdr, sizeof(ehdr));
+	if (error)
+		goto fail;
+
+	/* Dump program headers */
+	error = foreach_chunk(cb_dumphdr, di);
+	if (error < 0)
+		goto fail;
+	buf_flush(di);
+
+	/*
+	 * All headers are written using blocked I/O, so we know the
+	 * current offset is (still) block aligned. Skip the alignement
+	 * in the file to have the segment contents aligned at page
+	 * boundary. We cannot use MD_ALIGN on dumplo, because we don't
+	 * care and may very well be unaligned within the dump device.
+	 */
+	dumplo += hdrgap;
+
+	/* Dump memory chunks (updates dumplo) */
+	error = foreach_chunk(cb_dumpdata, di);
+	if (error < 0)
+		goto fail;
+
+	/* Dump trailer */
+	error = di->dumper(di->priv, &kdh, 0, dumplo, sizeof(kdh));
+	if (error)
+		goto fail;
+
+	/* Signal completion, signoff and exit stage left. */
+	di->dumper(di->priv, NULL, 0, 0, 0);
+	printf("\nDump complete\n");
+	return;
+
+ fail:
+	if (error < 0)
+		error = -error;
+
+	if (error == ECANCELED)
+		printf("\nDump aborted\n");
+	else
+		printf("\n** DUMP FAILED (ERROR %d) **\n", error);
+}
diff --git a/sys/powerpc/powerpc/elf_machdep.c b/sys/powerpc/powerpc/elf_machdep.c
index 69ac55b..4946b44 100644
--- a/sys/powerpc/powerpc/elf_machdep.c
+++ b/sys/powerpc/powerpc/elf_machdep.c
@@ -87,7 +87,8 @@ static Elf32_Brandinfo freebsd_brand_info = {
 	.interp_path	= "/libexec/ld-elf.so.1",
 	.sysvec		= &elf32_freebsd_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &elf32_freebsd_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 SYSINIT(elf32, SI_SUB_EXEC, SI_ORDER_ANY,
@@ -102,7 +103,8 @@ static Elf32_Brandinfo freebsd_brand_oinfo = {
 	.interp_path	= "/usr/libexec/ld-elf.so.1",
 	.sysvec		= &elf32_freebsd_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &elf32_freebsd_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 SYSINIT(oelf32, SI_SUB_EXEC, SI_ORDER_ANY,
diff --git a/sys/powerpc/powerpc/mem.c b/sys/powerpc/powerpc/mem.c
index b3c320b..22d99c4 100644
--- a/sys/powerpc/powerpc/mem.c
+++ b/sys/powerpc/powerpc/mem.c
@@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm.h>
 #include <vm/pmap.h>
 #include <vm/vm_extern.h>
+#include <vm/vm_page.h>
 
 #include <machine/memdev.h>
 
@@ -77,6 +78,8 @@ memrw(struct cdev *dev, struct uio *uio, int flags)
 	int error = 0;
 	vm_offset_t va, eva, off, v;
 	vm_prot_t prot;
+	struct vm_page m;
+	vm_page_t marr;
 	vm_size_t cnt;
 
 	cnt = 0;
@@ -102,14 +105,18 @@ kmem_direct_mapped:	v = uio->uio_offset;
 			cnt = min(cnt, PAGE_SIZE - off);
 			cnt = min(cnt, iov->iov_len);
 
-			if (mem_valid(v, cnt)
-			    && pmap_dev_direct_mapped(v, cnt)) {
+			if (mem_valid(v, cnt)) {
 				error = EFAULT;
 				break;
 			}
-
-			uiomove((void *)v, cnt, uio);
-			break;
+	
+			if (!pmap_dev_direct_mapped(v, cnt)) {
+				error = uiomove((void *)v, cnt, uio);
+			} else {
+				m.phys_addr = trunc_page(v);
+				marr = &m;
+				error = uiomove_fromphys(&marr, off, cnt, uio);
+			}
 		}
 		else if (dev2unit(dev) == CDEV_MINOR_KMEM) {
 			va = uio->uio_offset;
diff --git a/sys/powerpc/powerpc/mmu_if.m b/sys/powerpc/powerpc/mmu_if.m
index 4a8ffa8..4a5a37c 100644
--- a/sys/powerpc/powerpc/mmu_if.m
+++ b/sys/powerpc/powerpc/mmu_if.m
@@ -105,6 +105,11 @@ CODE {
 	{
 		return;
 	}
+
+	static struct pmap_md *mmu_null_scan_md(mmu_t mmu, struct pmap_md *p)
+	{
+		return (NULL);
+	}
 };
 
 
@@ -697,6 +702,18 @@ METHOD void bootstrap {
 	vm_offset_t	_end;
 };
 
+/**
+ * @brief Set up the MMU on the current CPU. Only called by the PMAP layer
+ * for alternate CPUs on SMP systems.
+ *
+ * @param _ap		Set to 1 if the CPU being set up is an AP
+ *
+ */
+METHOD void cpu_bootstrap {
+	mmu_t		_mmu;
+	int		_ap;
+};
+
 
 /**
  * @brief Create a kernel mapping for a given physical address range.
@@ -783,3 +800,50 @@ METHOD boolean_t page_executable {
 	vm_page_t	_pg;
 };
 
+
+/**
+ * @brief Create temporary memory mapping for use by dumpsys().
+ *
+ * @param _md		The memory chunk in which the mapping lies.
+ * @param _ofs		The offset within the chunk of the mapping.
+ * @param _sz		The requested size of the mapping.
+ *
+ * @retval vm_offset_t	The virtual address of the mapping.
+ *			
+ * The sz argument is modified to reflect the actual size of the
+ * mapping.
+ */
+METHOD vm_offset_t dumpsys_map {
+	mmu_t		_mmu;
+	struct pmap_md	*_md;
+	vm_size_t	_ofs;
+	vm_size_t	*_sz;
+};
+
+
+/**
+ * @brief Remove temporary dumpsys() mapping.
+ *
+ * @param _md		The memory chunk in which the mapping lies.
+ * @param _ofs		The offset within the chunk of the mapping.
+ * @param _va		The virtual address of the mapping.
+ */
+METHOD void dumpsys_unmap {
+	mmu_t		_mmu;
+	struct pmap_md	*_md;
+	vm_size_t	_ofs;
+	vm_offset_t	_va;
+};
+
+
+/**
+ * @brief Scan/iterate memory chunks.
+ *
+ * @param _prev		The previously returned chunk or NULL.
+ *
+ * @retval		The next (or first when _prev is NULL) chunk.
+ */
+METHOD struct pmap_md * scan_md {
+	mmu_t		_mmu;
+	struct pmap_md	*_prev;
+} DEFAULT mmu_null_scan_md;
diff --git a/sys/powerpc/powerpc/pmap_dispatch.c b/sys/powerpc/powerpc/pmap_dispatch.c
index 33e94d1..af657d4 100644
--- a/sys/powerpc/powerpc/pmap_dispatch.c
+++ b/sys/powerpc/powerpc/pmap_dispatch.c
@@ -52,6 +52,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_page.h>
 
 #include <machine/mmuvar.h>
+#include <machine/smp.h>
 
 #include "mmu_if.h"
 
@@ -406,6 +407,16 @@ pmap_bootstrap(vm_offset_t start, vm_offset_t end)
 	MMU_BOOTSTRAP(mmu_obj, start, end);
 }
 
+void
+pmap_cpu_bootstrap(int ap)
+{
+	/*
+	 * No KTR here because our console probably doesn't work yet
+	 */
+
+	return (MMU_CPU_BOOTSTRAP(mmu_obj, ap));
+}
+
 void *
 pmap_mapdev(vm_offset_t pa, vm_size_t size)
 {
@@ -454,6 +465,30 @@ pmap_page_executable(vm_page_t pg)
 	return (MMU_PAGE_EXECUTABLE(mmu_obj, pg));
 }
 
+vm_offset_t
+pmap_dumpsys_map(struct pmap_md *md, vm_size_t ofs, vm_size_t *sz)
+{
+
+	CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, md, ofs, *sz);
+	return (MMU_DUMPSYS_MAP(mmu_obj, md, ofs, sz));
+}
+
+void
+pmap_dumpsys_unmap(struct pmap_md *md, vm_size_t ofs, vm_offset_t va)
+{
+
+	CTR4(KTR_PMAP, "%s(%p, %#x, %#x)", __func__, md, ofs, va);
+	return (MMU_DUMPSYS_UNMAP(mmu_obj, md, ofs, va));
+}
+
+struct pmap_md *
+pmap_scan_md(struct pmap_md *prev)
+{
+
+	CTR2(KTR_PMAP, "%s(%p)", __func__, prev);
+	return (MMU_SCAN_MD(mmu_obj, prev));
+}
+
 /*
  * MMU install routines. Highest priority wins, equal priority also
  * overrides allowing last-set to win.
diff --git a/sys/powerpc/booke/uio_machdep.c b/sys/powerpc/powerpc/uio_machdep.c
index 2a88fd2..2a88fd2 100644
--- a/sys/powerpc/booke/uio_machdep.c
+++ b/sys/powerpc/powerpc/uio_machdep.c
diff --git a/sys/security/mac/mac_atalk.c b/sys/security/mac/mac_atalk.c
index 0992ee5..d45d29c 100644
--- a/sys/security/mac/mac_atalk.c
+++ b/sys/security/mac/mac_atalk.c
@@ -1,9 +1,12 @@
 /*-
- * Copyright (c) 2007 Robert N. M. Watson
+ * Copyright (c) 2007-2009 Robert N. M. Watson
  * All rights reserved.
  *
  * This software was developed by Robert Watson for the TrustedBSD Project.
  *
+ * This software was developed at the University of Cambridge Computer
+ * Laboratory with support from a grant from Google, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -61,6 +64,7 @@ mac_netatalk_aarp_send(struct ifnet *ifp, struct mbuf *m)
 	mlabel = mac_mbuf_to_label(m);
 
 	MAC_IFNET_LOCK(ifp);
-	MAC_PERFORM(netatalk_aarp_send, ifp, ifp->if_label, m, mlabel);
+	MAC_PERFORM_NOSLEEP(netatalk_aarp_send, ifp, ifp->if_label, m,
+	    mlabel);
 	MAC_IFNET_UNLOCK(ifp);
 }
diff --git a/sys/security/mac/mac_audit.c b/sys/security/mac/mac_audit.c
index 9bc2e27..62cfbdf 100644
--- a/sys/security/mac/mac_audit.c
+++ b/sys/security/mac/mac_audit.c
@@ -66,7 +66,7 @@ mac_cred_check_setaudit(struct ucred *cred, struct auditinfo *ai)
 {
 	int error;
 
-	MAC_CHECK(cred_check_setaudit, cred, ai);
+	MAC_CHECK_NOSLEEP(cred_check_setaudit, cred, ai);
 	MAC_CHECK_PROBE2(cred_check_setaudit, error, cred, ai);
 
 	return (error);
@@ -80,7 +80,7 @@ mac_cred_check_setaudit_addr(struct ucred *cred, struct auditinfo_addr *aia)
 {
 	int error;
 
-	MAC_CHECK(cred_check_setaudit_addr, cred, aia);
+	MAC_CHECK_NOSLEEP(cred_check_setaudit_addr, cred, aia);
 	MAC_CHECK_PROBE2(cred_check_setaudit_addr, error, cred, aia);
 
 	return (error);
@@ -93,7 +93,7 @@ mac_cred_check_setauid(struct ucred *cred, uid_t auid)
 {
 	int error;
 
-	MAC_CHECK(cred_check_setauid, cred, auid);
+	MAC_CHECK_NOSLEEP(cred_check_setauid, cred, auid);
 	MAC_CHECK_PROBE2(cred_check_setauid, error, cred, auid);
 
 	return (error);
@@ -107,7 +107,7 @@ mac_system_check_audit(struct ucred *cred, void *record, int length)
 {
 	int error;
 
-	MAC_CHECK(system_check_audit, cred, record, length);
+	MAC_CHECK_NOSLEEP(system_check_audit, cred, record, length);
 	MAC_CHECK_PROBE3(system_check_audit, error, cred, record, length);
 
 	return (error);
@@ -138,7 +138,7 @@ mac_system_check_auditon(struct ucred *cred, int cmd)
 {
 	int error;
 
-	MAC_CHECK(system_check_auditon, cred, cmd);
+	MAC_CHECK_NOSLEEP(system_check_auditon, cred, cmd);
 	MAC_CHECK_PROBE2(system_check_auditon, error, cred, cmd);
 
 	return (error);
diff --git a/sys/security/mac/mac_cred.c b/sys/security/mac/mac_cred.c
index 41c6e66..b6dcf9d 100644
--- a/sys/security/mac/mac_cred.c
+++ b/sys/security/mac/mac_cred.c
@@ -100,7 +100,7 @@ void
 mac_cred_label_free(struct label *label)
 {
 
-	MAC_PERFORM(cred_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(cred_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -127,7 +127,7 @@ void
 mac_cred_associate_nfsd(struct ucred *cred)
 {
 
-	MAC_PERFORM(cred_associate_nfsd, cred);
+	MAC_PERFORM_NOSLEEP(cred_associate_nfsd, cred);
 }
 
 /*
@@ -138,7 +138,7 @@ void
 mac_cred_create_swapper(struct ucred *cred)
 {
 
-	MAC_PERFORM(cred_create_swapper, cred);
+	MAC_PERFORM_NOSLEEP(cred_create_swapper, cred);
 }
 
 /*
@@ -149,7 +149,7 @@ void
 mac_cred_create_init(struct ucred *cred)
 {
 
-	MAC_PERFORM(cred_create_init, cred);
+	MAC_PERFORM_NOSLEEP(cred_create_init, cred);
 }
 
 int
@@ -182,7 +182,7 @@ void
 mac_cred_copy(struct ucred *src, struct ucred *dest)
 {
 
-	MAC_PERFORM(cred_copy_label, src->cr_label, dest->cr_label);
+	MAC_PERFORM_NOSLEEP(cred_copy_label, src->cr_label, dest->cr_label);
 }
 
 /*
@@ -194,7 +194,7 @@ void
 mac_cred_relabel(struct ucred *cred, struct label *newlabel)
 {
 
-	MAC_PERFORM(cred_relabel, cred, newlabel);
+	MAC_PERFORM_NOSLEEP(cred_relabel, cred, newlabel);
 }
 
 MAC_CHECK_PROBE_DEFINE2(cred_check_relabel, "struct ucred *",
@@ -205,7 +205,7 @@ mac_cred_check_relabel(struct ucred *cred, struct label *newlabel)
 {
 	int error;
 
-	MAC_CHECK(cred_check_relabel, cred, newlabel);
+	MAC_CHECK_NOSLEEP(cred_check_relabel, cred, newlabel);
 	MAC_CHECK_PROBE2(cred_check_relabel, error, cred, newlabel);
 
 	return (error);
@@ -218,7 +218,7 @@ mac_cred_check_setuid(struct ucred *cred, uid_t uid)
 {
 	int error;
 
-	MAC_CHECK(cred_check_setuid, cred, uid);
+	MAC_CHECK_NOSLEEP(cred_check_setuid, cred, uid);
 	MAC_CHECK_PROBE2(cred_check_setuid, error, cred, uid);
 
 	return (error);
@@ -231,7 +231,7 @@ mac_cred_check_seteuid(struct ucred *cred, uid_t euid)
 {
 	int error;
 
-	MAC_CHECK(cred_check_seteuid, cred, euid);
+	MAC_CHECK_NOSLEEP(cred_check_seteuid, cred, euid);
 	MAC_CHECK_PROBE2(cred_check_seteuid, error, cred, euid);
 
 	return (error);
@@ -244,7 +244,7 @@ mac_cred_check_setgid(struct ucred *cred, gid_t gid)
 {
 	int error;
 
-	MAC_CHECK(cred_check_setgid, cred, gid);
+	MAC_CHECK_NOSLEEP(cred_check_setgid, cred, gid);
 	MAC_CHECK_PROBE2(cred_check_setgid, error, cred, gid);
 
 	return (error);
@@ -257,7 +257,7 @@ mac_cred_check_setegid(struct ucred *cred, gid_t egid)
 {
 	int error;
 
-	MAC_CHECK(cred_check_setegid, cred, egid);
+	MAC_CHECK_NOSLEEP(cred_check_setegid, cred, egid);
 	MAC_CHECK_PROBE2(cred_check_setegid, error, cred, egid);
 
 	return (error);
@@ -271,7 +271,7 @@ mac_cred_check_setgroups(struct ucred *cred, int ngroups, gid_t *gidset)
 {
 	int error;
 
-	MAC_CHECK(cred_check_setgroups, cred, ngroups, gidset);
+	MAC_CHECK_NOSLEEP(cred_check_setgroups, cred, ngroups, gidset);
 	MAC_CHECK_PROBE3(cred_check_setgroups, error, cred, ngroups, gidset);
 
 	return (error);
@@ -285,7 +285,7 @@ mac_cred_check_setreuid(struct ucred *cred, uid_t ruid, uid_t euid)
 {
 	int error;
 
-	MAC_CHECK(cred_check_setreuid, cred, ruid, euid);
+	MAC_CHECK_NOSLEEP(cred_check_setreuid, cred, ruid, euid);
 	MAC_CHECK_PROBE3(cred_check_setreuid, error, cred, ruid, euid);
 
 	return (error);
@@ -299,7 +299,7 @@ mac_cred_check_setregid(struct ucred *cred, gid_t rgid, gid_t egid)
 {
 	int error;
 
-	MAC_CHECK(cred_check_setregid, cred, rgid, egid);
+	MAC_CHECK_NOSLEEP(cred_check_setregid, cred, rgid, egid);
 	MAC_CHECK_PROBE3(cred_check_setregid, error, cred, rgid, egid);
 
 	return (error);
@@ -314,7 +314,7 @@ mac_cred_check_setresuid(struct ucred *cred, uid_t ruid, uid_t euid,
 {
 	int error;
 
-	MAC_CHECK(cred_check_setresuid, cred, ruid, euid, suid);
+	MAC_CHECK_NOSLEEP(cred_check_setresuid, cred, ruid, euid, suid);
 	MAC_CHECK_PROBE4(cred_check_setresuid, error, cred, ruid, euid,
 	    suid);
 
@@ -330,7 +330,7 @@ mac_cred_check_setresgid(struct ucred *cred, gid_t rgid, gid_t egid,
 {
 	int error;
 
-	MAC_CHECK(cred_check_setresgid, cred, rgid, egid, sgid);
+	MAC_CHECK_NOSLEEP(cred_check_setresgid, cred, rgid, egid, sgid);
 	MAC_CHECK_PROBE4(cred_check_setresgid, error, cred, rgid, egid,
 	    sgid);
 
@@ -345,7 +345,7 @@ mac_cred_check_visible(struct ucred *cr1, struct ucred *cr2)
 {
 	int error;
 
-	MAC_CHECK(cred_check_visible, cr1, cr2);
+	MAC_CHECK_NOSLEEP(cred_check_visible, cr1, cr2);
 	MAC_CHECK_PROBE2(cred_check_visible, error, cr1, cr2);
 
 	return (error);
diff --git a/sys/security/mac/mac_framework.c b/sys/security/mac/mac_framework.c
index f434df8..1806e4a 100644
--- a/sys/security/mac/mac_framework.c
+++ b/sys/security/mac/mac_framework.c
@@ -76,10 +76,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/condvar.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
-#include <sys/mutex.h>
 #include <sys/mac.h>
 #include <sys/module.h>
+#include <sys/rwlock.h>
 #include <sys/sdt.h>
+#include <sys/sx.h>
 #include <sys/systm.h>
 #include <sys/sysctl.h>
 
@@ -154,164 +155,125 @@ SYSCTL_QUAD(_security_mac, OID_AUTO, labeled, CTLFLAG_RD, &mac_labeled, 0,
 MALLOC_DEFINE(M_MACTEMP, "mactemp", "MAC temporary label storage");
 
 /*
- * mac_static_policy_list holds a list of policy modules that are not loaded
- * while the system is "live", and cannot be unloaded.  These policies can be
- * invoked without holding the busy count.
+ * MAC policy modules are placed in one of two lists: mac_static_policy_list,
+ * for policies that are loaded early and cannot be unloaded, and
+ * mac_policy_list, which holds policies either loaded later in the boot
+ * cycle or that may be unloaded.  The static policy list does not require
+ * locks to iterate over, but the dynamic list requires synchronization.
+ * Support for dynamic policy loading can be compiled out using the
+ * MAC_STATIC kernel option.
  *
- * mac_policy_list stores the list of dynamic policies.  A busy count is
- * maintained for the list, stored in mac_policy_busy.  The busy count is
- * protected by mac_policy_mtx; the list may be modified only while the busy
- * count is 0, requiring that the lock be held to prevent new references to
- * the list from being acquired.  For almost all operations, incrementing the
- * busy count is sufficient to guarantee consistency, as the list cannot be
- * modified while the busy count is elevated.  For a few special operations
- * involving a change to the list of active policies, the mtx itself must be
- * held.  A condition variable, mac_policy_cv, is used to signal potential
- * exclusive consumers that they should try to acquire the lock if a first
- * attempt at exclusive access fails.
- *
- * This design intentionally avoids fairness, and may starve attempts to
- * acquire an exclusive lock on a busy system.  This is required because we
- * do not ever want acquiring a read reference to perform an unbounded length
- * sleep.  Read references are acquired in ithreads, network isrs, etc, and
- * any unbounded blocking could lead quickly to deadlock.
- *
- * Another reason for never blocking on read references is that the MAC
- * Framework may recurse: if a policy calls a VOP, for example, this might
- * lead to vnode life cycle operations (such as init/destroy).
- *
- * If the kernel option MAC_STATIC has been compiled in, all locking becomes
- * a no-op, and the global list of policies is not allowed to change after
- * early boot.
- *
- * XXXRW: Currently, we signal mac_policy_cv every time the framework becomes
- * unbusy and there is a thread waiting to enter it exclusively.  Since it 
- * may take some time before the thread runs, we may issue a lot of signals.
- * We should instead keep track of the fact that we've signalled, taking into 
- * account that the framework may be busy again by the time the thread runs, 
- * requiring us to re-signal. 
+ * The dynamic policy list is protected by two locks: modifying the list
+ * requires both locks to be held exclusively.  One of the locks,
+ * mac_policy_rw, is acquired over policy entry points that will never sleep;
+ * the other, mac_policy_sx, is acquire over policy entry points that may
+ * sleep.  The former category will be used when kernel locks may be held
+ * over calls to the MAC Framework, during network processing in ithreads,
+ * etc.  The latter will tend to involve potentially blocking memory
+ * allocations, extended attribute I/O, etc.
  */
 #ifndef MAC_STATIC
-static struct mtx mac_policy_mtx;
-static struct cv mac_policy_cv;
-static int mac_policy_count;
-static int mac_policy_wait;
+static struct rwlock mac_policy_rw;	/* Non-sleeping entry points. */
+static struct sx mac_policy_sx;		/* Sleeping entry points. */
 #endif
+
 struct mac_policy_list_head mac_policy_list;
 struct mac_policy_list_head mac_static_policy_list;
 
-/*
- * We manually invoke WITNESS_WARN() to allow Witness to generate warnings
- * even if we don't end up ever triggering the wait at run-time.  The
- * consumer of the exclusive interface must not hold any locks (other than
- * potentially Giant) since we may sleep for long (potentially indefinite)
- * periods of time waiting for the framework to become quiescent so that a
- * policy list change may be made.
- */
+static void	mac_policy_xlock(void);
+static void	mac_policy_xlock_assert(void);
+static void	mac_policy_xunlock(void);
+
 void
-mac_policy_grab_exclusive(void)
+mac_policy_slock_nosleep(void)
 {
 
 #ifndef MAC_STATIC
 	if (!mac_late)
 		return;
 
-	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
- 	    "mac_policy_grab_exclusive() at %s:%d", __FILE__, __LINE__);
-	mtx_lock(&mac_policy_mtx);
-	while (mac_policy_count != 0) {
-		mac_policy_wait++;
-		cv_wait(&mac_policy_cv, &mac_policy_mtx);
-		mac_policy_wait--;
-	}
+	rw_rlock(&mac_policy_rw);
 #endif
 }
 
 void
-mac_policy_assert_exclusive(void)
+mac_policy_slock_sleep(void)
 {
 
+	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
+ 	    "mac_policy_slock_sleep");
+
 #ifndef MAC_STATIC
 	if (!mac_late)
 		return;
 
-	mtx_assert(&mac_policy_mtx, MA_OWNED);
-	KASSERT(mac_policy_count == 0,
-	    ("mac_policy_assert_exclusive(): not exclusive"));
+	sx_slock(&mac_policy_sx);
 #endif
 }
 
 void
-mac_policy_release_exclusive(void)
+mac_policy_sunlock_nosleep(void)
 {
-#ifndef MAC_STATIC
-	int dowakeup;
 
+#ifndef MAC_STATIC
 	if (!mac_late)
 		return;
 
-	KASSERT(mac_policy_count == 0,
-	    ("mac_policy_release_exclusive(): not exclusive"));
-	dowakeup = (mac_policy_wait != 0);
-	mtx_unlock(&mac_policy_mtx);
-	if (dowakeup)
-		cv_signal(&mac_policy_cv);
+	rw_runlock(&mac_policy_rw);
 #endif
 }
 
 void
-mac_policy_list_busy(void)
+mac_policy_sunlock_sleep(void)
 {
 
 #ifndef MAC_STATIC
 	if (!mac_late)
 		return;
 
-	mtx_lock(&mac_policy_mtx);
-	mac_policy_count++;
-	mtx_unlock(&mac_policy_mtx);
+	sx_sunlock(&mac_policy_sx);
 #endif
 }
 
-int
-mac_policy_list_conditional_busy(void)
+static void
+mac_policy_xlock(void)
 {
-#ifndef MAC_STATIC
-	int ret;
 
+	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
+ 	    "mac_policy_xlock()");
+
+#ifndef MAC_STATIC
 	if (!mac_late)
-		return (1);
-
-	mtx_lock(&mac_policy_mtx);
-	if (!LIST_EMPTY(&mac_policy_list)) {
-		mac_policy_count++;
-		ret = 1;
-	} else
-		ret = 0;
-	mtx_unlock(&mac_policy_mtx);
-	return (ret);
-#else
-	return (1);
+		return;
+
+	sx_xlock(&mac_policy_sx);
+	rw_wlock(&mac_policy_rw);
 #endif
 }
 
-void
-mac_policy_list_unbusy(void)
+static void
+mac_policy_xunlock(void)
 {
-#ifndef MAC_STATIC
-	int dowakeup;
 
+#ifndef MAC_STATIC
 	if (!mac_late)
 		return;
 
-	mtx_lock(&mac_policy_mtx);
-	mac_policy_count--;
-	KASSERT(mac_policy_count >= 0, ("MAC_POLICY_LIST_LOCK"));
-	dowakeup = (mac_policy_count == 0 && mac_policy_wait != 0);
-	mtx_unlock(&mac_policy_mtx);
+	rw_wunlock(&mac_policy_rw);
+	sx_xunlock(&mac_policy_sx);
+#endif
+}
+
+static void
+mac_policy_xlock_assert(void)
+{
 
-	if (dowakeup)
-		cv_signal(&mac_policy_cv);
+#ifndef MAC_STATIC
+	if (!mac_late)
+		return;
+
+	rw_assert(&mac_policy_rw, RA_WLOCKED);
+	sx_assert(&mac_policy_sx, SA_XLOCKED);
 #endif
 }
 
@@ -327,8 +289,8 @@ mac_init(void)
 	mac_labelzone_init();
 
 #ifndef MAC_STATIC
-	mtx_init(&mac_policy_mtx, "mac_policy_mtx", NULL, MTX_DEF);
-	cv_init(&mac_policy_cv, "mac_policy_cv");
+	rw_init(&mac_policy_rw, "mac_policy_rw");
+	sx_init(&mac_policy_sx, "mac_policy_sx");
 #endif
 }
 
@@ -393,7 +355,7 @@ mac_policy_updateflags(void)
 {
 	struct mac_policy_conf *mpc;
 
-	mac_policy_assert_exclusive();
+	mac_policy_xlock_assert();
 
 	mac_labeled = 0;
 	LIST_FOREACH(mpc, &mac_static_policy_list, mpc_list)
@@ -414,7 +376,7 @@ mac_policy_register(struct mac_policy_conf *mpc)
 	 * We don't technically need exclusive access while !mac_late, but
 	 * hold it for assertion consistency.
 	 */
-	mac_policy_grab_exclusive();
+	mac_policy_xlock();
 
 	/*
 	 * If the module can potentially be unloaded, or we're loading late,
@@ -479,7 +441,7 @@ mac_policy_register(struct mac_policy_conf *mpc)
 	    mpc->mpc_name);
 
 out:
-	mac_policy_release_exclusive();
+	mac_policy_xunlock();
 	return (error);
 }
 
@@ -491,9 +453,9 @@ mac_policy_unregister(struct mac_policy_conf *mpc)
 	 * If we fail the load, we may get a request to unload.  Check to see
 	 * if we did the run-time registration, and if not, silently succeed.
 	 */
-	mac_policy_grab_exclusive();
+	mac_policy_xlock();
 	if ((mpc->mpc_runtime_flags & MPC_RUNTIME_FLAG_REGISTERED) == 0) {
-		mac_policy_release_exclusive();
+		mac_policy_xunlock();
 		return (0);
 	}
 #if 0
@@ -501,7 +463,7 @@ mac_policy_unregister(struct mac_policy_conf *mpc)
 	 * Don't allow unloading modules with private data.
 	 */
 	if (mpc->mpc_field_off != NULL) {
-		MAC_POLICY_LIST_UNLOCK();
+		mac_policy_xunlock();
 		return (EBUSY);
 	}
 #endif
@@ -510,7 +472,7 @@ mac_policy_unregister(struct mac_policy_conf *mpc)
 	 * its own definition.
 	 */
 	if ((mpc->mpc_loadtime_flags & MPC_LOADTIME_FLAG_UNLOADOK) == 0) {
-		mac_policy_release_exclusive();
+		mac_policy_xunlock();
 		return (EBUSY);
 	}
 	if (mpc->mpc_ops->mpo_destroy != NULL)
@@ -519,8 +481,7 @@ mac_policy_unregister(struct mac_policy_conf *mpc)
 	LIST_REMOVE(mpc, mpc_list);
 	mpc->mpc_runtime_flags &= ~MPC_RUNTIME_FLAG_REGISTERED;
 	mac_policy_updateflags();
-
-	mac_policy_release_exclusive();
+	mac_policy_xunlock();
 
 	SDT_PROBE(mac, kernel, policy, unregister, mpc, 0, 0, 0, 0);
 	printf("Security policy unload: %s (%s)\n", mpc->mpc_fullname,
diff --git a/sys/security/mac/mac_inet.c b/sys/security/mac/mac_inet.c
index b62938b..df21a16 100644
--- a/sys/security/mac/mac_inet.c
+++ b/sys/security/mac/mac_inet.c
@@ -84,9 +84,12 @@ mac_inpcb_label_alloc(int flag)
 	label = mac_labelzone_alloc(flag);
 	if (label == NULL)
 		return (NULL);
-	MAC_CHECK(inpcb_init_label, label, flag);
+	if (flag & M_WAITOK)
+		MAC_CHECK(inpcb_init_label, label, flag);
+	else
+		MAC_CHECK_NOSLEEP(inpcb_init_label, label, flag);
 	if (error) {
-		MAC_PERFORM(inpcb_destroy_label, label);
+		MAC_PERFORM_NOSLEEP(inpcb_destroy_label, label);
 		mac_labelzone_free(label);
 		return (NULL);
 	}
@@ -116,9 +119,12 @@ mac_ipq_label_alloc(int flag)
 	if (label == NULL)
 		return (NULL);
 
-	MAC_CHECK(ipq_init_label, label, flag);
+	if (flag & M_WAITOK)
+		MAC_CHECK(ipq_init_label, label, flag);
+	else
+		MAC_CHECK_NOSLEEP(ipq_init_label, label, flag);
 	if (error) {
-		MAC_PERFORM(ipq_destroy_label, label);
+		MAC_PERFORM_NOSLEEP(ipq_destroy_label, label);
 		mac_labelzone_free(label);
 		return (NULL);
 	}
@@ -142,7 +148,7 @@ static void
 mac_inpcb_label_free(struct label *label)
 {
 
-	MAC_PERFORM(inpcb_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(inpcb_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -160,7 +166,7 @@ static void
 mac_ipq_label_free(struct label *label)
 {
 
-	MAC_PERFORM(ipq_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(ipq_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -178,7 +184,8 @@ void
 mac_inpcb_create(struct socket *so, struct inpcb *inp)
 {
 
-	MAC_PERFORM(inpcb_create, so, so->so_label, inp, inp->inp_label);
+	MAC_PERFORM_NOSLEEP(inpcb_create, so, so->so_label, inp,
+	    inp->inp_label);
 }
 
 void
@@ -188,7 +195,7 @@ mac_ipq_reassemble(struct ipq *q, struct mbuf *m)
 
 	label = mac_mbuf_to_label(m);
 
-	MAC_PERFORM(ipq_reassemble, q, q->ipq_label, m, label);
+	MAC_PERFORM_NOSLEEP(ipq_reassemble, q, q->ipq_label, m, label);
 }
 
 void
@@ -199,7 +206,7 @@ mac_netinet_fragment(struct mbuf *m, struct mbuf *frag)
 	mlabel = mac_mbuf_to_label(m);
 	fraglabel = mac_mbuf_to_label(frag);
 
-	MAC_PERFORM(netinet_fragment, m, mlabel, frag, fraglabel);
+	MAC_PERFORM_NOSLEEP(netinet_fragment, m, mlabel, frag, fraglabel);
 }
 
 void
@@ -209,7 +216,7 @@ mac_ipq_create(struct mbuf *m, struct ipq *q)
 
 	label = mac_mbuf_to_label(m);
 
-	MAC_PERFORM(ipq_create, m, label, q, q->ipq_label);
+	MAC_PERFORM_NOSLEEP(ipq_create, m, label, q, q->ipq_label);
 }
 
 void
@@ -220,7 +227,8 @@ mac_inpcb_create_mbuf(struct inpcb *inp, struct mbuf *m)
 	INP_LOCK_ASSERT(inp);
 	mlabel = mac_mbuf_to_label(m);
 
-	MAC_PERFORM(inpcb_create_mbuf, inp, inp->inp_label, m, mlabel);
+	MAC_PERFORM_NOSLEEP(inpcb_create_mbuf, inp, inp->inp_label, m,
+	    mlabel);
 }
 
 int
@@ -232,7 +240,7 @@ mac_ipq_match(struct mbuf *m, struct ipq *q)
 	label = mac_mbuf_to_label(m);
 
 	result = 1;
-	MAC_BOOLEAN(ipq_match, &&, m, label, q, q->ipq_label);
+	MAC_BOOLEAN_NOSLEEP(ipq_match, &&, m, label, q, q->ipq_label);
 
 	return (result);
 }
@@ -245,7 +253,7 @@ mac_netinet_arp_send(struct ifnet *ifp, struct mbuf *m)
 	mlabel = mac_mbuf_to_label(m);
 
 	MAC_IFNET_LOCK(ifp);
-	MAC_PERFORM(netinet_arp_send, ifp, ifp->if_label, m, mlabel);
+	MAC_PERFORM_NOSLEEP(netinet_arp_send, ifp, ifp->if_label, m, mlabel);
 	MAC_IFNET_UNLOCK(ifp);
 }
 
@@ -257,7 +265,7 @@ mac_netinet_icmp_reply(struct mbuf *mrecv, struct mbuf *msend)
 	mrecvlabel = mac_mbuf_to_label(mrecv);
 	msendlabel = mac_mbuf_to_label(msend);
 
-	MAC_PERFORM(netinet_icmp_reply, mrecv, mrecvlabel, msend,
+	MAC_PERFORM_NOSLEEP(netinet_icmp_reply, mrecv, mrecvlabel, msend,
 	    msendlabel);
 }
 
@@ -268,7 +276,7 @@ mac_netinet_icmp_replyinplace(struct mbuf *m)
 
 	label = mac_mbuf_to_label(m);
 
-	MAC_PERFORM(netinet_icmp_replyinplace, m, label);
+	MAC_PERFORM_NOSLEEP(netinet_icmp_replyinplace, m, label);
 }
 
 void
@@ -279,7 +287,8 @@ mac_netinet_igmp_send(struct ifnet *ifp, struct mbuf *m)
 	mlabel = mac_mbuf_to_label(m);
 
 	MAC_IFNET_LOCK(ifp);
-	MAC_PERFORM(netinet_igmp_send, ifp, ifp->if_label, m, mlabel);
+	MAC_PERFORM_NOSLEEP(netinet_igmp_send, ifp, ifp->if_label, m,
+	    mlabel);
 	MAC_IFNET_UNLOCK(ifp);
 }
 
@@ -290,7 +299,7 @@ mac_netinet_tcp_reply(struct mbuf *m)
 
 	label = mac_mbuf_to_label(m);
 
-	MAC_PERFORM(netinet_tcp_reply, m, label);
+	MAC_PERFORM_NOSLEEP(netinet_tcp_reply, m, label);
 }
 
 void
@@ -300,7 +309,7 @@ mac_ipq_update(struct mbuf *m, struct ipq *q)
 
 	label = mac_mbuf_to_label(m);
 
-	MAC_PERFORM(ipq_update, m, label, q, q->ipq_label);
+	MAC_PERFORM_NOSLEEP(ipq_update, m, label, q, q->ipq_label);
 }
 
 MAC_CHECK_PROBE_DEFINE2(inpcb_check_deliver, "struct inpcb *",
@@ -316,7 +325,8 @@ mac_inpcb_check_deliver(struct inpcb *inp, struct mbuf *m)
 
 	label = mac_mbuf_to_label(m);
 
-	MAC_CHECK(inpcb_check_deliver, inp, inp->inp_label, m, label);
+	MAC_CHECK_NOSLEEP(inpcb_check_deliver, inp, inp->inp_label, m,
+	    label);
 	MAC_CHECK_PROBE2(inpcb_check_deliver, error, inp, m);
 
 	return (error);
@@ -332,7 +342,7 @@ mac_inpcb_check_visible(struct ucred *cred, struct inpcb *inp)
 
 	INP_LOCK_ASSERT(inp);
 
-	MAC_CHECK(inpcb_check_visible, cred, inp, inp->inp_label);
+	MAC_CHECK_NOSLEEP(inpcb_check_visible, cred, inp, inp->inp_label);
 	MAC_CHECK_PROBE2(inpcb_check_visible, error, cred, inp);
 
 	return (error);
@@ -344,7 +354,9 @@ mac_inpcb_sosetlabel(struct socket *so, struct inpcb *inp)
 
 	INP_WLOCK_ASSERT(inp);
 	SOCK_LOCK_ASSERT(so);
-	MAC_PERFORM(inpcb_sosetlabel, so, so->so_label, inp, inp->inp_label);
+
+	MAC_PERFORM_NOSLEEP(inpcb_sosetlabel, so, so->so_label, inp,
+	    inp->inp_label);
 }
 
 void
@@ -358,7 +370,7 @@ mac_netinet_firewall_reply(struct mbuf *mrecv, struct mbuf *msend)
 	mrecvlabel = mac_mbuf_to_label(mrecv);
 	msendlabel = mac_mbuf_to_label(msend);
 
-	MAC_PERFORM(netinet_firewall_reply, mrecv, mrecvlabel, msend,
+	MAC_PERFORM_NOSLEEP(netinet_firewall_reply, mrecv, mrecvlabel, msend,
 	    msendlabel);
 }
 
@@ -368,8 +380,10 @@ mac_netinet_firewall_send(struct mbuf *m)
 	struct label *label;
 
 	M_ASSERTPKTHDR(m);
+
 	label = mac_mbuf_to_label(m);
-	MAC_PERFORM(netinet_firewall_send, m, label);
+
+	MAC_PERFORM_NOSLEEP(netinet_firewall_send, m, label);
 }
 
 /*
@@ -386,7 +400,7 @@ mac_syncache_destroy(struct label **label)
 {
 
 	if (*label != NULL) {
-		MAC_PERFORM(syncache_destroy_label, *label);
+		MAC_PERFORM_NOSLEEP(syncache_destroy_label, *label);
 		mac_labelzone_free(*label);
 		*label = NULL;
 	}
@@ -408,9 +422,9 @@ mac_syncache_init(struct label **label)
 		 * MAC_PERFORM so we can propagate allocation failures back
 		 * to the syncache code.
 		 */
-		MAC_CHECK(syncache_init_label, *label, M_NOWAIT);
+		MAC_CHECK_NOSLEEP(syncache_init_label, *label, M_NOWAIT);
 		if (error) {
-			MAC_PERFORM(syncache_destroy_label, *label);
+			MAC_PERFORM_NOSLEEP(syncache_destroy_label, *label);
 			mac_labelzone_free(*label);
 		}
 		return (error);
@@ -424,7 +438,8 @@ mac_syncache_create(struct label *label, struct inpcb *inp)
 {
 
 	INP_WLOCK_ASSERT(inp);
-	MAC_PERFORM(syncache_create, label, inp);
+
+	MAC_PERFORM_NOSLEEP(syncache_create, label, inp);
 }
 
 void
@@ -433,6 +448,8 @@ mac_syncache_create_mbuf(struct label *sc_label, struct mbuf *m)
 	struct label *mlabel;
 
 	M_ASSERTPKTHDR(m);
+
 	mlabel = mac_mbuf_to_label(m);
-	MAC_PERFORM(syncache_create_mbuf, sc_label, m, mlabel);
+
+	MAC_PERFORM_NOSLEEP(syncache_create_mbuf, sc_label, m, mlabel);
 }
diff --git a/sys/security/mac/mac_inet6.c b/sys/security/mac/mac_inet6.c
index 9201b36..f804fe3 100644
--- a/sys/security/mac/mac_inet6.c
+++ b/sys/security/mac/mac_inet6.c
@@ -1,9 +1,12 @@
 /*-
- * Copyright (c) 2007-2008 Robert N. M. Watson
+ * Copyright (c) 2007-2009 Robert N. M. Watson
  * All rights reserved.
  *
  * This software was developed by Robert Watson for the TrustedBSD Project.
  *
+ * This software was developed at the University of Cambridge Computer
+ * Laboratory with support from a grant from Google, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -67,9 +70,12 @@ mac_ip6q_label_alloc(int flag)
 	if (label == NULL)
 		return (NULL);
 
-	MAC_CHECK(ip6q_init_label, label, flag);
+	if (flag & M_WAITOK)
+		MAC_CHECK(ip6q_init_label, label, flag);
+	else
+		MAC_CHECK_NOSLEEP(ip6q_init_label, label, flag);
 	if (error) {
-		MAC_PERFORM(ip6q_destroy_label, label);
+		MAC_PERFORM_NOSLEEP(ip6q_destroy_label, label);
 		mac_labelzone_free(label);
 		return (NULL);
 	}
@@ -93,7 +99,7 @@ static void
 mac_ip6q_label_free(struct label *label)
 {
 
-	MAC_PERFORM(ip6q_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(ip6q_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -114,7 +120,7 @@ mac_ip6q_reassemble(struct ip6q *q6, struct mbuf *m)
 
 	label = mac_mbuf_to_label(m);
 
-	MAC_PERFORM(ip6q_reassemble, q6, q6->ip6q_label, m, label);
+	MAC_PERFORM_NOSLEEP(ip6q_reassemble, q6, q6->ip6q_label, m, label);
 }
 
 void
@@ -124,7 +130,7 @@ mac_ip6q_create(struct mbuf *m, struct ip6q *q6)
 
 	label = mac_mbuf_to_label(m);
 
-	MAC_PERFORM(ip6q_create, m, label, q6, q6->ip6q_label);
+	MAC_PERFORM_NOSLEEP(ip6q_create, m, label, q6, q6->ip6q_label);
 }
 
 int
@@ -136,7 +142,7 @@ mac_ip6q_match(struct mbuf *m, struct ip6q *q6)
 	label = mac_mbuf_to_label(m);
 
 	result = 1;
-	MAC_BOOLEAN(ip6q_match, &&, m, label, q6, q6->ip6q_label);
+	MAC_BOOLEAN_NOSLEEP(ip6q_match, &&, m, label, q6, q6->ip6q_label);
 
 	return (result);
 }
@@ -148,7 +154,7 @@ mac_ip6q_update(struct mbuf *m, struct ip6q *q6)
 
 	label = mac_mbuf_to_label(m);
 
-	MAC_PERFORM(ip6q_update, m, label, q6, q6->ip6q_label);
+	MAC_PERFORM_NOSLEEP(ip6q_update, m, label, q6, q6->ip6q_label);
 }
 
 void
@@ -158,5 +164,6 @@ mac_netinet6_nd6_send(struct ifnet *ifp, struct mbuf *m)
 
 	mlabel = mac_mbuf_to_label(m);
 
-	MAC_PERFORM(netinet6_nd6_send, ifp, ifp->if_label, m, mlabel);
+	MAC_PERFORM_NOSLEEP(netinet6_nd6_send, ifp, ifp->if_label, m,
+	    mlabel);
 }
diff --git a/sys/security/mac/mac_internal.h b/sys/security/mac/mac_internal.h
index 34336fc..280c8b8 100644
--- a/sys/security/mac/mac_internal.h
+++ b/sys/security/mac/mac_internal.h
@@ -194,12 +194,10 @@ extern struct mtx			mac_ifnet_mtx;
  */
 int	mac_error_select(int error1, int error2);
 
-void	mac_policy_grab_exclusive(void);
-void	mac_policy_assert_exclusive(void);
-void	mac_policy_release_exclusive(void);
-void	mac_policy_list_busy(void);
-int	mac_policy_list_conditional_busy(void);
-void	mac_policy_list_unbusy(void);
+void	mac_policy_slock_nosleep(void);
+void	mac_policy_slock_sleep(void);
+void	mac_policy_sunlock_nosleep(void);
+void	mac_policy_sunlock_sleep(void);
 
 struct label	*mac_labelzone_alloc(int flags);
 void		 mac_labelzone_free(struct label *label);
@@ -255,13 +253,16 @@ int	vn_setlabel(struct vnode *vp, struct label *intlabel,
 	    struct ucred *cred);
 
 /*
+ * MAC Framework composition macros invoke all registered MAC policies for a
+ * specific entry point.  They come in two forms: one which permits policies
+ * to sleep/block, and another that does not.
+ *
  * MAC_CHECK performs the designated check by walking the policy module list
  * and checking with each as to how it feels about the request.  Note that it
  * returns its value via 'error' in the scope of the caller.
  */
 #define	MAC_CHECK(check, args...) do {					\
 	struct mac_policy_conf *mpc;					\
-	int entrycount;							\
 									\
 	error = 0;							\
 	LIST_FOREACH(mpc, &mac_static_policy_list, mpc_list) {		\
@@ -270,14 +271,37 @@ int	vn_setlabel(struct vnode *vp, struct label *intlabel,
 			    mpc->mpc_ops->mpo_ ## check (args),		\
 			    error);					\
 	}								\
-	if ((entrycount = mac_policy_list_conditional_busy()) != 0) {	\
+	if (!LIST_EMPTY(&mac_policy_list)) {				\
+		mac_policy_slock_sleep();				\
+		LIST_FOREACH(mpc, &mac_policy_list, mpc_list) {		\
+			if (mpc->mpc_ops->mpo_ ## check != NULL)	\
+				error = mac_error_select(		\
+				    mpc->mpc_ops->mpo_ ## check (args),	\
+				    error);				\
+		}							\
+		mac_policy_sunlock_sleep();				\
+	}								\
+} while (0)
+
+#define	MAC_CHECK_NOSLEEP(check, args...) do {				\
+	struct mac_policy_conf *mpc;					\
+									\
+	error = 0;							\
+	LIST_FOREACH(mpc, &mac_static_policy_list, mpc_list) {		\
+		if (mpc->mpc_ops->mpo_ ## check != NULL)		\
+			error = mac_error_select(			\
+			    mpc->mpc_ops->mpo_ ## check (args),		\
+			    error);					\
+	}								\
+	if (!LIST_EMPTY(&mac_policy_list)) {				\
+		mac_policy_slock_nosleep();				\
 		LIST_FOREACH(mpc, &mac_policy_list, mpc_list) {		\
 			if (mpc->mpc_ops->mpo_ ## check != NULL)	\
 				error = mac_error_select(		\
 				    mpc->mpc_ops->mpo_ ## check (args),	\
 				    error);				\
 		}							\
-		mac_policy_list_unbusy();				\
+		mac_policy_sunlock_nosleep();				\
 	}								\
 } while (0)
 
@@ -288,9 +312,8 @@ int	vn_setlabel(struct vnode *vp, struct label *intlabel,
  * EPERM.  Note that it returns its value via 'error' in the scope of the
  * caller.
  */
-#define	MAC_GRANT(check, args...) do {					\
+#define	MAC_GRANT_NOSLEEP(check, args...) do {				\
 	struct mac_policy_conf *mpc;					\
-	int entrycount;							\
 									\
 	error = EPERM;							\
 	LIST_FOREACH(mpc, &mac_static_policy_list, mpc_list) {		\
@@ -299,7 +322,8 @@ int	vn_setlabel(struct vnode *vp, struct label *intlabel,
 				error = 0;				\
 		}							\
 	}								\
-	if ((entrycount = mac_policy_list_conditional_busy()) != 0) {	\
+	if (!LIST_EMPTY(&mac_policy_list)) {				\
+		mac_policy_slock_nosleep();				\
 		LIST_FOREACH(mpc, &mac_policy_list, mpc_list) {		\
 			if (mpc->mpc_ops->mpo_ ## check != NULL) {	\
 				if (mpc->mpc_ops->mpo_ ## check (args)	\
@@ -307,7 +331,7 @@ int	vn_setlabel(struct vnode *vp, struct label *intlabel,
 					error = 0;			\
 			}						\
 		}							\
-		mac_policy_list_unbusy();				\
+		mac_policy_sunlock_nosleep();				\
 	}								\
 } while (0)
 
@@ -320,21 +344,41 @@ int	vn_setlabel(struct vnode *vp, struct label *intlabel,
  */
 #define	MAC_BOOLEAN(operation, composition, args...) do {		\
 	struct mac_policy_conf *mpc;					\
-	int entrycount;							\
 									\
 	LIST_FOREACH(mpc, &mac_static_policy_list, mpc_list) {		\
 		if (mpc->mpc_ops->mpo_ ## operation != NULL)		\
 			result = result composition			\
 			    mpc->mpc_ops->mpo_ ## operation (args);	\
 	}								\
-	if ((entrycount = mac_policy_list_conditional_busy()) != 0) {	\
+	if (!LIST_EMPTY(&mac_policy_list)) {				\
+		mac_policy_slock_sleep();				\
+		LIST_FOREACH(mpc, &mac_policy_list, mpc_list) {		\
+			if (mpc->mpc_ops->mpo_ ## operation != NULL)	\
+				result = result composition		\
+				    mpc->mpc_ops->mpo_ ## operation	\
+				    (args);				\
+		}							\
+		mac_policy_sunlock_sleep();				\
+	}								\
+} while (0)
+
+#define	MAC_BOOLEAN_NOSLEEP(operation, composition, args...) do {	\
+	struct mac_policy_conf *mpc;					\
+									\
+	LIST_FOREACH(mpc, &mac_static_policy_list, mpc_list) {		\
+		if (mpc->mpc_ops->mpo_ ## operation != NULL)		\
+			result = result composition			\
+			    mpc->mpc_ops->mpo_ ## operation (args);	\
+	}								\
+	if (!LIST_EMPTY(&mac_policy_list)) {				\
+		mac_policy_slock_nosleep();				\
 		LIST_FOREACH(mpc, &mac_policy_list, mpc_list) {		\
 			if (mpc->mpc_ops->mpo_ ## operation != NULL)	\
 				result = result composition		\
 				    mpc->mpc_ops->mpo_ ## operation	\
 				    (args);				\
 		}							\
-		mac_policy_list_unbusy();				\
+		mac_policy_sunlock_nosleep();				\
 	}								\
 } while (0)
 
@@ -425,18 +469,35 @@ int	vn_setlabel(struct vnode *vp, struct label *intlabel,
  */
 #define	MAC_PERFORM(operation, args...) do {				\
 	struct mac_policy_conf *mpc;					\
-	int entrycount;							\
 									\
 	LIST_FOREACH(mpc, &mac_static_policy_list, mpc_list) {		\
 		if (mpc->mpc_ops->mpo_ ## operation != NULL)		\
 			mpc->mpc_ops->mpo_ ## operation (args);		\
 	}								\
-	if ((entrycount = mac_policy_list_conditional_busy()) != 0) {	\
+	if (!LIST_EMPTY(&mac_policy_list)) {				\
+		mac_policy_slock_sleep();				\
+		LIST_FOREACH(mpc, &mac_policy_list, mpc_list) {		\
+			if (mpc->mpc_ops->mpo_ ## operation != NULL)	\
+				mpc->mpc_ops->mpo_ ## operation (args);	\
+		}							\
+		mac_policy_sunlock_sleep();				\
+	}								\
+} while (0)
+
+#define	MAC_PERFORM_NOSLEEP(operation, args...) do {			\
+	struct mac_policy_conf *mpc;					\
+									\
+	LIST_FOREACH(mpc, &mac_static_policy_list, mpc_list) {		\
+		if (mpc->mpc_ops->mpo_ ## operation != NULL)		\
+			mpc->mpc_ops->mpo_ ## operation (args);		\
+	}								\
+	if (!LIST_EMPTY(&mac_policy_list)) {				\
+		mac_policy_slock_nosleep();				\
 		LIST_FOREACH(mpc, &mac_policy_list, mpc_list) {		\
 			if (mpc->mpc_ops->mpo_ ## operation != NULL)	\
 				mpc->mpc_ops->mpo_ ## operation (args);	\
 		}							\
-		mac_policy_list_unbusy();				\
+		mac_policy_sunlock_nosleep();				\
 	}								\
 } while (0)
 
diff --git a/sys/security/mac/mac_net.c b/sys/security/mac/mac_net.c
index 4fccbd7..697d02a 100644
--- a/sys/security/mac/mac_net.c
+++ b/sys/security/mac/mac_net.c
@@ -153,9 +153,12 @@ mac_mbuf_tag_init(struct m_tag *tag, int flag)
 	label = (struct label *) (tag + 1);
 	mac_init_label(label);
 
-	MAC_CHECK(mbuf_init_label, label, flag);
+	if (flag & M_WAITOK)
+		MAC_CHECK(mbuf_init_label, label, flag);
+	else
+		MAC_CHECK_NOSLEEP(mbuf_init_label, label, flag);
 	if (error) {
-		MAC_PERFORM(mbuf_destroy_label, label);
+		MAC_PERFORM_NOSLEEP(mbuf_destroy_label, label);
 		mac_destroy_label(label);
 	}
 	return (error);
@@ -188,7 +191,7 @@ static void
 mac_bpfdesc_label_free(struct label *label)
 {
 
-	MAC_PERFORM(bpfdesc_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(bpfdesc_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -206,7 +209,7 @@ static void
 mac_ifnet_label_free(struct label *label)
 {
 
-	MAC_PERFORM(ifnet_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(ifnet_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -227,7 +230,7 @@ mac_mbuf_tag_destroy(struct m_tag *tag)
 
 	label = (struct label *)(tag+1);
 
-	MAC_PERFORM(mbuf_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(mbuf_destroy_label, label);
 	mac_destroy_label(label);
 }
 
@@ -247,7 +250,7 @@ mac_mbuf_tag_copy(struct m_tag *src, struct m_tag *dest)
 	 * mac_mbuf_tag_init() is called on the target tag in m_tag_copy(),
 	 * so we don't need to call it here.
 	 */
-	MAC_PERFORM(mbuf_copy_label, src_label, dest_label);
+	MAC_PERFORM_NOSLEEP(mbuf_copy_label, src_label, dest_label);
 }
 
 void
@@ -258,14 +261,14 @@ mac_mbuf_copy(struct mbuf *m_from, struct mbuf *m_to)
 	src_label = mac_mbuf_to_label(m_from);
 	dest_label = mac_mbuf_to_label(m_to);
 
-	MAC_PERFORM(mbuf_copy_label, src_label, dest_label);
+	MAC_PERFORM_NOSLEEP(mbuf_copy_label, src_label, dest_label);
 }
 
 static void
 mac_ifnet_copy_label(struct label *src, struct label *dest)
 {
 
-	MAC_PERFORM(ifnet_copy_label, src, dest);
+	MAC_PERFORM_NOSLEEP(ifnet_copy_label, src, dest);
 }
 
 static int
@@ -294,7 +297,7 @@ mac_ifnet_create(struct ifnet *ifp)
 {
 
 	MAC_IFNET_LOCK(ifp);
-	MAC_PERFORM(ifnet_create, ifp, ifp->if_label);
+	MAC_PERFORM_NOSLEEP(ifnet_create, ifp, ifp->if_label);
 	MAC_IFNET_UNLOCK(ifp);
 }
 
@@ -302,7 +305,7 @@ void
 mac_bpfdesc_create(struct ucred *cred, struct bpf_d *d)
 {
 
-	MAC_PERFORM(bpfdesc_create, cred, d, d->bd_label);
+	MAC_PERFORM_NOSLEEP(bpfdesc_create, cred, d, d->bd_label);
 }
 
 void
@@ -314,7 +317,7 @@ mac_bpfdesc_create_mbuf(struct bpf_d *d, struct mbuf *m)
 
 	label = mac_mbuf_to_label(m);
 
-	MAC_PERFORM(bpfdesc_create_mbuf, d, d->bd_label, m, label);
+	MAC_PERFORM_NOSLEEP(bpfdesc_create_mbuf, d, d->bd_label, m, label);
 }
 
 void
@@ -325,7 +328,7 @@ mac_ifnet_create_mbuf(struct ifnet *ifp, struct mbuf *m)
 	label = mac_mbuf_to_label(m);
 
 	MAC_IFNET_LOCK(ifp);
-	MAC_PERFORM(ifnet_create_mbuf, ifp, ifp->if_label, m, label);
+	MAC_PERFORM_NOSLEEP(ifnet_create_mbuf, ifp, ifp->if_label, m, label);
 	MAC_IFNET_UNLOCK(ifp);
 }
 
@@ -340,7 +343,8 @@ mac_bpfdesc_check_receive(struct bpf_d *d, struct ifnet *ifp)
 	BPFD_LOCK_ASSERT(d);
 
 	MAC_IFNET_LOCK(ifp);
-	MAC_CHECK(bpfdesc_check_receive, d, d->bd_label, ifp, ifp->if_label);
+	MAC_CHECK_NOSLEEP(bpfdesc_check_receive, d, d->bd_label, ifp,
+	    ifp->if_label);
 	MAC_CHECK_PROBE2(bpfdesc_check_receive, error, d, ifp);
 	MAC_IFNET_UNLOCK(ifp);
 
@@ -361,7 +365,8 @@ mac_ifnet_check_transmit(struct ifnet *ifp, struct mbuf *m)
 	label = mac_mbuf_to_label(m);
 
 	MAC_IFNET_LOCK(ifp);
-	MAC_CHECK(ifnet_check_transmit, ifp, ifp->if_label, m, label);
+	MAC_CHECK_NOSLEEP(ifnet_check_transmit, ifp, ifp->if_label, m,
+	    label);
 	MAC_CHECK_PROBE2(ifnet_check_transmit, error, ifp, m);
 	MAC_IFNET_UNLOCK(ifp);
 
@@ -458,14 +463,16 @@ mac_ifnet_ioctl_set(struct ucred *cred, struct ifreq *ifr, struct ifnet *ifp)
 	}
 
 	MAC_IFNET_LOCK(ifp);
-	MAC_CHECK(ifnet_check_relabel, cred, ifp, ifp->if_label, intlabel);
+	MAC_CHECK_NOSLEEP(ifnet_check_relabel, cred, ifp, ifp->if_label,
+	    intlabel);
 	if (error) {
 		MAC_IFNET_UNLOCK(ifp);
 		mac_ifnet_label_free(intlabel);
 		return (error);
 	}
 
-	MAC_PERFORM(ifnet_relabel, cred, ifp, ifp->if_label, intlabel);
+	MAC_PERFORM_NOSLEEP(ifnet_relabel, cred, ifp, ifp->if_label,
+	    intlabel);
 	MAC_IFNET_UNLOCK(ifp);
 
 	mac_ifnet_label_free(intlabel);
diff --git a/sys/security/mac/mac_pipe.c b/sys/security/mac/mac_pipe.c
index 921fd20..160a944 100644
--- a/sys/security/mac/mac_pipe.c
+++ b/sys/security/mac/mac_pipe.c
@@ -84,7 +84,7 @@ void
 mac_pipe_label_free(struct label *label)
 {
 
-	MAC_PERFORM(pipe_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(pipe_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -102,7 +102,7 @@ void
 mac_pipe_copy_label(struct label *src, struct label *dest)
 {
 
-	MAC_PERFORM(pipe_copy_label, src, dest);
+	MAC_PERFORM_NOSLEEP(pipe_copy_label, src, dest);
 }
 
 int
@@ -130,7 +130,7 @@ void
 mac_pipe_create(struct ucred *cred, struct pipepair *pp)
 {
 
-	MAC_PERFORM(pipe_create, cred, pp, pp->pp_label);
+	MAC_PERFORM_NOSLEEP(pipe_create, cred, pp, pp->pp_label);
 }
 
 static void
@@ -138,7 +138,7 @@ mac_pipe_relabel(struct ucred *cred, struct pipepair *pp,
     struct label *newlabel)
 {
 
-	MAC_PERFORM(pipe_relabel, cred, pp, pp->pp_label, newlabel);
+	MAC_PERFORM_NOSLEEP(pipe_relabel, cred, pp, pp->pp_label, newlabel);
 }
 
 MAC_CHECK_PROBE_DEFINE4(pipe_check_ioctl, "struct ucred *",
@@ -152,7 +152,8 @@ mac_pipe_check_ioctl(struct ucred *cred, struct pipepair *pp,
 
 	mtx_assert(&pp->pp_mtx, MA_OWNED);
 
-	MAC_CHECK(pipe_check_ioctl, cred, pp, pp->pp_label, cmd, data);
+	MAC_CHECK_NOSLEEP(pipe_check_ioctl, cred, pp, pp->pp_label, cmd,
+	    data);
 	MAC_CHECK_PROBE4(pipe_check_ioctl, error, cred, pp, cmd, data);
 
 	return (error);
@@ -168,7 +169,7 @@ mac_pipe_check_poll(struct ucred *cred, struct pipepair *pp)
 
 	mtx_assert(&pp->pp_mtx, MA_OWNED);
 
-	MAC_CHECK(pipe_check_poll, cred, pp, pp->pp_label);
+	MAC_CHECK_NOSLEEP(pipe_check_poll, cred, pp, pp->pp_label);
 	MAC_CHECK_PROBE2(pipe_check_poll, error, cred, pp);
 
 	return (error);
@@ -184,7 +185,7 @@ mac_pipe_check_read(struct ucred *cred, struct pipepair *pp)
 
 	mtx_assert(&pp->pp_mtx, MA_OWNED);
 
-	MAC_CHECK(pipe_check_read, cred, pp, pp->pp_label);
+	MAC_CHECK_NOSLEEP(pipe_check_read, cred, pp, pp->pp_label);
 	MAC_CHECK_PROBE2(pipe_check_read, error, cred, pp);
 
 	return (error);
@@ -201,7 +202,8 @@ mac_pipe_check_relabel(struct ucred *cred, struct pipepair *pp,
 
 	mtx_assert(&pp->pp_mtx, MA_OWNED);
 
-	MAC_CHECK(pipe_check_relabel, cred, pp, pp->pp_label, newlabel);
+	MAC_CHECK_NOSLEEP(pipe_check_relabel, cred, pp, pp->pp_label,
+	    newlabel);
 	MAC_CHECK_PROBE3(pipe_check_relabel, error, cred, pp, newlabel);
 
 	return (error);
@@ -217,7 +219,7 @@ mac_pipe_check_stat(struct ucred *cred, struct pipepair *pp)
 
 	mtx_assert(&pp->pp_mtx, MA_OWNED);
 
-	MAC_CHECK(pipe_check_stat, cred, pp, pp->pp_label);
+	MAC_CHECK_NOSLEEP(pipe_check_stat, cred, pp, pp->pp_label);
 	MAC_CHECK_PROBE2(pipe_check_stat, error, cred, pp);
 
 	return (error);
@@ -233,7 +235,7 @@ mac_pipe_check_write(struct ucred *cred, struct pipepair *pp)
 
 	mtx_assert(&pp->pp_mtx, MA_OWNED);
 
-	MAC_CHECK(pipe_check_write, cred, pp, pp->pp_label);
+	MAC_CHECK_NOSLEEP(pipe_check_write, cred, pp, pp->pp_label);
 	MAC_CHECK_PROBE2(pipe_check_write, error, cred, pp);
 
 	return (error);
diff --git a/sys/security/mac/mac_posix_sem.c b/sys/security/mac/mac_posix_sem.c
index 2e3560d..74a9ad9 100644
--- a/sys/security/mac/mac_posix_sem.c
+++ b/sys/security/mac/mac_posix_sem.c
@@ -80,7 +80,7 @@ static void
 mac_posixsem_label_free(struct label *label)
 {
 
-	MAC_PERFORM(posixsem_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(posixsem_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -98,7 +98,7 @@ void
 mac_posixsem_create(struct ucred *cred, struct ksem *ks)
 {
 
-	MAC_PERFORM(posixsem_create, cred, ks, ks->ks_label);
+	MAC_PERFORM_NOSLEEP(posixsem_create, cred, ks, ks->ks_label);
 }
 
 MAC_CHECK_PROBE_DEFINE2(posixsem_check_open, "struct ucred *",
@@ -109,7 +109,7 @@ mac_posixsem_check_open(struct ucred *cred, struct ksem *ks)
 {
 	int error;
 
-	MAC_CHECK(posixsem_check_open, cred, ks, ks->ks_label);
+	MAC_CHECK_NOSLEEP(posixsem_check_open, cred, ks, ks->ks_label);
 	MAC_CHECK_PROBE2(posixsem_check_open, error, cred, ks);
 
 	return (error);
@@ -124,8 +124,8 @@ mac_posixsem_check_getvalue(struct ucred *active_cred, struct ucred *file_cred,
 {
 	int error;
 
-	MAC_CHECK(posixsem_check_getvalue, active_cred, file_cred, ks,
-	    ks->ks_label);
+	MAC_CHECK_NOSLEEP(posixsem_check_getvalue, active_cred, file_cred,
+	    ks, ks->ks_label);
 	MAC_CHECK_PROBE3(posixsem_check_getvalue, error, active_cred,
 	    file_cred, ks);
 
@@ -141,7 +141,7 @@ mac_posixsem_check_post(struct ucred *active_cred, struct ucred *file_cred,
 {
 	int error;
 
-	MAC_CHECK(posixsem_check_post, active_cred, file_cred, ks,
+	MAC_CHECK_NOSLEEP(posixsem_check_post, active_cred, file_cred, ks,
 	    ks->ks_label);
 	MAC_CHECK_PROBE3(posixsem_check_post, error, active_cred, file_cred,
 	    ks);
@@ -158,7 +158,7 @@ mac_posixsem_check_stat(struct ucred *active_cred, struct ucred *file_cred,
 {
 	int error;
 
-	MAC_CHECK(posixsem_check_stat, active_cred, file_cred, ks,
+	MAC_CHECK_NOSLEEP(posixsem_check_stat, active_cred, file_cred, ks,
 	    ks->ks_label);
 	MAC_CHECK_PROBE3(posixsem_check_stat, error, active_cred, file_cred,
 	    ks);
@@ -174,7 +174,7 @@ mac_posixsem_check_unlink(struct ucred *cred, struct ksem *ks)
 {
 	int error;
 
-	MAC_CHECK(posixsem_check_unlink, cred, ks, ks->ks_label);
+	MAC_CHECK_NOSLEEP(posixsem_check_unlink, cred, ks, ks->ks_label);
 	MAC_CHECK_PROBE2(posixsem_check_unlink, error, cred, ks);
 
 	return (error);
@@ -189,7 +189,7 @@ mac_posixsem_check_wait(struct ucred *active_cred, struct ucred *file_cred,
 {
 	int error;
 
-	MAC_CHECK(posixsem_check_wait, active_cred, file_cred, ks,
+	MAC_CHECK_NOSLEEP(posixsem_check_wait, active_cred, file_cred, ks,
 	    ks->ks_label);
 	MAC_CHECK_PROBE3(posixsem_check_wait, error, active_cred, file_cred,
 	    ks);
diff --git a/sys/security/mac/mac_posix_shm.c b/sys/security/mac/mac_posix_shm.c
index 913cb43..bb9d9dd 100644
--- a/sys/security/mac/mac_posix_shm.c
+++ b/sys/security/mac/mac_posix_shm.c
@@ -79,7 +79,7 @@ static void
 mac_posixshm_label_free(struct label *label)
 {
 
-	MAC_PERFORM(posixshm_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(posixshm_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -97,7 +97,7 @@ void
 mac_posixshm_create(struct ucred *cred, struct shmfd *shmfd)
 {
 
-	MAC_PERFORM(posixshm_create, cred, shmfd, shmfd->shm_label);
+	MAC_PERFORM_NOSLEEP(posixshm_create, cred, shmfd, shmfd->shm_label);
 }
 
 MAC_CHECK_PROBE_DEFINE4(posixshm_check_mmap, "struct ucred *",
@@ -109,8 +109,8 @@ mac_posixshm_check_mmap(struct ucred *cred, struct shmfd *shmfd, int prot,
 {
 	int error;
 
-	MAC_CHECK(posixshm_check_mmap, cred, shmfd, shmfd->shm_label, prot,
-	    flags);
+	MAC_CHECK_NOSLEEP(posixshm_check_mmap, cred, shmfd, shmfd->shm_label,
+	    prot, flags);
 	MAC_CHECK_PROBE4(posixshm_check_mmap, error, cred, shmfd, prot,
 	    flags);
 
@@ -125,7 +125,7 @@ mac_posixshm_check_open(struct ucred *cred, struct shmfd *shmfd)
 {
 	int error;
 
-	MAC_CHECK(posixshm_check_open, cred, shmfd, shmfd->shm_label);
+	MAC_CHECK_NOSLEEP(posixshm_check_open, cred, shmfd, shmfd->shm_label);
 	MAC_CHECK_PROBE2(posixshm_check_open, error, cred, shmfd);
 
 	return (error);
@@ -140,7 +140,7 @@ mac_posixshm_check_stat(struct ucred *active_cred, struct ucred *file_cred,
 {
 	int error;
 
-	MAC_CHECK(posixshm_check_stat, active_cred, file_cred, shmfd,
+	MAC_CHECK_NOSLEEP(posixshm_check_stat, active_cred, file_cred, shmfd,
 	    shmfd->shm_label);
 	MAC_CHECK_PROBE3(posixshm_check_stat, error, active_cred, file_cred,
 	    shmfd);
@@ -157,8 +157,8 @@ mac_posixshm_check_truncate(struct ucred *active_cred, struct ucred *file_cred,
 {
 	int error;
 
-	MAC_CHECK(posixshm_check_truncate, active_cred, file_cred, shmfd,
-	    shmfd->shm_label);
+	MAC_CHECK_NOSLEEP(posixshm_check_truncate, active_cred, file_cred,
+	    shmfd, shmfd->shm_label);
 	MAC_CHECK_PROBE3(posixshm_check_truncate, error, active_cred,
 	    file_cred, shmfd);
 
@@ -173,7 +173,8 @@ mac_posixshm_check_unlink(struct ucred *cred, struct shmfd *shmfd)
 {
 	int error;
 
-	MAC_CHECK(posixshm_check_unlink, cred, shmfd, shmfd->shm_label);
+	MAC_CHECK_NOSLEEP(posixshm_check_unlink, cred, shmfd,
+	    shmfd->shm_label);
 	MAC_CHECK_PROBE2(posixshm_check_unlink, error, cred, shmfd);
 
 	return (error);
diff --git a/sys/security/mac/mac_priv.c b/sys/security/mac/mac_priv.c
index f12b020..afd64b9 100644
--- a/sys/security/mac/mac_priv.c
+++ b/sys/security/mac/mac_priv.c
@@ -72,7 +72,7 @@ mac_priv_check(struct ucred *cred, int priv)
 {
 	int error;
 
-	MAC_CHECK(priv_check, cred, priv);
+	MAC_CHECK_NOSLEEP(priv_check, cred, priv);
 	MAC_CHECK_PROBE2(priv_check, error, cred, priv);
 
 	return (error);
@@ -89,7 +89,7 @@ mac_priv_grant(struct ucred *cred, int priv)
 {
 	int error;
 
-	MAC_GRANT(priv_grant, cred, priv);
+	MAC_GRANT_NOSLEEP(priv_grant, cred, priv);
 	MAC_GRANT_PROBE2(priv_grant, error, cred, priv);
 
 	return (error);
diff --git a/sys/security/mac/mac_process.c b/sys/security/mac/mac_process.c
index 7faa7ae..1c8ec64 100644
--- a/sys/security/mac/mac_process.c
+++ b/sys/security/mac/mac_process.c
@@ -112,7 +112,7 @@ static void
 mac_proc_label_free(struct label *label)
 {
 
-	MAC_PERFORM(proc_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(proc_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -386,7 +386,7 @@ mac_proc_check_debug(struct ucred *cred, struct proc *p)
 
 	PROC_LOCK_ASSERT(p, MA_OWNED);
 
-	MAC_CHECK(proc_check_debug, cred, p);
+	MAC_CHECK_NOSLEEP(proc_check_debug, cred, p);
 	MAC_CHECK_PROBE2(proc_check_debug, error, cred, p);
 
 	return (error);
@@ -401,7 +401,7 @@ mac_proc_check_sched(struct ucred *cred, struct proc *p)
 
 	PROC_LOCK_ASSERT(p, MA_OWNED);
 
-	MAC_CHECK(proc_check_sched, cred, p);
+	MAC_CHECK_NOSLEEP(proc_check_sched, cred, p);
 	MAC_CHECK_PROBE2(proc_check_sched, error, cred, p);
 
 	return (error);
@@ -417,7 +417,7 @@ mac_proc_check_signal(struct ucred *cred, struct proc *p, int signum)
 
 	PROC_LOCK_ASSERT(p, MA_OWNED);
 
-	MAC_CHECK(proc_check_signal, cred, p, signum);
+	MAC_CHECK_NOSLEEP(proc_check_signal, cred, p, signum);
 	MAC_CHECK_PROBE3(proc_check_signal, error, cred, p, signum);
 
 	return (error);
@@ -432,7 +432,7 @@ mac_proc_check_wait(struct ucred *cred, struct proc *p)
 
 	PROC_LOCK_ASSERT(p, MA_OWNED);
 
-	MAC_CHECK(proc_check_wait, cred, p);
+	MAC_CHECK_NOSLEEP(proc_check_wait, cred, p);
 	MAC_CHECK_PROBE2(proc_check_wait, error, cred, p);
 
 	return (error);
diff --git a/sys/security/mac/mac_socket.c b/sys/security/mac/mac_socket.c
index e73ce12..fa24499 100644
--- a/sys/security/mac/mac_socket.c
+++ b/sys/security/mac/mac_socket.c
@@ -100,9 +100,12 @@ mac_socket_label_alloc(int flag)
 	if (label == NULL)
 		return (NULL);
 
-	MAC_CHECK(socket_init_label, label, flag);
+	if (flag & M_WAITOK)
+		MAC_CHECK(socket_init_label, label, flag);
+	else
+		MAC_CHECK_NOSLEEP(socket_init_label, label, flag);
 	if (error) {
-		MAC_PERFORM(socket_destroy_label, label);
+		MAC_PERFORM_NOSLEEP(socket_destroy_label, label);
 		mac_labelzone_free(label);
 		return (NULL);
 	}
@@ -119,9 +122,12 @@ mac_socketpeer_label_alloc(int flag)
 	if (label == NULL)
 		return (NULL);
 
-	MAC_CHECK(socketpeer_init_label, label, flag);
+	if (flag & M_WAITOK)
+		MAC_CHECK(socketpeer_init_label, label, flag);
+	else
+		MAC_CHECK_NOSLEEP(socketpeer_init_label, label, flag);
 	if (error) {
-		MAC_PERFORM(socketpeer_destroy_label, label);
+		MAC_PERFORM_NOSLEEP(socketpeer_destroy_label, label);
 		mac_labelzone_free(label);
 		return (NULL);
 	}
@@ -153,7 +159,7 @@ void
 mac_socket_label_free(struct label *label)
 {
 
-	MAC_PERFORM(socket_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(socket_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -161,7 +167,7 @@ static void
 mac_socketpeer_label_free(struct label *label)
 {
 
-	MAC_PERFORM(socketpeer_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(socketpeer_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -181,7 +187,7 @@ void
 mac_socket_copy_label(struct label *src, struct label *dest)
 {
 
-	MAC_PERFORM(socket_copy_label, src, dest);
+	MAC_PERFORM_NOSLEEP(socket_copy_label, src, dest);
 }
 
 int
@@ -220,7 +226,7 @@ void
 mac_socket_create(struct ucred *cred, struct socket *so)
 {
 
-	MAC_PERFORM(socket_create, cred, so, so->so_label);
+	MAC_PERFORM_NOSLEEP(socket_create, cred, so, so->so_label);
 }
 
 void
@@ -229,7 +235,7 @@ mac_socket_newconn(struct socket *oldso, struct socket *newso)
 
 	SOCK_LOCK_ASSERT(oldso);
 
-	MAC_PERFORM(socket_newconn, oldso, oldso->so_label, newso,
+	MAC_PERFORM_NOSLEEP(socket_newconn, oldso, oldso->so_label, newso,
 	    newso->so_label);
 }
 
@@ -240,7 +246,8 @@ mac_socket_relabel(struct ucred *cred, struct socket *so,
 
 	SOCK_LOCK_ASSERT(so);
 
-	MAC_PERFORM(socket_relabel, cred, so, so->so_label, newlabel);
+	MAC_PERFORM_NOSLEEP(socket_relabel, cred, so, so->so_label,
+	    newlabel);
 }
 
 void
@@ -252,7 +259,7 @@ mac_socketpeer_set_from_mbuf(struct mbuf *m, struct socket *so)
 
 	label = mac_mbuf_to_label(m);
 
-	MAC_PERFORM(socketpeer_set_from_mbuf, m, label, so,
+	MAC_PERFORM_NOSLEEP(socketpeer_set_from_mbuf, m, label, so,
 	    so->so_peerlabel);
 }
 
@@ -265,8 +272,8 @@ mac_socketpeer_set_from_socket(struct socket *oldso, struct socket *newso)
 	 * is the original, and one is the new.  However, it's called in both
 	 * directions, so we can't assert the lock here currently.
 	 */
-	MAC_PERFORM(socketpeer_set_from_socket, oldso, oldso->so_label,
-	    newso, newso->so_peerlabel);
+	MAC_PERFORM_NOSLEEP(socketpeer_set_from_socket, oldso,
+	    oldso->so_label, newso, newso->so_peerlabel);
 }
 
 void
@@ -278,7 +285,7 @@ mac_socket_create_mbuf(struct socket *so, struct mbuf *m)
 
 	label = mac_mbuf_to_label(m);
 
-	MAC_PERFORM(socket_create_mbuf, so, so->so_label, m, label);
+	MAC_PERFORM_NOSLEEP(socket_create_mbuf, so, so->so_label, m, label);
 }
 
 MAC_CHECK_PROBE_DEFINE2(socket_check_accept, "struct ucred *",
@@ -291,7 +298,7 @@ mac_socket_check_accept(struct ucred *cred, struct socket *so)
 
 	SOCK_LOCK_ASSERT(so);
 
-	MAC_CHECK(socket_check_accept, cred, so, so->so_label);
+	MAC_CHECK_NOSLEEP(socket_check_accept, cred, so, so->so_label);
 	MAC_CHECK_PROBE2(socket_check_accept, error, cred, so);
 
 	return (error);
@@ -308,7 +315,7 @@ mac_socket_check_bind(struct ucred *cred, struct socket *so,
 
 	SOCK_LOCK_ASSERT(so);
 
-	MAC_CHECK(socket_check_bind, cred, so, so->so_label, sa);
+	MAC_CHECK_NOSLEEP(socket_check_bind, cred, so, so->so_label, sa);
 	MAC_CHECK_PROBE3(socket_check_bind, error, cred, so, sa);
 
 	return (error);
@@ -325,7 +332,7 @@ mac_socket_check_connect(struct ucred *cred, struct socket *so,
 
 	SOCK_LOCK_ASSERT(so);
 
-	MAC_CHECK(socket_check_connect, cred, so, so->so_label, sa);
+	MAC_CHECK_NOSLEEP(socket_check_connect, cred, so, so->so_label, sa);
 	MAC_CHECK_PROBE3(socket_check_connect, error, cred, so, sa);
 
 	return (error);
@@ -339,7 +346,7 @@ mac_socket_check_create(struct ucred *cred, int domain, int type, int proto)
 {
 	int error;
 
-	MAC_CHECK(socket_check_create, cred, domain, type, proto);
+	MAC_CHECK_NOSLEEP(socket_check_create, cred, domain, type, proto);
 	MAC_CHECK_PROBE4(socket_check_create, error, cred, domain, type,
 	    proto);
 
@@ -359,7 +366,7 @@ mac_socket_check_deliver(struct socket *so, struct mbuf *m)
 
 	label = mac_mbuf_to_label(m);
 
-	MAC_CHECK(socket_check_deliver, so, so->so_label, m, label);
+	MAC_CHECK_NOSLEEP(socket_check_deliver, so, so->so_label, m, label);
 	MAC_CHECK_PROBE2(socket_check_deliver, error, so, m);
 
 	return (error);
@@ -375,7 +382,7 @@ mac_socket_check_listen(struct ucred *cred, struct socket *so)
 
 	SOCK_LOCK_ASSERT(so);
 
-	MAC_CHECK(socket_check_listen, cred, so, so->so_label);
+	MAC_CHECK_NOSLEEP(socket_check_listen, cred, so, so->so_label);
 	MAC_CHECK_PROBE2(socket_check_listen, error, cred, so);
 
 	return (error);
@@ -391,7 +398,7 @@ mac_socket_check_poll(struct ucred *cred, struct socket *so)
 
 	SOCK_LOCK_ASSERT(so);
 
-	MAC_CHECK(socket_check_poll, cred, so, so->so_label);
+	MAC_CHECK_NOSLEEP(socket_check_poll, cred, so, so->so_label);
 	MAC_CHECK_PROBE2(socket_check_poll, error, cred, so);
 
 	return (error);
@@ -407,7 +414,7 @@ mac_socket_check_receive(struct ucred *cred, struct socket *so)
 
 	SOCK_LOCK_ASSERT(so);
 
-	MAC_CHECK(socket_check_receive, cred, so, so->so_label);
+	MAC_CHECK_NOSLEEP(socket_check_receive, cred, so, so->so_label);
 	MAC_CHECK_PROBE2(socket_check_receive, error, cred, so);
 
 	return (error);
@@ -424,7 +431,8 @@ mac_socket_check_relabel(struct ucred *cred, struct socket *so,
 
 	SOCK_LOCK_ASSERT(so);
 
-	MAC_CHECK(socket_check_relabel, cred, so, so->so_label, newlabel);
+	MAC_CHECK_NOSLEEP(socket_check_relabel, cred, so, so->so_label,
+	    newlabel);
 	MAC_CHECK_PROBE3(socket_check_relabel, error, cred, so, newlabel);
 
 	return (error);
@@ -440,7 +448,7 @@ mac_socket_check_send(struct ucred *cred, struct socket *so)
 
 	SOCK_LOCK_ASSERT(so);
 
-	MAC_CHECK(socket_check_send, cred, so, so->so_label);
+	MAC_CHECK_NOSLEEP(socket_check_send, cred, so, so->so_label);
 	MAC_CHECK_PROBE2(socket_check_send, error, cred, so);
 
 	return (error);
@@ -456,7 +464,7 @@ mac_socket_check_stat(struct ucred *cred, struct socket *so)
 
 	SOCK_LOCK_ASSERT(so);
 
-	MAC_CHECK(socket_check_stat, cred, so, so->so_label);
+	MAC_CHECK_NOSLEEP(socket_check_stat, cred, so, so->so_label);
 	MAC_CHECK_PROBE2(socket_check_stat, error, cred, so);
 
 	return (error);
@@ -472,7 +480,7 @@ mac_socket_check_visible(struct ucred *cred, struct socket *so)
 
 	SOCK_LOCK_ASSERT(so);
 
-	MAC_CHECK(socket_check_visible, cred, so, so->so_label);
+	MAC_CHECK_NOSLEEP(socket_check_visible, cred, so, so->so_label);
 	MAC_CHECK_PROBE2(socket_check_visible, error, cred, so);
 
 	return (error);
diff --git a/sys/security/mac/mac_syscalls.c b/sys/security/mac/mac_syscalls.c
index c4b0606..905c43b 100644
--- a/sys/security/mac/mac_syscalls.c
+++ b/sys/security/mac/mac_syscalls.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 1999-2002, 2006 Robert N. M. Watson
+ * Copyright (c) 1999-2002, 2006, 2009 Robert N. M. Watson
  * Copyright (c) 2001 Ilmar S. Habibulin
  * Copyright (c) 2001-2005 Networks Associates Technology, Inc.
  * Copyright (c) 2005-2006 SPARTA, Inc.
@@ -17,6 +17,9 @@
  * This software was enhanced by SPARTA ISSO under SPAWAR contract 
  * N66001-04-C-6019 ("SEFOS").
  *
+ * This software was developed at the University of Cambridge Computer
+ * Laboratory with support from a grant from Google, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -617,7 +620,7 @@ mac_syscall(struct thread *td, struct mac_syscall_args *uap)
 {
 	struct mac_policy_conf *mpc;
 	char target[MAC_MAX_POLICY_NAME];
-	int entrycount, error;
+	int error;
 
 	error = copyinstr(uap->policy, target, sizeof(target), NULL);
 	if (error)
@@ -633,7 +636,8 @@ mac_syscall(struct thread *td, struct mac_syscall_args *uap)
 		}
 	}
 
-	if ((entrycount = mac_policy_list_conditional_busy()) != 0) {
+	if (!LIST_EMPTY(&mac_policy_list)) {
+		mac_policy_slock_sleep();
 		LIST_FOREACH(mpc, &mac_policy_list, mpc_list) {
 			if (strcmp(mpc->mpc_name, target) == 0 &&
 			    mpc->mpc_ops->mpo_syscall != NULL) {
@@ -642,7 +646,7 @@ mac_syscall(struct thread *td, struct mac_syscall_args *uap)
 				break;
 			}
 		}
-		mac_policy_list_unbusy();
+		mac_policy_sunlock_sleep();
 	}
 out:
 	return (error);
diff --git a/sys/security/mac/mac_system.c b/sys/security/mac/mac_system.c
index a8e351e..4a377c0 100644
--- a/sys/security/mac/mac_system.c
+++ b/sys/security/mac/mac_system.c
@@ -78,7 +78,7 @@ mac_kenv_check_dump(struct ucred *cred)
 {
 	int error;
 
-	MAC_CHECK(kenv_check_dump, cred);
+	MAC_CHECK_NOSLEEP(kenv_check_dump, cred);
 	MAC_CHECK_PROBE1(kenv_check_dump, error, cred);
 
 	return (error);
@@ -91,7 +91,7 @@ mac_kenv_check_get(struct ucred *cred, char *name)
 {
 	int error;
 
-	MAC_CHECK(kenv_check_get, cred, name);
+	MAC_CHECK_NOSLEEP(kenv_check_get, cred, name);
 	MAC_CHECK_PROBE2(kenv_check_get, error, cred, name);
 
 	return (error);
@@ -105,7 +105,7 @@ mac_kenv_check_set(struct ucred *cred, char *name, char *value)
 {
 	int error;
 
-	MAC_CHECK(kenv_check_set, cred, name, value);
+	MAC_CHECK_NOSLEEP(kenv_check_set, cred, name, value);
 	MAC_CHECK_PROBE3(kenv_check_set, error, cred, name, value);
 
 	return (error);
@@ -118,7 +118,7 @@ mac_kenv_check_unset(struct ucred *cred, char *name)
 {
 	int error;
 
-	MAC_CHECK(kenv_check_unset, cred, name);
+	MAC_CHECK_NOSLEEP(kenv_check_unset, cred, name);
 	MAC_CHECK_PROBE2(kenv_check_unset, error, cred, name);
 
 	return (error);
@@ -146,7 +146,7 @@ mac_kld_check_stat(struct ucred *cred)
 {
 	int error;
 
-	MAC_CHECK(kld_check_stat, cred);
+	MAC_CHECK_NOSLEEP(kld_check_stat, cred);
 	MAC_CHECK_PROBE1(kld_check_stat, error, cred);
 
 	return (error);
@@ -178,7 +178,7 @@ mac_system_check_reboot(struct ucred *cred, int howto)
 {
 	int error;
 
-	MAC_CHECK(system_check_reboot, cred, howto);
+	MAC_CHECK_NOSLEEP(system_check_reboot, cred, howto);
 	MAC_CHECK_PROBE2(system_check_reboot, error, cred, howto);
 
 	return (error);
@@ -229,7 +229,7 @@ mac_system_check_sysctl(struct ucred *cred, struct sysctl_oid *oidp,
 	 * XXXMAC: We would very much like to assert the SYSCTL_LOCK here,
 	 * but since it's not exported from kern_sysctl.c, we can't.
 	 */
-	MAC_CHECK(system_check_sysctl, cred, oidp, arg1, arg2, req);
+	MAC_CHECK_NOSLEEP(system_check_sysctl, cred, oidp, arg1, arg2, req);
 	MAC_CHECK_PROBE3(system_check_sysctl, error, cred, oidp, req);
 
 	return (error);
diff --git a/sys/security/mac/mac_sysv_msg.c b/sys/security/mac/mac_sysv_msg.c
index 1053871..a1d21d9 100644
--- a/sys/security/mac/mac_sysv_msg.c
+++ b/sys/security/mac/mac_sysv_msg.c
@@ -107,7 +107,7 @@ static void
 mac_sysv_msgmsg_label_free(struct label *label)
 {
 
-	MAC_PERFORM(sysvmsg_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(sysvmsg_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -125,7 +125,7 @@ static void
 mac_sysv_msgqueue_label_free(struct label *label)
 {
 
-	MAC_PERFORM(sysvmsq_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(sysvmsq_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -144,7 +144,7 @@ mac_sysvmsg_create(struct ucred *cred, struct msqid_kernel *msqkptr,
     struct msg *msgptr)
 {
 
-	MAC_PERFORM(sysvmsg_create, cred, msqkptr, msqkptr->label,
+	MAC_PERFORM_NOSLEEP(sysvmsg_create, cred, msqkptr, msqkptr->label,
 		msgptr, msgptr->label);
 }
 
@@ -152,21 +152,21 @@ void
 mac_sysvmsq_create(struct ucred *cred, struct msqid_kernel *msqkptr)
 {
 
-	MAC_PERFORM(sysvmsq_create, cred, msqkptr, msqkptr->label);
+	MAC_PERFORM_NOSLEEP(sysvmsq_create, cred, msqkptr, msqkptr->label);
 }
 
 void
 mac_sysvmsg_cleanup(struct msg *msgptr)
 {
 
-	MAC_PERFORM(sysvmsg_cleanup, msgptr->label);
+	MAC_PERFORM_NOSLEEP(sysvmsg_cleanup, msgptr->label);
 }
 
 void
 mac_sysvmsq_cleanup(struct msqid_kernel *msqkptr)
 {
 
-	MAC_PERFORM(sysvmsq_cleanup, msqkptr->label);
+	MAC_PERFORM_NOSLEEP(sysvmsq_cleanup, msqkptr->label);
 }
 
 MAC_CHECK_PROBE_DEFINE3(sysvmsq_check_msgmsq, "struct ucred *",
@@ -178,7 +178,7 @@ mac_sysvmsq_check_msgmsq(struct ucred *cred, struct msg *msgptr,
 {
 	int error;
 
-	MAC_CHECK(sysvmsq_check_msgmsq, cred, msgptr, msgptr->label,
+	MAC_CHECK_NOSLEEP(sysvmsq_check_msgmsq, cred, msgptr, msgptr->label,
 	    msqkptr, msqkptr->label);
 	MAC_CHECK_PROBE3(sysvmsq_check_msgmsq, error, cred, msgptr, msqkptr);
 
@@ -193,7 +193,7 @@ mac_sysvmsq_check_msgrcv(struct ucred *cred, struct msg *msgptr)
 {
 	int error;
 
-	MAC_CHECK(sysvmsq_check_msgrcv, cred, msgptr, msgptr->label);
+	MAC_CHECK_NOSLEEP(sysvmsq_check_msgrcv, cred, msgptr, msgptr->label);
 	MAC_CHECK_PROBE2(sysvmsq_check_msgrcv, error, cred, msgptr);
 
 	return (error);
@@ -207,7 +207,8 @@ mac_sysvmsq_check_msgrmid(struct ucred *cred, struct msg *msgptr)
 {
 	int error;
 
-	MAC_CHECK(sysvmsq_check_msgrmid, cred, msgptr, msgptr->label);
+	MAC_CHECK_NOSLEEP(sysvmsq_check_msgrmid, cred, msgptr,
+	    msgptr->label);
 	MAC_CHECK_PROBE2(sysvmsq_check_msgrmid, error, cred, msgptr);
 
 	return (error);
@@ -221,7 +222,8 @@ mac_sysvmsq_check_msqget(struct ucred *cred, struct msqid_kernel *msqkptr)
 {
 	int error;
 
-	MAC_CHECK(sysvmsq_check_msqget, cred, msqkptr, msqkptr->label);
+	MAC_CHECK_NOSLEEP(sysvmsq_check_msqget, cred, msqkptr,
+	    msqkptr->label);
 	MAC_CHECK_PROBE2(sysvmsq_check_msqget, error, cred, msqkptr);
 
 	return (error);
@@ -235,7 +237,8 @@ mac_sysvmsq_check_msqsnd(struct ucred *cred, struct msqid_kernel *msqkptr)
 {
 	int error;
 
-	MAC_CHECK(sysvmsq_check_msqsnd, cred, msqkptr, msqkptr->label);
+	MAC_CHECK_NOSLEEP(sysvmsq_check_msqsnd, cred, msqkptr,
+	    msqkptr->label);
 	MAC_CHECK_PROBE2(sysvmsq_check_msqsnd, error, cred, msqkptr);
 
 	return (error);
@@ -249,7 +252,8 @@ mac_sysvmsq_check_msqrcv(struct ucred *cred, struct msqid_kernel *msqkptr)
 {
 	int error;
 
-	MAC_CHECK(sysvmsq_check_msqrcv, cred, msqkptr, msqkptr->label);
+	MAC_CHECK_NOSLEEP(sysvmsq_check_msqrcv, cred, msqkptr,
+	    msqkptr->label);
 	MAC_CHECK_PROBE2(sysvmsq_check_msqrcv, error, cred, msqkptr);
 
 	return (error);
@@ -264,7 +268,8 @@ mac_sysvmsq_check_msqctl(struct ucred *cred, struct msqid_kernel *msqkptr,
 {
 	int error;
 
-	MAC_CHECK(sysvmsq_check_msqctl, cred, msqkptr, msqkptr->label, cmd);
+	MAC_CHECK_NOSLEEP(sysvmsq_check_msqctl, cred, msqkptr,
+	    msqkptr->label, cmd);
 	MAC_CHECK_PROBE3(sysvmsq_check_msqctl, error, cred, msqkptr, cmd);
 
 	return (error);
diff --git a/sys/security/mac/mac_sysv_sem.c b/sys/security/mac/mac_sysv_sem.c
index 9fc13fa..add17da 100644
--- a/sys/security/mac/mac_sysv_sem.c
+++ b/sys/security/mac/mac_sysv_sem.c
@@ -86,7 +86,7 @@ static void
 mac_sysv_sem_label_free(struct label *label)
 {
 
-	MAC_PERFORM(sysvsem_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(sysvsem_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -104,14 +104,14 @@ void
 mac_sysvsem_create(struct ucred *cred, struct semid_kernel *semakptr)
 {
 
-	MAC_PERFORM(sysvsem_create, cred, semakptr, semakptr->label);
+	MAC_PERFORM_NOSLEEP(sysvsem_create, cred, semakptr, semakptr->label);
 }
 
 void
 mac_sysvsem_cleanup(struct semid_kernel *semakptr)
 {
 
-	MAC_PERFORM(sysvsem_cleanup, semakptr->label);
+	MAC_PERFORM_NOSLEEP(sysvsem_cleanup, semakptr->label);
 }
 
 MAC_CHECK_PROBE_DEFINE3(sysvsem_check_semctl, "struct ucred *",
@@ -123,8 +123,8 @@ mac_sysvsem_check_semctl(struct ucred *cred, struct semid_kernel *semakptr,
 {
 	int error;
 
-	MAC_CHECK(sysvsem_check_semctl, cred, semakptr, semakptr->label,
-	    cmd);
+	MAC_CHECK_NOSLEEP(sysvsem_check_semctl, cred, semakptr,
+	    semakptr->label, cmd);
 	MAC_CHECK_PROBE3(sysvsem_check_semctl, error, cred, semakptr, cmd);
 
 	return (error);
@@ -138,7 +138,8 @@ mac_sysvsem_check_semget(struct ucred *cred, struct semid_kernel *semakptr)
 {
 	int error;
 
-	MAC_CHECK(sysvsem_check_semget, cred, semakptr, semakptr->label);
+	MAC_CHECK_NOSLEEP(sysvsem_check_semget, cred, semakptr,
+	    semakptr->label);
 
 	return (error);
 }
@@ -152,8 +153,8 @@ mac_sysvsem_check_semop(struct ucred *cred, struct semid_kernel *semakptr,
 {
 	int error;
 
-	MAC_CHECK(sysvsem_check_semop, cred, semakptr, semakptr->label,
-	    accesstype);
+	MAC_CHECK_NOSLEEP(sysvsem_check_semop, cred, semakptr,
+	    semakptr->label, accesstype);
 	MAC_CHECK_PROBE3(sysvsem_check_semop, error, cred, semakptr,
 	    accesstype);
 
diff --git a/sys/security/mac/mac_sysv_shm.c b/sys/security/mac/mac_sysv_shm.c
index d42cb0b..57c2264 100644
--- a/sys/security/mac/mac_sysv_shm.c
+++ b/sys/security/mac/mac_sysv_shm.c
@@ -86,7 +86,7 @@ static void
 mac_sysv_shm_label_free(struct label *label)
 {
 
-	MAC_PERFORM(sysvshm_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(sysvshm_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -104,14 +104,15 @@ void
 mac_sysvshm_create(struct ucred *cred, struct shmid_kernel *shmsegptr)
 {
 
-	MAC_PERFORM(sysvshm_create, cred, shmsegptr, shmsegptr->label);
+	MAC_PERFORM_NOSLEEP(sysvshm_create, cred, shmsegptr,
+	    shmsegptr->label);
 }
 
 void
 mac_sysvshm_cleanup(struct shmid_kernel *shmsegptr)
 {
 
-	MAC_PERFORM(sysvshm_cleanup, shmsegptr->label);
+	MAC_PERFORM_NOSLEEP(sysvshm_cleanup, shmsegptr->label);
 }
 
 MAC_CHECK_PROBE_DEFINE3(sysvshm_check_shmat, "struct ucred *",
@@ -123,8 +124,8 @@ mac_sysvshm_check_shmat(struct ucred *cred, struct shmid_kernel *shmsegptr,
 {
 	int error;
 
-	MAC_CHECK(sysvshm_check_shmat, cred, shmsegptr, shmsegptr->label,
-	    shmflg);
+	MAC_CHECK_NOSLEEP(sysvshm_check_shmat, cred, shmsegptr,
+	    shmsegptr->label, shmflg);
 	MAC_CHECK_PROBE3(sysvshm_check_shmat, error, cred, shmsegptr,
 	    shmflg);
 
@@ -140,8 +141,8 @@ mac_sysvshm_check_shmctl(struct ucred *cred, struct shmid_kernel *shmsegptr,
 {
 	int error;
 
-	MAC_CHECK(sysvshm_check_shmctl, cred, shmsegptr, shmsegptr->label,
-	    cmd);
+	MAC_CHECK_NOSLEEP(sysvshm_check_shmctl, cred, shmsegptr,
+	    shmsegptr->label, cmd);
 	MAC_CHECK_PROBE3(sysvshm_check_shmctl, error, cred, shmsegptr, cmd);
 
 	return (error);
@@ -155,7 +156,8 @@ mac_sysvshm_check_shmdt(struct ucred *cred, struct shmid_kernel *shmsegptr)
 {
 	int error;
 
-	MAC_CHECK(sysvshm_check_shmdt, cred, shmsegptr, shmsegptr->label);
+	MAC_CHECK_NOSLEEP(sysvshm_check_shmdt, cred, shmsegptr,
+	    shmsegptr->label);
 	MAC_CHECK_PROBE2(sysvshm_check_shmdt, error, cred, shmsegptr);
 
 	return (error);
@@ -170,8 +172,8 @@ mac_sysvshm_check_shmget(struct ucred *cred, struct shmid_kernel *shmsegptr,
 {
 	int error;
 
-	MAC_CHECK(sysvshm_check_shmget, cred, shmsegptr, shmsegptr->label,
-	    shmflg);
+	MAC_CHECK_NOSLEEP(sysvshm_check_shmget, cred, shmsegptr,
+	    shmsegptr->label, shmflg);
 	MAC_CHECK_PROBE3(sysvshm_check_shmget, error, cred, shmsegptr,
 	    shmflg);
 
diff --git a/sys/security/mac/mac_vfs.c b/sys/security/mac/mac_vfs.c
index 01afb81..0cb0f15 100644
--- a/sys/security/mac/mac_vfs.c
+++ b/sys/security/mac/mac_vfs.c
@@ -150,7 +150,7 @@ static void
 mac_devfs_label_free(struct label *label)
 {
 
-	MAC_PERFORM(devfs_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(devfs_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -168,7 +168,7 @@ static void
 mac_mount_label_free(struct label *label)
 {
 
-	MAC_PERFORM(mount_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(mount_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -186,7 +186,7 @@ void
 mac_vnode_label_free(struct label *label)
 {
 
-	MAC_PERFORM(vnode_destroy_label, label);
+	MAC_PERFORM_NOSLEEP(vnode_destroy_label, label);
 	mac_labelzone_free(label);
 }
 
@@ -204,7 +204,7 @@ void
 mac_vnode_copy_label(struct label *src, struct label *dest)
 {
 
-	MAC_PERFORM(vnode_copy_label, src, dest);
+	MAC_PERFORM_NOSLEEP(vnode_copy_label, src, dest);
 }
 
 int
@@ -232,7 +232,8 @@ void
 mac_devfs_update(struct mount *mp, struct devfs_dirent *de, struct vnode *vp)
 {
 
-	MAC_PERFORM(devfs_update, mp, de, de->de_label, vp, vp->v_label);
+	MAC_PERFORM_NOSLEEP(devfs_update, mp, de, de->de_label, vp,
+	    vp->v_label);
 }
 
 void
@@ -240,7 +241,7 @@ mac_devfs_vnode_associate(struct mount *mp, struct devfs_dirent *de,
     struct vnode *vp)
 {
 
-	MAC_PERFORM(devfs_vnode_associate, mp, mp->mnt_label, de,
+	MAC_PERFORM_NOSLEEP(devfs_vnode_associate, mp, mp->mnt_label, de,
 	    de->de_label, vp, vp->v_label);
 }
 
@@ -261,8 +262,8 @@ void
 mac_vnode_associate_singlelabel(struct mount *mp, struct vnode *vp)
 {
 
-	MAC_PERFORM(vnode_associate_singlelabel, mp, mp->mnt_label, vp,
-	    vp->v_label);
+	MAC_PERFORM_NOSLEEP(vnode_associate_singlelabel, mp, mp->mnt_label,
+	    vp, vp->v_label);
 }
 
 /*
@@ -360,8 +361,9 @@ mac_vnode_execve_will_transition(struct ucred *old, struct vnode *vp,
 	ASSERT_VOP_LOCKED(vp, "mac_vnode_execve_will_transition");
 
 	result = 0;
-	MAC_BOOLEAN(vnode_execve_will_transition, ||, old, vp, vp->v_label,
-	    interpvplabel, imgp, imgp->execlabel);
+	/* No sleeping since the process lock will be held by the caller. */
+	MAC_BOOLEAN_NOSLEEP(vnode_execve_will_transition, ||, old, vp,
+	    vp->v_label, interpvplabel, imgp, imgp->execlabel);
 
 	return (result);
 }
@@ -960,7 +962,7 @@ mac_mount_check_stat(struct ucred *cred, struct mount *mount)
 {
 	int error;
 
-	MAC_CHECK(mount_check_stat, cred, mount, mount->mnt_label);
+	MAC_CHECK_NOSLEEP(mount_check_stat, cred, mount, mount->mnt_label);
 	MAC_CHECK_PROBE2(mount_check_stat, error, cred, mount);
 
 	return (error);
@@ -971,7 +973,8 @@ mac_devfs_create_device(struct ucred *cred, struct mount *mp,
     struct cdev *dev, struct devfs_dirent *de)
 {
 
-	MAC_PERFORM(devfs_create_device, cred, mp, dev, de, de->de_label);
+	MAC_PERFORM_NOSLEEP(devfs_create_device, cred, mp, dev, de,
+	    de->de_label);
 }
 
 void
@@ -979,8 +982,8 @@ mac_devfs_create_symlink(struct ucred *cred, struct mount *mp,
     struct devfs_dirent *dd, struct devfs_dirent *de)
 {
 
-	MAC_PERFORM(devfs_create_symlink, cred, mp, dd, dd->de_label, de,
-	    de->de_label);
+	MAC_PERFORM_NOSLEEP(devfs_create_symlink, cred, mp, dd,
+	    dd->de_label, de, de->de_label);
 }
 
 void
@@ -988,8 +991,8 @@ mac_devfs_create_directory(struct mount *mp, char *dirname, int dirnamelen,
     struct devfs_dirent *de)
 {
 
-	MAC_PERFORM(devfs_create_directory, mp, dirname, dirnamelen, de,
-	    de->de_label);
+	MAC_PERFORM_NOSLEEP(devfs_create_directory, mp, dirname, dirnamelen,
+	    de, de->de_label);
 }
 
 /*
diff --git a/sys/security/mac_biba/mac_biba.c b/sys/security/mac_biba/mac_biba.c
index 41c1eea..97c3cbe 100644
--- a/sys/security/mac_biba/mac_biba.c
+++ b/sys/security/mac_biba/mac_biba.c
@@ -2892,11 +2892,11 @@ biba_vnode_check_open(struct ucred *cred, struct vnode *vp,
 	obj = SLOT(vplabel);
 
 	/* XXX privilege override for admin? */
-	if (accmode & (VREAD | VEXEC | VSTAT)) {
+	if (accmode & (VREAD | VEXEC | VSTAT_PERMS)) {
 		if (!biba_dominate_effective(obj, subj))
 			return (EACCES);
 	}
-	if (accmode & (VWRITE | VAPPEND | VADMIN)) {
+	if (accmode & VMODIFY_PERMS) {
 		if (!biba_dominate_effective(subj, obj))
 			return (EACCES);
 	}
diff --git a/sys/security/mac_bsdextended/mac_bsdextended.c b/sys/security/mac_bsdextended/mac_bsdextended.c
index ee72df4..cc36851 100644
--- a/sys/security/mac_bsdextended/mac_bsdextended.c
+++ b/sys/security/mac_bsdextended/mac_bsdextended.c
@@ -478,9 +478,9 @@ ugidfw_accmode2mbi(accmode_t accmode)
 		mbi |= MBI_WRITE;
 	if (accmode & VREAD)
 		mbi |= MBI_READ;
-	if (accmode & VADMIN)
+	if (accmode & VADMIN_PERMS)
 		mbi |= MBI_ADMIN;
-	if (accmode & VSTAT)
+	if (accmode & VSTAT_PERMS)
 		mbi |= MBI_STAT;
 	if (accmode & VAPPEND)
 		mbi |= MBI_APPEND;
diff --git a/sys/security/mac_mls/mac_mls.c b/sys/security/mac_mls/mac_mls.c
index 81030d7..6d13505 100644
--- a/sys/security/mac_mls/mac_mls.c
+++ b/sys/security/mac_mls/mac_mls.c
@@ -2515,11 +2515,11 @@ mls_vnode_check_open(struct ucred *cred, struct vnode *vp,
 	obj = SLOT(vplabel);
 
 	/* XXX privilege override for admin? */
-	if (accmode & (VREAD | VEXEC | VSTAT)) {
+	if (accmode & (VREAD | VEXEC | VSTAT_PERMS)) {
 		if (!mls_dominate_effective(subj, obj))
 			return (EACCES);
 	}
-	if (accmode & (VWRITE | VAPPEND | VADMIN)) {
+	if (accmode & VMODIFY_PERMS) {
 		if (!mls_dominate_effective(obj, subj))
 			return (EACCES);
 	}
diff --git a/sys/security/mac_portacl/mac_portacl.c b/sys/security/mac_portacl/mac_portacl.c
index aceda69..f54319a 100644
--- a/sys/security/mac_portacl/mac_portacl.c
+++ b/sys/security/mac_portacl/mac_portacl.c
@@ -341,10 +341,12 @@ sysctl_rules(SYSCTL_HANDLER_ARGS)
 	int error;
 
 	new_string = NULL;
-	if (req->newptr == NULL) {
+	if (req->newptr != NULL) {
 		new_string = malloc(MAC_RULE_STRING_LEN, M_PORTACL,
 		    M_WAITOK | M_ZERO);
+		mtx_lock(&rule_mtx);
 		strcpy(new_string, rule_string);
+		mtx_unlock(&rule_mtx);
 		string = new_string;
 	} else
 		string = rule_string;
diff --git a/sys/sparc64/central/central.c b/sys/sparc64/central/central.c
index 48cf065..4ded549 100644
--- a/sys/sparc64/central/central.c
+++ b/sys/sparc64/central/central.c
@@ -76,14 +76,15 @@ static device_method_t central_methods[] = {
 	/* Bus interface */
 	DEVMETHOD(bus_print_child,	central_print_child),
 	DEVMETHOD(bus_probe_nomatch,	central_probe_nomatch),
-	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
-	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_alloc_resource,	central_alloc_resource),
-	DEVMETHOD(bus_release_resource,	bus_generic_rl_release_resource),
 	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
-	DEVMETHOD(bus_get_resource_list, central_get_resource_list),
+	DEVMETHOD(bus_release_resource,	bus_generic_rl_release_resource),
+	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
+	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_get_resource,	bus_generic_rl_get_resource),
+	DEVMETHOD(bus_get_resource_list, central_get_resource_list),
+	DEVMETHOD(bus_child_pnpinfo_str, ofw_bus_gen_child_pnpinfo_str),
 
 	/* ofw_bus interface */
 	DEVMETHOD(ofw_bus_get_devinfo,	central_get_devinfo),
@@ -93,7 +94,7 @@ static device_method_t central_methods[] = {
 	DEVMETHOD(ofw_bus_get_node,	ofw_bus_gen_get_node),
 	DEVMETHOD(ofw_bus_get_type,	ofw_bus_gen_get_type),
 
-	{ NULL, NULL }
+	KOBJMETHOD_END
 };
 
 static driver_t central_driver = {
diff --git a/sys/sparc64/conf/GENERIC b/sys/sparc64/conf/GENERIC
index 0c051fb..46c363e 100644
--- a/sys/sparc64/conf/GENERIC
+++ b/sys/sparc64/conf/GENERIC
@@ -230,7 +230,6 @@ device		ukbd		# Keyboard
 #device		umass		# Disks/Mass storage - Requires scbus and da
 device		ums		# Mouse
 #device		urio		# Diamond Rio 500 MP3 player
-#device		uscanner	# Scanners
 # USB Ethernet, requires mii
 #device		aue		# ADMtek USB Ethernet
 #device		axe		# ASIX Electronics USB Ethernet
diff --git a/sys/sparc64/ebus/ebus.c b/sys/sparc64/ebus/ebus.c
index 9baa7a8..3bafaae 100644
--- a/sys/sparc64/ebus/ebus.c
+++ b/sys/sparc64/ebus/ebus.c
@@ -109,14 +109,14 @@ static device_method_t ebus_methods[] = {
 	/* Bus interface */
 	DEVMETHOD(bus_print_child,	ebus_print_child),
 	DEVMETHOD(bus_probe_nomatch,	ebus_probe_nomatch),
-	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
-	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_alloc_resource,	ebus_alloc_resource),
-	DEVMETHOD(bus_get_resource_list, ebus_get_resource_list),
 	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
 	DEVMETHOD(bus_release_resource,	ebus_release_resource),
+	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
+	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_get_resource,	bus_generic_rl_get_resource),
+	DEVMETHOD(bus_get_resource_list, ebus_get_resource_list),
 	DEVMETHOD(bus_child_pnpinfo_str, ofw_bus_gen_child_pnpinfo_str),
 
 	/* ofw_bus interface */
@@ -127,7 +127,7 @@ static device_method_t ebus_methods[] = {
 	DEVMETHOD(ofw_bus_get_node,	ofw_bus_gen_get_node),
 	DEVMETHOD(ofw_bus_get_type,	ofw_bus_gen_get_type),
 
-	{ 0, 0 }
+	KOBJMETHOD_END
 };
 
 static driver_t ebus_driver = {
diff --git a/sys/sparc64/fhc/fhc.c b/sys/sparc64/fhc/fhc.c
index 84edbd4..91416ec 100644
--- a/sys/sparc64/fhc/fhc.c
+++ b/sys/sparc64/fhc/fhc.c
@@ -90,14 +90,15 @@ static device_method_t fhc_methods[] = {
 	/* Bus interface */
 	DEVMETHOD(bus_print_child,	fhc_print_child),
 	DEVMETHOD(bus_probe_nomatch,	fhc_probe_nomatch),
-	DEVMETHOD(bus_setup_intr,	fhc_setup_intr),
-	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_alloc_resource,	fhc_alloc_resource),
-	DEVMETHOD(bus_release_resource,	bus_generic_rl_release_resource),
 	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
-	DEVMETHOD(bus_get_resource_list, fhc_get_resource_list),
+	DEVMETHOD(bus_release_resource,	bus_generic_rl_release_resource),
+	DEVMETHOD(bus_setup_intr,	fhc_setup_intr),
+	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_get_resource,	bus_generic_rl_get_resource),
+	DEVMETHOD(bus_get_resource_list, fhc_get_resource_list),
+	DEVMETHOD(bus_child_pnpinfo_str, ofw_bus_gen_child_pnpinfo_str),
 
 	/* ofw_bus interface */
 	DEVMETHOD(ofw_bus_get_devinfo,	fhc_get_devinfo),
@@ -107,7 +108,7 @@ static device_method_t fhc_methods[] = {
 	DEVMETHOD(ofw_bus_get_node,	ofw_bus_gen_get_node),
 	DEVMETHOD(ofw_bus_get_type,	ofw_bus_gen_get_type),
 
-	{ NULL, NULL }
+	KOBJMETHOD_END
 };
 
 static driver_t fhc_driver = {
@@ -165,9 +166,7 @@ fhc_attach(device_t dev)
 	int central;
 	int error;
 	int i;
-	int nintr;
-	int nreg;
-	int rid;
+	int j;
 
 	sc = device_get_softc(dev);
 	node = ofw_bus_get_node(dev);
@@ -177,9 +176,9 @@ fhc_attach(device_t dev)
 		central = 1;
 
 	for (i = 0; i < FHC_NREG; i++) {
-		rid = i;
+		j = i;
 		sc->sc_memres[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
-		    &rid, RF_ACTIVE);
+		    &j, RF_ACTIVE);
 		if (sc->sc_memres[i] == NULL) {
 			device_printf(dev, "cannot allocate resource %d\n", i);
 			error = ENXIO;
@@ -239,7 +238,8 @@ fhc_attach(device_t dev)
 		/*
 		 * Hunt through all the interrupt mapping regs and register
 		 * our interrupt controller for the corresponding interrupt
-		 * vectors.
+		 * vectors.  We do this early in order to be able to catch
+		 * stray interrupts.
 		 */
 		for (i = FHC_FANFAIL; i <= FHC_TOD; i++) {
 			fica = malloc(sizeof(*fica), M_DEVBUF, M_NOWAIT);
@@ -259,11 +259,13 @@ fhc_attach(device_t dev)
 			 * the IGN and the IGN is constant for all devices
 			 * on that FireHose controller.
 			 */
-			if (intr_controller_register(INTMAP_VEC(sc->sc_ign,
+			j = intr_controller_register(INTMAP_VEC(sc->sc_ign,
 			    INTINO(bus_read_4(fica->fica_memres, FHC_IMAP))),
-			    &fhc_ic, fica) != 0)
-				panic("%s: could not register interrupt "
-				    "controller for map %d", __func__, i);
+			    &fhc_ic, fica);
+			if (j != 0)
+				device_printf(dev, "could not register "
+				    "interrupt controller for map %d (%d)\n",
+				    i, j);
 		}
 	} else {
 		snprintf(ledname, sizeof(ledname), "board%d", board);
@@ -276,9 +278,9 @@ fhc_attach(device_t dev)
 			free(fdi, M_DEVBUF);
 			continue;
 		}
-		nreg = OF_getprop_alloc(child, "reg", sizeof(*reg),
+		i = OF_getprop_alloc(child, "reg", sizeof(*reg),
 		    (void **)&reg);
-		if (nreg == -1) {
+		if (i == -1) {
 			device_printf(dev, "<%s>: incomplete\n",
 			    fdi->fdi_obdinfo.obd_name);
 			ofw_bus_gen_destroy_devinfo(&fdi->fdi_obdinfo);
@@ -286,19 +288,19 @@ fhc_attach(device_t dev)
 			continue;
 		}
 		resource_list_init(&fdi->fdi_rl);
-		for (i = 0; i < nreg; i++)
-			resource_list_add(&fdi->fdi_rl, SYS_RES_MEMORY, i,
-			    reg[i].sbr_offset, reg[i].sbr_offset +
-			    reg[i].sbr_size, reg[i].sbr_size);
+		for (j = 0; j < i; j++)
+			resource_list_add(&fdi->fdi_rl, SYS_RES_MEMORY, j,
+			    reg[j].sbr_offset, reg[j].sbr_offset +
+			    reg[j].sbr_size, reg[j].sbr_size);
 		free(reg, M_OFWPROP);
 		if (central == 1) {
-			nintr = OF_getprop_alloc(child, "interrupts",
+			i = OF_getprop_alloc(child, "interrupts",
 			    sizeof(*intr), (void **)&intr);
-			if (nintr != -1) {
-				for (i = 0; i < nintr; i++) {
-					iv = INTMAP_VEC(sc->sc_ign, intr[i]);
+			if (i != -1) {
+				for (j = 0; j < i; j++) {
+					iv = INTMAP_VEC(sc->sc_ign, intr[j]);
 					resource_list_add(&fdi->fdi_rl,
-					    SYS_RES_IRQ, i, iv, iv, 1);
+					    SYS_RES_IRQ, j, iv, iv, 1);
 				}
 				free(intr, M_OFWPROP);
 			}
diff --git a/sys/sparc64/include/elf.h b/sys/sparc64/include/elf.h
index 108ade1..c0fcbee 100644
--- a/sys/sparc64/include/elf.h
+++ b/sys/sparc64/include/elf.h
@@ -78,16 +78,14 @@ __ElfType(Auxinfo);
 #define	AT_BASE		7	/* Interpreter's base address. */
 #define	AT_FLAGS	8	/* Flags (unused). */
 #define	AT_ENTRY	9	/* Where interpreter should transfer control. */
-/*
- * The following non-standard values are used in Linux ELF binaries.
- */
 #define	AT_NOTELF	10	/* Program is not ELF ?? */
 #define	AT_UID		11	/* Real uid. */
 #define	AT_EUID		12	/* Effective uid. */
 #define	AT_GID		13	/* Real gid. */
 #define	AT_EGID		14	/* Effective gid. */
+#define	AT_EXECPATH	15	/* Path to the executable. */
 
-#define	AT_COUNT	15	/* Count of defined aux entry types. */
+#define	AT_COUNT	16	/* Count of defined aux entry types. */
 
 /* Define "machine" characteristics */
 #if __ELF_WORD_SIZE == 32
diff --git a/sys/sparc64/include/trap.h b/sys/sparc64/include/trap.h
index f73340c..dd0e6dc 100644
--- a/sys/sparc64/include/trap.h
+++ b/sys/sparc64/include/trap.h
@@ -91,7 +91,7 @@
 
 #ifndef LOCORE
 void sun4u_set_traptable(void *tba_addr);
-extern const char *trap_msg[];
+extern const char *const trap_msg[];
 #endif
 
 #endif
diff --git a/sys/sparc64/isa/isa.c b/sys/sparc64/isa/isa.c
index be76e01..724b4a8 100644
--- a/sys/sparc64/isa/isa.c
+++ b/sys/sparc64/isa/isa.c
@@ -118,7 +118,7 @@ isa_init(device_t dev)
 static const struct {
 	const char	*name;
 	uint32_t	id;
-} ofw_isa_pnp_map[] = {
+} const ofw_isa_pnp_map[] = {
 	{ "SUNW,lomh",	0x0000ae4e }, /* SUN0000 */
 	{ "dma",	0x0002d041 }, /* PNP0200 */
 	{ "floppy",	0x0007d041 }, /* PNP0700 */
@@ -130,6 +130,7 @@ static const struct {
 	{ "rmc-comm",	0x0300ae4e }, /* SUN0003 */
 	{ "kb_ps2",	0x0303d041 }, /* PNP0303 */
 	{ "kdmouse",	0x030fd041 }, /* PNP0F03 */
+	{ "bscbus",	0x0400ae4e }, /* SUN0004 */
 	{ "power",	0x0c0cd041 }, /* PNP0C0C */
 	{ NULL,		0x0 }
 };
diff --git a/sys/sparc64/isa/ofw_isa.c b/sys/sparc64/isa/ofw_isa.c
index de90869..de5e4ce 100644
--- a/sys/sparc64/isa/ofw_isa.c
+++ b/sys/sparc64/isa/ofw_isa.c
@@ -40,8 +40,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/bus.h>
 
-#include <dev/ofw/openfirm.h>
 #include <dev/ofw/ofw_bus_subr.h>
+#include <dev/ofw/openfirm.h>
 
 #include <machine/bus.h>
 #include <machine/resource.h>
diff --git a/sys/sparc64/pci/apb.c b/sys/sparc64/pci/apb.c
index ba0698c..ad4dfca 100644
--- a/sys/sparc64/pci/apb.c
+++ b/sys/sparc64/pci/apb.c
@@ -90,9 +90,9 @@ static device_method_t apb_methods[] = {
 	DEVMETHOD(bus_read_ivar,	pcib_read_ivar),
 	DEVMETHOD(bus_write_ivar,	pcib_write_ivar),
 	DEVMETHOD(bus_alloc_resource,	apb_alloc_resource),
-	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
 	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
+	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
 	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 
@@ -105,7 +105,7 @@ static device_method_t apb_methods[] = {
 	/* ofw_bus interface */
 	DEVMETHOD(ofw_bus_get_node,	ofw_pcib_gen_get_node),
 
-	{ 0, 0 }
+	KOBJMETHOD_END
 };
 
 static devclass_t pcib_devclass;
diff --git a/sys/sparc64/pci/ofw_pcib.c b/sys/sparc64/pci/ofw_pcib.c
index 8a5b991..3be57e3 100644
--- a/sys/sparc64/pci/ofw_pcib.c
+++ b/sys/sparc64/pci/ofw_pcib.c
@@ -71,9 +71,9 @@ static device_method_t ofw_pcib_methods[] = {
 	DEVMETHOD(bus_read_ivar,	pcib_read_ivar),
 	DEVMETHOD(bus_write_ivar,	pcib_write_ivar),
 	DEVMETHOD(bus_alloc_resource,	pcib_alloc_resource),
-	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
 	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
+	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
 	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 
@@ -86,7 +86,7 @@ static device_method_t ofw_pcib_methods[] = {
 	/* ofw_bus interface */
 	DEVMETHOD(ofw_bus_get_node,	ofw_pcib_gen_get_node),
 
-	{ 0, 0 }
+	KOBJMETHOD_END
 };
 
 static devclass_t pcib_devclass;
diff --git a/sys/sparc64/pci/ofw_pcibus.c b/sys/sparc64/pci/ofw_pcibus.c
index 65a498a..bd2c7a6 100644
--- a/sys/sparc64/pci/ofw_pcibus.c
+++ b/sys/sparc64/pci/ofw_pcibus.c
@@ -68,8 +68,7 @@ static device_probe_t ofw_pcibus_probe;
 static device_attach_t ofw_pcibus_attach;
 static pci_assign_interrupt_t ofw_pcibus_assign_interrupt;
 static ofw_bus_get_devinfo_t ofw_pcibus_get_devinfo;
-static int ofw_pcibus_child_pnpinfo_str_method(device_t cbdev, device_t child, 
-    char *buf, size_t buflen);
+static bus_child_pnpinfo_str_t ofw_pcibus_pnpinfo_str;
 
 static device_method_t ofw_pcibus_methods[] = {
 	/* Device interface */
@@ -77,7 +76,7 @@ static device_method_t ofw_pcibus_methods[] = {
 	DEVMETHOD(device_attach,	ofw_pcibus_attach),
 
 	/* Bus interface */
-	DEVMETHOD(bus_child_pnpinfo_str, ofw_pcibus_child_pnpinfo_str_method),
+	DEVMETHOD(bus_child_pnpinfo_str, ofw_pcibus_pnpinfo_str),
 
 	/* PCI interface */
 	DEVMETHOD(pci_assign_interrupt, ofw_pcibus_assign_interrupt),
@@ -90,7 +89,7 @@ static device_method_t ofw_pcibus_methods[] = {
 	DEVMETHOD(ofw_bus_get_node,	ofw_bus_gen_get_node),
 	DEVMETHOD(ofw_bus_get_type,	ofw_bus_gen_get_type),
 
-	{ 0, 0 }
+	KOBJMETHOD_END
 };
 
 struct ofw_pcibus_devinfo {
@@ -306,16 +305,15 @@ ofw_pcibus_get_devinfo(device_t bus, device_t dev)
 }
 
 static int
-ofw_pcibus_child_pnpinfo_str_method(device_t cbdev, device_t child, char *buf,
+ofw_pcibus_pnpinfo_str(device_t dev, device_t child, char *buf,
     size_t buflen)
 {
-	pci_child_pnpinfo_str_method(cbdev, child, buf, buflen);
 
+	pci_child_pnpinfo_str_method(dev, child, buf, buflen);
 	if (ofw_bus_get_node(child) != -1)  {
-		strlcat(buf, " ", buflen); /* Separate info */
-		ofw_bus_gen_child_pnpinfo_str(cbdev, child, buf, buflen);
+		strlcat(buf, " ", buflen); /* Separate info. */
+		ofw_bus_gen_child_pnpinfo_str(dev, child, buf, buflen);
 	}
 
 	return (0);
 }
-
diff --git a/sys/sparc64/pci/psycho.c b/sys/sparc64/pci/psycho.c
index 505eec4..c8caec2 100644
--- a/sys/sparc64/pci/psycho.c
+++ b/sys/sparc64/pci/psycho.c
@@ -83,7 +83,7 @@ static void psycho_set_intr(struct psycho_softc *, u_int, bus_addr_t,
     driver_filter_t, driver_intr_t);
 static int psycho_find_intrmap(struct psycho_softc *, u_int, bus_addr_t *,
     bus_addr_t *, u_long *);
-static driver_filter_t psycho_dmasync;
+static driver_filter_t psycho_dma_sync_stub;
 static void psycho_intr_enable(void *);
 static void psycho_intr_disable(void *);
 static void psycho_intr_assign(void *);
@@ -150,7 +150,7 @@ static device_method_t psycho_methods[] = {
 	/* ofw_bus interface */
 	DEVMETHOD(ofw_bus_get_node,	psycho_get_node),
 
-	{ 0, 0 }
+	KOBJMETHOD_END
 };
 
 static devclass_t psycho_devclass;
@@ -175,7 +175,7 @@ struct psycho_icarg {
 	bus_addr_t		pica_clr;
 };
 
-struct psycho_dmasync {
+struct psycho_dma_sync {
 	struct psycho_softc	*pds_sc;
 	driver_filter_t		*pds_handler;	/* handler to call */
 	void			*pds_arg;	/* argument for the handler */
@@ -232,14 +232,14 @@ struct psycho_desc {
 	const char	*pd_name;
 };
 
-static const struct psycho_desc psycho_compats[] = {
+static const struct psycho_desc const psycho_compats[] = {
 	{ "pci108e,8000", PSYCHO_MODE_PSYCHO,	"Psycho compatible" },
 	{ "pci108e,a000", PSYCHO_MODE_SABRE,	"Sabre compatible" },
 	{ "pci108e,a001", PSYCHO_MODE_SABRE,	"Hummingbird compatible" },
 	{ NULL,		  0,			NULL }
 };
 
-static const struct psycho_desc psycho_models[] = {
+static const struct psycho_desc const psycho_models[] = {
 	{ "SUNW,psycho",  PSYCHO_MODE_PSYCHO,	"Psycho" },
 	{ "SUNW,sabre",   PSYCHO_MODE_SABRE,	"Sabre" },
 	{ NULL,		  0,			NULL }
@@ -296,8 +296,8 @@ psycho_attach(device_t dev)
 	phandle_t child, node;
 	uint32_t dvmabase, prop, prop_array[2];
 	int32_t rev;
-	u_int ver;
-	int i, n, nrange, rid;
+	u_int rerun, ver;
+	int i, n;
 
 	node = ofw_bus_get_node(dev);
 	sc = device_get_softc(dev);
@@ -315,7 +315,7 @@ psycho_attach(device_t dev)
 	 * (2) the shared Psycho configuration registers
 	 */
 	if (sc->sc_mode == PSYCHO_MODE_PSYCHO) {
-		rid = 2;
+		i = 2;
 		sc->sc_pcictl =
 		    bus_get_resource_start(dev, SYS_RES_MEMORY, 0) -
 		    bus_get_resource_start(dev, SYS_RES_MEMORY, 2);
@@ -332,18 +332,18 @@ psycho_attach(device_t dev)
 			/* NOTREACHED */
 		}
 	} else {
-		rid = 0;
+		i = 0;
 		sc->sc_pcictl = PSR_PCICTL0;
 		sc->sc_half = 0;
 	}
-	sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
+	sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &i,
 	    (sc->sc_mode == PSYCHO_MODE_PSYCHO ? RF_SHAREABLE : 0) |
 	    RF_ACTIVE);
 	if (sc->sc_mem_res == NULL)
 		panic("%s: could not allocate registers", __func__);
 
 	/*
-	 * Match other Psycho's that are already configured against
+	 * Match other Psychos that are already configured against
 	 * the base physical address.  This will be the same for a
 	 * pair of devices that share register space.
 	 */
@@ -363,6 +363,8 @@ psycho_attach(device_t dev)
 			panic("%s: could not malloc mutex", __func__);
 		mtx_init(sc->sc_mtx, "pcib_mtx", NULL, MTX_SPIN);
 	} else {
+		if (sc->sc_mode != PSYCHO_MODE_PSYCHO)
+			panic("%s: no partner expected", __func__);
 		if (mtx_initialized(osc->sc_mtx) == 0)
 			panic("%s: mutex not initialized", __func__);
 		sc->sc_mtx = osc->sc_mtx;
@@ -408,16 +410,17 @@ psycho_attach(device_t dev)
 	case 0:
 		dr |= DIAG_RTRY_DIS;
 		dr &= ~DIAG_DWSYNC_DIS;
-		/* XXX need to also disable rerun of the streaming buffers. */
+		rerun = 0;
 		break;
 	case 1:
 		csr &= ~PCICTL_ARB_PARK;
 		dr |= DIAG_RTRY_DIS | DIAG_DWSYNC_DIS;
-		/* XXX need to also disable rerun of the streaming buffers. */
+		rerun = 0;
 		break;
 	default:
 		dr |= DIAG_DWSYNC_DIS;
 		dr &= ~DIAG_RTRY_DIS;
+		rerun = 1;
 		break;
 	}
 
@@ -460,13 +463,12 @@ psycho_attach(device_t dev)
 	    rman_manage_region(&sc->sc_pci_mem_rman, 0, PSYCHO_MEM_SIZE) != 0)
 		panic("%s: failed to set up memory rman", __func__);
 
-	nrange = OF_getprop_alloc(node, "ranges", sizeof(*range),
-	    (void **)&range);
+	n = OF_getprop_alloc(node, "ranges", sizeof(*range), (void **)&range);
 	/*
 	 * Make sure that the expected ranges are present.  The
 	 * OFW_PCI_CS_MEM64 one is not currently used though.
 	 */
-	if (nrange != PSYCHO_NRANGE)
+	if (n != PSYCHO_NRANGE)
 		panic("%s: unsupported number of ranges", __func__);
 	/*
 	 * Find the addresses of the various bus spaces.
@@ -493,7 +495,8 @@ psycho_attach(device_t dev)
 		/*
 		 * Hunt through all the interrupt mapping regs and register
 		 * our interrupt controller for the corresponding interrupt
-		 * vectors.
+		 * vectors.  We do this early in order to be able to catch
+		 * stray interrupts.
 		 */
 		for (n = 0; n <= PSYCHO_MAX_INO; n++) {
 			if (psycho_find_intrmap(sc, n, &intrmap, &intrclr,
@@ -523,22 +526,23 @@ psycho_attach(device_t dev)
 			    INTMAP_ENABLE(INTMAP_VEC(sc->sc_ign, n),
 			    PCPU_GET(mid)));
 #endif
-			if (intr_controller_register(INTMAP_VEC(sc->sc_ign, n),
-			    &psycho_ic, pica) != 0)
-				panic("%s: could not register interrupt "
-				    "controller for INO %d", __func__, n);
+			i = intr_controller_register(INTMAP_VEC(sc->sc_ign, n),
+			    &psycho_ic, pica);
+			if (i != 0)
+				device_printf(dev, "could not register "
+				    "interrupt controller for INO %d (%d)\n",
+				    n, i);
 		}
 
-		if (sc->sc_mode == PSYCHO_MODE_PSYCHO) {
-			/* Initialize the counter-timer. */
+		if (sc->sc_mode == PSYCHO_MODE_PSYCHO)
 			sparc64_counter_init(device_get_nameunit(dev),
 			    rman_get_bustag(sc->sc_mem_res),
 			    rman_get_bushandle(sc->sc_mem_res), PSR_TC0);
-		}
 
 		/*
 		 * Set up IOMMU and PCI configuration if we're the first
-		 * of a pair of Psycho's to arrive here.
+		 * of a pair of Psychos to arrive here or a Hummingbird
+		 * or Sabre.
 		 *
 		 * We should calculate a TSB size based on amount of RAM
 		 * and number of bus controllers and number and type of
@@ -556,10 +560,10 @@ psycho_attach(device_t dev)
 		else
 			sc->sc_is->is_pmaxaddr =
 			    IOMMU_MAXADDR(PSYCHO_IOMMU_BITS);
-		sc->sc_is->is_sb[0] = 0;
-		sc->sc_is->is_sb[1] = 0;
+		sc->sc_is->is_sb[0] = sc->sc_is->is_sb[1] = 0;
 		if (OF_getproplen(node, "no-streaming-cache") < 0)
 			sc->sc_is->is_sb[0] = sc->sc_pcictl + PCR_STRBUF;
+		sc->sc_is->is_flags |= (rerun != 1) ? IOMMU_RERUN_DISABLE : 0;
 		psycho_iommu_init(sc, 3, dvmabase);
 	} else {
 		/* Just copy IOMMU state, config tag and address. */
@@ -694,12 +698,20 @@ psycho_set_intr(struct psycho_softc *sc, u_int index, bus_addr_t intrmap,
 	rid = index;
 	sc->sc_irq_res[index] = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ,
 	    &rid, RF_ACTIVE);
+	if (sc->sc_irq_res[index] == NULL && intrmap >= PSR_POWER_INT_MAP) {
+		/*
+		 * These interrupts aren't mandatory and not available
+		 * with all controllers (not even Psychos).
+		 */
+		return;
+	}
 	if (sc->sc_irq_res[index] == NULL ||
 	    INTIGN(vec = rman_get_start(sc->sc_irq_res[index])) != sc->sc_ign ||
 	    INTVEC(PSYCHO_READ8(sc, intrmap)) != vec ||
 	    intr_vectors[vec].iv_ic != &psycho_ic ||
-	    bus_setup_intr(sc->sc_dev, sc->sc_irq_res[index], INTR_TYPE_MISC,
-	    filt, intr, sc, &sc->sc_ihand[index]) != 0)
+	    bus_setup_intr(sc->sc_dev, sc->sc_irq_res[index],
+	    INTR_TYPE_MISC | INTR_FAST, filt, intr, sc,
+	    &sc->sc_ihand[index]) != 0)
 		panic("%s: failed to set up interrupt %d", __func__, index);
 }
 
@@ -1065,9 +1077,9 @@ psycho_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
 }
 
 static int
-psycho_dmasync(void *arg)
+psycho_dma_sync_stub(void *arg)
 {
-	struct psycho_dmasync *pds = arg;
+	struct psycho_dma_sync *pds = arg;
 
 	(void)PCIB_READ_CONFIG(pds->pds_ppb, pds->pds_bus, pds->pds_slot,
 	    pds->pds_func, PCIR_VENDOR, 2);
@@ -1125,7 +1137,7 @@ psycho_setup_intr(device_t dev, device_t child, struct resource *ires,
 	devclass_t pci_devclass;
 	device_t cdev, pdev, pcidev;
 	struct psycho_softc *sc;
-	struct psycho_dmasync *pds;
+	struct psycho_dma_sync *pds;
 	u_long vec;
 	int error;
 
@@ -1142,17 +1154,12 @@ psycho_setup_intr(device_t dev, device_t child, struct resource *ires,
 	}
 
 	/*
-	 * The Sabre-APB-combination has a bug where it does not drain
-	 * DMA write data for devices behind additional PCI-PCI bridges
-	 * underneath the APB PCI-PCI bridge.  The workaround is to do
-	 * a read on the farest PCI-PCI bridge followed by a read of the
-	 * PCI DMA write sync register of the Sabre.
-	 * XXX installing the wrapper for an affected device and the
-	 * actual workaround in psycho_dmasync() should be moved to
-	 * psycho(4)-specific bus_dma_tag_create() and bus_dmamap_sync()
-	 * methods, respectively, once DMA tag creation is newbus'ified,
-	 * so the workaround isn't only applied for interrupt handlers
-	 * but also for polling(4) callbacks.
+	 * The Sabre-APB-combination does not automatically flush DMA
+	 * write data for devices behind additional PCI-PCI bridges
+	 * underneath the APB PCI-PCI bridge.  The procedure for a
+	 * manual flush is to do a PIO read on the far side of the
+	 * farthest PCI-PCI bridge followed by a read of the PCI DMA
+	 * write sync register of the Sabre.
 	 */
 	if (sc->sc_mode == PSYCHO_MODE_SABRE) {
 		pds = malloc(sizeof(*pds), M_DEVBUF, M_NOWAIT | M_ZERO);
@@ -1191,20 +1198,20 @@ psycho_setup_intr(device_t dev, device_t child, struct resource *ires,
 			pds->pds_func = pci_get_function(pcidev);
 			if (bootverbose)
 				device_printf(dev, "installed DMA sync "
-				    "workaround for device %d.%d on bus %d\n",
+				    "wrapper for device %d.%d on bus %d\n",
 				    pds->pds_slot, pds->pds_func,
 				    pds->pds_bus);
 			if (intr == NULL) {
 				pds->pds_handler = filt;
 				error = bus_generic_setup_intr(dev, child,
-				    ires, flags, psycho_dmasync, intr, pds,
-				    cookiep);
+				    ires, flags, psycho_dma_sync_stub, intr,
+				    pds, cookiep);
 			} else {
 				pds->pds_handler = (driver_filter_t *)intr;
 				error = bus_generic_setup_intr(dev, child,
 				    ires, flags, filt,
-				    (driver_intr_t *)psycho_dmasync, pds,
-				    cookiep);
+				    (driver_intr_t *)psycho_dma_sync_stub,
+				    pds, cookiep);
 			}
 		} else
 			error = bus_generic_setup_intr(dev, child, ires,
@@ -1226,7 +1233,7 @@ psycho_teardown_intr(device_t dev, device_t child, struct resource *vec,
     void *cookie)
 {
 	struct psycho_softc *sc;
-	struct psycho_dmasync *pds;
+	struct psycho_dma_sync *pds;
 	int error;
 
 	sc = device_get_softc(dev);
@@ -1314,8 +1321,8 @@ psycho_activate_resource(device_t bus, device_t child, int type, int rid,
 		    type, rid, r));
 	if (type == SYS_RES_MEMORY) {
 		/*
-		 * Need to memory-map the device space, as some drivers depend
-		 * on the virtual address being set and useable.
+		 * Need to memory-map the device space, as some drivers
+		 * depend on the virtual address being set and usable.
 		 */
 		error = sparc64_bus_mem_map(rman_get_bustag(r),
 		    rman_get_bushandle(r), rman_get_size(r), 0, 0, &p);
diff --git a/sys/sparc64/pci/psychovar.h b/sys/sparc64/pci/psychovar.h
index ef4286b..04d81ad 100644
--- a/sys/sparc64/pci/psychovar.h
+++ b/sys/sparc64/pci/psychovar.h
@@ -49,8 +49,8 @@ struct psycho_softc {
 
 	phandle_t			sc_node;	/* Firmware node */
 	u_int				sc_mode;
-#define	PSYCHO_MODE_SABRE	1
-#define	PSYCHO_MODE_PSYCHO	2
+#define	PSYCHO_MODE_SABRE		0
+#define	PSYCHO_MODE_PSYCHO		1
 
 	/* Bus A or B of a psycho pair? */
 	u_int				sc_half;
diff --git a/sys/sparc64/pci/schizo.c b/sys/sparc64/pci/schizo.c
index 2425294..7085171 100644
--- a/sys/sparc64/pci/schizo.c
+++ b/sys/sparc64/pci/schizo.c
@@ -148,7 +148,7 @@ static device_method_t schizo_methods[] = {
 	/* ofw_bus interface */
 	DEVMETHOD(ofw_bus_get_node,	schizo_get_node),
 
-	{ 0, 0 }
+	KOBJMETHOD_END
 };
 
 static devclass_t schizo_devclass;
@@ -449,14 +449,17 @@ schizo_attach(device_t dev)
 		tc_init(tc);
 	}
 
-	/* Set up the IOMMU.  Both Schizo and Tomatillo have one per PBM. */
+	/*
+	 * Set up the IOMMU.  Schizo, Tomatillo and XMITS all have
+	 * one per PBM.  Schizo and XMITS additionally have a streaming
+	 * buffer, in Schizo version < 5 (i.e. revision < 2.3) it's
+	 * affected by several errata and basically unusable though.
+	 */
 	sc->sc_is.is_pmaxaddr = IOMMU_MAXADDR(STX_IOMMU_BITS);
-	sc->sc_is.is_sb[0] = 0;
-	sc->sc_is.is_sb[1] = 0;
-#ifdef notyet
-	if (OF_getproplen(node, "no-streaming-cache") < 0)
+	sc->sc_is.is_sb[0] = sc->sc_is.is_sb[1] = 0;
+	if (OF_getproplen(node, "no-streaming-cache") < 0 &&
+	    !(sc->sc_mode == SCHIZO_MODE_SCZ && sc->sc_ver < 5))
 		sc->sc_is.is_sb[0] = STX_PCI_STRBUF;
-#endif
 
 #define	TSBCASE(x)							\
 	case (IOTSB_BASESZ << (x)) << (IO_PAGE_SHIFT - IOTTE_SHIFT):	\
@@ -1154,8 +1157,9 @@ schizo_setup_intr(device_t dev, device_t child, struct resource *ires,
 			    "controller for vector 0x%lx (%d)\n", vec, error);
 			return (error);
 		}
-		device_printf(dev, "belatedly registered as interrupt "
-		    "controller for vector 0x%lx\n", vec);
+		if (bootverbose)
+			device_printf(dev, "belatedly registered as "
+			    "interrupt controller for vector 0x%lx\n", vec);
 	} else {
 		device_printf(dev,
 		    "invalid interrupt controller for vector 0x%lx\n", vec);
diff --git a/sys/sparc64/sbus/dma_sbus.c b/sys/sparc64/sbus/dma_sbus.c
index 4623995..b4c2e5e 100644
--- a/sys/sparc64/sbus/dma_sbus.c
+++ b/sys/sparc64/sbus/dma_sbus.c
@@ -122,14 +122,15 @@ static device_method_t dma_methods[] = {
 	/* Bus interface */
 	DEVMETHOD(bus_print_child,	dma_print_child),
 	DEVMETHOD(bus_probe_nomatch,	dma_probe_nomatch),
-	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
-	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_alloc_resource,	bus_generic_rl_alloc_resource),
-	DEVMETHOD(bus_release_resource, bus_generic_rl_release_resource),
 	DEVMETHOD(bus_activate_resource, bus_generic_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
-	DEVMETHOD(bus_get_resource_list, dma_get_resource_list),
+	DEVMETHOD(bus_release_resource, bus_generic_rl_release_resource),
+	DEVMETHOD(bus_setup_intr,	bus_generic_setup_intr),
+	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_get_resource,	bus_generic_rl_get_resource),
+	DEVMETHOD(bus_get_resource_list, dma_get_resource_list),
+	DEVMETHOD(bus_child_pnpinfo_str, ofw_bus_gen_child_pnpinfo_str),
 
 	/* ofw_bus interface */
 	DEVMETHOD(ofw_bus_get_devinfo,	dma_get_devinfo),
@@ -139,7 +140,7 @@ static device_method_t dma_methods[] = {
 	DEVMETHOD(ofw_bus_get_node,	ofw_bus_gen_get_node),
 	DEVMETHOD(ofw_bus_get_type,	ofw_bus_gen_get_type),
 
-	{ 0, 0 }
+	KOBJMETHOD_END
 };
 
 static driver_t dma_driver = {
diff --git a/sys/sparc64/sbus/sbus.c b/sys/sparc64/sbus/sbus.c
index d6b8393..dd36f03 100644
--- a/sys/sparc64/sbus/sbus.c
+++ b/sys/sparc64/sbus/sbus.c
@@ -190,7 +190,7 @@ static bus_deactivate_resource_t sbus_deactivate_resource;
 static bus_get_dma_tag_t sbus_get_dma_tag;
 static ofw_bus_get_devinfo_t sbus_get_devinfo;
 
-static int sbus_inlist(const char *, const char **);
+static int sbus_inlist(const char *, const char *const *);
 static struct sbus_devinfo * sbus_setup_dinfo(device_t, struct sbus_softc *,
     phandle_t);
 static void sbus_destroy_dinfo(struct sbus_devinfo *);
@@ -217,16 +217,16 @@ static device_method_t sbus_methods[] = {
 	DEVMETHOD(bus_print_child,	sbus_print_child),
 	DEVMETHOD(bus_probe_nomatch,	sbus_probe_nomatch),
 	DEVMETHOD(bus_read_ivar,	sbus_read_ivar),
-	DEVMETHOD(bus_setup_intr, 	sbus_setup_intr),
-	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_alloc_resource,	sbus_alloc_resource),
 	DEVMETHOD(bus_activate_resource,	sbus_activate_resource),
 	DEVMETHOD(bus_deactivate_resource,	sbus_deactivate_resource),
 	DEVMETHOD(bus_release_resource,	sbus_release_resource),
-	DEVMETHOD(bus_get_resource_list, sbus_get_resource_list),
+	DEVMETHOD(bus_setup_intr, 	sbus_setup_intr),
+	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_get_resource,	bus_generic_rl_get_resource),
-	DEVMETHOD(bus_get_dma_tag,	sbus_get_dma_tag),
+	DEVMETHOD(bus_get_resource_list, sbus_get_resource_list),
 	DEVMETHOD(bus_child_pnpinfo_str, ofw_bus_gen_child_pnpinfo_str),
+	DEVMETHOD(bus_get_dma_tag,	sbus_get_dma_tag),
 
 	/* ofw_bus interface */
 	DEVMETHOD(ofw_bus_get_devinfo,	sbus_get_devinfo),
@@ -236,7 +236,7 @@ static device_method_t sbus_methods[] = {
 	DEVMETHOD(ofw_bus_get_node,	ofw_bus_gen_get_node),
 	DEVMETHOD(ofw_bus_get_type,	ofw_bus_gen_get_type),
 
-	{ 0, 0 }
+	KOBJMETHOD_END
 };
 
 static driver_t sbus_driver = {
@@ -266,14 +266,14 @@ struct sbus_icarg {
 	bus_addr_t		sica_clr;
 };
 
-static const char *sbus_order_first[] = {
+static const char *const sbus_order_first[] = {
 	"auxio",
 	"dma",
 	NULL
 };
 
 static int
-sbus_inlist(const char *name, const char **list)
+sbus_inlist(const char *name, const char *const *list)
 {
 	int i;
 
@@ -313,32 +313,33 @@ sbus_attach(device_t dev)
 	bus_size_t size;
 	u_long vec;
 	phandle_t child, node;
-	int clock, i, intr, rid;
+	uint32_t prop;
+	int i, j;
 
 	sc = device_get_softc(dev);
 	sc->sc_dev = dev;
 	node = ofw_bus_get_node(dev);
 
-	rid = 0;
-	sc->sc_sysio_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
+	i = 0;
+	sc->sc_sysio_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &i,
 	    RF_ACTIVE);
 	if (sc->sc_sysio_res == NULL)
 		panic("%s: cannot allocate device memory", __func__);
 
-	if (OF_getprop(node, "interrupts", &intr, sizeof(intr)) == -1)
+	if (OF_getprop(node, "interrupts", &prop, sizeof(prop)) == -1)
 		panic("%s: cannot get IGN", __func__);
-	sc->sc_ign = INTIGN(intr);
+	sc->sc_ign = INTIGN(prop);
 	sc->sc_cbustag = sbus_alloc_bustag(sc);
 
 	/*
 	 * Record clock frequency for synchronous SCSI.
 	 * IS THIS THE CORRECT DEFAULT??
 	 */
-	if (OF_getprop(node, "clock-frequency", &clock, sizeof(clock)) == -1)
-		clock = 25000000;
-	sc->sc_clockfreq = clock;
-	clock /= 1000;
-	device_printf(dev, "clock %d.%03d MHz\n", clock / 1000, clock % 1000);
+	if (OF_getprop(node, "clock-frequency", &prop, sizeof(prop)) == -1)
+		prop = 25000000;
+	sc->sc_clockfreq = prop;
+	prop /= 1000;
+	device_printf(dev, "clock %d.%03d MHz\n", prop / 1000, prop % 1000);
 
 	/*
 	 * Collect address translations from the OBP.
@@ -362,9 +363,9 @@ sbus_attach(device_t dev)
 		sc->sc_rd[i].rd_slot = range[i].cspace;
 		sc->sc_rd[i].rd_coffset = range[i].coffset;
 		sc->sc_rd[i].rd_cend = sc->sc_rd[i].rd_coffset + size;
-		rid = resource_list_add_next(rl, SYS_RES_MEMORY, phys,
+		j = resource_list_add_next(rl, SYS_RES_MEMORY, phys,
 		    phys + size - 1, size);
-		if ((res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
+		if ((res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &j,
 		    RF_ACTIVE)) == NULL)
 			panic("%s: cannot allocate decoded range", __func__);
 		sc->sc_rd[i].rd_bushandle = rman_get_bushandle(res);
@@ -381,11 +382,12 @@ sbus_attach(device_t dev)
 
 	/*
 	 * Get the SBus burst transfer size if burst transfers are supported.
-	 * XXX: is the default correct?
 	 */
-	if (OF_getprop(node, "burst-sizes", &sc->sc_burst,
+	if (OF_getprop(node, "up-burst-sizes", &sc->sc_burst,
 	    sizeof(sc->sc_burst)) == -1 || sc->sc_burst == 0)
-		sc->sc_burst = SBUS_BURST_DEF;
+		sc->sc_burst =
+		    (SBUS_BURST64_DEF << SBUS_BURST64_SHIFT) | SBUS_BURST_DEF;
+
 
 	/* initalise the IOMMU */
 
@@ -421,6 +423,7 @@ sbus_attach(device_t dev)
  	/*
 	 * Hunt through all the interrupt mapping regs and register our
 	 * interrupt controller for the corresponding interrupt vectors.
+	 * We do this early in order to be able to catch stray interrupts.
 	 */
 	for (i = 0; i <= SBUS_MAX_INO; i++) {
 		if (sbus_find_intrmap(sc, i, &intrmap, &intrclr) == 0)
@@ -439,31 +442,32 @@ sbus_attach(device_t dev)
 		    (u_long)intrmap, (u_long)SYSIO_READ8(sc, intrmap),
 		    (u_long)intrclr);
 #endif
-		if (intr_controller_register(INTMAP_VEC(sc->sc_ign, i),
-		    &sbus_ic, sica) != 0)
-			panic("%s: could not register interrupt controller "
-			    "for INO %d", __func__, i);
+		j = intr_controller_register(INTMAP_VEC(sc->sc_ign, i),
+		    &sbus_ic, sica);
+		if (j != 0)
+			device_printf(dev, "could not register interrupt "
+			    "controller for INO %d (%d)\n", i, j);
 	}
 
 	/* Enable the over-temperature and power-fail interrupts. */
-	rid = 4;
-	sc->sc_ot_ires = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
+	i = 4;
+	sc->sc_ot_ires = bus_alloc_resource_any(dev, SYS_RES_IRQ, &i,
 	    RF_ACTIVE);
 	if (sc->sc_ot_ires == NULL ||
 	    INTIGN(vec = rman_get_start(sc->sc_ot_ires)) != sc->sc_ign ||
 	    INTVEC(SYSIO_READ8(sc, SBR_THERM_INT_MAP)) != vec ||
 	    intr_vectors[vec].iv_ic != &sbus_ic ||
-	    bus_setup_intr(dev, sc->sc_ot_ires, INTR_TYPE_MISC,
+	    bus_setup_intr(dev, sc->sc_ot_ires, INTR_TYPE_MISC | INTR_FAST,
 	    NULL, sbus_overtemp, sc, &sc->sc_ot_ihand) != 0)
 		panic("%s: failed to set up temperature interrupt", __func__);
-	rid = 3;
-	sc->sc_pf_ires = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
+	i = 3;
+	sc->sc_pf_ires = bus_alloc_resource_any(dev, SYS_RES_IRQ, &i,
 	    RF_ACTIVE);
 	if (sc->sc_pf_ires == NULL ||
 	    INTIGN(vec = rman_get_start(sc->sc_pf_ires)) != sc->sc_ign ||
 	    INTVEC(SYSIO_READ8(sc, SBR_POWER_INT_MAP)) != vec ||
 	    intr_vectors[vec].iv_ic != &sbus_ic ||
-	    bus_setup_intr(dev, sc->sc_pf_ires, INTR_TYPE_MISC,
+	    bus_setup_intr(dev, sc->sc_pf_ires, INTR_TYPE_MISC | INTR_FAST,
 	    NULL, sbus_pwrfail, sc, &sc->sc_pf_ihand) != 0)
 		panic("%s: failed to set up power fail interrupt", __func__);
 
@@ -849,8 +853,8 @@ sbus_activate_resource(device_t bus, device_t child, int type, int rid,
 	}
 	if (type == SYS_RES_MEMORY) {
 		/*
-		 * Need to memory-map the device space, as some drivers depend
-		 * on the virtual address being set and useable.
+		 * Need to memory-map the device space, as some drivers
+		 * depend on the virtual address being set and usable.
 		 */
 		error = sparc64_bus_mem_map(rman_get_bustag(r),
 		    rman_get_bushandle(r), rman_get_size(r), 0, 0, &p);
diff --git a/sys/sparc64/sbus/sbusvar.h b/sys/sparc64/sbus/sbusvar.h
index e4f2895..8d6a54e 100644
--- a/sys/sparc64/sbus/sbusvar.h
+++ b/sys/sparc64/sbus/sbusvar.h
@@ -77,7 +77,7 @@
  */
 
 #ifndef _SPARC64_SBUS_SBUSVAR_H_
-#define _SPARC64_SBUS_SBUSVAR_H_
+#define	_SPARC64_SBUS_SBUSVAR_H_
 
 /*
  * Macros for probe order
@@ -88,18 +88,24 @@
 /*
  * PROM-reported DMA burst sizes for the SBus
  */
-#define SBUS_BURST_1	0x1
-#define SBUS_BURST_2	0x2
-#define SBUS_BURST_4	0x4
-#define SBUS_BURST_8	0x8
-#define SBUS_BURST_16	0x10
-#define SBUS_BURST_32	0x20
-#define SBUS_BURST_64	0x40
+#define	SBUS_BURST_1		(1 << 0)
+#define	SBUS_BURST_2		(1 << 1)
+#define	SBUS_BURST_4		(1 << 2)
+#define	SBUS_BURST_8		(1 << 3)
+#define	SBUS_BURST_16		(1 << 4)
+#define	SBUS_BURST_32		(1 << 5)
+#define	SBUS_BURST_64		(1 << 6)
+#define	SBUS_BURST_MASK		((1 << SBUS_BURST_SIZE) - 1)
+#define	SBUS_BURST_SIZE		16
+#define	SBUS_BURST64_MASK	(SBUS_BURST_MASK << SBUS_BURST64_SHIFT)
+#define	SBUS_BURST64_SHIFT	16
 
 /* Used if no burst sizes are specified for the bus. */
 #define	SBUS_BURST_DEF \
 	(SBUS_BURST_1 | SBUS_BURST_2 | SBUS_BURST_4 | SBUS_BURST_8 | 	\
-	SBUS_BURST_16)
+	SBUS_BURST_16 | SBUS_BURST_32 | SBUS_BURST_64)
+#define	SBUS_BURST64_DEF \
+	(SBUS_BURST_8 | SBUS_BURST_16 | SBUS_BURST_32 | SBUS_BURST_64)
 
 enum sbus_device_ivars {
 	SBUS_IVAR_BURSTSZ,
@@ -111,7 +117,7 @@ enum sbus_device_ivars {
 /*
  * Simplified accessors for sbus devices
  */
-#define SBUS_ACCESSOR(var, ivar, type) \
+#define	SBUS_ACCESSOR(var, ivar, type) \
 	__BUS_ACCESSOR(sbus, var, SBUS, ivar, type)
 
 SBUS_ACCESSOR(burstsz,		BURSTSZ,	int)
diff --git a/sys/sparc64/sparc64/db_disasm.c b/sys/sparc64/sparc64/db_disasm.c
index 1c244e0..a20219e 100644
--- a/sys/sparc64/sparc64/db_disasm.c
+++ b/sys/sparc64/sparc64/db_disasm.c
@@ -82,14 +82,14 @@ struct sparc_insn {
 	  const char* format;
 };
 
-static const char* regs[] = {
+static const char *const regs[] = {
 	"g0", "g1", "g2", "g3", "g4", "g5", "g6", "g7",
 	"o0", "o1", "o2", "o3", "o4", "o5", "sp", "o7",
 	"l0", "l1", "l2", "l3", "l4", "l5", "l6", "l7",
 	"i0", "i1", "i2", "i3", "i4", "i5", "fp", "i7"
 };
 
-static const char* priv_regs[] = {
+static const char *const priv_regs[] = {
 	"tpc", "tnpc", "tstate", "tt", "tick", "tba", "pstate", "tl",
 	"pil", "cwp", "cansave", "canrestore", "cleanwin", "otherwin",
 	"wstate", "fq",
@@ -97,18 +97,18 @@ static const char* priv_regs[] = {
 	"", "", "", "", "", "", "", "ver"
 };
 
-static const char* state_regs[] = {
+static const char *const state_regs[] = {
 	"y", "", "ccr", "asi", "tick", "pc", "fprs", "asr",
 	"", "", "", "", "", "", "", "",
 	"pcr", "pic", "dcr", "gsr", "set_softint", "clr_softint", "softint",
 	"tick_cmpr", "sys_tick", "sys_tick_cmpr", "", "", "", "", "", "", ""
 };
 
-static const char* ccodes[] = {
+static const char *const ccodes[] = {
 	"fcc0", "fcc1", "fcc2", "fcc3", "icc", "", "xcc", ""
 };
 
-static const char* prefetch[] = {
+static const char *const prefetch[] = {
 	"n_reads", "one_read", "n_writes", "one_write", "page"
 };
 
diff --git a/sys/sparc64/sparc64/eeprom.c b/sys/sparc64/sparc64/eeprom.c
index c707018..223b96c 100644
--- a/sys/sparc64/sparc64/eeprom.c
+++ b/sys/sparc64/sparc64/eeprom.c
@@ -91,7 +91,7 @@ static device_method_t eeprom_methods[] = {
 	DEVMETHOD(clock_gettime,	mk48txx_gettime),
 	DEVMETHOD(clock_settime,	mk48txx_settime),
 
-	{ 0, 0 }
+	KOBJMETHOD_END
 };
 
 static driver_t eeprom_driver = {
diff --git a/sys/sparc64/sparc64/elf_machdep.c b/sys/sparc64/sparc64/elf_machdep.c
index a956c5c..b0a6998 100644
--- a/sys/sparc64/sparc64/elf_machdep.c
+++ b/sys/sparc64/sparc64/elf_machdep.c
@@ -99,7 +99,8 @@ static Elf64_Brandinfo freebsd_brand_info = {
 	.interp_path	= "/libexec/ld-elf.so.1",
 	.sysvec		= &elf64_freebsd_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &elf64_freebsd_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 SYSINIT(elf64, SI_SUB_EXEC, SI_ORDER_ANY,
@@ -114,7 +115,8 @@ static Elf64_Brandinfo freebsd_brand_oinfo = {
 	.interp_path	= "/usr/libexec/ld-elf.so.1",
 	.sysvec		= &elf64_freebsd_sysvec,
 	.interp_newpath	= NULL,
-	.flags		= BI_CAN_EXEC_DYN,
+	.brand_note	= &elf64_freebsd_brandnote,
+	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
 };
 
 SYSINIT(oelf64, SI_SUB_EXEC, SI_ORDER_ANY,
diff --git a/sys/sparc64/sparc64/jbusppm.c b/sys/sparc64/sparc64/jbusppm.c
index 902424c..c08b64d 100644
--- a/sys/sparc64/sparc64/jbusppm.c
+++ b/sys/sparc64/sparc64/jbusppm.c
@@ -78,7 +78,7 @@ static device_method_t jbusppm_methods[] = {
 	DEVMETHOD(device_probe,		jbusppm_probe),
 	DEVMETHOD(device_attach,	jbusppm_attach),
 
-	{ NULL, NULL }
+	KOBJMETHOD_END
 };
 
 static devclass_t jbusppm_devclass;
diff --git a/sys/sparc64/sparc64/mp_machdep.c b/sys/sparc64/sparc64/mp_machdep.c
index 5470c11..ee4ffe7 100644
--- a/sys/sparc64/sparc64/mp_machdep.c
+++ b/sys/sparc64/sparc64/mp_machdep.c
@@ -413,8 +413,6 @@ cpu_mp_shutdown(void)
 			break;
 		}
 	}
-	/* XXX: delay a bit to allow the CPUs to actually enter the PROM. */
-	DELAY(100000);
 	critical_exit();
 }
 
@@ -505,11 +503,7 @@ spitfire_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2)
 		 */
 		DELAY(2);
 	}
-	if (
-#ifdef KDB
-	    kdb_active ||
-#endif
-	    panicstr != NULL)
+	if (kdb_active != 0 || panicstr != NULL)
 		printf("%s: couldn't send IPI to module 0x%u\n",
 		    __func__, mid);
 	else
@@ -581,11 +575,7 @@ cheetah_ipi_selected(u_int cpus, u_long d0, u_long d1, u_long d2)
 		 */
 		DELAY(2 * mp_ncpus);
 	}
-	if (
-#ifdef KDB
-	    kdb_active ||
-#endif
-	    panicstr != NULL)
+	if (kdb_active != 0 || panicstr != NULL)
 		printf("%s: couldn't send IPI (cpus=0x%u ids=0x%lu)\n",
 		    __func__, cpus, ids);
 	else
diff --git a/sys/sparc64/sparc64/nexus.c b/sys/sparc64/sparc64/nexus.c
index 6784cde..58c0805 100644
--- a/sys/sparc64/sparc64/nexus.c
+++ b/sys/sparc64/sparc64/nexus.c
@@ -122,12 +122,12 @@ static device_method_t nexus_methods[] = {
 	DEVMETHOD(bus_release_resource,	nexus_release_resource),
 	DEVMETHOD(bus_setup_intr,	nexus_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	nexus_teardown_intr),
-#ifdef SMP
-	DEVMETHOD(bus_bind_intr,	nexus_bind_intr),
-#endif
 	DEVMETHOD(bus_set_resource,	bus_generic_rl_set_resource),
 	DEVMETHOD(bus_get_resource,	bus_generic_rl_get_resource),
 	DEVMETHOD(bus_get_resource_list, nexus_get_resource_list),
+#ifdef SMP
+	DEVMETHOD(bus_bind_intr,	nexus_bind_intr),
+#endif
 	DEVMETHOD(bus_get_dma_tag,	nexus_get_dma_tag),
 
 	/* ofw_bus interface */
@@ -138,7 +138,7 @@ static device_method_t nexus_methods[] = {
 	DEVMETHOD(ofw_bus_get_node,	ofw_bus_gen_get_node),
 	DEVMETHOD(ofw_bus_get_type,	ofw_bus_gen_get_type),
 
-	{ 0, 0 }
+	KOBJMETHOD_END
 };
 
 static devclass_t nexus_devclass;
diff --git a/sys/sparc64/sparc64/rtc.c b/sys/sparc64/sparc64/rtc.c
index fe7f17e..5bd196b 100644
--- a/sys/sparc64/sparc64/rtc.c
+++ b/sys/sparc64/sparc64/rtc.c
@@ -89,7 +89,7 @@ static device_method_t rtc_ebus_methods[] = {
 	DEVMETHOD(clock_gettime,	mc146818_gettime),
 	DEVMETHOD(clock_settime,	mc146818_settime),
 
-	{ 0, 0 }
+	KOBJMETHOD_END
 };
 
 static driver_t rtc_ebus_driver = {
diff --git a/sys/sparc64/sparc64/sc_machdep.c b/sys/sparc64/sparc64/sc_machdep.c
index f56f669..107ebd2 100644
--- a/sys/sparc64/sparc64/sc_machdep.c
+++ b/sys/sparc64/sparc64/sc_machdep.c
@@ -57,7 +57,7 @@ static device_method_t sc_methods[] = {
 	DEVMETHOD(device_probe,		sc_probe),
 	DEVMETHOD(device_attach,	sc_attach),
 
-	{ 0, 0 }
+	KOBJMETHOD_END
 };
 
 static driver_t sc_driver = {
diff --git a/sys/sparc64/sparc64/schppm.c b/sys/sparc64/sparc64/schppm.c
index 7431062..ac0ff75 100644
--- a/sys/sparc64/sparc64/schppm.c
+++ b/sys/sparc64/sparc64/schppm.c
@@ -73,7 +73,7 @@ static device_method_t schppm_methods[] = {
 	DEVMETHOD(device_probe,		schppm_probe),
 	DEVMETHOD(device_attach,	schppm_attach),
 
-	{ NULL, NULL }
+	KOBJMETHOD_END
 };
 
 static devclass_t schppm_devclass;
diff --git a/sys/sparc64/sparc64/trap.c b/sys/sparc64/sparc64/trap.c
index c10dbc4..702e4f1 100644
--- a/sys/sparc64/sparc64/trap.c
+++ b/sys/sparc64/sparc64/trap.c
@@ -71,7 +71,6 @@ __FBSDID("$FreeBSD$");
 #include <security/audit/audit.h>
 
 #include <dev/ofw/openfirm.h>
-#include <machine/ofw_machdep.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
@@ -84,7 +83,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/cpu.h>
 #include <machine/frame.h>
 #include <machine/intr_machdep.h>
-#include <machine/pcb.h>
+#include <machine/ofw_machdep.h>
 #include <machine/smp.h>
 #include <machine/trap.h>
 #include <machine/tstate.h>
@@ -114,7 +113,7 @@ extern char fas_nofault_end[];
 
 extern char *syscallnames[];
 
-const char *trap_msg[] = {
+const char *const trap_msg[] = {
 	"reserved",
 	"instruction access exception",
 	"instruction access error",
@@ -329,15 +328,12 @@ trap(struct trapframe *tf)
 		KASSERT((tf->tf_type & T_KERNEL) != 0,
 		    ("trap: kernel trap isn't"));
 
-#ifdef KDB
 		if (kdb_active) {
 			kdb_reenter();
 			return;
 		}
-#endif
 
 		switch (tf->tf_type & ~T_KERNEL) {
-#ifdef KDB
 		case T_BREAKPOINT:
 		case T_KSTACK_FAULT:
 			error = (kdb_trap(tf->tf_type, 0, tf) == 0);
@@ -349,7 +345,6 @@ trap(struct trapframe *tf)
 			error = db_watch_trap(tf);
 			break;
 #endif
-#endif
 		case T_DATA_MISS:
 		case T_DATA_PROTECTION:
 		case T_INSTRUCTION_MISS:
@@ -409,7 +404,6 @@ static int
 trap_pfault(struct thread *td, struct trapframe *tf)
 {
 	struct vmspace *vm;
-	struct pcb *pcb;
 	struct proc *p;
 	vm_offset_t va;
 	vm_prot_t prot;
@@ -429,7 +423,6 @@ trap_pfault(struct thread *td, struct trapframe *tf)
 
 	rv = KERN_SUCCESS;
 	ctx = TLB_TAR_CTX(tf->tf_tar);
-	pcb = td->td_pcb;
 	type = tf->tf_type & ~T_KERNEL;
 	va = TLB_TAR_VA(tf->tf_tar);
 
@@ -559,11 +552,6 @@ syscall(struct trapframe *tf)
 
 	PCPU_INC(cnt.v_syscall);
 
-	narg = 0;
-	error = 0;
-	reg = 0;
-	regcnt = REG_MAXARGS;
-
 	td->td_pticks = 0;
 	td->td_frame = tf;
 	if (td->td_ucred != p->p_ucred)
@@ -577,6 +565,8 @@ syscall(struct trapframe *tf)
 	tpc = tf->tf_tpc;
 	TF_DONE(tf);
 
+	reg = 0;
+	regcnt = REG_MAXARGS;
 	if (p->p_sysent->sv_prepsyscall) {
 		/*
 		 * The prep code is MP aware.
diff --git a/sys/sparc64/sparc64/upa.c b/sys/sparc64/sparc64/upa.c
index 06ae244..7e81c89 100644
--- a/sys/sparc64/sparc64/upa.c
+++ b/sys/sparc64/sparc64/upa.c
@@ -135,6 +135,7 @@ static device_method_t upa_methods[] = {
 	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_get_resource,	bus_generic_rl_get_resource),
 	DEVMETHOD(bus_get_resource_list, upa_get_resource_list),
+	DEVMETHOD(bus_child_pnpinfo_str, ofw_bus_gen_child_pnpinfo_str),
 
 	/* ofw_bus interface */
 	DEVMETHOD(ofw_bus_get_devinfo,	upa_get_devinfo),
@@ -144,7 +145,7 @@ static device_method_t upa_methods[] = {
 	DEVMETHOD(ofw_bus_get_node,	ofw_bus_gen_get_node),
 	DEVMETHOD(ofw_bus_get_type,	ofw_bus_gen_get_type),
 
-	{ NULL, NULL }
+	KOBJMETHOD_END
 };
 
 static devclass_t upa_devclass;
@@ -241,7 +242,7 @@ upa_attach(device_t dev)
 				    "pci108e,8001") == 0 &&
 				    ((bus_get_resource_start(children[j],
 				    SYS_RES_MEMORY, 0) >> 20) & 1) == 1) {
-				    	schizo = children[j];
+					schizo = children[j];
 					break;
 				}
 			}
@@ -287,9 +288,10 @@ upa_attach(device_t dev)
 		goto fail;
 	}
 
- 	/*
+	/*
 	 * Hunt through all the interrupt mapping regs and register our
 	 * interrupt controller for the corresponding interrupt vectors.
+	 * We do this early in order to be able to catch stray interrupts.
 	 */
 	for (i = UPA_INO_BASE; i <= UPA_INO_MAX; i++) {
 		imr = 0;
@@ -312,10 +314,11 @@ upa_attach(device_t dev)
 		device_printf(dev, "intr map (INO %d) IMR%d: %#lx\n",
 		    i, imr, (u_long)UPA_READ(sc, imr, 0x0));
 #endif
-		if (intr_controller_register(INTMAP_VEC(sc->sc_ign, i),
-		    &upa_ic, uica) != 0)
-			panic("%s: could not register interrupt controller "
-			    "for INO %d", __func__, i);
+		j = intr_controller_register(INTMAP_VEC(sc->sc_ign, i),
+		    &upa_ic, uica);
+		if (j != 0)
+			device_printf(dev, "could not register interrupt "
+			    "controller for INO %d (%d)\n", i, j);
 	}
 
 	/* Make sure the power level is appropriate for normal operation. */
@@ -345,13 +348,13 @@ upa_attach(device_t dev)
 			device_printf(dev,
 			    "could not determine upa-portid of child 0x%lx\n",
 			    (unsigned long)child);
-		    	continue;
+			continue;
 		}
 		if (portid > 1) {
 			device_printf(dev,
 			    "upa-portid %d of child 0x%lx invalid\n", portid,
 			    (unsigned long)child);
-		    	continue;
+			continue;
 		}
 		if ((udi = upa_setup_dinfo(dev, sc, child, portid)) == NULL)
 			continue;
@@ -493,12 +496,12 @@ upa_setup_intr(device_t dev, device_t child, struct resource *ires, int flags,
 	/*
 	 * Make sure the vector is fully specified and we registered
 	 * our interrupt controller for it.
- 	 */
+	 */
 	vec = rman_get_start(ires);
 	if (INTIGN(vec) != sc->sc_ign || intr_vectors[vec].iv_ic != &upa_ic) {
 		device_printf(dev, "invalid interrupt vector 0x%lx\n", vec);
- 		return (EINVAL);
- 	}
+		return (EINVAL);
+	}
 	return (bus_generic_setup_intr(dev, child, ires, flags, filt, func,
 	    arg, cookiep));
 }
diff --git a/sys/sun4v/conf/GENERIC b/sys/sun4v/conf/GENERIC
index fac7e04..2291087 100644
--- a/sys/sun4v/conf/GENERIC
+++ b/sys/sun4v/conf/GENERIC
@@ -191,7 +191,6 @@ device		ulpt		# Printer
 #device		umass		# Disks/Mass storage - Requires scbus and da
 device		ums		# Mouse
 #device		urio		# Diamond Rio 500 MP3 player
-#device		uscanner	# Scanners
 # USB Ethernet, requires mii
 #device		aue		# ADMtek USB Ethernet
 #device		axe		# ASIX Electronics USB Ethernet
diff --git a/sys/sun4v/include/elf.h b/sys/sun4v/include/elf.h
index 108ade1..c0fcbee 100644
--- a/sys/sun4v/include/elf.h
+++ b/sys/sun4v/include/elf.h
@@ -78,16 +78,14 @@ __ElfType(Auxinfo);
 #define	AT_BASE		7	/* Interpreter's base address. */
 #define	AT_FLAGS	8	/* Flags (unused). */
 #define	AT_ENTRY	9	/* Where interpreter should transfer control. */
-/*
- * The following non-standard values are used in Linux ELF binaries.
- */
 #define	AT_NOTELF	10	/* Program is not ELF ?? */
 #define	AT_UID		11	/* Real uid. */
 #define	AT_EUID		12	/* Effective uid. */
 #define	AT_GID		13	/* Real gid. */
 #define	AT_EGID		14	/* Effective gid. */
+#define	AT_EXECPATH	15	/* Path to the executable. */
 
-#define	AT_COUNT	15	/* Count of defined aux entry types. */
+#define	AT_COUNT	16	/* Count of defined aux entry types. */
 
 /* Define "machine" characteristics */
 #if __ELF_WORD_SIZE == 32
diff --git a/sys/sun4v/include/trap.h b/sys/sun4v/include/trap.h
index 42efb8b..8922a35 100644
--- a/sys/sun4v/include/trap.h
+++ b/sys/sun4v/include/trap.h
@@ -144,7 +144,7 @@
 
 
 #ifndef LOCORE
-extern const char *trap_msg[];
+extern const char *const trap_msg[];
 extern void set_mmfsa_traptable(void *, uint64_t);
 extern void trap_init(void);
 #endif
diff --git a/sys/sun4v/sun4v/trap.c b/sys/sun4v/sun4v/trap.c
index bac4f55..ffa0e8c 100644
--- a/sys/sun4v/sun4v/trap.c
+++ b/sys/sun4v/sun4v/trap.c
@@ -81,7 +81,6 @@
 #include <machine/cpu.h>
 #include <machine/frame.h>
 #include <machine/intr_machdep.h>
-#include <machine/pcb.h>
 #include <machine/smp.h>
 #include <machine/trap.h>
 #include <machine/tstate.h>
@@ -119,7 +118,7 @@ extern char fas_nofault_end[];
 
 extern char *syscallnames[];
 
-const char *trap_msg[] = {
+const char *const trap_msg[] = {
 	"reserved",
 	"instruction access exception",
 	"instruction access error",
@@ -390,21 +389,17 @@ trap(struct trapframe *tf, int64_t type, uint64_t data)
 		    ("trap: kernel trap isn't - trap: %ld:%s: 0x%lx at 0x%lx on cpu=%d\n", 
 		     trapno, trap_msg[trapno], data, tf->tf_tpc, curcpu));
 
-#ifdef KDB
 		if (kdb_active) {
 			kdb_reenter();
 			return;
 		}
-#endif
 
 		switch (trapno) {
-#ifdef KDB
 		case T_BREAKPOINT:
 		case T_KSTACK_FAULT:
 			error = (kdb_trap(trapno, 0, tf) == 0);
 			TF_DONE(tf);
 			break;
-#endif
 		case T_DATA_MISS:
 		case T_DATA_PROTECTION:
 		case T_INSTRUCTION_MISS:
@@ -460,7 +455,6 @@ static int
 trap_pfault(struct thread *td, struct trapframe *tf, int64_t type, uint64_t data)
 {
 	struct vmspace *vm;
-	struct pcb *pcb;
 	struct proc *p;
 	vm_offset_t va;
 	vm_prot_t prot;
@@ -475,7 +469,6 @@ trap_pfault(struct thread *td, struct trapframe *tf, int64_t type, uint64_t data
 
 	rv = KERN_SUCCESS;
 	ctx = TLB_TAR_CTX(data);
-	pcb = td->td_pcb;
 	type = type & ~T_KERNEL;
 	va = TLB_TAR_VA(data);
 
@@ -603,11 +596,6 @@ syscall(struct trapframe *tf)
 
 	PCPU_INC(cnt.v_syscall);
 
-	narg = 0;
-	error = 0;
-	reg = 0;
-	regcnt = REG_MAXARGS;
-
 	td->td_pticks = 0;
 	td->td_frame = tf;
 	if (td->td_ucred != p->p_ucred)
@@ -621,6 +609,8 @@ syscall(struct trapframe *tf)
 	tpc = tf->tf_tpc;
 	TF_DONE(tf);
 
+	reg = 0;
+	regcnt = REG_MAXARGS;
 	if (p->p_sysent->sv_prepsyscall) {
 		/*
 		 * The prep code is MP aware.
diff --git a/sys/sys/_pthreadtypes.h b/sys/sys/_pthreadtypes.h
index 6337540..76049ee 100644
--- a/sys/sys/_pthreadtypes.h
+++ b/sys/sys/_pthreadtypes.h
@@ -61,7 +61,10 @@ struct pthread_spinlock;
  * or assignment operators for the types pthread_attr_t, pthread_cond_t,
  * pthread_condattr_t, pthread_mutex_t, pthread_mutexattr_t.
  */
+#ifndef _PTHREAD_T_DECLARED
 typedef struct	pthread			*pthread_t;
+#define	_PTHREAD_T_DECLARED
+#endif
 typedef struct	pthread_attr		*pthread_attr_t;
 typedef struct	pthread_mutex		*pthread_mutex_t;
 typedef struct	pthread_mutex_attr	*pthread_mutexattr_t;
diff --git a/sys/sys/aio.h b/sys/sys/aio.h
index a474b8d..5a8779b 100644
--- a/sys/sys/aio.h
+++ b/sys/sys/aio.h
@@ -19,7 +19,6 @@
 #ifndef _SYS_AIO_H_
 #define	_SYS_AIO_H_
 
-#include <sys/time.h>
 #include <sys/types.h>
 #include <sys/signal.h>
 
@@ -79,6 +78,8 @@ typedef struct aiocb {
 
 #ifndef _KERNEL
 
+struct timespec;
+
 __BEGIN_DECLS
 /*
  * Asynchronously read from a file
@@ -123,7 +124,9 @@ int	aio_cancel(int, struct aiocb *);
  */
 int	aio_suspend(const struct aiocb * const[], int, const struct timespec *);
 
+#ifdef __BSD_VISIBLE
 int	aio_waitcomplete(struct aiocb **, struct timespec *);
+#endif
 
 int	aio_fsync(int op, struct aiocb *aiocbp);
 __END_DECLS
diff --git a/sys/sys/buf.h b/sys/sys/buf.h
index 7c3dd4e..7ca6ffa 100644
--- a/sys/sys/buf.h
+++ b/sys/sys/buf.h
@@ -443,6 +443,7 @@ buf_countdeps(struct buf *bp, int i)
  */
 #define	GB_LOCK_NOWAIT	0x0001		/* Fail if we block on a buf lock. */
 #define	GB_NOCREAT	0x0002		/* Don't create a buf if not found. */
+#define	GB_NOWAIT_BD	0x0004		/* Do not wait for bufdaemon */
 
 #ifdef _KERNEL
 extern int	nbuf;			/* The number of buffer headers */
@@ -487,7 +488,7 @@ struct buf *     getpbuf(int *);
 struct buf *incore(struct bufobj *, daddr_t);
 struct buf *gbincore(struct bufobj *, daddr_t);
 struct buf *getblk(struct vnode *, daddr_t, int, int, int, int);
-struct buf *geteblk(int);
+struct buf *geteblk(int, int);
 int	bufwait(struct buf *);
 int	bufwrite(struct buf *);
 void	bufdone(struct buf *);
diff --git a/sys/sys/dtrace_bsd.h b/sys/sys/dtrace_bsd.h
index e0adbe6..f323284 100644
--- a/sys/sys/dtrace_bsd.h
+++ b/sys/sys/dtrace_bsd.h
@@ -32,8 +32,11 @@
 #define	_SYS_DTRACE_BSD_H
 
 /* Forward definitions: */
+struct mbuf;
 struct trapframe;
 struct thread;
+struct vattr;
+struct vnode;
 
 /*
  * Cyclic clock function type definition used to hook the cyclic
@@ -93,6 +96,55 @@ typedef	void (*dtrace_malloc_probe_func_t)(u_int32_t, uintptr_t arg0,
 
 extern dtrace_malloc_probe_func_t   dtrace_malloc_probe;
 
+/* dtnfsclient NFSv3 access cache provider hooks. */
+typedef void (*dtrace_nfsclient_accesscache_flush_probe_func_t)(uint32_t,
+    struct vnode *);
+extern dtrace_nfsclient_accesscache_flush_probe_func_t
+    dtrace_nfsclient_accesscache_flush_done_probe;
+
+typedef void (*dtrace_nfsclient_accesscache_get_probe_func_t)(uint32_t,
+    struct vnode *, uid_t, uint32_t);
+extern dtrace_nfsclient_accesscache_get_probe_func_t
+    dtrace_nfsclient_accesscache_get_hit_probe,
+    dtrace_nfsclient_accesscache_get_miss_probe;
+
+typedef void (*dtrace_nfsclient_accesscache_load_probe_func_t)(uint32_t,
+    struct vnode *, uid_t, uint32_t, int);
+extern dtrace_nfsclient_accesscache_load_probe_func_t
+    dtrace_nfsclient_accesscache_load_done_probe;
+
+/* dtnfsclient NFSv[23] attribute cache provider hooks. */
+typedef void (*dtrace_nfsclient_attrcache_flush_probe_func_t)(uint32_t,
+    struct vnode *);
+extern dtrace_nfsclient_attrcache_flush_probe_func_t
+    dtrace_nfsclient_attrcache_flush_done_probe;
+
+typedef void (*dtrace_nfsclient_attrcache_get_hit_probe_func_t)(uint32_t,
+    struct vnode *, struct vattr *);
+extern dtrace_nfsclient_attrcache_get_hit_probe_func_t
+    dtrace_nfsclient_attrcache_get_hit_probe;
+
+typedef void (*dtrace_nfsclient_attrcache_get_miss_probe_func_t)(uint32_t,
+    struct vnode *);
+extern dtrace_nfsclient_attrcache_get_miss_probe_func_t
+    dtrace_nfsclient_attrcache_get_miss_probe;
+
+typedef void (*dtrace_nfsclient_attrcache_load_probe_func_t)(uint32_t,
+    struct vnode *, struct vattr *, int);
+extern dtrace_nfsclient_attrcache_load_probe_func_t
+    dtrace_nfsclient_attrcache_load_done_probe;
+
+/* dtnfsclient NFSv[23] RPC provider hooks. */
+typedef void (*dtrace_nfsclient_nfs23_start_probe_func_t)(uint32_t,
+    struct vnode *, struct mbuf *, struct ucred *, int);
+extern dtrace_nfsclient_nfs23_start_probe_func_t
+    dtrace_nfsclient_nfs23_start_probe;
+
+typedef void (*dtrace_nfsclient_nfs23_done_probe_func_t)(uint32_t,
+    struct vnode *, struct mbuf *, struct ucred *, int, int);
+extern dtrace_nfsclient_nfs23_done_probe_func_t
+    dtrace_nfsclient_nfs23_done_probe;
+
 /*
  * Functions which allow the dtrace module to check that the kernel 
  * hooks have been compiled with sufficient space for it's private
diff --git a/sys/sys/elf_common.h b/sys/sys/elf_common.h
index 34c0b3a..f642c71 100644
--- a/sys/sys/elf_common.h
+++ b/sys/sys/elf_common.h
@@ -427,6 +427,7 @@ typedef struct {
  * built, these entries will need to be adjusted.
  */
 #define	DT_ADDRRNGLO	0x6ffffe00
+#define	DT_GNU_HASH	0x6ffffef5	/* GNU-style hash table */
 #define	DT_CONFIG	0x6ffffefa	/* configuration information */
 #define	DT_DEPAUDIT	0x6ffffefb	/* dependency auditing */
 #define	DT_AUDIT	0x6ffffefc	/* object auditing */
@@ -466,6 +467,12 @@ typedef struct {
 				   executable contains code using a static
 				   thread-local storage scheme. */
 
+/* Values for DT_FLAGS_1 */
+#define	DF_1_BIND_NOW	0x00000001	/* Same as DF_BIND_NOW */
+#define	DF_1_GLOBAL	0x00000002	/* Set the RTLD_GLOBAL for object */
+#define	DF_1_NODELETE	0x00000008	/* Set the RTLD_NODELETE for object */
+#define	DF_1_ORIGIN	0x00000080	/* Process $ORIGIN */
+
 /* Values for n_type.  Used in core files. */
 #define	NT_PRSTATUS	1	/* Process status. */
 #define	NT_FPREGSET	2	/* Floating point registers. */
diff --git a/sys/sys/imgact.h b/sys/sys/imgact.h
index 011a7ae..e6acc00 100644
--- a/sys/sys/imgact.h
+++ b/sys/sys/imgact.h
@@ -66,6 +66,9 @@ struct image_params {
 	size_t auxarg_size;
 	struct image_args *args;	/* system call arguments */
 	struct sysentvec *sysent;	/* system entry vector */
+	char *execpath;
+	unsigned long execpathp;
+	char *freepath;
 };
 
 #ifdef _KERNEL
diff --git a/sys/sys/imgact_elf.h b/sys/sys/imgact_elf.h
index deb5b10..60979d9 100644
--- a/sys/sys/imgact_elf.h
+++ b/sys/sys/imgact_elf.h
@@ -55,6 +55,13 @@ typedef struct {
 } __ElfN(Auxargs);
 
 typedef struct {
+	Elf_Note	hdr;
+	const char *	vendor;
+	int		flags;
+#define	BN_CAN_FETCH_OSREL	0x0001
+} Elf_Brandnote;
+
+typedef struct {
 	int brand;
 	int machine;
 	const char *compat_3_brand;	/* pre Binutils 2.10 method (FBSD 3) */
@@ -63,7 +70,9 @@ typedef struct {
 	struct sysentvec *sysvec;
 	const char *interp_newpath;
 	int flags;
+	Elf_Brandnote *brand_note;
 #define	BI_CAN_EXEC_DYN	0x0001
+#define	BI_BRAND_NOTE	0x0002
 } __ElfN(Brandinfo);
 
 __ElfType(Auxargs);
@@ -81,7 +90,7 @@ int	__elfN(coredump)(struct thread *, struct vnode *, off_t);
 void	__elfN(dump_thread)(struct thread *, void *, size_t *);
 
 extern int __elfN(fallback_brand);
-
+extern Elf_Brandnote __elfN(freebsd_brandnote);
 #endif /* _KERNEL */
 
 #endif /* !_SYS_IMGACT_ELF_H_ */
diff --git a/sys/sys/jail.h b/sys/sys/jail.h
index 4212603..4560456 100644
--- a/sys/sys/jail.h
+++ b/sys/sys/jail.h
@@ -7,7 +7,6 @@
  * ----------------------------------------------------------------------------
  *
  * $FreeBSD$
- *
  */
 
 #ifndef _SYS_JAIL_H_
@@ -32,7 +31,7 @@ struct jail {
 	struct in_addr	*ip4;
 	struct in6_addr	*ip6;
 };
-#define	JAIL_API_VERSION 2
+#define	JAIL_API_VERSION	2
 
 /*
  * For all xprison structs, always keep the pr_version an int and
@@ -54,8 +53,8 @@ struct xprison {
 	int		 pr_state;
 	cpusetid_t	 pr_cpusetid;
 	char		 pr_path[MAXPATHLEN];
-	char 		 pr_host[MAXHOSTNAMELEN];
-	char 		 pr_name[MAXHOSTNAMELEN];
+	char		 pr_host[MAXHOSTNAMELEN];
+	char		 pr_name[MAXHOSTNAMELEN];
 	uint32_t	 pr_ip4s;
 	uint32_t	 pr_ip6s;
 #if 0
@@ -128,8 +127,8 @@ struct prison {
 	char		 pr_path[MAXPATHLEN];		/* (c) chroot path */
 	struct cpuset	*pr_cpuset;			/* (p) cpuset */
 	struct vnode	*pr_root;			/* (c) vnode to rdir */
-	char 		 pr_host[MAXHOSTNAMELEN];	/* (p) jail hostname */
-	char 		 pr_name[MAXHOSTNAMELEN];	/* (c) admin jail name */
+	char		 pr_host[MAXHOSTNAMELEN];	/* (p) jail hostname */
+	char		 pr_name[MAXHOSTNAMELEN];	/* (c) admin jail name */
 	void		*pr_linux;			/* (p) linux abi */
 	int		 pr_securelevel;		/* (p) securelevel */
 	struct task	 pr_task;			/* (d) destroy task */
diff --git a/sys/sys/kernel.h b/sys/sys/kernel.h
index 16fab6c..fa397c4 100644
--- a/sys/sys/kernel.h
+++ b/sys/sys/kernel.h
@@ -163,6 +163,7 @@ enum sysinit_sub_id {
 	SI_SUB_SWAP		= 0xc000000,	/* swap */
 	SI_SUB_INTRINSIC_POST	= 0xd000000,	/* proc 0 cleanup*/
 	SI_SUB_SYSCALLS		= 0xd800000,	/* register system calls */
+	SI_SUB_VIMAGE_DONE	= 0xdc00000,	/* vnet registration complete */
 	SI_SUB_KTHREAD_INIT	= 0xe000000,	/* init process*/
 	SI_SUB_KTHREAD_PAGE	= 0xe400000,	/* pageout daemon*/
 	SI_SUB_KTHREAD_VM	= 0xe800000,	/* vm daemon*/
diff --git a/sys/sys/kerneldump.h b/sys/sys/kerneldump.h
index 3f3eacd..e3421de 100644
--- a/sys/sys/kerneldump.h
+++ b/sys/sys/kerneldump.h
@@ -67,12 +67,13 @@ struct kerneldumpheader {
 #define	KERNELDUMPVERSION	1
 	uint32_t	architectureversion;
 #define	KERNELDUMP_ALPHA_VERSION	1
-#define	KERNELDUMP_I386_VERSION	2
-#define	KERNELDUMP_IA64_VERSION	1
-#define	KERNELDUMP_SPARC64_VERSION	1
 #define	KERNELDUMP_AMD64_VERSION	2
-#define	KERNELDUMP_ARM_VERSION	1
-#define	KERNELDUMP_TEXT_VERSION	1
+#define	KERNELDUMP_ARM_VERSION		1
+#define	KERNELDUMP_I386_VERSION		2
+#define	KERNELDUMP_IA64_VERSION		1
+#define	KERNELDUMP_POWERPC_VERSION	1
+#define	KERNELDUMP_SPARC64_VERSION	1
+#define	KERNELDUMP_TEXT_VERSION		1
 	uint64_t	dumplength;		/* excl headers */
 	uint64_t	dumptime;
 	uint32_t	blocksize;
diff --git a/sys/sys/lock_profile.h b/sys/sys/lock_profile.h
index 2d9df3b..7b36ed5 100644
--- a/sys/sys/lock_profile.h
+++ b/sys/sys/lock_profile.h
@@ -43,11 +43,13 @@ LIST_HEAD(lpohead, lock_profile_object);
 u_int64_t nanoseconds(void);
 #endif
 
-extern int lock_prof_enable;
+extern volatile int lock_prof_enable;
 
 void lock_profile_obtain_lock_success(struct lock_object *lo, int contested,
     uint64_t waittime, const char *file, int line);
 void lock_profile_release_lock(struct lock_object *lo);
+void lock_profile_thread_exit(struct thread *td);
+
 
 static inline void
 lock_profile_obtain_lock_failed(struct lock_object *lo, int *contested,
@@ -61,21 +63,10 @@ lock_profile_obtain_lock_failed(struct lock_object *lo, int *contested,
 
 #else /* !LOCK_PROFILING */
 
-static inline void
-lock_profile_release_lock(struct lock_object *lo)
-{
-}
-
-static inline void
-lock_profile_obtain_lock_failed(struct lock_object *lo, int *contested, uint64_t *waittime)
-{
-}
-
-static inline void
-lock_profile_obtain_lock_success(struct lock_object *lo, int contested, uint64_t waittime,  
-    const char *file, int line)
-{
-}
+#define	lock_profile_release_lock(lo)
+#define lock_profile_obtain_lock_failed(lo, contested, waittime)
+#define lock_profile_obtain_lock_success(lo, contested, waittime, file, line)
+#define	lock_profile_thread_exit(td)
 
 #endif  /* !LOCK_PROFILING */
 
diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h
index 05b28de2..e7f0b7e 100644
--- a/sys/sys/mbuf.h
+++ b/sys/sys/mbuf.h
@@ -199,6 +199,7 @@ struct mbuf {
 #define	M_PROTO6	0x00080000 /* protocol-specific */
 #define	M_PROTO7	0x00100000 /* protocol-specific */
 #define	M_PROTO8	0x00200000 /* protocol-specific */
+#define	M_FLOWID	0x00400000 /* flowid is valid */
 /*
  * For RELENG_{6,7} steal these flags for limited multiple routing table
  * support. In RELENG_8 and beyond, use just one flag and a tag.
diff --git a/sys/sys/memrange.h b/sys/sys/memrange.h
index ace778d..c90104f 100644
--- a/sys/sys/memrange.h
+++ b/sys/sys/memrange.h
@@ -52,6 +52,7 @@ struct mem_range_ops
 	void	(*init)(struct mem_range_softc *sc);
 	int	(*set)(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg);
     	void	(*initAP)(struct mem_range_softc *sc);
+    	void	(*reinit)(struct mem_range_softc *sc);
 };
 
 struct mem_range_softc 
@@ -68,4 +69,3 @@ extern int mem_range_attr_get(struct mem_range_desc *mrd, int *arg);
 extern int mem_range_attr_set(struct mem_range_desc *mrd, int *arg);
 
 #endif
-
diff --git a/sys/sys/param.h b/sys/sys/param.h
index e8b26c7..f02853c 100644
--- a/sys/sys/param.h
+++ b/sys/sys/param.h
@@ -57,7 +57,7 @@
  *		is created, otherwise 1.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 800070	/* Master, propagated to newvers */
+#define __FreeBSD_version 800077	/* Master, propagated to newvers */
 
 #ifndef LOCORE
 #include <sys/types.h>
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index b2bfadd..b2cb011 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -345,7 +345,7 @@ do {									\
 #define	TDP_OLDMASK	0x00000001 /* Need to restore mask after suspend. */
 #define	TDP_INKTR	0x00000002 /* Thread is currently in KTR code. */
 #define	TDP_INKTRACE	0x00000004 /* Thread is currently in KTRACE code. */
-#define	TDP_UNUSED8	0x00000008 /* available */
+#define	TDP_BUFNEED	0x00000008 /* Do not recurse into the buf flush */
 #define	TDP_COWINPROGRESS 0x00000010 /* Snapshot copy-on-write in progress. */
 #define	TDP_ALTSTACK	0x00000020 /* Have alternate signal stack. */
 #define	TDP_DEADLKTREAT	0x00000040 /* Lock aquisition - deadlock treatment. */
diff --git a/sys/sys/sem.h b/sys/sys/sem.h
index ecd78a3..d8d6e92 100644
--- a/sys/sys/sem.h
+++ b/sys/sys/sem.h
@@ -117,7 +117,9 @@ void	semexit(struct proc *p);
 #else /* ! _KERNEL */
 
 __BEGIN_DECLS
+#if __BSD_VISIBLE
 int semsys(int, ...);
+#endif
 int semctl(int, int, int, ...);
 int semget(key_t, int, int);
 int semop(int, struct sembuf *, size_t);
diff --git a/sys/sys/shm.h b/sys/sys/shm.h
index c4b1369..e6f2f76 100644
--- a/sys/sys/shm.h
+++ b/sys/sys/shm.h
@@ -138,7 +138,9 @@ typedef __size_t        size_t;
 #endif
 
 __BEGIN_DECLS
+#ifdef __BSD_VISIBLE
 int shmsys(int, ...);
+#endif
 void *shmat(int, const void *, int);
 int shmget(key_t, size_t, int);
 int shmctl(int, int, struct shmid_ds *);
diff --git a/sys/sys/smp.h b/sys/sys/smp.h
index 8306d52..500516f 100644
--- a/sys/sys/smp.h
+++ b/sys/sys/smp.h
@@ -122,6 +122,9 @@ void	forward_signal(struct thread *);
 void	forward_roundrobin(void);
 int	restart_cpus(cpumask_t);
 int	stop_cpus(cpumask_t);
+#if defined(__amd64__)
+int	suspend_cpus(cpumask_t);
+#endif
 void	smp_rendezvous_action(void);
 extern	struct mtx smp_ipi_mtx;
 
diff --git a/sys/sys/stat.h b/sys/sys/stat.h
index b310641..856d674 100644
--- a/sys/sys/stat.h
+++ b/sys/sys/stat.h
@@ -312,8 +312,13 @@ int	chflags(const char *, unsigned long);
 int	chmod(const char *, mode_t);
 #if __BSD_VISIBLE
 int	fchflags(int, unsigned long);
+#endif
+#if __POSIX_VISIBLE >= 200112
 int	fchmod(int, mode_t);
 #endif
+#if __POSIX_VISIBLE >= 200809
+int	fchmodat(int, const char *, mode_t, int);
+#endif
 int	fstat(int, struct stat *);
 #if __BSD_VISIBLE
 int	lchflags(const char *, int);
@@ -334,6 +339,8 @@ mode_t	umask(mode_t);
 int	fstatat(int, const char *, struct stat *, int);
 int	mkdirat(int, const char *, mode_t);
 int	mkfifoat(int, const char *, mode_t);
+#endif
+#if __BSD_VISIBLE || __XSI_VISIBLE >= 700
 int	mknodat(int, const char *, mode_t, dev_t);
 #endif
 __END_DECLS
diff --git a/sys/sys/syslog.h b/sys/sys/syslog.h
index fe2d0fe..6f12831 100644
--- a/sys/sys/syslog.h
+++ b/sys/sys/syslog.h
@@ -193,7 +193,9 @@ void	closelog(void);
 void	openlog(const char *, int, int);
 int	setlogmask(int);
 void	syslog(int, const char *, ...) __printflike(2, 3);
+#if __BSD_VISIBLE
 void	vsyslog(int, const char *, __va_list) __printflike(2, 0);
+#endif
 __END_DECLS
 
 #endif /* !_KERNEL */
diff --git a/sys/sys/termios.h b/sys/sys/termios.h
index bc071fb..192c3f2 100644
--- a/sys/sys/termios.h
+++ b/sys/sys/termios.h
@@ -273,10 +273,10 @@ int	tcsendbreak(int, int);
 pid_t	tcgetsid(int);
 #endif
 
-#ifndef _POSIX_SOURCE
+#if __BSD_VISIBLE
 void	cfmakeraw(struct termios *);
 int	cfsetspeed(struct termios *, speed_t);
-#endif /* !_POSIX_SOURCE */
+#endif
 __END_DECLS
 
 #endif /* !_KERNEL */
diff --git a/sys/sys/time.h b/sys/sys/time.h
index 7512840..0185c31 100644
--- a/sys/sys/time.h
+++ b/sys/sys/time.h
@@ -316,17 +316,25 @@ int	tvtohz(struct timeval *tv);
 #include <time.h>
 
 #include <sys/cdefs.h>
+#include <sys/select.h>
 
 __BEGIN_DECLS
+int	setitimer(int, const struct itimerval *, struct itimerval *);
+int	utimes(const char *, const struct timeval *);
+
+#if __BSD_VISIBLE
 int	adjtime(const struct timeval *, struct timeval *);
 int	futimes(int, const struct timeval *);
-int	getitimer(int, struct itimerval *);
-int	gettimeofday(struct timeval *, struct timezone *);
+int	futimesat(int, const char *, const struct timeval [2]);
 int	lutimes(const char *, const struct timeval *);
-int	setitimer(int, const struct itimerval *, struct itimerval *);
 int	settimeofday(const struct timeval *, const struct timezone *);
-int	utimes(const char *, const struct timeval *);
-int	futimesat(int, const char *, const struct timeval [2]);
+#endif
+
+#if __XSI_VISIBLE
+int	getitimer(int, struct itimerval *);
+int	gettimeofday(struct timeval *, struct timezone *);
+#endif
+
 __END_DECLS
 
 #endif /* !_KERNEL */
diff --git a/sys/sys/uio.h b/sys/sys/uio.h
index c46cc6f..871f93a 100644
--- a/sys/sys/uio.h
+++ b/sys/sys/uio.h
@@ -106,8 +106,10 @@ int	uiomoveco(void *cp, int n, struct uio *uio, int disposable);
 __BEGIN_DECLS
 ssize_t	readv(int, const struct iovec *, int);
 ssize_t	writev(int, const struct iovec *, int);
+#if __BSD_VISIBLE
 ssize_t	preadv(int, const struct iovec *, int, off_t);
 ssize_t	pwritev(int, const struct iovec *, int, off_t);
+#endif
 __END_DECLS
 
 #endif /* _KERNEL */
diff --git a/sys/sys/vimage.h b/sys/sys/vimage.h
index ea51e49..7781b47 100644
--- a/sys/sys/vimage.h
+++ b/sys/sys/vimage.h
@@ -35,6 +35,15 @@
 
 #include <sys/queue.h>
 
+#if defined(VIMAGE) && defined(VIMAGE_GLOBALS)
+#error "You cannot have both option VIMAGE and option VIMAGE_GLOBALS!"
+#endif
+
+typedef int vnet_attach_fn(const void *);
+typedef int vnet_detach_fn(const void *);
+
+#ifndef VIMAGE_GLOBALS
+
 struct kld_sym_lookup;
 
 struct vnet_symmap {
@@ -42,27 +51,78 @@ struct vnet_symmap {
 	void	*base;
 	size_t	size;
 };
+typedef struct vnet_symmap vnet_symmap_t;
 
 struct vnet_modinfo {
+	u_int				 vmi_id;
+	u_int				 vmi_dependson;
 	char				*vmi_name;
+	vnet_attach_fn			*vmi_iattach;
+	vnet_detach_fn			*vmi_idetach;
+	size_t				 vmi_struct_size;
 	struct vnet_symmap		*vmi_symmap;
 };
+typedef struct vnet_modinfo vnet_modinfo_t;
 
 struct vnet_modlink {
-	TAILQ_ENTRY(vnet_modlink)	vml_mod_le;
+	TAILQ_ENTRY(vnet_modlink)	 vml_mod_le;
 	const struct vnet_modinfo	*vml_modinfo;
+	const void			*vml_iarg;
+	const char			*vml_iname;
 };
 
-#define VNET_MOD_DECLARE(m_name_uc, m_name_lc, m_iattach, m_idetach, 	\
-    m_dependson, m_symmap)						\
-	static const struct vnet_modinfo vnet_##m_name_lc##_modinfo = {	\
-		.vmi_name		= #m_name_lc,			\
-		.vmi_symmap		= m_symmap			\
-};
+#define	VNET_SYMMAP(mod, name)						\
+	{ #name, &(vnet_ ## mod ## _0._ ## name),			\
+	sizeof(vnet_ ## mod ## _0._ ## name) }
 
-#if defined(VIMAGE) && defined(VIMAGE_GLOBALS)
-#error "You cannot have both option VIMAGE and option VIMAGE_GLOBALS!"
-#endif
+#define	VNET_SYMMAP_END		{ NULL, 0 }
+
+/* stateful modules */
+#define	VNET_MOD_NET		 0	/* MUST be 0 - implicit dependency */
+#define	VNET_MOD_NETGRAPH	 1
+#define	VNET_MOD_INET		 2
+#define	VNET_MOD_INET6		 3
+#define	VNET_MOD_IPSEC		 4
+#define	VNET_MOD_IPFW		 5
+#define	VNET_MOD_DUMMYNET	 6
+#define	VNET_MOD_PF		 7
+#define	VNET_MOD_ALTQ		 8
+#define	VNET_MOD_IPX		 9
+#define	VNET_MOD_ATALK		10
+#define	VNET_MOD_ACCF_HTTP	11
+#define	VNET_MOD_IGMP		12
+
+/* stateless modules */
+#define	VNET_MOD_NG_ETHER	20
+#define	VNET_MOD_NG_IFACE	21
+#define	VNET_MOD_NG_EIFACE	22
+#define	VNET_MOD_ESP		23
+#define	VNET_MOD_IPIP		24
+#define	VNET_MOD_AH		25
+#define	VNET_MOD_IPCOMP	 	26	
+#define	VNET_MOD_GIF		27
+#define	VNET_MOD_ARP		28
+#define	VNET_MOD_RTABLE		29
+#define	VNET_MOD_LOIF		30
+#define	VNET_MOD_DOMAIN		31
+#define	VNET_MOD_DYNAMIC_START	32
+#define	VNET_MOD_MAX		64
+
+/* Sysctl virtualization macros need these name mappings bellow */
+#define	V_MOD_vnet_net		VNET_MOD_NET
+#define	V_MOD_vnet_netgraph	VNET_MOD_NETGRAPH
+#define	V_MOD_vnet_inet		VNET_MOD_INET
+#define	V_MOD_vnet_inet6	VNET_MOD_INET6
+#define	V_MOD_vnet_ipfw		VNET_MOD_IPFW
+#define	V_MOD_vnet_pf		VNET_MOD_PF
+#define	V_MOD_vnet_gif		VNET_MOD_GIF
+#define	V_MOD_vnet_ipsec	VNET_MOD_IPSEC
+
+int	vi_symlookup(struct kld_sym_lookup *, char *);
+void	vnet_mod_register(const struct vnet_modinfo *);
+void	vnet_mod_register_multi(const struct vnet_modinfo *, void *, char *);
+
+#endif /* !VIMAGE_GLOBALS */
 
 #ifdef VIMAGE_GLOBALS
 #define	VSYM(base, sym) (sym)
@@ -74,13 +134,8 @@ struct vnet_modlink {
 #endif
 #endif
 
-#define VNET_SYMMAP(mod, name)						\
-	{ #name, &(vnet_ ## mod ## _0._ ## name),			\
-	sizeof(vnet_ ## mod ## _0._ ## name) }
-
-#define VNET_SYMMAP_END		{ NULL, 0 }
-
 /* Non-VIMAGE null-macros */
+#define	IS_DEFAULT_VNET(arg) 1
 #define	CURVNET_SET(arg)
 #define	CURVNET_SET_QUIET(arg)
 #define	CURVNET_RESTORE()
@@ -109,9 +164,6 @@ struct vnet_modlink {
 #define	G_hostname		VPROCG(hostname) /* global hostname */
 #define	V_domainname		VPROCG(domainname)
 
-int	vi_symlookup(struct kld_sym_lookup *, char *);
-void	vnet_mod_register(const struct vnet_modinfo *);
-
 /*
  * Size-guards for the vimage structures.
  * If you need to update the values you MUST increment __FreeBSD_version.
diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h
index 53f45c6..0a3d1dc 100644
--- a/sys/sys/vnode.h
+++ b/sys/sys/vnode.h
@@ -135,7 +135,7 @@ struct vnode {
 	 */
 	LIST_HEAD(, namecache) v_cache_src;	/* c Cache entries from us */
 	TAILQ_HEAD(, namecache) v_cache_dst;	/* c Cache entries to us */
-	struct	vnode *v_dd;			/* c .. vnode */
+	struct namecache *v_cache_dd;		/* c Cache entry for .. vnode */
 
 	/*
 	 * clustering stuff
@@ -308,15 +308,48 @@ struct vattr {
 #define IO_SEQSHIFT	16		/* seq heuristic in upper 16 bits */
 
 /*
- *  Flags for accmode_t.
+ * Flags for accmode_t.
  */
-#define	VEXEC	000100		/* execute/search permission */
-#define	VWRITE	000200		/* write permission */
-#define	VREAD	000400		/* read permission */
-#define	VADMIN	010000		/* permission to administer */
-#define	VSTAT	020000		/* permission to retrieve attrs */
-#define	VAPPEND	040000		/* permission to write/append */
-#define	VALLPERM	(VEXEC | VWRITE | VREAD | VADMIN | VSTAT | VAPPEND)
+#define	VEXEC			000000000100 /* execute/search permission */
+#define	VWRITE			000000000200 /* write permission */
+#define	VREAD			000000000400 /* read permission */
+#define	VADMIN			000000010000 /* being the file owner */
+#define	VAPPEND			000000040000 /* permission to write/append */
+/*
+ * VEXPLICIT_DENY makes VOP_ACCESS(9) return EPERM or EACCES only
+ * if permission was denied explicitly, by a "deny" rule in NFS4 ACL,
+ * and 0 otherwise.  This never happens with ordinary unix access rights
+ * or POSIX.1e ACLs.  Obviously, VEXPLICIT_DENY must be OR-ed with
+ * some other V* constant.
+ */
+#define	VEXPLICIT_DENY		000000100000
+#define	VREAD_NAMED_ATTRS 	000000200000 /* not used */
+#define	VWRITE_NAMED_ATTRS 	000000400000 /* not used */
+#define	VDELETE_CHILD	 	000001000000
+#define	VREAD_ATTRIBUTES 	000002000000 /* permission to stat(2) */
+#define	VWRITE_ATTRIBUTES 	000004000000 /* change {m,c,a}time */
+#define	VDELETE		 	000010000000
+#define	VREAD_ACL	 	000020000000 /* read ACL and file mode */
+#define	VWRITE_ACL	 	000040000000 /* change ACL and/or file mode */
+#define	VWRITE_OWNER	 	000100000000 /* change file owner */
+#define	VSYNCHRONIZE	 	000200000000 /* not used */
+
+/*
+ * Permissions that were traditionally granted only to the file owner.
+ */
+#define VADMIN_PERMS	(VADMIN | VWRITE_ATTRIBUTES | VWRITE_ACL | \
+    VWRITE_OWNER)
+
+/*
+ * Permissions that were traditionally granted to everyone.
+ */
+#define VSTAT_PERMS	(VREAD_ATTRIBUTES | VREAD_ACL)
+
+/*
+ * Permissions that allow to change the state of the file in any way.
+ */
+#define VMODIFY_PERMS	(VWRITE | VAPPEND | VADMIN_PERMS | VDELETE_CHILD | \
+    VDELETE)
 
 /*
  * Token indicating no attribute value yet assigned.
@@ -379,14 +412,6 @@ extern	struct uma_zone *namei_zone;
 extern	int prtactive;			/* nonzero to call vprint() */
 extern	struct vattr va_null;		/* predefined null vattr structure */
 
-/*
- * Macro/function to check for client cache inconsistency w.r.t. leasing.
- */
-#define	LEASE_READ	0x1		/* Check lease for readers */
-#define	LEASE_WRITE	0x2		/* Check lease for modifiers */
-
-extern void	(*lease_updatetime)(int deltat);
-
 #define	VI_LOCK(vp)	mtx_lock(&(vp)->v_interlock)
 #define	VI_LOCK_FLAGS(vp, flags) mtx_lock_flags(&(vp)->v_interlock, (flags))
 #define	VI_TRYLOCK(vp)	mtx_trylock(&(vp)->v_interlock)
@@ -554,8 +579,6 @@ struct uio;
 struct vattr;
 struct vnode;
 
-extern int	(*lease_check_hook)(struct vop_lease_args *);
-
 /* cache_* may belong in namei.h. */
 void	cache_enter(struct vnode *dvp, struct vnode *vp,
 	    struct componentname *cnp);
@@ -574,7 +597,6 @@ int	insmntque1(struct vnode *vp, struct mount *mp,
 	    void (*dtr)(struct vnode *, void *), void *dtr_arg);
 int	insmntque(struct vnode *vp, struct mount *mp);
 u_quad_t init_va_filerev(void);
-int	lease_check(struct vop_lease_args *ap);
 int	speedup_syncer(void);
 #define textvp_fullpath(p, rb, rfb) \
 	vn_fullpath(FIRST_THREAD_IN_PROC(p), (p)->p_textvp, rb, rfb)
diff --git a/sys/tools/vnode_if.awk b/sys/tools/vnode_if.awk
index b250c92..86c6bdc 100644
--- a/sys/tools/vnode_if.awk
+++ b/sys/tools/vnode_if.awk
@@ -165,12 +165,18 @@ if (hfile) {
 
 if (cfile) {
 	printc(common_head \
+	    "#include \"opt_kdtrace.h\"\n" \
+	    "\n" \
 	    "#include <sys/param.h>\n" \
 	    "#include <sys/event.h>\n" \
+	    "#include <sys/kernel.h>\n" \
 	    "#include <sys/mount.h>\n" \
+	    "#include <sys/sdt.h>\n" \
 	    "#include <sys/systm.h>\n" \
 	    "#include <sys/vnode.h>\n" \
 	    "\n" \
+	    "SDT_PROVIDER_DECLARE(vfs);\n" \
+	    "\n" \
 	    "struct vnodeop_desc vop_default_desc = {\n" \
 	    "	\"default\",\n" \
 	    "	0,\n" \
@@ -348,6 +354,10 @@ while ((getline < srcfile) > 0) {
 		printc("\tVDESC_NO_OFFSET");
 		printc("};");
 
+		printc("\n");
+		printc("SDT_PROBE_DEFINE2(vfs, vop, " name ", entry, \"struct vnode *\", \"struct " name "_args *\");\n");
+		printc("SDT_PROBE_DEFINE3(vfs, vop, " name ", return, \"struct vnode *\", \"struct " name "_args *\", \"int\");\n");
+
 		# Print out function.
 		printc("\nint\n" uname "_AP(struct " name "_args *a)");
 		printc("{");
@@ -364,6 +374,7 @@ while ((getline < srcfile) > 0) {
 		printc("\t    vop->"name" == NULL && vop->vop_bypass == NULL)")
 		printc("\t\tvop = vop->vop_default;")
 		printc("\tVNASSERT(vop != NULL, a->a_" args[0]", (\"No "name"(%p, %p)\", a->a_" args[0]", a));")
+		printc("\tSDT_PROBE(vfs, vop, " name ", entry, a->a_" args[0] ", a, 0, 0, 0);\n");
 		for (i = 0; i < numargs; ++i)
 			add_debug_code(name, args[i], "Entry", "\t");
 		add_pre(name);
@@ -372,6 +383,7 @@ while ((getline < srcfile) > 0) {
 		printc("\telse")
 		printc("\t\trc = vop->vop_bypass(&a->a_gen);")
 		printc(ctrstr);
+		printc("\tSDT_PROBE(vfs, vop, " name ", return, a->a_" args[0] ", a, rc, 0, 0);\n");
 		printc("\tif (rc == 0) {");
 		for (i = 0; i < numargs; ++i)
 			add_debug_code(name, args[i], "OK", "\t\t");
diff --git a/sys/ufs/ffs/ffs_snapshot.c b/sys/ufs/ffs/ffs_snapshot.c
index b45532e..a6e73f5 100644
--- a/sys/ufs/ffs/ffs_snapshot.c
+++ b/sys/ufs/ffs/ffs_snapshot.c
@@ -278,7 +278,6 @@ restart:
 			return (error);
 		goto restart;
 	}
-	VOP_LEASE(nd.ni_dvp, td, KERNCRED, LEASE_WRITE);
 	error = VOP_CREATE(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vat);
 	VOP_UNLOCK(nd.ni_dvp, 0);
 	if (error) {
diff --git a/sys/ufs/ffs/ffs_softdep.c b/sys/ufs/ffs/ffs_softdep.c
index 15ceef8..5d959ac4 100644
--- a/sys/ufs/ffs/ffs_softdep.c
+++ b/sys/ufs/ffs/ffs_softdep.c
@@ -3309,7 +3309,7 @@ newdirrem(bp, dp, ip, isrmdir, prevdirremp)
 	 * the number of freefile and freeblks structures.
 	 */
 	ACQUIRE_LOCK(&lk);
-	if (num_dirrem > max_softdeps / 2)
+	if (!(ip->i_flags & SF_SNAPSHOT) && num_dirrem > max_softdeps / 2)
 		(void) request_cleanup(ITOV(dp)->v_mount, FLUSH_REMOVE);
 	num_dirrem += 1;
 	FREE_LOCK(&lk);
diff --git a/sys/ufs/ffs/ffs_vfsops.c b/sys/ufs/ffs/ffs_vfsops.c
index 73535fc..94752c7 100644
--- a/sys/ufs/ffs/ffs_vfsops.c
+++ b/sys/ufs/ffs/ffs_vfsops.c
@@ -1451,6 +1451,7 @@ ffs_vgetf(mp, ino, flags, vpp, ffs_flags)
 	ip->i_fs = fs;
 	ip->i_dev = dev;
 	ip->i_number = ino;
+	ip->i_ea_refs = 0;
 #ifdef QUOTA
 	{
 		int i;
@@ -1844,7 +1845,9 @@ ffs_bufwrite(struct buf *bp)
 		    ("bufwrite: needs chained iodone (%p)", bp->b_iodone));
 
 		/* get a new block */
-		newbp = geteblk(bp->b_bufsize);
+		newbp = geteblk(bp->b_bufsize, GB_NOWAIT_BD);
+		if (newbp == NULL)
+			goto normal_write;
 
 		/*
 		 * set it to be identical to the old block.  We have to
@@ -1884,6 +1887,7 @@ ffs_bufwrite(struct buf *bp)
 	}
 
 	/* Let the normal bufwrite do the rest for us */
+normal_write:
 	return (bufwrite(bp));
 }
 
diff --git a/sys/ufs/ffs/ffs_vnops.c b/sys/ufs/ffs/ffs_vnops.c
index ca2efa6..1abb994 100644
--- a/sys/ufs/ffs/ffs_vnops.c
+++ b/sys/ufs/ffs/ffs_vnops.c
@@ -1225,6 +1225,35 @@ ffs_rdextattr(u_char **p, struct vnode *vp, struct thread *td, int extra)
 	return (0);
 }
 
+static void
+ffs_lock_ea(struct vnode *vp)
+{
+	struct inode *ip;
+
+	ip = VTOI(vp);
+	VI_LOCK(vp);
+	while (ip->i_flag & IN_EA_LOCKED) {
+		ip->i_flag |= IN_EA_LOCKWAIT;
+		msleep(&ip->i_ea_refs, &vp->v_interlock, PINOD + 2, "ufs_ea",
+		    0);
+	}
+	ip->i_flag |= IN_EA_LOCKED;
+	VI_UNLOCK(vp);
+}
+
+static void
+ffs_unlock_ea(struct vnode *vp)
+{
+	struct inode *ip;
+
+	ip = VTOI(vp);
+	VI_LOCK(vp);
+	if (ip->i_flag & IN_EA_LOCKWAIT)
+		wakeup(&ip->i_ea_refs);
+	ip->i_flag &= ~(IN_EA_LOCKED | IN_EA_LOCKWAIT);
+	VI_UNLOCK(vp);
+}
+
 static int
 ffs_open_ea(struct vnode *vp, struct ucred *cred, struct thread *td)
 {
@@ -1234,14 +1263,22 @@ ffs_open_ea(struct vnode *vp, struct ucred *cred, struct thread *td)
 
 	ip = VTOI(vp);
 
-	if (ip->i_ea_area != NULL)
-		return (EBUSY);
+	ffs_lock_ea(vp);
+	if (ip->i_ea_area != NULL) {
+		ip->i_ea_refs++;
+		ffs_unlock_ea(vp);
+		return (0);
+	}
 	dp = ip->i_din2;
 	error = ffs_rdextattr(&ip->i_ea_area, vp, td, 0);
-	if (error)
+	if (error) {
+		ffs_unlock_ea(vp);
 		return (error);
+	}
 	ip->i_ea_len = dp->di_extsize;
 	ip->i_ea_error = 0;
+	ip->i_ea_refs++;
+	ffs_unlock_ea(vp);
 	return (0);
 }
 
@@ -1258,11 +1295,16 @@ ffs_close_ea(struct vnode *vp, int commit, struct ucred *cred, struct thread *td
 	struct ufs2_dinode *dp;
 
 	ip = VTOI(vp);
-	if (ip->i_ea_area == NULL)
+
+	ffs_lock_ea(vp);
+	if (ip->i_ea_area == NULL) {
+		ffs_unlock_ea(vp);
 		return (EINVAL);
+	}
 	dp = ip->i_din2;
 	error = ip->i_ea_error;
 	if (commit && error == 0) {
+		ASSERT_VOP_ELOCKED(vp, "ffs_close_ea commit");
 		if (cred == NOCRED)
 			cred =  vp->v_mount->mnt_cred;
 		liovec.iov_base = ip->i_ea_area;
@@ -1279,10 +1321,13 @@ ffs_close_ea(struct vnode *vp, int commit, struct ucred *cred, struct thread *td
 			error = ffs_truncate(vp, 0, IO_EXT, cred, td);
 		error = ffs_extwrite(vp, &luio, IO_EXT | IO_SYNC, cred);
 	}
-	free(ip->i_ea_area, M_TEMP);
-	ip->i_ea_area = NULL;
-	ip->i_ea_len = 0;
-	ip->i_ea_error = 0;
+	if (--ip->i_ea_refs == 0) {
+		free(ip->i_ea_area, M_TEMP);
+		ip->i_ea_area = NULL;
+		ip->i_ea_len = 0;
+		ip->i_ea_error = 0;
+	}
+	ffs_unlock_ea(vp);
 	return (error);
 }
 
@@ -1392,7 +1437,6 @@ vop_deleteextattr {
 	uint32_t ealength, ul;
 	int ealen, olen, eapad1, eapad2, error, i, easize;
 	u_char *eae, *p;
-	int stand_alone;
 
 	ip = VTOI(ap->a_vp);
 	fs = ip->i_fs;
@@ -1409,19 +1453,19 @@ vop_deleteextattr {
 	error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace,
 	    ap->a_cred, ap->a_td, VWRITE);
 	if (error) {
+
+		/*
+		 * ffs_lock_ea is not needed there, because the vnode
+		 * must be exclusively locked.
+		 */
 		if (ip->i_ea_area != NULL && ip->i_ea_error == 0)
 			ip->i_ea_error = error;
 		return (error);
 	}
 
-	if (ip->i_ea_area == NULL) {
-		error = ffs_open_ea(ap->a_vp, ap->a_cred, ap->a_td);
-		if (error)
-			return (error);
-		stand_alone = 1;
-	} else {
-		stand_alone = 0;
-	}
+	error = ffs_open_ea(ap->a_vp, ap->a_cred, ap->a_td);
+	if (error)
+		return (error);
 
 	ealength = eapad1 = ealen = eapad2 = 0;
 
@@ -1434,8 +1478,7 @@ vop_deleteextattr {
 	if (olen == -1) {
 		/* delete but nonexistent */
 		free(eae, M_TEMP);
-		if (stand_alone)
-			ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
+		ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
 		return(ENOATTR);
 	}
 	bcopy(p, &ul, sizeof ul);
@@ -1446,9 +1489,8 @@ vop_deleteextattr {
 	}
 	if (easize > NXADDR * fs->fs_bsize) {
 		free(eae, M_TEMP);
-		if (stand_alone)
-			ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
-		else if (ip->i_ea_error == 0)
+		ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
+		if (ip->i_ea_area != NULL && ip->i_ea_error == 0)
 			ip->i_ea_error = ENOSPC;
 		return(ENOSPC);
 	}
@@ -1456,8 +1498,7 @@ vop_deleteextattr {
 	ip->i_ea_area = eae;
 	ip->i_ea_len = easize;
 	free(p, M_TEMP);
-	if (stand_alone)
-		error = ffs_close_ea(ap->a_vp, 1, ap->a_cred, ap->a_td);
+	error = ffs_close_ea(ap->a_vp, 1, ap->a_cred, ap->a_td);
 	return(error);
 }
 
@@ -1482,7 +1523,7 @@ vop_getextattr {
 	struct fs *fs;
 	u_char *eae, *p;
 	unsigned easize;
-	int error, ealen, stand_alone;
+	int error, ealen;
 
 	ip = VTOI(ap->a_vp);
 	fs = ip->i_fs;
@@ -1495,14 +1536,10 @@ vop_getextattr {
 	if (error)
 		return (error);
 
-	if (ip->i_ea_area == NULL) {
-		error = ffs_open_ea(ap->a_vp, ap->a_cred, ap->a_td);
-		if (error)
-			return (error);
-		stand_alone = 1;
-	} else {
-		stand_alone = 0;
-	}
+	error = ffs_open_ea(ap->a_vp, ap->a_cred, ap->a_td);
+	if (error)
+		return (error);
+
 	eae = ip->i_ea_area;
 	easize = ip->i_ea_len;
 
@@ -1516,8 +1553,8 @@ vop_getextattr {
 			error = uiomove(p, ealen, ap->a_uio);
 	} else
 		error = ENOATTR;
-	if (stand_alone)
-		ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
+
+	ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
 	return(error);
 }
 
@@ -1542,7 +1579,7 @@ vop_listextattr {
 	u_char *eae, *p, *pe, *pn;
 	unsigned easize;
 	uint32_t ul;
-	int error, ealen, stand_alone;
+	int error, ealen;
 
 	ip = VTOI(ap->a_vp);
 	fs = ip->i_fs;
@@ -1555,14 +1592,9 @@ vop_listextattr {
 	if (error)
 		return (error);
 
-	if (ip->i_ea_area == NULL) {
-		error = ffs_open_ea(ap->a_vp, ap->a_cred, ap->a_td);
-		if (error)
-			return (error);
-		stand_alone = 1;
-	} else {
-		stand_alone = 0;
-	}
+	error = ffs_open_ea(ap->a_vp, ap->a_cred, ap->a_td);
+	if (error)
+		return (error);
 	eae = ip->i_ea_area;
 	easize = ip->i_ea_len;
 
@@ -1586,8 +1618,7 @@ vop_listextattr {
 			error = uiomove(p, ealen + 1, ap->a_uio);
 		}
 	}
-	if (stand_alone)
-		ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
+	ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
 	return(error);
 }
 
@@ -1612,7 +1643,6 @@ vop_setextattr {
 	uint32_t ealength, ul;
 	int ealen, olen, eapad1, eapad2, error, i, easize;
 	u_char *eae, *p;
-	int stand_alone;
 
 	ip = VTOI(ap->a_vp);
 	fs = ip->i_fs;
@@ -1633,19 +1663,19 @@ vop_setextattr {
 	error = extattr_check_cred(ap->a_vp, ap->a_attrnamespace,
 	    ap->a_cred, ap->a_td, VWRITE);
 	if (error) {
+
+		/*
+		 * ffs_lock_ea is not needed there, because the vnode
+		 * must be exclusively locked.
+		 */
 		if (ip->i_ea_area != NULL && ip->i_ea_error == 0)
 			ip->i_ea_error = error;
 		return (error);
 	}
 
-	if (ip->i_ea_area == NULL) {
-		error = ffs_open_ea(ap->a_vp, ap->a_cred, ap->a_td);
-		if (error)
-			return (error);
-		stand_alone = 1;
-	} else {
-		stand_alone = 0;
-	}
+	error = ffs_open_ea(ap->a_vp, ap->a_cred, ap->a_td);
+	if (error)
+		return (error);
 
 	ealen = ap->a_uio->uio_resid;
 	ealength = sizeof(uint32_t) + 3 + strlen(ap->a_name);
@@ -1677,9 +1707,8 @@ vop_setextattr {
 	}
 	if (easize > NXADDR * fs->fs_bsize) {
 		free(eae, M_TEMP);
-		if (stand_alone)
-			ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
-		else if (ip->i_ea_error == 0)
+		ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
+		if (ip->i_ea_area != NULL && ip->i_ea_error == 0)
 			ip->i_ea_error = ENOSPC;
 		return(ENOSPC);
 	}
@@ -1695,9 +1724,8 @@ vop_setextattr {
 	error = uiomove(p, ealen, ap->a_uio);
 	if (error) {
 		free(eae, M_TEMP);
-		if (stand_alone)
-			ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
-		else if (ip->i_ea_error == 0)
+		ffs_close_ea(ap->a_vp, 0, ap->a_cred, ap->a_td);
+		if (ip->i_ea_area != NULL && ip->i_ea_error == 0)
 			ip->i_ea_error = error;
 		return(error);
 	}
@@ -1708,8 +1736,7 @@ vop_setextattr {
 	ip->i_ea_area = eae;
 	ip->i_ea_len = easize;
 	free(p, M_TEMP);
-	if (stand_alone)
-		error = ffs_close_ea(ap->a_vp, 1, ap->a_cred, ap->a_td);
+	error = ffs_close_ea(ap->a_vp, 1, ap->a_cred, ap->a_td);
 	return(error);
 }
 
diff --git a/sys/ufs/ufs/inode.h b/sys/ufs/ufs/inode.h
index 39f46d0a..5c4732d 100644
--- a/sys/ufs/ufs/inode.h
+++ b/sys/ufs/ufs/inode.h
@@ -94,6 +94,7 @@ struct inode {
 	u_char	  *i_ea_area;	/* Pointer to malloced copy of EA area */
 	unsigned  i_ea_len;	/* Length of i_ea_area */
 	int	  i_ea_error;	/* First errno in transaction */
+	int	  i_ea_refs;	/* Number of users of EA area */
 
 	/*
 	 * Copies from the on-disk dinode itself.
@@ -125,6 +126,8 @@ struct inode {
 #define	IN_SPACECOUNTED	0x0080		/* Blocks to be freed in free count. */
 #define	IN_LAZYACCESS	0x0100		/* Process IN_ACCESS after the
 					   suspension finished */
+#define	IN_EA_LOCKED	0x0200
+#define	IN_EA_LOCKWAIT	0x0400
 
 #define i_devvp i_ump->um_devvp
 #define i_umbufobj i_ump->um_bo
diff --git a/sys/vm/vm_extern.h b/sys/vm/vm_extern.h
index 0a54372..475a20e 100644
--- a/sys/vm/vm_extern.h
+++ b/sys/vm/vm_extern.h
@@ -42,19 +42,6 @@ struct vnode;
 
 #ifdef _KERNEL
 
-#ifdef TYPEDEF_FOR_UAP
-int getpagesize(struct thread *, void *, int *);
-int madvise(struct thread *, void *, int *);
-int mincore(struct thread *, void *, int *);
-int mprotect(struct thread *, void *, int *);
-int msync(struct thread *, void *, int *);
-int munmap(struct thread *, void *, int *);
-int obreak(struct thread *, void *, int *);
-int sbrk(struct thread *, void *, int *);
-int sstk(struct thread *, void *, int *);
-int swapon(struct thread *, void *, int *);
-#endif			/* TYPEDEF_FOR_UAP */
-
 int kernacc(void *, int, int);
 vm_offset_t kmem_alloc(vm_map_t, vm_size_t);
 vm_offset_t kmem_alloc_nofault(vm_map_t, vm_size_t);
diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c
index f395be8..718c890 100644
--- a/sys/vm/vm_map.c
+++ b/sys/vm/vm_map.c
@@ -2217,6 +2217,16 @@ vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end,
 		 *
 		 */
 		if (entry->wired_count == 0) {
+			if ((entry->protection & (VM_PROT_READ|VM_PROT_EXECUTE))
+			    == 0) {
+				if ((flags & VM_MAP_WIRE_HOLESOK) == 0) {
+					end = entry->end;
+					rv = KERN_INVALID_ADDRESS;
+					goto done;
+				}
+				entry->eflags |= MAP_ENTRY_WIRE_SKIPPED;
+				goto next_entry;
+			}
 			entry->wired_count++;
 			saved_start = entry->start;
 			saved_end = entry->end;
@@ -2274,6 +2284,7 @@ vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end,
 		 * Check the map for holes in the specified region.
 		 * If VM_MAP_WIRE_HOLESOK was specified, skip this check.
 		 */
+	next_entry:
 		if (((flags & VM_MAP_WIRE_HOLESOK) == 0) &&
 		    (entry->end < end && (entry->next == &map->header ||
 		    entry->next->start > entry->end))) {
@@ -2295,6 +2306,8 @@ done:
 	}
 	entry = first_entry;
 	while (entry != &map->header && entry->start < end) {
+		if ((entry->eflags & MAP_ENTRY_WIRE_SKIPPED) != 0)
+			goto next_entry_done;
 		if (rv == KERN_SUCCESS) {
 			if (user_wire)
 				entry->eflags |= MAP_ENTRY_USER_WIRED;
@@ -2317,9 +2330,10 @@ done:
 				    entry->object.vm_object->type == OBJT_DEVICE);
 			}
 		}
+	next_entry_done:
 		KASSERT(entry->eflags & MAP_ENTRY_IN_TRANSITION,
 			("vm_map_wire: in-transition flag missing"));
-		entry->eflags &= ~MAP_ENTRY_IN_TRANSITION;
+		entry->eflags &= ~(MAP_ENTRY_IN_TRANSITION|MAP_ENTRY_WIRE_SKIPPED);
 		if (entry->eflags & MAP_ENTRY_NEEDS_WAKEUP) {
 			entry->eflags &= ~MAP_ENTRY_NEEDS_WAKEUP;
 			need_wakeup = TRUE;
diff --git a/sys/vm/vm_map.h b/sys/vm/vm_map.h
index 34b183b..70c3a0b 100644
--- a/sys/vm/vm_map.h
+++ b/sys/vm/vm_map.h
@@ -137,6 +137,8 @@ struct vm_map_entry {
 #define	MAP_ENTRY_GROWS_DOWN		0x1000	/* Top-down stacks */
 #define	MAP_ENTRY_GROWS_UP		0x2000	/* Bottom-up stacks */
 
+#define	MAP_ENTRY_WIRE_SKIPPED		0x4000
+
 #ifdef	_KERNEL
 static __inline u_char
 vm_map_entry_behavior(vm_map_entry_t entry)
diff --git a/sys/vm/vm_mmap.c b/sys/vm/vm_mmap.c
index 2d668c4..6cc0acc 100644
--- a/sys/vm/vm_mmap.c
+++ b/sys/vm/vm_mmap.c
@@ -620,10 +620,6 @@ mprotect(td, uap)
 	addr = (vm_offset_t) uap->addr;
 	size = uap->len;
 	prot = uap->prot & VM_PROT_ALL;
-#if defined(VM_PROT_READ_IS_EXEC)
-	if (prot & VM_PROT_READ)
-		prot |= VM_PROT_EXECUTE;
-#endif
 
 	pageoff = (addr & PAGE_MASK);
 	addr -= pageoff;
@@ -1441,14 +1437,6 @@ vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot,
 	if (flags & MAP_NOCORE)
 		docow |= MAP_DISABLE_COREDUMP;
 
-#if defined(VM_PROT_READ_IS_EXEC)
-	if (prot & VM_PROT_READ)
-		prot |= VM_PROT_EXECUTE;
-
-	if (maxprot & VM_PROT_READ)
-		maxprot |= VM_PROT_EXECUTE;
-#endif
-
 	if (flags & MAP_STACK)
 		rv = vm_map_stack(map, *addr, size, prot, maxprot,
 		    docow | MAP_STACK_GROWS_DOWN);
diff --git a/sys/vm/vm_reserv.c b/sys/vm/vm_reserv.c
index 4ea9079..d9e908f 100644
--- a/sys/vm/vm_reserv.c
+++ b/sys/vm/vm_reserv.c
@@ -138,8 +138,8 @@ static vm_reserv_t vm_reserv_array;
  * The partially-populated reservation queue
  *
  * This queue enables the fast recovery of an unused cached or free small page
- * from a partially-populated reservation.  The head of this queue is either
- * the least-recently-populated or most-recently-depopulated reservation.
+ * from a partially-populated reservation.  The reservation at the head of
+ * this queue is the least-recently-changed, partially-populated reservation.
  *
  * Access to this queue is synchronized by the free page queue lock.
  */
@@ -209,7 +209,7 @@ sysctl_vm_reserv_partpopq(SYSCTL_HANDLER_ARGS)
 /*
  * Reduces the given reservation's population count.  If the population count
  * becomes zero, the reservation is destroyed.  Additionally, moves the
- * reservation to the head of the partially-populated reservations queue if the
+ * reservation to the tail of the partially-populated reservations queue if the
  * population count is non-zero.
  *
  * The free page queue lock must be held.
@@ -235,7 +235,7 @@ vm_reserv_depopulate(vm_reserv_t rv)
 		vm_reserv_freed++;
 	} else {
 		rv->inpartpopq = TRUE;
-		TAILQ_INSERT_HEAD(&vm_rvq_partpop, rv, partpopq);
+		TAILQ_INSERT_TAIL(&vm_rvq_partpop, rv, partpopq);
 	}
 }
 
diff --git a/sys/vm/vm_unix.c b/sys/vm/vm_unix.c
index cd98be9..6bb9917 100644
--- a/sys/vm/vm_unix.c
+++ b/sys/vm/vm_unix.c
@@ -117,7 +117,7 @@ obreak(td, uap)
 			goto done;
 		}
 		rv = vm_map_insert(&vm->vm_map, NULL, 0, old, new,
-		    VM_PROT_ALL, VM_PROT_ALL, 0);
+		    VM_PROT_RW, VM_PROT_ALL, 0);
 		if (rv != KERN_SUCCESS) {
 			error = ENOMEM;
 			goto done;
diff --git a/sys/xen/evtchn/evtchn.c b/sys/xen/evtchn/evtchn.c
index 61b738b..59d24b2 100644
--- a/sys/xen/evtchn/evtchn.c
+++ b/sys/xen/evtchn/evtchn.c
@@ -512,7 +512,7 @@ bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain,
 int 
 bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
     const char *devname, driver_filter_t filter, driver_intr_t handler,
-    unsigned long irqflags, unsigned int *irqp)
+    void *arg, unsigned long irqflags, unsigned int *irqp)
 {
 	unsigned int irq;
 	int error;
diff --git a/sys/xen/reboot.c b/sys/xen/reboot.c
index 892dfbf..04ba132 100644
--- a/sys/xen/reboot.c
+++ b/sys/xen/reboot.c
@@ -176,9 +176,9 @@ xen_suspend()
 	/*
 	 * Bind us to CPU 0 and stop any other VCPUs.
 	 */
-	mtx_lock_spin(&sched_lock);
+	thread_lock(curthread);
 	sched_bind(curthread, 0);
-	mtx_unlock_spin(&sched_lock);
+	thread_unlock(curthread);
 	KASSERT(PCPU_GET(cpuid) == 0, ("xen_suspend: not running on cpu 0"));
 
 	map = PCPU_GET(other_cpus) & ~stopped_cpus;
@@ -188,8 +188,10 @@ xen_suspend()
 
 	if (DEVICE_SUSPEND(root_bus) != 0) {
 		printf("xen_suspend: device_suspend failed\n");
+#ifdef SMP
 		if (map)
 			restart_cpus(map);
+#endif
 		return;
 	}
 
@@ -253,7 +255,9 @@ xen_suspend()
 	DEVICE_RESUME(root_bus);
 
 #ifdef SMP
+	thread_lock(curthread);
 	sched_unbind(curthread);
+	thread_unlock(curthread);
 	if (map)
 		restart_cpus(map);
 #endif