MFC

856 files changed, 37700 insertions, 12532 deletions

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index d2e4aad..31dbb3f 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -1431,11 +1431,11 @@ cpususpend_handler(void)
 	while (!CPU_ISSET(cpu, &started_cpus))
 		ia32_pause();
 
-	CPU_CLR_ATOMIC(cpu, &started_cpus);
-
 	/* Resume MCA and local APIC */
 	mca_resume();
 	lapic_setup(0);
+
+	CPU_CLR_ATOMIC(cpu, &started_cpus);
 }
 
 /*
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index a7b1e2b..5436b6f 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -102,6 +102,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_vm.h"
 
 #include <sys/param.h>
+#include <sys/bus.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/ktr.h>
@@ -133,6 +134,8 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_reserv.h>
 #include <vm/uma.h>
 
+#include <machine/intr_machdep.h>
+#include <machine/apicvar.h>
 #include <machine/cpu.h>
 #include <machine/cputypes.h>
 #include <machine/md_var.h>
@@ -1151,6 +1154,15 @@ pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
 	    eva - sva < PMAP_CLFLUSH_THRESHOLD) {
 
 		/*
+		 * XXX: Some CPUs fault, hang, or trash the local APIC
+		 * registers if we use CLFLUSH on the local APIC
+		 * range.  The local APIC is always uncached, so we
+		 * don't need to flush for that range anyway.
+		 */
+		if (pmap_kextract(sva) == lapic_paddr)
+			return;
+
+		/*
 		 * Otherwise, do per-cache line flush.  Use the mfence
 		 * instruction to insure that previous stores are
 		 * included in the write-back.  The processor
diff --git a/sys/amd64/conf/GENERIC b/sys/amd64/conf/GENERIC
index 48f41b3..5819a0d 100644
--- a/sys/amd64/conf/GENERIC
+++ b/sys/amd64/conf/GENERIC
@@ -197,7 +197,6 @@ device		uart		# Generic UART driver
 device		ppc
 device		ppbus		# Parallel port bus (required)
 device		lpt		# Printer
-device		plip		# TCP/IP over parallel
 device		ppi		# Parallel port interface device
 #device		vpo		# Requires scbus and da
 
@@ -332,3 +331,11 @@ device		snd_via8233	# VIA VT8233x Audio
 device		mmc		# MMC/SD bus
 device		mmcsd		# MMC/SD memory card
 device		sdhci		# Generic PCI SD Host Controller
+
+# VirtIO support
+device		virtio		# Generic VirtIO bus (required)
+device		virtio_pci	# VirtIO PCI device
+device		vtnet		# VirtIO Ethernet device
+device		virtio_blk	# VirtIO Block device
+device		virtio_scsi	# VirtIO SCSI device
+device		virtio_balloon	# VirtIO Memory Balloon device
diff --git a/sys/amd64/conf/NOTES b/sys/amd64/conf/NOTES
index 6562981..a4371f7 100644
--- a/sys/amd64/conf/NOTES
+++ b/sys/amd64/conf/NOTES
@@ -440,6 +440,15 @@ device		safe		# SafeNet 1141
 options 	SAFE_DEBUG	# enable debugging support: hw.safe.debug
 options 	SAFE_RNDTEST	# enable rndtest support
 
+#
+# VirtIO support
+device		virtio		# Generic VirtIO bus (required)
+device		virtio_pci	# VirtIO PCI Interface
+device		vtnet		# VirtIO Ethernet device
+device		virtio_blk	# VirtIO Block device
+device		virtio_scsi	# VirtIO SCSI device
+device		virtio_balloon	# VirtIO Memory Balloon device
+
 #####################################################################
 
 #
diff --git a/sys/amd64/include/intr_machdep.h b/sys/amd64/include/intr_machdep.h
index 700e35f..8671605 100644
--- a/sys/amd64/include/intr_machdep.h
+++ b/sys/amd64/include/intr_machdep.h
@@ -94,7 +94,7 @@ struct pic {
 	int (*pic_config_intr)(struct intsrc *, enum intr_trigger,
 	    enum intr_polarity);
 	int (*pic_assign_cpu)(struct intsrc *, u_int apic_id);
-	STAILQ_ENTRY(pic) pics;
+	TAILQ_ENTRY(pic) pics;
 };
 
 /* Flags for pic_disable_source() */
diff --git a/sys/amd64/include/vmm.h b/sys/amd64/include/vmm.h
new file mode 100644
index 0000000..ec94083
--- /dev/null
+++ b/sys/amd64/include/vmm.h
@@ -0,0 +1,293 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VMM_H_
+#define	_VMM_H_
+
+#ifdef _KERNEL
+
+#define	VM_MAX_NAMELEN	32
+
+struct vm;
+struct vm_memory_segment;
+struct seg_desc;
+struct vm_exit;
+struct vm_run;
+struct vlapic;
+
+enum x2apic_state;
+
+typedef int	(*vmm_init_func_t)(void);
+typedef int	(*vmm_cleanup_func_t)(void);
+typedef void *	(*vmi_init_func_t)(struct vm *vm); /* instance specific apis */
+typedef int	(*vmi_run_func_t)(void *vmi, int vcpu, register_t rip);
+typedef void	(*vmi_cleanup_func_t)(void *vmi);
+typedef int	(*vmi_mmap_set_func_t)(void *vmi, vm_paddr_t gpa,
+				       vm_paddr_t hpa, size_t length,
+				       vm_memattr_t attr, int prot,
+				       boolean_t superpages_ok);
+typedef vm_paddr_t (*vmi_mmap_get_func_t)(void *vmi, vm_paddr_t gpa);
+typedef int	(*vmi_get_register_t)(void *vmi, int vcpu, int num,
+				      uint64_t *retval);
+typedef int	(*vmi_set_register_t)(void *vmi, int vcpu, int num,
+				      uint64_t val);
+typedef int	(*vmi_get_desc_t)(void *vmi, int vcpu, int num,
+				  struct seg_desc *desc);
+typedef int	(*vmi_set_desc_t)(void *vmi, int vcpu, int num,
+				  struct seg_desc *desc);
+typedef int	(*vmi_inject_event_t)(void *vmi, int vcpu,
+				      int type, int vector,
+				      uint32_t code, int code_valid);
+typedef int	(*vmi_get_cap_t)(void *vmi, int vcpu, int num, int *retval);
+typedef int	(*vmi_set_cap_t)(void *vmi, int vcpu, int num, int val);
+
+struct vmm_ops {
+	vmm_init_func_t		init;		/* module wide initialization */
+	vmm_cleanup_func_t	cleanup;
+
+	vmi_init_func_t		vminit;		/* vm-specific initialization */
+	vmi_run_func_t		vmrun;
+	vmi_cleanup_func_t	vmcleanup;
+	vmi_mmap_set_func_t	vmmmap_set;
+	vmi_mmap_get_func_t	vmmmap_get;
+	vmi_get_register_t	vmgetreg;
+	vmi_set_register_t	vmsetreg;
+	vmi_get_desc_t		vmgetdesc;
+	vmi_set_desc_t		vmsetdesc;
+	vmi_inject_event_t	vminject;
+	vmi_get_cap_t		vmgetcap;
+	vmi_set_cap_t		vmsetcap;
+};
+
+extern struct vmm_ops vmm_ops_intel;
+extern struct vmm_ops vmm_ops_amd;
+
+struct vm *vm_create(const char *name);
+void vm_destroy(struct vm *vm);
+const char *vm_name(struct vm *vm);
+int vm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len);
+int vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa);
+int vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len);
+vm_paddr_t vm_gpa2hpa(struct vm *vm, vm_paddr_t gpa, size_t size);
+int vm_gpabase2memseg(struct vm *vm, vm_paddr_t gpabase,
+	      struct vm_memory_segment *seg);
+int vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval);
+int vm_set_register(struct vm *vm, int vcpu, int reg, uint64_t val);
+int vm_get_seg_desc(struct vm *vm, int vcpu, int reg,
+		    struct seg_desc *ret_desc);
+int vm_set_seg_desc(struct vm *vm, int vcpu, int reg,
+		    struct seg_desc *desc);
+int vm_get_pinning(struct vm *vm, int vcpu, int *cpuid);
+int vm_set_pinning(struct vm *vm, int vcpu, int cpuid);
+int vm_run(struct vm *vm, struct vm_run *vmrun);
+int vm_inject_event(struct vm *vm, int vcpu, int type,
+		    int vector, uint32_t error_code, int error_code_valid);
+int vm_inject_nmi(struct vm *vm, int vcpu);
+int vm_nmi_pending(struct vm *vm, int vcpuid);
+void vm_nmi_clear(struct vm *vm, int vcpuid);
+uint64_t *vm_guest_msrs(struct vm *vm, int cpu);
+struct vlapic *vm_lapic(struct vm *vm, int cpu);
+int vm_get_capability(struct vm *vm, int vcpu, int type, int *val);
+int vm_set_capability(struct vm *vm, int vcpu, int type, int val);
+int vm_get_x2apic_state(struct vm *vm, int vcpu, enum x2apic_state *state);
+int vm_set_x2apic_state(struct vm *vm, int vcpu, enum x2apic_state state);
+void vm_activate_cpu(struct vm *vm, int vcpu);
+cpuset_t vm_active_cpus(struct vm *vm);
+struct vm_exit *vm_exitinfo(struct vm *vm, int vcpuid);
+
+/*
+ * Return 1 if device indicated by bus/slot/func is supposed to be a
+ * pci passthrough device.
+ *
+ * Return 0 otherwise.
+ */
+int vmm_is_pptdev(int bus, int slot, int func);
+
+void *vm_iommu_domain(struct vm *vm);
+
+enum vcpu_state {
+	VCPU_IDLE,
+	VCPU_RUNNING,
+	VCPU_CANNOT_RUN,
+};
+
+int vcpu_set_state(struct vm *vm, int vcpu, enum vcpu_state state);
+enum vcpu_state vcpu_get_state(struct vm *vm, int vcpu);
+
+static int __inline
+vcpu_is_running(struct vm *vm, int vcpu)
+{
+	return (vcpu_get_state(vm, vcpu) == VCPU_RUNNING);
+}
+
+void *vcpu_stats(struct vm *vm, int vcpu);
+void vm_interrupt_hostcpu(struct vm *vm, int vcpu);
+
+#endif	/* KERNEL */
+
+#include <machine/vmm_instruction_emul.h>
+
+#define	VM_MAXCPU	8			/* maximum virtual cpus */
+
+/*
+ * Identifiers for events that can be injected into the VM
+ */
+enum vm_event_type {
+	VM_EVENT_NONE,
+	VM_HW_INTR,
+	VM_NMI,
+	VM_HW_EXCEPTION,
+	VM_SW_INTR,
+	VM_PRIV_SW_EXCEPTION,
+	VM_SW_EXCEPTION,
+	VM_EVENT_MAX
+};
+
+/*
+ * Identifiers for architecturally defined registers.
+ */
+enum vm_reg_name {
+	VM_REG_GUEST_RAX,
+	VM_REG_GUEST_RBX,
+	VM_REG_GUEST_RCX,
+	VM_REG_GUEST_RDX,
+	VM_REG_GUEST_RSI,
+	VM_REG_GUEST_RDI,
+	VM_REG_GUEST_RBP,
+	VM_REG_GUEST_R8,
+	VM_REG_GUEST_R9,
+	VM_REG_GUEST_R10,
+	VM_REG_GUEST_R11,
+	VM_REG_GUEST_R12,
+	VM_REG_GUEST_R13,
+	VM_REG_GUEST_R14,
+	VM_REG_GUEST_R15,
+	VM_REG_GUEST_CR0,
+	VM_REG_GUEST_CR3,
+	VM_REG_GUEST_CR4,
+	VM_REG_GUEST_DR7,
+	VM_REG_GUEST_RSP,
+	VM_REG_GUEST_RIP,
+	VM_REG_GUEST_RFLAGS,
+	VM_REG_GUEST_ES,
+	VM_REG_GUEST_CS,
+	VM_REG_GUEST_SS,
+	VM_REG_GUEST_DS,
+	VM_REG_GUEST_FS,
+	VM_REG_GUEST_GS,
+	VM_REG_GUEST_LDTR,
+	VM_REG_GUEST_TR,
+	VM_REG_GUEST_IDTR,
+	VM_REG_GUEST_GDTR,
+	VM_REG_GUEST_EFER,
+	VM_REG_LAST
+};
+
+/*
+ * Identifiers for optional vmm capabilities
+ */
+enum vm_cap_type {
+	VM_CAP_HALT_EXIT,
+	VM_CAP_MTRAP_EXIT,
+	VM_CAP_PAUSE_EXIT,
+	VM_CAP_UNRESTRICTED_GUEST,
+	VM_CAP_MAX
+};
+
+enum x2apic_state {
+	X2APIC_ENABLED,
+	X2APIC_AVAILABLE,
+	X2APIC_DISABLED,
+	X2APIC_STATE_LAST
+};
+
+/*
+ * The 'access' field has the format specified in Table 21-2 of the Intel
+ * Architecture Manual vol 3b.
+ *
+ * XXX The contents of the 'access' field are architecturally defined except
+ * bit 16 - Segment Unusable.
+ */
+struct seg_desc {
+	uint64_t	base;
+	uint32_t	limit;
+	uint32_t	access;
+};
+
+enum vm_exitcode {
+	VM_EXITCODE_INOUT,
+	VM_EXITCODE_VMX,
+	VM_EXITCODE_BOGUS,
+	VM_EXITCODE_RDMSR,
+	VM_EXITCODE_WRMSR,
+	VM_EXITCODE_HLT,
+	VM_EXITCODE_MTRAP,
+	VM_EXITCODE_PAUSE,
+	VM_EXITCODE_PAGING,
+	VM_EXITCODE_SPINUP_AP,
+	VM_EXITCODE_MAX
+};
+
+struct vm_exit {
+	enum vm_exitcode	exitcode;
+	int			inst_length;	/* 0 means unknown */
+	uint64_t		rip;
+	union {
+		struct {
+			uint16_t	bytes:3;	/* 1 or 2 or 4 */
+			uint16_t	in:1;		/* out is 0, in is 1 */
+			uint16_t	string:1;
+			uint16_t	rep:1;
+			uint16_t	port;
+			uint32_t	eax;		/* valid for out */
+		} inout;
+		struct {
+			uint64_t	gpa;
+			struct vie	vie;
+		} paging;
+		/*
+		 * VMX specific payload. Used when there is no "better"
+		 * exitcode to represent the VM-exit.
+		 */
+		struct {
+			int		error;		/* vmx inst error */
+			uint32_t	exit_reason;
+			uint64_t	exit_qualification;
+		} vmx;
+		struct {
+			uint32_t	code;		/* ecx value */
+			uint64_t	wval;
+		} msr;
+		struct {
+			int		vcpu;
+			uint64_t	rip;
+		} spinup_ap;
+	} u;
+};
+
+#endif	/* _VMM_H_ */
diff --git a/sys/amd64/include/vmm_dev.h b/sys/amd64/include/vmm_dev.h
new file mode 100644
index 0000000..2311673
--- /dev/null
+++ b/sys/amd64/include/vmm_dev.h
@@ -0,0 +1,215 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_VMM_DEV_H_
+#define	_VMM_DEV_H_
+
+#ifdef _KERNEL
+void	vmmdev_init(void);
+int	vmmdev_cleanup(void);
+#endif
+
+struct vm_memory_segment {
+	vm_paddr_t	gpa;	/* in */
+	size_t		len;	/* in */
+};
+
+struct vm_register {
+	int		cpuid;
+	int		regnum;		/* enum vm_reg_name */
+	uint64_t	regval;
+};
+
+struct vm_seg_desc {			/* data or code segment */
+	int		cpuid;
+	int		regnum;		/* enum vm_reg_name */
+	struct seg_desc desc;
+};
+
+struct vm_pin {
+	int		vm_cpuid;
+	int		host_cpuid;	/* -1 to unpin */
+};
+
+struct vm_run {
+	int		cpuid;
+	uint64_t	rip;		/* start running here */
+	struct vm_exit	vm_exit;
+};
+
+struct vm_event {
+	int		cpuid;
+	enum vm_event_type type;
+	int		vector;
+	uint32_t	error_code;
+	int		error_code_valid;
+};
+
+struct vm_lapic_irq {
+	int		cpuid;
+	int		vector;
+};
+
+struct vm_capability {
+	int		cpuid;
+	enum vm_cap_type captype;
+	int		capval;
+	int		allcpus;
+};
+
+struct vm_pptdev {
+	int		bus;
+	int		slot;
+	int		func;
+};
+
+struct vm_pptdev_mmio {
+	int		bus;
+	int		slot;
+	int		func;
+	vm_paddr_t	gpa;
+	vm_paddr_t	hpa;
+	size_t		len;
+};
+
+struct vm_pptdev_msi {
+	int		vcpu;
+	int		bus;
+	int		slot;
+	int		func;
+	int		numvec;		/* 0 means disabled */
+	int		vector;
+	int		destcpu;
+};
+
+struct vm_pptdev_msix {
+	int		vcpu;
+	int		bus;
+	int		slot;
+	int		func;
+	int		idx;
+	uint32_t	msg;
+	uint32_t	vector_control;
+	uint64_t	addr;
+};
+
+struct vm_nmi {
+	int		cpuid;
+};
+
+#define	MAX_VM_STATS	64
+struct vm_stats {
+	int		cpuid;				/* in */
+	int		num_entries;			/* out */
+	struct timeval	tv;
+	uint64_t	statbuf[MAX_VM_STATS];
+};
+
+struct vm_stat_desc {
+	int		index;				/* in */
+	char		desc[128];			/* out */
+};
+
+struct vm_x2apic {
+	int			cpuid;
+	enum x2apic_state	state;
+};
+
+enum {
+	IOCNUM_RUN,
+	IOCNUM_SET_PINNING,
+	IOCNUM_GET_PINNING,
+	IOCNUM_MAP_MEMORY,
+	IOCNUM_GET_MEMORY_SEG,
+	IOCNUM_SET_REGISTER,
+	IOCNUM_GET_REGISTER,
+	IOCNUM_SET_SEGMENT_DESCRIPTOR,
+	IOCNUM_GET_SEGMENT_DESCRIPTOR,
+	IOCNUM_INJECT_EVENT,
+	IOCNUM_LAPIC_IRQ,
+	IOCNUM_SET_CAPABILITY,
+	IOCNUM_GET_CAPABILITY,
+	IOCNUM_BIND_PPTDEV,
+	IOCNUM_UNBIND_PPTDEV,
+	IOCNUM_MAP_PPTDEV_MMIO,
+	IOCNUM_PPTDEV_MSI,
+	IOCNUM_PPTDEV_MSIX,
+	IOCNUM_INJECT_NMI,
+	IOCNUM_VM_STATS,
+	IOCNUM_VM_STAT_DESC,
+	IOCNUM_SET_X2APIC_STATE,
+	IOCNUM_GET_X2APIC_STATE,
+};
+
+#define	VM_RUN		\
+	_IOWR('v', IOCNUM_RUN, struct vm_run)
+#define	VM_SET_PINNING	\
+	_IOW('v', IOCNUM_SET_PINNING, struct vm_pin)
+#define	VM_GET_PINNING	\
+	_IOWR('v', IOCNUM_GET_PINNING, struct vm_pin)
+#define	VM_MAP_MEMORY	\
+	_IOWR('v', IOCNUM_MAP_MEMORY, struct vm_memory_segment)
+#define	VM_GET_MEMORY_SEG \
+	_IOWR('v', IOCNUM_GET_MEMORY_SEG, struct vm_memory_segment)
+#define	VM_SET_REGISTER \
+	_IOW('v', IOCNUM_SET_REGISTER, struct vm_register)
+#define	VM_GET_REGISTER \
+	_IOWR('v', IOCNUM_GET_REGISTER, struct vm_register)
+#define	VM_SET_SEGMENT_DESCRIPTOR \
+	_IOW('v', IOCNUM_SET_SEGMENT_DESCRIPTOR, struct vm_seg_desc)
+#define	VM_GET_SEGMENT_DESCRIPTOR \
+	_IOWR('v', IOCNUM_GET_SEGMENT_DESCRIPTOR, struct vm_seg_desc)
+#define	VM_INJECT_EVENT	\
+	_IOW('v', IOCNUM_INJECT_EVENT, struct vm_event)
+#define	VM_LAPIC_IRQ 		\
+	_IOW('v', IOCNUM_LAPIC_IRQ, struct vm_lapic_irq)
+#define	VM_SET_CAPABILITY \
+	_IOW('v', IOCNUM_SET_CAPABILITY, struct vm_capability)
+#define	VM_GET_CAPABILITY \
+	_IOWR('v', IOCNUM_GET_CAPABILITY, struct vm_capability)
+#define	VM_BIND_PPTDEV \
+	_IOW('v', IOCNUM_BIND_PPTDEV, struct vm_pptdev)
+#define	VM_UNBIND_PPTDEV \
+	_IOW('v', IOCNUM_UNBIND_PPTDEV, struct vm_pptdev)
+#define	VM_MAP_PPTDEV_MMIO \
+	_IOW('v', IOCNUM_MAP_PPTDEV_MMIO, struct vm_pptdev_mmio)
+#define	VM_PPTDEV_MSI \
+	_IOW('v', IOCNUM_PPTDEV_MSI, struct vm_pptdev_msi)
+#define	VM_PPTDEV_MSIX \
+	_IOW('v', IOCNUM_PPTDEV_MSIX, struct vm_pptdev_msix)
+#define VM_INJECT_NMI \
+	_IOW('v', IOCNUM_INJECT_NMI, struct vm_nmi)
+#define	VM_STATS \
+	_IOWR('v', IOCNUM_VM_STATS, struct vm_stats)
+#define	VM_STAT_DESC \
+	_IOWR('v', IOCNUM_VM_STAT_DESC, struct vm_stat_desc)
+#define	VM_SET_X2APIC_STATE \
+	_IOW('v', IOCNUM_SET_X2APIC_STATE, struct vm_x2apic)
+#define	VM_GET_X2APIC_STATE \
+	_IOWR('v', IOCNUM_GET_X2APIC_STATE, struct vm_x2apic)
+#endif
diff --git a/sys/amd64/include/vmm_instruction_emul.h b/sys/amd64/include/vmm_instruction_emul.h
new file mode 100644
index 0000000..4c7a346
--- /dev/null
+++ b/sys/amd64/include/vmm_instruction_emul.h
@@ -0,0 +1,114 @@
+/*-
+ * Copyright (c) 2012 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_VMM_INSTRUCTION_EMUL_H_
+#define	_VMM_INSTRUCTION_EMUL_H_
+
+/*
+ * The data structures 'vie' and 'vie_op' are meant to be opaque to the
+ * consumers of instruction decoding. The only reason why their contents
+ * need to be exposed is because they are part of the 'vm_exit' structure.
+ */
+struct vie_op {
+	uint8_t		op_byte;	/* actual opcode byte */
+	uint8_t		op_type;	/* type of operation (e.g. MOV) */
+	uint16_t	op_flags;
+};
+
+#define	VIE_INST_SIZE	15
+struct vie {
+	uint8_t		inst[VIE_INST_SIZE];	/* instruction bytes */
+	uint8_t		num_valid;		/* size of the instruction */
+	uint8_t		num_processed;
+
+	uint8_t		rex_w:1,		/* REX prefix */
+			rex_r:1,
+			rex_x:1,
+			rex_b:1,
+			rex_present:1;
+
+	uint8_t		mod:2,			/* ModRM byte */
+			reg:4,
+			rm:4;
+
+	uint8_t		ss:2,			/* SIB byte */
+			index:4,
+			base:4;
+
+	uint8_t		disp_bytes;
+	uint8_t		imm_bytes;
+
+	uint8_t		scale;
+	int		base_register;		/* VM_REG_GUEST_xyz */
+	int		index_register;		/* VM_REG_GUEST_xyz */
+
+	int64_t		displacement;		/* optional addr displacement */
+	int64_t		immediate;		/* optional immediate operand */
+
+	uint8_t		decoded;	/* set to 1 if successfully decoded */
+
+	struct vie_op	op;			/* opcode description */
+};
+
+/*
+ * Callback functions to read and write memory regions.
+ */
+typedef int (*mem_region_read_t)(void *vm, int cpuid, uint64_t gpa,
+				 uint64_t *rval, int rsize, void *arg);
+
+typedef int (*mem_region_write_t)(void *vm, int cpuid, uint64_t gpa,
+				  uint64_t wval, int wsize, void *arg);
+
+/*
+ * Emulate the decoded 'vie' instruction.
+ *
+ * The callbacks 'mrr' and 'mrw' emulate reads and writes to the memory region
+ * containing 'gpa'. 'mrarg' is an opaque argument that is passed into the
+ * callback functions.
+ *
+ * 'void *vm' should be 'struct vm *' when called from kernel context and
+ * 'struct vmctx *' when called from user context.
+ * s
+ */
+int vmm_emulate_instruction(void *vm, int cpuid, uint64_t gpa, struct vie *vie,
+			    mem_region_read_t mrr, mem_region_write_t mrw,
+			    void *mrarg);
+
+#ifdef _KERNEL
+/*
+ * APIs to fetch and decode the instruction from nested page fault handler.
+ */
+int vmm_fetch_instruction(struct vm *vm, int cpuid,
+			  uint64_t rip, int inst_length, uint64_t cr3,
+			  struct vie *vie);
+
+int vmm_decode_instruction(struct vm *vm, int cpuid,
+			   uint64_t gla, struct vie *vie);
+#endif	/* _KERNEL */
+
+#endif	/* _VMM_INSTRUCTION_EMUL_H_ */
diff --git a/sys/amd64/linux32/linux.h b/sys/amd64/linux32/linux.h
index 2c269d3..7b52a64 100644
--- a/sys/amd64/linux32/linux.h
+++ b/sys/amd64/linux32/linux.h
@@ -107,11 +107,6 @@ typedef struct {
 /*
  * Miscellaneous
  */
-#define	LINUX_NAME_MAX		255
-#define	LINUX_MAX_UTSNAME	65
-
-#define	LINUX_CTL_MAXNAME	10
-
 #define	LINUX_AT_COUNT		16	/* Count of used aux entry types.
 					 * Keep this synchronized with
 					 * elf_linux_fixup() code.
@@ -127,11 +122,6 @@ struct l___sysctl_args
 	l_ulong		__spare[4];
 } __packed;
 
-/* Scheduling policies */
-#define	LINUX_SCHED_OTHER	0
-#define	LINUX_SCHED_FIFO	1
-#define	LINUX_SCHED_RR		2
-
 /* Resource limits */
 #define	LINUX_RLIMIT_CPU	0
 #define	LINUX_RLIMIT_FSIZE	1
@@ -265,15 +255,6 @@ struct l_statfs64 {
         l_int           f_spare[6];
 } __packed;
 
-struct l_new_utsname {
-	char	sysname[LINUX_MAX_UTSNAME];
-	char	nodename[LINUX_MAX_UTSNAME];
-	char	release[LINUX_MAX_UTSNAME];
-	char	version[LINUX_MAX_UTSNAME];
-	char	machine[LINUX_MAX_UTSNAME];
-	char	domainname[LINUX_MAX_UTSNAME];
-} __packed;
-
 /*
  * Signalling
  */
@@ -535,27 +516,9 @@ struct l_rt_sigframe {
 	l_handler_t 		sf_handler;
 } __packed;
 
-extern int bsd_to_linux_signal[];
-extern int linux_to_bsd_signal[];
 extern struct sysentvec elf_linux_sysvec;
 
 /*
- * Pluggable ioctl handlers
- */
-struct linux_ioctl_args;
-struct thread;
-
-typedef int linux_ioctl_function_t(struct thread *, struct linux_ioctl_args *);
-
-struct linux_ioctl_handler {
-	linux_ioctl_function_t *func;
-	int	low, high;
-};
-
-int	linux_ioctl_register_handler(struct linux_ioctl_handler *h);
-int	linux_ioctl_unregister_handler(struct linux_ioctl_handler *h);
-
-/*
  * open/fcntl flags
  */
 #define	LINUX_O_RDONLY		00000000
@@ -597,65 +560,6 @@ int	linux_ioctl_unregister_handler(struct linux_ioctl_handler *h);
 #define	LINUX_F_WRLCK		1
 #define	LINUX_F_UNLCK		2
 
-/*
- * posix_fadvise advice
- */
-#define	LINUX_POSIX_FADV_NORMAL		0
-#define	LINUX_POSIX_FADV_RANDOM		1
-#define	LINUX_POSIX_FADV_SEQUENTIAL    	2
-#define	LINUX_POSIX_FADV_WILLNEED      	3
-#define	LINUX_POSIX_FADV_DONTNEED      	4
-#define	LINUX_POSIX_FADV_NOREUSE       	5
-
-/*
- * mount flags
- */
-#define	LINUX_MS_RDONLY		0x0001
-#define	LINUX_MS_NOSUID		0x0002
-#define	LINUX_MS_NODEV		0x0004
-#define	LINUX_MS_NOEXEC		0x0008
-#define	LINUX_MS_REMOUNT	0x0020
-
-/*
- * SystemV IPC defines
- */
-#define	LINUX_SEMOP		1
-#define	LINUX_SEMGET		2
-#define	LINUX_SEMCTL		3
-#define	LINUX_MSGSND		11
-#define	LINUX_MSGRCV		12
-#define	LINUX_MSGGET		13
-#define	LINUX_MSGCTL		14
-#define	LINUX_SHMAT		21
-#define	LINUX_SHMDT		22
-#define	LINUX_SHMGET		23
-#define	LINUX_SHMCTL		24
-
-#define	LINUX_IPC_RMID		0
-#define	LINUX_IPC_SET		1
-#define	LINUX_IPC_STAT		2
-#define	LINUX_IPC_INFO		3
-
-#define	LINUX_SHM_LOCK		11
-#define	LINUX_SHM_UNLOCK	12
-#define	LINUX_SHM_STAT		13
-#define	LINUX_SHM_INFO		14
-
-#define	LINUX_SHM_RDONLY	0x1000
-#define	LINUX_SHM_RND		0x2000
-#define	LINUX_SHM_REMAP		0x4000
-
-/* semctl commands */
-#define	LINUX_GETPID		11
-#define	LINUX_GETVAL		12
-#define	LINUX_GETALL		13
-#define	LINUX_GETNCNT		14
-#define	LINUX_GETZCNT		15
-#define	LINUX_SETVAL		16
-#define	LINUX_SETALL		17
-#define	LINUX_SEM_STAT		18
-#define	LINUX_SEM_INFO		19
-
 union l_semun {
 	l_int		val;
 	l_uintptr_t	buf;
@@ -667,25 +571,6 @@ union l_semun {
 /*
  * Socket defines
  */
-#define	LINUX_SOCKET 		1
-#define	LINUX_BIND		2
-#define	LINUX_CONNECT 		3
-#define	LINUX_LISTEN 		4
-#define	LINUX_ACCEPT 		5
-#define	LINUX_GETSOCKNAME	6
-#define	LINUX_GETPEERNAME	7
-#define	LINUX_SOCKETPAIR	8
-#define	LINUX_SEND		9
-#define	LINUX_RECV		10
-#define	LINUX_SENDTO 		11
-#define	LINUX_RECVFROM 		12
-#define	LINUX_SHUTDOWN 		13
-#define	LINUX_SETSOCKOPT	14
-#define	LINUX_GETSOCKOPT	15
-#define	LINUX_SENDMSG		16
-#define	LINUX_RECVMSG		17
-#define	LINUX_ACCEPT4		18
-
 #define	LINUX_SOL_SOCKET	1
 #define	LINUX_SOL_IP		0
 #define	LINUX_SOL_IPX		256
@@ -714,17 +599,6 @@ union l_semun {
 #define	LINUX_SO_TIMESTAMP	29
 #define	LINUX_SO_ACCEPTCONN	30
 
-#define	LINUX_IP_TOS		1
-#define	LINUX_IP_TTL		2
-#define	LINUX_IP_HDRINCL	3
-#define	LINUX_IP_OPTIONS	4
-
-#define	LINUX_IP_MULTICAST_IF		32
-#define	LINUX_IP_MULTICAST_TTL		33
-#define	LINUX_IP_MULTICAST_LOOP		34
-#define	LINUX_IP_ADD_MEMBERSHIP		35
-#define	LINUX_IP_DROP_MEMBERSHIP	36
-
 struct l_sockaddr {
 	l_ushort	sa_family;
 	char		sa_data[14];
@@ -890,30 +764,6 @@ struct l_user_desc {
 #define	LINUX_GET_USEABLE(desc)		\
 	(((desc)->b >> LINUX_ENTRY_B_USEABLE) & 1)
 
-#define	LINUX_CLOCK_REALTIME		0
-#define	LINUX_CLOCK_MONOTONIC		1
-#define	LINUX_CLOCK_PROCESS_CPUTIME_ID	2
-#define	LINUX_CLOCK_THREAD_CPUTIME_ID	3
-#define	LINUX_CLOCK_REALTIME_HR		4
-#define	LINUX_CLOCK_MONOTONIC_HR	5
-
-#define	LINUX_CLONE_VM			0x00000100
-#define	LINUX_CLONE_FS			0x00000200
-#define	LINUX_CLONE_FILES		0x00000400
-#define	LINUX_CLONE_SIGHAND		0x00000800
-#define	LINUX_CLONE_PID			0x00001000	/* No longer exist in Linux */
-#define	LINUX_CLONE_VFORK		0x00004000
-#define	LINUX_CLONE_PARENT		0x00008000
-#define	LINUX_CLONE_THREAD		0x00010000
-#define	LINUX_CLONE_SETTLS		0x00080000
-#define	LINUX_CLONE_PARENT_SETTID	0x00100000
-#define	LINUX_CLONE_CHILD_CLEARTID	0x00200000
-#define	LINUX_CLONE_CHILD_SETTID	0x01000000
-
-#define	LINUX_THREADING_FLAGS					\
-	(LINUX_CLONE_VM | LINUX_CLONE_FS | LINUX_CLONE_FILES |	\
-	LINUX_CLONE_SIGHAND | LINUX_CLONE_THREAD)
-
 struct iovec;
 
 struct l_iovec32 {
@@ -935,7 +785,4 @@ struct linux_robust_list_head {
 	l_uintptr_t			pending_list;
 };
 
-int linux_set_upcall_kse(struct thread *td, register_t stack);
-int linux_set_cloned_tls(struct thread *td, void *desc);
-
 #endif /* !_AMD64_LINUX_H_ */
diff --git a/sys/amd64/linux32/linux32_sysvec.c b/sys/amd64/linux32/linux32_sysvec.c
index 5afc9ce..42500da 100644
--- a/sys/amd64/linux32/linux32_sysvec.c
+++ b/sys/amd64/linux32/linux32_sysvec.c
@@ -78,6 +78,7 @@ __FBSDID("$FreeBSD$");
 #include <amd64/linux32/linux32_proto.h>
 #include <compat/linux/linux_emul.h>
 #include <compat/linux/linux_futex.h>
+#include <compat/linux/linux_ioctl.h>
 #include <compat/linux/linux_mib.h>
 #include <compat/linux/linux_misc.h>
 #include <compat/linux/linux_signal.h>
diff --git a/sys/amd64/vmm/amd/amdv.c b/sys/amd64/vmm/amd/amdv.c
new file mode 100644
index 0000000..dc071d3
--- /dev/null
+++ b/sys/amd64/vmm/amd/amdv.c
@@ -0,0 +1,265 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/errno.h>
+#include <sys/smp.h>
+
+#include <machine/vmm.h>
+#include "io/iommu.h"
+
+static int
+amdv_init(void)
+{
+
+	printf("amdv_init: not implemented\n");
+	return (ENXIO);
+}
+
+static int
+amdv_cleanup(void)
+{
+
+	printf("amdv_cleanup: not implemented\n");
+	return (ENXIO);
+}
+
+static void *
+amdv_vminit(struct vm *vm)
+{
+
+	printf("amdv_vminit: not implemented\n");
+	return (NULL);
+}
+
+static int
+amdv_vmrun(void *arg, int vcpu, register_t rip)
+{
+
+	printf("amdv_vmrun: not implemented\n");
+	return (ENXIO);
+}
+
+static void
+amdv_vmcleanup(void *arg)
+{
+
+	printf("amdv_vmcleanup: not implemented\n");
+	return;
+}
+
+static int
+amdv_vmmmap_set(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, size_t length,
+	    vm_memattr_t attr, int prot, boolean_t spok)
+{
+
+	printf("amdv_vmmmap_set: not implemented\n");
+	return (EINVAL);
+}
+
+static vm_paddr_t
+amdv_vmmmap_get(void *arg, vm_paddr_t gpa)
+{
+
+	printf("amdv_vmmmap_get: not implemented\n");
+	return (EINVAL);
+}
+
+static int
+amdv_getreg(void *arg, int vcpu, int regnum, uint64_t *retval)
+{
+	
+	printf("amdv_getreg: not implemented\n");
+	return (EINVAL);
+}
+
+static int
+amdv_setreg(void *arg, int vcpu, int regnum, uint64_t val)
+{
+	
+	printf("amdv_setreg: not implemented\n");
+	return (EINVAL);
+}
+
+static int
+amdv_getdesc(void *vmi, int vcpu, int num, struct seg_desc *desc)
+{
+
+	printf("amdv_get_desc: not implemented\n");
+	return (EINVAL);
+}
+
+static int
+amdv_setdesc(void *vmi, int vcpu, int num, struct seg_desc *desc)
+{
+
+	printf("amdv_get_desc: not implemented\n");
+	return (EINVAL);
+}
+
+static int
+amdv_inject_event(void *vmi, int vcpu, int type, int vector,
+		  uint32_t error_code, int error_code_valid)
+{
+
+	printf("amdv_inject_event: not implemented\n");
+	return (EINVAL);
+}
+
+static int
+amdv_getcap(void *arg, int vcpu, int type, int *retval)
+{
+
+	printf("amdv_getcap: not implemented\n");
+	return (EINVAL);
+}
+
+static int
+amdv_setcap(void *arg, int vcpu, int type, int val)
+{
+
+	printf("amdv_setcap: not implemented\n");
+	return (EINVAL);
+}
+
+struct vmm_ops vmm_ops_amd = {
+	amdv_init,
+	amdv_cleanup,
+	amdv_vminit,
+	amdv_vmrun,
+	amdv_vmcleanup,
+	amdv_vmmmap_set,
+	amdv_vmmmap_get,
+	amdv_getreg,
+	amdv_setreg,
+	amdv_getdesc,
+	amdv_setdesc,
+	amdv_inject_event,
+	amdv_getcap,
+	amdv_setcap
+};
+
+static int
+amd_iommu_init(void)
+{
+
+	printf("amd_iommu_init: not implemented\n");
+	return (ENXIO);
+}
+
+static void
+amd_iommu_cleanup(void)
+{
+
+	printf("amd_iommu_cleanup: not implemented\n");
+}
+
+static void
+amd_iommu_enable(void)
+{
+
+	printf("amd_iommu_enable: not implemented\n");
+}
+
+static void
+amd_iommu_disable(void)
+{
+
+	printf("amd_iommu_disable: not implemented\n");
+}
+
+static void *
+amd_iommu_create_domain(vm_paddr_t maxaddr)
+{
+
+	printf("amd_iommu_create_domain: not implemented\n");
+	return (NULL);
+}
+
+static void
+amd_iommu_destroy_domain(void *domain)
+{
+
+	printf("amd_iommu_destroy_domain: not implemented\n");
+}
+
+static uint64_t
+amd_iommu_create_mapping(void *domain, vm_paddr_t gpa, vm_paddr_t hpa,
+			 uint64_t len)
+{
+
+	printf("amd_iommu_create_mapping: not implemented\n");
+	return (0);
+}
+
+static uint64_t
+amd_iommu_remove_mapping(void *domain, vm_paddr_t gpa, uint64_t len)
+{
+
+	printf("amd_iommu_remove_mapping: not implemented\n");
+	return (0);
+}
+
+static void
+amd_iommu_add_device(void *domain, int bus, int slot, int func)
+{
+
+	printf("amd_iommu_add_device: not implemented\n");
+}
+
+static void
+amd_iommu_remove_device(void *domain, int bus, int slot, int func)
+{
+
+	printf("amd_iommu_remove_device: not implemented\n");
+}
+
+static void
+amd_iommu_invalidate_tlb(void *domain)
+{
+
+	printf("amd_iommu_invalidate_tlb: not implemented\n");
+}
+
+struct iommu_ops iommu_ops_amd = {
+	amd_iommu_init,
+	amd_iommu_cleanup,
+	amd_iommu_enable,
+	amd_iommu_disable,
+	amd_iommu_create_domain,
+	amd_iommu_destroy_domain,
+	amd_iommu_create_mapping,
+	amd_iommu_remove_mapping,
+	amd_iommu_add_device,
+	amd_iommu_remove_device,
+	amd_iommu_invalidate_tlb,
+};
diff --git a/sys/amd64/vmm/intel/ept.c b/sys/amd64/vmm/intel/ept.c
new file mode 100644
index 0000000..4f91601
--- /dev/null
+++ b/sys/amd64/vmm/intel/ept.c
@@ -0,0 +1,392 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/errno.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/smp.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/param.h>
+#include <machine/cpufunc.h>
+#include <machine/pmap.h>
+#include <machine/vmparam.h>
+
+#include <machine/vmm.h>
+#include "vmx_cpufunc.h"
+#include "vmx_msr.h"
+#include "vmx.h"
+#include "ept.h"
+
+#define	EPT_PWL4(cap)			((cap) & (1UL << 6))
+#define	EPT_MEMORY_TYPE_WB(cap)		((cap) & (1UL << 14))
+#define	EPT_PDE_SUPERPAGE(cap)		((cap) & (1UL << 16))	/* 2MB pages */
+#define	EPT_PDPTE_SUPERPAGE(cap)	((cap) & (1UL << 17))	/* 1GB pages */
+#define	INVVPID_SUPPORTED(cap)		((cap) & (1UL << 32))
+#define	INVEPT_SUPPORTED(cap)		((cap) & (1UL << 20))
+
+#define	INVVPID_ALL_TYPES_MASK		0xF0000000000UL
+#define	INVVPID_ALL_TYPES_SUPPORTED(cap)	\
+	(((cap) & INVVPID_ALL_TYPES_MASK) == INVVPID_ALL_TYPES_MASK)
+
+#define	INVEPT_ALL_TYPES_MASK		0x6000000UL
+#define	INVEPT_ALL_TYPES_SUPPORTED(cap)		\
+	(((cap) & INVEPT_ALL_TYPES_MASK) == INVEPT_ALL_TYPES_MASK)
+
+#define	EPT_PG_RD			(1 << 0)
+#define	EPT_PG_WR			(1 << 1)
+#define	EPT_PG_EX			(1 << 2)
+#define	EPT_PG_MEMORY_TYPE(x)		((x) << 3)
+#define	EPT_PG_IGNORE_PAT		(1 << 6)
+#define	EPT_PG_SUPERPAGE		(1 << 7)
+
+#define	EPT_ADDR_MASK			((uint64_t)-1 << 12)
+
+MALLOC_DECLARE(M_VMX);
+
+static uint64_t page_sizes_mask;
+
+int
+ept_init(void)
+{
+	int page_shift;
+	uint64_t cap;
+
+	cap = rdmsr(MSR_VMX_EPT_VPID_CAP);
+
+	/*
+	 * Verify that:
+	 * - page walk length is 4 steps
+	 * - extended page tables can be laid out in write-back memory
+	 * - invvpid instruction with all possible types is supported
+	 * - invept instruction with all possible types is supported
+	 */
+	if (!EPT_PWL4(cap) ||
+	    !EPT_MEMORY_TYPE_WB(cap) ||
+	    !INVVPID_SUPPORTED(cap) ||
+	    !INVVPID_ALL_TYPES_SUPPORTED(cap) ||
+	    !INVEPT_SUPPORTED(cap) ||
+	    !INVEPT_ALL_TYPES_SUPPORTED(cap))
+		return (EINVAL);
+
+	/* Set bits in 'page_sizes_mask' for each valid page size */
+	page_shift = PAGE_SHIFT;
+	page_sizes_mask = 1UL << page_shift;		/* 4KB page */
+
+	page_shift += 9;
+	if (EPT_PDE_SUPERPAGE(cap))
+		page_sizes_mask |= 1UL << page_shift;	/* 2MB superpage */
+
+	page_shift += 9;
+	if (EPT_PDPTE_SUPERPAGE(cap))
+		page_sizes_mask |= 1UL << page_shift;	/* 1GB superpage */
+
+	return (0);
+}
+
+#if 0
+static void
+ept_dump(uint64_t *ptp, int nlevels)
+{
+	int i, t, tabs;
+	uint64_t *ptpnext, ptpval;
+
+	if (--nlevels < 0)
+		return;
+
+	tabs = 3 - nlevels;
+	for (t = 0; t < tabs; t++)
+		printf("\t");
+	printf("PTP = %p\n", ptp);
+
+	for (i = 0; i < 512; i++) {
+		ptpval = ptp[i];
+
+		if (ptpval == 0)
+			continue;
+		
+		for (t = 0; t < tabs; t++)
+			printf("\t");
+		printf("%3d 0x%016lx\n", i, ptpval);
+
+		if (nlevels != 0 && (ptpval & EPT_PG_SUPERPAGE) == 0) {
+			ptpnext = (uint64_t *)
+				  PHYS_TO_DMAP(ptpval & EPT_ADDR_MASK);
+			ept_dump(ptpnext, nlevels);
+		}
+	}
+}
+#endif
+
+static size_t
+ept_create_mapping(uint64_t *ptp, vm_paddr_t gpa, vm_paddr_t hpa, size_t length,
+		   vm_memattr_t attr, vm_prot_t prot, boolean_t spok)
+{
+	int spshift, ptpshift, ptpindex, nlevels;
+
+	/*
+	 * Compute the size of the mapping that we can accomodate.
+	 *
+	 * This is based on three factors:
+	 * - super page sizes supported by the processor
+	 * - alignment of the region starting at 'gpa' and 'hpa'
+	 * - length of the region 'len'
+	 */
+	spshift = PAGE_SHIFT;
+	if (spok)
+		spshift += (EPT_PWLEVELS - 1) * 9;
+	while (spshift >= PAGE_SHIFT) {
+		uint64_t spsize = 1UL << spshift;
+		if ((page_sizes_mask & spsize) != 0 &&
+		    (gpa & (spsize - 1)) == 0 &&
+		    (hpa & (spsize - 1)) == 0 &&
+		    length >= spsize) {
+			break;
+		}
+		spshift -= 9;
+	}
+
+	if (spshift < PAGE_SHIFT) {
+		panic("Invalid spshift for gpa 0x%016lx, hpa 0x%016lx, "
+		      "length 0x%016lx, page_sizes_mask 0x%016lx",
+		      gpa, hpa, length, page_sizes_mask);
+	}
+
+	nlevels = EPT_PWLEVELS;
+	while (--nlevels >= 0) {
+		ptpshift = PAGE_SHIFT + nlevels * 9;
+		ptpindex = (gpa >> ptpshift) & 0x1FF;
+
+		/* We have reached the leaf mapping */
+		if (spshift >= ptpshift)
+			break;
+
+		/*
+		 * We are working on a non-leaf page table page.
+		 *
+		 * Create the next level page table page if necessary and point
+		 * to it from the current page table.
+		 */
+		if (ptp[ptpindex] == 0) {
+			void *nlp = malloc(PAGE_SIZE, M_VMX, M_WAITOK | M_ZERO);
+			ptp[ptpindex] = vtophys(nlp);
+			ptp[ptpindex] |= EPT_PG_RD | EPT_PG_WR | EPT_PG_EX;
+		}
+
+		/* Work our way down to the next level page table page */
+		ptp = (uint64_t *)PHYS_TO_DMAP(ptp[ptpindex] & EPT_ADDR_MASK);
+	}
+
+	if ((gpa & ((1UL << ptpshift) - 1)) != 0) {
+		panic("ept_create_mapping: gpa 0x%016lx and ptpshift %d "
+		      "mismatch\n", gpa, ptpshift);
+	}
+
+	if (prot != VM_PROT_NONE) {
+		/* Do the mapping */
+		ptp[ptpindex] = hpa;
+
+		/* Apply the access controls */
+		if (prot & VM_PROT_READ)
+			ptp[ptpindex] |= EPT_PG_RD;
+		if (prot & VM_PROT_WRITE)
+			ptp[ptpindex] |= EPT_PG_WR;
+		if (prot & VM_PROT_EXECUTE)
+			ptp[ptpindex] |= EPT_PG_EX;
+
+		/*
+		 * XXX should we enforce this memory type by setting the
+		 * ignore PAT bit to 1.
+		 */
+		ptp[ptpindex] |= EPT_PG_MEMORY_TYPE(attr);
+
+		if (nlevels > 0)
+			ptp[ptpindex] |= EPT_PG_SUPERPAGE;
+	} else {
+		/* Remove the mapping */
+		ptp[ptpindex] = 0;
+	}
+
+	return (1UL << ptpshift);
+}
+
+static vm_paddr_t
+ept_lookup_mapping(uint64_t *ptp, vm_paddr_t gpa)
+{
+	int nlevels, ptpshift, ptpindex;
+	uint64_t ptpval, hpabase, pgmask;
+
+	nlevels = EPT_PWLEVELS;
+	while (--nlevels >= 0) {
+		ptpshift = PAGE_SHIFT + nlevels * 9;
+		ptpindex = (gpa >> ptpshift) & 0x1FF;
+
+		ptpval = ptp[ptpindex];
+
+		/* Cannot make progress beyond this point */
+		if ((ptpval & (EPT_PG_RD | EPT_PG_WR | EPT_PG_EX)) == 0)
+			break;
+
+		if (nlevels == 0 || (ptpval & EPT_PG_SUPERPAGE)) {
+			pgmask = (1UL << ptpshift) - 1;
+			hpabase = ptpval & ~pgmask;
+			return (hpabase | (gpa & pgmask));
+		}
+
+		/* Work our way down to the next level page table page */
+		ptp = (uint64_t *)PHYS_TO_DMAP(ptpval & EPT_ADDR_MASK);
+	}
+
+	return ((vm_paddr_t)-1);
+}
+
+static void
+ept_free_pt_entry(pt_entry_t pte)
+{
+	if (pte == 0)
+		return;
+
+	/* sanity check */
+	if ((pte & EPT_PG_SUPERPAGE) != 0)
+		panic("ept_free_pt_entry: pte cannot have superpage bit");
+
+	return;
+}
+
+static void
+ept_free_pd_entry(pd_entry_t pde)
+{
+	pt_entry_t	*pt;
+	int		i;
+
+	if (pde == 0)
+		return;
+
+	if ((pde & EPT_PG_SUPERPAGE) == 0) {
+		pt = (pt_entry_t *)PHYS_TO_DMAP(pde & EPT_ADDR_MASK);
+		for (i = 0; i < NPTEPG; i++)
+			ept_free_pt_entry(pt[i]);
+		free(pt, M_VMX);	/* free the page table page */
+	}
+}
+
+static void
+ept_free_pdp_entry(pdp_entry_t pdpe)
+{
+	pd_entry_t 	*pd;
+	int		 i;
+
+	if (pdpe == 0)
+		return;
+
+	if ((pdpe & EPT_PG_SUPERPAGE) == 0) {
+		pd = (pd_entry_t *)PHYS_TO_DMAP(pdpe & EPT_ADDR_MASK);
+		for (i = 0; i < NPDEPG; i++)
+			ept_free_pd_entry(pd[i]);
+		free(pd, M_VMX);	/* free the page directory page */
+	}
+}
+
+static void
+ept_free_pml4_entry(pml4_entry_t pml4e)
+{
+	pdp_entry_t	*pdp;
+	int		i;
+
+	if (pml4e == 0)
+		return;
+
+	if ((pml4e & EPT_PG_SUPERPAGE) == 0) {
+		pdp = (pdp_entry_t *)PHYS_TO_DMAP(pml4e & EPT_ADDR_MASK);
+		for (i = 0; i < NPDPEPG; i++)
+			ept_free_pdp_entry(pdp[i]);
+		free(pdp, M_VMX);	/* free the page directory ptr page */
+	}
+}
+
+void
+ept_vmcleanup(struct vmx *vmx)
+{
+	int 		 i;
+
+	for (i = 0; i < NPML4EPG; i++)
+		ept_free_pml4_entry(vmx->pml4ept[i]);
+}
+
+int
+ept_vmmmap_set(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, size_t len,
+		vm_memattr_t attr, int prot, boolean_t spok)
+{
+	size_t n;
+	struct vmx *vmx = arg;
+
+	while (len > 0) {
+		n = ept_create_mapping(vmx->pml4ept, gpa, hpa, len, attr,
+				       prot, spok);
+		len -= n;
+		gpa += n;
+		hpa += n;
+	}
+
+	return (0);
+}
+
+vm_paddr_t
+ept_vmmmap_get(void *arg, vm_paddr_t gpa)
+{
+	vm_paddr_t hpa;
+	struct vmx *vmx;
+
+	vmx = arg;
+	hpa = ept_lookup_mapping(vmx->pml4ept, gpa);
+	return (hpa);
+}
+
+static void
+invept_single_context(void *arg)
+{
+	struct invept_desc desc = *(struct invept_desc *)arg;
+
+	invept(INVEPT_TYPE_SINGLE_CONTEXT, desc);
+}
+
+void
+ept_invalidate_mappings(u_long pml4ept)
+{
+	struct invept_desc invept_desc = { 0 };
+
+	invept_desc.eptp = EPTP(pml4ept);
+
+	smp_rendezvous(NULL, invept_single_context, NULL, &invept_desc);
+}
diff --git a/sys/amd64/vmm/intel/ept.h b/sys/amd64/vmm/intel/ept.h
new file mode 100644
index 0000000..2d7258d
--- /dev/null
+++ b/sys/amd64/vmm/intel/ept.h
@@ -0,0 +1,43 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_EPT_H_
+#define	_EPT_H_
+
+struct vmx;
+
+#define	EPT_PWLEVELS	4		/* page walk levels */
+#define	EPTP(pml4)	((pml4) | (EPT_PWLEVELS - 1) << 3 | PAT_WRITE_BACK)
+
+int	ept_init(void);
+int	ept_vmmmap_set(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, size_t length,
+	    vm_memattr_t attr, int prot, boolean_t allow_superpage_mappings);
+vm_paddr_t ept_vmmmap_get(void *arg, vm_paddr_t gpa);
+void	ept_invalidate_mappings(u_long ept_pml4);
+void	ept_vmcleanup(struct vmx *vmx);
+#endif
diff --git a/sys/amd64/vmm/intel/vmcs.c b/sys/amd64/vmm/intel/vmcs.c
new file mode 100644
index 0000000..a5784dd
--- /dev/null
+++ b/sys/amd64/vmm/intel/vmcs.c
@@ -0,0 +1,551 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "opt_ddb.h"
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/pcpu.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/segments.h>
+#include <machine/pmap.h>
+
+#include <machine/vmm.h>
+#include "vmm_host.h"
+#include "vmcs.h"
+#include "vmx_cpufunc.h"
+#include "ept.h"
+#include "vmx.h"
+
+#ifdef DDB
+#include <ddb/ddb.h>
+#endif
+
+static uint64_t
+vmcs_fix_regval(uint32_t encoding, uint64_t val)
+{
+
+	switch (encoding) {
+	case VMCS_GUEST_CR0:
+		val = vmx_fix_cr0(val);
+		break;
+	case VMCS_GUEST_CR4:
+		val = vmx_fix_cr4(val);
+		break;
+	default:
+		break;
+	}
+	return (val);
+}
+
+static uint32_t
+vmcs_field_encoding(int ident)
+{
+	switch (ident) {
+	case VM_REG_GUEST_CR0:
+		return (VMCS_GUEST_CR0);
+	case VM_REG_GUEST_CR3:
+		return (VMCS_GUEST_CR3);
+	case VM_REG_GUEST_CR4:
+		return (VMCS_GUEST_CR4);
+	case VM_REG_GUEST_DR7:
+		return (VMCS_GUEST_DR7);
+	case VM_REG_GUEST_RSP:
+		return (VMCS_GUEST_RSP);
+	case VM_REG_GUEST_RIP:
+		return (VMCS_GUEST_RIP);
+	case VM_REG_GUEST_RFLAGS:
+		return (VMCS_GUEST_RFLAGS);
+	case VM_REG_GUEST_ES:
+		return (VMCS_GUEST_ES_SELECTOR);
+	case VM_REG_GUEST_CS:
+		return (VMCS_GUEST_CS_SELECTOR);
+	case VM_REG_GUEST_SS:
+		return (VMCS_GUEST_SS_SELECTOR);
+	case VM_REG_GUEST_DS:
+		return (VMCS_GUEST_DS_SELECTOR);
+	case VM_REG_GUEST_FS:
+		return (VMCS_GUEST_FS_SELECTOR);
+	case VM_REG_GUEST_GS:
+		return (VMCS_GUEST_GS_SELECTOR);
+	case VM_REG_GUEST_TR:
+		return (VMCS_GUEST_TR_SELECTOR);
+	case VM_REG_GUEST_LDTR:
+		return (VMCS_GUEST_LDTR_SELECTOR);
+	case VM_REG_GUEST_EFER:
+		return (VMCS_GUEST_IA32_EFER);
+	default:
+		return (-1);
+	}
+
+}
+
+static int
+vmcs_seg_desc_encoding(int seg, uint32_t *base, uint32_t *lim, uint32_t *acc)
+{
+
+	switch (seg) {
+	case VM_REG_GUEST_ES:
+		*base = VMCS_GUEST_ES_BASE;
+		*lim = VMCS_GUEST_ES_LIMIT;
+		*acc = VMCS_GUEST_ES_ACCESS_RIGHTS;
+		break;
+	case VM_REG_GUEST_CS:
+		*base = VMCS_GUEST_CS_BASE;
+		*lim = VMCS_GUEST_CS_LIMIT;
+		*acc = VMCS_GUEST_CS_ACCESS_RIGHTS;
+		break;
+	case VM_REG_GUEST_SS:
+		*base = VMCS_GUEST_SS_BASE;
+		*lim = VMCS_GUEST_SS_LIMIT;
+		*acc = VMCS_GUEST_SS_ACCESS_RIGHTS;
+		break;
+	case VM_REG_GUEST_DS:
+		*base = VMCS_GUEST_DS_BASE;
+		*lim = VMCS_GUEST_DS_LIMIT;
+		*acc = VMCS_GUEST_DS_ACCESS_RIGHTS;
+		break;
+	case VM_REG_GUEST_FS:
+		*base = VMCS_GUEST_FS_BASE;
+		*lim = VMCS_GUEST_FS_LIMIT;
+		*acc = VMCS_GUEST_FS_ACCESS_RIGHTS;
+		break;
+	case VM_REG_GUEST_GS:
+		*base = VMCS_GUEST_GS_BASE;
+		*lim = VMCS_GUEST_GS_LIMIT;
+		*acc = VMCS_GUEST_GS_ACCESS_RIGHTS;
+		break;
+	case VM_REG_GUEST_TR:
+		*base = VMCS_GUEST_TR_BASE;
+		*lim = VMCS_GUEST_TR_LIMIT;
+		*acc = VMCS_GUEST_TR_ACCESS_RIGHTS;
+		break;
+	case VM_REG_GUEST_LDTR:
+		*base = VMCS_GUEST_LDTR_BASE;
+		*lim = VMCS_GUEST_LDTR_LIMIT;
+		*acc = VMCS_GUEST_LDTR_ACCESS_RIGHTS;
+		break;
+	case VM_REG_GUEST_IDTR:
+		*base = VMCS_GUEST_IDTR_BASE;
+		*lim = VMCS_GUEST_IDTR_LIMIT;
+		*acc = VMCS_INVALID_ENCODING;
+		break;
+	case VM_REG_GUEST_GDTR:
+		*base = VMCS_GUEST_GDTR_BASE;
+		*lim = VMCS_GUEST_GDTR_LIMIT;
+		*acc = VMCS_INVALID_ENCODING;
+		break;
+	default:
+		return (EINVAL);
+	}
+
+	return (0);
+}
+
+int
+vmcs_getreg(struct vmcs *vmcs, int ident, uint64_t *retval)
+{
+	int error;
+	uint32_t encoding;
+
+	/*
+	 * If we need to get at vmx-specific state in the VMCS we can bypass
+	 * the translation of 'ident' to 'encoding' by simply setting the
+	 * sign bit. As it so happens the upper 16 bits are reserved (i.e
+	 * set to 0) in the encodings for the VMCS so we are free to use the
+	 * sign bit.
+	 */
+	if (ident < 0)
+		encoding = ident & 0x7fffffff;
+	else
+		encoding = vmcs_field_encoding(ident);
+
+	if (encoding == (uint32_t)-1)
+		return (EINVAL);
+
+	VMPTRLD(vmcs);
+	error = vmread(encoding, retval);
+	VMCLEAR(vmcs);
+	return (error);
+}
+
+int
+vmcs_setreg(struct vmcs *vmcs, int ident, uint64_t val)
+{
+	int error;
+	uint32_t encoding;
+
+	if (ident < 0)
+		encoding = ident & 0x7fffffff;
+	else
+		encoding = vmcs_field_encoding(ident);
+
+	if (encoding == (uint32_t)-1)
+		return (EINVAL);
+
+	val = vmcs_fix_regval(encoding, val);
+
+	VMPTRLD(vmcs);
+	error = vmwrite(encoding, val);
+	VMCLEAR(vmcs);
+	return (error);
+}
+
+int
+vmcs_setdesc(struct vmcs *vmcs, int seg, struct seg_desc *desc)
+{
+	int error;
+	uint32_t base, limit, access;
+
+	error = vmcs_seg_desc_encoding(seg, &base, &limit, &access);
+	if (error != 0)
+		panic("vmcs_setdesc: invalid segment register %d", seg);
+
+	VMPTRLD(vmcs);
+	if ((error = vmwrite(base, desc->base)) != 0)
+		goto done;
+
+	if ((error = vmwrite(limit, desc->limit)) != 0)
+		goto done;
+
+	if (access != VMCS_INVALID_ENCODING) {
+		if ((error = vmwrite(access, desc->access)) != 0)
+			goto done;
+	}
+done:
+	VMCLEAR(vmcs);
+	return (error);
+}
+
+int
+vmcs_getdesc(struct vmcs *vmcs, int seg, struct seg_desc *desc)
+{
+	int error;
+	uint32_t base, limit, access;
+	uint64_t u64;
+
+	error = vmcs_seg_desc_encoding(seg, &base, &limit, &access);
+	if (error != 0)
+		panic("vmcs_getdesc: invalid segment register %d", seg);
+
+	VMPTRLD(vmcs);
+	if ((error = vmread(base, &u64)) != 0)
+		goto done;
+	desc->base = u64;
+
+	if ((error = vmread(limit, &u64)) != 0)
+		goto done;
+	desc->limit = u64;
+
+	if (access != VMCS_INVALID_ENCODING) {
+		if ((error = vmread(access, &u64)) != 0)
+			goto done;
+		desc->access = u64;
+	}
+done:
+	VMCLEAR(vmcs);
+	return (error);
+}
+
+int
+vmcs_set_msr_save(struct vmcs *vmcs, u_long g_area, u_int g_count)
+{
+	int error;
+
+	VMPTRLD(vmcs);
+
+	/*
+	 * Guest MSRs are saved in the VM-exit MSR-store area.
+	 * Guest MSRs are loaded from the VM-entry MSR-load area.
+	 * Both areas point to the same location in memory.
+	 */
+	if ((error = vmwrite(VMCS_EXIT_MSR_STORE, g_area)) != 0)
+		goto done;
+	if ((error = vmwrite(VMCS_EXIT_MSR_STORE_COUNT, g_count)) != 0)
+		goto done;
+
+	if ((error = vmwrite(VMCS_ENTRY_MSR_LOAD, g_area)) != 0)
+		goto done;
+	if ((error = vmwrite(VMCS_ENTRY_MSR_LOAD_COUNT, g_count)) != 0)
+		goto done;
+
+	error = 0;
+done:
+	VMCLEAR(vmcs);
+	return (error);
+}
+
+int
+vmcs_set_defaults(struct vmcs *vmcs,
+		  u_long host_rip, u_long host_rsp, u_long ept_pml4,
+		  uint32_t pinbased_ctls, uint32_t procbased_ctls,
+		  uint32_t procbased_ctls2, uint32_t exit_ctls,
+		  uint32_t entry_ctls, u_long msr_bitmap, uint16_t vpid)
+{
+	int error, codesel, datasel, tsssel;
+	u_long cr0, cr4, efer;
+	uint64_t eptp, pat, fsbase, idtrbase;
+	uint32_t exc_bitmap;
+
+	codesel = vmm_get_host_codesel();
+	datasel = vmm_get_host_datasel();
+	tsssel = vmm_get_host_tsssel();
+
+	/*
+	 * Make sure we have a "current" VMCS to work with.
+	 */
+	VMPTRLD(vmcs);
+
+	/*
+	 * Load the VMX controls
+	 */
+	if ((error = vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls)) != 0)
+		goto done;
+	if ((error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls)) != 0)
+		goto done;
+	if ((error = vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2)) != 0)
+		goto done;
+	if ((error = vmwrite(VMCS_EXIT_CTLS, exit_ctls)) != 0)
+		goto done;
+	if ((error = vmwrite(VMCS_ENTRY_CTLS, entry_ctls)) != 0)
+		goto done;
+
+	/* Guest state */
+
+	/* Initialize guest IA32_PAT MSR with the default value */
+	pat = PAT_VALUE(0, PAT_WRITE_BACK)	|
+	      PAT_VALUE(1, PAT_WRITE_THROUGH)	|
+	      PAT_VALUE(2, PAT_UNCACHED)	|
+	      PAT_VALUE(3, PAT_UNCACHEABLE)	|
+	      PAT_VALUE(4, PAT_WRITE_BACK)	|
+	      PAT_VALUE(5, PAT_WRITE_THROUGH)	|
+	      PAT_VALUE(6, PAT_UNCACHED)	|
+	      PAT_VALUE(7, PAT_UNCACHEABLE);
+	if ((error = vmwrite(VMCS_GUEST_IA32_PAT, pat)) != 0)
+		goto done;
+
+	/* Host state */
+
+	/* Initialize host IA32_PAT MSR */
+	pat = vmm_get_host_pat();
+	if ((error = vmwrite(VMCS_HOST_IA32_PAT, pat)) != 0)
+		goto done;
+
+	/* Load the IA32_EFER MSR */
+	efer = vmm_get_host_efer();
+	if ((error = vmwrite(VMCS_HOST_IA32_EFER, efer)) != 0)
+		goto done;
+
+	/* Load the control registers */
+
+	cr0 = vmm_get_host_cr0();
+	if ((error = vmwrite(VMCS_HOST_CR0, cr0)) != 0)
+		goto done;
+	
+	cr4 = vmm_get_host_cr4() | CR4_VMXE;
+	if ((error = vmwrite(VMCS_HOST_CR4, cr4)) != 0)
+		goto done;
+
+	/* Load the segment selectors */
+	if ((error = vmwrite(VMCS_HOST_ES_SELECTOR, datasel)) != 0)
+		goto done;
+
+	if ((error = vmwrite(VMCS_HOST_CS_SELECTOR, codesel)) != 0)
+		goto done;
+
+	if ((error = vmwrite(VMCS_HOST_SS_SELECTOR, datasel)) != 0)
+		goto done;
+
+	if ((error = vmwrite(VMCS_HOST_DS_SELECTOR, datasel)) != 0)
+		goto done;
+
+	if ((error = vmwrite(VMCS_HOST_FS_SELECTOR, datasel)) != 0)
+		goto done;
+
+	if ((error = vmwrite(VMCS_HOST_GS_SELECTOR, datasel)) != 0)
+		goto done;
+
+	if ((error = vmwrite(VMCS_HOST_TR_SELECTOR, tsssel)) != 0)
+		goto done;
+
+	/*
+	 * Load the Base-Address for %fs and idtr.
+	 *
+	 * Note that we exclude %gs, tss and gdtr here because their base
+	 * address is pcpu specific.
+	 */
+	fsbase = vmm_get_host_fsbase();
+	if ((error = vmwrite(VMCS_HOST_FS_BASE, fsbase)) != 0)
+		goto done;
+
+	idtrbase = vmm_get_host_idtrbase();
+	if ((error = vmwrite(VMCS_HOST_IDTR_BASE, idtrbase)) != 0)
+		goto done;
+
+	/* instruction pointer */
+	if ((error = vmwrite(VMCS_HOST_RIP, host_rip)) != 0)
+		goto done;
+
+	/* stack pointer */
+	if ((error = vmwrite(VMCS_HOST_RSP, host_rsp)) != 0)
+		goto done;
+
+	/* eptp */
+	eptp = EPTP(ept_pml4);
+	if ((error = vmwrite(VMCS_EPTP, eptp)) != 0)
+		goto done;
+
+	/* vpid */
+	if ((error = vmwrite(VMCS_VPID, vpid)) != 0)
+		goto done;
+
+	/* msr bitmap */
+	if ((error = vmwrite(VMCS_MSR_BITMAP, msr_bitmap)) != 0)
+		goto done;
+
+	/* exception bitmap */
+	exc_bitmap = 1 << IDT_MC;
+	if ((error = vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap)) != 0)
+		goto done;
+
+	/* link pointer */
+	if ((error = vmwrite(VMCS_LINK_POINTER, ~0)) != 0)
+		goto done;
+done:
+	VMCLEAR(vmcs);
+	return (error);
+}
+
+uint64_t
+vmcs_read(uint32_t encoding)
+{
+	int error;
+	uint64_t val;
+
+	error = vmread(encoding, &val);
+	if (error != 0)
+		panic("vmcs_read(%u) error %d", encoding, error);
+
+	return (val);
+}
+
+#ifdef DDB
+extern int vmxon_enabled[];
+
+DB_SHOW_COMMAND(vmcs, db_show_vmcs)
+{
+	uint64_t cur_vmcs, val;
+	uint32_t exit;
+
+	if (!vmxon_enabled[curcpu]) {
+		db_printf("VMX not enabled\n");
+		return;
+	}
+
+	if (have_addr) {
+		db_printf("Only current VMCS supported\n");
+		return;
+	}
+
+	vmptrst(&cur_vmcs);
+	if (cur_vmcs == VMCS_INITIAL) {
+		db_printf("No current VM context\n");
+		return;
+	}
+	db_printf("VMCS: %jx\n", cur_vmcs);
+	db_printf("VPID: %lu\n", vmcs_read(VMCS_VPID));
+	db_printf("Activity: ");
+	val = vmcs_read(VMCS_GUEST_ACTIVITY);
+	switch (val) {
+	case 0:
+		db_printf("Active");
+		break;
+	case 1:
+		db_printf("HLT");
+		break;
+	case 2:
+		db_printf("Shutdown");
+		break;
+	case 3:
+		db_printf("Wait for SIPI");
+		break;
+	default:
+		db_printf("Unknown: %#lx", val);
+	}
+	db_printf("\n");
+	exit = vmcs_read(VMCS_EXIT_REASON);
+	if (exit & 0x80000000)
+		db_printf("Entry Failure Reason: %u\n", exit & 0xffff);
+	else
+		db_printf("Exit Reason: %u\n", exit & 0xffff);
+	db_printf("Qualification: %#lx\n", vmcs_exit_qualification());
+	db_printf("Guest Linear Address: %#lx\n",
+	    vmcs_read(VMCS_GUEST_LINEAR_ADDRESS));
+	switch (exit & 0x8000ffff) {
+	case EXIT_REASON_EXCEPTION:
+	case EXIT_REASON_EXT_INTR:
+		val = vmcs_read(VMCS_EXIT_INTERRUPTION_INFO);
+		db_printf("Interrupt Type: ");
+		switch (val >> 8 & 0x7) {
+		case 0:
+			db_printf("external");
+			break;
+		case 2:
+			db_printf("NMI");
+			break;
+		case 3:
+			db_printf("HW exception");
+			break;
+		case 4:
+			db_printf("SW exception");
+			break;
+		default:
+			db_printf("?? %lu", val >> 8 & 0x7);
+			break;
+		}
+		db_printf("  Vector: %lu", val & 0xff);
+		if (val & 0x800)
+			db_printf("  Error Code: %lx",
+			    vmcs_read(VMCS_EXIT_INTERRUPTION_ERROR));
+		db_printf("\n");
+		break;
+	case EXIT_REASON_EPT_FAULT:
+	case EXIT_REASON_EPT_MISCONFIG:
+		db_printf("Guest Physical Address: %#lx\n",
+		    vmcs_read(VMCS_GUEST_PHYSICAL_ADDRESS));
+		break;
+	}
+	db_printf("VM-instruction error: %#lx\n", vmcs_instruction_error());
+}
+#endif
diff --git a/sys/amd64/vmm/intel/vmcs.h b/sys/amd64/vmm/intel/vmcs.h
new file mode 100644
index 0000000..f39eed2
--- /dev/null
+++ b/sys/amd64/vmm/intel/vmcs.h
@@ -0,0 +1,338 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VMCS_H_
+#define	_VMCS_H_
+
+#ifdef _KERNEL
+struct vmcs {
+	uint32_t	identifier;
+	uint32_t	abort_code;
+	char		_impl_specific[PAGE_SIZE - sizeof(uint32_t) * 2];
+};
+CTASSERT(sizeof(struct vmcs) == PAGE_SIZE);
+
+/* MSR save region is composed of an array of 'struct msr_entry' */
+struct msr_entry {
+	uint32_t	index;
+	uint32_t	reserved;
+	uint64_t	val;
+
+};
+
+int vmcs_set_msr_save(struct vmcs *vmcs, u_long g_area, u_int g_count);
+int	vmcs_set_defaults(struct vmcs *vmcs, u_long host_rip, u_long host_rsp,
+			  u_long ept_pml4,
+			  uint32_t pinbased_ctls, uint32_t procbased_ctls,
+			  uint32_t procbased_ctls2, uint32_t exit_ctls,
+			  uint32_t entry_ctls, u_long msr_bitmap,
+			  uint16_t vpid);
+int	vmcs_getreg(struct vmcs *vmcs, int ident, uint64_t *retval);
+int	vmcs_setreg(struct vmcs *vmcs, int ident, uint64_t val);
+int	vmcs_getdesc(struct vmcs *vmcs, int ident,
+		     struct seg_desc *desc);
+int	vmcs_setdesc(struct vmcs *vmcs, int ident,
+		     struct seg_desc *desc);
+uint64_t vmcs_read(uint32_t encoding);
+
+#define	vmexit_instruction_length()	vmcs_read(VMCS_EXIT_INSTRUCTION_LENGTH)
+#define	vmcs_guest_rip()		vmcs_read(VMCS_GUEST_RIP)
+#define	vmcs_instruction_error()	vmcs_read(VMCS_INSTRUCTION_ERROR)
+#define	vmcs_exit_reason()		(vmcs_read(VMCS_EXIT_REASON) & 0xffff)
+#define	vmcs_exit_qualification()	vmcs_read(VMCS_EXIT_QUALIFICATION)
+#define	vmcs_guest_cr3()		vmcs_read(VMCS_GUEST_CR3)
+#define	vmcs_gpa()			vmcs_read(VMCS_GUEST_PHYSICAL_ADDRESS)
+#define	vmcs_gla()			vmcs_read(VMCS_GUEST_LINEAR_ADDRESS)
+
+#endif	/* _KERNEL */
+
+#define	VMCS_INITIAL			0xffffffffffffffff
+
+#define	VMCS_IDENT(encoding)		((encoding) | 0x80000000)
+/*
+ * VMCS field encodings from Appendix H, Intel Architecture Manual Vol3B.
+ */
+#define	VMCS_INVALID_ENCODING		0xffffffff
+
+/* 16-bit control fields */
+#define	VMCS_VPID			0x00000000
+
+/* 16-bit guest-state fields */
+#define	VMCS_GUEST_ES_SELECTOR		0x00000800
+#define	VMCS_GUEST_CS_SELECTOR		0x00000802
+#define	VMCS_GUEST_SS_SELECTOR		0x00000804
+#define	VMCS_GUEST_DS_SELECTOR		0x00000806
+#define	VMCS_GUEST_FS_SELECTOR		0x00000808
+#define	VMCS_GUEST_GS_SELECTOR		0x0000080A
+#define	VMCS_GUEST_LDTR_SELECTOR	0x0000080C
+#define	VMCS_GUEST_TR_SELECTOR		0x0000080E
+
+/* 16-bit host-state fields */
+#define	VMCS_HOST_ES_SELECTOR		0x00000C00
+#define	VMCS_HOST_CS_SELECTOR		0x00000C02
+#define	VMCS_HOST_SS_SELECTOR		0x00000C04
+#define	VMCS_HOST_DS_SELECTOR		0x00000C06
+#define	VMCS_HOST_FS_SELECTOR		0x00000C08
+#define	VMCS_HOST_GS_SELECTOR		0x00000C0A
+#define	VMCS_HOST_TR_SELECTOR		0x00000C0C
+
+/* 64-bit control fields */
+#define	VMCS_IO_BITMAP_A		0x00002000
+#define	VMCS_IO_BITMAP_B		0x00002002
+#define	VMCS_MSR_BITMAP			0x00002004
+#define	VMCS_EXIT_MSR_STORE		0x00002006
+#define	VMCS_EXIT_MSR_LOAD		0x00002008
+#define	VMCS_ENTRY_MSR_LOAD		0x0000200A
+#define	VMCS_EXECUTIVE_VMCS		0x0000200C
+#define	VMCS_TSC_OFFSET			0x00002010
+#define	VMCS_VIRTUAL_APIC		0x00002012
+#define	VMCS_APIC_ACCESS		0x00002014
+#define	VMCS_EPTP			0x0000201A
+
+/* 64-bit read-only fields */
+#define	VMCS_GUEST_PHYSICAL_ADDRESS	0x00002400
+
+/* 64-bit guest-state fields */
+#define	VMCS_LINK_POINTER		0x00002800
+#define	VMCS_GUEST_IA32_DEBUGCTL	0x00002802
+#define	VMCS_GUEST_IA32_PAT		0x00002804
+#define	VMCS_GUEST_IA32_EFER		0x00002806
+#define	VMCS_GUEST_IA32_PERF_GLOBAL_CTRL 0x00002808
+#define	VMCS_GUEST_PDPTE0		0x0000280A
+#define	VMCS_GUEST_PDPTE1		0x0000280C
+#define	VMCS_GUEST_PDPTE2		0x0000280E
+#define	VMCS_GUEST_PDPTE3		0x00002810
+
+/* 64-bit host-state fields */
+#define	VMCS_HOST_IA32_PAT		0x00002C00
+#define	VMCS_HOST_IA32_EFER		0x00002C02
+#define	VMCS_HOST_IA32_PERF_GLOBAL_CTRL	0x00002C04
+
+/* 32-bit control fields */
+#define	VMCS_PIN_BASED_CTLS		0x00004000
+#define	VMCS_PRI_PROC_BASED_CTLS	0x00004002
+#define	VMCS_EXCEPTION_BITMAP		0x00004004
+#define	VMCS_PF_ERROR_MASK		0x00004006
+#define	VMCS_PF_ERROR_MATCH		0x00004008
+#define	VMCS_CR3_TARGET_COUNT		0x0000400A
+#define	VMCS_EXIT_CTLS			0x0000400C
+#define	VMCS_EXIT_MSR_STORE_COUNT	0x0000400E
+#define	VMCS_EXIT_MSR_LOAD_COUNT	0x00004010
+#define	VMCS_ENTRY_CTLS			0x00004012
+#define	VMCS_ENTRY_MSR_LOAD_COUNT	0x00004014
+#define	VMCS_ENTRY_INTR_INFO		0x00004016
+#define	VMCS_ENTRY_EXCEPTION_ERROR	0x00004018
+#define	VMCS_ENTRY_INST_LENGTH		0x0000401A
+#define	VMCS_TPR_THRESHOLD		0x0000401C
+#define	VMCS_SEC_PROC_BASED_CTLS	0x0000401E
+#define	VMCS_PLE_GAP			0x00004020
+#define	VMCS_PLE_WINDOW			0x00004022
+
+/* 32-bit read-only data fields */
+#define	VMCS_INSTRUCTION_ERROR		0x00004400
+#define	VMCS_EXIT_REASON		0x00004402
+#define	VMCS_EXIT_INTERRUPTION_INFO	0x00004404
+#define	VMCS_EXIT_INTERRUPTION_ERROR	0x00004406
+#define	VMCS_IDT_VECTORING_INFO		0x00004408
+#define	VMCS_IDT_VECTORING_ERROR	0x0000440A
+#define	VMCS_EXIT_INSTRUCTION_LENGTH	0x0000440C
+#define	VMCS_EXIT_INSTRUCTION_INFO	0x0000440E
+
+/* 32-bit guest-state fields */
+#define	VMCS_GUEST_ES_LIMIT		0x00004800
+#define	VMCS_GUEST_CS_LIMIT		0x00004802
+#define	VMCS_GUEST_SS_LIMIT		0x00004804
+#define	VMCS_GUEST_DS_LIMIT		0x00004806
+#define	VMCS_GUEST_FS_LIMIT		0x00004808
+#define	VMCS_GUEST_GS_LIMIT		0x0000480A
+#define	VMCS_GUEST_LDTR_LIMIT		0x0000480C
+#define	VMCS_GUEST_TR_LIMIT		0x0000480E
+#define	VMCS_GUEST_GDTR_LIMIT		0x00004810
+#define	VMCS_GUEST_IDTR_LIMIT		0x00004812
+#define	VMCS_GUEST_ES_ACCESS_RIGHTS	0x00004814
+#define	VMCS_GUEST_CS_ACCESS_RIGHTS	0x00004816
+#define	VMCS_GUEST_SS_ACCESS_RIGHTS	0x00004818
+#define	VMCS_GUEST_DS_ACCESS_RIGHTS	0x0000481A
+#define	VMCS_GUEST_FS_ACCESS_RIGHTS	0x0000481C
+#define	VMCS_GUEST_GS_ACCESS_RIGHTS	0x0000481E
+#define	VMCS_GUEST_LDTR_ACCESS_RIGHTS	0x00004820
+#define	VMCS_GUEST_TR_ACCESS_RIGHTS	0x00004822
+#define	VMCS_GUEST_INTERRUPTIBILITY	0x00004824
+#define	VMCS_GUEST_ACTIVITY		0x00004826
+#define VMCS_GUEST_SMBASE		0x00004828
+#define	VMCS_GUEST_IA32_SYSENTER_CS	0x0000482A
+#define	VMCS_PREEMPTION_TIMER_VALUE	0x0000482E
+
+/* 32-bit host state fields */
+#define	VMCS_HOST_IA32_SYSENTER_CS	0x00004C00
+
+/* Natural Width control fields */
+#define	VMCS_CR0_MASK			0x00006000
+#define	VMCS_CR4_MASK			0x00006002
+#define	VMCS_CR0_SHADOW			0x00006004
+#define	VMCS_CR4_SHADOW			0x00006006
+#define	VMCS_CR3_TARGET0		0x00006008
+#define	VMCS_CR3_TARGET1		0x0000600A
+#define	VMCS_CR3_TARGET2		0x0000600C
+#define	VMCS_CR3_TARGET3		0x0000600E
+
+/* Natural Width read-only fields */
+#define	VMCS_EXIT_QUALIFICATION		0x00006400
+#define	VMCS_IO_RCX			0x00006402
+#define	VMCS_IO_RSI			0x00006404
+#define	VMCS_IO_RDI			0x00006406
+#define	VMCS_IO_RIP			0x00006408
+#define	VMCS_GUEST_LINEAR_ADDRESS	0x0000640A
+
+/* Natural Width guest-state fields */
+#define	VMCS_GUEST_CR0			0x00006800
+#define	VMCS_GUEST_CR3			0x00006802
+#define	VMCS_GUEST_CR4			0x00006804
+#define	VMCS_GUEST_ES_BASE		0x00006806
+#define	VMCS_GUEST_CS_BASE		0x00006808
+#define	VMCS_GUEST_SS_BASE		0x0000680A
+#define	VMCS_GUEST_DS_BASE		0x0000680C
+#define	VMCS_GUEST_FS_BASE		0x0000680E
+#define	VMCS_GUEST_GS_BASE		0x00006810
+#define	VMCS_GUEST_LDTR_BASE		0x00006812
+#define	VMCS_GUEST_TR_BASE		0x00006814
+#define	VMCS_GUEST_GDTR_BASE		0x00006816
+#define	VMCS_GUEST_IDTR_BASE		0x00006818
+#define	VMCS_GUEST_DR7			0x0000681A
+#define	VMCS_GUEST_RSP			0x0000681C
+#define	VMCS_GUEST_RIP			0x0000681E
+#define	VMCS_GUEST_RFLAGS		0x00006820
+#define	VMCS_GUEST_PENDING_DBG_EXCEPTIONS 0x00006822
+#define	VMCS_GUEST_IA32_SYSENTER_ESP	0x00006824
+#define	VMCS_GUEST_IA32_SYSENTER_EIP	0x00006826
+
+/* Natural Width host-state fields */
+#define	VMCS_HOST_CR0			0x00006C00
+#define	VMCS_HOST_CR3			0x00006C02
+#define	VMCS_HOST_CR4			0x00006C04
+#define	VMCS_HOST_FS_BASE		0x00006C06
+#define	VMCS_HOST_GS_BASE		0x00006C08
+#define	VMCS_HOST_TR_BASE		0x00006C0A
+#define	VMCS_HOST_GDTR_BASE		0x00006C0C
+#define	VMCS_HOST_IDTR_BASE		0x00006C0E
+#define	VMCS_HOST_IA32_SYSENTER_ESP	0x00006C10
+#define	VMCS_HOST_IA32_SYSENTER_EIP	0x00006C12
+#define	VMCS_HOST_RSP			0x00006C14
+#define	VMCS_HOST_RIP			0x00006c16
+
+/*
+ * VM instruction error numbers
+ */
+#define	VMRESUME_WITH_NON_LAUNCHED_VMCS	5
+
+/*
+ * VMCS exit reasons
+ */
+#define EXIT_REASON_EXCEPTION		0
+#define EXIT_REASON_EXT_INTR		1
+#define EXIT_REASON_TRIPLE_FAULT	2
+#define EXIT_REASON_INIT		3
+#define EXIT_REASON_SIPI		4
+#define EXIT_REASON_IO_SMI		5
+#define EXIT_REASON_SMI			6
+#define EXIT_REASON_INTR_WINDOW		7
+#define EXIT_REASON_NMI_WINDOW		8
+#define EXIT_REASON_TASK_SWITCH		9
+#define EXIT_REASON_CPUID		10
+#define EXIT_REASON_GETSEC		11
+#define EXIT_REASON_HLT			12
+#define EXIT_REASON_INVD		13
+#define EXIT_REASON_INVLPG		14
+#define EXIT_REASON_RDPMC		15
+#define EXIT_REASON_RDTSC		16
+#define EXIT_REASON_RSM			17
+#define EXIT_REASON_VMCALL		18
+#define EXIT_REASON_VMCLEAR		19
+#define EXIT_REASON_VMLAUNCH		20
+#define EXIT_REASON_VMPTRLD		21
+#define EXIT_REASON_VMPTRST		22
+#define EXIT_REASON_VMREAD		23
+#define EXIT_REASON_VMRESUME		24
+#define EXIT_REASON_VMWRITE		25
+#define EXIT_REASON_VMXOFF		26
+#define EXIT_REASON_VMXON		27
+#define EXIT_REASON_CR_ACCESS		28
+#define EXIT_REASON_DR_ACCESS		29
+#define EXIT_REASON_INOUT		30
+#define EXIT_REASON_RDMSR		31
+#define EXIT_REASON_WRMSR		32
+#define EXIT_REASON_INVAL_VMCS		33
+#define EXIT_REASON_INVAL_MSR		34
+#define EXIT_REASON_MWAIT		36
+#define EXIT_REASON_MTF			37
+#define EXIT_REASON_MONITOR		39
+#define EXIT_REASON_PAUSE		40
+#define EXIT_REASON_MCE			41
+#define EXIT_REASON_TPR			43
+#define EXIT_REASON_APIC		44
+#define EXIT_REASON_GDTR_IDTR		46
+#define EXIT_REASON_LDTR_TR		47
+#define EXIT_REASON_EPT_FAULT		48
+#define EXIT_REASON_EPT_MISCONFIG	49
+#define EXIT_REASON_INVEPT		50
+#define EXIT_REASON_RDTSCP		51
+#define EXIT_REASON_VMX_PREEMPT		52
+#define EXIT_REASON_INVVPID		53
+#define EXIT_REASON_WBINVD		54
+#define EXIT_REASON_XSETBV		55
+
+/*
+ * VMCS interrupt information fields
+ */
+#define	VMCS_INTERRUPTION_INFO_VALID	(1U << 31)
+#define	VMCS_INTERRUPTION_INFO_HW_INTR	(0 << 8)
+#define	VMCS_INTERRUPTION_INFO_NMI	(2 << 8)
+
+/*
+ * VMCS Guest interruptibility field
+ */
+#define	VMCS_INTERRUPTIBILITY_STI_BLOCKING	(1 << 0)
+#define	VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING	(1 << 1)
+#define	VMCS_INTERRUPTIBILITY_SMI_BLOCKING	(1 << 2)
+#define	VMCS_INTERRUPTIBILITY_NMI_BLOCKING	(1 << 3)
+
+/*
+ * Exit qualification for EXIT_REASON_INVAL_VMCS
+ */
+#define	EXIT_QUAL_NMI_WHILE_STI_BLOCKING	3
+
+/*
+ * Exit qualification for EPT violation
+ */
+#define	EPT_VIOLATION_DATA_READ		(1UL << 0)
+#define	EPT_VIOLATION_DATA_WRITE	(1UL << 1)
+#define	EPT_VIOLATION_INST_FETCH	(1UL << 2)
+#define	EPT_VIOLATION_GLA_VALID		(1UL << 7)
+#define	EPT_VIOLATION_XLAT_VALID	(1UL << 8)
+
+#endif
diff --git a/sys/amd64/vmm/intel/vmx.c b/sys/amd64/vmm/intel/vmx.c
new file mode 100644
index 0000000..287ac8c
--- /dev/null
+++ b/sys/amd64/vmm/intel/vmx.c
@@ -0,0 +1,1852 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/smp.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/pcpu.h>
+#include <sys/proc.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/psl.h>
+#include <machine/cpufunc.h>
+#include <machine/md_var.h>
+#include <machine/pmap.h>
+#include <machine/segments.h>
+#include <machine/specialreg.h>
+#include <machine/vmparam.h>
+
+#include <x86/apicreg.h>
+
+#include <machine/vmm.h>
+#include "vmm_host.h"
+#include "vmm_lapic.h"
+#include "vmm_msr.h"
+#include "vmm_ktr.h"
+#include "vmm_stat.h"
+
+#include "vmx_msr.h"
+#include "ept.h"
+#include "vmx_cpufunc.h"
+#include "vmx.h"
+#include "x86.h"
+#include "vmx_controls.h"
+
+#define	PINBASED_CTLS_ONE_SETTING					\
+	(PINBASED_EXTINT_EXITING	|				\
+	 PINBASED_NMI_EXITING		|				\
+	 PINBASED_VIRTUAL_NMI)
+#define	PINBASED_CTLS_ZERO_SETTING	0
+
+#define PROCBASED_CTLS_WINDOW_SETTING					\
+	(PROCBASED_INT_WINDOW_EXITING	|				\
+	 PROCBASED_NMI_WINDOW_EXITING)
+
+#define	PROCBASED_CTLS_ONE_SETTING 					\
+	(PROCBASED_SECONDARY_CONTROLS	|				\
+	 PROCBASED_IO_EXITING		|				\
+	 PROCBASED_MSR_BITMAPS		|				\
+	 PROCBASED_CTLS_WINDOW_SETTING)
+#define	PROCBASED_CTLS_ZERO_SETTING	\
+	(PROCBASED_CR3_LOAD_EXITING |	\
+	PROCBASED_CR3_STORE_EXITING |	\
+	PROCBASED_IO_BITMAPS)
+
+#define	PROCBASED_CTLS2_ONE_SETTING	PROCBASED2_ENABLE_EPT
+#define	PROCBASED_CTLS2_ZERO_SETTING	0
+
+#define VM_EXIT_CTLS_ONE_SETTING_NO_PAT					\
+	(VM_EXIT_HOST_LMA			|			\
+	VM_EXIT_SAVE_EFER			|			\
+	VM_EXIT_LOAD_EFER)
+
+#define	VM_EXIT_CTLS_ONE_SETTING					\
+	(VM_EXIT_CTLS_ONE_SETTING_NO_PAT       	|			\
+	VM_EXIT_SAVE_PAT			|			\
+	VM_EXIT_LOAD_PAT)
+#define	VM_EXIT_CTLS_ZERO_SETTING	VM_EXIT_SAVE_DEBUG_CONTROLS
+
+#define	VM_ENTRY_CTLS_ONE_SETTING_NO_PAT	VM_ENTRY_LOAD_EFER
+
+#define	VM_ENTRY_CTLS_ONE_SETTING					\
+	(VM_ENTRY_CTLS_ONE_SETTING_NO_PAT     	|			\
+	VM_ENTRY_LOAD_PAT)
+#define	VM_ENTRY_CTLS_ZERO_SETTING					\
+	(VM_ENTRY_LOAD_DEBUG_CONTROLS		|			\
+	VM_ENTRY_INTO_SMM			|			\
+	VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
+
+#define	guest_msr_rw(vmx, msr) \
+	msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_RW)
+
+#define	HANDLED		1
+#define	UNHANDLED	0
+
+MALLOC_DEFINE(M_VMX, "vmx", "vmx");
+
+int vmxon_enabled[MAXCPU];
+static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
+
+static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
+static uint32_t exit_ctls, entry_ctls;
+
+static uint64_t cr0_ones_mask, cr0_zeros_mask;
+static uint64_t cr4_ones_mask, cr4_zeros_mask;
+
+static volatile u_int nextvpid;
+
+static int vmx_no_patmsr;
+
+/*
+ * Virtual NMI blocking conditions.
+ *
+ * Some processor implementations also require NMI to be blocked if
+ * the STI_BLOCKING bit is set. It is possible to detect this at runtime
+ * based on the (exit_reason,exit_qual) tuple being set to 
+ * (EXIT_REASON_INVAL_VMCS, EXIT_QUAL_NMI_WHILE_STI_BLOCKING).
+ *
+ * We take the easy way out and also include STI_BLOCKING as one of the
+ * gating items for vNMI injection.
+ */
+static uint64_t nmi_blocking_bits = VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING |
+				    VMCS_INTERRUPTIBILITY_NMI_BLOCKING |
+				    VMCS_INTERRUPTIBILITY_STI_BLOCKING;
+
+/*
+ * Optional capabilities
+ */
+static int cap_halt_exit;
+static int cap_pause_exit;
+static int cap_unrestricted_guest;
+static int cap_monitor_trap;
+ 
+/* statistics */
+static VMM_STAT_DEFINE(VCPU_MIGRATIONS, "vcpu migration across host cpus");
+static VMM_STAT_DEFINE(VMEXIT_EXTINT, "vm exits due to external interrupt");
+static VMM_STAT_DEFINE(VMEXIT_HLT_IGNORED, "number of times hlt was ignored");
+static VMM_STAT_DEFINE(VMEXIT_HLT, "number of times hlt was intercepted");
+
+#ifdef KTR
+static const char *
+exit_reason_to_str(int reason)
+{
+	static char reasonbuf[32];
+
+	switch (reason) {
+	case EXIT_REASON_EXCEPTION:
+		return "exception";
+	case EXIT_REASON_EXT_INTR:
+		return "extint";
+	case EXIT_REASON_TRIPLE_FAULT:
+		return "triplefault";
+	case EXIT_REASON_INIT:
+		return "init";
+	case EXIT_REASON_SIPI:
+		return "sipi";
+	case EXIT_REASON_IO_SMI:
+		return "iosmi";
+	case EXIT_REASON_SMI:
+		return "smi";
+	case EXIT_REASON_INTR_WINDOW:
+		return "intrwindow";
+	case EXIT_REASON_NMI_WINDOW:
+		return "nmiwindow";
+	case EXIT_REASON_TASK_SWITCH:
+		return "taskswitch";
+	case EXIT_REASON_CPUID:
+		return "cpuid";
+	case EXIT_REASON_GETSEC:
+		return "getsec";
+	case EXIT_REASON_HLT:
+		return "hlt";
+	case EXIT_REASON_INVD:
+		return "invd";
+	case EXIT_REASON_INVLPG:
+		return "invlpg";
+	case EXIT_REASON_RDPMC:
+		return "rdpmc";
+	case EXIT_REASON_RDTSC:
+		return "rdtsc";
+	case EXIT_REASON_RSM:
+		return "rsm";
+	case EXIT_REASON_VMCALL:
+		return "vmcall";
+	case EXIT_REASON_VMCLEAR:
+		return "vmclear";
+	case EXIT_REASON_VMLAUNCH:
+		return "vmlaunch";
+	case EXIT_REASON_VMPTRLD:
+		return "vmptrld";
+	case EXIT_REASON_VMPTRST:
+		return "vmptrst";
+	case EXIT_REASON_VMREAD:
+		return "vmread";
+	case EXIT_REASON_VMRESUME:
+		return "vmresume";
+	case EXIT_REASON_VMWRITE:
+		return "vmwrite";
+	case EXIT_REASON_VMXOFF:
+		return "vmxoff";
+	case EXIT_REASON_VMXON:
+		return "vmxon";
+	case EXIT_REASON_CR_ACCESS:
+		return "craccess";
+	case EXIT_REASON_DR_ACCESS:
+		return "draccess";
+	case EXIT_REASON_INOUT:
+		return "inout";
+	case EXIT_REASON_RDMSR:
+		return "rdmsr";
+	case EXIT_REASON_WRMSR:
+		return "wrmsr";
+	case EXIT_REASON_INVAL_VMCS:
+		return "invalvmcs";
+	case EXIT_REASON_INVAL_MSR:
+		return "invalmsr";
+	case EXIT_REASON_MWAIT:
+		return "mwait";
+	case EXIT_REASON_MTF:
+		return "mtf";
+	case EXIT_REASON_MONITOR:
+		return "monitor";
+	case EXIT_REASON_PAUSE:
+		return "pause";
+	case EXIT_REASON_MCE:
+		return "mce";
+	case EXIT_REASON_TPR:
+		return "tpr";
+	case EXIT_REASON_APIC:
+		return "apic";
+	case EXIT_REASON_GDTR_IDTR:
+		return "gdtridtr";
+	case EXIT_REASON_LDTR_TR:
+		return "ldtrtr";
+	case EXIT_REASON_EPT_FAULT:
+		return "eptfault";
+	case EXIT_REASON_EPT_MISCONFIG:
+		return "eptmisconfig";
+	case EXIT_REASON_INVEPT:
+		return "invept";
+	case EXIT_REASON_RDTSCP:
+		return "rdtscp";
+	case EXIT_REASON_VMX_PREEMPT:
+		return "vmxpreempt";
+	case EXIT_REASON_INVVPID:
+		return "invvpid";
+	case EXIT_REASON_WBINVD:
+		return "wbinvd";
+	case EXIT_REASON_XSETBV:
+		return "xsetbv";
+	default:
+		snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
+		return (reasonbuf);
+	}
+}
+
+#ifdef SETJMP_TRACE
+static const char *
+vmx_setjmp_rc2str(int rc)
+{
+	switch (rc) {
+	case VMX_RETURN_DIRECT:
+		return "direct";
+	case VMX_RETURN_LONGJMP:
+		return "longjmp";
+	case VMX_RETURN_VMRESUME:
+		return "vmresume";
+	case VMX_RETURN_VMLAUNCH:
+		return "vmlaunch";
+	case VMX_RETURN_AST:
+		return "ast";
+	default:
+		return "unknown";
+	}
+}
+
+#define	SETJMP_TRACE(vmx, vcpu, vmxctx, regname)			  \
+	VMM_CTR1((vmx)->vm, (vcpu), "setjmp trace " #regname " 0x%016lx", \
+		 (vmxctx)->regname)
+
+static void
+vmx_setjmp_trace(struct vmx *vmx, int vcpu, struct vmxctx *vmxctx, int rc)
+{
+	uint64_t host_rip, host_rsp;
+
+	if (vmxctx != &vmx->ctx[vcpu])
+		panic("vmx_setjmp_trace: invalid vmxctx %p; should be %p",
+			vmxctx, &vmx->ctx[vcpu]);
+
+	VMM_CTR1((vmx)->vm, (vcpu), "vmxctx = %p", vmxctx);
+	VMM_CTR2((vmx)->vm, (vcpu), "setjmp return code %s(%d)",
+		 vmx_setjmp_rc2str(rc), rc);
+
+	host_rsp = host_rip = ~0;
+	vmread(VMCS_HOST_RIP, &host_rip);
+	vmread(VMCS_HOST_RSP, &host_rsp);
+	VMM_CTR2((vmx)->vm, (vcpu), "vmcs host_rip 0x%016lx, host_rsp 0x%016lx",
+		 host_rip, host_rsp);
+
+	SETJMP_TRACE(vmx, vcpu, vmxctx, host_r15);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, host_r14);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, host_r13);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, host_r12);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, host_rbp);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, host_rsp);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, host_rbx);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, host_rip);
+
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rdi);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rsi);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rdx);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rcx);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r8);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r9);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rax);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rbx);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_rbp);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r10);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r11);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r12);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r13);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r14);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_r15);
+	SETJMP_TRACE(vmx, vcpu, vmxctx, guest_cr2);
+}
+#endif
+#else
+static void __inline
+vmx_setjmp_trace(struct vmx *vmx, int vcpu, struct vmxctx *vmxctx, int rc)
+{
+	return;
+}
+#endif	/* KTR */
+
+u_long
+vmx_fix_cr0(u_long cr0)
+{
+
+	return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
+}
+
+u_long
+vmx_fix_cr4(u_long cr4)
+{
+
+	return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
+}
+
+static void
+msr_save_area_init(struct msr_entry *g_area, int *g_count)
+{
+	int cnt;
+
+	static struct msr_entry guest_msrs[] = {
+		{ MSR_KGSBASE, 0, 0 },
+	};
+
+	cnt = sizeof(guest_msrs) / sizeof(guest_msrs[0]);
+	if (cnt > GUEST_MSR_MAX_ENTRIES)
+		panic("guest msr save area overrun");
+	bcopy(guest_msrs, g_area, sizeof(guest_msrs));
+	*g_count = cnt;
+}
+
+static void
+vmx_disable(void *arg __unused)
+{
+	struct invvpid_desc invvpid_desc = { 0 };
+	struct invept_desc invept_desc = { 0 };
+
+	if (vmxon_enabled[curcpu]) {
+		/*
+		 * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
+		 *
+		 * VMXON or VMXOFF are not required to invalidate any TLB
+		 * caching structures. This prevents potential retention of
+		 * cached information in the TLB between distinct VMX episodes.
+		 */
+		invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
+		invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
+		vmxoff();
+	}
+	load_cr4(rcr4() & ~CR4_VMXE);
+}
+
+static int
+vmx_cleanup(void)
+{
+
+	smp_rendezvous(NULL, vmx_disable, NULL, NULL);
+
+	return (0);
+}
+
+static void
+vmx_enable(void *arg __unused)
+{
+	int error;
+
+	load_cr4(rcr4() | CR4_VMXE);
+
+	*(uint32_t *)vmxon_region[curcpu] = vmx_revision();
+	error = vmxon(vmxon_region[curcpu]);
+	if (error == 0)
+		vmxon_enabled[curcpu] = 1;
+}
+
+static int
+vmx_init(void)
+{
+	int error;
+	uint64_t fixed0, fixed1, feature_control;
+	uint32_t tmp;
+
+	/* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
+	if (!(cpu_feature2 & CPUID2_VMX)) {
+		printf("vmx_init: processor does not support VMX operation\n");
+		return (ENXIO);
+	}
+
+	/*
+	 * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
+	 * are set (bits 0 and 2 respectively).
+	 */
+	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
+	if ((feature_control & 0x5) != 0x5) {
+		printf("vmx_init: VMX operation disabled by BIOS\n");
+		return (ENXIO);
+	}
+
+	/* Check support for primary processor-based VM-execution controls */
+	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
+			       MSR_VMX_TRUE_PROCBASED_CTLS,
+			       PROCBASED_CTLS_ONE_SETTING,
+			       PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
+	if (error) {
+		printf("vmx_init: processor does not support desired primary "
+		       "processor-based controls\n");
+		return (error);
+	}
+
+	/* Clear the processor-based ctl bits that are set on demand */
+	procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
+
+	/* Check support for secondary processor-based VM-execution controls */
+	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
+			       MSR_VMX_PROCBASED_CTLS2,
+			       PROCBASED_CTLS2_ONE_SETTING,
+			       PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
+	if (error) {
+		printf("vmx_init: processor does not support desired secondary "
+		       "processor-based controls\n");
+		return (error);
+	}
+
+	/* Check support for VPID */
+	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
+			       PROCBASED2_ENABLE_VPID, 0, &tmp);
+	if (error == 0)
+		procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
+
+	/* Check support for pin-based VM-execution controls */
+	error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
+			       MSR_VMX_TRUE_PINBASED_CTLS,
+			       PINBASED_CTLS_ONE_SETTING,
+			       PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
+	if (error) {
+		printf("vmx_init: processor does not support desired "
+		       "pin-based controls\n");
+		return (error);
+	}
+
+	/* Check support for VM-exit controls */
+	error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
+			       VM_EXIT_CTLS_ONE_SETTING,
+			       VM_EXIT_CTLS_ZERO_SETTING,
+			       &exit_ctls);
+	if (error) {
+		/* Try again without the PAT MSR bits */
+		error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS,
+				       MSR_VMX_TRUE_EXIT_CTLS,
+				       VM_EXIT_CTLS_ONE_SETTING_NO_PAT,
+				       VM_EXIT_CTLS_ZERO_SETTING,
+				       &exit_ctls);
+		if (error) {
+			printf("vmx_init: processor does not support desired "
+			       "exit controls\n");
+			return (error);
+		} else {
+			if (bootverbose)
+				printf("vmm: PAT MSR access not supported\n");
+			guest_msr_valid(MSR_PAT);
+			vmx_no_patmsr = 1;
+		}
+	}
+
+	/* Check support for VM-entry controls */
+	if (!vmx_no_patmsr) {
+		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
+				       MSR_VMX_TRUE_ENTRY_CTLS,
+				       VM_ENTRY_CTLS_ONE_SETTING,
+				       VM_ENTRY_CTLS_ZERO_SETTING,
+				       &entry_ctls);
+	} else {
+		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
+				       MSR_VMX_TRUE_ENTRY_CTLS,
+				       VM_ENTRY_CTLS_ONE_SETTING_NO_PAT,
+				       VM_ENTRY_CTLS_ZERO_SETTING,
+				       &entry_ctls);
+	}
+
+	if (error) {
+		printf("vmx_init: processor does not support desired "
+		       "entry controls\n");
+		       return (error);
+	}
+
+	/*
+	 * Check support for optional features by testing them
+	 * as individual bits
+	 */
+	cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
+					MSR_VMX_TRUE_PROCBASED_CTLS,
+					PROCBASED_HLT_EXITING, 0,
+					&tmp) == 0);
+
+	cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
+					MSR_VMX_PROCBASED_CTLS,
+					PROCBASED_MTF, 0,
+					&tmp) == 0);
+
+	cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
+					 MSR_VMX_TRUE_PROCBASED_CTLS,
+					 PROCBASED_PAUSE_EXITING, 0,
+					 &tmp) == 0);
+
+	cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
+					MSR_VMX_PROCBASED_CTLS2,
+					PROCBASED2_UNRESTRICTED_GUEST, 0,
+				        &tmp) == 0);
+
+	/* Initialize EPT */
+	error = ept_init();
+	if (error) {
+		printf("vmx_init: ept initialization failed (%d)\n", error);
+		return (error);
+	}
+
+	/*
+	 * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
+	 */
+	fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
+	fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
+	cr0_ones_mask = fixed0 & fixed1;
+	cr0_zeros_mask = ~fixed0 & ~fixed1;
+
+	/*
+	 * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
+	 * if unrestricted guest execution is allowed.
+	 */
+	if (cap_unrestricted_guest)
+		cr0_ones_mask &= ~(CR0_PG | CR0_PE);
+
+	/*
+	 * Do not allow the guest to set CR0_NW or CR0_CD.
+	 */
+	cr0_zeros_mask |= (CR0_NW | CR0_CD);
+
+	fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
+	fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
+	cr4_ones_mask = fixed0 & fixed1;
+	cr4_zeros_mask = ~fixed0 & ~fixed1;
+
+	/* enable VMX operation */
+	smp_rendezvous(NULL, vmx_enable, NULL, NULL);
+
+	return (0);
+}
+
+/*
+ * If this processor does not support VPIDs then simply return 0.
+ *
+ * Otherwise generate the next value of VPID to use. Any value is alright
+ * as long as it is non-zero.
+ *
+ * We always execute in VMX non-root context with EPT enabled. Thus all
+ * combined mappings are tagged with the (EP4TA, VPID, PCID) tuple. This
+ * in turn means that multiple VMs can share the same VPID as long as
+ * they have distinct EPT page tables.
+ *
+ * XXX
+ * We should optimize this so that it returns VPIDs that are not in
+ * use. Then we will not unnecessarily invalidate mappings in
+ * vmx_set_pcpu_defaults() just because two or more vcpus happen to
+ * use the same 'vpid'.
+ */
+static uint16_t
+vmx_vpid(void)
+{
+	uint16_t vpid = 0;
+
+	if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) != 0) {
+		do {
+			vpid = atomic_fetchadd_int(&nextvpid, 1);
+		} while (vpid == 0);
+	}
+
+	return (vpid);
+}
+
+static int
+vmx_setup_cr_shadow(int which, struct vmcs *vmcs)
+{
+	int error, mask_ident, shadow_ident;
+	uint64_t mask_value, shadow_value;
+
+	if (which != 0 && which != 4)
+		panic("vmx_setup_cr_shadow: unknown cr%d", which);
+
+	if (which == 0) {
+		mask_ident = VMCS_CR0_MASK;
+		mask_value = cr0_ones_mask | cr0_zeros_mask;
+		shadow_ident = VMCS_CR0_SHADOW;
+		shadow_value = cr0_ones_mask;
+	} else {
+		mask_ident = VMCS_CR4_MASK;
+		mask_value = cr4_ones_mask | cr4_zeros_mask;
+		shadow_ident = VMCS_CR4_SHADOW;
+		shadow_value = cr4_ones_mask;
+	}
+
+	error = vmcs_setreg(vmcs, VMCS_IDENT(mask_ident), mask_value);
+	if (error)
+		return (error);
+
+	error = vmcs_setreg(vmcs, VMCS_IDENT(shadow_ident), shadow_value);
+	if (error)
+		return (error);
+
+	return (0);
+}
+#define	vmx_setup_cr0_shadow(vmcs)	vmx_setup_cr_shadow(0, (vmcs))
+#define	vmx_setup_cr4_shadow(vmcs)	vmx_setup_cr_shadow(4, (vmcs))
+
+static void *
+vmx_vminit(struct vm *vm)
+{
+	uint16_t vpid;
+	int i, error, guest_msr_count;
+	struct vmx *vmx;
+
+	vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
+	if ((uintptr_t)vmx & PAGE_MASK) {
+		panic("malloc of struct vmx not aligned on %d byte boundary",
+		      PAGE_SIZE);
+	}
+	vmx->vm = vm;
+
+	/*
+	 * Clean up EPTP-tagged guest physical and combined mappings
+	 *
+	 * VMX transitions are not required to invalidate any guest physical
+	 * mappings. So, it may be possible for stale guest physical mappings
+	 * to be present in the processor TLBs.
+	 *
+	 * Combined mappings for this EP4TA are also invalidated for all VPIDs.
+	 */
+	ept_invalidate_mappings(vtophys(vmx->pml4ept));
+
+	msr_bitmap_initialize(vmx->msr_bitmap);
+
+	/*
+	 * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
+	 * The guest FSBASE and GSBASE are saved and restored during
+	 * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
+	 * always restored from the vmcs host state area on vm-exit.
+	 *
+	 * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
+	 * how they are saved/restored so can be directly accessed by the
+	 * guest.
+	 *
+	 * Guest KGSBASE is saved and restored in the guest MSR save area.
+	 * Host KGSBASE is restored before returning to userland from the pcb.
+	 * There will be a window of time when we are executing in the host
+	 * kernel context with a value of KGSBASE from the guest. This is ok
+	 * because the value of KGSBASE is inconsequential in kernel context.
+	 *
+	 * MSR_EFER is saved and restored in the guest VMCS area on a
+	 * VM exit and entry respectively. It is also restored from the
+	 * host VMCS area on a VM exit.
+	 */
+	if (guest_msr_rw(vmx, MSR_GSBASE) ||
+	    guest_msr_rw(vmx, MSR_FSBASE) ||
+	    guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
+	    guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
+	    guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
+	    guest_msr_rw(vmx, MSR_KGSBASE) ||
+	    guest_msr_rw(vmx, MSR_EFER))
+		panic("vmx_vminit: error setting guest msr access");
+
+	/*
+	 * MSR_PAT is saved and restored in the guest VMCS are on a VM exit
+	 * and entry respectively. It is also restored from the host VMCS
+	 * area on a VM exit. However, if running on a system with no
+	 * MSR_PAT save/restore support, leave access disabled so accesses
+	 * will be trapped.
+	 */
+	if (!vmx_no_patmsr && guest_msr_rw(vmx, MSR_PAT))
+		panic("vmx_vminit: error setting guest pat msr access");
+
+	for (i = 0; i < VM_MAXCPU; i++) {
+		vmx->vmcs[i].identifier = vmx_revision();
+		error = vmclear(&vmx->vmcs[i]);
+		if (error != 0) {
+			panic("vmx_vminit: vmclear error %d on vcpu %d\n",
+			      error, i);
+		}
+
+		vpid = vmx_vpid();
+
+		error = vmcs_set_defaults(&vmx->vmcs[i],
+					  (u_long)vmx_longjmp,
+					  (u_long)&vmx->ctx[i],
+					  vtophys(vmx->pml4ept),
+					  pinbased_ctls,
+					  procbased_ctls,
+					  procbased_ctls2,
+					  exit_ctls, entry_ctls,
+					  vtophys(vmx->msr_bitmap),
+					  vpid);
+
+		if (error != 0)
+			panic("vmx_vminit: vmcs_set_defaults error %d", error);
+
+		vmx->cap[i].set = 0;
+		vmx->cap[i].proc_ctls = procbased_ctls;
+
+		vmx->state[i].lastcpu = -1;
+		vmx->state[i].vpid = vpid;
+
+		msr_save_area_init(vmx->guest_msrs[i], &guest_msr_count);
+
+		error = vmcs_set_msr_save(&vmx->vmcs[i],
+					  vtophys(vmx->guest_msrs[i]),
+					  guest_msr_count);
+		if (error != 0)
+			panic("vmcs_set_msr_save error %d", error);
+
+		error = vmx_setup_cr0_shadow(&vmx->vmcs[i]);
+		if (error != 0)
+			panic("vmx_setup_cr0_shadow %d", error);
+
+		error = vmx_setup_cr4_shadow(&vmx->vmcs[i]);
+		if (error != 0)
+			panic("vmx_setup_cr4_shadow %d", error);
+	}
+
+	return (vmx);
+}
+
+static int
+vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx)
+{
+	int handled, func;
+	
+	func = vmxctx->guest_rax;
+
+	handled = x86_emulate_cpuid(vm, vcpu,
+				    (uint32_t*)(&vmxctx->guest_rax),
+				    (uint32_t*)(&vmxctx->guest_rbx),
+				    (uint32_t*)(&vmxctx->guest_rcx),
+				    (uint32_t*)(&vmxctx->guest_rdx));
+	return (handled);
+}
+
+static __inline void
+vmx_run_trace(struct vmx *vmx, int vcpu)
+{
+#ifdef KTR
+	VMM_CTR1(vmx->vm, vcpu, "Resume execution at 0x%0lx", vmcs_guest_rip());
+#endif
+}
+
+static __inline void
+vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
+	       int handled)
+{
+#ifdef KTR
+	VMM_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx",
+		 handled ? "handled" : "unhandled",
+		 exit_reason_to_str(exit_reason), rip);
+#endif
+}
+
+static __inline void
+vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip)
+{
+#ifdef KTR
+	VMM_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip);
+#endif
+}
+
+static int
+vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu)
+{
+	int error, lastcpu;
+	struct vmxstate *vmxstate;
+	struct invvpid_desc invvpid_desc = { 0 };
+
+	vmxstate = &vmx->state[vcpu];
+	lastcpu = vmxstate->lastcpu;
+	vmxstate->lastcpu = curcpu;
+
+	if (lastcpu == curcpu) {
+		error = 0;
+		goto done;
+	}
+
+	vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
+
+	error = vmwrite(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
+	if (error != 0)
+		goto done;
+
+	error = vmwrite(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
+	if (error != 0)
+		goto done;
+
+	error = vmwrite(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
+	if (error != 0)
+		goto done;
+
+	/*
+	 * If we are using VPIDs then invalidate all mappings tagged with 'vpid'
+	 *
+	 * We do this because this vcpu was executing on a different host
+	 * cpu when it last ran. We do not track whether it invalidated
+	 * mappings associated with its 'vpid' during that run. So we must
+	 * assume that the mappings associated with 'vpid' on 'curcpu' are
+	 * stale and invalidate them.
+	 *
+	 * Note that we incur this penalty only when the scheduler chooses to
+	 * move the thread associated with this vcpu between host cpus.
+	 *
+	 * Note also that this will invalidate mappings tagged with 'vpid'
+	 * for "all" EP4TAs.
+	 */
+	if (vmxstate->vpid != 0) {
+		invvpid_desc.vpid = vmxstate->vpid;
+		invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
+	}
+done:
+	return (error);
+}
+
+static void 
+vm_exit_update_rip(struct vm_exit *vmexit)
+{
+	int error;
+
+	error = vmwrite(VMCS_GUEST_RIP, vmexit->rip + vmexit->inst_length);
+	if (error)
+		panic("vmx_run: error %d writing to VMCS_GUEST_RIP", error);
+}
+
+/*
+ * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
+ */
+CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
+
+static void __inline
+vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
+{
+	int error;
+
+	vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
+
+	error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
+	if (error)
+		panic("vmx_set_int_window_exiting: vmwrite error %d", error);
+}
+
+static void __inline
+vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
+{
+	int error;
+
+	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
+
+	error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
+	if (error)
+		panic("vmx_clear_int_window_exiting: vmwrite error %d", error);
+}
+
+static void __inline
+vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
+{
+	int error;
+
+	vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
+
+	error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
+	if (error)
+		panic("vmx_set_nmi_window_exiting: vmwrite error %d", error);
+}
+
+static void __inline
+vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
+{
+	int error;
+
+	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
+
+	error = vmwrite(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
+	if (error)
+		panic("vmx_clear_nmi_window_exiting: vmwrite error %d", error);
+}
+
+static int
+vmx_inject_nmi(struct vmx *vmx, int vcpu)
+{
+	int error;
+	uint64_t info, interruptibility;
+
+	/* Bail out if no NMI requested */
+	if (!vm_nmi_pending(vmx->vm, vcpu))
+		return (0);
+
+	error = vmread(VMCS_GUEST_INTERRUPTIBILITY, &interruptibility);
+	if (error) {
+		panic("vmx_inject_nmi: vmread(interruptibility) %d",
+			error);
+	}
+	if (interruptibility & nmi_blocking_bits)
+		goto nmiblocked;
+
+	/*
+	 * Inject the virtual NMI. The vector must be the NMI IDT entry
+	 * or the VMCS entry check will fail.
+	 */
+	info = VMCS_INTERRUPTION_INFO_NMI | VMCS_INTERRUPTION_INFO_VALID;
+	info |= IDT_NMI;
+
+	error = vmwrite(VMCS_ENTRY_INTR_INFO, info);
+	if (error)
+		panic("vmx_inject_nmi: vmwrite(intrinfo) %d", error);
+
+	VMM_CTR0(vmx->vm, vcpu, "Injecting vNMI");
+
+	/* Clear the request */
+	vm_nmi_clear(vmx->vm, vcpu);
+	return (1);
+
+nmiblocked:
+	/*
+	 * Set the NMI Window Exiting execution control so we can inject
+	 * the virtual NMI as soon as blocking condition goes away.
+	 */
+	vmx_set_nmi_window_exiting(vmx, vcpu);
+
+	VMM_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
+	return (1);
+}
+
+static void
+vmx_inject_interrupts(struct vmx *vmx, int vcpu)
+{
+	int error, vector;
+	uint64_t info, rflags, interruptibility;
+
+	const int HWINTR_BLOCKED = VMCS_INTERRUPTIBILITY_STI_BLOCKING |
+				   VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING;
+
+	/*
+	 * If there is already an interrupt pending then just return.
+	 *
+	 * This could happen if an interrupt was injected on a prior
+	 * VM entry but the actual entry into guest mode was aborted
+	 * because of a pending AST.
+	 */
+	error = vmread(VMCS_ENTRY_INTR_INFO, &info);
+	if (error)
+		panic("vmx_inject_interrupts: vmread(intrinfo) %d", error);
+	if (info & VMCS_INTERRUPTION_INFO_VALID)
+		return;
+
+	/*
+	 * NMI injection has priority so deal with those first
+	 */
+	if (vmx_inject_nmi(vmx, vcpu))
+		return;
+
+	/* Ask the local apic for a vector to inject */
+	vector = lapic_pending_intr(vmx->vm, vcpu);
+	if (vector < 0)
+		return;
+
+	if (vector < 32 || vector > 255)
+		panic("vmx_inject_interrupts: invalid vector %d\n", vector);
+
+	/* Check RFLAGS.IF and the interruptibility state of the guest */
+	error = vmread(VMCS_GUEST_RFLAGS, &rflags);
+	if (error)
+		panic("vmx_inject_interrupts: vmread(rflags) %d", error);
+
+	if ((rflags & PSL_I) == 0)
+		goto cantinject;
+
+	error = vmread(VMCS_GUEST_INTERRUPTIBILITY, &interruptibility);
+	if (error) {
+		panic("vmx_inject_interrupts: vmread(interruptibility) %d",
+			error);
+	}
+	if (interruptibility & HWINTR_BLOCKED)
+		goto cantinject;
+
+	/* Inject the interrupt */
+	info = VMCS_INTERRUPTION_INFO_HW_INTR | VMCS_INTERRUPTION_INFO_VALID;
+	info |= vector;
+	error = vmwrite(VMCS_ENTRY_INTR_INFO, info);
+	if (error)
+		panic("vmx_inject_interrupts: vmwrite(intrinfo) %d", error);
+
+	/* Update the Local APIC ISR */
+	lapic_intr_accepted(vmx->vm, vcpu, vector);
+
+	VMM_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
+
+	return;
+
+cantinject:
+	/*
+	 * Set the Interrupt Window Exiting execution control so we can inject
+	 * the interrupt as soon as blocking condition goes away.
+	 */
+	vmx_set_int_window_exiting(vmx, vcpu);
+
+	VMM_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
+}
+
+static int
+vmx_emulate_cr_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
+{
+	int error, cr, vmcs_guest_cr;
+	uint64_t regval, ones_mask, zeros_mask;
+	const struct vmxctx *vmxctx;
+
+	/* We only handle mov to %cr0 or %cr4 at this time */
+	if ((exitqual & 0xf0) != 0x00)
+		return (UNHANDLED);
+
+	cr = exitqual & 0xf;
+	if (cr != 0 && cr != 4)
+		return (UNHANDLED);
+
+	vmxctx = &vmx->ctx[vcpu];
+
+	/*
+	 * We must use vmwrite() directly here because vmcs_setreg() will
+	 * call vmclear(vmcs) as a side-effect which we certainly don't want.
+	 */
+	switch ((exitqual >> 8) & 0xf) {
+	case 0:
+		regval = vmxctx->guest_rax;
+		break;
+	case 1:
+		regval = vmxctx->guest_rcx;
+		break;
+	case 2:
+		regval = vmxctx->guest_rdx;
+		break;
+	case 3:
+		regval = vmxctx->guest_rbx;
+		break;
+	case 4:
+		error = vmread(VMCS_GUEST_RSP, &regval);
+		if (error) {
+			panic("vmx_emulate_cr_access: "
+			      "error %d reading guest rsp", error);
+		}
+		break;
+	case 5:
+		regval = vmxctx->guest_rbp;
+		break;
+	case 6:
+		regval = vmxctx->guest_rsi;
+		break;
+	case 7:
+		regval = vmxctx->guest_rdi;
+		break;
+	case 8:
+		regval = vmxctx->guest_r8;
+		break;
+	case 9:
+		regval = vmxctx->guest_r9;
+		break;
+	case 10:
+		regval = vmxctx->guest_r10;
+		break;
+	case 11:
+		regval = vmxctx->guest_r11;
+		break;
+	case 12:
+		regval = vmxctx->guest_r12;
+		break;
+	case 13:
+		regval = vmxctx->guest_r13;
+		break;
+	case 14:
+		regval = vmxctx->guest_r14;
+		break;
+	case 15:
+		regval = vmxctx->guest_r15;
+		break;
+	}
+
+	if (cr == 0) {
+		ones_mask = cr0_ones_mask;
+		zeros_mask = cr0_zeros_mask;
+		vmcs_guest_cr = VMCS_GUEST_CR0;
+	} else {
+		ones_mask = cr4_ones_mask;
+		zeros_mask = cr4_zeros_mask;
+		vmcs_guest_cr = VMCS_GUEST_CR4;
+	}
+	regval |= ones_mask;
+	regval &= ~zeros_mask;
+	error = vmwrite(vmcs_guest_cr, regval);
+	if (error) {
+		panic("vmx_emulate_cr_access: error %d writing cr%d",
+		      error, cr);
+	}
+
+	return (HANDLED);
+}
+
+static int
+vmx_ept_fault(struct vm *vm, int cpu,
+	      uint64_t gla, uint64_t gpa, uint64_t rip, int inst_length,
+	      uint64_t cr3, uint64_t ept_qual, struct vie *vie)
+{
+	int read, write, error;
+
+	/* EPT violation on an instruction fetch doesn't make sense here */
+	if (ept_qual & EPT_VIOLATION_INST_FETCH)
+		return (UNHANDLED);
+
+	/* EPT violation must be a read fault or a write fault */
+	read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
+	write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
+	if ((read | write) == 0)
+		return (UNHANDLED);
+
+	/*
+	 * The EPT violation must have been caused by accessing a
+	 * guest-physical address that is a translation of a guest-linear
+	 * address.
+	 */
+	if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
+	    (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
+		return (UNHANDLED);
+	}
+
+	/* Fetch, decode and emulate the faulting instruction */
+	if (vmm_fetch_instruction(vm, cpu, rip, inst_length, cr3, vie) != 0)
+		return (UNHANDLED);
+
+	if (vmm_decode_instruction(vm, cpu, gla, vie) != 0)
+		return (UNHANDLED);
+
+	/*
+	 * Check if this is a local apic access
+	 */
+	if (gpa < DEFAULT_APIC_BASE || gpa >= DEFAULT_APIC_BASE + PAGE_SIZE)
+		return (UNHANDLED);
+
+	error = vmm_emulate_instruction(vm, cpu, gpa, vie,
+					lapic_mmio_read, lapic_mmio_write, 0);
+
+	return (error ? UNHANDLED : HANDLED);
+}
+
+static int
+vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
+{
+	int error, handled;
+	struct vmcs *vmcs;
+	struct vmxctx *vmxctx;
+	uint32_t eax, ecx, edx;
+	uint64_t qual, gla, gpa, cr3, intr_info;
+
+	handled = 0;
+	vmcs = &vmx->vmcs[vcpu];
+	vmxctx = &vmx->ctx[vcpu];
+	qual = vmexit->u.vmx.exit_qualification;
+	vmexit->exitcode = VM_EXITCODE_BOGUS;
+
+	switch (vmexit->u.vmx.exit_reason) {
+	case EXIT_REASON_CR_ACCESS:
+		handled = vmx_emulate_cr_access(vmx, vcpu, qual);
+		break;
+	case EXIT_REASON_RDMSR:
+		ecx = vmxctx->guest_rcx;
+		error = emulate_rdmsr(vmx->vm, vcpu, ecx);
+		if (error) {
+			vmexit->exitcode = VM_EXITCODE_RDMSR;
+			vmexit->u.msr.code = ecx;
+		} else
+			handled = 1;
+		break;
+	case EXIT_REASON_WRMSR:
+		eax = vmxctx->guest_rax;
+		ecx = vmxctx->guest_rcx;
+		edx = vmxctx->guest_rdx;
+		error = emulate_wrmsr(vmx->vm, vcpu, ecx,
+					(uint64_t)edx << 32 | eax);
+		if (error) {
+			vmexit->exitcode = VM_EXITCODE_WRMSR;
+			vmexit->u.msr.code = ecx;
+			vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
+		} else
+			handled = 1;
+		break;
+	case EXIT_REASON_HLT:
+		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
+		/*
+		 * If there is an event waiting to be injected then there is
+		 * no need to 'hlt'.
+		 */
+		error = vmread(VMCS_ENTRY_INTR_INFO, &intr_info);
+		if (error)
+			panic("vmx_exit_process: vmread(intrinfo) %d", error);
+
+		if (intr_info & VMCS_INTERRUPTION_INFO_VALID) {
+			handled = 1;
+			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT_IGNORED, 1);
+		} else
+			vmexit->exitcode = VM_EXITCODE_HLT;
+		break;
+	case EXIT_REASON_MTF:
+		vmexit->exitcode = VM_EXITCODE_MTRAP;
+		break;
+	case EXIT_REASON_PAUSE:
+		vmexit->exitcode = VM_EXITCODE_PAUSE;
+		break;
+	case EXIT_REASON_INTR_WINDOW:
+		vmx_clear_int_window_exiting(vmx, vcpu);
+		VMM_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
+		/* FALLTHRU */
+	case EXIT_REASON_EXT_INTR:
+		/*
+		 * External interrupts serve only to cause VM exits and allow
+		 * the host interrupt handler to run.
+		 *
+		 * If this external interrupt triggers a virtual interrupt
+		 * to a VM, then that state will be recorded by the
+		 * host interrupt handler in the VM's softc. We will inject
+		 * this virtual interrupt during the subsequent VM enter.
+		 */
+
+		/*
+		 * This is special. We want to treat this as an 'handled'
+		 * VM-exit but not increment the instruction pointer.
+		 */
+		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
+		return (1);
+	case EXIT_REASON_NMI_WINDOW:
+		/* Exit to allow the pending virtual NMI to be injected */
+		vmx_clear_nmi_window_exiting(vmx, vcpu);
+		VMM_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
+		return (1);
+	case EXIT_REASON_INOUT:
+		vmexit->exitcode = VM_EXITCODE_INOUT;
+		vmexit->u.inout.bytes = (qual & 0x7) + 1;
+		vmexit->u.inout.in = (qual & 0x8) ? 1 : 0;
+		vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
+		vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
+		vmexit->u.inout.port = (uint16_t)(qual >> 16);
+		vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
+		break;
+	case EXIT_REASON_CPUID:
+		handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx);
+		break;
+	case EXIT_REASON_EPT_FAULT:
+		gla = vmcs_gla();
+		gpa = vmcs_gpa();
+		cr3 = vmcs_guest_cr3();
+		handled = vmx_ept_fault(vmx->vm, vcpu, gla, gpa,
+					vmexit->rip, vmexit->inst_length,
+					cr3, qual, &vmexit->u.paging.vie);
+		if (!handled) {
+			vmexit->exitcode = VM_EXITCODE_PAGING;
+			vmexit->u.paging.gpa = gpa;
+		}
+		break;
+	default:
+		break;
+	}
+
+	if (handled) {
+		/*
+		 * It is possible that control is returned to userland
+		 * even though we were able to handle the VM exit in the
+		 * kernel.
+		 *
+		 * In such a case we want to make sure that the userland
+		 * restarts guest execution at the instruction *after*
+		 * the one we just processed. Therefore we update the
+		 * guest rip in the VMCS and in 'vmexit'.
+		 */
+		vm_exit_update_rip(vmexit);
+		vmexit->rip += vmexit->inst_length;
+		vmexit->inst_length = 0;
+
+		/*
+		 * Special case for spinning up an AP - exit to userspace to
+		 * give the controlling process a chance to intercept and
+		 * spin up a thread for the AP.
+		 */
+		if (vmexit->exitcode == VM_EXITCODE_SPINUP_AP)
+			handled = 0;
+	} else {
+		if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
+			/*
+			 * If this VM exit was not claimed by anybody then
+			 * treat it as a generic VMX exit.
+			 */
+			vmexit->exitcode = VM_EXITCODE_VMX;
+			vmexit->u.vmx.error = 0;
+		} else {
+			/*
+			 * The exitcode and collateral have been populated.
+			 * The VM exit will be processed further in userland.
+			 */
+		}
+	}
+	return (handled);
+}
+
+static int
+vmx_run(void *arg, int vcpu, register_t rip)
+{
+	int error, vie, rc, handled, astpending;
+	uint32_t exit_reason;
+	struct vmx *vmx;
+	struct vmxctx *vmxctx;
+	struct vmcs *vmcs;
+	struct vm_exit *vmexit;
+	
+	vmx = arg;
+	vmcs = &vmx->vmcs[vcpu];
+	vmxctx = &vmx->ctx[vcpu];
+	vmxctx->launched = 0;
+
+	astpending = 0;
+	vmexit = vm_exitinfo(vmx->vm, vcpu);
+
+	/*
+	 * XXX Can we avoid doing this every time we do a vm run?
+	 */
+	VMPTRLD(vmcs);
+
+	/*
+	 * XXX
+	 * We do this every time because we may setup the virtual machine
+	 * from a different process than the one that actually runs it.
+	 *
+	 * If the life of a virtual machine was spent entirely in the context
+	 * of a single process we could do this once in vmcs_set_defaults().
+	 */
+	if ((error = vmwrite(VMCS_HOST_CR3, rcr3())) != 0)
+		panic("vmx_run: error %d writing to VMCS_HOST_CR3", error);
+
+	if ((error = vmwrite(VMCS_GUEST_RIP, rip)) != 0)
+		panic("vmx_run: error %d writing to VMCS_GUEST_RIP", error);
+
+	if ((error = vmx_set_pcpu_defaults(vmx, vcpu)) != 0)
+		panic("vmx_run: error %d setting up pcpu defaults", error);
+
+	do {
+		lapic_timer_tick(vmx->vm, vcpu);
+		vmx_inject_interrupts(vmx, vcpu);
+		vmx_run_trace(vmx, vcpu);
+		rc = vmx_setjmp(vmxctx);
+#ifdef SETJMP_TRACE
+		vmx_setjmp_trace(vmx, vcpu, vmxctx, rc);
+#endif
+		switch (rc) {
+		case VMX_RETURN_DIRECT:
+			if (vmxctx->launched == 0) {
+				vmxctx->launched = 1;
+				vmx_launch(vmxctx);
+			} else
+				vmx_resume(vmxctx);
+			panic("vmx_launch/resume should not return");
+			break;
+		case VMX_RETURN_LONGJMP:
+			break;			/* vm exit */
+		case VMX_RETURN_AST:
+			astpending = 1;
+			break;
+		case VMX_RETURN_VMRESUME:
+			vie = vmcs_instruction_error();
+			if (vmxctx->launch_error == VM_FAIL_INVALID ||
+			    vie != VMRESUME_WITH_NON_LAUNCHED_VMCS) {
+				printf("vmresume error %d vmcs inst error %d\n",
+					vmxctx->launch_error, vie);
+				goto err_exit;
+			}
+			vmx_launch(vmxctx);	/* try to launch the guest */
+			panic("vmx_launch should not return");
+			break;
+		case VMX_RETURN_VMLAUNCH:
+			vie = vmcs_instruction_error();
+#if 1
+			printf("vmlaunch error %d vmcs inst error %d\n",
+				vmxctx->launch_error, vie);
+#endif
+			goto err_exit;
+		default:
+			panic("vmx_setjmp returned %d", rc);
+		}
+		
+		/* enable interrupts */
+		enable_intr();
+
+		/* collect some basic information for VM exit processing */
+		vmexit->rip = rip = vmcs_guest_rip();
+		vmexit->inst_length = vmexit_instruction_length();
+		vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
+		vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
+
+		if (astpending) {
+			handled = 1;
+			vmexit->inst_length = 0;
+			vmexit->exitcode = VM_EXITCODE_BOGUS;
+			vmx_astpending_trace(vmx, vcpu, rip);
+			break;
+		}
+
+		handled = vmx_exit_process(vmx, vcpu, vmexit);
+		vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled);
+
+	} while (handled);
+
+	/*
+	 * If a VM exit has been handled then the exitcode must be BOGUS
+	 * If a VM exit is not handled then the exitcode must not be BOGUS
+	 */
+	if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
+	    (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
+		panic("Mismatch between handled (%d) and exitcode (%d)",
+		      handled, vmexit->exitcode);
+	}
+
+	VMM_CTR1(vmx->vm, vcpu, "goto userland: exitcode %d",vmexit->exitcode);
+
+	/*
+	 * XXX
+	 * We need to do this to ensure that any VMCS state cached by the
+	 * processor is flushed to memory. We need to do this in case the
+	 * VM moves to a different cpu the next time it runs.
+	 *
+	 * Can we avoid doing this?
+	 */
+	VMCLEAR(vmcs);
+	return (0);
+
+err_exit:
+	vmexit->exitcode = VM_EXITCODE_VMX;
+	vmexit->u.vmx.exit_reason = (uint32_t)-1;
+	vmexit->u.vmx.exit_qualification = (uint32_t)-1;
+	vmexit->u.vmx.error = vie;
+	VMCLEAR(vmcs);
+	return (ENOEXEC);
+}
+
+static void
+vmx_vmcleanup(void *arg)
+{
+	int error;
+	struct vmx *vmx = arg;
+
+	/*
+	 * XXXSMP we also need to clear the VMCS active on the other vcpus.
+	 */
+	error = vmclear(&vmx->vmcs[0]);
+	if (error != 0)
+		panic("vmx_vmcleanup: vmclear error %d on vcpu 0", error);
+
+	ept_vmcleanup(vmx);
+	free(vmx, M_VMX);
+
+	return;
+}
+
+static register_t *
+vmxctx_regptr(struct vmxctx *vmxctx, int reg)
+{
+
+	switch (reg) {
+	case VM_REG_GUEST_RAX:
+		return (&vmxctx->guest_rax);
+	case VM_REG_GUEST_RBX:
+		return (&vmxctx->guest_rbx);
+	case VM_REG_GUEST_RCX:
+		return (&vmxctx->guest_rcx);
+	case VM_REG_GUEST_RDX:
+		return (&vmxctx->guest_rdx);
+	case VM_REG_GUEST_RSI:
+		return (&vmxctx->guest_rsi);
+	case VM_REG_GUEST_RDI:
+		return (&vmxctx->guest_rdi);
+	case VM_REG_GUEST_RBP:
+		return (&vmxctx->guest_rbp);
+	case VM_REG_GUEST_R8:
+		return (&vmxctx->guest_r8);
+	case VM_REG_GUEST_R9:
+		return (&vmxctx->guest_r9);
+	case VM_REG_GUEST_R10:
+		return (&vmxctx->guest_r10);
+	case VM_REG_GUEST_R11:
+		return (&vmxctx->guest_r11);
+	case VM_REG_GUEST_R12:
+		return (&vmxctx->guest_r12);
+	case VM_REG_GUEST_R13:
+		return (&vmxctx->guest_r13);
+	case VM_REG_GUEST_R14:
+		return (&vmxctx->guest_r14);
+	case VM_REG_GUEST_R15:
+		return (&vmxctx->guest_r15);
+	default:
+		break;
+	}
+	return (NULL);
+}
+
+static int
+vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
+{
+	register_t *regp;
+
+	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
+		*retval = *regp;
+		return (0);
+	} else
+		return (EINVAL);
+}
+
+static int
+vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
+{
+	register_t *regp;
+
+	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
+		*regp = val;
+		return (0);
+	} else
+		return (EINVAL);
+}
+
+static int
+vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
+{
+	struct vmx *vmx = arg;
+
+	if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0)
+		return (0);
+
+	/*
+	 * If the vcpu is running then don't mess with the VMCS.
+	 *
+	 * vmcs_getreg will VMCLEAR the vmcs when it is done which will cause
+	 * the subsequent vmlaunch/vmresume to fail.
+	 */
+	if (vcpu_is_running(vmx->vm, vcpu))
+		panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
+
+	return (vmcs_getreg(&vmx->vmcs[vcpu], reg, retval));
+}
+
+static int
+vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
+{
+	int error;
+	uint64_t ctls;
+	struct vmx *vmx = arg;
+
+	/*
+	 * XXX Allow caller to set contents of the guest registers saved in
+	 * the 'vmxctx' even though the vcpu might be running. We need this
+	 * specifically to support the rdmsr emulation that will set the
+	 * %eax and %edx registers during vm exit processing.
+	 */
+	if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0)
+		return (0);
+
+	/*
+	 * If the vcpu is running then don't mess with the VMCS.
+	 *
+	 * vmcs_setreg will VMCLEAR the vmcs when it is done which will cause
+	 * the subsequent vmlaunch/vmresume to fail.
+	 */
+	if (vcpu_is_running(vmx->vm, vcpu))
+		panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
+
+	error = vmcs_setreg(&vmx->vmcs[vcpu], reg, val);
+
+	if (error == 0) {
+		/*
+		 * If the "load EFER" VM-entry control is 1 then the
+		 * value of EFER.LMA must be identical to "IA-32e mode guest"
+		 * bit in the VM-entry control.
+		 */
+		if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
+		    (reg == VM_REG_GUEST_EFER)) {
+			vmcs_getreg(&vmx->vmcs[vcpu],
+				    VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
+			if (val & EFER_LMA)
+				ctls |= VM_ENTRY_GUEST_LMA;
+			else
+				ctls &= ~VM_ENTRY_GUEST_LMA;
+			vmcs_setreg(&vmx->vmcs[vcpu],
+				    VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
+		}
+	}
+
+	return (error);
+}
+
+static int
+vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
+{
+	struct vmx *vmx = arg;
+
+	return (vmcs_getdesc(&vmx->vmcs[vcpu], reg, desc));
+}
+
+static int
+vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
+{
+	struct vmx *vmx = arg;
+
+	return (vmcs_setdesc(&vmx->vmcs[vcpu], reg, desc));
+}
+
+static int
+vmx_inject(void *arg, int vcpu, int type, int vector, uint32_t code,
+	   int code_valid)
+{
+	int error;
+	uint64_t info;
+	struct vmx *vmx = arg;
+	struct vmcs *vmcs = &vmx->vmcs[vcpu];
+
+	static uint32_t type_map[VM_EVENT_MAX] = {
+		0x1,		/* VM_EVENT_NONE */
+		0x0,		/* VM_HW_INTR */
+		0x2,		/* VM_NMI */
+		0x3,		/* VM_HW_EXCEPTION */
+		0x4,		/* VM_SW_INTR */
+		0x5,		/* VM_PRIV_SW_EXCEPTION */
+		0x6,		/* VM_SW_EXCEPTION */
+	};
+
+	/*
+	 * If there is already an exception pending to be delivered to the
+	 * vcpu then just return.
+	 */
+	error = vmcs_getreg(vmcs, VMCS_IDENT(VMCS_ENTRY_INTR_INFO), &info);
+	if (error)
+		return (error);
+
+	if (info & VMCS_INTERRUPTION_INFO_VALID)
+		return (EAGAIN);
+
+	info = vector | (type_map[type] << 8) | (code_valid ? 1 << 11 : 0);
+	info |= VMCS_INTERRUPTION_INFO_VALID;
+	error = vmcs_setreg(vmcs, VMCS_IDENT(VMCS_ENTRY_INTR_INFO), info);
+	if (error != 0)
+		return (error);
+
+	if (code_valid) {
+		error = vmcs_setreg(vmcs,
+				    VMCS_IDENT(VMCS_ENTRY_EXCEPTION_ERROR),
+				    code);
+	}
+	return (error);
+}
+
+static int
+vmx_getcap(void *arg, int vcpu, int type, int *retval)
+{
+	struct vmx *vmx = arg;
+	int vcap;
+	int ret;
+
+	ret = ENOENT;
+
+	vcap = vmx->cap[vcpu].set;
+
+	switch (type) {
+	case VM_CAP_HALT_EXIT:
+		if (cap_halt_exit)
+			ret = 0;
+		break;
+	case VM_CAP_PAUSE_EXIT:
+		if (cap_pause_exit)
+			ret = 0;
+		break;
+	case VM_CAP_MTRAP_EXIT:
+		if (cap_monitor_trap)
+			ret = 0;
+		break;
+	case VM_CAP_UNRESTRICTED_GUEST:
+		if (cap_unrestricted_guest)
+			ret = 0;
+		break;
+	default:
+		break;
+	}
+
+	if (ret == 0)
+		*retval = (vcap & (1 << type)) ? 1 : 0;
+
+	return (ret);
+}
+
+static int
+vmx_setcap(void *arg, int vcpu, int type, int val)
+{
+	struct vmx *vmx = arg;
+	struct vmcs *vmcs = &vmx->vmcs[vcpu];
+	uint32_t baseval;
+	uint32_t *pptr;
+	int error;
+	int flag;
+	int reg;
+	int retval;
+
+	retval = ENOENT;
+	pptr = NULL;
+
+	switch (type) {
+	case VM_CAP_HALT_EXIT:
+		if (cap_halt_exit) {
+			retval = 0;
+			pptr = &vmx->cap[vcpu].proc_ctls;
+			baseval = *pptr;
+			flag = PROCBASED_HLT_EXITING;
+			reg = VMCS_PRI_PROC_BASED_CTLS;
+		}
+		break;
+	case VM_CAP_MTRAP_EXIT:
+		if (cap_monitor_trap) {
+			retval = 0;
+			pptr = &vmx->cap[vcpu].proc_ctls;
+			baseval = *pptr;
+			flag = PROCBASED_MTF;
+			reg = VMCS_PRI_PROC_BASED_CTLS;
+		}
+		break;
+	case VM_CAP_PAUSE_EXIT:
+		if (cap_pause_exit) {
+			retval = 0;
+			pptr = &vmx->cap[vcpu].proc_ctls;
+			baseval = *pptr;
+			flag = PROCBASED_PAUSE_EXITING;
+			reg = VMCS_PRI_PROC_BASED_CTLS;
+		}
+		break;
+	case VM_CAP_UNRESTRICTED_GUEST:
+		if (cap_unrestricted_guest) {
+			retval = 0;
+			baseval = procbased_ctls2;
+			flag = PROCBASED2_UNRESTRICTED_GUEST;
+			reg = VMCS_SEC_PROC_BASED_CTLS;
+		}
+		break;
+	default:
+		break;
+	}
+
+	if (retval == 0) {
+		if (val) {
+			baseval |= flag;
+		} else {
+			baseval &= ~flag;
+		}
+		VMPTRLD(vmcs);
+		error = vmwrite(reg, baseval);
+		VMCLEAR(vmcs);
+
+		if (error) {
+			retval = error;
+		} else {
+			/*
+			 * Update optional stored flags, and record
+			 * setting
+			 */
+			if (pptr != NULL) {
+				*pptr = baseval;
+			}
+
+			if (val) {
+				vmx->cap[vcpu].set |= (1 << type);
+			} else {
+				vmx->cap[vcpu].set &= ~(1 << type);
+			}
+		}
+	}
+
+        return (retval);
+}
+
+struct vmm_ops vmm_ops_intel = {
+	vmx_init,
+	vmx_cleanup,
+	vmx_vminit,
+	vmx_run,
+	vmx_vmcleanup,
+	ept_vmmmap_set,
+	ept_vmmmap_get,
+	vmx_getreg,
+	vmx_setreg,
+	vmx_getdesc,
+	vmx_setdesc,
+	vmx_inject,
+	vmx_getcap,
+	vmx_setcap
+};
diff --git a/sys/amd64/vmm/intel/vmx.h b/sys/amd64/vmm/intel/vmx.h
new file mode 100644
index 0000000..c7cd567
--- /dev/null
+++ b/sys/amd64/vmm/intel/vmx.h
@@ -0,0 +1,120 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VMX_H_
+#define	_VMX_H_
+
+#include "vmcs.h"
+
+#define	GUEST_MSR_MAX_ENTRIES	64		/* arbitrary */
+
+struct vmxctx {
+	register_t	tmpstk[32];		/* vmx_return() stack */
+	register_t	tmpstktop;
+
+	register_t	guest_rdi;		/* Guest state */
+	register_t	guest_rsi;
+	register_t	guest_rdx;
+	register_t	guest_rcx;
+	register_t	guest_r8;
+	register_t	guest_r9;
+	register_t	guest_rax;
+	register_t	guest_rbx;
+	register_t	guest_rbp;
+	register_t	guest_r10;
+	register_t	guest_r11;
+	register_t	guest_r12;
+	register_t	guest_r13;
+	register_t	guest_r14;
+	register_t	guest_r15;
+	register_t	guest_cr2;
+
+	register_t	host_r15;		/* Host state */
+	register_t	host_r14;
+	register_t	host_r13;
+	register_t	host_r12;
+	register_t	host_rbp;
+	register_t	host_rsp;
+	register_t	host_rbx;
+	register_t	host_rip;
+	/*
+	 * XXX todo debug registers and fpu state
+	 */
+	
+	int		launched;		/* vmcs launch state */
+	int		launch_error;
+};
+
+struct vmxcap {
+	int	set;
+	uint32_t proc_ctls;
+};
+
+struct vmxstate {
+	int	lastcpu;	/* host cpu that this 'vcpu' last ran on */
+	uint16_t vpid;
+};
+
+/* virtual machine softc */
+struct vmx {
+	pml4_entry_t	pml4ept[NPML4EPG];
+	struct vmcs	vmcs[VM_MAXCPU];	/* one vmcs per virtual cpu */
+	char		msr_bitmap[PAGE_SIZE];
+	struct msr_entry guest_msrs[VM_MAXCPU][GUEST_MSR_MAX_ENTRIES];
+	struct vmxctx	ctx[VM_MAXCPU];
+	struct vmxcap	cap[VM_MAXCPU];
+	struct vmxstate	state[VM_MAXCPU];
+	struct vm	*vm;
+};
+CTASSERT((offsetof(struct vmx, pml4ept) & PAGE_MASK) == 0);
+CTASSERT((offsetof(struct vmx, vmcs) & PAGE_MASK) == 0);
+CTASSERT((offsetof(struct vmx, msr_bitmap) & PAGE_MASK) == 0);
+CTASSERT((offsetof(struct vmx, guest_msrs) & 15) == 0);
+
+#define	VMX_RETURN_DIRECT	0
+#define	VMX_RETURN_LONGJMP	1
+#define	VMX_RETURN_VMRESUME	2
+#define	VMX_RETURN_VMLAUNCH	3
+#define	VMX_RETURN_AST		4
+/*
+ * vmx_setjmp() returns:
+ * - 0 when it returns directly
+ * - 1 when it returns from vmx_longjmp
+ * - 2 when it returns from vmx_resume (which would only be in the error case)
+ * - 3 when it returns from vmx_launch (which would only be in the error case)
+ * - 4 when it returns from vmx_resume or vmx_launch because of AST pending
+ */
+int	vmx_setjmp(struct vmxctx *ctx);
+void	vmx_longjmp(void);			/* returns via vmx_setjmp */
+void	vmx_launch(struct vmxctx *ctx) __dead2;	/* may return via vmx_setjmp */
+void	vmx_resume(struct vmxctx *ctx) __dead2;	/* may return via vmx_setjmp */
+
+u_long	vmx_fix_cr0(u_long cr0);
+u_long	vmx_fix_cr4(u_long cr4);
+
+#endif
diff --git a/sys/amd64/vmm/intel/vmx_controls.h b/sys/amd64/vmm/intel/vmx_controls.h
new file mode 100644
index 0000000..31f29f8
--- /dev/null
+++ b/sys/amd64/vmm/intel/vmx_controls.h
@@ -0,0 +1,92 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VMX_CONTROLS_H_
+#define	_VMX_CONTROLS_H_
+
+/* Pin-Based VM-Execution Controls */
+#define	PINBASED_EXTINT_EXITING		(1 << 0)
+#define	PINBASED_NMI_EXITING		(1 << 3)
+#define	PINBASED_VIRTUAL_NMI		(1 << 5)
+#define	PINBASED_PREMPTION_TIMER	(1 << 6)
+
+/* Primary Processor-Based VM-Execution Controls */
+#define	PROCBASED_INT_WINDOW_EXITING	(1 << 2)
+#define	PROCBASED_TSC_OFFSET		(1 << 3)
+#define	PROCBASED_HLT_EXITING		(1 << 7)
+#define	PROCBASED_INVLPG_EXITING	(1 << 9)
+#define	PROCBASED_MWAIT_EXITING		(1 << 10)
+#define	PROCBASED_RDPMC_EXITING		(1 << 11)
+#define	PROCBASED_RDTSC_EXITING		(1 << 12)
+#define	PROCBASED_CR3_LOAD_EXITING	(1 << 15)
+#define	PROCBASED_CR3_STORE_EXITING	(1 << 16)
+#define	PROCBASED_CR8_LOAD_EXITING	(1 << 19)
+#define	PROCBASED_CR8_STORE_EXITING	(1 << 20)
+#define	PROCBASED_USE_TPR_SHADOW	(1 << 21)
+#define	PROCBASED_NMI_WINDOW_EXITING	(1 << 22)
+#define PROCBASED_MOV_DR_EXITING	(1 << 23)
+#define	PROCBASED_IO_EXITING		(1 << 24)
+#define	PROCBASED_IO_BITMAPS		(1 << 25)
+#define	PROCBASED_MTF			(1 << 27)
+#define	PROCBASED_MSR_BITMAPS		(1 << 28)
+#define	PROCBASED_MONITOR_EXITING	(1 << 29)
+#define	PROCBASED_PAUSE_EXITING		(1 << 30)
+#define	PROCBASED_SECONDARY_CONTROLS	(1 << 31)
+
+/* Secondary Processor-Based VM-Execution Controls */
+#define	PROCBASED2_VIRTUALIZE_APIC	(1 << 0)
+#define	PROCBASED2_ENABLE_EPT		(1 << 1)
+#define	PROCBASED2_DESC_TABLE_EXITING	(1 << 2)
+#define	PROCBASED2_ENABLE_RDTSCP	(1 << 3)
+#define	PROCBASED2_VIRTUALIZE_X2APIC	(1 << 4)
+#define	PROCBASED2_ENABLE_VPID		(1 << 5)
+#define	PROCBASED2_WBINVD_EXITING	(1 << 6)
+#define	PROCBASED2_UNRESTRICTED_GUEST	(1 << 7)
+#define	PROCBASED2_PAUSE_LOOP_EXITING	(1 << 10)
+
+/* VM Exit Controls */
+#define	VM_EXIT_SAVE_DEBUG_CONTROLS	(1 << 2)
+#define	VM_EXIT_HOST_LMA		(1 << 9)
+#define	VM_EXIT_LOAD_PERF_GLOBAL_CTRL	(1 << 12)
+#define	VM_EXIT_ACKNOWLEDGE_INTERRUPT	(1 << 15)
+#define	VM_EXIT_SAVE_PAT		(1 << 18)
+#define	VM_EXIT_LOAD_PAT		(1 << 19)
+#define	VM_EXIT_SAVE_EFER		(1 << 20)
+#define	VM_EXIT_LOAD_EFER		(1 << 21)
+#define	VM_EXIT_SAVE_PREEMPTION_TIMER	(1 << 22)
+
+/* VM Entry Controls */
+#define	VM_ENTRY_LOAD_DEBUG_CONTROLS	(1 << 2)
+#define	VM_ENTRY_GUEST_LMA		(1 << 9)
+#define	VM_ENTRY_INTO_SMM		(1 << 10)
+#define	VM_ENTRY_DEACTIVATE_DUAL_MONITOR (1 << 11)
+#define	VM_ENTRY_LOAD_PERF_GLOBAL_CTRL	(1 << 13)
+#define	VM_ENTRY_LOAD_PAT		(1 << 14)
+#define	VM_ENTRY_LOAD_EFER		(1 << 15)
+
+#endif
diff --git a/sys/amd64/vmm/intel/vmx_cpufunc.h b/sys/amd64/vmm/intel/vmx_cpufunc.h
new file mode 100644
index 0000000..2e66443
--- /dev/null
+++ b/sys/amd64/vmm/intel/vmx_cpufunc.h
@@ -0,0 +1,218 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_VMX_CPUFUNC_H_
+#define	_VMX_CPUFUNC_H_
+
+struct vmcs;
+
+/*
+ * Section 5.2 "Conventions" from Intel Architecture Manual 2B.
+ *
+ *			error
+ * VMsucceed		  0
+ * VMFailInvalid	  1
+ * VMFailValid		  2	see also VMCS VM-Instruction Error Field
+ */
+#define	VM_SUCCESS		0
+#define	VM_FAIL_INVALID		1
+#define	VM_FAIL_VALID		2
+#define	VMX_SET_ERROR_CODE \
+	"	jnc 1f;"						\
+	"	mov $1, %[error];"	/* CF: error = 1 */		\
+	"	jmp 3f;"						\
+	"1:	jnz 2f;"						\
+	"	mov $2, %[error];"	/* ZF: error = 2 */		\
+	"	jmp 3f;"						\
+	"2:	mov $0, %[error];"					\
+	"3:"
+
+/* returns 0 on success and non-zero on failure */
+static __inline int
+vmxon(char *region)
+{
+	int error;
+	uint64_t addr;
+
+	addr = vtophys(region);
+	__asm __volatile("vmxon %[addr];"
+			 VMX_SET_ERROR_CODE
+			 : [error] "=r" (error)
+			 : [addr] "m" (*(uint64_t *)&addr)
+			 : "memory");
+
+	return (error);
+}
+
+/* returns 0 on success and non-zero on failure */
+static __inline int
+vmclear(struct vmcs *vmcs)
+{
+	int error;
+	uint64_t addr;
+
+	addr = vtophys(vmcs);
+	__asm __volatile("vmclear %[addr];"
+			 VMX_SET_ERROR_CODE
+			 : [error] "=r" (error)
+			 : [addr] "m" (*(uint64_t *)&addr)
+			 : "memory");
+	return (error);
+}
+
+static __inline void
+vmxoff(void)
+{
+
+	__asm __volatile("vmxoff");
+}
+
+static __inline void
+vmptrst(uint64_t *addr)
+{
+
+	__asm __volatile("vmptrst %[addr]" :: [addr]"m" (*addr) : "memory");
+}
+
+static __inline int
+vmptrld(struct vmcs *vmcs)
+{
+	int error;
+	uint64_t addr;
+
+	addr = vtophys(vmcs);
+	__asm __volatile("vmptrld %[addr];"
+			 VMX_SET_ERROR_CODE
+			 : [error] "=r" (error)
+			 : [addr] "m" (*(uint64_t *)&addr)
+			 : "memory");
+	return (error);
+}
+
+static __inline int
+vmwrite(uint64_t reg, uint64_t val)
+{
+	int error;
+
+	__asm __volatile("vmwrite %[val], %[reg];"
+			 VMX_SET_ERROR_CODE
+			 : [error] "=r" (error)
+			 : [val] "r" (val), [reg] "r" (reg)
+			 : "memory");
+
+	return (error);
+}
+
+static __inline int
+vmread(uint64_t r, uint64_t *addr)
+{
+	int error;
+
+	__asm __volatile("vmread %[r], %[addr];"
+			 VMX_SET_ERROR_CODE
+			 : [error] "=r" (error)
+			 : [r] "r" (r), [addr] "m" (*addr)
+			 : "memory");
+
+	return (error);
+}
+
+static void __inline
+VMCLEAR(struct vmcs *vmcs)
+{
+	int err;
+
+	err = vmclear(vmcs);
+	if (err != 0)
+		panic("%s: vmclear(%p) error %d", __func__, vmcs, err);
+
+	critical_exit();
+}
+
+static void __inline
+VMPTRLD(struct vmcs *vmcs)
+{
+	int err;
+
+	critical_enter();
+
+	err = vmptrld(vmcs);
+	if (err != 0)
+		panic("%s: vmptrld(%p) error %d", __func__, vmcs, err);
+}
+
+#define	INVVPID_TYPE_ADDRESS		0UL
+#define	INVVPID_TYPE_SINGLE_CONTEXT	1UL
+#define	INVVPID_TYPE_ALL_CONTEXTS	2UL
+
+struct invvpid_desc {
+	uint16_t	vpid;
+	uint16_t	_res1;
+	uint32_t	_res2;
+	uint64_t	linear_addr;
+};
+CTASSERT(sizeof(struct invvpid_desc) == 16);
+
+static void __inline
+invvpid(uint64_t type, struct invvpid_desc desc)
+{
+	int error;
+
+	__asm __volatile("invvpid %[desc], %[type];"
+			 VMX_SET_ERROR_CODE
+			 : [error] "=r" (error)
+			 : [desc] "m" (desc), [type] "r" (type)
+			 : "memory");
+
+	if (error)
+		panic("invvpid error %d", error);
+}
+
+#define	INVEPT_TYPE_SINGLE_CONTEXT	1UL
+#define	INVEPT_TYPE_ALL_CONTEXTS	2UL
+struct invept_desc {
+	uint64_t	eptp;
+	uint64_t	_res;
+};
+CTASSERT(sizeof(struct invept_desc) == 16);
+
+static void __inline
+invept(uint64_t type, struct invept_desc desc)
+{
+	int error;
+
+	__asm __volatile("invept %[desc], %[type];"
+			 VMX_SET_ERROR_CODE
+			 : [error] "=r" (error)
+			 : [desc] "m" (desc), [type] "r" (type)
+			 : "memory");
+
+	if (error)
+		panic("invept error %d", error);
+}
+#endif
diff --git a/sys/amd64/vmm/intel/vmx_genassym.c b/sys/amd64/vmm/intel/vmx_genassym.c
new file mode 100644
index 0000000..823a05d
--- /dev/null
+++ b/sys/amd64/vmm/intel/vmx_genassym.c
@@ -0,0 +1,89 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/proc.h>
+#include <sys/assym.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/pmap.h>
+
+#include <machine/vmm.h>
+#include "vmx.h"
+#include "vmx_cpufunc.h"
+
+ASSYM(VMXCTX_TMPSTKTOP, offsetof(struct vmxctx, tmpstktop));
+ASSYM(VMXCTX_GUEST_RDI, offsetof(struct vmxctx, guest_rdi));
+ASSYM(VMXCTX_GUEST_RSI, offsetof(struct vmxctx, guest_rsi));
+ASSYM(VMXCTX_GUEST_RDX, offsetof(struct vmxctx, guest_rdx));
+ASSYM(VMXCTX_GUEST_RCX, offsetof(struct vmxctx, guest_rcx));
+ASSYM(VMXCTX_GUEST_R8, offsetof(struct vmxctx, guest_r8));
+ASSYM(VMXCTX_GUEST_R9, offsetof(struct vmxctx, guest_r9));
+ASSYM(VMXCTX_GUEST_RAX, offsetof(struct vmxctx, guest_rax));
+ASSYM(VMXCTX_GUEST_RBX, offsetof(struct vmxctx, guest_rbx));
+ASSYM(VMXCTX_GUEST_RBP, offsetof(struct vmxctx, guest_rbp));
+ASSYM(VMXCTX_GUEST_R10, offsetof(struct vmxctx, guest_r10));
+ASSYM(VMXCTX_GUEST_R11, offsetof(struct vmxctx, guest_r11));
+ASSYM(VMXCTX_GUEST_R12, offsetof(struct vmxctx, guest_r12));
+ASSYM(VMXCTX_GUEST_R13, offsetof(struct vmxctx, guest_r13));
+ASSYM(VMXCTX_GUEST_R14, offsetof(struct vmxctx, guest_r14));
+ASSYM(VMXCTX_GUEST_R15, offsetof(struct vmxctx, guest_r15));
+ASSYM(VMXCTX_GUEST_CR2, offsetof(struct vmxctx, guest_cr2));
+
+ASSYM(VMXCTX_HOST_R15, offsetof(struct vmxctx, host_r15));
+ASSYM(VMXCTX_HOST_R14, offsetof(struct vmxctx, host_r14));
+ASSYM(VMXCTX_HOST_R13, offsetof(struct vmxctx, host_r13));
+ASSYM(VMXCTX_HOST_R12, offsetof(struct vmxctx, host_r12));
+ASSYM(VMXCTX_HOST_RBP, offsetof(struct vmxctx, host_rbp));
+ASSYM(VMXCTX_HOST_RSP, offsetof(struct vmxctx, host_rsp));
+ASSYM(VMXCTX_HOST_RBX, offsetof(struct vmxctx, host_rbx));
+ASSYM(VMXCTX_HOST_RIP, offsetof(struct vmxctx, host_rip));
+
+ASSYM(VMXCTX_LAUNCH_ERROR, offsetof(struct vmxctx, launch_error));
+
+ASSYM(VM_SUCCESS,	VM_SUCCESS);
+ASSYM(VM_FAIL_INVALID,	VM_FAIL_INVALID);
+ASSYM(VM_FAIL_VALID,	VM_FAIL_VALID);
+
+ASSYM(VMX_RETURN_DIRECT,	VMX_RETURN_DIRECT);
+ASSYM(VMX_RETURN_LONGJMP,	VMX_RETURN_LONGJMP);
+ASSYM(VMX_RETURN_VMRESUME,	VMX_RETURN_VMRESUME);
+ASSYM(VMX_RETURN_VMLAUNCH,	VMX_RETURN_VMLAUNCH);
+ASSYM(VMX_RETURN_AST,		VMX_RETURN_AST);
+
+ASSYM(TDF_ASTPENDING, TDF_ASTPENDING);
+ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED);
+ASSYM(TD_FLAGS, offsetof(struct thread, td_flags));
+ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread));
diff --git a/sys/amd64/vmm/intel/vmx_msr.c b/sys/amd64/vmm/intel/vmx_msr.c
new file mode 100644
index 0000000..2aba63c
--- /dev/null
+++ b/sys/amd64/vmm/intel/vmx_msr.c
@@ -0,0 +1,172 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+
+#include <machine/cpufunc.h>
+
+#include "vmx_msr.h"
+
+static boolean_t
+vmx_ctl_allows_one_setting(uint64_t msr_val, int bitpos)
+{
+
+	if (msr_val & (1UL << (bitpos + 32)))
+		return (TRUE);
+	else
+		return (FALSE);
+}
+
+static boolean_t
+vmx_ctl_allows_zero_setting(uint64_t msr_val, int bitpos)
+{
+
+	if ((msr_val & (1UL << bitpos)) == 0)
+		return (TRUE);
+	else
+		return (FALSE);
+}
+
+uint32_t
+vmx_revision(void)
+{
+
+	return (rdmsr(MSR_VMX_BASIC) & 0xffffffff);
+}
+
+/*
+ * Generate a bitmask to be used for the VMCS execution control fields.
+ *
+ * The caller specifies what bits should be set to one in 'ones_mask'
+ * and what bits should be set to zero in 'zeros_mask'. The don't-care
+ * bits are set to the default value. The default values are obtained
+ * based on "Algorithm 3" in Section 27.5.1 "Algorithms for Determining
+ * VMX Capabilities".
+ *
+ * Returns zero on success and non-zero on error.
+ */
+int
+vmx_set_ctlreg(int ctl_reg, int true_ctl_reg, uint32_t ones_mask,
+	       uint32_t zeros_mask, uint32_t *retval)
+{
+	int i;
+	uint64_t val, trueval;
+	boolean_t true_ctls_avail, one_allowed, zero_allowed;
+
+	/* We cannot ask the same bit to be set to both '1' and '0' */
+	if ((ones_mask ^ zeros_mask) != (ones_mask | zeros_mask))
+		return (EINVAL);
+
+	if (rdmsr(MSR_VMX_BASIC) & (1UL << 55))
+		true_ctls_avail = TRUE;
+	else
+		true_ctls_avail = FALSE;
+
+	val = rdmsr(ctl_reg);
+	if (true_ctls_avail)
+		trueval = rdmsr(true_ctl_reg);		/* step c */
+	else
+		trueval = val;				/* step a */
+
+	for (i = 0; i < 32; i++) {
+		one_allowed = vmx_ctl_allows_one_setting(trueval, i);
+		zero_allowed = vmx_ctl_allows_zero_setting(trueval, i);
+
+		KASSERT(one_allowed || zero_allowed,
+			("invalid zero/one setting for bit %d of ctl 0x%0x, "
+			 "truectl 0x%0x\n", i, ctl_reg, true_ctl_reg));
+
+		if (zero_allowed && !one_allowed) {		/* b(i),c(i) */
+			if (ones_mask & (1 << i))
+				return (EINVAL);
+			*retval &= ~(1 << i);
+		} else if (one_allowed && !zero_allowed) {	/* b(i),c(i) */
+			if (zeros_mask & (1 << i))
+				return (EINVAL);
+			*retval |= 1 << i;
+		} else {
+			if (zeros_mask & (1 << i))	/* b(ii),c(ii) */
+				*retval &= ~(1 << i);
+			else if (ones_mask & (1 << i)) /* b(ii), c(ii) */
+				*retval |= 1 << i;
+			else if (!true_ctls_avail)
+				*retval &= ~(1 << i);	/* b(iii) */
+			else if (vmx_ctl_allows_zero_setting(val, i))/* c(iii)*/
+				*retval &= ~(1 << i);
+			else if (vmx_ctl_allows_one_setting(val, i)) /* c(iv) */
+				*retval |= 1 << i;
+			else {
+				panic("vmx_set_ctlreg: unable to determine "
+				      "correct value of ctl bit %d for msr "
+				      "0x%0x and true msr 0x%0x", i, ctl_reg,
+				      true_ctl_reg);
+			}
+		}
+	}
+
+	return (0);
+}
+
+void
+msr_bitmap_initialize(char *bitmap)
+{
+
+	memset(bitmap, 0xff, PAGE_SIZE);
+}
+
+int
+msr_bitmap_change_access(char *bitmap, u_int msr, int access)
+{
+	int byte, bit;
+
+	if (msr <= 0x00001FFF)
+		byte = msr / 8;
+	else if (msr >= 0xC0000000 && msr <= 0xC0001FFF)
+		byte = 1024 + (msr - 0xC0000000) / 8;
+	else
+		return (EINVAL);
+
+	bit = msr & 0x7;
+
+	if (access & MSR_BITMAP_ACCESS_READ)
+		bitmap[byte] &= ~(1 << bit);
+	else
+		bitmap[byte] |= 1 << bit;
+
+	byte += 2048;
+	if (access & MSR_BITMAP_ACCESS_WRITE)
+		bitmap[byte] &= ~(1 << bit);
+	else
+		bitmap[byte] |= 1 << bit;
+
+	return (0);
+}
diff --git a/sys/amd64/vmm/intel/vmx_msr.h b/sys/amd64/vmm/intel/vmx_msr.h
new file mode 100644
index 0000000..e6379a9
--- /dev/null
+++ b/sys/amd64/vmm/intel/vmx_msr.h
@@ -0,0 +1,78 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VMX_MSR_H_
+#define	_VMX_MSR_H_
+
+#define	MSR_VMX_BASIC			0x480
+#define	MSR_VMX_EPT_VPID_CAP		0x48C
+
+#define	MSR_VMX_PROCBASED_CTLS		0x482
+#define	MSR_VMX_TRUE_PROCBASED_CTLS	0x48E
+
+#define	MSR_VMX_PINBASED_CTLS		0x481
+#define	MSR_VMX_TRUE_PINBASED_CTLS	0x48D
+
+#define	MSR_VMX_PROCBASED_CTLS2		0x48B
+
+#define	MSR_VMX_EXIT_CTLS		0x483
+#define	MSR_VMX_TRUE_EXIT_CTLS		0x48f
+
+#define	MSR_VMX_ENTRY_CTLS		0x484
+#define	MSR_VMX_TRUE_ENTRY_CTLS		0x490
+
+#define	MSR_VMX_CR0_FIXED0		0x486
+#define	MSR_VMX_CR0_FIXED1		0x487
+
+#define	MSR_VMX_CR4_FIXED0		0x488
+#define	MSR_VMX_CR4_FIXED1		0x489
+
+uint32_t vmx_revision(void);
+
+int vmx_set_ctlreg(int ctl_reg, int true_ctl_reg, uint32_t ones_mask,
+		   uint32_t zeros_mask, uint32_t *retval);
+
+/*
+ * According to Section 21.10.4 "Software Access to Related Structures",
+ * changes to data structures pointed to by the VMCS must be made only when
+ * there is no logical processor with a current VMCS that points to the
+ * data structure.
+ *
+ * This pretty much limits us to configuring the MSR bitmap before VMCS
+ * initialization for SMP VMs. Unless of course we do it the hard way - which
+ * would involve some form of synchronization between the vcpus to vmclear
+ * all VMCSs' that point to the bitmap.
+ */
+#define	MSR_BITMAP_ACCESS_NONE	0x0
+#define	MSR_BITMAP_ACCESS_READ	0x1
+#define	MSR_BITMAP_ACCESS_WRITE	0x2
+#define	MSR_BITMAP_ACCESS_RW	(MSR_BITMAP_ACCESS_READ|MSR_BITMAP_ACCESS_WRITE)
+void	msr_bitmap_initialize(char *bitmap);
+int	msr_bitmap_change_access(char *bitmap, u_int msr, int access);
+
+#endif
diff --git a/sys/amd64/vmm/intel/vmx_support.S b/sys/amd64/vmm/intel/vmx_support.S
new file mode 100644
index 0000000..4ba582a
--- /dev/null
+++ b/sys/amd64/vmm/intel/vmx_support.S
@@ -0,0 +1,246 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <machine/asmacros.h>
+
+#include "vmx_assym.s"
+
+/*
+ * Disable interrupts before updating %rsp in VMX_CHECK_AST or
+ * VMX_GUEST_RESTORE.
+ *
+ * The location that %rsp points to is a 'vmxctx' and not a
+ * real stack so we don't want an interrupt handler to trash it
+ */
+#define	VMX_DISABLE_INTERRUPTS		cli
+
+/*
+ * If the thread hosting the vcpu has an ast pending then take care of it
+ * by returning from vmx_setjmp() with a return value of VMX_RETURN_AST.
+ *
+ * Assumes that %rdi holds a pointer to the 'vmxctx' and that interrupts
+ * are disabled.
+ */
+#define	VMX_CHECK_AST							\
+	movq	PCPU(CURTHREAD),%rax;					\
+	testl	$TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax);	\
+	je	9f;							\
+	movq	$VMX_RETURN_AST,%rsi;					\
+	movq	%rdi,%rsp;						\
+	addq	$VMXCTX_TMPSTKTOP,%rsp;					\
+	callq	vmx_return;						\
+9:
+
+/*
+ * Assumes that %rdi holds a pointer to the 'vmxctx'.
+ *
+ * On "return" all registers are updated to reflect guest state. The two
+ * exceptions are %rip and %rsp. These registers are atomically switched
+ * by hardware from the guest area of the vmcs.
+ *
+ * We modify %rsp to point to the 'vmxctx' so we can use it to restore
+ * host context in case of an error with 'vmlaunch' or 'vmresume'.
+ */
+#define	VMX_GUEST_RESTORE						\
+	movq	%rdi,%rsp;						\
+	movq	VMXCTX_GUEST_CR2(%rdi),%rsi;				\
+	movq	%rsi,%cr2;						\
+	movq	VMXCTX_GUEST_RSI(%rdi),%rsi;				\
+	movq	VMXCTX_GUEST_RDX(%rdi),%rdx;				\
+	movq	VMXCTX_GUEST_RCX(%rdi),%rcx;				\
+	movq	VMXCTX_GUEST_R8(%rdi),%r8;				\
+	movq	VMXCTX_GUEST_R9(%rdi),%r9;				\
+	movq	VMXCTX_GUEST_RAX(%rdi),%rax;				\
+	movq	VMXCTX_GUEST_RBX(%rdi),%rbx;				\
+	movq	VMXCTX_GUEST_RBP(%rdi),%rbp;				\
+	movq	VMXCTX_GUEST_R10(%rdi),%r10;				\
+	movq	VMXCTX_GUEST_R11(%rdi),%r11;				\
+	movq	VMXCTX_GUEST_R12(%rdi),%r12;				\
+	movq	VMXCTX_GUEST_R13(%rdi),%r13;				\
+	movq	VMXCTX_GUEST_R14(%rdi),%r14;				\
+	movq	VMXCTX_GUEST_R15(%rdi),%r15;				\
+	movq	VMXCTX_GUEST_RDI(%rdi),%rdi; /* restore rdi the last */
+
+#define	VM_INSTRUCTION_ERROR(reg)					\
+	jnc 	1f;							\
+	movl 	$VM_FAIL_INVALID,reg;		/* CF is set */		\
+	jmp 	3f;							\
+1:	jnz 	2f;							\
+	movl 	$VM_FAIL_VALID,reg;		/* ZF is set */		\
+	jmp 	3f;							\
+2:	movl 	$VM_SUCCESS,reg;					\
+3:	movl	reg,VMXCTX_LAUNCH_ERROR(%rsp)
+
+	.text
+/*
+ * int vmx_setjmp(ctxp)
+ * %rdi = ctxp
+ *
+ * Return value is '0' when it returns directly from here.
+ * Return value is '1' when it returns after a vm exit through vmx_longjmp.
+ */
+ENTRY(vmx_setjmp)
+	movq	(%rsp),%rax			/* return address */
+	movq    %r15,VMXCTX_HOST_R15(%rdi)
+	movq    %r14,VMXCTX_HOST_R14(%rdi)
+	movq    %r13,VMXCTX_HOST_R13(%rdi)
+	movq    %r12,VMXCTX_HOST_R12(%rdi)
+	movq    %rbp,VMXCTX_HOST_RBP(%rdi)
+	movq    %rsp,VMXCTX_HOST_RSP(%rdi)
+	movq    %rbx,VMXCTX_HOST_RBX(%rdi)
+	movq    %rax,VMXCTX_HOST_RIP(%rdi)
+
+	/*
+	 * XXX save host debug registers
+	 */
+	movl	$VMX_RETURN_DIRECT,%eax
+	ret
+END(vmx_setjmp)
+
+/*
+ * void vmx_return(struct vmxctx *ctxp, int retval)
+ * %rdi = ctxp
+ * %rsi = retval
+ * Return to vmm context through vmx_setjmp() with a value of 'retval'.
+ */
+ENTRY(vmx_return)
+	/* Restore host context. */
+	movq	VMXCTX_HOST_R15(%rdi),%r15
+	movq	VMXCTX_HOST_R14(%rdi),%r14
+	movq	VMXCTX_HOST_R13(%rdi),%r13
+	movq	VMXCTX_HOST_R12(%rdi),%r12
+	movq	VMXCTX_HOST_RBP(%rdi),%rbp
+	movq	VMXCTX_HOST_RSP(%rdi),%rsp
+	movq	VMXCTX_HOST_RBX(%rdi),%rbx
+	movq	VMXCTX_HOST_RIP(%rdi),%rax
+	movq	%rax,(%rsp)			/* return address */
+
+	/*
+	 * XXX restore host debug registers
+	 */
+	movl	%esi,%eax
+	ret
+END(vmx_return)
+
+/*
+ * void vmx_longjmp(void)
+ * %rsp points to the struct vmxctx
+ */
+ENTRY(vmx_longjmp)
+	/*
+	 * Save guest state that is not automatically saved in the vmcs.
+	 */
+	movq	%rdi,VMXCTX_GUEST_RDI(%rsp)
+	movq	%rsi,VMXCTX_GUEST_RSI(%rsp)
+	movq	%rdx,VMXCTX_GUEST_RDX(%rsp)
+	movq	%rcx,VMXCTX_GUEST_RCX(%rsp)
+	movq	%r8,VMXCTX_GUEST_R8(%rsp)
+	movq	%r9,VMXCTX_GUEST_R9(%rsp)
+	movq	%rax,VMXCTX_GUEST_RAX(%rsp)
+	movq	%rbx,VMXCTX_GUEST_RBX(%rsp)
+	movq	%rbp,VMXCTX_GUEST_RBP(%rsp)
+	movq	%r10,VMXCTX_GUEST_R10(%rsp)
+	movq	%r11,VMXCTX_GUEST_R11(%rsp)
+	movq	%r12,VMXCTX_GUEST_R12(%rsp)
+	movq	%r13,VMXCTX_GUEST_R13(%rsp)
+	movq	%r14,VMXCTX_GUEST_R14(%rsp)
+	movq	%r15,VMXCTX_GUEST_R15(%rsp)
+
+	movq	%cr2,%rdi
+	movq	%rdi,VMXCTX_GUEST_CR2(%rsp)
+
+	movq	%rsp,%rdi
+	movq	$VMX_RETURN_LONGJMP,%rsi
+
+	addq	$VMXCTX_TMPSTKTOP,%rsp
+	callq	vmx_return
+END(vmx_longjmp)
+
+/*
+ * void vmx_resume(struct vmxctx *ctxp)
+ * %rdi = ctxp
+ *
+ * Although the return type is a 'void' this function may return indirectly
+ * through vmx_setjmp() with a return value of 2.
+ */
+ENTRY(vmx_resume)
+	VMX_DISABLE_INTERRUPTS
+
+	VMX_CHECK_AST
+
+	/*
+	 * Restore guest state that is not automatically loaded from the vmcs.
+	 */
+	VMX_GUEST_RESTORE
+
+	vmresume
+
+	/*
+	 * Capture the reason why vmresume failed.
+	 */
+	VM_INSTRUCTION_ERROR(%eax)
+
+	/* Return via vmx_setjmp with return value of VMX_RETURN_VMRESUME */
+	movq	%rsp,%rdi
+	movq	$VMX_RETURN_VMRESUME,%rsi
+
+	addq	$VMXCTX_TMPSTKTOP,%rsp
+	callq	vmx_return
+END(vmx_resume)
+
+/*
+ * void vmx_launch(struct vmxctx *ctxp)
+ * %rdi = ctxp
+ *
+ * Although the return type is a 'void' this function may return indirectly
+ * through vmx_setjmp() with a return value of 3.
+ */
+ENTRY(vmx_launch)
+	VMX_DISABLE_INTERRUPTS
+
+	VMX_CHECK_AST
+
+	/*
+	 * Restore guest state that is not automatically loaded from the vmcs.
+	 */
+	VMX_GUEST_RESTORE
+
+	vmlaunch
+
+	/*
+	 * Capture the reason why vmlaunch failed.
+	 */
+	VM_INSTRUCTION_ERROR(%eax)
+
+	/* Return via vmx_setjmp with return value of VMX_RETURN_VMLAUNCH */
+	movq	%rsp,%rdi
+	movq	$VMX_RETURN_VMLAUNCH,%rsi
+
+	addq	$VMXCTX_TMPSTKTOP,%rsp
+	callq	vmx_return
+END(vmx_launch)
diff --git a/sys/amd64/vmm/intel/vtd.c b/sys/amd64/vmm/intel/vtd.c
new file mode 100644
index 0000000..ef0e9bc
--- /dev/null
+++ b/sys/amd64/vmm/intel/vtd.c
@@ -0,0 +1,677 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <dev/pci/pcireg.h>
+
+#include <machine/pmap.h>
+#include <machine/vmparam.h>
+#include <machine/pci_cfgreg.h>
+
+#include "io/iommu.h"
+
+/*
+ * Documented in the "Intel Virtualization Technology for Directed I/O",
+ * Architecture Spec, September 2008.
+ */
+
+/* Section 10.4 "Register Descriptions" */
+struct vtdmap {
+	volatile uint32_t	version;
+	volatile uint32_t	res0;
+	volatile uint64_t	cap;
+	volatile uint64_t	ext_cap;
+	volatile uint32_t	gcr;
+	volatile uint32_t	gsr;
+	volatile uint64_t	rta;
+	volatile uint64_t	ccr;
+};
+
+#define	VTD_CAP_SAGAW(cap)	(((cap) >> 8) & 0x1F)
+#define	VTD_CAP_ND(cap)		((cap) & 0x7)
+#define	VTD_CAP_CM(cap)		(((cap) >> 7) & 0x1)
+#define	VTD_CAP_SPS(cap)	(((cap) >> 34) & 0xF)
+#define	VTD_CAP_RWBF(cap)	(((cap) >> 4) & 0x1)
+
+#define	VTD_ECAP_DI(ecap)	(((ecap) >> 2) & 0x1)
+#define	VTD_ECAP_COHERENCY(ecap) ((ecap) & 0x1)
+#define	VTD_ECAP_IRO(ecap)	(((ecap) >> 8) & 0x3FF)
+
+#define	VTD_GCR_WBF		(1 << 27)
+#define	VTD_GCR_SRTP		(1 << 30)
+#define	VTD_GCR_TE		(1 << 31)
+
+#define	VTD_GSR_WBFS		(1 << 27)
+#define	VTD_GSR_RTPS		(1 << 30)
+#define	VTD_GSR_TES		(1 << 31)
+
+#define	VTD_CCR_ICC		(1UL << 63)	/* invalidate context cache */
+#define	VTD_CCR_CIRG_GLOBAL	(1UL << 61)	/* global invalidation */
+
+#define	VTD_IIR_IVT		(1UL << 63)	/* invalidation IOTLB */
+#define	VTD_IIR_IIRG_GLOBAL	(1ULL << 60)	/* global IOTLB invalidation */
+#define	VTD_IIR_IIRG_DOMAIN	(2ULL << 60)	/* domain IOTLB invalidation */
+#define	VTD_IIR_IIRG_PAGE	(3ULL << 60)	/* page IOTLB invalidation */
+#define	VTD_IIR_DRAIN_READS	(1ULL << 49)	/* drain pending DMA reads */
+#define	VTD_IIR_DRAIN_WRITES	(1ULL << 48)	/* drain pending DMA writes */
+#define	VTD_IIR_DOMAIN_P	32
+
+#define	VTD_ROOT_PRESENT	0x1
+#define	VTD_CTX_PRESENT		0x1
+#define	VTD_CTX_TT_ALL		(1UL << 2)
+
+#define	VTD_PTE_RD		(1UL << 0)
+#define	VTD_PTE_WR		(1UL << 1)
+#define	VTD_PTE_SUPERPAGE	(1UL << 7)
+#define	VTD_PTE_ADDR_M		(0x000FFFFFFFFFF000UL)
+
+struct domain {
+	uint64_t	*ptp;		/* first level page table page */
+	int		pt_levels;	/* number of page table levels */
+	int		addrwidth;	/* 'AW' field in context entry */
+	int		spsmask;	/* supported super page sizes */
+	u_int		id;		/* domain id */
+	vm_paddr_t	maxaddr;	/* highest address to be mapped */
+	SLIST_ENTRY(domain) next;
+};
+
+static SLIST_HEAD(, domain) domhead;
+
+#define	DRHD_MAX_UNITS	8
+static int		drhd_num;
+static struct vtdmap	*vtdmaps[DRHD_MAX_UNITS];
+static int		max_domains;
+typedef int		(*drhd_ident_func_t)(void);
+
+static uint64_t root_table[PAGE_SIZE / sizeof(uint64_t)] __aligned(4096);
+static uint64_t ctx_tables[256][PAGE_SIZE / sizeof(uint64_t)] __aligned(4096);
+
+static MALLOC_DEFINE(M_VTD, "vtd", "vtd");
+
+/*
+ * Config space register definitions from the "Intel 5520 and 5500" datasheet.
+ */
+static int
+tylersburg_vtd_ident(void)
+{
+	int units, nlbus;
+	uint16_t did, vid;
+	uint32_t miscsts, vtbar;
+
+	const int bus = 0;
+	const int slot = 20;
+	const int func = 0;
+
+	units = 0;
+
+	vid = pci_cfgregread(bus, slot, func, PCIR_VENDOR, 2);
+	did = pci_cfgregread(bus, slot, func, PCIR_DEVICE, 2);
+	if (vid != 0x8086 || did != 0x342E)
+		goto done;
+
+	/*
+	 * Check if this is a dual IOH configuration.
+	 */
+	miscsts = pci_cfgregread(bus, slot, func, 0x9C, 4);
+	if (miscsts & (1 << 25))
+		nlbus = pci_cfgregread(bus, slot, func, 0x160, 1);
+	else	
+		nlbus = -1;
+
+	vtbar = pci_cfgregread(bus, slot, func, 0x180, 4);
+	if (vtbar & 0x1) {
+		vtdmaps[units++] = (struct vtdmap *)
+					PHYS_TO_DMAP(vtbar & 0xffffe000);
+	} else if (bootverbose)
+		printf("VT-d unit in legacy IOH is disabled!\n");
+
+	if (nlbus != -1) {
+		vtbar = pci_cfgregread(nlbus, slot, func, 0x180, 4);
+		if (vtbar & 0x1) {
+			vtdmaps[units++] = (struct vtdmap *)
+					   PHYS_TO_DMAP(vtbar & 0xffffe000);
+		} else if (bootverbose)
+			printf("VT-d unit in non-legacy IOH is disabled!\n");
+	}
+done:
+	return (units);
+}
+
+static drhd_ident_func_t drhd_ident_funcs[] = {
+	tylersburg_vtd_ident,
+	NULL
+};
+
+static int
+vtd_max_domains(struct vtdmap *vtdmap)
+{
+	int nd;
+
+	nd = VTD_CAP_ND(vtdmap->cap);
+
+	switch (nd) {
+	case 0:
+		return (16);
+	case 1:
+		return (64);
+	case 2:
+		return (256);
+	case 3:
+		return (1024);
+	case 4:
+		return (4 * 1024);
+	case 5:
+		return (16 * 1024);
+	case 6:
+		return (64 * 1024);
+	default:
+		panic("vtd_max_domains: invalid value of nd (0x%0x)", nd);
+	}
+}
+
+static u_int
+domain_id(void)
+{
+	u_int id;
+	struct domain *dom;
+
+	/* Skip domain id 0 - it is reserved when Caching Mode field is set */
+	for (id = 1; id < max_domains; id++) {
+		SLIST_FOREACH(dom, &domhead, next) {
+			if (dom->id == id)
+				break;
+		}
+		if (dom == NULL)
+			break;		/* found it */
+	}
+	
+	if (id >= max_domains)
+		panic("domain ids exhausted");
+
+	return (id);
+}
+
+static void
+vtd_wbflush(struct vtdmap *vtdmap)
+{
+
+	if (VTD_ECAP_COHERENCY(vtdmap->ext_cap) == 0)
+		pmap_invalidate_cache();
+
+	if (VTD_CAP_RWBF(vtdmap->cap)) {
+		vtdmap->gcr = VTD_GCR_WBF;
+		while ((vtdmap->gsr & VTD_GSR_WBFS) != 0)
+			;
+	}
+}
+
+static void
+vtd_ctx_global_invalidate(struct vtdmap *vtdmap)
+{
+
+	vtdmap->ccr = VTD_CCR_ICC | VTD_CCR_CIRG_GLOBAL;
+	while ((vtdmap->ccr & VTD_CCR_ICC) != 0)
+		;
+}
+
+static void
+vtd_iotlb_global_invalidate(struct vtdmap *vtdmap)
+{
+	int offset;
+	volatile uint64_t *iotlb_reg, val;
+
+	vtd_wbflush(vtdmap);
+
+	offset = VTD_ECAP_IRO(vtdmap->ext_cap) * 16;
+	iotlb_reg = (volatile uint64_t *)((caddr_t)vtdmap + offset + 8);
+	
+	*iotlb_reg =  VTD_IIR_IVT | VTD_IIR_IIRG_GLOBAL |
+		      VTD_IIR_DRAIN_READS | VTD_IIR_DRAIN_WRITES;
+
+	while (1) {
+		val = *iotlb_reg;
+		if ((val & VTD_IIR_IVT) == 0)
+			break;
+	}
+}
+
+static void
+vtd_translation_enable(struct vtdmap *vtdmap)
+{
+
+	vtdmap->gcr = VTD_GCR_TE;
+	while ((vtdmap->gsr & VTD_GSR_TES) == 0)
+		;
+}
+
+static void
+vtd_translation_disable(struct vtdmap *vtdmap)
+{
+
+	vtdmap->gcr = 0;
+	while ((vtdmap->gsr & VTD_GSR_TES) != 0)
+		;
+}
+
+static int
+vtd_init(void)
+{
+	int i, units;
+	struct vtdmap *vtdmap;
+	vm_paddr_t ctx_paddr;
+	
+	for (i = 0; drhd_ident_funcs[i] != NULL; i++) {
+		units = (*drhd_ident_funcs[i])();
+		if (units > 0)
+			break;
+	}
+
+	if (units <= 0)
+		return (ENXIO);
+
+	drhd_num = units;
+	vtdmap = vtdmaps[0];
+
+	if (VTD_CAP_CM(vtdmap->cap) != 0)
+		panic("vtd_init: invalid caching mode");
+
+	max_domains = vtd_max_domains(vtdmap);
+
+	/*
+	 * Set up the root-table to point to the context-entry tables
+	 */
+	for (i = 0; i < 256; i++) {
+		ctx_paddr = vtophys(ctx_tables[i]);
+		if (ctx_paddr & PAGE_MASK)
+			panic("ctx table (0x%0lx) not page aligned", ctx_paddr);
+
+		root_table[i * 2] = ctx_paddr | VTD_ROOT_PRESENT;
+	}
+
+	return (0);
+}
+
+static void
+vtd_cleanup(void)
+{
+}
+
+static void
+vtd_enable(void)
+{
+	int i;
+	struct vtdmap *vtdmap;
+
+	for (i = 0; i < drhd_num; i++) {
+		vtdmap = vtdmaps[i];
+		vtd_wbflush(vtdmap);
+
+		/* Update the root table address */
+		vtdmap->rta = vtophys(root_table);
+		vtdmap->gcr = VTD_GCR_SRTP;
+		while ((vtdmap->gsr & VTD_GSR_RTPS) == 0)
+			;
+
+		vtd_ctx_global_invalidate(vtdmap);
+		vtd_iotlb_global_invalidate(vtdmap);
+
+		vtd_translation_enable(vtdmap);
+	}
+}
+
+static void
+vtd_disable(void)
+{
+	int i;
+	struct vtdmap *vtdmap;
+
+	for (i = 0; i < drhd_num; i++) {
+		vtdmap = vtdmaps[i];
+		vtd_translation_disable(vtdmap);
+	}
+}
+
+static void
+vtd_add_device(void *arg, int bus, int slot, int func)
+{
+	int idx;
+	uint64_t *ctxp;
+	struct domain *dom = arg;
+	vm_paddr_t pt_paddr;
+	struct vtdmap *vtdmap;
+
+	if (bus < 0 || bus > PCI_BUSMAX ||
+	    slot < 0 || slot > PCI_SLOTMAX ||
+	    func < 0 || func > PCI_FUNCMAX)
+		panic("vtd_add_device: invalid bsf %d/%d/%d", bus, slot, func);
+
+	vtdmap = vtdmaps[0];
+	ctxp = ctx_tables[bus];
+	pt_paddr = vtophys(dom->ptp);
+	idx = (slot << 3 | func) * 2;
+
+	if (ctxp[idx] & VTD_CTX_PRESENT) {
+		panic("vtd_add_device: device %d/%d/%d is already owned by "
+		      "domain %d", bus, slot, func,
+		      (uint16_t)(ctxp[idx + 1] >> 8));
+	}
+
+	/*
+	 * Order is important. The 'present' bit is set only after all fields
+	 * of the context pointer are initialized.
+	 */
+	ctxp[idx + 1] = dom->addrwidth | (dom->id << 8);
+
+	if (VTD_ECAP_DI(vtdmap->ext_cap))
+		ctxp[idx] = VTD_CTX_TT_ALL;
+	else
+		ctxp[idx] = 0;
+
+	ctxp[idx] |= pt_paddr | VTD_CTX_PRESENT;
+
+	/*
+	 * 'Not Present' entries are not cached in either the Context Cache
+	 * or in the IOTLB, so there is no need to invalidate either of them.
+	 */
+}
+
+static void
+vtd_remove_device(void *arg, int bus, int slot, int func)
+{
+	int i, idx;
+	uint64_t *ctxp;
+	struct vtdmap *vtdmap;
+
+	if (bus < 0 || bus > PCI_BUSMAX ||
+	    slot < 0 || slot > PCI_SLOTMAX ||
+	    func < 0 || func > PCI_FUNCMAX)
+		panic("vtd_add_device: invalid bsf %d/%d/%d", bus, slot, func);
+
+	ctxp = ctx_tables[bus];
+	idx = (slot << 3 | func) * 2;
+
+	/*
+	 * Order is important. The 'present' bit is must be cleared first.
+	 */
+	ctxp[idx] = 0;
+	ctxp[idx + 1] = 0;
+
+	/*
+	 * Invalidate the Context Cache and the IOTLB.
+	 *
+	 * XXX use device-selective invalidation for Context Cache
+	 * XXX use domain-selective invalidation for IOTLB
+	 */
+	for (i = 0; i < drhd_num; i++) {
+		vtdmap = vtdmaps[i];
+		vtd_ctx_global_invalidate(vtdmap);
+		vtd_iotlb_global_invalidate(vtdmap);
+	}
+}
+
+#define	CREATE_MAPPING	0
+#define	REMOVE_MAPPING	1
+
+static uint64_t
+vtd_update_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len,
+		   int remove)
+{
+	struct domain *dom;
+	int i, spshift, ptpshift, ptpindex, nlevels;
+	uint64_t spsize, *ptp;
+
+	dom = arg;
+	ptpindex = 0;
+	ptpshift = 0;
+
+	if (gpa & PAGE_MASK)
+		panic("vtd_create_mapping: unaligned gpa 0x%0lx", gpa);
+
+	if (hpa & PAGE_MASK)
+		panic("vtd_create_mapping: unaligned hpa 0x%0lx", hpa);
+
+	if (len & PAGE_MASK)
+		panic("vtd_create_mapping: unaligned len 0x%0lx", len);
+
+	/*
+	 * Compute the size of the mapping that we can accomodate.
+	 *
+	 * This is based on three factors:
+	 * - supported super page size
+	 * - alignment of the region starting at 'gpa' and 'hpa'
+	 * - length of the region 'len'
+	 */
+	spshift = 48;
+	for (i = 3; i >= 0; i--) {
+		spsize = 1UL << spshift;
+		if ((dom->spsmask & (1 << i)) != 0 &&
+		    (gpa & (spsize - 1)) == 0 &&
+		    (hpa & (spsize - 1)) == 0 &&
+		    (len >= spsize)) {
+			break;
+		}
+		spshift -= 9;
+	}
+
+	ptp = dom->ptp;
+	nlevels = dom->pt_levels;
+	while (--nlevels >= 0) {
+		ptpshift = 12 + nlevels * 9;
+		ptpindex = (gpa >> ptpshift) & 0x1FF;
+
+		/* We have reached the leaf mapping */
+		if (spshift >= ptpshift) {
+			break;
+		}
+
+		/*
+		 * We are working on a non-leaf page table page.
+		 *
+		 * Create a downstream page table page if necessary and point
+		 * to it from the current page table.
+		 */
+		if (ptp[ptpindex] == 0) {
+			void *nlp = malloc(PAGE_SIZE, M_VTD, M_WAITOK | M_ZERO);
+			ptp[ptpindex] = vtophys(nlp)| VTD_PTE_RD | VTD_PTE_WR;
+		}
+
+		ptp = (uint64_t *)PHYS_TO_DMAP(ptp[ptpindex] & VTD_PTE_ADDR_M);
+	}
+
+	if ((gpa & ((1UL << ptpshift) - 1)) != 0)
+		panic("gpa 0x%lx and ptpshift %d mismatch", gpa, ptpshift);
+
+	/*
+	 * Update the 'gpa' -> 'hpa' mapping
+	 */
+	if (remove) {
+		ptp[ptpindex] = 0;
+	} else {
+		ptp[ptpindex] = hpa | VTD_PTE_RD | VTD_PTE_WR;
+
+		if (nlevels > 0)
+			ptp[ptpindex] |= VTD_PTE_SUPERPAGE;
+	}
+
+	return (1UL << ptpshift);
+}
+
+static uint64_t
+vtd_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len)
+{
+
+	return (vtd_update_mapping(arg, gpa, hpa, len, CREATE_MAPPING));
+}
+
+static uint64_t
+vtd_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len)
+{
+
+	return (vtd_update_mapping(arg, gpa, 0, len, REMOVE_MAPPING));
+}
+
+static void
+vtd_invalidate_tlb(void *dom)
+{
+	int i;
+	struct vtdmap *vtdmap;
+
+	/*
+	 * Invalidate the IOTLB.
+	 * XXX use domain-selective invalidation for IOTLB
+	 */
+	for (i = 0; i < drhd_num; i++) {
+		vtdmap = vtdmaps[i];
+		vtd_iotlb_global_invalidate(vtdmap);
+	}
+}
+
+static void *
+vtd_create_domain(vm_paddr_t maxaddr)
+{
+	struct domain *dom;
+	vm_paddr_t addr;
+	int tmp, i, gaw, agaw, sagaw, res, pt_levels, addrwidth;
+	struct vtdmap *vtdmap;
+
+	if (drhd_num <= 0)
+		panic("vtd_create_domain: no dma remapping hardware available");
+
+	vtdmap = vtdmaps[0];
+
+	/*
+	 * Calculate AGAW.
+	 * Section 3.4.2 "Adjusted Guest Address Width", Architecture Spec.
+	 */
+	addr = 0;
+	for (gaw = 0; addr < maxaddr; gaw++)
+		addr = 1ULL << gaw;
+
+	res = (gaw - 12) % 9;
+	if (res == 0)
+		agaw = gaw;
+	else
+		agaw = gaw + 9 - res;
+
+	if (agaw > 64)
+		agaw = 64;
+
+	/*
+	 * Select the smallest Supported AGAW and the corresponding number
+	 * of page table levels.
+	 */
+	pt_levels = 2;
+	sagaw = 30;
+	addrwidth = 0;
+	tmp = VTD_CAP_SAGAW(vtdmap->cap);
+	for (i = 0; i < 5; i++) {
+		if ((tmp & (1 << i)) != 0 && sagaw >= agaw)
+			break;
+		pt_levels++;
+		addrwidth++;
+		sagaw += 9;
+		if (sagaw > 64)
+			sagaw = 64;
+	}
+
+	if (i >= 5) {
+		panic("vtd_create_domain: SAGAW 0x%lx does not support AGAW %d",
+		      VTD_CAP_SAGAW(vtdmap->cap), agaw);
+	}
+
+	dom = malloc(sizeof(struct domain), M_VTD, M_ZERO | M_WAITOK);
+	dom->pt_levels = pt_levels;
+	dom->addrwidth = addrwidth;
+	dom->spsmask = VTD_CAP_SPS(vtdmap->cap);
+	dom->id = domain_id();
+	dom->maxaddr = maxaddr;
+	dom->ptp = malloc(PAGE_SIZE, M_VTD, M_ZERO | M_WAITOK);
+	if ((uintptr_t)dom->ptp & PAGE_MASK)
+		panic("vtd_create_domain: ptp (%p) not page aligned", dom->ptp);
+
+	SLIST_INSERT_HEAD(&domhead, dom, next);
+
+	return (dom);
+}
+
+static void
+vtd_free_ptp(uint64_t *ptp, int level)
+{
+	int i;
+	uint64_t *nlp;
+
+	if (level > 1) {
+		for (i = 0; i < 512; i++) {
+			if ((ptp[i] & (VTD_PTE_RD | VTD_PTE_WR)) == 0)
+				continue;
+			if ((ptp[i] & VTD_PTE_SUPERPAGE) != 0)
+				continue;
+			nlp = (uint64_t *)PHYS_TO_DMAP(ptp[i] & VTD_PTE_ADDR_M);
+			vtd_free_ptp(nlp, level - 1);
+		}
+	}
+
+	bzero(ptp, PAGE_SIZE);
+	free(ptp, M_VTD);
+}
+
+static void
+vtd_destroy_domain(void *arg)
+{
+	struct domain *dom;
+	
+	dom = arg;
+
+	SLIST_REMOVE(&domhead, dom, domain, next);
+	vtd_free_ptp(dom->ptp, dom->pt_levels);
+	free(dom, M_VTD);
+}
+
+struct iommu_ops iommu_ops_intel = {
+	vtd_init,
+	vtd_cleanup,
+	vtd_enable,
+	vtd_disable,
+	vtd_create_domain,
+	vtd_destroy_domain,
+	vtd_create_mapping,
+	vtd_remove_mapping,
+	vtd_add_device,
+	vtd_remove_device,
+	vtd_invalidate_tlb,
+};
diff --git a/sys/amd64/vmm/io/iommu.c b/sys/amd64/vmm/io/iommu.c
new file mode 100644
index 0000000..c8447cc
--- /dev/null
+++ b/sys/amd64/vmm/io/iommu.c
@@ -0,0 +1,277 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+
+#include <dev/pci/pcivar.h>
+#include <dev/pci/pcireg.h>
+
+#include <machine/md_var.h>
+
+#include "vmm_util.h"
+#include "vmm_mem.h"
+#include "iommu.h"
+
+static boolean_t iommu_avail;
+static struct iommu_ops *ops;
+static void *host_domain;
+
+static __inline int
+IOMMU_INIT(void)
+{
+	if (ops != NULL)
+		return ((*ops->init)());
+	else
+		return (ENXIO);
+}
+
+static __inline void
+IOMMU_CLEANUP(void)
+{
+	if (ops != NULL && iommu_avail)
+		(*ops->cleanup)();
+}
+
+static __inline void *
+IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr)
+{
+
+	if (ops != NULL && iommu_avail)
+		return ((*ops->create_domain)(maxaddr));
+	else
+		return (NULL);
+}
+
+static __inline void
+IOMMU_DESTROY_DOMAIN(void *dom)
+{
+
+	if (ops != NULL && iommu_avail)
+		(*ops->destroy_domain)(dom);
+}
+
+static __inline uint64_t
+IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len)
+{
+
+	if (ops != NULL && iommu_avail)
+		return ((*ops->create_mapping)(domain, gpa, hpa, len));
+	else
+		return (len);		/* XXX */
+}
+
+static __inline uint64_t
+IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len)
+{
+
+	if (ops != NULL && iommu_avail)
+		return ((*ops->remove_mapping)(domain, gpa, len));
+	else
+		return (len);		/* XXX */
+}
+
+static __inline void
+IOMMU_ADD_DEVICE(void *domain, int bus, int slot, int func)
+{
+
+	if (ops != NULL && iommu_avail)
+		(*ops->add_device)(domain, bus, slot, func);
+}
+
+static __inline void
+IOMMU_REMOVE_DEVICE(void *domain, int bus, int slot, int func)
+{
+
+	if (ops != NULL && iommu_avail)
+		(*ops->remove_device)(domain, bus, slot, func);
+}
+
+static __inline void
+IOMMU_INVALIDATE_TLB(void *domain)
+{
+
+	if (ops != NULL && iommu_avail)
+		(*ops->invalidate_tlb)(domain);
+}
+
+static __inline void
+IOMMU_ENABLE(void)
+{
+
+	if (ops != NULL && iommu_avail)
+		(*ops->enable)();
+}
+
+static __inline void
+IOMMU_DISABLE(void)
+{
+
+	if (ops != NULL && iommu_avail)
+		(*ops->disable)();
+}
+
+void
+iommu_init(void)
+{
+	int error, bus, slot, func;
+	vm_paddr_t maxaddr;
+	const char *name;
+	device_t dev;
+
+	if (vmm_is_intel())
+		ops = &iommu_ops_intel;
+	else if (vmm_is_amd())
+		ops = &iommu_ops_amd;
+	else
+		ops = NULL;
+
+	error = IOMMU_INIT();
+	if (error)
+		return;
+
+	iommu_avail = TRUE;
+
+	/*
+	 * Create a domain for the devices owned by the host
+	 */
+	maxaddr = vmm_mem_maxaddr();
+	host_domain = IOMMU_CREATE_DOMAIN(maxaddr);
+	if (host_domain == NULL)
+		panic("iommu_init: unable to create a host domain");
+
+	/*
+	 * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to
+	 * the host
+	 */
+	iommu_create_mapping(host_domain, 0, 0, maxaddr);
+
+	for (bus = 0; bus <= PCI_BUSMAX; bus++) {
+		for (slot = 0; slot <= PCI_SLOTMAX; slot++) {
+			for (func = 0; func <= PCI_FUNCMAX; func++) {
+				dev = pci_find_dbsf(0, bus, slot, func);
+				if (dev == NULL)
+					continue;
+
+				/* skip passthrough devices */
+				name = device_get_name(dev);
+				if (name != NULL && strcmp(name, "ppt") == 0)
+					continue;
+
+				/* everything else belongs to the host domain */
+				iommu_add_device(host_domain, bus, slot, func);
+			}
+		}
+	}
+	IOMMU_ENABLE();
+
+}
+
+void
+iommu_cleanup(void)
+{
+	IOMMU_DISABLE();
+	IOMMU_DESTROY_DOMAIN(host_domain);
+	IOMMU_CLEANUP();
+}
+
+void *
+iommu_create_domain(vm_paddr_t maxaddr)
+{
+
+	return (IOMMU_CREATE_DOMAIN(maxaddr));
+}
+
+void
+iommu_destroy_domain(void *dom)
+{
+
+	IOMMU_DESTROY_DOMAIN(dom);
+}
+
+void
+iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len)
+{
+	uint64_t mapped, remaining;
+
+	remaining = len;
+
+	while (remaining > 0) {
+		mapped = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining);
+		gpa += mapped;
+		hpa += mapped;
+		remaining -= mapped;
+	}
+}
+
+void
+iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len)
+{
+	uint64_t unmapped, remaining;
+
+	remaining = len;
+
+	while (remaining > 0) {
+		unmapped = IOMMU_REMOVE_MAPPING(dom, gpa, remaining);
+		gpa += unmapped;
+		remaining -= unmapped;
+	}
+}
+
+void *
+iommu_host_domain(void)
+{
+
+	return (host_domain);
+}
+
+void
+iommu_add_device(void *dom, int bus, int slot, int func)
+{
+
+	IOMMU_ADD_DEVICE(dom, bus, slot, func);
+}
+
+void
+iommu_remove_device(void *dom, int bus, int slot, int func)
+{
+
+	IOMMU_REMOVE_DEVICE(dom, bus, slot, func);
+}
+
+void
+iommu_invalidate_tlb(void *domain)
+{
+
+	IOMMU_INVALIDATE_TLB(domain);
+}
diff --git a/sys/amd64/vmm/io/iommu.h b/sys/amd64/vmm/io/iommu.h
new file mode 100644
index 0000000..d5c1d6e
--- /dev/null
+++ b/sys/amd64/vmm/io/iommu.h
@@ -0,0 +1,75 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _IO_IOMMU_H_
+#define	_IO_IOMMU_H_
+
+typedef int (*iommu_init_func_t)(void);
+typedef void (*iommu_cleanup_func_t)(void);
+typedef void (*iommu_enable_func_t)(void);
+typedef void (*iommu_disable_func_t)(void);
+typedef void *(*iommu_create_domain_t)(vm_paddr_t maxaddr);
+typedef void (*iommu_destroy_domain_t)(void *domain);
+typedef uint64_t (*iommu_create_mapping_t)(void *domain, vm_paddr_t gpa,
+					   vm_paddr_t hpa, uint64_t len);
+typedef uint64_t (*iommu_remove_mapping_t)(void *domain, vm_paddr_t gpa,
+					   uint64_t len);
+typedef void (*iommu_add_device_t)(void *domain, int bus, int slot, int func);
+typedef void (*iommu_remove_device_t)(void *dom, int bus, int slot, int func);
+typedef void (*iommu_invalidate_tlb_t)(void *dom);
+
+struct iommu_ops {
+	iommu_init_func_t	init;		/* module wide */
+	iommu_cleanup_func_t	cleanup;
+	iommu_enable_func_t	enable;
+	iommu_disable_func_t	disable;
+
+	iommu_create_domain_t	create_domain;	/* domain-specific */
+	iommu_destroy_domain_t	destroy_domain;
+	iommu_create_mapping_t	create_mapping;
+	iommu_remove_mapping_t	remove_mapping;
+	iommu_add_device_t	add_device;
+	iommu_remove_device_t	remove_device;
+	iommu_invalidate_tlb_t	invalidate_tlb;
+};
+
+extern struct iommu_ops iommu_ops_intel;
+extern struct iommu_ops iommu_ops_amd;
+
+void	iommu_init(void);
+void	iommu_cleanup(void);
+void	*iommu_host_domain(void);
+void	*iommu_create_domain(vm_paddr_t maxaddr);
+void	iommu_destroy_domain(void *dom);
+void	iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa,
+			     size_t len);
+void	iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len);
+void	iommu_add_device(void *dom, int bus, int slot, int func);
+void	iommu_remove_device(void *dom, int bus, int slot, int func);
+void	iommu_invalidate_tlb(void *domain);
+#endif
diff --git a/sys/amd64/vmm/io/ppt.c b/sys/amd64/vmm/io/ppt.c
new file mode 100644
index 0000000..4a05985
--- /dev/null
+++ b/sys/amd64/vmm/io/ppt.c
@@ -0,0 +1,619 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/bus.h>
+#include <sys/pciio.h>
+#include <sys/rman.h>
+#include <sys/smp.h>
+
+#include <dev/pci/pcivar.h>
+#include <dev/pci/pcireg.h>
+
+#include <machine/resource.h>
+
+#include <machine/vmm.h>
+#include <machine/vmm_dev.h>
+
+#include "vmm_lapic.h"
+#include "vmm_ktr.h"
+
+#include "iommu.h"
+#include "ppt.h"
+
+/* XXX locking */
+
+#define	MAX_PPTDEVS	(sizeof(pptdevs) / sizeof(pptdevs[0]))
+#define	MAX_MSIMSGS	32
+
+/*
+ * If the MSI-X table is located in the middle of a BAR then that MMIO
+ * region gets split into two segments - one segment above the MSI-X table
+ * and the other segment below the MSI-X table - with a hole in place of
+ * the MSI-X table so accesses to it can be trapped and emulated.
+ *
+ * So, allocate a MMIO segment for each BAR register + 1 additional segment.
+ */
+#define	MAX_MMIOSEGS	((PCIR_MAX_BAR_0 + 1) + 1)
+
+MALLOC_DEFINE(M_PPTMSIX, "pptmsix", "Passthru MSI-X resources");
+
+struct pptintr_arg {				/* pptintr(pptintr_arg) */
+	struct pptdev	*pptdev;
+	int		vec;
+	int 		vcpu;
+};
+
+static struct pptdev {
+	device_t	dev;
+	struct vm	*vm;			/* owner of this device */
+	struct vm_memory_segment mmio[MAX_MMIOSEGS];
+	struct {
+		int	num_msgs;		/* guest state */
+
+		int	startrid;		/* host state */
+		struct resource *res[MAX_MSIMSGS];
+		void	*cookie[MAX_MSIMSGS];
+		struct pptintr_arg arg[MAX_MSIMSGS];
+	} msi;
+
+	struct {
+		int num_msgs;
+		int startrid;
+		int msix_table_rid;
+		struct resource *msix_table_res;
+		struct resource **res;
+		void **cookie;
+		struct pptintr_arg *arg;
+	} msix;
+} pptdevs[64];
+
+static int num_pptdevs;
+
+static int
+ppt_probe(device_t dev)
+{
+	int bus, slot, func;
+	struct pci_devinfo *dinfo;
+
+	dinfo = (struct pci_devinfo *)device_get_ivars(dev);
+
+	bus = pci_get_bus(dev);
+	slot = pci_get_slot(dev);
+	func = pci_get_function(dev);
+
+	/*
+	 * To qualify as a pci passthrough device a device must:
+	 * - be allowed by administrator to be used in this role
+	 * - be an endpoint device
+	 */
+	if (vmm_is_pptdev(bus, slot, func) &&
+	    (dinfo->cfg.hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_NORMAL)
+		return (0);
+	else
+		return (ENXIO);
+}
+
+static int
+ppt_attach(device_t dev)
+{
+	int n;
+
+	if (num_pptdevs >= MAX_PPTDEVS) {
+		printf("ppt_attach: maximum number of pci passthrough devices "
+		       "exceeded\n");
+		return (ENXIO);
+	}
+
+	n = num_pptdevs++;
+	pptdevs[n].dev = dev;
+
+	if (bootverbose)
+		device_printf(dev, "attached\n");
+
+	return (0);
+}
+
+static int
+ppt_detach(device_t dev)
+{
+	/*
+	 * XXX check whether there are any pci passthrough devices assigned
+	 * to guests before we allow this driver to detach.
+	 */
+
+	return (0);
+}
+
+static device_method_t ppt_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		ppt_probe),
+	DEVMETHOD(device_attach,	ppt_attach),
+	DEVMETHOD(device_detach,	ppt_detach),
+	{0, 0}
+};
+
+static devclass_t ppt_devclass;
+DEFINE_CLASS_0(ppt, ppt_driver, ppt_methods, 0);
+DRIVER_MODULE(ppt, pci, ppt_driver, ppt_devclass, NULL, NULL);
+
+static struct pptdev *
+ppt_find(int bus, int slot, int func)
+{
+	device_t dev;
+	int i, b, s, f;
+
+	for (i = 0; i < num_pptdevs; i++) {
+		dev = pptdevs[i].dev;
+		b = pci_get_bus(dev);
+		s = pci_get_slot(dev);
+		f = pci_get_function(dev);
+		if (bus == b && slot == s && func == f)
+			return (&pptdevs[i]);
+	}
+	return (NULL);
+}
+
+static void
+ppt_unmap_mmio(struct vm *vm, struct pptdev *ppt)
+{
+	int i;
+	struct vm_memory_segment *seg;
+
+	for (i = 0; i < MAX_MMIOSEGS; i++) {
+		seg = &ppt->mmio[i];
+		if (seg->len == 0)
+			continue;
+		(void)vm_unmap_mmio(vm, seg->gpa, seg->len);
+		bzero(seg, sizeof(struct vm_memory_segment));
+	}
+}
+
+static void
+ppt_teardown_msi(struct pptdev *ppt)
+{
+	int i, rid;
+	void *cookie;
+	struct resource *res;
+
+	if (ppt->msi.num_msgs == 0)
+		return;
+
+	for (i = 0; i < ppt->msi.num_msgs; i++) {
+		rid = ppt->msi.startrid + i;
+		res = ppt->msi.res[i];
+		cookie = ppt->msi.cookie[i];
+
+		if (cookie != NULL)
+			bus_teardown_intr(ppt->dev, res, cookie);
+
+		if (res != NULL)
+			bus_release_resource(ppt->dev, SYS_RES_IRQ, rid, res);
+		
+		ppt->msi.res[i] = NULL;
+		ppt->msi.cookie[i] = NULL;
+	}
+
+	if (ppt->msi.startrid == 1)
+		pci_release_msi(ppt->dev);
+
+	ppt->msi.num_msgs = 0;
+}
+
+static void 
+ppt_teardown_msix_intr(struct pptdev *ppt, int idx)
+{
+	int rid;
+	struct resource *res;
+	void *cookie;
+
+	rid = ppt->msix.startrid + idx;
+	res = ppt->msix.res[idx];
+	cookie = ppt->msix.cookie[idx];
+
+	if (cookie != NULL) 
+		bus_teardown_intr(ppt->dev, res, cookie);
+
+	if (res != NULL) 
+		bus_release_resource(ppt->dev, SYS_RES_IRQ, rid, res);
+
+	ppt->msix.res[idx] = NULL;
+	ppt->msix.cookie[idx] = NULL;
+}
+
+static void 
+ppt_teardown_msix(struct pptdev *ppt)
+{
+	int i;
+
+	if (ppt->msix.num_msgs == 0) 
+		return;
+
+	for (i = 0; i < ppt->msix.num_msgs; i++) 
+		ppt_teardown_msix_intr(ppt, i);
+
+	if (ppt->msix.msix_table_res) {
+		bus_release_resource(ppt->dev, SYS_RES_MEMORY, 
+				     ppt->msix.msix_table_rid,
+				     ppt->msix.msix_table_res);
+		ppt->msix.msix_table_res = NULL;
+		ppt->msix.msix_table_rid = 0;
+	}
+
+	free(ppt->msix.res, M_PPTMSIX);
+	free(ppt->msix.cookie, M_PPTMSIX);
+	free(ppt->msix.arg, M_PPTMSIX);
+
+	pci_release_msi(ppt->dev);
+
+	ppt->msix.num_msgs = 0;
+}
+
+int
+ppt_assign_device(struct vm *vm, int bus, int slot, int func)
+{
+	struct pptdev *ppt;
+
+	ppt = ppt_find(bus, slot, func);
+	if (ppt != NULL) {
+		/*
+		 * If this device is owned by a different VM then we
+		 * cannot change its owner.
+		 */
+		if (ppt->vm != NULL && ppt->vm != vm)
+			return (EBUSY);
+
+		ppt->vm = vm;
+		iommu_add_device(vm_iommu_domain(vm), bus, slot, func);
+		return (0);
+	}
+	return (ENOENT);
+}
+
+int
+ppt_unassign_device(struct vm *vm, int bus, int slot, int func)
+{
+	struct pptdev *ppt;
+
+	ppt = ppt_find(bus, slot, func);
+	if (ppt != NULL) {
+		/*
+		 * If this device is not owned by this 'vm' then bail out.
+		 */
+		if (ppt->vm != vm)
+			return (EBUSY);
+		ppt_unmap_mmio(vm, ppt);
+		ppt_teardown_msi(ppt);
+		ppt_teardown_msix(ppt);
+		iommu_remove_device(vm_iommu_domain(vm), bus, slot, func);
+		ppt->vm = NULL;
+		return (0);
+	}
+	return (ENOENT);
+}
+
+int
+ppt_unassign_all(struct vm *vm)
+{
+	int i, bus, slot, func;
+	device_t dev;
+
+	for (i = 0; i < num_pptdevs; i++) {
+		if (pptdevs[i].vm == vm) {
+			dev = pptdevs[i].dev;
+			bus = pci_get_bus(dev);
+			slot = pci_get_slot(dev);
+			func = pci_get_function(dev);
+			ppt_unassign_device(vm, bus, slot, func);
+		}
+	}
+
+	return (0);
+}
+
+int
+ppt_map_mmio(struct vm *vm, int bus, int slot, int func,
+	     vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
+{
+	int i, error;
+	struct vm_memory_segment *seg;
+	struct pptdev *ppt;
+
+	ppt = ppt_find(bus, slot, func);
+	if (ppt != NULL) {
+		if (ppt->vm != vm)
+			return (EBUSY);
+
+		for (i = 0; i < MAX_MMIOSEGS; i++) {
+			seg = &ppt->mmio[i];
+			if (seg->len == 0) {
+				error = vm_map_mmio(vm, gpa, len, hpa);
+				if (error == 0) {
+					seg->gpa = gpa;
+					seg->len = len;
+				}
+				return (error);
+			}
+		}
+		return (ENOSPC);
+	}
+	return (ENOENT);
+}
+
+static int
+pptintr(void *arg)
+{
+	int vec;
+	struct pptdev *ppt;
+	struct pptintr_arg *pptarg;
+	
+	pptarg = arg;
+	ppt = pptarg->pptdev;
+	vec = pptarg->vec;
+
+	if (ppt->vm != NULL)
+		(void) lapic_set_intr(ppt->vm, pptarg->vcpu, vec);
+	else {
+		/*
+		 * XXX
+		 * This is not expected to happen - panic?
+		 */
+	}
+
+	/*
+	 * For legacy interrupts give other filters a chance in case
+	 * the interrupt was not generated by the passthrough device.
+	 */
+	if (ppt->msi.startrid == 0)
+		return (FILTER_STRAY);
+	else
+		return (FILTER_HANDLED);
+}
+
+/*
+ * XXX
+ * When we try to free the MSI resource the kernel will bind the thread to
+ * the host cpu was originally handling the MSI. The function freeing the
+ * MSI vector (apic_free_vector()) will panic the kernel if the thread
+ * is already bound to a cpu.
+ * 
+ * So, we temporarily unbind the vcpu thread before freeing the MSI resource.
+ */
+static void
+PPT_TEARDOWN_MSI(struct vm *vm, int vcpu, struct pptdev *ppt)
+{
+	int pincpu = -1;
+
+	vm_get_pinning(vm, vcpu, &pincpu);
+
+	if (pincpu >= 0)
+		vm_set_pinning(vm, vcpu, -1);
+
+	ppt_teardown_msi(ppt);
+
+	if (pincpu >= 0)
+		vm_set_pinning(vm, vcpu, pincpu);
+}
+
+int
+ppt_setup_msi(struct vm *vm, int vcpu, int bus, int slot, int func,
+	      int destcpu, int vector, int numvec)
+{
+	int i, rid, flags;
+	int msi_count, startrid, error, tmp;
+	struct pptdev *ppt;
+
+	if ((destcpu >= VM_MAXCPU || destcpu < 0) ||
+	    (vector < 0 || vector > 255) ||
+	    (numvec < 0 || numvec > MAX_MSIMSGS))
+		return (EINVAL);
+
+	ppt = ppt_find(bus, slot, func);
+	if (ppt == NULL)
+		return (ENOENT);
+	if (ppt->vm != vm)		/* Make sure we own this device */
+		return (EBUSY);
+
+	/* Free any allocated resources */
+	PPT_TEARDOWN_MSI(vm, vcpu, ppt);
+
+	if (numvec == 0)		/* nothing more to do */
+		return (0);
+
+	flags = RF_ACTIVE;
+	msi_count = pci_msi_count(ppt->dev);
+	if (msi_count == 0) {
+		startrid = 0;		/* legacy interrupt */
+		msi_count = 1;
+		flags |= RF_SHAREABLE;
+	} else
+		startrid = 1;		/* MSI */
+
+	/*
+	 * The device must be capable of supporting the number of vectors
+	 * the guest wants to allocate.
+	 */
+	if (numvec > msi_count)
+		return (EINVAL);
+
+	/*
+	 * Make sure that we can allocate all the MSI vectors that are needed
+	 * by the guest.
+	 */
+	if (startrid == 1) {
+		tmp = numvec;
+		error = pci_alloc_msi(ppt->dev, &tmp);
+		if (error)
+			return (error);
+		else if (tmp != numvec) {
+			pci_release_msi(ppt->dev);
+			return (ENOSPC);
+		} else {
+			/* success */
+		}
+	}
+	
+	ppt->msi.startrid = startrid;
+
+	/*
+	 * Allocate the irq resource and attach it to the interrupt handler.
+	 */
+	for (i = 0; i < numvec; i++) {
+		ppt->msi.num_msgs = i + 1;
+		ppt->msi.cookie[i] = NULL;
+
+		rid = startrid + i;
+		ppt->msi.res[i] = bus_alloc_resource_any(ppt->dev, SYS_RES_IRQ,
+							 &rid, flags);
+		if (ppt->msi.res[i] == NULL)
+			break;
+
+		ppt->msi.arg[i].pptdev = ppt;
+		ppt->msi.arg[i].vec = vector + i;
+		ppt->msi.arg[i].vcpu = destcpu;
+
+		error = bus_setup_intr(ppt->dev, ppt->msi.res[i],
+				       INTR_TYPE_NET | INTR_MPSAFE,
+				       pptintr, NULL, &ppt->msi.arg[i],
+				       &ppt->msi.cookie[i]);
+		if (error != 0)
+			break;
+	}
+	
+	if (i < numvec) {
+		PPT_TEARDOWN_MSI(vm, vcpu, ppt);
+		return (ENXIO);
+	}
+
+	return (0);
+}
+
+int
+ppt_setup_msix(struct vm *vm, int vcpu, int bus, int slot, int func,
+	       int idx, uint32_t msg, uint32_t vector_control, uint64_t addr)
+{
+	struct pptdev *ppt;
+	struct pci_devinfo *dinfo;
+	int numvec, alloced, rid, error;
+	size_t res_size, cookie_size, arg_size;
+
+	ppt = ppt_find(bus, slot, func);
+	if (ppt == NULL)
+		return (ENOENT);
+	if (ppt->vm != vm)		/* Make sure we own this device */
+		return (EBUSY);
+
+	dinfo = device_get_ivars(ppt->dev);
+	if (!dinfo) 
+		return (ENXIO);
+
+	/* 
+	 * First-time configuration:
+	 * 	Allocate the MSI-X table
+	 *	Allocate the IRQ resources
+	 *	Set up some variables in ppt->msix
+	 */
+	if (ppt->msix.num_msgs == 0) {
+		numvec = pci_msix_count(ppt->dev);
+		if (numvec <= 0)
+			return (EINVAL);
+
+		ppt->msix.startrid = 1;
+		ppt->msix.num_msgs = numvec;
+
+		res_size = numvec * sizeof(ppt->msix.res[0]);
+		cookie_size = numvec * sizeof(ppt->msix.cookie[0]);
+		arg_size = numvec * sizeof(ppt->msix.arg[0]);
+
+		ppt->msix.res = malloc(res_size, M_PPTMSIX, M_WAITOK | M_ZERO);
+		ppt->msix.cookie = malloc(cookie_size, M_PPTMSIX,
+					  M_WAITOK | M_ZERO);
+		ppt->msix.arg = malloc(arg_size, M_PPTMSIX, M_WAITOK | M_ZERO);
+
+		rid = dinfo->cfg.msix.msix_table_bar;
+		ppt->msix.msix_table_res = bus_alloc_resource_any(ppt->dev,
+					       SYS_RES_MEMORY, &rid, RF_ACTIVE);
+
+		if (ppt->msix.msix_table_res == NULL) {
+			ppt_teardown_msix(ppt);
+			return (ENOSPC);
+		}
+		ppt->msix.msix_table_rid = rid;
+
+		alloced = numvec;
+		error = pci_alloc_msix(ppt->dev, &alloced);
+		if (error || alloced != numvec) {
+			ppt_teardown_msix(ppt);
+			return (error == 0 ? ENOSPC: error);
+		}
+	}
+
+	if ((vector_control & PCIM_MSIX_VCTRL_MASK) == 0) {
+		/* Tear down the IRQ if it's already set up */
+		ppt_teardown_msix_intr(ppt, idx);
+
+		/* Allocate the IRQ resource */
+		ppt->msix.cookie[idx] = NULL;
+		rid = ppt->msix.startrid + idx;
+		ppt->msix.res[idx] = bus_alloc_resource_any(ppt->dev, SYS_RES_IRQ,
+							    &rid, RF_ACTIVE);
+		if (ppt->msix.res[idx] == NULL)
+			return (ENXIO);
+	
+		ppt->msix.arg[idx].pptdev = ppt;
+		ppt->msix.arg[idx].vec = msg;
+		ppt->msix.arg[idx].vcpu = (addr >> 12) & 0xFF;
+	
+		/* Setup the MSI-X interrupt */
+		error = bus_setup_intr(ppt->dev, ppt->msix.res[idx],
+				       INTR_TYPE_NET | INTR_MPSAFE,
+				       pptintr, NULL, &ppt->msix.arg[idx],
+				       &ppt->msix.cookie[idx]);
+	
+		if (error != 0) {
+			bus_teardown_intr(ppt->dev, ppt->msix.res[idx], ppt->msix.cookie[idx]);
+			bus_release_resource(ppt->dev, SYS_RES_IRQ, rid, ppt->msix.res[idx]);
+			ppt->msix.cookie[idx] = NULL;
+			ppt->msix.res[idx] = NULL;
+			return (ENXIO);
+		}
+	} else {
+		/* Masked, tear it down if it's already been set up */
+		ppt_teardown_msix_intr(ppt, idx);
+	}
+
+	return (0);
+}
+
diff --git a/sys/amd64/vmm/io/ppt.h b/sys/amd64/vmm/io/ppt.h
new file mode 100644
index 0000000..63c8228
--- /dev/null
+++ b/sys/amd64/vmm/io/ppt.h
@@ -0,0 +1,41 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _IO_PPT_H_
+#define	_IO_PPT_H_
+
+int	ppt_assign_device(struct vm *vm, int bus, int slot, int func);
+int	ppt_unassign_device(struct vm *vm, int bus, int slot, int func);
+int	ppt_unassign_all(struct vm *vm);
+int	ppt_map_mmio(struct vm *vm, int bus, int slot, int func,
+		     vm_paddr_t gpa, size_t len, vm_paddr_t hpa);
+int	ppt_setup_msi(struct vm *vm, int vcpu, int bus, int slot, int func,
+		      int destcpu, int vector, int numvec);
+int	ppt_setup_msix(struct vm *vm, int vcpu, int bus, int slot, int func,
+		       int idx, uint32_t msg, uint32_t vector_control, uint64_t addr);
+#endif
diff --git a/sys/amd64/vmm/io/vdev.c b/sys/amd64/vmm/io/vdev.c
new file mode 100644
index 0000000..cd6c5d1
--- /dev/null
+++ b/sys/amd64/vmm/io/vdev.c
@@ -0,0 +1,270 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+
+#include "vdev.h"
+
+struct vdev {
+	SLIST_ENTRY(vdev) 	 entry;
+	struct vdev_ops 	*ops;
+	void			*dev;
+};
+static SLIST_HEAD(, vdev)	vdev_head;
+static int 		  	vdev_count;
+
+struct vdev_region {
+	SLIST_ENTRY(vdev_region) 	 entry;
+	struct vdev_ops 		*ops;
+	void				*dev;
+	struct io_region		*io;
+};
+static SLIST_HEAD(, vdev_region)	 region_head;
+static int 		  		 region_count;
+
+static MALLOC_DEFINE(M_VDEV, "vdev", "vdev");
+
+#define VDEV_INIT 	(0)
+#define VDEV_RESET	(1)
+#define VDEV_HALT	(2)
+
+// static const char* vdev_event_str[] = {"VDEV_INIT", "VDEV_RESET", "VDEV_HALT"};
+
+static int
+vdev_system_event(int event)
+{
+	struct vdev 	*vd;
+	int		 rc;
+
+	// TODO: locking
+	SLIST_FOREACH(vd, &vdev_head, entry) {
+		// printf("%s : %s Device %s\n", __func__, vdev_event_str[event], vd->ops->name);
+		switch (event) {
+			case VDEV_INIT:
+				rc = vd->ops->init(vd->dev);
+				break;
+			case VDEV_RESET:
+				rc = vd->ops->reset(vd->dev);
+				break;
+			case VDEV_HALT:
+				rc = vd->ops->halt(vd->dev);
+				break;
+			default:
+				break;
+		}
+		if (rc) {
+			printf("vdev %s init failed rc=%d\n",
+			    vd->ops->name, rc);
+			return rc;
+		}
+	}
+	return 0;
+}
+
+int
+vdev_init(void)
+{
+	return vdev_system_event(VDEV_INIT);
+}
+
+int
+vdev_reset(void)
+{
+	return vdev_system_event(VDEV_RESET);
+}
+
+int
+vdev_halt(void)
+{
+	return vdev_system_event(VDEV_HALT);
+}
+
+void
+vdev_vm_init(void)
+{
+	SLIST_INIT(&vdev_head);
+	vdev_count = 0;
+
+	SLIST_INIT(&region_head);
+	region_count = 0;
+}
+void
+vdev_vm_cleanup(void)
+{
+	struct vdev *vd;
+     
+	// TODO: locking
+	while (!SLIST_EMPTY(&vdev_head)) {
+		vd = SLIST_FIRST(&vdev_head);
+		SLIST_REMOVE_HEAD(&vdev_head, entry);
+		free(vd, M_VDEV);
+		vdev_count--;
+	}
+}
+
+int
+vdev_register(struct vdev_ops *ops, void *dev)
+{
+	struct vdev *vd;
+	vd = malloc(sizeof(*vd), M_VDEV, M_WAITOK | M_ZERO); 
+	vd->ops = ops;
+	vd->dev = dev;
+	
+	// TODO: locking
+	SLIST_INSERT_HEAD(&vdev_head, vd, entry); 
+	vdev_count++;
+	return 0;
+}
+
+void
+vdev_unregister(void *dev)
+{
+	struct vdev 	*vd, *found;
+
+	found = NULL;
+	// TODO: locking
+	SLIST_FOREACH(vd, &vdev_head, entry) {
+		if (vd->dev == dev) {
+			found = vd;
+		}
+	}
+
+	if (found) {
+		SLIST_REMOVE(&vdev_head, found, vdev, entry);
+		free(found, M_VDEV);
+	}
+}
+
+#define IN_RANGE(val, start, end)	\
+    (((val) >= (start)) && ((val) < (end)))
+
+static struct vdev_region*
+vdev_find_region(struct io_region *io, void *dev) 
+{
+	struct 		vdev_region *region, *found;
+	uint64_t	region_base;
+	uint64_t	region_end;
+
+	found = NULL;
+
+	// TODO: locking
+	// FIXME: we should verify we are in the context the current
+	// 	  vcpu here as well.
+	SLIST_FOREACH(region, &region_head, entry) {
+		region_base = region->io->base;
+		region_end = region_base + region->io->len;
+		if (IN_RANGE(io->base, region_base, region_end) &&
+		    IN_RANGE(io->base+io->len, region_base, region_end+1) &&
+		    (dev && dev == region->dev)) {
+			found = region;
+			break;
+		}
+	}
+	return found;
+}
+
+int
+vdev_register_region(struct vdev_ops *ops, void *dev, struct io_region *io)
+{
+	struct vdev_region *region;
+
+	region = vdev_find_region(io, dev);
+	if (region) {
+		return -EEXIST;
+	}
+
+	region = malloc(sizeof(*region), M_VDEV, M_WAITOK | M_ZERO);
+	region->io = io;
+	region->ops = ops;
+	region->dev = dev;
+
+	// TODO: locking
+	SLIST_INSERT_HEAD(&region_head, region, entry); 
+	region_count++;
+
+	return 0;
+}
+
+void
+vdev_unregister_region(void *dev, struct io_region *io)
+{
+	struct vdev_region *region;
+
+	region = vdev_find_region(io, dev);
+	
+	if (region) {
+		SLIST_REMOVE(&region_head, region, vdev_region, entry);
+		free(region, M_VDEV);
+		region_count--;
+	}
+}
+
+static int
+vdev_memrw(uint64_t gpa, opsize_t size, uint64_t *data, int read)
+{
+	struct vdev_region 	*region;
+	struct io_region	 io;
+	region_attr_t		 attr;
+	int			 rc;
+
+	io.base = gpa;
+	io.len = size;
+
+	region = vdev_find_region(&io, NULL);
+	if (!region)
+		return -EINVAL;
+	
+	attr = (read) ? MMIO_READ : MMIO_WRITE;
+	if (!(region->io->attr & attr))
+		return -EPERM;
+
+	if (read)
+		rc = region->ops->memread(region->dev, gpa, size, data);
+	else 
+		rc = region->ops->memwrite(region->dev, gpa, size, *data);
+
+	return rc;
+}
+
+int
+vdev_memread(uint64_t gpa, opsize_t size, uint64_t *data)
+{
+	return vdev_memrw(gpa, size, data, 1);
+}
+
+int
+vdev_memwrite(uint64_t gpa, opsize_t size, uint64_t data)
+{
+	return vdev_memrw(gpa, size, &data, 0);
+}
diff --git a/sys/amd64/vmm/io/vdev.h b/sys/amd64/vmm/io/vdev.h
new file mode 100644
index 0000000..6feeba8
--- /dev/null
+++ b/sys/amd64/vmm/io/vdev.h
@@ -0,0 +1,84 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VDEV_H_
+#define	_VDEV_H_
+
+typedef enum {
+	BYTE	= 1,
+	WORD	= 2,
+	DWORD	= 4,
+	QWORD	= 8,
+} opsize_t;
+
+typedef enum {
+	MMIO_READ = 1,
+	MMIO_WRITE = 2,
+} region_attr_t;
+
+struct io_region {
+	uint64_t	base;
+	uint64_t	len;
+	region_attr_t	attr;
+	int		vcpu;
+};
+
+typedef int (*vdev_init_t)(void* dev);
+typedef int (*vdev_reset_t)(void* dev);
+typedef int (*vdev_halt_t)(void* dev);
+typedef int (*vdev_memread_t)(void* dev, uint64_t gpa, opsize_t size, uint64_t *data);
+typedef int (*vdev_memwrite_t)(void* dev, uint64_t gpa, opsize_t size, uint64_t data);
+
+
+struct vdev_ops {
+	const char	*name;
+	vdev_init_t	init;
+	vdev_reset_t	reset;
+	vdev_halt_t	halt;
+	vdev_memread_t	memread;
+	vdev_memwrite_t	memwrite;
+};
+
+
+void vdev_vm_init(void);
+void vdev_vm_cleanup(void);
+
+int  vdev_register(struct vdev_ops *ops, void *dev);
+void vdev_unregister(void *dev);
+
+int  vdev_register_region(struct vdev_ops *ops, void *dev, struct io_region *io);
+void vdev_unregister_region(void *dev, struct io_region *io);
+
+int vdev_init(void);
+int vdev_reset(void);
+int vdev_halt(void);
+int vdev_memread(uint64_t gpa, opsize_t size, uint64_t *data);
+int vdev_memwrite(uint64_t gpa, opsize_t size, uint64_t data);
+
+#endif	/* _VDEV_H_ */
+
diff --git a/sys/amd64/vmm/io/vlapic.c b/sys/amd64/vmm/io/vlapic.c
new file mode 100644
index 0000000..15fc6c2
--- /dev/null
+++ b/sys/amd64/vmm/io/vlapic.c
@@ -0,0 +1,901 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/systm.h>
+#include <sys/smp.h>
+
+#include <machine/clock.h>
+#include <x86/specialreg.h>
+#include <x86/apicreg.h>
+
+#include <machine/vmm.h>
+
+#include "vmm_lapic.h"
+#include "vmm_ktr.h"
+#include "vdev.h"
+#include "vlapic.h"
+
+#define	VLAPIC_CTR0(vlapic, format)					\
+	VMM_CTR0((vlapic)->vm, (vlapic)->vcpuid, format)
+
+#define	VLAPIC_CTR1(vlapic, format, p1)					\
+	VMM_CTR1((vlapic)->vm, (vlapic)->vcpuid, format, p1)
+
+#define	VLAPIC_CTR_IRR(vlapic, msg)					\
+do {									\
+	uint32_t *irrptr = &(vlapic)->apic.irr0;			\
+	irrptr[0] = irrptr[0];	/* silence compiler */			\
+	VLAPIC_CTR1((vlapic), msg " irr0 0x%08x", irrptr[0 << 2]);	\
+	VLAPIC_CTR1((vlapic), msg " irr1 0x%08x", irrptr[1 << 2]);	\
+	VLAPIC_CTR1((vlapic), msg " irr2 0x%08x", irrptr[2 << 2]);	\
+	VLAPIC_CTR1((vlapic), msg " irr3 0x%08x", irrptr[3 << 2]);	\
+	VLAPIC_CTR1((vlapic), msg " irr4 0x%08x", irrptr[4 << 2]);	\
+	VLAPIC_CTR1((vlapic), msg " irr5 0x%08x", irrptr[5 << 2]);	\
+	VLAPIC_CTR1((vlapic), msg " irr6 0x%08x", irrptr[6 << 2]);	\
+	VLAPIC_CTR1((vlapic), msg " irr7 0x%08x", irrptr[7 << 2]);	\
+} while (0)
+
+#define	VLAPIC_CTR_ISR(vlapic, msg)					\
+do {									\
+	uint32_t *isrptr = &(vlapic)->apic.isr0;			\
+	isrptr[0] = isrptr[0];	/* silence compiler */			\
+	VLAPIC_CTR1((vlapic), msg " isr0 0x%08x", isrptr[0 << 2]);	\
+	VLAPIC_CTR1((vlapic), msg " isr1 0x%08x", isrptr[1 << 2]);	\
+	VLAPIC_CTR1((vlapic), msg " isr2 0x%08x", isrptr[2 << 2]);	\
+	VLAPIC_CTR1((vlapic), msg " isr3 0x%08x", isrptr[3 << 2]);	\
+	VLAPIC_CTR1((vlapic), msg " isr4 0x%08x", isrptr[4 << 2]);	\
+	VLAPIC_CTR1((vlapic), msg " isr5 0x%08x", isrptr[5 << 2]);	\
+	VLAPIC_CTR1((vlapic), msg " isr6 0x%08x", isrptr[6 << 2]);	\
+	VLAPIC_CTR1((vlapic), msg " isr7 0x%08x", isrptr[7 << 2]);	\
+} while (0)
+
+static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
+
+#define	PRIO(x)			((x) >> 4)
+
+#define VLAPIC_VERSION		(16)
+#define VLAPIC_MAXLVT_ENTRIES	(5)
+
+#define	x2apic(vlapic)	(((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
+
+enum boot_state {
+	BS_INIT,
+	BS_SIPI,
+	BS_RUNNING
+};
+
+struct vlapic {
+	struct vm		*vm;
+	int			vcpuid;
+
+	struct io_region	*mmio;
+	struct vdev_ops		*ops;
+	struct LAPIC		 apic;
+
+	int			 esr_update;
+
+	int			 divisor;
+	int			 ccr_ticks;
+
+	/*
+	 * The 'isrvec_stk' is a stack of vectors injected by the local apic.
+	 * A vector is popped from the stack when the processor does an EOI.
+	 * The vector on the top of the stack is used to compute the
+	 * Processor Priority in conjunction with the TPR.
+	 */
+	uint8_t			 isrvec_stk[ISRVEC_STK_SIZE];
+	int			 isrvec_stk_top;
+
+	uint64_t		msr_apicbase;
+	enum boot_state		boot_state;
+};
+
+#define VLAPIC_BUS_FREQ	tsc_freq
+
+static int
+vlapic_timer_divisor(uint32_t dcr)
+{
+	switch (dcr & 0xB) {
+	case APIC_TDCR_2:
+		return (2);
+	case APIC_TDCR_4:
+		return (4);
+	case APIC_TDCR_8:
+		return (8);
+	case APIC_TDCR_16:
+		return (16);
+	case APIC_TDCR_32:
+		return (32);
+	case APIC_TDCR_64:
+		return (64);
+	case APIC_TDCR_128:
+		return (128);
+	default:
+		panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
+	}
+}
+
+static void
+vlapic_mask_lvts(uint32_t *lvts, int num_lvt)
+{
+	int i;
+	for (i = 0; i < num_lvt; i++) {
+		*lvts |= APIC_LVT_M;
+		lvts += 4;
+	}
+}
+
+#if 0
+static inline void
+vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
+{
+	printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
+	    *lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
+	    *lvt & APIC_LVTT_M);
+}
+#endif
+
+static uint64_t
+vlapic_get_ccr(struct vlapic *vlapic)
+{
+	struct LAPIC    *lapic = &vlapic->apic;
+	return lapic->ccr_timer;
+}
+
+static void
+vlapic_update_errors(struct vlapic *vlapic)
+{
+	struct LAPIC    *lapic = &vlapic->apic;
+	lapic->esr = 0; // XXX 
+}
+
+static void
+vlapic_init_ipi(struct vlapic *vlapic)
+{
+	struct LAPIC    *lapic = &vlapic->apic;
+	lapic->version = VLAPIC_VERSION;
+	lapic->version |= (VLAPIC_MAXLVT_ENTRIES < MAXLVTSHIFT);
+	lapic->dfr = 0xffffffff;
+	lapic->svr = APIC_SVR_VECTOR;
+	vlapic_mask_lvts(&lapic->lvt_timer, VLAPIC_MAXLVT_ENTRIES+1);
+}
+
+static int
+vlapic_op_reset(void* dev)
+{
+	struct vlapic 	*vlapic = (struct vlapic*)dev;
+	struct LAPIC	*lapic = &vlapic->apic;
+
+	memset(lapic, 0, sizeof(*lapic));
+	lapic->apr = vlapic->vcpuid;
+	vlapic_init_ipi(vlapic);
+	vlapic->divisor = vlapic_timer_divisor(lapic->dcr_timer);
+
+	if (vlapic->vcpuid == 0)
+		vlapic->boot_state = BS_RUNNING;	/* BSP */
+	else
+		vlapic->boot_state = BS_INIT;		/* AP */
+	
+	return 0;
+
+}
+
+static int
+vlapic_op_init(void* dev)
+{
+	struct vlapic *vlapic = (struct vlapic*)dev;
+	vdev_register_region(vlapic->ops, vlapic, vlapic->mmio);
+	return vlapic_op_reset(dev);
+}
+
+static int
+vlapic_op_halt(void* dev)
+{
+	struct vlapic *vlapic = (struct vlapic*)dev;
+	vdev_unregister_region(vlapic, vlapic->mmio);
+	return 0;
+
+}
+
+void
+vlapic_set_intr_ready(struct vlapic *vlapic, int vector)
+{
+	struct LAPIC	*lapic = &vlapic->apic;
+	uint32_t	*irrptr;
+	int		idx;
+
+	if (vector < 0 || vector >= 256)
+		panic("vlapic_set_intr_ready: invalid vector %d\n", vector);
+
+	idx = (vector / 32) * 4;
+	irrptr = &lapic->irr0;
+	atomic_set_int(&irrptr[idx], 1 << (vector % 32));
+	VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
+}
+
+static void
+vlapic_start_timer(struct vlapic *vlapic, uint32_t elapsed)
+{
+	uint32_t icr_timer;
+
+	icr_timer = vlapic->apic.icr_timer;
+
+	vlapic->ccr_ticks = ticks;
+	if (elapsed < icr_timer)
+		vlapic->apic.ccr_timer = icr_timer - elapsed;
+	else {
+		/*
+		 * This can happen when the guest is trying to run its local
+		 * apic timer higher that the setting of 'hz' in the host.
+		 *
+		 * We deal with this by running the guest local apic timer
+		 * at the rate of the host's 'hz' setting.
+		 */
+		vlapic->apic.ccr_timer = 0;
+	}
+}
+
+static __inline uint32_t *
+vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
+{
+	struct LAPIC	*lapic = &vlapic->apic;
+	int 		 i;
+
+	if (offset < APIC_OFFSET_TIMER_LVT || offset > APIC_OFFSET_ERROR_LVT) {
+		panic("vlapic_get_lvt: invalid LVT\n");
+	}
+	i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
+	return ((&lapic->lvt_timer) + i);;
+}
+
+#if 1
+static void
+dump_isrvec_stk(struct vlapic *vlapic)
+{
+	int i;
+	uint32_t *isrptr;
+
+	isrptr = &vlapic->apic.isr0;
+	for (i = 0; i < 8; i++)
+		printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
+
+	for (i = 0; i <= vlapic->isrvec_stk_top; i++)
+		printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
+}
+#endif
+
+/*
+ * Algorithm adopted from section "Interrupt, Task and Processor Priority"
+ * in Intel Architecture Manual Vol 3a.
+ */
+static void
+vlapic_update_ppr(struct vlapic *vlapic)
+{
+	int isrvec, tpr, ppr;
+
+	/*
+	 * Note that the value on the stack at index 0 is always 0.
+	 *
+	 * This is a placeholder for the value of ISRV when none of the
+	 * bits is set in the ISRx registers.
+	 */
+	isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
+	tpr = vlapic->apic.tpr;
+
+#if 1
+	{
+		int i, lastprio, curprio, vector, idx;
+		uint32_t *isrptr;
+
+		if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
+			panic("isrvec_stk is corrupted: %d", isrvec);
+
+		/*
+		 * Make sure that the priority of the nested interrupts is
+		 * always increasing.
+		 */
+		lastprio = -1;
+		for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
+			curprio = PRIO(vlapic->isrvec_stk[i]);
+			if (curprio <= lastprio) {
+				dump_isrvec_stk(vlapic);
+				panic("isrvec_stk does not satisfy invariant");
+			}
+			lastprio = curprio;
+		}
+
+		/*
+		 * Make sure that each bit set in the ISRx registers has a
+		 * corresponding entry on the isrvec stack.
+		 */
+		i = 1;
+		isrptr = &vlapic->apic.isr0;
+		for (vector = 0; vector < 256; vector++) {
+			idx = (vector / 32) * 4;
+			if (isrptr[idx] & (1 << (vector % 32))) {
+				if (i > vlapic->isrvec_stk_top ||
+				    vlapic->isrvec_stk[i] != vector) {
+					dump_isrvec_stk(vlapic);
+					panic("ISR and isrvec_stk out of sync");
+				}
+				i++;
+			}
+		}
+	}
+#endif
+
+	if (PRIO(tpr) >= PRIO(isrvec))
+		ppr = tpr;
+	else
+		ppr = isrvec & 0xf0;
+
+	vlapic->apic.ppr = ppr;
+	VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
+}
+
+static void
+vlapic_process_eoi(struct vlapic *vlapic)
+{
+	struct LAPIC	*lapic = &vlapic->apic;
+	uint32_t	*isrptr;
+	int		i, idx, bitpos;
+
+	isrptr = &lapic->isr0;
+
+	/*
+	 * The x86 architecture reserves the the first 32 vectors for use
+	 * by the processor.
+	 */
+	for (i = 7; i > 0; i--) {
+		idx = i * 4;
+		bitpos = fls(isrptr[idx]);
+		if (bitpos != 0) {
+			if (vlapic->isrvec_stk_top <= 0) {
+				panic("invalid vlapic isrvec_stk_top %d",
+				      vlapic->isrvec_stk_top);
+			}
+			isrptr[idx] &= ~(1 << (bitpos - 1));
+			VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
+			vlapic->isrvec_stk_top--;
+			vlapic_update_ppr(vlapic);
+			return;
+		}
+	}
+}
+
+static __inline int
+vlapic_get_lvt_field(uint32_t *lvt, uint32_t mask)
+{
+	return (*lvt & mask);
+}
+
+static __inline int
+vlapic_periodic_timer(struct vlapic *vlapic)
+{
+	uint32_t *lvt;
+	
+	lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
+
+	return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
+}
+
+static void
+vlapic_fire_timer(struct vlapic *vlapic)
+{
+	int vector;
+	uint32_t *lvt;
+	
+	lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
+
+	if (!vlapic_get_lvt_field(lvt, APIC_LVTT_M)) {
+		vector = vlapic_get_lvt_field(lvt,APIC_LVTT_VECTOR);
+		vlapic_set_intr_ready(vlapic, vector);
+	}
+}
+
+static int
+lapic_process_icr(struct vlapic *vlapic, uint64_t icrval)
+{
+	int i;
+	cpuset_t dmask;
+	uint32_t dest, vec, mode;
+	struct vlapic *vlapic2;
+	struct vm_exit *vmexit;
+	
+	if (x2apic(vlapic))
+		dest = icrval >> 32;
+	else
+		dest = icrval >> (32 + 24);
+	vec = icrval & APIC_VECTOR_MASK;
+	mode = icrval & APIC_DELMODE_MASK;
+
+	if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
+		switch (icrval & APIC_DEST_MASK) {
+		case APIC_DEST_DESTFLD:
+			CPU_SETOF(dest, &dmask);
+			break;
+		case APIC_DEST_SELF:
+			CPU_SETOF(vlapic->vcpuid, &dmask);
+			break;
+		case APIC_DEST_ALLISELF:
+			dmask = vm_active_cpus(vlapic->vm);
+			break;
+		case APIC_DEST_ALLESELF:
+			dmask = vm_active_cpus(vlapic->vm);
+			CPU_CLR(vlapic->vcpuid, &dmask);
+			break;
+		}
+
+		while ((i = cpusetobj_ffs(&dmask)) != 0) {
+			i--;
+			CPU_CLR(i, &dmask);
+			if (mode == APIC_DELMODE_FIXED)
+				lapic_set_intr(vlapic->vm, i, vec);
+			else
+				vm_inject_nmi(vlapic->vm, i);
+		}
+
+		return (0);	/* handled completely in the kernel */
+	}
+
+	if (mode == APIC_DELMODE_INIT) {
+		if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
+			return (0);
+
+		if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
+			vlapic2 = vm_lapic(vlapic->vm, dest);
+
+			/* move from INIT to waiting-for-SIPI state */
+			if (vlapic2->boot_state == BS_INIT) {
+				vlapic2->boot_state = BS_SIPI;
+			}
+
+			return (0);
+		}
+	}
+
+	if (mode == APIC_DELMODE_STARTUP) {
+		if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
+			vlapic2 = vm_lapic(vlapic->vm, dest);
+
+			/*
+			 * Ignore SIPIs in any state other than wait-for-SIPI
+			 */
+			if (vlapic2->boot_state != BS_SIPI)
+				return (0);
+
+			vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
+			vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
+			vmexit->u.spinup_ap.vcpu = dest;
+			vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
+
+			/*
+			 * XXX this assumes that the startup IPI always succeeds
+			 */
+			vlapic2->boot_state = BS_RUNNING;
+			vm_activate_cpu(vlapic2->vm, dest);
+
+			return (0);
+		}
+	}
+
+	/*
+	 * This will cause a return to userland.
+	 */
+	return (1);
+}
+
+int
+vlapic_pending_intr(struct vlapic *vlapic)
+{
+	struct LAPIC	*lapic = &vlapic->apic;
+	int	  	 idx, i, bitpos, vector;
+	uint32_t	*irrptr, val;
+
+	irrptr = &lapic->irr0;
+
+	/*
+	 * The x86 architecture reserves the the first 32 vectors for use
+	 * by the processor.
+	 */
+	for (i = 7; i > 0; i--) {
+		idx = i * 4;
+		val = atomic_load_acq_int(&irrptr[idx]);
+		bitpos = fls(val);
+		if (bitpos != 0) {
+			vector = i * 32 + (bitpos - 1);
+			if (PRIO(vector) > PRIO(lapic->ppr)) {
+				VLAPIC_CTR1(vlapic, "pending intr %d", vector);
+				return (vector);
+			} else 
+				break;
+		}
+	}
+	VLAPIC_CTR0(vlapic, "no pending intr");
+	return (-1);
+}
+
+void
+vlapic_intr_accepted(struct vlapic *vlapic, int vector)
+{
+	struct LAPIC	*lapic = &vlapic->apic;
+	uint32_t	*irrptr, *isrptr;
+	int		idx, stk_top;
+
+	/*
+	 * clear the ready bit for vector being accepted in irr 
+	 * and set the vector as in service in isr.
+	 */
+	idx = (vector / 32) * 4;
+
+	irrptr = &lapic->irr0;
+	atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
+	VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
+
+	isrptr = &lapic->isr0;
+	isrptr[idx] |= 1 << (vector % 32);
+	VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
+
+	/*
+	 * Update the PPR
+	 */
+	vlapic->isrvec_stk_top++;
+
+	stk_top = vlapic->isrvec_stk_top;
+	if (stk_top >= ISRVEC_STK_SIZE)
+		panic("isrvec_stk_top overflow %d", stk_top);
+
+	vlapic->isrvec_stk[stk_top] = vector;
+	vlapic_update_ppr(vlapic);
+}
+
+int
+vlapic_op_mem_read(void* dev, uint64_t gpa, opsize_t size, uint64_t *data)
+{
+	struct vlapic 	*vlapic = (struct vlapic*)dev;
+	struct LAPIC	*lapic = &vlapic->apic;
+	uint64_t	 offset = gpa & ~(PAGE_SIZE);
+	uint32_t	*reg;
+	int		 i;
+
+	if (offset > sizeof(*lapic)) {
+		*data = 0;
+		return 0;
+	}
+	
+	offset &= ~3;
+	switch(offset)
+	{
+		case APIC_OFFSET_ID:
+			if (x2apic(vlapic))
+				*data = vlapic->vcpuid;
+			else
+				*data = vlapic->vcpuid << 24;
+			break;
+		case APIC_OFFSET_VER:
+			*data = lapic->version;
+			break;
+		case APIC_OFFSET_TPR:
+			*data = lapic->tpr;
+			break;
+		case APIC_OFFSET_APR:
+			*data = lapic->apr;
+			break;
+		case APIC_OFFSET_PPR:
+			*data = lapic->ppr;
+			break;
+		case APIC_OFFSET_EOI:
+			*data = lapic->eoi;
+			break;
+		case APIC_OFFSET_LDR:
+			*data = lapic->ldr;
+			break;
+		case APIC_OFFSET_DFR:
+			*data = lapic->dfr;
+			break;
+		case APIC_OFFSET_SVR:
+			*data = lapic->svr;
+			break;
+		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
+			i = (offset - APIC_OFFSET_ISR0) >> 2;
+			reg = &lapic->isr0;
+			*data = *(reg + i);
+			break;
+		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
+			i = (offset - APIC_OFFSET_TMR0) >> 2;
+			reg = &lapic->tmr0;
+			*data = *(reg + i);
+			break;
+		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
+			i = (offset - APIC_OFFSET_IRR0) >> 2;
+			reg = &lapic->irr0;
+			*data = atomic_load_acq_int(reg + i);
+			break;
+		case APIC_OFFSET_ESR:
+			*data = lapic->esr;
+			break;
+		case APIC_OFFSET_ICR_LOW: 
+			*data = lapic->icr_lo;
+			break;
+		case APIC_OFFSET_ICR_HI: 
+			*data = lapic->icr_hi;
+			break;
+		case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
+			reg = vlapic_get_lvt(vlapic, offset);	
+			*data = *(reg);
+			break;
+		case APIC_OFFSET_ICR:
+			*data = lapic->icr_timer;
+			break;
+		case APIC_OFFSET_CCR:
+			*data = vlapic_get_ccr(vlapic);
+			break;
+		case APIC_OFFSET_DCR:
+			*data = lapic->dcr_timer;
+			break;
+		case APIC_OFFSET_RRR:
+		default:
+			*data = 0;
+			break;
+	}
+	return 0;
+}
+
+int
+vlapic_op_mem_write(void* dev, uint64_t gpa, opsize_t size, uint64_t data)
+{
+	struct vlapic 	*vlapic = (struct vlapic*)dev;
+	struct LAPIC	*lapic = &vlapic->apic;
+	uint64_t	 offset = gpa & ~(PAGE_SIZE);
+	uint32_t	*reg;
+	int		retval;
+
+	if (offset > sizeof(*lapic)) {
+		return 0;
+	}
+
+	retval = 0;
+	offset &= ~3;
+	switch(offset)
+	{
+		case APIC_OFFSET_ID:
+			break;
+		case APIC_OFFSET_TPR:
+			lapic->tpr = data & 0xff;
+			vlapic_update_ppr(vlapic);
+			break;
+		case APIC_OFFSET_EOI:
+			vlapic_process_eoi(vlapic);
+			break;
+		case APIC_OFFSET_LDR:
+			break;
+		case APIC_OFFSET_DFR:
+			break;
+		case APIC_OFFSET_SVR:
+			lapic->svr = data;
+			break;
+		case APIC_OFFSET_ICR_LOW: 
+			if (!x2apic(vlapic)) {
+				data &= 0xffffffff;
+				data |= (uint64_t)lapic->icr_hi << 32;
+			}
+			retval = lapic_process_icr(vlapic, data);
+			break;
+		case APIC_OFFSET_ICR_HI:
+			if (!x2apic(vlapic)) {
+				retval = 0;
+				lapic->icr_hi = data;
+			}
+			break;
+		case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
+			reg = vlapic_get_lvt(vlapic, offset);	
+			if (!(lapic->svr & APIC_SVR_ENABLE)) {
+				data |= APIC_LVT_M;
+			}
+			*reg = data;
+			// vlapic_dump_lvt(offset, reg);
+			break;
+		case APIC_OFFSET_ICR:
+			lapic->icr_timer = data;
+			vlapic_start_timer(vlapic, 0);
+			break;
+
+		case APIC_OFFSET_DCR:
+			lapic->dcr_timer = data;
+			vlapic->divisor = vlapic_timer_divisor(data);
+			break;
+
+		case APIC_OFFSET_ESR:
+			vlapic_update_errors(vlapic);
+			break;
+		case APIC_OFFSET_VER:
+		case APIC_OFFSET_APR:
+		case APIC_OFFSET_PPR:
+		case APIC_OFFSET_RRR:
+		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
+		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
+		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
+		case APIC_OFFSET_CCR:
+		default:
+			// Read only.
+			break;
+	}
+
+	return (retval);
+}
+
+int
+vlapic_timer_tick(struct vlapic *vlapic)
+{
+	int curticks, delta, periodic, fired;
+	uint32_t ccr;
+	uint32_t decrement, leftover;
+
+restart:
+	curticks = ticks;
+	delta = curticks - vlapic->ccr_ticks;
+
+	/* Local APIC timer is disabled */
+	if (vlapic->apic.icr_timer == 0)
+		return (-1);
+
+	/* One-shot mode and timer has already counted down to zero */
+	periodic = vlapic_periodic_timer(vlapic);
+	if (!periodic && vlapic->apic.ccr_timer == 0)
+		return (-1);
+	/*
+	 * The 'curticks' and 'ccr_ticks' are out of sync by more than
+	 * 2^31 ticks. We deal with this by restarting the timer.
+	 */
+	if (delta < 0) {
+		vlapic_start_timer(vlapic, 0);
+		goto restart;
+	}
+
+	fired = 0;
+	decrement = (VLAPIC_BUS_FREQ / vlapic->divisor) / hz;
+
+	vlapic->ccr_ticks = curticks;
+	ccr = vlapic->apic.ccr_timer;
+
+	while (delta-- > 0) {
+		if (ccr > decrement) {
+			ccr -= decrement;
+			continue;
+		}
+
+		/* Trigger the local apic timer interrupt */
+		vlapic_fire_timer(vlapic);
+		if (periodic) {
+			leftover = decrement - ccr;
+			vlapic_start_timer(vlapic, leftover);
+			ccr = vlapic->apic.ccr_timer;
+		} else {
+			/*
+			 * One-shot timer has counted down to zero.
+			 */
+			ccr = 0;
+		}
+		fired = 1;
+		break;
+	}
+
+	vlapic->apic.ccr_timer = ccr;
+
+	if (!fired)
+		return ((ccr / decrement) + 1);
+	else
+		return (0);
+}
+
+struct vdev_ops vlapic_dev_ops = {
+	.name = "vlapic",
+	.init = vlapic_op_init,
+	.reset = vlapic_op_reset,
+	.halt = vlapic_op_halt,
+	.memread = vlapic_op_mem_read,
+	.memwrite = vlapic_op_mem_write,
+};
+static struct io_region vlapic_mmio[VM_MAXCPU];
+
+struct vlapic *
+vlapic_init(struct vm *vm, int vcpuid)
+{
+	struct vlapic 		*vlapic;
+
+	vlapic = malloc(sizeof(struct vlapic), M_VLAPIC, M_WAITOK | M_ZERO);
+	vlapic->vm = vm;
+	vlapic->vcpuid = vcpuid;
+
+	vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
+
+	if (vcpuid == 0)
+		vlapic->msr_apicbase |= APICBASE_BSP;
+
+	vlapic->ops = &vlapic_dev_ops;
+
+	vlapic->mmio = vlapic_mmio + vcpuid;
+	vlapic->mmio->base = DEFAULT_APIC_BASE;
+	vlapic->mmio->len = PAGE_SIZE;
+	vlapic->mmio->attr = MMIO_READ|MMIO_WRITE;
+	vlapic->mmio->vcpu = vcpuid;
+
+	vdev_register(&vlapic_dev_ops, vlapic);
+
+	vlapic_op_init(vlapic);
+
+	return (vlapic);
+}
+
+void
+vlapic_cleanup(struct vlapic *vlapic)
+{
+	vlapic_op_halt(vlapic);
+	vdev_unregister(vlapic);
+	free(vlapic, M_VLAPIC);
+}
+
+uint64_t
+vlapic_get_apicbase(struct vlapic *vlapic)
+{
+
+	return (vlapic->msr_apicbase);
+}
+
+void
+vlapic_set_apicbase(struct vlapic *vlapic, uint64_t val)
+{
+	int err;
+	enum x2apic_state state;
+
+	err = vm_get_x2apic_state(vlapic->vm, vlapic->vcpuid, &state);
+	if (err)
+		panic("vlapic_set_apicbase: err %d fetching x2apic state", err);
+
+	if (state == X2APIC_DISABLED)
+		val &= ~APICBASE_X2APIC;
+
+	vlapic->msr_apicbase = val;
+}
+
+void
+vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
+{
+	struct vlapic *vlapic;
+
+	vlapic = vm_lapic(vm, vcpuid);
+
+	if (state == X2APIC_DISABLED)
+		vlapic->msr_apicbase &= ~APICBASE_X2APIC;
+}
diff --git a/sys/amd64/vmm/io/vlapic.h b/sys/amd64/vmm/io/vlapic.h
new file mode 100644
index 0000000..00de019
--- /dev/null
+++ b/sys/amd64/vmm/io/vlapic.h
@@ -0,0 +1,111 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VLAPIC_H_
+#define	_VLAPIC_H_
+
+#include "vdev.h"
+
+struct vm;
+  
+/*
+ * Map of APIC Registers:       Offset  Description          		 	Access
+ */
+#define APIC_OFFSET_ID 		0x20    // Local APIC ID               		R/W
+#define APIC_OFFSET_VER 	0x30    // Local APIC Version              	R
+#define APIC_OFFSET_TPR 	0x80    // Task Priority Register          	R/W
+#define APIC_OFFSET_APR 	0x90    // Arbitration Priority Register   	R
+#define APIC_OFFSET_PPR 	0xA0    // Processor Priority Register     	R
+#define APIC_OFFSET_EOI 	0xB0    // EOI Register                    	W
+#define APIC_OFFSET_RRR 	0xC0    // Remote read                     	R
+#define APIC_OFFSET_LDR 	0xD0    // Logical Destination             	R/W
+#define APIC_OFFSET_DFR 	0xE0    // Destination Format Register     	0..27 R;  28..31 R/W
+#define APIC_OFFSET_SVR 	0xF0    // Spurious Interrupt Vector Reg.  	0..3  R;  4..9   R/W
+#define APIC_OFFSET_ISR0 	0x100   // ISR  000-031                    	R
+#define APIC_OFFSET_ISR1 	0x110   // ISR  032-063                    	R
+#define APIC_OFFSET_ISR2 	0x120   // ISR  064-095                    	R
+#define APIC_OFFSET_ISR3 	0x130   // ISR  095-128                    	R
+#define APIC_OFFSET_ISR4 	0x140   // ISR  128-159                    	R
+#define APIC_OFFSET_ISR5 	0x150   // ISR  160-191                    	R
+#define APIC_OFFSET_ISR6 	0x160   // ISR  192-223                    	R
+#define APIC_OFFSET_ISR7 	0x170   // ISR  224-255                    	R
+#define APIC_OFFSET_TMR0 	0x180   // TMR  000-031                    	R
+#define APIC_OFFSET_TMR1 	0x190   // TMR  032-063                    	R
+#define APIC_OFFSET_TMR2 	0x1A0   // TMR  064-095                    	R
+#define APIC_OFFSET_TMR3 	0x1B0   // TMR  095-128                    	R
+#define APIC_OFFSET_TMR4 	0x1C0   // TMR  128-159                    	R
+#define APIC_OFFSET_TMR5 	0x1D0   // TMR  160-191                    	R
+#define APIC_OFFSET_TMR6 	0x1E0   // TMR  192-223                    	R
+#define APIC_OFFSET_TMR7 	0x1F0   // TMR  224-255                    	R
+#define APIC_OFFSET_IRR0 	0x200   // IRR  000-031                    	R
+#define APIC_OFFSET_IRR1 	0x210   // IRR  032-063                    	R
+#define APIC_OFFSET_IRR2 	0x220   // IRR  064-095                    	R
+#define APIC_OFFSET_IRR3 	0x230   // IRR  095-128                    	R
+#define APIC_OFFSET_IRR4 	0x240   // IRR  128-159                    	R
+#define APIC_OFFSET_IRR5 	0x250   // IRR  160-191                    	R
+#define APIC_OFFSET_IRR6 	0x260   // IRR  192-223                    	R
+#define APIC_OFFSET_IRR7 	0x270   // IRR  224-255                    	R
+#define APIC_OFFSET_ESR		0x280   // Error Status Register           	R
+#define APIC_OFFSET_ICR_LOW 	0x300   // Interrupt Command Reg. (0-31)   	R/W
+#define APIC_OFFSET_ICR_HI 	0x310   // Interrupt Command Reg. (32-63)  	R/W
+#define APIC_OFFSET_TIMER_LVT 	0x320   // Local Vector Table (Timer)      	R/W
+#define APIC_OFFSET_THERM_LVT 	0x330   // Local Vector Table (Thermal)    	R/W (PIV+)
+#define APIC_OFFSET_PERF_LVT 	0x340   // Local Vector Table (Performance) 	R/W (P6+)
+#define APIC_OFFSET_LINT0_LVT 	0x350   // Local Vector Table (LINT0)      	R/W
+#define APIC_OFFSET_LINT1_LVT 	0x360 	// Local Vector Table (LINT1)      	R/W
+#define APIC_OFFSET_ERROR_LVT 	0x370   // Local Vector Table (ERROR)      	R/W
+#define APIC_OFFSET_ICR 	0x380   // Initial Count Reg. for Timer    	R/W
+#define APIC_OFFSET_CCR 	0x390   // Current Count of Timer          	R
+#define APIC_OFFSET_DCR 	0x3E0   // Timer Divide Configuration Reg. 	R/W
+
+/*
+ * 16 priority levels with at most one vector injected per level.
+ */
+#define	ISRVEC_STK_SIZE		(16 + 1)
+
+enum x2apic_state;
+
+struct vlapic *vlapic_init(struct vm *vm, int vcpuid);
+void vlapic_cleanup(struct vlapic *vlapic);
+
+int vlapic_op_mem_write(void* dev, uint64_t gpa,
+    			opsize_t size, uint64_t data);
+
+int vlapic_op_mem_read(void* dev, uint64_t gpa,
+    			opsize_t size, uint64_t *data);
+
+int vlapic_pending_intr(struct vlapic *vlapic);
+void vlapic_intr_accepted(struct vlapic *vlapic, int vector);
+void vlapic_set_intr_ready(struct vlapic *vlapic, int vector);
+int vlapic_timer_tick(struct vlapic *vlapic);
+
+uint64_t vlapic_get_apicbase(struct vlapic *vlapic);
+void vlapic_set_apicbase(struct vlapic *vlapic, uint64_t val);
+void vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state s);
+
+#endif	/* _VLAPIC_H_ */
diff --git a/sys/amd64/vmm/vmm.c b/sys/amd64/vmm/vmm.c
new file mode 100644
index 0000000..82d4baa
--- /dev/null
+++ b/sys/amd64/vmm/vmm.c
@@ -0,0 +1,1038 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/sysctl.h>
+#include <sys/malloc.h>
+#include <sys/pcpu.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/sched.h>
+#include <sys/smp.h>
+#include <sys/systm.h>
+
+#include <vm/vm.h>
+
+#include <machine/vm.h>
+#include <machine/pcb.h>
+#include <machine/smp.h>
+#include <x86/apicreg.h>
+
+#include <machine/vmm.h>
+#include "vmm_host.h"
+#include "vmm_mem.h"
+#include "vmm_util.h"
+#include <machine/vmm_dev.h>
+#include "vlapic.h"
+#include "vmm_msr.h"
+#include "vmm_ipi.h"
+#include "vmm_stat.h"
+#include "vmm_lapic.h"
+
+#include "io/ppt.h"
+#include "io/iommu.h"
+
+struct vlapic;
+
+struct vcpu {
+	int		flags;
+	enum vcpu_state	state;
+	struct mtx	mtx;
+	int		pincpu;		/* host cpuid this vcpu is bound to */
+	int		hostcpu;	/* host cpuid this vcpu last ran on */
+	uint64_t	guest_msrs[VMM_MSR_NUM];
+	struct vlapic	*vlapic;
+	int		 vcpuid;
+	struct savefpu	*guestfpu;	/* guest fpu state */
+	void		*stats;
+	struct vm_exit	exitinfo;
+	enum x2apic_state x2apic_state;
+	int		nmi_pending;
+};
+#define	VCPU_F_PINNED	0x0001
+
+#define	VCPU_PINCPU(vm, vcpuid)	\
+    ((vm->vcpu[vcpuid].flags & VCPU_F_PINNED) ? vm->vcpu[vcpuid].pincpu : -1)
+
+#define	VCPU_UNPIN(vm, vcpuid)	(vm->vcpu[vcpuid].flags &= ~VCPU_F_PINNED)
+
+#define	VCPU_PIN(vm, vcpuid, host_cpuid)				\
+do {									\
+	vm->vcpu[vcpuid].flags |= VCPU_F_PINNED;			\
+	vm->vcpu[vcpuid].pincpu = host_cpuid;				\
+} while(0)
+
+#define	vcpu_lock_init(v)	mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN)
+#define	vcpu_lock(v)		mtx_lock_spin(&((v)->mtx))
+#define	vcpu_unlock(v)		mtx_unlock_spin(&((v)->mtx))
+
+#define	VM_MAX_MEMORY_SEGMENTS	2
+
+struct vm {
+	void		*cookie;	/* processor-specific data */
+	void		*iommu;		/* iommu-specific data */
+	struct vcpu	vcpu[VM_MAXCPU];
+	int		num_mem_segs;
+	struct vm_memory_segment mem_segs[VM_MAX_MEMORY_SEGMENTS];
+	char		name[VM_MAX_NAMELEN];
+
+	/*
+	 * Set of active vcpus.
+	 * An active vcpu is one that has been started implicitly (BSP) or
+	 * explicitly (AP) by sending it a startup ipi.
+	 */
+	cpuset_t	active_cpus;
+};
+
+static struct vmm_ops *ops;
+#define	VMM_INIT()	(ops != NULL ? (*ops->init)() : 0)
+#define	VMM_CLEANUP()	(ops != NULL ? (*ops->cleanup)() : 0)
+
+#define	VMINIT(vm)	(ops != NULL ? (*ops->vminit)(vm): NULL)
+#define	VMRUN(vmi, vcpu, rip) \
+	(ops != NULL ? (*ops->vmrun)(vmi, vcpu, rip) : ENXIO)
+#define	VMCLEANUP(vmi)	(ops != NULL ? (*ops->vmcleanup)(vmi) : NULL)
+#define	VMMMAP_SET(vmi, gpa, hpa, len, attr, prot, spm)			\
+    	(ops != NULL ? 							\
+    	(*ops->vmmmap_set)(vmi, gpa, hpa, len, attr, prot, spm) :	\
+	ENXIO)
+#define	VMMMAP_GET(vmi, gpa) \
+	(ops != NULL ? (*ops->vmmmap_get)(vmi, gpa) : ENXIO)
+#define	VMGETREG(vmi, vcpu, num, retval)		\
+	(ops != NULL ? (*ops->vmgetreg)(vmi, vcpu, num, retval) : ENXIO)
+#define	VMSETREG(vmi, vcpu, num, val)		\
+	(ops != NULL ? (*ops->vmsetreg)(vmi, vcpu, num, val) : ENXIO)
+#define	VMGETDESC(vmi, vcpu, num, desc)		\
+	(ops != NULL ? (*ops->vmgetdesc)(vmi, vcpu, num, desc) : ENXIO)
+#define	VMSETDESC(vmi, vcpu, num, desc)		\
+	(ops != NULL ? (*ops->vmsetdesc)(vmi, vcpu, num, desc) : ENXIO)
+#define	VMINJECT(vmi, vcpu, type, vec, ec, ecv)	\
+	(ops != NULL ? (*ops->vminject)(vmi, vcpu, type, vec, ec, ecv) : ENXIO)
+#define	VMGETCAP(vmi, vcpu, num, retval)	\
+	(ops != NULL ? (*ops->vmgetcap)(vmi, vcpu, num, retval) : ENXIO)
+#define	VMSETCAP(vmi, vcpu, num, val)		\
+	(ops != NULL ? (*ops->vmsetcap)(vmi, vcpu, num, val) : ENXIO)
+
+#define	fpu_start_emulating()	load_cr0(rcr0() | CR0_TS)
+#define	fpu_stop_emulating()	clts()
+
+static MALLOC_DEFINE(M_VM, "vm", "vm");
+CTASSERT(VMM_MSR_NUM <= 64);	/* msr_mask can keep track of up to 64 msrs */
+
+/* statistics */
+static VMM_STAT_DEFINE(VCPU_TOTAL_RUNTIME, "vcpu total runtime");
+
+static void
+vcpu_cleanup(struct vcpu *vcpu)
+{
+	vlapic_cleanup(vcpu->vlapic);
+	vmm_stat_free(vcpu->stats);	
+	fpu_save_area_free(vcpu->guestfpu);
+}
+
+static void
+vcpu_init(struct vm *vm, uint32_t vcpu_id)
+{
+	struct vcpu *vcpu;
+	
+	vcpu = &vm->vcpu[vcpu_id];
+
+	vcpu_lock_init(vcpu);
+	vcpu->hostcpu = NOCPU;
+	vcpu->vcpuid = vcpu_id;
+	vcpu->vlapic = vlapic_init(vm, vcpu_id);
+	vm_set_x2apic_state(vm, vcpu_id, X2APIC_ENABLED);
+	vcpu->guestfpu = fpu_save_area_alloc();
+	fpu_save_area_reset(vcpu->guestfpu);
+	vcpu->stats = vmm_stat_alloc();
+}
+
+struct vm_exit *
+vm_exitinfo(struct vm *vm, int cpuid)
+{
+	struct vcpu *vcpu;
+
+	if (cpuid < 0 || cpuid >= VM_MAXCPU)
+		panic("vm_exitinfo: invalid cpuid %d", cpuid);
+
+	vcpu = &vm->vcpu[cpuid];
+
+	return (&vcpu->exitinfo);
+}
+
+static int
+vmm_init(void)
+{
+	int error;
+
+	vmm_host_state_init();
+	vmm_ipi_init();
+
+	error = vmm_mem_init();
+	if (error)
+		return (error);
+	
+	if (vmm_is_intel())
+		ops = &vmm_ops_intel;
+	else if (vmm_is_amd())
+		ops = &vmm_ops_amd;
+	else
+		return (ENXIO);
+
+	vmm_msr_init();
+
+	return (VMM_INIT());
+}
+
+static int
+vmm_handler(module_t mod, int what, void *arg)
+{
+	int error;
+
+	switch (what) {
+	case MOD_LOAD:
+		vmmdev_init();
+		iommu_init();
+		error = vmm_init();
+		break;
+	case MOD_UNLOAD:
+		error = vmmdev_cleanup();
+		if (error == 0) {
+			iommu_cleanup();
+			vmm_ipi_cleanup();
+			error = VMM_CLEANUP();
+		}
+		break;
+	default:
+		error = 0;
+		break;
+	}
+	return (error);
+}
+
+static moduledata_t vmm_kmod = {
+	"vmm",
+	vmm_handler,
+	NULL
+};
+
+/*
+ * vmm initialization has the following dependencies:
+ *
+ * - iommu initialization must happen after the pci passthru driver has had
+ *   a chance to attach to any passthru devices (after SI_SUB_CONFIGURE).
+ *
+ * - VT-x initialization requires smp_rendezvous() and therefore must happen
+ *   after SMP is fully functional (after SI_SUB_SMP).
+ */
+DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY);
+MODULE_VERSION(vmm, 1);
+
+SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL);
+
+struct vm *
+vm_create(const char *name)
+{
+	int i;
+	struct vm *vm;
+	vm_paddr_t maxaddr;
+
+	const int BSP = 0;
+
+	if (name == NULL || strlen(name) >= VM_MAX_NAMELEN)
+		return (NULL);
+
+	vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO);
+	strcpy(vm->name, name);
+	vm->cookie = VMINIT(vm);
+
+	for (i = 0; i < VM_MAXCPU; i++) {
+		vcpu_init(vm, i);
+		guest_msrs_init(vm, i);
+	}
+
+	maxaddr = vmm_mem_maxaddr();
+	vm->iommu = iommu_create_domain(maxaddr);
+	vm_activate_cpu(vm, BSP);
+
+	return (vm);
+}
+
+static void
+vm_free_mem_seg(struct vm *vm, struct vm_memory_segment *seg)
+{
+	size_t len;
+	vm_paddr_t hpa;
+	void *host_domain;
+
+	host_domain = iommu_host_domain();
+
+	len = 0;
+	while (len < seg->len) {
+		hpa = vm_gpa2hpa(vm, seg->gpa + len, PAGE_SIZE);
+		if (hpa == (vm_paddr_t)-1) {
+			panic("vm_free_mem_segs: cannot free hpa "
+			      "associated with gpa 0x%016lx", seg->gpa + len);
+		}
+
+		/*
+		 * Remove the 'gpa' to 'hpa' mapping in VMs domain.
+		 * And resurrect the 1:1 mapping for 'hpa' in 'host_domain'.
+		 */
+		iommu_remove_mapping(vm->iommu, seg->gpa + len, PAGE_SIZE);
+		iommu_create_mapping(host_domain, hpa, hpa, PAGE_SIZE);
+
+		vmm_mem_free(hpa, PAGE_SIZE);
+
+		len += PAGE_SIZE;
+	}
+
+	/*
+	 * Invalidate cached translations associated with 'vm->iommu' since
+	 * we have now moved some pages from it.
+	 */
+	iommu_invalidate_tlb(vm->iommu);
+
+	bzero(seg, sizeof(struct vm_memory_segment));
+}
+
+void
+vm_destroy(struct vm *vm)
+{
+	int i;
+
+	ppt_unassign_all(vm);
+
+	for (i = 0; i < vm->num_mem_segs; i++)
+		vm_free_mem_seg(vm, &vm->mem_segs[i]);
+
+	vm->num_mem_segs = 0;
+
+	for (i = 0; i < VM_MAXCPU; i++)
+		vcpu_cleanup(&vm->vcpu[i]);
+
+	iommu_destroy_domain(vm->iommu);
+
+	VMCLEANUP(vm->cookie);
+
+	free(vm, M_VM);
+}
+
+const char *
+vm_name(struct vm *vm)
+{
+	return (vm->name);
+}
+
+int
+vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
+{
+	const boolean_t spok = TRUE;	/* superpage mappings are ok */
+
+	return (VMMMAP_SET(vm->cookie, gpa, hpa, len, VM_MEMATTR_UNCACHEABLE,
+			   VM_PROT_RW, spok));
+}
+
+int
+vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len)
+{
+	const boolean_t spok = TRUE;	/* superpage mappings are ok */
+
+	return (VMMMAP_SET(vm->cookie, gpa, 0, len, 0,
+			   VM_PROT_NONE, spok));
+}
+
+/*
+ * Returns TRUE if 'gpa' is available for allocation and FALSE otherwise
+ */
+static boolean_t
+vm_gpa_available(struct vm *vm, vm_paddr_t gpa)
+{
+	int i;
+	vm_paddr_t gpabase, gpalimit;
+
+	if (gpa & PAGE_MASK)
+		panic("vm_gpa_available: gpa (0x%016lx) not page aligned", gpa);
+
+	for (i = 0; i < vm->num_mem_segs; i++) {
+		gpabase = vm->mem_segs[i].gpa;
+		gpalimit = gpabase + vm->mem_segs[i].len;
+		if (gpa >= gpabase && gpa < gpalimit)
+			return (FALSE);
+	}
+
+	return (TRUE);
+}
+
+int
+vm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len)
+{
+	int error, available, allocated;
+	struct vm_memory_segment *seg;
+	vm_paddr_t g, hpa;
+	void *host_domain;
+
+	const boolean_t spok = TRUE;	/* superpage mappings are ok */
+
+	if ((gpa & PAGE_MASK) || (len & PAGE_MASK) || len == 0)
+		return (EINVAL);
+	
+	available = allocated = 0;
+	g = gpa;
+	while (g < gpa + len) {
+		if (vm_gpa_available(vm, g))
+			available++;
+		else
+			allocated++;
+
+		g += PAGE_SIZE;
+	}
+
+	/*
+	 * If there are some allocated and some available pages in the address
+	 * range then it is an error.
+	 */
+	if (allocated && available)
+		return (EINVAL);
+
+	/*
+	 * If the entire address range being requested has already been
+	 * allocated then there isn't anything more to do.
+	 */
+	if (allocated && available == 0)
+		return (0);
+
+	if (vm->num_mem_segs >= VM_MAX_MEMORY_SEGMENTS)
+		return (E2BIG);
+
+	host_domain = iommu_host_domain();
+
+	seg = &vm->mem_segs[vm->num_mem_segs];
+
+	error = 0;
+	seg->gpa = gpa;
+	seg->len = 0;
+	while (seg->len < len) {
+		hpa = vmm_mem_alloc(PAGE_SIZE);
+		if (hpa == 0) {
+			error = ENOMEM;
+			break;
+		}
+
+		error = VMMMAP_SET(vm->cookie, gpa + seg->len, hpa, PAGE_SIZE,
+				   VM_MEMATTR_WRITE_BACK, VM_PROT_ALL, spok);
+		if (error)
+			break;
+
+		/*
+		 * Remove the 1:1 mapping for 'hpa' from the 'host_domain'.
+		 * Add mapping for 'gpa + seg->len' to 'hpa' in the VMs domain.
+		 */
+		iommu_remove_mapping(host_domain, hpa, PAGE_SIZE);
+		iommu_create_mapping(vm->iommu, gpa + seg->len, hpa, PAGE_SIZE);
+
+		seg->len += PAGE_SIZE;
+	}
+
+	if (error) {
+		vm_free_mem_seg(vm, seg);
+		return (error);
+	}
+
+	/*
+	 * Invalidate cached translations associated with 'host_domain' since
+	 * we have now moved some pages from it.
+	 */
+	iommu_invalidate_tlb(host_domain);
+
+	vm->num_mem_segs++;
+
+	return (0);
+}
+
+vm_paddr_t
+vm_gpa2hpa(struct vm *vm, vm_paddr_t gpa, size_t len)
+{
+	vm_paddr_t nextpage;
+
+	nextpage = rounddown(gpa + PAGE_SIZE, PAGE_SIZE);
+	if (len > nextpage - gpa)
+		panic("vm_gpa2hpa: invalid gpa/len: 0x%016lx/%lu", gpa, len);
+
+	return (VMMMAP_GET(vm->cookie, gpa));
+}
+
+int
+vm_gpabase2memseg(struct vm *vm, vm_paddr_t gpabase,
+		  struct vm_memory_segment *seg)
+{
+	int i;
+
+	for (i = 0; i < vm->num_mem_segs; i++) {
+		if (gpabase == vm->mem_segs[i].gpa) {
+			*seg = vm->mem_segs[i];
+			return (0);
+		}
+	}
+	return (-1);
+}
+
+int
+vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval)
+{
+
+	if (vcpu < 0 || vcpu >= VM_MAXCPU)
+		return (EINVAL);
+
+	if (reg >= VM_REG_LAST)
+		return (EINVAL);
+
+	return (VMGETREG(vm->cookie, vcpu, reg, retval));
+}
+
+int
+vm_set_register(struct vm *vm, int vcpu, int reg, uint64_t val)
+{
+
+	if (vcpu < 0 || vcpu >= VM_MAXCPU)
+		return (EINVAL);
+
+	if (reg >= VM_REG_LAST)
+		return (EINVAL);
+
+	return (VMSETREG(vm->cookie, vcpu, reg, val));
+}
+
+static boolean_t
+is_descriptor_table(int reg)
+{
+
+	switch (reg) {
+	case VM_REG_GUEST_IDTR:
+	case VM_REG_GUEST_GDTR:
+		return (TRUE);
+	default:
+		return (FALSE);
+	}
+}
+
+static boolean_t
+is_segment_register(int reg)
+{
+	
+	switch (reg) {
+	case VM_REG_GUEST_ES:
+	case VM_REG_GUEST_CS:
+	case VM_REG_GUEST_SS:
+	case VM_REG_GUEST_DS:
+	case VM_REG_GUEST_FS:
+	case VM_REG_GUEST_GS:
+	case VM_REG_GUEST_TR:
+	case VM_REG_GUEST_LDTR:
+		return (TRUE);
+	default:
+		return (FALSE);
+	}
+}
+
+int
+vm_get_seg_desc(struct vm *vm, int vcpu, int reg,
+		struct seg_desc *desc)
+{
+
+	if (vcpu < 0 || vcpu >= VM_MAXCPU)
+		return (EINVAL);
+
+	if (!is_segment_register(reg) && !is_descriptor_table(reg))
+		return (EINVAL);
+
+	return (VMGETDESC(vm->cookie, vcpu, reg, desc));
+}
+
+int
+vm_set_seg_desc(struct vm *vm, int vcpu, int reg,
+		struct seg_desc *desc)
+{
+	if (vcpu < 0 || vcpu >= VM_MAXCPU)
+		return (EINVAL);
+
+	if (!is_segment_register(reg) && !is_descriptor_table(reg))
+		return (EINVAL);
+
+	return (VMSETDESC(vm->cookie, vcpu, reg, desc));
+}
+
+int
+vm_get_pinning(struct vm *vm, int vcpuid, int *cpuid)
+{
+
+	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
+		return (EINVAL);
+
+	*cpuid = VCPU_PINCPU(vm, vcpuid);
+
+	return (0);
+}
+
+int
+vm_set_pinning(struct vm *vm, int vcpuid, int host_cpuid)
+{
+	struct thread *td;
+
+	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
+		return (EINVAL);
+
+	td = curthread;		/* XXXSMP only safe when muxing vcpus */
+
+	/* unpin */
+	if (host_cpuid < 0) {
+		VCPU_UNPIN(vm, vcpuid);
+		thread_lock(td);
+		sched_unbind(td);
+		thread_unlock(td);
+		return (0);
+	}
+
+	if (CPU_ABSENT(host_cpuid))
+		return (EINVAL);
+
+	/*
+	 * XXX we should check that 'host_cpuid' has not already been pinned
+	 * by another vm.
+	 */
+	thread_lock(td);
+	sched_bind(td, host_cpuid);
+	thread_unlock(td);
+	VCPU_PIN(vm, vcpuid, host_cpuid);
+
+	return (0);
+}
+
+static void
+restore_guest_fpustate(struct vcpu *vcpu)
+{
+
+	/* flush host state to the pcb */
+	fpuexit(curthread);
+
+	/* restore guest FPU state */
+	fpu_stop_emulating();
+	fpurestore(vcpu->guestfpu);
+
+	/*
+	 * The FPU is now "dirty" with the guest's state so turn on emulation
+	 * to trap any access to the FPU by the host.
+	 */
+	fpu_start_emulating();
+}
+
+static void
+save_guest_fpustate(struct vcpu *vcpu)
+{
+
+	if ((rcr0() & CR0_TS) == 0)
+		panic("fpu emulation not enabled in host!");
+
+	/* save guest FPU state */
+	fpu_stop_emulating();
+	fpusave(vcpu->guestfpu);
+	fpu_start_emulating();
+}
+
+static VMM_STAT_DEFINE(VCPU_IDLE_TICKS, "number of ticks vcpu was idle");
+
+int
+vm_run(struct vm *vm, struct vm_run *vmrun)
+{
+	int error, vcpuid, sleepticks, t;
+	struct vcpu *vcpu;
+	struct pcb *pcb;
+	uint64_t tscval, rip;
+	struct vm_exit *vme;
+
+	vcpuid = vmrun->cpuid;
+
+	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
+		return (EINVAL);
+
+	vcpu = &vm->vcpu[vcpuid];
+	vme = &vmrun->vm_exit;
+	rip = vmrun->rip;
+restart:
+	critical_enter();
+
+	tscval = rdtsc();
+
+	pcb = PCPU_GET(curpcb);
+	set_pcb_flags(pcb, PCB_FULL_IRET);
+
+	restore_guest_msrs(vm, vcpuid);	
+	restore_guest_fpustate(vcpu);
+
+	vcpu->hostcpu = curcpu;
+	error = VMRUN(vm->cookie, vcpuid, rip);
+	vcpu->hostcpu = NOCPU;
+
+	save_guest_fpustate(vcpu);
+	restore_host_msrs(vm, vcpuid);
+
+	vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval);
+
+	/* copy the exit information */
+	bcopy(&vcpu->exitinfo, vme, sizeof(struct vm_exit));
+
+	critical_exit();
+
+	/*
+	 * Oblige the guest's desire to 'hlt' by sleeping until the vcpu
+	 * is ready to run.
+	 */
+	if (error == 0 && vme->exitcode == VM_EXITCODE_HLT) {
+		vcpu_lock(vcpu);
+
+		/*
+		 * Figure out the number of host ticks until the next apic
+		 * timer interrupt in the guest.
+		 */
+		sleepticks = lapic_timer_tick(vm, vcpuid);
+
+		/*
+		 * If the guest local apic timer is disabled then sleep for
+		 * a long time but not forever.
+		 */
+		if (sleepticks < 0)
+			sleepticks = hz;
+
+		/*
+		 * Do a final check for pending NMI or interrupts before
+		 * really putting this thread to sleep.
+		 *
+		 * These interrupts could have happened any time after we
+		 * returned from VMRUN() and before we grabbed the vcpu lock.
+		 */
+		if (!vm_nmi_pending(vm, vcpuid) &&
+		    lapic_pending_intr(vm, vcpuid) < 0) {
+			if (sleepticks <= 0)
+				panic("invalid sleepticks %d", sleepticks);
+			t = ticks;
+			msleep_spin(vcpu, &vcpu->mtx, "vmidle", sleepticks);
+			vmm_stat_incr(vm, vcpuid, VCPU_IDLE_TICKS, ticks - t);
+		}
+
+		vcpu_unlock(vcpu);
+
+		rip = vme->rip + vme->inst_length;
+		goto restart;
+	}
+
+	return (error);
+}
+
+int
+vm_inject_event(struct vm *vm, int vcpuid, int type,
+		int vector, uint32_t code, int code_valid)
+{
+	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
+		return (EINVAL);
+
+	if ((type > VM_EVENT_NONE && type < VM_EVENT_MAX) == 0)
+		return (EINVAL);
+
+	if (vector < 0 || vector > 255)
+		return (EINVAL);
+
+	return (VMINJECT(vm->cookie, vcpuid, type, vector, code, code_valid));
+}
+
+static VMM_STAT_DEFINE(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu");
+
+int
+vm_inject_nmi(struct vm *vm, int vcpuid)
+{
+	struct vcpu *vcpu;
+
+	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
+		return (EINVAL);
+
+	vcpu = &vm->vcpu[vcpuid];
+
+	vcpu->nmi_pending = 1;
+	vm_interrupt_hostcpu(vm, vcpuid);
+	return (0);
+}
+
+int
+vm_nmi_pending(struct vm *vm, int vcpuid)
+{
+	struct vcpu *vcpu;
+
+	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
+		panic("vm_nmi_pending: invalid vcpuid %d", vcpuid);
+
+	vcpu = &vm->vcpu[vcpuid];
+
+	return (vcpu->nmi_pending);
+}
+
+void
+vm_nmi_clear(struct vm *vm, int vcpuid)
+{
+	struct vcpu *vcpu;
+
+	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
+		panic("vm_nmi_pending: invalid vcpuid %d", vcpuid);
+
+	vcpu = &vm->vcpu[vcpuid];
+
+	if (vcpu->nmi_pending == 0)
+		panic("vm_nmi_clear: inconsistent nmi_pending state");
+
+	vcpu->nmi_pending = 0;
+	vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1);
+}
+
+int
+vm_get_capability(struct vm *vm, int vcpu, int type, int *retval)
+{
+	if (vcpu < 0 || vcpu >= VM_MAXCPU)
+		return (EINVAL);
+
+	if (type < 0 || type >= VM_CAP_MAX)
+		return (EINVAL);
+
+	return (VMGETCAP(vm->cookie, vcpu, type, retval));
+}
+
+int
+vm_set_capability(struct vm *vm, int vcpu, int type, int val)
+{
+	if (vcpu < 0 || vcpu >= VM_MAXCPU)
+		return (EINVAL);
+
+	if (type < 0 || type >= VM_CAP_MAX)
+		return (EINVAL);
+
+	return (VMSETCAP(vm->cookie, vcpu, type, val));
+}
+
+uint64_t *
+vm_guest_msrs(struct vm *vm, int cpu)
+{
+	return (vm->vcpu[cpu].guest_msrs);
+}
+
+struct vlapic *
+vm_lapic(struct vm *vm, int cpu)
+{
+	return (vm->vcpu[cpu].vlapic);
+}
+
+boolean_t
+vmm_is_pptdev(int bus, int slot, int func)
+{
+	int found, i, n;
+	int b, s, f;
+	char *val, *cp, *cp2;
+
+	/*
+	 * XXX
+	 * The length of an environment variable is limited to 128 bytes which
+	 * puts an upper limit on the number of passthru devices that may be
+	 * specified using a single environment variable.
+	 *
+	 * Work around this by scanning multiple environment variable
+	 * names instead of a single one - yuck!
+	 */
+	const char *names[] = { "pptdevs", "pptdevs2", "pptdevs3", NULL };
+
+	/* set pptdevs="1/2/3 4/5/6 7/8/9 10/11/12" */
+	found = 0;
+	for (i = 0; names[i] != NULL && !found; i++) {
+		cp = val = getenv(names[i]);
+		while (cp != NULL && *cp != '\0') {
+			if ((cp2 = strchr(cp, ' ')) != NULL)
+				*cp2 = '\0';
+
+			n = sscanf(cp, "%d/%d/%d", &b, &s, &f);
+			if (n == 3 && bus == b && slot == s && func == f) {
+				found = 1;
+				break;
+			}
+		
+			if (cp2 != NULL)
+				*cp2++ = ' ';
+
+			cp = cp2;
+		}
+		freeenv(val);
+	}
+	return (found);
+}
+
+void *
+vm_iommu_domain(struct vm *vm)
+{
+
+	return (vm->iommu);
+}
+
+int
+vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state state)
+{
+	int error;
+	struct vcpu *vcpu;
+
+	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
+		panic("vm_set_run_state: invalid vcpuid %d", vcpuid);
+
+	vcpu = &vm->vcpu[vcpuid];
+
+	vcpu_lock(vcpu);
+
+	/*
+	 * The following state transitions are allowed:
+	 * IDLE -> RUNNING -> IDLE
+	 * IDLE -> CANNOT_RUN -> IDLE
+	 */
+	if ((vcpu->state == VCPU_IDLE && state != VCPU_IDLE) ||
+	    (vcpu->state != VCPU_IDLE && state == VCPU_IDLE)) {
+		error = 0;
+		vcpu->state = state;
+	} else {
+		error = EBUSY;
+	}
+
+	vcpu_unlock(vcpu);
+
+	return (error);
+}
+
+enum vcpu_state
+vcpu_get_state(struct vm *vm, int vcpuid)
+{
+	struct vcpu *vcpu;
+	enum vcpu_state state;
+
+	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
+		panic("vm_get_run_state: invalid vcpuid %d", vcpuid);
+
+	vcpu = &vm->vcpu[vcpuid];
+
+	vcpu_lock(vcpu);
+	state = vcpu->state;
+	vcpu_unlock(vcpu);
+
+	return (state);
+}
+
+void
+vm_activate_cpu(struct vm *vm, int vcpuid)
+{
+
+	if (vcpuid >= 0 && vcpuid < VM_MAXCPU)
+		CPU_SET(vcpuid, &vm->active_cpus);
+}
+
+cpuset_t
+vm_active_cpus(struct vm *vm)
+{
+
+	return (vm->active_cpus);
+}
+
+void *
+vcpu_stats(struct vm *vm, int vcpuid)
+{
+
+	return (vm->vcpu[vcpuid].stats);
+}
+
+int
+vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state)
+{
+	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
+		return (EINVAL);
+
+	*state = vm->vcpu[vcpuid].x2apic_state;
+
+	return (0);
+}
+
+int
+vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
+{
+	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
+		return (EINVAL);
+
+	if (state < 0 || state >= X2APIC_STATE_LAST)
+		return (EINVAL);
+
+	vm->vcpu[vcpuid].x2apic_state = state;
+
+	vlapic_set_x2apic_state(vm, vcpuid, state);
+
+	return (0);
+}
+
+void
+vm_interrupt_hostcpu(struct vm *vm, int vcpuid)
+{
+	int hostcpu;
+	struct vcpu *vcpu;
+
+	vcpu = &vm->vcpu[vcpuid];
+
+	vcpu_lock(vcpu);
+	hostcpu = vcpu->hostcpu;
+	if (hostcpu == NOCPU) {
+		/*
+		 * If the vcpu is 'RUNNING' but without a valid 'hostcpu' then
+		 * the host thread must be sleeping waiting for an event to
+		 * kick the vcpu out of 'hlt'.
+		 *
+		 * XXX this is racy because the condition exists right before
+		 * and after calling VMRUN() in vm_run(). The wakeup() is
+		 * benign in this case.
+		 */
+		if (vcpu->state == VCPU_RUNNING)
+			wakeup_one(vcpu);
+	} else {
+		if (vcpu->state != VCPU_RUNNING)
+			panic("invalid vcpu state %d", vcpu->state);
+		if (hostcpu != curcpu)
+			ipi_cpu(hostcpu, vmm_ipinum);
+	}
+	vcpu_unlock(vcpu);
+}
diff --git a/sys/amd64/vmm/vmm_dev.c b/sys/amd64/vmm/vmm_dev.c
new file mode 100644
index 0000000..0150ebd
--- /dev/null
+++ b/sys/amd64/vmm/vmm_dev.c
@@ -0,0 +1,538 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/queue.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/malloc.h>
+#include <sys/conf.h>
+#include <sys/sysctl.h>
+#include <sys/libkern.h>
+#include <sys/ioccom.h>
+#include <sys/mman.h>
+#include <sys/uio.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/pmap.h>
+#include <machine/vmparam.h>
+
+#include <machine/vmm.h>
+#include "vmm_lapic.h"
+#include "vmm_stat.h"
+#include "vmm_mem.h"
+#include "io/ppt.h"
+#include <machine/vmm_dev.h>
+
+struct vmmdev_softc {
+	struct vm	*vm;		/* vm instance cookie */
+	struct cdev	*cdev;
+	SLIST_ENTRY(vmmdev_softc) link;
+};
+static SLIST_HEAD(, vmmdev_softc) head;
+
+static struct mtx vmmdev_mtx;
+
+static MALLOC_DEFINE(M_VMMDEV, "vmmdev", "vmmdev");
+
+SYSCTL_DECL(_hw_vmm);
+
+static struct vmmdev_softc *
+vmmdev_lookup(const char *name)
+{
+	struct vmmdev_softc *sc;
+
+#ifdef notyet	/* XXX kernel is not compiled with invariants */
+	mtx_assert(&vmmdev_mtx, MA_OWNED);
+#endif
+
+	SLIST_FOREACH(sc, &head, link) {
+		if (strcmp(name, vm_name(sc->vm)) == 0)
+			break;
+	}
+
+	return (sc);
+}
+
+static struct vmmdev_softc *
+vmmdev_lookup2(struct cdev *cdev)
+{
+
+	return (cdev->si_drv1);
+}
+
+static int
+vmmdev_rw(struct cdev *cdev, struct uio *uio, int flags)
+{
+	int error, off, c;
+	vm_paddr_t hpa, gpa;
+	struct vmmdev_softc *sc;
+
+	static char zerobuf[PAGE_SIZE];
+
+	error = 0;
+	mtx_lock(&vmmdev_mtx);
+	sc = vmmdev_lookup2(cdev);
+	if (sc == NULL)
+		error = ENXIO;
+
+	while (uio->uio_resid > 0 && error == 0) {
+		gpa = uio->uio_offset;
+		off = gpa & PAGE_MASK;
+		c = min(uio->uio_resid, PAGE_SIZE - off);
+
+		/*
+		 * The VM has a hole in its physical memory map. If we want to
+		 * use 'dd' to inspect memory beyond the hole we need to
+		 * provide bogus data for memory that lies in the hole.
+		 *
+		 * Since this device does not support lseek(2), dd(1) will
+		 * read(2) blocks of data to simulate the lseek(2).
+		 */
+		hpa = vm_gpa2hpa(sc->vm, gpa, c);
+		if (hpa == (vm_paddr_t)-1) {
+			if (uio->uio_rw == UIO_READ)
+				error = uiomove(zerobuf, c, uio);
+			else
+				error = EFAULT;
+		} else
+			error = uiomove((void *)PHYS_TO_DMAP(hpa), c, uio);
+	}
+
+	mtx_unlock(&vmmdev_mtx);
+	return (error);
+}
+
+static int
+vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
+	     struct thread *td)
+{
+	int error, vcpu, state_changed;
+	enum vcpu_state new_state;
+	struct vmmdev_softc *sc;
+	struct vm_memory_segment *seg;
+	struct vm_register *vmreg;
+	struct vm_seg_desc* vmsegdesc;
+	struct vm_pin *vmpin;
+	struct vm_run *vmrun;
+	struct vm_event *vmevent;
+	struct vm_lapic_irq *vmirq;
+	struct vm_capability *vmcap;
+	struct vm_pptdev *pptdev;
+	struct vm_pptdev_mmio *pptmmio;
+	struct vm_pptdev_msi *pptmsi;
+	struct vm_pptdev_msix *pptmsix;
+	struct vm_nmi *vmnmi;
+	struct vm_stats *vmstats;
+	struct vm_stat_desc *statdesc;
+	struct vm_x2apic *x2apic;
+
+	sc = vmmdev_lookup2(cdev);
+	if (sc == NULL)
+		return (ENXIO);
+
+	vcpu = -1;
+	state_changed = 0;
+
+	/*
+	 * Some VMM ioctls can operate only on vcpus that are not running.
+	 */
+	switch (cmd) {
+	case VM_RUN:
+	case VM_SET_PINNING:
+	case VM_GET_REGISTER:
+	case VM_SET_REGISTER:
+	case VM_GET_SEGMENT_DESCRIPTOR:
+	case VM_SET_SEGMENT_DESCRIPTOR:
+	case VM_INJECT_EVENT:
+	case VM_GET_CAPABILITY:
+	case VM_SET_CAPABILITY:
+	case VM_PPTDEV_MSI:
+	case VM_PPTDEV_MSIX:
+	case VM_SET_X2APIC_STATE:
+		/*
+		 * XXX fragile, handle with care
+		 * Assumes that the first field of the ioctl data is the vcpu.
+		 */
+		vcpu = *(int *)data;
+		if (vcpu < 0 || vcpu >= VM_MAXCPU) {
+			error = EINVAL;
+			goto done;
+		}
+
+		if (cmd == VM_RUN)
+			new_state = VCPU_RUNNING;
+		else
+			new_state = VCPU_CANNOT_RUN;
+
+		error = vcpu_set_state(sc->vm, vcpu, new_state);
+		if (error)
+			goto done;
+
+		state_changed = 1;
+		break;
+
+	case VM_MAP_PPTDEV_MMIO:
+	case VM_BIND_PPTDEV:
+	case VM_UNBIND_PPTDEV:
+	case VM_MAP_MEMORY:
+		/*
+		 * ioctls that operate on the entire virtual machine must
+		 * prevent all vcpus from running.
+		 */
+		error = 0;
+		for (vcpu = 0; vcpu < VM_MAXCPU; vcpu++) {
+			error = vcpu_set_state(sc->vm, vcpu, VCPU_CANNOT_RUN);
+			if (error)
+				break;
+		}
+
+		if (error) {
+			while (--vcpu >= 0)
+				vcpu_set_state(sc->vm, vcpu, VCPU_IDLE);
+			goto done;
+		}
+
+		state_changed = 2;
+		break;
+
+	default:
+		break;
+	}
+
+	switch(cmd) {
+	case VM_RUN:
+		vmrun = (struct vm_run *)data;
+		error = vm_run(sc->vm, vmrun);
+		break;
+	case VM_STAT_DESC: {
+		const char *desc;
+		statdesc = (struct vm_stat_desc *)data;
+		desc = vmm_stat_desc(statdesc->index);
+		if (desc != NULL) {
+			error = 0;
+			strlcpy(statdesc->desc, desc, sizeof(statdesc->desc));
+		} else
+			error = EINVAL;
+		break;
+	}
+	case VM_STATS: {
+		CTASSERT(MAX_VM_STATS >= MAX_VMM_STAT_TYPES);
+		vmstats = (struct vm_stats *)data;
+		getmicrotime(&vmstats->tv);
+		error = vmm_stat_copy(sc->vm, vmstats->cpuid,
+				      &vmstats->num_entries, vmstats->statbuf);
+		break;
+	}
+	case VM_PPTDEV_MSI:
+		pptmsi = (struct vm_pptdev_msi *)data;
+		error = ppt_setup_msi(sc->vm, pptmsi->vcpu,
+				      pptmsi->bus, pptmsi->slot, pptmsi->func,
+				      pptmsi->destcpu, pptmsi->vector,
+				      pptmsi->numvec);
+		break;
+	case VM_PPTDEV_MSIX:
+		pptmsix = (struct vm_pptdev_msix *)data;
+		error = ppt_setup_msix(sc->vm, pptmsix->vcpu,
+				       pptmsix->bus, pptmsix->slot, 
+				       pptmsix->func, pptmsix->idx,
+				       pptmsix->msg, pptmsix->vector_control,
+				       pptmsix->addr);
+		break;
+	case VM_MAP_PPTDEV_MMIO:
+		pptmmio = (struct vm_pptdev_mmio *)data;
+		error = ppt_map_mmio(sc->vm, pptmmio->bus, pptmmio->slot,
+				     pptmmio->func, pptmmio->gpa, pptmmio->len,
+				     pptmmio->hpa);
+		break;
+	case VM_BIND_PPTDEV:
+		pptdev = (struct vm_pptdev *)data;
+		error = ppt_assign_device(sc->vm, pptdev->bus, pptdev->slot,
+					  pptdev->func);
+		break;
+	case VM_UNBIND_PPTDEV:
+		pptdev = (struct vm_pptdev *)data;
+		error = ppt_unassign_device(sc->vm, pptdev->bus, pptdev->slot,
+					    pptdev->func);
+		break;
+	case VM_INJECT_EVENT:
+		vmevent = (struct vm_event *)data;
+		error = vm_inject_event(sc->vm, vmevent->cpuid, vmevent->type,
+					vmevent->vector,
+					vmevent->error_code,
+					vmevent->error_code_valid);
+		break;
+	case VM_INJECT_NMI:
+		vmnmi = (struct vm_nmi *)data;
+		error = vm_inject_nmi(sc->vm, vmnmi->cpuid);
+		break;
+	case VM_LAPIC_IRQ:
+		vmirq = (struct vm_lapic_irq *)data;
+		error = lapic_set_intr(sc->vm, vmirq->cpuid, vmirq->vector);
+		break;
+	case VM_SET_PINNING:
+		vmpin = (struct vm_pin *)data;
+		error = vm_set_pinning(sc->vm, vmpin->vm_cpuid,
+				       vmpin->host_cpuid);
+		break;
+	case VM_GET_PINNING:
+		vmpin = (struct vm_pin *)data;
+		error = vm_get_pinning(sc->vm, vmpin->vm_cpuid,
+				       &vmpin->host_cpuid);
+		break;
+	case VM_MAP_MEMORY:
+		seg = (struct vm_memory_segment *)data;
+		error = vm_malloc(sc->vm, seg->gpa, seg->len);
+		break;
+	case VM_GET_MEMORY_SEG:
+		seg = (struct vm_memory_segment *)data;
+		seg->len = 0;
+		(void)vm_gpabase2memseg(sc->vm, seg->gpa, seg);
+		error = 0;
+		break;
+	case VM_GET_REGISTER:
+		vmreg = (struct vm_register *)data;
+		error = vm_get_register(sc->vm, vmreg->cpuid, vmreg->regnum,
+					&vmreg->regval);
+		break;
+	case VM_SET_REGISTER:
+		vmreg = (struct vm_register *)data;
+		error = vm_set_register(sc->vm, vmreg->cpuid, vmreg->regnum,
+					vmreg->regval);
+		break;
+	case VM_SET_SEGMENT_DESCRIPTOR:
+		vmsegdesc = (struct vm_seg_desc *)data;
+		error = vm_set_seg_desc(sc->vm, vmsegdesc->cpuid,
+					vmsegdesc->regnum,
+					&vmsegdesc->desc);
+		break;
+	case VM_GET_SEGMENT_DESCRIPTOR:
+		vmsegdesc = (struct vm_seg_desc *)data;
+		error = vm_get_seg_desc(sc->vm, vmsegdesc->cpuid,
+					vmsegdesc->regnum,
+					&vmsegdesc->desc);
+		break;
+	case VM_GET_CAPABILITY:
+		vmcap = (struct vm_capability *)data;
+		error = vm_get_capability(sc->vm, vmcap->cpuid,
+					  vmcap->captype,
+					  &vmcap->capval);
+		break;
+	case VM_SET_CAPABILITY:
+		vmcap = (struct vm_capability *)data;
+		error = vm_set_capability(sc->vm, vmcap->cpuid,
+					  vmcap->captype,
+					  vmcap->capval);
+		break;
+	case VM_SET_X2APIC_STATE:
+		x2apic = (struct vm_x2apic *)data;
+		error = vm_set_x2apic_state(sc->vm,
+					    x2apic->cpuid, x2apic->state);
+		break;
+	case VM_GET_X2APIC_STATE:
+		x2apic = (struct vm_x2apic *)data;
+		error = vm_get_x2apic_state(sc->vm,
+					    x2apic->cpuid, &x2apic->state);
+		break;
+	default:
+		error = ENOTTY;
+		break;
+	}
+
+	if (state_changed == 1) {
+		vcpu_set_state(sc->vm, vcpu, VCPU_IDLE);
+	} else if (state_changed == 2) {
+		for (vcpu = 0; vcpu < VM_MAXCPU; vcpu++)
+			vcpu_set_state(sc->vm, vcpu, VCPU_IDLE);
+	}
+
+done:
+	return (error);
+}
+
+static int
+vmmdev_mmap(struct cdev *cdev, vm_ooffset_t offset, vm_paddr_t *paddr,
+    int nprot, vm_memattr_t *memattr)
+{
+	int error;
+	struct vmmdev_softc *sc;
+
+	error = -1;
+	mtx_lock(&vmmdev_mtx);
+
+	sc = vmmdev_lookup2(cdev);
+	if (sc != NULL && (nprot & PROT_EXEC) == 0) {
+		*paddr = vm_gpa2hpa(sc->vm, (vm_paddr_t)offset, PAGE_SIZE);
+		if (*paddr != (vm_paddr_t)-1)
+			error = 0;
+	}
+
+	mtx_unlock(&vmmdev_mtx);
+
+	return (error);
+}
+
+static void
+vmmdev_destroy(struct vmmdev_softc *sc, boolean_t unlink)
+{
+
+	/*
+	 * XXX must stop virtual machine instances that may be still
+	 * running and cleanup their state.
+	 */
+	if (sc->cdev)
+		destroy_dev(sc->cdev);
+
+	if (sc->vm)
+		vm_destroy(sc->vm);
+
+	if (unlink) {
+		mtx_lock(&vmmdev_mtx);
+		SLIST_REMOVE(&head, sc, vmmdev_softc, link);
+		mtx_unlock(&vmmdev_mtx);
+	}
+
+	free(sc, M_VMMDEV);
+}
+
+static int
+sysctl_vmm_destroy(SYSCTL_HANDLER_ARGS)
+{
+	int error;
+	char buf[VM_MAX_NAMELEN];
+	struct vmmdev_softc *sc;
+
+	strlcpy(buf, "beavis", sizeof(buf));
+	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
+	if (error != 0 || req->newptr == NULL)
+		return (error);
+
+	/*
+	 * XXX TODO if any process has this device open then fail
+	 */
+
+	mtx_lock(&vmmdev_mtx);
+	sc = vmmdev_lookup(buf);
+	if (sc == NULL) {
+		mtx_unlock(&vmmdev_mtx);
+		return (EINVAL);
+	}
+
+	sc->cdev->si_drv1 = NULL;
+	mtx_unlock(&vmmdev_mtx);
+
+	vmmdev_destroy(sc, TRUE);
+
+	return (0);
+}
+SYSCTL_PROC(_hw_vmm, OID_AUTO, destroy, CTLTYPE_STRING | CTLFLAG_RW,
+	    NULL, 0, sysctl_vmm_destroy, "A", NULL);
+
+static struct cdevsw vmmdevsw = {
+	.d_name		= "vmmdev",
+	.d_version	= D_VERSION,
+	.d_ioctl	= vmmdev_ioctl,
+	.d_mmap		= vmmdev_mmap,
+	.d_read		= vmmdev_rw,
+	.d_write	= vmmdev_rw,
+};
+
+static int
+sysctl_vmm_create(SYSCTL_HANDLER_ARGS)
+{
+	int error;
+	struct vm *vm;
+	struct vmmdev_softc *sc, *sc2;
+	char buf[VM_MAX_NAMELEN];
+
+	strlcpy(buf, "beavis", sizeof(buf));
+	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
+	if (error != 0 || req->newptr == NULL)
+		return (error);
+
+	mtx_lock(&vmmdev_mtx);
+	sc = vmmdev_lookup(buf);
+	mtx_unlock(&vmmdev_mtx);
+	if (sc != NULL)
+		return (EEXIST);
+
+	vm = vm_create(buf);
+	if (vm == NULL)
+		return (EINVAL);
+
+	sc = malloc(sizeof(struct vmmdev_softc), M_VMMDEV, M_WAITOK | M_ZERO);
+	sc->vm = vm;
+
+	/*
+	 * Lookup the name again just in case somebody sneaked in when we
+	 * dropped the lock.
+	 */
+	mtx_lock(&vmmdev_mtx);
+	sc2 = vmmdev_lookup(buf);
+	if (sc2 == NULL)
+		SLIST_INSERT_HEAD(&head, sc, link);
+	mtx_unlock(&vmmdev_mtx);
+
+	if (sc2 != NULL) {
+		vmmdev_destroy(sc, FALSE);
+		return (EEXIST);
+	}
+
+	sc->cdev = make_dev(&vmmdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
+			    "vmm/%s", buf);
+	sc->cdev->si_drv1 = sc;
+
+	return (0);
+}
+SYSCTL_PROC(_hw_vmm, OID_AUTO, create, CTLTYPE_STRING | CTLFLAG_RW,
+	    NULL, 0, sysctl_vmm_create, "A", NULL);
+
+void
+vmmdev_init(void)
+{
+	mtx_init(&vmmdev_mtx, "vmm device mutex", NULL, MTX_DEF);
+}
+
+int
+vmmdev_cleanup(void)
+{
+	int error;
+
+	if (SLIST_EMPTY(&head))
+		error = 0;
+	else
+		error = EBUSY;
+
+	return (error);
+}
diff --git a/sys/amd64/vmm/vmm_host.c b/sys/amd64/vmm/vmm_host.c
new file mode 100644
index 0000000..8dfef73
--- /dev/null
+++ b/sys/amd64/vmm/vmm_host.c
@@ -0,0 +1,124 @@
+/*-
+ * Copyright (c) 2012 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/pcpu.h>
+
+#include <machine/cpufunc.h>
+#include <machine/segments.h>
+#include <machine/specialreg.h>
+
+#include "vmm_host.h"
+
+static uint64_t vmm_host_efer, vmm_host_pat, vmm_host_cr0, vmm_host_cr4;
+
+void
+vmm_host_state_init(void)
+{
+
+	vmm_host_efer = rdmsr(MSR_EFER);
+	vmm_host_pat = rdmsr(MSR_PAT);
+
+	/*
+	 * We always want CR0.TS to be set when the processor does a VM exit.
+	 *
+	 * With emulation turned on unconditionally after a VM exit, we are
+	 * able to trap inadvertent use of the FPU until the guest FPU state
+	 * has been safely squirreled away.
+	 */
+	vmm_host_cr0 = rcr0() | CR0_TS;
+
+	vmm_host_cr4 = rcr4();
+}
+
+uint64_t
+vmm_get_host_pat(void)
+{
+
+	return (vmm_host_pat);
+}
+
+uint64_t
+vmm_get_host_efer(void)
+{
+
+	return (vmm_host_efer);
+}
+
+uint64_t
+vmm_get_host_cr0(void)
+{
+
+	return (vmm_host_cr0);
+}
+
+uint64_t
+vmm_get_host_cr4(void)
+{
+
+	return (vmm_host_cr4);
+}
+
+uint64_t
+vmm_get_host_datasel(void)
+{
+
+	return (GSEL(GDATA_SEL, SEL_KPL));
+
+}
+
+uint64_t
+vmm_get_host_codesel(void)
+{
+
+	return (GSEL(GCODE_SEL, SEL_KPL));
+}
+
+uint64_t
+vmm_get_host_tsssel(void)
+{
+
+	return (GSEL(GPROC0_SEL, SEL_KPL));
+}
+
+uint64_t
+vmm_get_host_fsbase(void)
+{
+
+	return (0);
+}
+
+uint64_t
+vmm_get_host_idtrbase(void)
+{
+
+	return (r_idt.rd_base);
+}
diff --git a/sys/amd64/vmm/vmm_host.h b/sys/amd64/vmm/vmm_host.h
new file mode 100644
index 0000000..839f54a
--- /dev/null
+++ b/sys/amd64/vmm/vmm_host.h
@@ -0,0 +1,75 @@
+/*-
+ * Copyright (c) 2012 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_VMM_HOST_H_
+#define	_VMM_HOST_H_
+
+#ifndef	_KERNEL
+#error "no user-servicable parts inside"
+#endif
+
+void vmm_host_state_init(void);
+
+uint64_t vmm_get_host_pat(void);
+uint64_t vmm_get_host_efer(void);
+uint64_t vmm_get_host_cr0(void);
+uint64_t vmm_get_host_cr4(void);
+uint64_t vmm_get_host_datasel(void);
+uint64_t vmm_get_host_codesel(void);
+uint64_t vmm_get_host_tsssel(void);
+uint64_t vmm_get_host_fsbase(void);
+uint64_t vmm_get_host_idtrbase(void);
+
+/*
+ * Inline access to host state that is used on every VM entry
+ */
+static __inline uint64_t
+vmm_get_host_trbase(void)
+{
+
+	return ((uint64_t)PCPU_GET(tssp));
+}
+
+static __inline uint64_t
+vmm_get_host_gdtrbase(void)
+{
+
+	return ((uint64_t)&gdt[NGDT * curcpu]);
+}
+
+struct pcpu;
+extern struct pcpu __pcpu[];
+
+static __inline uint64_t
+vmm_get_host_gsbase(void)
+{
+
+	return ((uint64_t)&__pcpu[curcpu]);
+}
+
+#endif
diff --git a/sys/amd64/vmm/vmm_instruction_emul.c b/sys/amd64/vmm/vmm_instruction_emul.c
new file mode 100644
index 0000000..40748ea
--- /dev/null
+++ b/sys/amd64/vmm/vmm_instruction_emul.c
@@ -0,0 +1,868 @@
+/*-
+ * Copyright (c) 2012 Sandvine, Inc.
+ * Copyright (c) 2012 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#ifdef _KERNEL
+#include <sys/param.h>
+#include <sys/pcpu.h>
+#include <sys/systm.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/pmap.h>
+#include <machine/vmparam.h>
+#include <machine/vmm.h>
+#else	/* !_KERNEL */
+#include <sys/types.h>
+#include <sys/errno.h>
+
+#include <machine/vmm.h>
+
+#include <vmmapi.h>
+#endif	/* _KERNEL */
+
+
+
+/* struct vie_op.op_type */
+enum {
+	VIE_OP_TYPE_NONE = 0,
+	VIE_OP_TYPE_MOV,
+	VIE_OP_TYPE_AND,
+	VIE_OP_TYPE_LAST
+};
+
+/* struct vie_op.op_flags */
+#define	VIE_OP_F_IMM		(1 << 0)	/* immediate operand present */
+#define	VIE_OP_F_IMM8		(1 << 1)	/* 8-bit immediate operand */
+
+static const struct vie_op one_byte_opcodes[256] = {
+	[0x88] = {
+		.op_byte = 0x88,
+		.op_type = VIE_OP_TYPE_MOV,
+	},
+	[0x89] = {
+		.op_byte = 0x89,
+		.op_type = VIE_OP_TYPE_MOV,
+	},
+	[0x8B] = {
+		.op_byte = 0x8B,
+		.op_type = VIE_OP_TYPE_MOV,
+	},
+	[0xC7] = {
+		.op_byte = 0xC7,
+		.op_type = VIE_OP_TYPE_MOV,
+		.op_flags = VIE_OP_F_IMM,
+	},
+	[0x23] = {
+		.op_byte = 0x23,
+		.op_type = VIE_OP_TYPE_AND,
+	},
+	[0x81] = {
+		/* XXX Group 1 extended opcode - not just AND */
+		.op_byte = 0x81,
+		.op_type = VIE_OP_TYPE_AND,
+		.op_flags = VIE_OP_F_IMM,
+	}
+};
+
+/* struct vie.mod */
+#define	VIE_MOD_INDIRECT		0
+#define	VIE_MOD_INDIRECT_DISP8		1
+#define	VIE_MOD_INDIRECT_DISP32		2
+#define	VIE_MOD_DIRECT			3
+
+/* struct vie.rm */
+#define	VIE_RM_SIB			4
+#define	VIE_RM_DISP32			5
+
+#define	GB				(1024 * 1024 * 1024)
+
+static enum vm_reg_name gpr_map[16] = {
+	VM_REG_GUEST_RAX,
+	VM_REG_GUEST_RCX,
+	VM_REG_GUEST_RDX,
+	VM_REG_GUEST_RBX,
+	VM_REG_GUEST_RSP,
+	VM_REG_GUEST_RBP,
+	VM_REG_GUEST_RSI,
+	VM_REG_GUEST_RDI,
+	VM_REG_GUEST_R8,
+	VM_REG_GUEST_R9,
+	VM_REG_GUEST_R10,
+	VM_REG_GUEST_R11,
+	VM_REG_GUEST_R12,
+	VM_REG_GUEST_R13,
+	VM_REG_GUEST_R14,
+	VM_REG_GUEST_R15
+};
+
+static uint64_t size2mask[] = {
+	[1] = 0xff,
+	[2] = 0xffff,
+	[4] = 0xffffffff,
+	[8] = 0xffffffffffffffff,
+};
+
+static int
+vie_valid_register(enum vm_reg_name reg)
+{
+#ifdef _KERNEL
+	/*
+	 * XXX
+	 * The operand register in which we store the result of the
+	 * read must be a GPR that we can modify even if the vcpu
+	 * is "running". All the GPRs qualify except for %rsp.
+	 *
+	 * This is a limitation of the vm_set_register() API
+	 * and can be fixed if necessary.
+	 */
+	if (reg == VM_REG_GUEST_RSP)
+		return (0);
+#endif
+	return (1);
+}
+
+static int
+vie_read_register(void *vm, int vcpuid, enum vm_reg_name reg, uint64_t *rval)
+{
+	int error;
+
+	if (!vie_valid_register(reg))
+		return (EINVAL);
+
+	error = vm_get_register(vm, vcpuid, reg, rval);
+
+	return (error);
+}
+
+static int
+vie_read_bytereg(void *vm, int vcpuid, struct vie *vie, uint8_t *rval)
+{
+	uint64_t val;
+	int error, rshift;
+	enum vm_reg_name reg;
+
+	rshift = 0;
+	reg = gpr_map[vie->reg];
+
+	/*
+	 * 64-bit mode imposes limitations on accessing legacy byte registers.
+	 *
+	 * The legacy high-byte registers cannot be addressed if the REX
+	 * prefix is present. In this case the values 4, 5, 6 and 7 of the
+	 * 'ModRM:reg' field address %spl, %bpl, %sil and %dil respectively.
+	 *
+	 * If the REX prefix is not present then the values 4, 5, 6 and 7
+	 * of the 'ModRM:reg' field address the legacy high-byte registers,
+	 * %ah, %ch, %dh and %bh respectively.
+	 */
+	if (!vie->rex_present) {
+		if (vie->reg & 0x4) {
+			/*
+			 * Obtain the value of %ah by reading %rax and shifting
+			 * right by 8 bits (same for %bh, %ch and %dh).
+			 */
+			rshift = 8;
+			reg = gpr_map[vie->reg & 0x3];
+		}
+	}
+
+	if (!vie_valid_register(reg))
+		return (EINVAL);
+
+	error = vm_get_register(vm, vcpuid, reg, &val);
+	*rval = val >> rshift;
+	return (error);
+}
+
+static int
+vie_update_register(void *vm, int vcpuid, enum vm_reg_name reg,
+		    uint64_t val, int size)
+{
+	int error;
+	uint64_t origval;
+
+	if (!vie_valid_register(reg))
+		return (EINVAL);
+
+	switch (size) {
+	case 1:
+	case 2:
+		error = vie_read_register(vm, vcpuid, reg, &origval);
+		if (error)
+			return (error);
+		val &= size2mask[size];
+		val |= origval & ~size2mask[size];
+		break;
+	case 4:
+		val &= 0xffffffffUL;
+		break;
+	case 8:
+		break;
+	default:
+		return (EINVAL);
+	}
+
+	error = vm_set_register(vm, vcpuid, reg, val);
+	return (error);
+}
+
+/*
+ * The following simplifying assumptions are made during emulation:
+ *
+ * - guest is in 64-bit mode
+ *   - default address size is 64-bits
+ *   - default operand size is 32-bits
+ *
+ * - operand size override is not supported
+ *
+ * - address size override is not supported
+ */
+static int
+emulate_mov(void *vm, int vcpuid, uint64_t gpa, struct vie *vie,
+	    mem_region_read_t memread, mem_region_write_t memwrite, void *arg)
+{
+	int error, size;
+	enum vm_reg_name reg;
+	uint8_t byte;
+	uint64_t val;
+
+	size = 4;
+	error = EINVAL;
+
+	switch (vie->op.op_byte) {
+	case 0x88:
+		/*
+		 * MOV byte from reg (ModRM:reg) to mem (ModRM:r/m)
+		 * 88/r:	mov r/m8, r8
+		 * REX + 88/r:	mov r/m8, r8 (%ah, %ch, %dh, %bh not available)
+		 */
+		size = 1;
+		error = vie_read_bytereg(vm, vcpuid, vie, &byte);
+		if (error == 0)
+			error = memwrite(vm, vcpuid, gpa, byte, size, arg);
+		break;
+	case 0x89:
+		/*
+		 * MOV from reg (ModRM:reg) to mem (ModRM:r/m)
+		 * 89/r:	mov r/m32, r32
+		 * REX.W + 89/r	mov r/m64, r64
+		 */
+		if (vie->rex_w)
+			size = 8;
+		reg = gpr_map[vie->reg];
+		error = vie_read_register(vm, vcpuid, reg, &val);
+		if (error == 0) {
+			val &= size2mask[size];
+			error = memwrite(vm, vcpuid, gpa, val, size, arg);
+		}
+		break;
+	case 0x8B:
+		/*
+		 * MOV from mem (ModRM:r/m) to reg (ModRM:reg)
+		 * 8B/r:	mov r32, r/m32
+		 * REX.W 8B/r:	mov r64, r/m64
+		 */
+		if (vie->rex_w)
+			size = 8;
+		error = memread(vm, vcpuid, gpa, &val, size, arg);
+		if (error == 0) {
+			reg = gpr_map[vie->reg];
+			error = vie_update_register(vm, vcpuid, reg, val, size);
+		}
+		break;
+	case 0xC7:
+		/*
+		 * MOV from imm32 to mem (ModRM:r/m)
+		 * C7/0		mov r/m32, imm32
+		 * REX.W + C7/0	mov r/m64, imm32 (sign-extended to 64-bits)
+		 */
+		val = vie->immediate;		/* already sign-extended */
+
+		if (vie->rex_w)
+			size = 8;
+
+		if (size != 8)
+			val &= size2mask[size];
+
+		error = memwrite(vm, vcpuid, gpa, val, size, arg);
+		break;
+	default:
+		break;
+	}
+
+	return (error);
+}
+
+static int
+emulate_and(void *vm, int vcpuid, uint64_t gpa, struct vie *vie,
+	    mem_region_read_t memread, mem_region_write_t memwrite, void *arg)
+{
+	int error, size;
+	enum vm_reg_name reg;
+	uint64_t val1, val2;
+
+	size = 4;
+	error = EINVAL;
+
+	switch (vie->op.op_byte) {
+	case 0x23:
+		/*
+		 * AND reg (ModRM:reg) and mem (ModRM:r/m) and store the
+		 * result in reg.
+		 *
+		 * 23/r		and r32, r/m32
+		 * REX.W + 23/r	and r64, r/m64
+		 */
+		if (vie->rex_w)
+			size = 8;
+
+		/* get the first operand */
+		reg = gpr_map[vie->reg];
+		error = vie_read_register(vm, vcpuid, reg, &val1);
+		if (error)
+			break;
+
+		/* get the second operand */
+		error = memread(vm, vcpuid, gpa, &val2, size, arg);
+		if (error)
+			break;
+
+		/* perform the operation and write the result */
+		val1 &= val2;
+		error = vie_update_register(vm, vcpuid, reg, val1, size);
+		break;
+	case 0x81:
+		/*
+		 * AND reg (ModRM:reg) with immediate and store the
+		 * result in reg
+		 *
+		 * 81/          and r/m32, imm32
+		 * REX.W + 81/  and r/m64, imm32 sign-extended to 64
+		 *
+		 * Currently, only the AND operation of the 0x81 opcode
+		 * is implemented (ModRM:reg = b100).
+		 */
+		if ((vie->reg & 7) != 4)
+			break;
+
+		if (vie->rex_w)
+			size = 8;
+		
+		/* get the first operand */
+                error = memread(vm, vcpuid, gpa, &val1, size, arg);
+                if (error)
+			break;
+
+                /*
+		 * perform the operation with the pre-fetched immediate
+		 * operand and write the result
+		 */
+                val1 &= vie->immediate;
+                error = memwrite(vm, vcpuid, gpa, val1, size, arg);
+		break;
+	default:
+		break;
+	}
+	return (error);
+}
+
+int
+vmm_emulate_instruction(void *vm, int vcpuid, uint64_t gpa, struct vie *vie,
+			mem_region_read_t memread, mem_region_write_t memwrite,
+			void *memarg)
+{
+	int error;
+
+	if (!vie->decoded)
+		return (EINVAL);
+
+	switch (vie->op.op_type) {
+	case VIE_OP_TYPE_MOV:
+		error = emulate_mov(vm, vcpuid, gpa, vie,
+				    memread, memwrite, memarg);
+		break;
+	case VIE_OP_TYPE_AND:
+		error = emulate_and(vm, vcpuid, gpa, vie,
+				    memread, memwrite, memarg);
+		break;
+	default:
+		error = EINVAL;
+		break;
+	}
+
+	return (error);
+}
+
+#ifdef _KERNEL
+static void
+vie_init(struct vie *vie)
+{
+
+	bzero(vie, sizeof(struct vie));
+
+	vie->base_register = VM_REG_LAST;
+	vie->index_register = VM_REG_LAST;
+}
+
+static int
+gla2gpa(struct vm *vm, uint64_t gla, uint64_t ptpphys,
+	uint64_t *gpa, uint64_t *gpaend)
+{
+	vm_paddr_t hpa;
+	int nlevels, ptpshift, ptpindex;
+	uint64_t *ptpbase, pte, pgsize;
+
+	/*
+	 * XXX assumes 64-bit guest with 4 page walk levels
+	 */
+	nlevels = 4;
+	while (--nlevels >= 0) {
+		/* Zero out the lower 12 bits and the upper 12 bits */
+		ptpphys >>= 12; ptpphys <<= 24; ptpphys >>= 12;
+
+		hpa = vm_gpa2hpa(vm, ptpphys, PAGE_SIZE);
+		if (hpa == -1)
+			goto error;
+
+		ptpbase = (uint64_t *)PHYS_TO_DMAP(hpa);
+
+		ptpshift = PAGE_SHIFT + nlevels * 9;
+		ptpindex = (gla >> ptpshift) & 0x1FF;
+		pgsize = 1UL << ptpshift;
+
+		pte = ptpbase[ptpindex];
+
+		if ((pte & PG_V) == 0)
+			goto error;
+
+		if (pte & PG_PS) {
+			if (pgsize > 1 * GB)
+				goto error;
+			else
+				break;
+		}
+
+		ptpphys = pte;
+	}
+
+	/* Zero out the lower 'ptpshift' bits and the upper 12 bits */
+	pte >>= ptpshift; pte <<= (ptpshift + 12); pte >>= 12;
+	*gpa = pte | (gla & (pgsize - 1));
+	*gpaend = pte + pgsize;
+	return (0);
+
+error:
+	return (-1);
+}
+
+int
+vmm_fetch_instruction(struct vm *vm, int cpuid, uint64_t rip, int inst_length,
+		      uint64_t cr3, struct vie *vie)
+{
+	int n, err;
+	uint64_t hpa, gpa, gpaend, off;
+
+	/*
+	 * XXX cache previously fetched instructions using 'rip' as the tag
+	 */
+
+	if (inst_length > VIE_INST_SIZE)
+		panic("vmm_fetch_instruction: invalid length %d", inst_length);
+
+	vie_init(vie);
+
+	/* Copy the instruction into 'vie' */
+	while (vie->num_valid < inst_length) {
+		err = gla2gpa(vm, rip, cr3, &gpa, &gpaend);
+		if (err)
+			break;
+
+		off = gpa & PAGE_MASK;
+		n = min(inst_length - vie->num_valid, PAGE_SIZE - off);
+
+		hpa = vm_gpa2hpa(vm, gpa, n);
+		if (hpa == -1)
+			break;
+
+		bcopy((void *)PHYS_TO_DMAP(hpa), &vie->inst[vie->num_valid], n);
+
+		rip += n;
+		vie->num_valid += n;
+	}
+
+	if (vie->num_valid == inst_length)
+		return (0);
+	else
+		return (-1);
+}
+
+static int
+vie_peek(struct vie *vie, uint8_t *x)
+{
+
+	if (vie->num_processed < vie->num_valid) {
+		*x = vie->inst[vie->num_processed];
+		return (0);
+	} else
+		return (-1);
+}
+
+static void
+vie_advance(struct vie *vie)
+{
+
+	vie->num_processed++;
+}
+
+static int
+decode_rex(struct vie *vie)
+{
+	uint8_t x;
+
+	if (vie_peek(vie, &x))
+		return (-1);
+
+	if (x >= 0x40 && x <= 0x4F) {
+		vie->rex_present = 1;
+
+		vie->rex_w = x & 0x8 ? 1 : 0;
+		vie->rex_r = x & 0x4 ? 1 : 0;
+		vie->rex_x = x & 0x2 ? 1 : 0;
+		vie->rex_b = x & 0x1 ? 1 : 0;
+
+		vie_advance(vie);
+	}
+
+	return (0);
+}
+
+static int
+decode_opcode(struct vie *vie)
+{
+	uint8_t x;
+
+	if (vie_peek(vie, &x))
+		return (-1);
+
+	vie->op = one_byte_opcodes[x];
+
+	if (vie->op.op_type == VIE_OP_TYPE_NONE)
+		return (-1);
+
+	vie_advance(vie);
+	return (0);
+}
+
+/*
+ * XXX assuming 32-bit or 64-bit guest
+ */
+static int
+decode_modrm(struct vie *vie)
+{
+	uint8_t x;
+
+	if (vie_peek(vie, &x))
+		return (-1);
+
+	vie->mod = (x >> 6) & 0x3;
+	vie->rm =  (x >> 0) & 0x7;
+	vie->reg = (x >> 3) & 0x7;
+
+	/*
+	 * A direct addressing mode makes no sense in the context of an EPT
+	 * fault. There has to be a memory access involved to cause the
+	 * EPT fault.
+	 */
+	if (vie->mod == VIE_MOD_DIRECT)
+		return (-1);
+
+	if ((vie->mod == VIE_MOD_INDIRECT && vie->rm == VIE_RM_DISP32) ||
+	    (vie->mod != VIE_MOD_DIRECT && vie->rm == VIE_RM_SIB)) {
+		/*
+		 * Table 2-5: Special Cases of REX Encodings
+		 *
+		 * mod=0, r/m=5 is used in the compatibility mode to
+		 * indicate a disp32 without a base register.
+		 *
+		 * mod!=3, r/m=4 is used in the compatibility mode to
+		 * indicate that the SIB byte is present.
+		 *
+		 * The 'b' bit in the REX prefix is don't care in
+		 * this case.
+		 */
+	} else {
+		vie->rm |= (vie->rex_b << 3);
+	}
+
+	vie->reg |= (vie->rex_r << 3);
+
+	/* SIB */
+	if (vie->mod != VIE_MOD_DIRECT && vie->rm == VIE_RM_SIB)
+		goto done;
+
+	vie->base_register = gpr_map[vie->rm];
+
+	switch (vie->mod) {
+	case VIE_MOD_INDIRECT_DISP8:
+		vie->disp_bytes = 1;
+		break;
+	case VIE_MOD_INDIRECT_DISP32:
+		vie->disp_bytes = 4;
+		break;
+	case VIE_MOD_INDIRECT:
+		if (vie->rm == VIE_RM_DISP32) {
+			vie->disp_bytes = 4;
+			vie->base_register = VM_REG_LAST;	/* no base */
+		}
+		break;
+	}
+
+	/* Figure out immediate operand size (if any) */
+	if (vie->op.op_flags & VIE_OP_F_IMM)
+		vie->imm_bytes = 4;
+	else if (vie->op.op_flags & VIE_OP_F_IMM8)
+		vie->imm_bytes = 1;
+
+done:
+	vie_advance(vie);
+
+	return (0);
+}
+
+static int
+decode_sib(struct vie *vie)
+{
+	uint8_t x;
+
+	/* Proceed only if SIB byte is present */
+	if (vie->mod == VIE_MOD_DIRECT || vie->rm != VIE_RM_SIB)
+		return (0);
+
+	if (vie_peek(vie, &x))
+		return (-1);
+
+	/* De-construct the SIB byte */
+	vie->ss = (x >> 6) & 0x3;
+	vie->index = (x >> 3) & 0x7;
+	vie->base = (x >> 0) & 0x7;
+
+	/* Apply the REX prefix modifiers */
+	vie->index |= vie->rex_x << 3;
+	vie->base |= vie->rex_b << 3;
+
+	switch (vie->mod) {
+	case VIE_MOD_INDIRECT_DISP8:
+		vie->disp_bytes = 1;
+		break;
+	case VIE_MOD_INDIRECT_DISP32:
+		vie->disp_bytes = 4;
+		break;
+	}
+
+	if (vie->mod == VIE_MOD_INDIRECT &&
+	    (vie->base == 5 || vie->base == 13)) {
+		/*
+		 * Special case when base register is unused if mod = 0
+		 * and base = %rbp or %r13.
+		 *
+		 * Documented in:
+		 * Table 2-3: 32-bit Addressing Forms with the SIB Byte
+		 * Table 2-5: Special Cases of REX Encodings
+		 */
+		vie->disp_bytes = 4;
+	} else {
+		vie->base_register = gpr_map[vie->base];
+	}
+
+	/*
+	 * All encodings of 'index' are valid except for %rsp (4).
+	 *
+	 * Documented in:
+	 * Table 2-3: 32-bit Addressing Forms with the SIB Byte
+	 * Table 2-5: Special Cases of REX Encodings
+	 */
+	if (vie->index != 4)
+		vie->index_register = gpr_map[vie->index];
+
+	/* 'scale' makes sense only in the context of an index register */
+	if (vie->index_register < VM_REG_LAST)
+		vie->scale = 1 << vie->ss;
+
+	vie_advance(vie);
+
+	return (0);
+}
+
+static int
+decode_displacement(struct vie *vie)
+{
+	int n, i;
+	uint8_t x;
+
+	union {
+		char	buf[4];
+		int8_t	signed8;
+		int32_t	signed32;
+	} u;
+
+	if ((n = vie->disp_bytes) == 0)
+		return (0);
+
+	if (n != 1 && n != 4)
+		panic("decode_displacement: invalid disp_bytes %d", n);
+
+	for (i = 0; i < n; i++) {
+		if (vie_peek(vie, &x))
+			return (-1);
+
+		u.buf[i] = x;
+		vie_advance(vie);
+	}
+
+	if (n == 1)
+		vie->displacement = u.signed8;		/* sign-extended */
+	else
+		vie->displacement = u.signed32;		/* sign-extended */
+
+	return (0);
+}
+
+static int
+decode_immediate(struct vie *vie)
+{
+	int i, n;
+	uint8_t x;
+	union {
+		char	buf[4];
+		int8_t	signed8;
+		int32_t	signed32;
+	} u;
+
+	if ((n = vie->imm_bytes) == 0)
+		return (0);
+
+	if (n != 1 && n != 4)
+		panic("decode_immediate: invalid imm_bytes %d", n);
+
+	for (i = 0; i < n; i++) {
+		if (vie_peek(vie, &x))
+			return (-1);
+
+		u.buf[i] = x;
+		vie_advance(vie);
+	}
+	
+	if (n == 1)
+		vie->immediate = u.signed8;		/* sign-extended */
+	else
+		vie->immediate = u.signed32;		/* sign-extended */
+
+	return (0);
+}
+
+#define	VERIFY_GLA
+/*
+ * Verify that the 'guest linear address' provided as collateral of the nested
+ * page table fault matches with our instruction decoding.
+ */
+#ifdef VERIFY_GLA
+static int
+verify_gla(struct vm *vm, int cpuid, uint64_t gla, struct vie *vie)
+{
+	int error;
+	uint64_t base, idx;
+
+	base = 0;
+	if (vie->base_register != VM_REG_LAST) {
+		error = vm_get_register(vm, cpuid, vie->base_register, &base);
+		if (error) {
+			printf("verify_gla: error %d getting base reg %d\n",
+				error, vie->base_register);
+			return (-1);
+		}
+	}
+
+	idx = 0;
+	if (vie->index_register != VM_REG_LAST) {
+		error = vm_get_register(vm, cpuid, vie->index_register, &idx);
+		if (error) {
+			printf("verify_gla: error %d getting index reg %d\n",
+				error, vie->index_register);
+			return (-1);
+		}
+	}
+
+	if (base + vie->scale * idx + vie->displacement != gla) {
+		printf("verify_gla mismatch: "
+		       "base(0x%0lx), scale(%d), index(0x%0lx), "
+		       "disp(0x%0lx), gla(0x%0lx)\n",
+		       base, vie->scale, idx, vie->displacement, gla);
+		return (-1);
+	}
+
+	return (0);
+}
+#endif	/* VERIFY_GLA */
+
+int
+vmm_decode_instruction(struct vm *vm, int cpuid, uint64_t gla, struct vie *vie)
+{
+
+	if (decode_rex(vie))
+		return (-1);
+
+	if (decode_opcode(vie))
+		return (-1);
+
+	if (decode_modrm(vie))
+		return (-1);
+
+	if (decode_sib(vie))
+		return (-1);
+
+	if (decode_displacement(vie))
+		return (-1);
+	
+	if (decode_immediate(vie))
+		return (-1);
+
+#ifdef VERIFY_GLA
+	if (verify_gla(vm, cpuid, gla, vie))
+		return (-1);
+#endif
+
+	vie->decoded = 1;	/* success */
+
+	return (0);
+}
+#endif	/* _KERNEL */
diff --git a/sys/amd64/vmm/vmm_ipi.c b/sys/amd64/vmm/vmm_ipi.c
new file mode 100644
index 0000000..643d326
--- /dev/null
+++ b/sys/amd64/vmm/vmm_ipi.c
@@ -0,0 +1,93 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/proc.h>
+#include <sys/bus.h>
+
+#include <machine/intr_machdep.h>
+#include <machine/apicvar.h>
+#include <machine/segments.h>
+#include <machine/md_var.h>
+
+#include <machine/vmm.h>
+#include "vmm_ipi.h"
+
+extern inthand_t IDTVEC(rsvd), IDTVEC(justreturn);
+
+/*
+ * The default is to use the IPI_AST to interrupt a vcpu.
+ */
+int vmm_ipinum = IPI_AST;
+
+CTASSERT(APIC_SPURIOUS_INT == 255);
+
+void
+vmm_ipi_init(void)
+{
+	int idx;
+	uintptr_t func;
+	struct gate_descriptor *ip;
+
+	/*
+	 * Search backwards from the highest IDT vector available for use
+	 * as our IPI vector. We install the 'justreturn' handler at that
+	 * vector and use it to interrupt the vcpus.
+	 *
+	 * We do this because the IPI_AST is heavyweight and saves all
+	 * registers in the trapframe. This is overkill for our use case
+	 * which is simply to EOI the interrupt and return.
+	 */
+	idx = APIC_SPURIOUS_INT;
+	while (--idx >= APIC_IPI_INTS) {
+		ip = &idt[idx];
+		func = ((long)ip->gd_hioffset << 16 | ip->gd_looffset);
+		if (func == (uintptr_t)&IDTVEC(rsvd)) {
+			vmm_ipinum = idx;
+			setidt(vmm_ipinum, IDTVEC(justreturn), SDT_SYSIGT,
+			       SEL_KPL, 0);
+			break;
+		}
+	}
+	
+	if (vmm_ipinum != IPI_AST && bootverbose) {
+		printf("vmm_ipi_init: installing ipi handler to interrupt "
+		       "vcpus at vector %d\n", vmm_ipinum);
+	}
+}
+
+void
+vmm_ipi_cleanup(void)
+{
+	if (vmm_ipinum != IPI_AST)
+		setidt(vmm_ipinum, IDTVEC(rsvd), SDT_SYSIGT, SEL_KPL, 0);
+}
diff --git a/sys/amd64/vmm/vmm_ipi.h b/sys/amd64/vmm/vmm_ipi.h
new file mode 100644
index 0000000..91552e3
--- /dev/null
+++ b/sys/amd64/vmm/vmm_ipi.h
@@ -0,0 +1,39 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VMM_IPI_H_
+#define _VMM_IPI_H_
+
+struct vm;
+
+extern int vmm_ipinum;
+
+void	vmm_ipi_init(void);
+void	vmm_ipi_cleanup(void);
+
+#endif
diff --git a/sys/amd64/vmm/vmm_ktr.h b/sys/amd64/vmm/vmm_ktr.h
new file mode 100644
index 0000000..e691c61
--- /dev/null
+++ b/sys/amd64/vmm/vmm_ktr.h
@@ -0,0 +1,51 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VMM_KTR_H_
+#define	_VMM_KTR_H_
+
+#include <sys/ktr.h>
+#include <sys/pcpu.h>
+
+#define	KTR_VMM	KTR_GEN
+
+#define	VMM_CTR0(vm, vcpuid, format)					\
+CTR3(KTR_VMM, "vm %s-%d(%d): " format, vm_name((vm)), (vcpuid), curcpu)
+
+#define	VMM_CTR1(vm, vcpuid, format, p1)				\
+CTR4(KTR_VMM, "vm %s-%d(%d): " format, vm_name((vm)), (vcpuid), curcpu, \
+			(p1))
+
+#define	VMM_CTR2(vm, vcpuid, format, p1, p2)				\
+CTR5(KTR_VMM, "vm %s-%d(%d): " format, vm_name((vm)), (vcpuid), curcpu, \
+			(p1), (p2))
+
+#define	VMM_CTR3(vm, vcpuid, format, p1, p2, p3)			\
+CTR6(KTR_VMM, "vm %s-%d(%d): " format, vm_name((vm)), (vcpuid), curcpu, \
+			(p1), (p2), (p3))
+#endif
diff --git a/sys/amd64/vmm/vmm_lapic.c b/sys/amd64/vmm/vmm_lapic.c
new file mode 100644
index 0000000..d024b71
--- /dev/null
+++ b/sys/amd64/vmm/vmm_lapic.c
@@ -0,0 +1,201 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/smp.h>
+
+#include <x86/specialreg.h>
+#include <x86/apicreg.h>
+
+#include <machine/vmm.h>
+#include "vmm_ipi.h"
+#include "vmm_lapic.h"
+#include "vlapic.h"
+
+int
+lapic_pending_intr(struct vm *vm, int cpu)
+{
+	struct vlapic *vlapic;
+
+	vlapic = vm_lapic(vm, cpu);
+
+	return (vlapic_pending_intr(vlapic));
+}
+
+void
+lapic_intr_accepted(struct vm *vm, int cpu, int vector)
+{
+	struct vlapic *vlapic;
+
+	vlapic = vm_lapic(vm, cpu);
+
+	vlapic_intr_accepted(vlapic, vector);
+}
+
+int
+lapic_set_intr(struct vm *vm, int cpu, int vector)
+{
+	struct vlapic *vlapic;
+
+	if (cpu < 0 || cpu >= VM_MAXCPU)
+		return (EINVAL);
+
+	if (vector < 32 || vector > 255)
+		return (EINVAL);
+
+	vlapic = vm_lapic(vm, cpu);
+	vlapic_set_intr_ready(vlapic, vector);
+
+	vm_interrupt_hostcpu(vm, cpu);
+
+	return (0);
+}
+
+int
+lapic_timer_tick(struct vm *vm, int cpu)
+{
+	struct vlapic *vlapic;
+
+	vlapic = vm_lapic(vm, cpu);
+
+	return (vlapic_timer_tick(vlapic));
+}
+
+static boolean_t
+x2apic_msr(u_int msr)
+{
+	if (msr >= 0x800 && msr <= 0xBFF)
+		return (TRUE);
+	else
+		return (FALSE);
+}
+
+static u_int
+x2apic_msr_to_regoff(u_int msr)
+{
+
+	return ((msr - 0x800) << 4);
+}
+
+boolean_t
+lapic_msr(u_int msr)
+{
+
+	if (x2apic_msr(msr) || (msr == MSR_APICBASE))
+		return (TRUE);
+	else
+		return (FALSE);
+}
+
+int
+lapic_rdmsr(struct vm *vm, int cpu, u_int msr, uint64_t *rval)
+{
+	int error;
+	u_int offset;
+	struct vlapic *vlapic;
+
+	vlapic = vm_lapic(vm, cpu);
+
+	if (msr == MSR_APICBASE) {
+		*rval = vlapic_get_apicbase(vlapic);
+		error = 0;
+	} else {
+		offset = x2apic_msr_to_regoff(msr);
+		error = vlapic_op_mem_read(vlapic, offset, DWORD, rval);
+	}
+
+	return (error);
+}
+
+int
+lapic_wrmsr(struct vm *vm, int cpu, u_int msr, uint64_t val)
+{
+	int error;
+	u_int offset;
+	struct vlapic *vlapic;
+
+	vlapic = vm_lapic(vm, cpu);
+
+	if (msr == MSR_APICBASE) {
+		vlapic_set_apicbase(vlapic, val);
+		error = 0;
+	} else {
+		offset = x2apic_msr_to_regoff(msr);
+		error = vlapic_op_mem_write(vlapic, offset, DWORD, val);
+	}
+
+	return (error);
+}
+
+int
+lapic_mmio_write(void *vm, int cpu, uint64_t gpa, uint64_t wval, int size,
+		 void *arg)
+{
+	int error;
+	uint64_t off;
+	struct vlapic *vlapic;
+
+	off = gpa - DEFAULT_APIC_BASE;
+
+	/*
+	 * Memory mapped local apic accesses must be 4 bytes wide and
+	 * aligned on a 16-byte boundary.
+	 */
+	if (size != 4 || off & 0xf)
+		return (EINVAL);
+
+	vlapic = vm_lapic(vm, cpu);
+	error = vlapic_op_mem_write(vlapic, off, DWORD, wval);
+	return (error);
+}
+
+int
+lapic_mmio_read(void *vm, int cpu, uint64_t gpa, uint64_t *rval, int size,
+		void *arg)
+{
+	int error;
+	uint64_t off;
+	struct vlapic *vlapic;
+
+	off = gpa - DEFAULT_APIC_BASE;
+
+	/*
+	 * Memory mapped local apic accesses must be 4 bytes wide and
+	 * aligned on a 16-byte boundary.
+	 */
+	if (size != 4 || off & 0xf)
+		return (EINVAL);
+
+	vlapic = vm_lapic(vm, cpu);
+	error = vlapic_op_mem_read(vlapic, off, DWORD, rval);
+	return (error);
+}
diff --git a/sys/amd64/vmm/vmm_lapic.h b/sys/amd64/vmm/vmm_lapic.h
new file mode 100644
index 0000000..a79912e
--- /dev/null
+++ b/sys/amd64/vmm/vmm_lapic.h
@@ -0,0 +1,71 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VMM_LAPIC_H_
+#define	_VMM_LAPIC_H_
+
+struct vm;
+
+boolean_t lapic_msr(u_int num);
+int	lapic_rdmsr(struct vm *vm, int cpu, u_int msr, uint64_t *rval);
+int	lapic_wrmsr(struct vm *vm, int cpu, u_int msr, uint64_t wval);
+
+int	lapic_mmio_read(void *vm, int cpu, uint64_t gpa,
+			uint64_t *rval, int size, void *arg);
+int	lapic_mmio_write(void *vm, int cpu, uint64_t gpa,
+			 uint64_t wval, int size, void *arg);
+
+int	lapic_timer_tick(struct vm *vm, int cpu);
+
+/*
+ * Returns a vector between 32 and 255 if an interrupt is pending in the
+ * IRR that can be delivered based on the current state of ISR and TPR.
+ *
+ * Note that the vector does not automatically transition to the ISR as a
+ * result of calling this function.
+ *
+ * Returns -1 if there is no eligible vector that can be delivered to the
+ * guest at this time.
+ */
+int	lapic_pending_intr(struct vm *vm, int cpu);
+
+/*
+ * Transition 'vector' from IRR to ISR. This function is called with the
+ * vector returned by 'lapic_pending_intr()' when the guest is able to
+ * accept this interrupt (i.e. RFLAGS.IF = 1 and no conditions exist that
+ * block interrupt delivery).
+ */
+void	lapic_intr_accepted(struct vm *vm, int cpu, int vector);
+
+/*
+ * Signals to the LAPIC that an interrupt at 'vector' needs to be generated
+ * to the 'cpu', the state is recorded in IRR.
+ */
+int	lapic_set_intr(struct vm *vm, int cpu, int vector);
+
+#endif
diff --git a/sys/amd64/vmm/vmm_mem.c b/sys/amd64/vmm/vmm_mem.c
new file mode 100644
index 0000000..04f99b1
--- /dev/null
+++ b/sys/amd64/vmm/vmm_mem.c
@@ -0,0 +1,135 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/linker.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/kernel.h>
+#include <sys/sysctl.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_page.h>
+#include <vm/vm_pageout.h>
+
+#include <machine/md_var.h>
+#include <machine/metadata.h>
+#include <machine/pc/bios.h>
+#include <machine/vmparam.h>
+#include <machine/pmap.h>
+
+#include "vmm_util.h"
+#include "vmm_mem.h"
+
+SYSCTL_DECL(_hw_vmm);
+
+static u_long pages_allocated;
+SYSCTL_ULONG(_hw_vmm, OID_AUTO, pages_allocated, CTLFLAG_RD,
+	     &pages_allocated, 0, "4KB pages allocated");
+
+static void
+update_pages_allocated(int howmany)
+{
+	pages_allocated += howmany;	/* XXX locking? */
+}
+
+int
+vmm_mem_init(void)
+{
+
+	return (0);
+}
+
+vm_paddr_t
+vmm_mem_alloc(size_t size)
+{
+	int flags;
+	vm_page_t m;
+	vm_paddr_t pa;
+
+	if (size != PAGE_SIZE)
+		panic("vmm_mem_alloc: invalid allocation size %lu", size);
+
+	flags = VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
+		VM_ALLOC_ZERO;
+
+	while (1) {
+		/*
+		 * XXX need policy to determine when to back off the allocation
+		 */
+		m = vm_page_alloc(NULL, 0, flags);
+		if (m == NULL)
+			VM_WAIT;
+		else
+			break;
+	}
+
+	pa = VM_PAGE_TO_PHYS(m);
+	
+	if ((m->flags & PG_ZERO) == 0)
+		pagezero((void *)PHYS_TO_DMAP(pa));
+	m->valid = VM_PAGE_BITS_ALL;
+
+	update_pages_allocated(1);
+
+	return (pa);
+}
+
+void
+vmm_mem_free(vm_paddr_t base, size_t length)
+{
+	vm_page_t m;
+
+	if (base & PAGE_MASK) {
+		panic("vmm_mem_free: base 0x%0lx must be aligned on a "
+		      "0x%0x boundary\n", base, PAGE_SIZE);
+	}
+
+	if (length != PAGE_SIZE)
+		panic("vmm_mem_free: invalid length %lu", length);
+
+	m = PHYS_TO_VM_PAGE(base);
+	m->wire_count--;
+	vm_page_free(m);
+	atomic_subtract_int(&cnt.v_wire_count, 1);
+
+	update_pages_allocated(-1);
+}
+
+vm_paddr_t
+vmm_mem_maxaddr(void)
+{
+
+	return (ptoa(Maxmem));
+}
diff --git a/sys/amd64/vmm/vmm_mem.h b/sys/amd64/vmm/vmm_mem.h
new file mode 100644
index 0000000..7d45c74
--- /dev/null
+++ b/sys/amd64/vmm/vmm_mem.h
@@ -0,0 +1,37 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_VMM_MEM_H_
+#define	_VMM_MEM_H_
+
+int		vmm_mem_init(void);
+vm_paddr_t	vmm_mem_alloc(size_t size);
+void		vmm_mem_free(vm_paddr_t start, size_t size);
+vm_paddr_t	vmm_mem_maxaddr(void);
+
+#endif
diff --git a/sys/amd64/vmm/vmm_msr.c b/sys/amd64/vmm/vmm_msr.c
new file mode 100644
index 0000000..d97c819
--- /dev/null
+++ b/sys/amd64/vmm/vmm_msr.c
@@ -0,0 +1,254 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/smp.h>
+
+#include <machine/specialreg.h>
+
+#include <machine/vmm.h>
+#include "vmm_lapic.h"
+#include "vmm_msr.h"
+
+#define	VMM_MSR_F_EMULATE	0x01
+#define	VMM_MSR_F_READONLY	0x02
+#define VMM_MSR_F_INVALID	0x04  /* guest_msr_valid() can override this */
+
+struct vmm_msr {
+	int		num;
+	int		flags;
+	uint64_t	hostval;
+};
+
+static struct vmm_msr vmm_msr[] = {
+	{ MSR_LSTAR,	0 },
+	{ MSR_CSTAR,	0 },
+	{ MSR_STAR,	0 },
+	{ MSR_SF_MASK,	0 },
+	{ MSR_PAT,      VMM_MSR_F_EMULATE | VMM_MSR_F_INVALID },
+	{ MSR_BIOS_SIGN,VMM_MSR_F_EMULATE },
+	{ MSR_MCG_CAP,	VMM_MSR_F_EMULATE | VMM_MSR_F_READONLY },
+};
+
+#define	vmm_msr_num	(sizeof(vmm_msr) / sizeof(vmm_msr[0]))
+CTASSERT(VMM_MSR_NUM >= vmm_msr_num);
+
+#define	readonly_msr(idx)	\
+	((vmm_msr[(idx)].flags & VMM_MSR_F_READONLY) != 0)
+
+#define	emulated_msr(idx)	\
+	((vmm_msr[(idx)].flags & VMM_MSR_F_EMULATE) != 0)
+
+#define invalid_msr(idx)	\
+	((vmm_msr[(idx)].flags & VMM_MSR_F_INVALID) != 0)
+
+void
+vmm_msr_init(void)
+{
+	int i;
+
+	for (i = 0; i < vmm_msr_num; i++) {
+		if (emulated_msr(i))
+			continue;
+		/*
+		 * XXX this assumes that the value of the host msr does not
+		 * change after we have cached it.
+		 */
+		vmm_msr[i].hostval = rdmsr(vmm_msr[i].num);
+	}
+}
+
+void
+guest_msrs_init(struct vm *vm, int cpu)
+{
+	int i;
+	uint64_t *guest_msrs;
+
+	guest_msrs = vm_guest_msrs(vm, cpu);
+	
+	for (i = 0; i < vmm_msr_num; i++) {
+		switch (vmm_msr[i].num) {
+		case MSR_LSTAR:
+		case MSR_CSTAR:
+		case MSR_STAR:
+		case MSR_SF_MASK:
+		case MSR_BIOS_SIGN:
+		case MSR_MCG_CAP:
+			guest_msrs[i] = 0;
+			break;
+		case MSR_PAT:
+			guest_msrs[i] = PAT_VALUE(0, PAT_WRITE_BACK)      |
+				PAT_VALUE(1, PAT_WRITE_THROUGH)   |
+				PAT_VALUE(2, PAT_UNCACHED)        |
+				PAT_VALUE(3, PAT_UNCACHEABLE)     |
+				PAT_VALUE(4, PAT_WRITE_BACK)      |
+				PAT_VALUE(5, PAT_WRITE_THROUGH)   |
+				PAT_VALUE(6, PAT_UNCACHED)        |
+				PAT_VALUE(7, PAT_UNCACHEABLE);
+			break;
+		default:
+			panic("guest_msrs_init: missing initialization for msr "
+			      "0x%0x", vmm_msr[i].num);
+		}
+	}
+}
+
+static int
+msr_num_to_idx(u_int num)
+{
+	int i;
+
+	for (i = 0; i < vmm_msr_num; i++)
+		if (vmm_msr[i].num == num)
+			return (i);
+
+	return (-1);
+}
+
+int
+emulate_wrmsr(struct vm *vm, int cpu, u_int num, uint64_t val)
+{
+	int idx;
+	uint64_t *guest_msrs;
+
+	if (lapic_msr(num))
+		return (lapic_wrmsr(vm, cpu, num, val));
+
+	idx = msr_num_to_idx(num);
+	if (idx < 0 || invalid_msr(idx))
+		return (EINVAL);
+
+	if (!readonly_msr(idx)) {
+		guest_msrs = vm_guest_msrs(vm, cpu);
+
+		/* Stash the value */
+		guest_msrs[idx] = val;
+
+		/* Update processor state for non-emulated MSRs */
+		if (!emulated_msr(idx))
+			wrmsr(vmm_msr[idx].num, val);
+	}
+
+	return (0);
+}
+
+int
+emulate_rdmsr(struct vm *vm, int cpu, u_int num)
+{
+	int error, idx;
+	uint32_t eax, edx;
+	uint64_t result, *guest_msrs;
+
+	if (lapic_msr(num)) {
+		error = lapic_rdmsr(vm, cpu, num, &result);
+		goto done;
+	}
+
+	idx = msr_num_to_idx(num);
+	if (idx < 0 || invalid_msr(idx)) {
+		error = EINVAL;
+		goto done;
+	}
+
+	guest_msrs = vm_guest_msrs(vm, cpu);
+	result = guest_msrs[idx];
+
+	/*
+	 * If this is not an emulated msr register make sure that the processor
+	 * state matches our cached state.
+	 */
+	if (!emulated_msr(idx) && (rdmsr(num) != result)) {
+		panic("emulate_rdmsr: msr 0x%0x has inconsistent cached "
+		      "(0x%016lx) and actual (0x%016lx) values", num,
+		      result, rdmsr(num));
+	}
+
+	error = 0;
+
+done:
+	if (error == 0) {
+		eax = result;
+		edx = result >> 32;
+		error = vm_set_register(vm, cpu, VM_REG_GUEST_RAX, eax);
+		if (error)
+			panic("vm_set_register(rax) error %d", error);
+		error = vm_set_register(vm, cpu, VM_REG_GUEST_RDX, edx);
+		if (error)
+			panic("vm_set_register(rdx) error %d", error);
+	}
+	return (error);
+}
+
+void
+restore_guest_msrs(struct vm *vm, int cpu)
+{
+	int i;
+	uint64_t *guest_msrs;
+
+	guest_msrs = vm_guest_msrs(vm, cpu);
+
+	for (i = 0; i < vmm_msr_num; i++) {
+		if (emulated_msr(i))
+			continue;
+		else
+			wrmsr(vmm_msr[i].num, guest_msrs[i]);
+	}
+}
+
+void
+restore_host_msrs(struct vm *vm, int cpu)
+{
+	int i;
+
+	for (i = 0; i < vmm_msr_num; i++) {
+		if (emulated_msr(i))
+			continue;
+		else
+			wrmsr(vmm_msr[i].num, vmm_msr[i].hostval);
+	}
+}
+
+/*
+ * Must be called by the CPU-specific code before any guests are
+ * created
+ */
+void
+guest_msr_valid(int msr)
+{
+	int i;
+
+	for (i = 0; i < vmm_msr_num; i++) {
+		if (vmm_msr[i].num == msr && invalid_msr(i)) {
+			vmm_msr[i].flags &= ~VMM_MSR_F_INVALID;
+		}
+	}
+}
diff --git a/sys/amd64/vmm/vmm_msr.h b/sys/amd64/vmm/vmm_msr.h
new file mode 100644
index 0000000..8a1fda3
--- /dev/null
+++ b/sys/amd64/vmm/vmm_msr.h
@@ -0,0 +1,43 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_VMM_MSR_H_
+#define	_VMM_MSR_H_
+
+#define	VMM_MSR_NUM	16
+struct vm;
+
+void	vmm_msr_init(void);
+int	emulate_wrmsr(struct vm *vm, int vcpu, u_int msr, uint64_t val);
+int	emulate_rdmsr(struct vm *vm, int vcpu, u_int msr);
+void	guest_msrs_init(struct vm *vm, int cpu);
+void	guest_msr_valid(int msr);
+void	restore_host_msrs(struct vm *vm, int cpu);
+void	restore_guest_msrs(struct vm *vm, int cpu);
+
+#endif
diff --git a/sys/amd64/vmm/vmm_stat.c b/sys/amd64/vmm/vmm_stat.c
new file mode 100644
index 0000000..ae60979
--- /dev/null
+++ b/sys/amd64/vmm/vmm_stat.c
@@ -0,0 +1,104 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/smp.h>
+
+#include <machine/vmm.h>
+#include "vmm_stat.h"
+
+static int vstnum;
+static struct vmm_stat_type *vsttab[MAX_VMM_STAT_TYPES];
+
+static MALLOC_DEFINE(M_VMM_STAT, "vmm stat", "vmm stat");
+
+void
+vmm_stat_init(void *arg)
+{
+	struct vmm_stat_type *vst = arg;
+
+	/* We require all stats to identify themselves with a description */
+	if (vst->desc == NULL)
+		return;
+
+	if (vstnum >= MAX_VMM_STAT_TYPES) {
+		printf("Cannot accomodate vmm stat type \"%s\"!\n", vst->desc);
+		return;
+	}
+
+	vst->index = vstnum;
+	vsttab[vstnum++] = vst;
+}
+
+int
+vmm_stat_copy(struct vm *vm, int vcpu, int *num_stats, uint64_t *buf)
+{
+	int i;
+	uint64_t *stats;
+
+	if (vcpu < 0 || vcpu >= VM_MAXCPU)
+		return (EINVAL);
+		
+	stats = vcpu_stats(vm, vcpu);
+	for (i = 0; i < vstnum; i++)
+		buf[i] = stats[i];
+	*num_stats = vstnum;
+	return (0);
+}
+
+void *
+vmm_stat_alloc(void)
+{
+	u_long size;
+	
+	size = vstnum * sizeof(uint64_t);
+
+	return (malloc(size, M_VMM_STAT, M_ZERO | M_WAITOK));
+}
+
+void
+vmm_stat_free(void *vp)
+{
+	free(vp, M_VMM_STAT);
+}
+
+const char *
+vmm_stat_desc(int index)
+{
+
+	if (index >= 0 && index < vstnum)
+		return (vsttab[index]->desc);
+	else
+		return (NULL);
+}
diff --git a/sys/amd64/vmm/vmm_stat.h b/sys/amd64/vmm/vmm_stat.h
new file mode 100644
index 0000000..7c075a6
--- /dev/null
+++ b/sys/amd64/vmm/vmm_stat.h
@@ -0,0 +1,71 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VMM_STAT_H_
+#define	_VMM_STAT_H_
+
+struct vm;
+
+#define	MAX_VMM_STAT_TYPES	64		/* arbitrary */
+
+struct vmm_stat_type {
+	const char	*desc;		/* description of statistic */
+	int	index;			/* position in the stats buffer */
+};
+
+void	vmm_stat_init(void *arg);
+
+#define	VMM_STAT_DEFINE(type, desc)					\
+	struct vmm_stat_type type[1] = {				\
+		{ desc, -1 }						\
+	};								\
+	SYSINIT(type##_stat, SI_SUB_KLD, SI_ORDER_ANY, vmm_stat_init, type)
+
+void	*vmm_stat_alloc(void);
+void 	vmm_stat_free(void *vp);
+
+/*
+ * 'buf' should be at least fit 'MAX_VMM_STAT_TYPES' entries
+ */
+int	vmm_stat_copy(struct vm *vm, int vcpu, int *num_stats, uint64_t *buf);
+const char *vmm_stat_desc(int index);
+
+static void __inline
+vmm_stat_incr(struct vm *vm, int vcpu, struct vmm_stat_type *vst, uint64_t x)
+{
+#ifdef	VMM_KEEP_STATS
+	uint64_t *stats = vcpu_stats(vm, vcpu);
+	if (vst->index >= 0)
+		stats[vst->index] += x;
+#endif
+}
+
+#endif
diff --git a/sys/amd64/vmm/vmm_support.S b/sys/amd64/vmm/vmm_support.S
new file mode 100644
index 0000000..2afc608
--- /dev/null
+++ b/sys/amd64/vmm/vmm_support.S
@@ -0,0 +1,42 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#define	LOCORE
+
+#include <machine/asmacros.h>
+
+#define	LA_EOI	0xB0
+
+	.text
+	SUPERALIGN_TEXT
+IDTVEC(justreturn)
+	pushq	%rax
+	movq	lapic, %rax
+	movl	$0, LA_EOI(%rax)
+	popq	%rax
+	iretq
diff --git a/sys/amd64/vmm/vmm_util.c b/sys/amd64/vmm/vmm_util.c
new file mode 100644
index 0000000..f245f92
--- /dev/null
+++ b/sys/amd64/vmm/vmm_util.c
@@ -0,0 +1,111 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/libkern.h>
+
+#include <machine/md_var.h>
+
+#include "vmm_util.h"
+
+boolean_t
+vmm_is_intel(void)
+{
+
+	if (strcmp(cpu_vendor, "GenuineIntel") == 0)
+		return (TRUE);
+	else
+		return (FALSE);
+}
+
+boolean_t
+vmm_is_amd(void)
+{
+	if (strcmp(cpu_vendor, "AuthenticAMD") == 0)
+		return (TRUE);
+	else
+		return (FALSE);
+}
+
+boolean_t
+vmm_supports_1G_pages(void)
+{
+	unsigned int regs[4];
+
+	/*
+	 * CPUID.80000001:EDX[bit 26] = 1 indicates support for 1GB pages
+	 *
+	 * Both Intel and AMD support this bit.
+	 */
+	if (cpu_exthigh >= 0x80000001) {
+		do_cpuid(0x80000001, regs);
+		if (regs[3] & (1 << 26))
+			return (TRUE);
+	}
+	return (FALSE);
+}
+
+#include <sys/proc.h>
+#include <machine/frame.h>
+#define	DUMP_REG(x)	printf(#x "\t\t0x%016lx\n", (long)(tf->tf_ ## x))
+#define	DUMP_SEG(x)	printf(#x "\t\t0x%04x\n", (unsigned)(tf->tf_ ## x))
+void
+dump_trapframe(struct trapframe *tf)
+{
+	DUMP_REG(rdi);
+	DUMP_REG(rsi);
+	DUMP_REG(rdx);
+	DUMP_REG(rcx);
+	DUMP_REG(r8);
+	DUMP_REG(r9);
+	DUMP_REG(rax);
+	DUMP_REG(rbx);
+	DUMP_REG(rbp);
+	DUMP_REG(r10);
+	DUMP_REG(r11);
+	DUMP_REG(r12);
+	DUMP_REG(r13);
+	DUMP_REG(r14);
+	DUMP_REG(r15);
+	DUMP_REG(trapno);
+	DUMP_REG(addr);
+	DUMP_REG(flags);
+	DUMP_REG(err);
+	DUMP_REG(rip);
+	DUMP_REG(rflags);
+	DUMP_REG(rsp);
+	DUMP_SEG(cs);
+	DUMP_SEG(ss);
+	DUMP_SEG(fs);
+	DUMP_SEG(gs);
+	DUMP_SEG(es);
+	DUMP_SEG(ds);
+}
diff --git a/sys/amd64/vmm/vmm_util.h b/sys/amd64/vmm/vmm_util.h
new file mode 100644
index 0000000..7f82332
--- /dev/null
+++ b/sys/amd64/vmm/vmm_util.h
@@ -0,0 +1,40 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VMM_UTIL_H_
+#define	_VMM_UTIL_H_
+
+struct trapframe;
+
+boolean_t	vmm_is_intel(void);
+boolean_t	vmm_is_amd(void);
+boolean_t	vmm_supports_1G_pages(void);
+
+void		dump_trapframe(struct trapframe *tf);
+
+#endif
diff --git a/sys/amd64/vmm/x86.c b/sys/amd64/vmm/x86.c
new file mode 100644
index 0000000..94abe09
--- /dev/null
+++ b/sys/amd64/vmm/x86.c
@@ -0,0 +1,202 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <sys/cpuset.h>
+
+#include <machine/cpufunc.h>
+#include <machine/md_var.h>
+#include <machine/specialreg.h>
+
+#include <machine/vmm.h>
+
+#include "x86.h"
+
+#define	CPUID_VM_HIGH		0x40000000
+
+static const char bhyve_id[12] = "BHyVE BHyVE ";
+
+int
+x86_emulate_cpuid(struct vm *vm, int vcpu_id,
+		  uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx)
+{
+	int error;
+	unsigned int 	func, regs[4];
+	enum x2apic_state x2apic_state;
+
+	func = *eax;
+
+	/*
+	 * Requests for invalid CPUID levels should map to the highest
+	 * available level instead.
+	 */
+	if (cpu_exthigh != 0 && *eax >= 0x80000000) {
+		if (*eax > cpu_exthigh)
+			*eax = cpu_exthigh;
+	} else if (*eax >= 0x40000000) {
+		if (*eax > CPUID_VM_HIGH)
+			*eax = CPUID_VM_HIGH;
+	} else if (*eax > cpu_high) {
+		*eax = cpu_high;
+	}
+
+	/*
+	 * In general the approach used for CPU topology is to
+	 * advertise a flat topology where all CPUs are packages with
+	 * no multi-core or SMT.
+	 */
+	switch (func) {
+		case CPUID_0000_0000:
+		case CPUID_0000_0002:
+		case CPUID_0000_0003:
+		case CPUID_0000_000A:
+			cpuid_count(*eax, *ecx, regs);
+			break;
+
+		case CPUID_8000_0000:
+		case CPUID_8000_0001:
+		case CPUID_8000_0002:
+		case CPUID_8000_0003:
+		case CPUID_8000_0004:
+		case CPUID_8000_0006:
+		case CPUID_8000_0007:
+		case CPUID_8000_0008:
+			cpuid_count(*eax, *ecx, regs);
+			break;
+
+		case CPUID_0000_0001:
+			do_cpuid(1, regs);
+
+			error = vm_get_x2apic_state(vm, vcpu_id, &x2apic_state);
+			if (error) {
+				panic("x86_emulate_cpuid: error %d "
+				      "fetching x2apic state", error);
+			}
+
+			/*
+			 * Override the APIC ID only in ebx
+			 */
+			regs[1] &= ~(CPUID_LOCAL_APIC_ID);
+			regs[1] |= (vcpu_id << CPUID_0000_0001_APICID_SHIFT);
+
+			/*
+			 * Don't expose VMX, SpeedStep or TME capability.
+			 * Advertise x2APIC capability and Hypervisor guest.
+			 */
+			regs[2] &= ~(CPUID2_VMX | CPUID2_EST | CPUID2_TM2);
+
+			regs[2] |= CPUID2_HV;
+
+			if (x2apic_state != X2APIC_DISABLED)
+				regs[2] |= CPUID2_X2APIC;
+
+			/*
+			 * Hide xsave/osxsave/avx until the FPU save/restore
+			 * issues are resolved
+			 */
+			regs[2] &= ~(CPUID2_XSAVE | CPUID2_OSXSAVE |
+				     CPUID2_AVX);
+
+			/*
+			 * Hide monitor/mwait until we know how to deal with
+			 * these instructions.
+			 */
+			regs[2] &= ~CPUID2_MON;
+
+			/*
+			 * Hide thermal monitoring
+			 */
+			regs[3] &= ~(CPUID_ACPI | CPUID_TM);
+			
+			/*
+			 * Machine check handling is done in the host.
+			 * Hide MTRR capability.
+			 */
+			regs[3] &= ~(CPUID_MCA | CPUID_MCE | CPUID_MTRR);
+
+			/*
+			 * Disable multi-core.
+			 */
+			regs[1] &= ~CPUID_HTT_CORES;
+			regs[3] &= ~CPUID_HTT;
+			break;
+
+		case CPUID_0000_0004:
+			do_cpuid(4, regs);
+
+			/*
+			 * Do not expose topology.
+			 */
+			regs[0] &= 0xffff8000;
+			regs[0] |= 0x04008000;
+			break;
+
+		case CPUID_0000_0006:
+		case CPUID_0000_0007:
+			/*
+			 * Handle the access, but report 0 for
+			 * all options
+			 */
+			regs[0] = 0;
+			regs[1] = 0;
+			regs[2] = 0;
+			regs[3] = 0;
+			break;
+
+		case CPUID_0000_000B:
+			/*
+			 * Processor topology enumeration
+			 */
+			regs[0] = 0;
+			regs[1] = 0;
+			regs[2] = *ecx & 0xff;
+			regs[3] = vcpu_id;
+			break;
+
+		case 0x40000000:
+			regs[0] = CPUID_VM_HIGH;
+			bcopy(bhyve_id, &regs[1], 4);
+			bcopy(bhyve_id, &regs[2], 4);
+			bcopy(bhyve_id, &regs[3], 4);
+			break;
+		default:
+			/* XXX: Leaf 5? */
+			return (0);
+	}
+
+	*eax = regs[0];
+	*ebx = regs[1];
+	*ecx = regs[2];
+	*edx = regs[3];
+	return (1);
+}
diff --git a/sys/amd64/vmm/x86.h b/sys/amd64/vmm/x86.h
new file mode 100644
index 0000000..368e967
--- /dev/null
+++ b/sys/amd64/vmm/x86.h
@@ -0,0 +1,64 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _X86_H_
+#define	_X86_H_
+
+#define CPUID_0000_0000 (0x0)
+#define CPUID_0000_0001	(0x1)
+#define CPUID_0000_0002 (0x2)
+#define CPUID_0000_0003 (0x3)
+#define CPUID_0000_0004 (0x4)
+#define CPUID_0000_0006 (0x6)
+#define CPUID_0000_0007 (0x7)
+#define	CPUID_0000_000A	(0xA)
+#define	CPUID_0000_000B	(0xB)
+#define CPUID_8000_0000	(0x80000000)
+#define CPUID_8000_0001	(0x80000001)
+#define CPUID_8000_0002	(0x80000002)
+#define CPUID_8000_0003	(0x80000003)
+#define CPUID_8000_0004	(0x80000004)
+#define CPUID_8000_0006	(0x80000006)
+#define CPUID_8000_0007	(0x80000007)
+#define CPUID_8000_0008	(0x80000008)
+
+/*
+ * CPUID instruction Fn0000_0001:
+ */
+#define CPUID_0000_0001_APICID_MASK			(0xff<<24)
+#define CPUID_0000_0001_APICID_SHIFT			24
+
+/*
+ * CPUID instruction Fn0000_0001 ECX
+ */
+#define CPUID_0000_0001_FEAT0_VMX	(1<<5)
+
+int x86_emulate_cpuid(struct vm *vm, int vcpu_id, uint32_t *eax, uint32_t *ebx,
+		      uint32_t *ecx, uint32_t *edx);
+
+#endif
diff --git a/sys/arm/allwinner/a10_clk.c b/sys/arm/allwinner/a10_clk.c
new file mode 100644
index 0000000..2d7699e
--- /dev/null
+++ b/sys/arm/allwinner/a10_clk.c
@@ -0,0 +1,171 @@
+/*-
+ * Copyright (c) 2013 Ganbold Tsagaankhuu <ganbold@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/* Simple clock driver for Allwinner A10 */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/malloc.h>
+#include <sys/rman.h>
+#include <sys/timeet.h>
+#include <sys/timetc.h>
+#include <sys/watchdog.h>
+#include <machine/bus.h>
+#include <machine/cpu.h>
+#include <machine/frame.h>
+#include <machine/intr.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <machine/bus.h>
+#include <machine/fdt.h>
+
+#include "a10_clk.h"
+
+struct a10_ccm_softc {
+	struct resource		*res;
+	bus_space_tag_t		bst;
+	bus_space_handle_t	bsh;
+};
+
+static struct a10_ccm_softc *a10_ccm_sc = NULL;
+
+#define ccm_read_4(reg)			\
+	bus_space_read_4(a10_ccm_sc->bst, a10_ccm_sc->bsh, reg)
+#define ccm_write_4(reg, val)		\
+	bus_space_write_4(a10_ccm_sc->bst, a10_ccm_sc->bsh, reg, val)
+
+static int
+a10_ccm_probe(device_t dev)
+{
+	if (ofw_bus_is_compatible(dev, "allwinner,sun4i-ccm")) {
+		device_set_desc(dev, "Allwinner Clock Control Module");
+		return(BUS_PROBE_DEFAULT);
+	}
+
+	return (ENXIO);
+}
+
+static int
+a10_ccm_attach(device_t dev)
+{
+	struct a10_ccm_softc *sc = device_get_softc(dev);
+	int rid = 0;
+
+	if (a10_ccm_sc)
+		return (ENXIO);
+
+	sc->res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
+	if (!sc->res) {
+		device_printf(dev, "could not allocate resource\n");
+		return (ENXIO);
+	}
+
+	sc->bst = rman_get_bustag(sc->res);
+	sc->bsh = rman_get_bushandle(sc->res);
+
+	a10_ccm_sc = sc;
+
+	return (0);
+}
+
+static device_method_t a10_ccm_methods[] = {
+	DEVMETHOD(device_probe,		a10_ccm_probe),
+	DEVMETHOD(device_attach,	a10_ccm_attach),
+	{ 0, 0 }
+};
+
+static driver_t a10_ccm_driver = {
+	"a10_ccm",
+	a10_ccm_methods,
+	sizeof(struct a10_ccm_softc),
+};
+
+static devclass_t a10_ccm_devclass;
+
+DRIVER_MODULE(a10_ccm, simplebus, a10_ccm_driver, a10_ccm_devclass, 0, 0);
+
+int
+a10_clk_usb_activate(void)
+{
+	struct a10_ccm_softc *sc = a10_ccm_sc;
+	uint32_t reg_value = 0;
+
+	if (sc == NULL)
+		return ENXIO;
+
+	/* Gating AHB clock for USB */
+	reg_value = ccm_read_4(CCM_AHB_GATING0);
+	reg_value |= CCM_AHB_GATING_USB0; /* AHB clock gate usb0 */
+	reg_value |= CCM_AHB_GATING_EHCI1; /* AHB clock gate ehci1 */
+	ccm_write_4(CCM_AHB_GATING0, reg_value);
+
+	/* Enable clock for USB */
+	reg_value = ccm_read_4(CCM_USB_CLK);
+	reg_value |= CCM_USB_PHY; /* USBPHY */
+	reg_value |= CCM_USB0_RESET; /* disable reset for USB0 */
+	reg_value |= CCM_USB1_RESET; /* disable reset for USB1 */
+	reg_value |= CCM_USB2_RESET; /* disable reset for USB2 */
+	ccm_write_4(CCM_USB_CLK, reg_value);
+
+	return (0);
+}
+
+int
+a10_clk_usb_deactivate(void)
+{
+	struct a10_ccm_softc *sc = a10_ccm_sc;
+	uint32_t reg_value = 0;
+
+	if (sc == NULL)
+		return ENXIO;
+
+	/* Disable clock for USB */
+	reg_value = ccm_read_4(CCM_USB_CLK);
+	reg_value &= ~CCM_USB_PHY; /* USBPHY */
+	reg_value &= ~CCM_USB0_RESET; /* reset for USB0 */
+	reg_value &= ~CCM_USB1_RESET; /* reset for USB1 */
+	reg_value &= ~CCM_USB2_RESET; /* reset for USB2 */
+	ccm_write_4(CCM_USB_CLK, reg_value);
+
+	/* Disable gating AHB clock for USB */
+	reg_value = ccm_read_4(CCM_AHB_GATING0);
+	reg_value &= ~CCM_AHB_GATING_USB0; /* disable AHB clock gate usb0 */
+	reg_value &= ~CCM_AHB_GATING_EHCI1; /* disable AHB clock gate ehci1 */
+	ccm_write_4(CCM_AHB_GATING0, reg_value);
+
+	return (0);
+}
+
diff --git a/sys/arm/allwinner/a10_clk.h b/sys/arm/allwinner/a10_clk.h
new file mode 100644
index 0000000..7316a22
--- /dev/null
+++ b/sys/arm/allwinner/a10_clk.h
@@ -0,0 +1,114 @@
+/*-
+ * Copyright (c) 2013 Ganbold Tsagaankhuu <ganbold@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _A10_CLK_H_
+#define _A10_CLK_H_
+
+#define CCMU_BASE		0xe1c20000
+
+#define CCM_PLL1_CFG		0x0000
+#define CCM_PLL1_TUN		0x0004
+#define CCM_PLL2_CFG		0x0008
+#define CCM_PLL2_TUN		0x000c
+#define CCM_PLL3_CFG		0x0010
+#define CCM_PLL3_TUN		0x0014
+#define CCM_PLL4_CFG		0x0018
+#define CCM_PLL4_TUN		0x001c
+#define CCM_PLL5_CFG		0x0020
+#define CCM_PLL5_TUN		0x0024
+#define CCM_PLL6_CFG		0x0028
+#define CCM_PLL6_TUN		0x002c
+#define CCM_PLL7_CFG		0x0030
+#define CCM_PLL7_TUN		0x0034
+#define CCM_PLL1_TUN2		0x0038
+#define CCM_PLL5_TUN2		0x003c
+#define CCM_PLL_LOCK_DBG	0x004c
+#define CCM_OSC24M_CFG		0x0050
+#define CCM_CPU_AHB_APB0_CFG	0x0054
+#define CCM_APB1_CLK_DIV	0x0058
+#define CCM_AXI_GATING		0x005c
+#define CCM_AHB_GATING0		0x0060
+#define CCM_AHB_GATING1		0x0064
+#define CCM_APB0_GATING		0x0068
+#define CCM_APB1_GATING		0x006c
+#define CCM_NAND_SCLK_CFG	0x0080
+#define CCM_MS_SCLK_CFG		0x0084
+#define CCM_MMC0_SCLK_CFG	0x0088
+#define CCM_MMC1_SCLK_CFG	0x008c
+#define CCM_MMC2_SCLK_CFG	0x0090
+#define CCM_MMC3_SCLK_CFG	0x0094
+#define CCM_TS_CLK		0x0098
+#define CCM_SS_CLK		0x009c
+#define CCM_SPI0_CLK		0x00a0
+#define CCM_SPI1_CLK		0x00a4
+#define CCM_SPI2_CLK		0x00a8
+#define CCM_PATA_CLK		0x00ac
+#define CCM_IR0_CLK		0x00b0
+#define CCM_IR1_CLK		0x00b4
+#define CCM_IIS_CLK		0x00b8
+#define CCM_AC97_CLK		0x00bc
+#define CCM_SPDIF_CLK		0x00c0
+#define CCM_KEYPAD_CLK		0x00c4
+#define CCM_SATA_CLK		0x00c8
+#define CCM_USB_CLK		0x00cc
+#define CCM_GPS_CLK		0x00d0
+#define CCM_SPI3_CLK		0x00d4
+#define CCM_DRAM_CLK		0x0100
+#define CCM_BE0_SCLK		0x0104
+#define CCM_BE1_SCLK		0x0108
+#define CCM_FE0_CLK		0x010c
+#define CCM_FE1_CLK		0x0110
+#define CCM_MP_CLK		0x0114
+#define CCM_LCD0_CH0_CLK	0x0118
+#define CCM_LCD1_CH0_CLK	0x011c
+#define CCM_CSI_ISP_CLK		0x0120
+#define CCM_TVD_CLK		0x0128
+#define CCM_LCD0_CH1_CLK	0x012c
+#define CCM_LCD1_CH1_CLK	0x0130
+#define CCM_CS0_CLK		0x0134
+#define CCM_CS1_CLK		0x0138
+#define CCM_VE_CLK		0x013c
+#define CCM_AUDIO_CODEC_CLK	0x0140
+#define CCM_AVS_CLK		0x0144
+#define CCM_ACE_CLK		0x0148
+#define CCM_LVDS_CLK		0x014c
+#define CCM_HDMI_CLK		0x0150
+#define CCM_MALI400_CLK		0x0154
+
+#define CCM_AHB_GATING_USB0	(1 << 0)
+#define CCM_AHB_GATING_EHCI1	(1 << 3)
+
+#define CCM_USB_PHY		(1 << 8)
+#define CCM_USB0_RESET		(1 << 0)
+#define CCM_USB1_RESET		(1 << 1)
+#define CCM_USB2_RESET		(1 << 2)
+
+int a10_clk_usb_activate(void);
+int a10_clk_usb_deactivate(void);
+
+#endif /* _A10_CLK_H_ */
diff --git a/sys/arm/allwinner/a10_ehci.c b/sys/arm/allwinner/a10_ehci.c
new file mode 100644
index 0000000..d3a90d0
--- /dev/null
+++ b/sys/arm/allwinner/a10_ehci.c
@@ -0,0 +1,279 @@
+/*-
+ * Copyright (c) 2012 Ganbold Tsagaankhuu <ganbold@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Allwinner A10 attachment driver for the USB Enhanced Host Controller.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_bus.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/rman.h>
+#include <sys/condvar.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+
+#include <machine/bus.h>
+#include <dev/ofw/ofw_bus.h> 
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <dev/usb/usb.h> 
+#include <dev/usb/usbdi.h>
+
+#include <dev/usb/usb_core.h>
+#include <dev/usb/usb_busdma.h>
+#include <dev/usb/usb_process.h>
+#include <dev/usb/usb_util.h>
+
+#include <dev/usb/usb_controller.h>
+#include <dev/usb/usb_bus.h>
+#include <dev/usb/controller/ehci.h>
+#include <dev/usb/controller/ehcireg.h>
+
+#include "a10_clk.h"
+
+#define EHCI_HC_DEVSTR			"Allwinner Integrated USB 2.0 controller"
+
+#define SW_USB_PMU_IRQ_ENABLE		0x800
+
+#define SW_SDRAM_REG_HPCR_USB1		(0x250 + ((1 << 2) * 4))
+#define SW_SDRAM_REG_HPCR_USB2		(0x250 + ((1 << 2) * 5))
+#define SW_SDRAM_BP_HPCR_ACCESS		(1 << 0)
+
+#define SW_ULPI_BYPASS			(1 << 0)
+#define SW_AHB_INCRX_ALIGN		(1 << 8)
+#define	SW_AHB_INCR4			(1 << 9)
+#define SW_AHB_INCR8			(1 << 10)
+
+#define A10_READ_4(sc, reg)		\
+	bus_space_read_4((sc)->sc_io_tag, (sc)->sc_io_hdl, reg)
+
+#define A10_WRITE_4(sc, reg, data)	\
+	bus_space_write_4((sc)->sc_io_tag, (sc)->sc_io_hdl, reg, data)
+
+static device_attach_t a10_ehci_attach;
+static device_detach_t a10_ehci_detach;
+
+bs_r_1_proto(reversed);
+bs_w_1_proto(reversed);
+
+static int
+a10_ehci_probe(device_t self)
+{
+	if (!ofw_bus_is_compatible(self, "allwinner,usb-ehci")) 
+		return (ENXIO);
+
+	device_set_desc(self, EHCI_HC_DEVSTR);
+
+	return (BUS_PROBE_DEFAULT);
+}
+
+static int
+a10_ehci_attach(device_t self)
+{
+	ehci_softc_t *sc = device_get_softc(self);
+	bus_space_handle_t bsh;
+	int err;
+	int rid;
+	uint32_t reg_value = 0;
+
+	/* initialise some bus fields */
+	sc->sc_bus.parent = self;
+	sc->sc_bus.devices = sc->sc_devices;
+	sc->sc_bus.devices_max = EHCI_MAX_DEVICES;
+
+	/* get all DMA memory */
+	if (usb_bus_mem_alloc_all(&sc->sc_bus,
+	    USB_GET_DMA_TAG(self), &ehci_iterate_hw_softc)) {
+		return (ENOMEM);
+	}
+
+	sc->sc_bus.usbrev = USB_REV_2_0;
+
+	rid = 0;
+	sc->sc_io_res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
+	if (!sc->sc_io_res) {
+		device_printf(self, "Could not map memory\n");
+		goto error;
+	}
+
+	sc->sc_io_tag = rman_get_bustag(sc->sc_io_res);
+	sc->sc_io_hdl = rman_get_bushandle(sc->sc_io_res);
+	bsh = rman_get_bushandle(sc->sc_io_res);
+
+	sc->sc_io_size = rman_get_size(sc->sc_io_res);
+
+	if (bus_space_subregion(sc->sc_io_tag, bsh, 0x00,
+	    sc->sc_io_size, &sc->sc_io_hdl) != 0)
+		panic("%s: unable to subregion USB host registers",
+		    device_get_name(self));
+
+	rid = 0;
+	sc->sc_irq_res = bus_alloc_resource_any(self, SYS_RES_IRQ, &rid,
+	    RF_SHAREABLE | RF_ACTIVE);
+	if (sc->sc_irq_res == NULL) {
+		device_printf(self, "Could not allocate irq\n");
+		goto error;
+	}
+	sc->sc_bus.bdev = device_add_child(self, "usbus", -1);
+	if (!sc->sc_bus.bdev) {
+		device_printf(self, "Could not add USB device\n");
+		goto error;
+	}
+	device_set_ivars(sc->sc_bus.bdev, &sc->sc_bus);
+	device_set_desc(sc->sc_bus.bdev, EHCI_HC_DEVSTR);
+
+	sprintf(sc->sc_vendor, "Allwinner");
+
+	err = bus_setup_intr(self, sc->sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
+	    NULL, (driver_intr_t *)ehci_interrupt, sc, &sc->sc_intr_hdl);
+	if (err) {
+		device_printf(self, "Could not setup irq, %d\n", err);
+		sc->sc_intr_hdl = NULL;
+		goto error;
+	}
+
+	sc->sc_flags |= EHCI_SCFLG_DONTRESET;
+
+	/* Enable clock for USB */
+	a10_clk_usb_activate();
+
+	/* Enable passby */
+	reg_value = A10_READ_4(sc, SW_USB_PMU_IRQ_ENABLE);
+	reg_value |= SW_AHB_INCR8; /* AHB INCR8 enable */
+	reg_value |= SW_AHB_INCR4; /* AHB burst type INCR4 enable */
+	reg_value |= SW_AHB_INCRX_ALIGN; /* AHB INCRX align enable */
+	reg_value |= SW_ULPI_BYPASS; /* ULPI bypass enable */
+	A10_WRITE_4(sc, SW_USB_PMU_IRQ_ENABLE, reg_value);
+
+	/* Configure port */
+	reg_value = A10_READ_4(sc, SW_SDRAM_REG_HPCR_USB2);
+	reg_value |= SW_SDRAM_BP_HPCR_ACCESS;
+	A10_WRITE_4(sc, SW_SDRAM_REG_HPCR_USB2, reg_value);
+
+	err = ehci_init(sc);
+	if (!err) {
+		err = device_probe_and_attach(sc->sc_bus.bdev);
+	}
+	if (err) {
+		device_printf(self, "USB init failed err=%d\n", err);
+		goto error;
+	}
+	return (0);
+
+error:
+	a10_ehci_detach(self);
+	return (ENXIO);
+}
+
+static int
+a10_ehci_detach(device_t self)
+{
+	ehci_softc_t *sc = device_get_softc(self);
+	device_t bdev;
+	int err;
+	uint32_t reg_value = 0;
+
+	if (sc->sc_bus.bdev) {
+		bdev = sc->sc_bus.bdev;
+		device_detach(bdev);
+		device_delete_child(self, bdev);
+	}
+	/* during module unload there are lots of children leftover */
+	device_delete_children(self);
+
+	if (sc->sc_irq_res && sc->sc_intr_hdl) {
+		/*
+		 * only call ehci_detach() after ehci_init()
+		 */
+		ehci_detach(sc);
+
+		err = bus_teardown_intr(self, sc->sc_irq_res, sc->sc_intr_hdl);
+
+		if (err)
+			/* XXX or should we panic? */
+			device_printf(self, "Could not tear down irq, %d\n",
+			    err);
+		sc->sc_intr_hdl = NULL;
+	}
+
+	if (sc->sc_irq_res) {
+		bus_release_resource(self, SYS_RES_IRQ, 0, sc->sc_irq_res);
+		sc->sc_irq_res = NULL;
+	}
+	if (sc->sc_io_res) {
+		bus_release_resource(self, SYS_RES_MEMORY, 0,
+		    sc->sc_io_res);
+		sc->sc_io_res = NULL;
+	}
+	usb_bus_mem_free_all(&sc->sc_bus, &ehci_iterate_hw_softc);
+
+	/* Disable configure port */
+	reg_value = A10_READ_4(sc, SW_SDRAM_REG_HPCR_USB2);
+	reg_value &= ~SW_SDRAM_BP_HPCR_ACCESS;
+	A10_WRITE_4(sc, SW_SDRAM_REG_HPCR_USB2, reg_value);
+
+	/* Disable passby */
+	reg_value = A10_READ_4(sc, SW_USB_PMU_IRQ_ENABLE);
+	reg_value &= ~SW_AHB_INCR8; /* AHB INCR8 disable */
+	reg_value &= ~SW_AHB_INCR4; /* AHB burst type INCR4 disable */
+	reg_value &= ~SW_AHB_INCRX_ALIGN; /* AHB INCRX align disable */
+	reg_value &= ~SW_ULPI_BYPASS; /* ULPI bypass disable */
+	A10_WRITE_4(sc, SW_USB_PMU_IRQ_ENABLE, reg_value);
+
+	/* Disable clock for USB */
+	a10_clk_usb_deactivate();
+
+	return (0);
+}
+
+static device_method_t ehci_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe, a10_ehci_probe),
+	DEVMETHOD(device_attach, a10_ehci_attach),
+	DEVMETHOD(device_detach, a10_ehci_detach),
+	DEVMETHOD(device_suspend, bus_generic_suspend),
+	DEVMETHOD(device_resume, bus_generic_resume),
+	DEVMETHOD(device_shutdown, bus_generic_shutdown),
+
+	DEVMETHOD_END
+};
+
+static driver_t ehci_driver = {
+	.name = "ehci",
+	.methods = ehci_methods,
+	.size = sizeof(ehci_softc_t),
+};
+
+static devclass_t ehci_devclass;
+
+DRIVER_MODULE(ehci, simplebus, ehci_driver, ehci_devclass, 0, 0);
+MODULE_DEPEND(ehci, usb, 1, 1, 1);
diff --git a/sys/arm/allwinner/a10_machdep.c b/sys/arm/allwinner/a10_machdep.c
new file mode 100644
index 0000000..1ad9ea1
--- /dev/null
+++ b/sys/arm/allwinner/a10_machdep.c
@@ -0,0 +1,122 @@
+/*-
+ * Copyright (c) 2012 Ganbold Tsagaankhuu <ganbold@gmail.com>
+ * All rights reserved.
+ *
+ * This code is derived from software written for Brini by Mark Brinicombe
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * from: FreeBSD: //depot/projects/arm/src/sys/arm/ti/ti_machdep.c
+ */
+
+#include "opt_ddb.h"
+#include "opt_platform.h"
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#define _ARM32_BUS_DMA_PRIVATE
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/bus.h>
+#include <machine/frame.h> /* For trapframe_t, used in <machine/machdep.h> */
+#include <machine/machdep.h>
+#include <machine/pmap.h>
+
+#include <dev/fdt/fdt_common.h>
+
+/* Start of address space used for bootstrap map */
+#define DEVMAP_BOOTSTRAP_MAP_START      0xE0000000
+
+void (*a10_cpu_reset)(void);
+
+vm_offset_t
+initarm_lastaddr(void)
+{
+
+	a10_cpu_reset = NULL;
+	return (DEVMAP_BOOTSTRAP_MAP_START - ARM_NOCACHE_KVA_SIZE);
+}
+
+void
+initarm_gpio_init(void)
+{
+}
+
+void
+initarm_late_init(void)
+{
+}
+
+#define FDT_DEVMAP_MAX		(1 + 2 + 1 + 1)
+static struct pmap_devmap fdt_devmap[FDT_DEVMAP_MAX] = {
+	{ 0, 0, 0, 0, 0, }
+};
+
+/*
+ * Construct pmap_devmap[] with DT-derived config data.
+ */
+int
+platform_devmap_init(void)
+{
+	int i = 0;
+
+	fdt_devmap[i].pd_va =   0xE1C00000;
+	fdt_devmap[i].pd_pa =   0x01C00000;
+	fdt_devmap[i].pd_size = 0x00400000;	/* 4 MB */
+	fdt_devmap[i].pd_prot = VM_PROT_READ | VM_PROT_WRITE;
+	fdt_devmap[i].pd_cache = PTE_DEVICE;
+
+	i++;
+
+	pmap_devmap_bootstrap_table = &fdt_devmap[0];
+
+	return (0);
+}
+
+struct arm32_dma_range *
+bus_dma_get_range(void)
+{
+	return (NULL);
+}
+
+int
+bus_dma_get_range_nb(void)
+{
+	return (0);
+}
+
+void
+cpu_reset()
+{
+	if (a10_cpu_reset)
+		(*a10_cpu_reset)();
+	else
+		printf("no cpu_reset implementation\n");
+	printf("Reset failed!\n");
+	while (1);
+}
diff --git a/sys/arm/allwinner/aintc.c b/sys/arm/allwinner/aintc.c
new file mode 100644
index 0000000..bfa35f5
--- /dev/null
+++ b/sys/arm/allwinner/aintc.c
@@ -0,0 +1,211 @@
+/*-
+ * Copyright (c) 2012 Ganbold Tsagaankhuu <ganbold@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/module.h>
+#include <sys/rman.h>
+#include <machine/bus.h>
+#include <machine/intr.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+/**
+ * Interrupt controller registers
+ *
+ */
+#define SW_INT_VECTOR_REG		0x00
+#define SW_INT_BASE_ADR_REG		0x04
+#define SW_INT_PROTECTION_REG		0x08
+#define SW_INT_NMI_CTRL_REG		0x0c
+
+#define SW_INT_IRQ_PENDING_REG0		0x10
+#define SW_INT_IRQ_PENDING_REG1		0x14
+#define SW_INT_IRQ_PENDING_REG2		0x18
+
+#define SW_INT_FIQ_PENDING_REG0		0x20
+#define SW_INT_FIQ_PENDING_REG1		0x24
+#define SW_INT_FIQ_PENDING_REG2		0x28
+
+#define SW_INT_SELECT_REG0		0x30
+#define SW_INT_SELECT_REG1		0x34
+#define SW_INT_SELECT_REG2		0x38
+
+#define SW_INT_ENABLE_REG0		0x40
+#define SW_INT_ENABLE_REG1		0x44
+#define SW_INT_ENABLE_REG2		0x48
+
+#define SW_INT_MASK_REG0		0x50
+#define SW_INT_MASK_REG1		0x54
+#define SW_INT_MASK_REG2		0x58
+
+#define SW_INT_IRQNO_ENMI		0
+
+#define SW_INT_IRQ_PENDING_REG(_b)	(0x10 + ((_b) * 4))
+#define SW_INT_FIQ_PENDING_REG(_b)	(0x20 + ((_b) * 4))
+#define SW_INT_SELECT_REG(_b)		(0x30 + ((_b) * 4))
+#define SW_INT_ENABLE_REG(_b)		(0x40 + ((_b) * 4))
+#define SW_INT_MASK_REG(_b)		(0x50 + ((_b) * 4))
+
+struct a10_aintc_softc {
+	device_t		sc_dev;
+	struct resource *	aintc_res;
+	bus_space_tag_t		aintc_bst;
+	bus_space_handle_t	aintc_bsh;
+	uint8_t			ver;
+};
+
+static struct a10_aintc_softc *a10_aintc_sc = NULL;
+
+#define	aintc_read_4(reg)	\
+	bus_space_read_4(a10_aintc_sc->aintc_bst, a10_aintc_sc->aintc_bsh, reg)
+#define	aintc_write_4(reg, val)		\
+	bus_space_write_4(a10_aintc_sc->aintc_bst, a10_aintc_sc->aintc_bsh, reg, val)
+
+static int
+a10_aintc_probe(device_t dev)
+{
+	if (!ofw_bus_is_compatible(dev, "allwinner,sun4i-ic"))
+		return (ENXIO);
+	device_set_desc(dev, "A10 AINTC Interrupt Controller");
+	return (BUS_PROBE_DEFAULT);
+}
+
+static int
+a10_aintc_attach(device_t dev)
+{
+	struct a10_aintc_softc *sc = device_get_softc(dev);
+	int rid = 0;
+	int i;
+	
+	sc->sc_dev = dev;
+
+	if (a10_aintc_sc)
+		return (ENXIO);
+
+	sc->aintc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
+	if (!sc->aintc_res) {
+		device_printf(dev, "could not allocate resource\n");
+		return (ENXIO);
+	}
+
+	sc->aintc_bst = rman_get_bustag(sc->aintc_res);
+	sc->aintc_bsh = rman_get_bushandle(sc->aintc_res);
+
+	a10_aintc_sc = sc;
+
+	/* Disable & clear all interrupts */
+	for (i = 0; i < 3; i++) {
+		aintc_write_4(SW_INT_ENABLE_REG(i), 0);
+		aintc_write_4(SW_INT_MASK_REG(i), 0xffffffff);
+	}
+	/* enable protection mode*/
+	aintc_write_4(SW_INT_PROTECTION_REG, 0x01);
+
+	/* config the external interrupt source type*/
+	aintc_write_4(SW_INT_NMI_CTRL_REG, 0x00);
+
+	return (0);
+}
+
+static device_method_t a10_aintc_methods[] = {
+	DEVMETHOD(device_probe,		a10_aintc_probe),
+	DEVMETHOD(device_attach,	a10_aintc_attach),
+	{ 0, 0 }
+};
+
+static driver_t a10_aintc_driver = {
+	"aintc",
+	a10_aintc_methods,
+	sizeof(struct a10_aintc_softc),
+};
+
+static devclass_t a10_aintc_devclass;
+
+DRIVER_MODULE(aintc, simplebus, a10_aintc_driver, a10_aintc_devclass, 0, 0);
+
+int
+arm_get_next_irq(int last_irq)
+{
+	uint32_t value;
+	int i, b;
+
+	for (i = 0; i < 3; i++) {
+		value = aintc_read_4(SW_INT_IRQ_PENDING_REG(i));
+		for (b = 0; b < 32; b++)
+			if (value & (1 << b)) {
+				return (i * 32 + b);
+			}
+	}
+
+	return (-1);
+}
+
+void
+arm_mask_irq(uintptr_t nb)
+{
+	uint32_t bit, block, value;
+	
+	bit = (nb % 32);
+	block = (nb / 32);
+
+	value = aintc_read_4(SW_INT_ENABLE_REG(block));
+	value &= ~(1 << bit);
+	aintc_write_4(SW_INT_ENABLE_REG(block), value);
+
+	value = aintc_read_4(SW_INT_MASK_REG(block));
+	value |= (1 << bit);
+	aintc_write_4(SW_INT_MASK_REG(block), value);
+}
+
+void
+arm_unmask_irq(uintptr_t nb)
+{
+	uint32_t bit, block, value;
+
+	bit = (nb % 32);
+	block = (nb / 32);
+
+	value = aintc_read_4(SW_INT_ENABLE_REG(block));
+	value |= (1 << bit);
+	aintc_write_4(SW_INT_ENABLE_REG(block), value);
+
+	value = aintc_read_4(SW_INT_MASK_REG(block));
+	value &= ~(1 << bit);
+	aintc_write_4(SW_INT_MASK_REG(block), value);
+
+	if(nb == SW_INT_IRQNO_ENMI) /* must clear pending bit when enabled */
+		aintc_write_4(SW_INT_IRQ_PENDING_REG(0), (1 << SW_INT_IRQNO_ENMI));
+}
diff --git a/sys/arm/allwinner/bus_space.c b/sys/arm/allwinner/bus_space.c
new file mode 100644
index 0000000..4cce820
--- /dev/null
+++ b/sys/arm/allwinner/bus_space.c
@@ -0,0 +1,113 @@
+/*-
+ * Copyright (C) 2012 FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of MARVELL nor the names of contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+
+#include <machine/bus.h>
+
+/* Prototypes for all the bus_space structure functions */
+bs_protos(generic);
+bs_protos(generic_armv4);
+
+struct bus_space _base_tag = {
+	/* cookie */
+	.bs_cookie	= (void *) 0,
+	
+	/* mapping/unmapping */
+	.bs_map		= generic_bs_map,
+	.bs_unmap	= generic_bs_unmap,
+	.bs_subregion	= generic_bs_subregion,
+	
+	/* allocation/deallocation */
+	.bs_alloc	= generic_bs_alloc,
+	.bs_free	= generic_bs_free,
+	
+	/* barrier */
+	.bs_barrier	= generic_bs_barrier,
+	
+	/* read (single) */
+	.bs_r_1		= generic_bs_r_1,
+	.bs_r_2		= generic_armv4_bs_r_2,
+	.bs_r_4		= generic_bs_r_4,
+	.bs_r_8		= NULL,
+	
+	/* read multiple */
+	.bs_rm_1	= generic_bs_rm_1,
+	.bs_rm_2	= generic_armv4_bs_rm_2,
+	.bs_rm_4	= generic_bs_rm_4,
+	.bs_rm_8	= NULL,
+	
+	/* read region */
+	.bs_rr_1	= generic_bs_rr_1,
+	.bs_rr_2	= generic_armv4_bs_rr_2,
+	.bs_rr_4	= generic_bs_rr_4,
+	.bs_rr_8	= NULL,
+	
+	/* write (single) */
+	.bs_w_1		= generic_bs_w_1,
+	.bs_w_2		= generic_armv4_bs_w_2,
+	.bs_w_4		= generic_bs_w_4,
+	.bs_w_8		= NULL,
+	
+	/* write multiple */
+	.bs_wm_1	= generic_bs_wm_1,
+	.bs_wm_2	= generic_armv4_bs_wm_2,
+	.bs_wm_4	= generic_bs_wm_4,
+	.bs_wm_8	= NULL,
+	
+	/* write region */
+	.bs_wr_1	= generic_bs_wr_1,
+	.bs_wr_2	= generic_armv4_bs_wr_2,
+	.bs_wr_4	= generic_bs_wr_4,
+	.bs_wr_8	= NULL,
+	
+	/* set multiple */
+	/* XXX not implemented */
+	
+	/* set region */
+	.bs_sr_1	= NULL,
+	.bs_sr_2	= generic_armv4_bs_sr_2,
+	.bs_sr_4	= generic_bs_sr_4,
+	.bs_sr_8	= NULL,
+	
+	/* copy */
+	.bs_c_1		= NULL,
+	.bs_c_2		= generic_armv4_bs_c_2,
+	.bs_c_4		= NULL,
+	.bs_c_8		= NULL,
+};
+
+bus_space_tag_t fdtbus_bs_tag = &_base_tag;
diff --git a/sys/arm/allwinner/common.c b/sys/arm/allwinner/common.c
new file mode 100644
index 0000000..eb49479
--- /dev/null
+++ b/sys/arm/allwinner/common.c
@@ -0,0 +1,63 @@
+/*-
+ * Copyright (c) 2012 Ganbold Tsagaankhuu <ganbold@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+
+#include <machine/bus.h>
+#include <machine/fdt.h>
+#include <machine/vmparam.h>
+
+struct fdt_fixup_entry fdt_fixup_table[] = {
+	{ NULL, NULL }
+};
+
+static int
+fdt_aintc_decode_ic(phandle_t node, pcell_t *intr, int *interrupt, int *trig,
+    int *pol)
+{
+	if (!fdt_is_compatible(node, "allwinner,sun4i-ic"))
+		return (ENXIO);
+
+	*interrupt = fdt32_to_cpu(intr[0]);
+	*trig = INTR_TRIGGER_CONFORM;
+	*pol = INTR_POLARITY_CONFORM;
+
+	return (0);
+}
+
+fdt_pic_decode_t fdt_pic_table[] = {
+	&fdt_aintc_decode_ic,
+	NULL
+};
diff --git a/sys/arm/allwinner/console.c b/sys/arm/allwinner/console.c
new file mode 100644
index 0000000..52d7376
--- /dev/null
+++ b/sys/arm/allwinner/console.c
@@ -0,0 +1,142 @@
+/*-
+ * Copyright (c) 2012 Ganbold Tsagaankhuu <ganbold@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/* Simple UART console driver for Allwinner A10 */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/cons.h>
+#include <sys/consio.h>
+#include <sys/kernel.h>
+
+#ifndef	A10_UART_BASE
+#define	A10_UART_BASE	0xe1c28000 	/* UART0 */
+#endif
+
+#define	REG_SHIFT	2
+
+#define	UART_DLL	0	/* Out: Divisor Latch Low */
+#define	UART_DLM	1	/* Out: Divisor Latch High */
+#define	UART_FCR	2	/* Out: FIFO Control Register */
+#define	UART_LCR	3	/* Out: Line Control Register */
+#define	UART_MCR	4	/* Out: Modem Control Register */
+#define	UART_LSR	5	/* In:  Line Status Register */
+#define	UART_LSR_THRE	0x20	/* Transmit-hold-register empty */
+#define	UART_LSR_DR	0x01	/* Receiver data ready */
+#define	UART_MSR	6	/* In:  Modem Status Register */
+#define	UART_SCR	7	/* I/O: Scratch Register */
+
+static uint32_t
+uart_getreg(uint32_t *bas)
+{
+	return *((volatile uint32_t *)(bas)) & 0xff;
+}
+
+static void
+uart_setreg(uint32_t *bas, uint32_t val)
+{
+	*((volatile uint32_t *)(bas)) = val;
+}
+
+static int
+ub_getc(void)
+{
+	while ((uart_getreg((uint32_t *)(A10_UART_BASE + 
+	    (UART_LSR << REG_SHIFT))) & UART_LSR_DR) == 0);
+		__asm __volatile("nop");
+
+	return (uart_getreg((uint32_t *)A10_UART_BASE) & 0xff);
+}
+
+static void
+ub_putc(unsigned char c)
+{
+	if (c == '\n')
+		ub_putc('\r');
+
+	while ((uart_getreg((uint32_t *)(A10_UART_BASE + 
+	    (UART_LSR << REG_SHIFT))) & UART_LSR_THRE) == 0)
+		__asm __volatile("nop");
+
+	uart_setreg((uint32_t *)A10_UART_BASE, c);
+}
+
+static cn_probe_t	uart_cnprobe;
+static cn_init_t	uart_cninit;
+static cn_term_t	uart_cnterm;
+static cn_getc_t	uart_cngetc;
+static cn_putc_t	uart_cnputc;
+static cn_grab_t	uart_cngrab;
+static cn_ungrab_t	uart_cnungrab;
+
+static void
+uart_cngrab(struct consdev *cp)
+{
+}
+
+static void
+uart_cnungrab(struct consdev *cp)
+{
+}
+
+
+static void
+uart_cnprobe(struct consdev *cp)
+{
+	sprintf(cp->cn_name, "uart");
+	cp->cn_pri = CN_NORMAL;
+}
+
+static void
+uart_cninit(struct consdev *cp)
+{
+	uart_setreg((uint32_t *)(A10_UART_BASE + 
+	    (UART_FCR << REG_SHIFT)), 0x06);
+}
+
+void
+uart_cnputc(struct consdev *cp, int c)
+{
+	ub_putc(c);
+}
+
+int
+uart_cngetc(struct consdev * cp)
+{
+	return ub_getc();
+}
+
+static void
+uart_cnterm(struct consdev * cp)
+{
+}
+
+CONSOLE_DRIVER(uart);
+
diff --git a/sys/arm/allwinner/files.a10 b/sys/arm/allwinner/files.a10
new file mode 100644
index 0000000..9d056fc
--- /dev/null
+++ b/sys/arm/allwinner/files.a10
@@ -0,0 +1,19 @@
+# $FreeBSD$
+kern/kern_clocksource.c			standard
+
+arm/arm/bus_space_asm_generic.S		standard
+arm/arm/bus_space_generic.c		standard
+arm/arm/cpufunc_asm_armv5.S		standard
+arm/arm/cpufunc_asm_arm10.S		standard
+arm/arm/cpufunc_asm_arm11.S		standard
+arm/arm/cpufunc_asm_armv7.S		standard
+arm/arm/irq_dispatch.S			standard
+
+arm/allwinner/a10_clk.c			standard
+arm/allwinner/a10_ehci.c		optional	ehci
+arm/allwinner/timer.c			standard
+arm/allwinner/aintc.c			standard
+arm/allwinner/bus_space.c		standard
+arm/allwinner/common.c			standard
+arm/allwinner/console.c			standard
+arm/allwinner/a10_machdep.c		standard
diff --git a/sys/arm/allwinner/std.a10 b/sys/arm/allwinner/std.a10
new file mode 100644
index 0000000..f698b34
--- /dev/null
+++ b/sys/arm/allwinner/std.a10
@@ -0,0 +1,21 @@
+# Allwinner A10 common options
+#$FreeBSD$
+
+cpu		CPU_CORTEXA
+machine 	arm armv6
+makeoption	ARM_LITTLE_ENDIAN
+
+# Physical memory starts at 0x40200000.  We assume images are loaded at
+# 0x40200000, e.g. from u-boot with 'fatload mmc 0 0x40200000 kernel'
+#
+#
+options		PHYSADDR=0x40000000
+
+makeoptions	KERNPHYSADDR=0x40200000
+options		KERNPHYSADDR=0x40200000
+makeoptions	KERNVIRTADDR=0xc0200000
+options		KERNVIRTADDR=0xc0200000
+
+options		STARTUP_PAGETABLE_ADDR=0x48000000
+
+files		"../allwinner/files.a10"
diff --git a/sys/arm/allwinner/timer.c b/sys/arm/allwinner/timer.c
new file mode 100644
index 0000000..4da3517
--- /dev/null
+++ b/sys/arm/allwinner/timer.c
@@ -0,0 +1,379 @@
+/*-
+ * Copyright (c) 2012 Ganbold Tsagaankhuu <ganbold@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/malloc.h>
+#include <sys/rman.h>
+#include <sys/timeet.h>
+#include <sys/timetc.h>
+#include <sys/watchdog.h>
+#include <machine/bus.h>
+#include <machine/cpu.h>
+#include <machine/frame.h>
+#include <machine/intr.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <machine/bus.h>
+#include <machine/fdt.h>
+
+#include <sys/kdb.h>
+
+/**
+ * Timer registers addr
+ *
+ */
+#define SW_TIMER_IRQ_EN_REG 	0x00
+#define SW_TIMER_IRQ_STA_REG 	0x04
+#define SW_TIMER0_CTRL_REG 	0x10
+#define SW_TIMER0_INT_VALUE_REG	0x14
+#define SW_TIMER0_CUR_VALUE_REG	0x18
+
+#define SW_COUNTER64LO_REG	0xa4
+#define SW_COUNTER64HI_REG	0xa8
+#define CNT64_CTRL_REG		0xa0
+
+#define CNT64_RL_EN		0x02 /* read latch enable */
+
+#define TIMER_ENABLE		(1<<0)
+#define TIMER_AUTORELOAD	(1<<1)
+#define TIMER_OSC24M		(1<<2) /* oscillator = 24mhz */
+#define TIMER_PRESCALAR		(4<<4) /* prescalar = 16 */
+
+#define SYS_TIMER_CLKSRC	24000000 /* clock source */
+
+struct a10_timer_softc {
+	device_t 	sc_dev;
+	struct resource *res[2];
+	bus_space_tag_t sc_bst;
+	bus_space_handle_t sc_bsh;
+	void 		*sc_ih;		/* interrupt handler */
+	uint32_t 	sc_period;
+	uint32_t 	timer0_freq;
+	struct eventtimer et;
+};
+
+int a10_timer_get_timerfreq(struct a10_timer_softc *);
+
+#define timer_read_4(sc, reg)	\
+	bus_space_read_4(sc->sc_bst, sc->sc_bsh, reg)
+#define timer_write_4(sc, reg, val)	\
+	bus_space_write_4(sc->sc_bst, sc->sc_bsh, reg, val)
+
+static u_int	a10_timer_get_timecount(struct timecounter *);
+static int	a10_timer_timer_start(struct eventtimer *,
+    struct bintime *, struct bintime *);
+static int	a10_timer_timer_stop(struct eventtimer *);
+
+static uint64_t timer_read_counter64(void);
+
+static int a10_timer_initialized = 0;
+static int a10_timer_hardclock(void *);
+static int a10_timer_probe(device_t);
+static int a10_timer_attach(device_t);
+
+static struct timecounter a10_timer_timecounter = {
+	.tc_name           = "a10_timer timer0",
+	.tc_get_timecount  = a10_timer_get_timecount,
+	.tc_counter_mask   = ~0u,
+	.tc_frequency      = 0,
+	.tc_quality        = 1000,
+};
+
+struct a10_timer_softc *a10_timer_sc = NULL;
+
+static struct resource_spec a10_timer_spec[] = {
+	{ SYS_RES_MEMORY,	0,	RF_ACTIVE },
+	{ SYS_RES_IRQ,		0,	RF_ACTIVE },
+	{ -1, 0 }
+};
+
+static uint64_t
+timer_read_counter64(void)
+{
+	uint32_t lo, hi;
+
+	/* Latch counter, wait for it to be ready to read. */
+	timer_write_4(a10_timer_sc, CNT64_CTRL_REG, CNT64_RL_EN);
+	while (timer_read_4(a10_timer_sc, CNT64_CTRL_REG) & CNT64_RL_EN)
+		continue;
+
+	hi = timer_read_4(a10_timer_sc, SW_COUNTER64HI_REG);
+	lo = timer_read_4(a10_timer_sc, SW_COUNTER64LO_REG);
+
+	return (((uint64_t)hi << 32) | lo);
+}
+
+static int
+a10_timer_probe(device_t dev)
+{
+
+	if (!ofw_bus_is_compatible(dev, "allwinner,sun4i-timer"))
+		return (ENXIO);
+
+	device_set_desc(dev, "Allwinner A10 timer");
+	return (BUS_PROBE_DEFAULT);
+}
+
+static int
+a10_timer_attach(device_t dev)
+{
+	struct a10_timer_softc *sc;
+	int err;
+	uint32_t val;
+
+	sc = device_get_softc(dev);
+
+	if (bus_alloc_resources(dev, a10_timer_spec, sc->res)) {
+		device_printf(dev, "could not allocate resources\n");
+		return (ENXIO);
+	}
+
+	sc->sc_dev = dev;
+	sc->sc_bst = rman_get_bustag(sc->res[0]);
+	sc->sc_bsh = rman_get_bushandle(sc->res[0]);
+
+	/* Setup and enable the timer interrupt */
+	err = bus_setup_intr(dev, sc->res[1], INTR_TYPE_CLK, a10_timer_hardclock,
+	    NULL, sc, &sc->sc_ih);
+	if (err != 0) {
+		bus_release_resources(dev, a10_timer_spec, sc->res);
+		device_printf(dev, "Unable to setup the clock irq handler, "
+		    "err = %d\n", err);
+		return (ENXIO);
+	}
+
+	/* Set clock source to OSC24M, 16 pre-division */
+	val = timer_read_4(sc, SW_TIMER0_CTRL_REG);
+	val |= TIMER_PRESCALAR | TIMER_OSC24M;
+	timer_write_4(sc, SW_TIMER0_CTRL_REG, val);
+
+	/* Enable timer0 */
+	val = timer_read_4(sc, SW_TIMER_IRQ_EN_REG);
+	val |= TIMER_ENABLE;
+	timer_write_4(sc, SW_TIMER_IRQ_EN_REG, val);
+
+	sc->timer0_freq = SYS_TIMER_CLKSRC;
+
+	/* Set desired frequency in event timer and timecounter */
+	sc->et.et_frequency = sc->timer0_freq;
+	sc->et.et_name = "a10_timer Eventtimer";
+	sc->et.et_flags = ET_FLAGS_ONESHOT | ET_FLAGS_PERIODIC;
+	sc->et.et_quality = 1000;
+	sc->et.et_min_period.sec = 0;
+	sc->et.et_min_period.frac =
+	    ((0x00000005LLU << 32) / sc->et.et_frequency) << 32;
+	sc->et.et_max_period.sec = 0xfffffff0U / sc->et.et_frequency;
+	sc->et.et_max_period.frac =
+	    ((0xfffffffeLLU << 32) / sc->et.et_frequency) << 32;
+	sc->et.et_start = a10_timer_timer_start;
+	sc->et.et_stop = a10_timer_timer_stop;
+	sc->et.et_priv = sc;
+	et_register(&sc->et);
+
+	if (device_get_unit(dev) == 0)
+		a10_timer_sc = sc;
+
+	a10_timer_timecounter.tc_frequency = sc->timer0_freq;
+	tc_init(&a10_timer_timecounter);
+
+	if (bootverbose) {
+		device_printf(sc->sc_dev, "clock: hz=%d stathz = %d\n", hz, stathz);
+
+		device_printf(sc->sc_dev, "event timer clock frequency %u\n", 
+		    sc->timer0_freq);
+		device_printf(sc->sc_dev, "timecounter clock frequency %lld\n", 
+		    a10_timer_timecounter.tc_frequency);
+	}
+
+	a10_timer_initialized = 1;
+
+	return (0);
+}
+
+static int
+a10_timer_timer_start(struct eventtimer *et, struct bintime *first,
+    struct bintime *period)
+{
+	struct a10_timer_softc *sc;
+	uint32_t count;
+	uint32_t val;
+
+	sc = (struct a10_timer_softc *)et->et_priv;
+
+	sc->sc_period = 0;
+
+	if (period != NULL) {
+		sc->sc_period = (sc->et.et_frequency * (period->frac >> 32)) >> 32;
+		sc->sc_period += sc->et.et_frequency * period->sec;
+	}
+	if (first == NULL)
+		count = sc->sc_period;
+	else {
+		count = (sc->et.et_frequency * (first->frac >> 32)) >> 32;
+		if (first->sec != 0)
+			count += sc->et.et_frequency * first->sec;
+	}
+
+	/* Update timer values */
+	timer_write_4(sc, SW_TIMER0_INT_VALUE_REG, sc->sc_period);
+	timer_write_4(sc, SW_TIMER0_CUR_VALUE_REG, count);
+
+	val = timer_read_4(sc, SW_TIMER0_CTRL_REG);
+	if (first == NULL) {
+		/* periodic */
+		val |= TIMER_AUTORELOAD;
+	} else {
+		/* oneshot */
+		val &= ~TIMER_AUTORELOAD;
+	}
+	/* Enable timer0 */
+	val |= TIMER_ENABLE;
+	timer_write_4(sc, SW_TIMER0_CTRL_REG, val);
+
+	return (0);
+}
+
+static int
+a10_timer_timer_stop(struct eventtimer *et)
+{
+	struct a10_timer_softc *sc;
+	uint32_t val;
+
+	sc = (struct a10_timer_softc *)et->et_priv;
+
+	/* Disable timer0 */
+	val = timer_read_4(sc, SW_TIMER0_CTRL_REG);
+	val &= ~TIMER_ENABLE;
+	timer_write_4(sc, SW_TIMER0_CTRL_REG, val);
+
+	sc->sc_period = 0;
+
+	return (0);
+}
+
+int
+a10_timer_get_timerfreq(struct a10_timer_softc *sc)
+{
+	return (sc->timer0_freq);
+}
+
+void
+cpu_initclocks(void)
+{
+	cpu_initclocks_bsp();
+}
+
+static int
+a10_timer_hardclock(void *arg)
+{
+	struct a10_timer_softc *sc;
+	uint32_t val;
+
+	sc = (struct a10_timer_softc *)arg;
+
+	/* Clear interrupt pending bit. */
+	timer_write_4(sc, SW_TIMER_IRQ_STA_REG, 0x1);
+
+	val = timer_read_4(sc, SW_TIMER0_CTRL_REG);
+	/*
+	 * Disabled autoreload and sc_period > 0 means 
+	 * timer_start was called with non NULL first value.
+	 * Now we will set periodic timer with the given period 
+	 * value.
+	 */
+	if ((val & (1<<1)) == 0 && sc->sc_period > 0) {
+		/* Update timer */
+		timer_write_4(sc, SW_TIMER0_CUR_VALUE_REG, sc->sc_period);
+
+		/* Make periodic and enable */
+		val |= TIMER_AUTORELOAD | TIMER_ENABLE;
+		timer_write_4(sc, SW_TIMER0_CTRL_REG, val);
+	}
+
+	if (sc->et.et_active)
+		sc->et.et_event_cb(&sc->et, sc->et.et_arg);
+
+	return (FILTER_HANDLED);
+}
+
+u_int
+a10_timer_get_timecount(struct timecounter *tc)
+{
+
+	if (a10_timer_sc == NULL)
+		return (0);
+
+	return ((u_int)timer_read_counter64());
+}
+
+static device_method_t a10_timer_methods[] = {
+	DEVMETHOD(device_probe,		a10_timer_probe),
+	DEVMETHOD(device_attach,	a10_timer_attach),
+
+	DEVMETHOD_END
+};
+
+static driver_t a10_timer_driver = {
+	"a10_timer",
+	a10_timer_methods,
+	sizeof(struct a10_timer_softc),
+};
+
+static devclass_t a10_timer_devclass;
+
+DRIVER_MODULE(a10_timer, simplebus, a10_timer_driver, a10_timer_devclass, 0, 0);
+
+void
+DELAY(int usec)
+{
+	uint32_t counter;
+	uint64_t end, now;
+
+	if (!a10_timer_initialized) {
+		for (; usec > 0; usec--)
+			for (counter = 50; counter > 0; counter--)
+				cpufunc_nullop();
+		return;
+	}
+
+	now = timer_read_counter64();
+	end = now + (a10_timer_sc->timer0_freq / 1000000) * (usec + 1);
+
+	while (now < end)
+		now = timer_read_counter64();
+}
+
diff --git a/sys/arm/arm/busdma_machdep.c b/sys/arm/arm/busdma_machdep.c
index 42566e8..fa7afd4 100644
--- a/sys/arm/arm/busdma_machdep.c
+++ b/sys/arm/arm/busdma_machdep.c
@@ -849,9 +849,6 @@ bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t *segs,
 	vm_offset_t vaddr = (vm_offset_t)buf;
 	int seg;
 	int error = 0;
-	pd_entry_t *pde;
-	pt_entry_t pte;
-	pt_entry_t *ptep;
 
 	lastaddr = *lastaddrp;
 	bmask = ~(dmat->boundary - 1);
@@ -868,34 +865,9 @@ bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t *segs,
 	for (seg = *segp; buflen > 0 ; ) {
 		/*
 		 * Get the physical address for this segment.
-		 *
-		 * XXX Don't support checking for coherent mappings
-		 * XXX in user address space.
 		 */
 		if (__predict_true(pmap == pmap_kernel())) {
-			if (pmap_get_pde_pte(pmap, vaddr, &pde, &ptep) == FALSE)
-				return (EFAULT);
-
-			if (__predict_false(pmap_pde_section(pde))) {
-				if (*pde & L1_S_SUPERSEC)
-					curaddr = (*pde & L1_SUP_FRAME) |
-					    (vaddr & L1_SUP_OFFSET);
-				else
-					curaddr = (*pde & L1_S_FRAME) |
-					    (vaddr & L1_S_OFFSET);
-			} else {
-				pte = *ptep;
-				KASSERT((pte & L2_TYPE_MASK) != L2_TYPE_INV,
-				    ("INV type"));
-				if (__predict_false((pte & L2_TYPE_MASK)
-						    == L2_TYPE_L)) {
-					curaddr = (pte & L2_L_FRAME) |
-					    (vaddr & L2_L_OFFSET);
-				} else {
-					curaddr = (pte & L2_S_FRAME) |
-					    (vaddr & L2_S_OFFSET);
-				}
-			}
+			curaddr = pmap_kextract(vaddr);
 		} else {
 			curaddr = pmap_extract(pmap, vaddr);
 			map->flags &= ~DMAMAP_COHERENT;
diff --git a/sys/arm/arm/cpufunc.c b/sys/arm/arm/cpufunc.c
index 5dc2679..1d15e27 100644
--- a/sys/arm/arm/cpufunc.c
+++ b/sys/arm/arm/cpufunc.c
@@ -1146,7 +1146,7 @@ struct cpu_functions cortexa_cpufuncs = {
 	/* Other functions */
 	
 	cpufunc_nullop,                 /* flush_prefetchbuf    */
-	arm11_drain_writebuf,           /* drain_writebuf       */
+	armv7_drain_writebuf,           /* drain_writebuf       */
 	cpufunc_nullop,                 /* flush_brnchtgt_C     */
 	(void *)cpufunc_nullop,         /* flush_brnchtgt_E     */
 	
@@ -1157,7 +1157,7 @@ struct cpu_functions cortexa_cpufuncs = {
 	cpufunc_null_fixup,             /* dataabt_fixup        */
 	cpufunc_null_fixup,             /* prefetchabt_fixup    */
 	
-	arm11_context_switch,           /* context_switch       */
+	armv7_context_switch,           /* context_switch       */
 	
 	cortexa_setup                     /* cpu setup            */
 };
diff --git a/sys/arm/arm/cpufunc_asm_arm10.S b/sys/arm/arm/cpufunc_asm_arm10.S
index 22da6aa..2ef999c 100644
--- a/sys/arm/arm/cpufunc_asm_arm10.S
+++ b/sys/arm/arm/cpufunc_asm_arm10.S
@@ -87,7 +87,7 @@ ENTRY_NP(arm10_icache_sync_range)
 	mcr	p15, 0, r0, c7, c10, 1	/* Clean D cache SE with VA */
 	add	r0, r0, ip
 	subs	r1, r1, ip
-	bpl	.Larm10_sync_next
+	bhi	.Larm10_sync_next
 	mcr	p15, 0, r0, c7, c10, 4	/* drain the write buffer */
 	bx	lr
 
@@ -108,12 +108,10 @@ ENTRY_NP(arm10_icache_sync_all)
 	orr	ip, s_max, i_max
 .Lnext_index:
 	mcr	p15, 0, ip, c7, c10, 2	/* Clean D cache SE with Set/Index */
-	sub	ip, ip, i_inc
-	tst	ip, i_max		/* Index 0 is last one */
-	bne	.Lnext_index		/* Next index */
-	mcr	p15, 0, ip, c7, c10, 2	/* Clean D cache SE with Set/Index */
+	subs	ip, ip, i_inc
+	bhs	.Lnext_index		/* Next index */
 	subs	s_max, s_max, s_inc
-	bpl	.Lnext_set		/* Next set */
+	bhs	.Lnext_set		/* Next set */
 	mcr	p15, 0, r0, c7, c10, 4	/* drain the write buffer */
 	bx	lr
 
@@ -133,7 +131,7 @@ ENTRY(arm10_dcache_wb_range)
 	mcr	p15, 0, r0, c7, c10, 1	/* Clean D cache SE with VA */
 	add	r0, r0, ip
 	subs	r1, r1, ip
-	bpl	.Larm10_wb_next
+	bhi	.Larm10_wb_next
 	mcr	p15, 0, r0, c7, c10, 4	/* drain the write buffer */
 	bx	lr
 	
@@ -150,7 +148,7 @@ ENTRY(arm10_dcache_wbinv_range)
 	mcr	p15, 0, r0, c7, c14, 1	/* Purge D cache SE with VA */
 	add	r0, r0, ip
 	subs	r1, r1, ip
-	bpl	.Larm10_wbinv_next
+	bhi	.Larm10_wbinv_next
 	mcr	p15, 0, r0, c7, c10, 4	/* drain the write buffer */
 	bx	lr
 	
@@ -171,7 +169,7 @@ ENTRY(arm10_dcache_inv_range)
 	mcr	p15, 0, r0, c7, c6, 1	/* Invalidate D cache SE with VA */
 	add	r0, r0, ip
 	subs	r1, r1, ip
-	bpl	.Larm10_inv_next
+	bhi	.Larm10_inv_next
 	mcr	p15, 0, r0, c7, c10, 4	/* drain the write buffer */
 	bx	lr
 
@@ -189,7 +187,7 @@ ENTRY(arm10_idcache_wbinv_range)
 	mcr	p15, 0, r0, c7, c14, 1	/* Purge D cache SE with VA */
 	add	r0, r0, ip
 	subs	r1, r1, ip
-	bpl	.Larm10_id_wbinv_next
+	bhi	.Larm10_id_wbinv_next
 	mcr	p15, 0, r0, c7, c10, 4	/* drain the write buffer */
 	bx	lr
 
@@ -211,12 +209,10 @@ ENTRY(arm10_dcache_wbinv_all)
 	orr	ip, s_max, i_max
 .Lnext_index_inv:
 	mcr	p15, 0, ip, c7, c14, 2	/* Purge D cache SE with Set/Index */
-	sub	ip, ip, i_inc
-	tst	ip, i_max		/* Index 0 is last one */
-	bne	.Lnext_index_inv		/* Next index */
-	mcr	p15, 0, ip, c7, c14, 2	/* Purge D cache SE with Set/Index */
+	subs	ip, ip, i_inc
+	bhs	.Lnext_index_inv		/* Next index */
 	subs	s_max, s_max, s_inc
-	bpl	.Lnext_set_inv		/* Next set */
+	bhs	.Lnext_set_inv		/* Next set */
 	mcr	p15, 0, r0, c7, c10, 4	/* drain the write buffer */
 	bx	lr
 
diff --git a/sys/arm/arm/cpufunc_asm_arm9.S b/sys/arm/arm/cpufunc_asm_arm9.S
index 291d3f7..ae9fe00 100644
--- a/sys/arm/arm/cpufunc_asm_arm9.S
+++ b/sys/arm/arm/cpufunc_asm_arm9.S
@@ -81,7 +81,7 @@ ENTRY_NP(arm9_icache_sync_range)
 	mcr	p15, 0, r0, c7, c10, 1	/* Clean D cache SE with VA */
 	add	r0, r0, ip
 	subs	r1, r1, ip
-	bpl	.Larm9_sync_next
+	bhi	.Larm9_sync_next
 	mov	pc, lr
 
 ENTRY_NP(arm9_icache_sync_all)
@@ -101,12 +101,10 @@ ENTRY_NP(arm9_icache_sync_all)
 	orr	ip, s_max, i_max
 .Lnext_index:
 	mcr	p15, 0, ip, c7, c10, 2	/* Clean D cache SE with Set/Index */
-	sub	ip, ip, i_inc
-	tst	ip, i_max		/* Index 0 is last one */
-	bne	.Lnext_index		/* Next index */
-	mcr	p15, 0, ip, c7, c10, 2	/* Clean D cache SE with Set/Index */
+	subs	ip, ip, i_inc
+	bhs	.Lnext_index		/* Next index */
 	subs	s_max, s_max, s_inc
-	bpl	.Lnext_set		/* Next set */
+	bhs	.Lnext_set		/* Next set */
 	mov	pc, lr
 
 .Larm9_line_size:
@@ -125,7 +123,7 @@ ENTRY(arm9_dcache_wb_range)
 	mcr	p15, 0, r0, c7, c10, 1	/* Clean D cache SE with VA */
 	add	r0, r0, ip
 	subs	r1, r1, ip
-	bpl	.Larm9_wb_next
+	bhi	.Larm9_wb_next
 	mov	pc, lr
 	
 ENTRY(arm9_dcache_wbinv_range)
@@ -141,7 +139,7 @@ ENTRY(arm9_dcache_wbinv_range)
 	mcr	p15, 0, r0, c7, c14, 1	/* Purge D cache SE with VA */
 	add	r0, r0, ip
 	subs	r1, r1, ip
-	bpl	.Larm9_wbinv_next
+	bhi	.Larm9_wbinv_next
 	mov	pc, lr
 	
 /*
@@ -161,7 +159,7 @@ ENTRY(arm9_dcache_inv_range)
 	mcr	p15, 0, r0, c7, c6, 1	/* Invalidate D cache SE with VA */
 	add	r0, r0, ip
 	subs	r1, r1, ip
-	bpl	.Larm9_inv_next
+	bhi	.Larm9_inv_next
 	mov	pc, lr
 
 ENTRY(arm9_idcache_wbinv_range)
@@ -178,7 +176,7 @@ ENTRY(arm9_idcache_wbinv_range)
 	mcr	p15, 0, r0, c7, c14, 1	/* Purge D cache SE with VA */
 	add	r0, r0, ip
 	subs	r1, r1, ip
-	bpl	.Larm9_id_wbinv_next
+	bhi	.Larm9_id_wbinv_next
 	mov	pc, lr
 
 ENTRY_NP(arm9_idcache_wbinv_all)
@@ -199,12 +197,10 @@ ENTRY(arm9_dcache_wbinv_all)
 	orr	ip, s_max, i_max
 .Lnext_index_inv:
 	mcr	p15, 0, ip, c7, c14, 2	/* Purge D cache SE with Set/Index */
-	sub	ip, ip, i_inc
-	tst	ip, i_max		/* Index 0 is last one */
-	bne	.Lnext_index_inv		/* Next index */
-	mcr	p15, 0, ip, c7, c14, 2	/* Purge D cache SE with Set/Index */
+	subs	ip, ip, i_inc
+	bhs	.Lnext_index_inv		/* Next index */
 	subs	s_max, s_max, s_inc
-	bpl	.Lnext_set_inv		/* Next set */
+	bhs	.Lnext_set_inv		/* Next set */
 	mov	pc, lr
 
 .Larm9_cache_data:
diff --git a/sys/arm/arm/db_trace.c b/sys/arm/arm/db_trace.c
index d6e1c3a..89ec507 100644
--- a/sys/arm/arm/db_trace.c
+++ b/sys/arm/arm/db_trace.c
@@ -50,6 +50,395 @@ __FBSDID("$FreeBSD$");
 #include <ddb/db_sym.h>
 #include <ddb/db_output.h>
 
+#ifdef __ARM_EABI__
+/*
+ * Definitions for the instruction interpreter.
+ *
+ * The ARM EABI specifies how to perform the frame unwinding in the
+ * Exception Handling ABI for the ARM Architecture document. To perform
+ * the unwind we need to know the initial frame pointer, stack pointer,
+ * link register and program counter. We then find the entry within the
+ * index table that points to the function the program counter is within.
+ * This gives us either a list of three instructions to process, a 31-bit
+ * relative offset to a table of instructions, or a value telling us
+ * we can't unwind any further.
+ *
+ * When we have the instructions to process we need to decode them
+ * following table 4 in section 9.3. This describes a collection of bit
+ * patterns to encode that steps to take to update the stack pointer and
+ * link register to the correct values at the start of the function.
+ */
+
+/* A special case when we are unable to unwind past this function */
+#define	EXIDX_CANTUNWIND	1
+
+/* The register names */
+#define	FP	11
+#define	SP	13
+#define	LR	14
+#define	PC	15
+
+/*
+ * These are set in the linker script. Their addresses will be
+ * either the start or end of the exception table or index.
+ */
+extern int extab_start, extab_end, exidx_start, exidx_end;
+
+/*
+ * Entry types.
+ * These are the only entry types that have been seen in the kernel.
+ */
+#define	ENTRY_MASK	0xff000000
+#define	ENTRY_ARM_SU16	0x80000000
+#define	ENTRY_ARM_LU16	0x81000000
+
+/* Instruction masks. */
+#define	INSN_VSP_MASK		0xc0
+#define	INSN_VSP_SIZE_MASK	0x3f
+#define	INSN_STD_MASK		0xf0
+#define	INSN_STD_DATA_MASK	0x0f
+#define	INSN_POP_TYPE_MASK	0x08
+#define	INSN_POP_COUNT_MASK	0x07
+#define	INSN_VSP_LARGE_INC_MASK	0xff
+
+/* Instruction definitions */
+#define	INSN_VSP_INC		0x00
+#define	INSN_VSP_DEC		0x40
+#define	INSN_POP_MASKED		0x80
+#define	INSN_VSP_REG		0x90
+#define	INSN_POP_COUNT		0xa0
+#define	INSN_FINISH		0xb0
+#define	INSN_VSP_LARGE_INC	0xb2
+
+/* An item in the exception index table */
+struct unwind_idx {
+	uint32_t offset;
+	uint32_t insn;
+};
+
+/* The state of the unwind process */
+struct unwind_state {
+	uint32_t registers[16];
+	uint32_t start_pc;
+	uint32_t *insn;
+	u_int entries;
+	u_int byte;
+	uint16_t update_mask;
+};
+
+/* We need to provide these but never use them */
+void __aeabi_unwind_cpp_pr0(void);
+void __aeabi_unwind_cpp_pr1(void);
+void __aeabi_unwind_cpp_pr2(void);
+
+void
+__aeabi_unwind_cpp_pr0(void)
+{
+	panic("__aeabi_unwind_cpp_pr0");
+}
+
+void
+__aeabi_unwind_cpp_pr1(void)
+{
+	panic("__aeabi_unwind_cpp_pr1");
+}
+
+void
+__aeabi_unwind_cpp_pr2(void)
+{
+	panic("__aeabi_unwind_cpp_pr2");
+}
+
+/* Expand a 31-bit signed value to a 32-bit signed value */
+static __inline int32_t
+db_expand_prel31(uint32_t prel31)
+{
+
+	return ((int32_t)(prel31 & 0x7fffffffu) << 1) / 2;
+}
+
+/*
+ * Perform a binary search of the index table to find the function
+ * with the largest address that doesn't exceed addr.
+ */
+static struct unwind_idx *
+db_find_index(uint32_t addr)
+{
+	unsigned int min, mid, max;
+	struct unwind_idx *start;
+	struct unwind_idx *item;
+	int32_t prel31_addr;
+	uint32_t func_addr;
+
+	start = (struct unwind_idx *)&exidx_start;
+
+	min = 0;
+	max = (&exidx_end - &exidx_start) / 2;
+
+	while (min != max) {
+		mid = min + (max - min + 1) / 2;
+
+		item = &start[mid];
+
+	 	prel31_addr = db_expand_prel31(item->offset);
+		func_addr = (uint32_t)&item->offset + prel31_addr;
+
+		if (func_addr <= addr) {
+			min = mid;
+		} else {
+			max = mid - 1;
+		}
+	}
+
+	return &start[min];
+}
+
+/* Reads the next byte from the instruction list */
+static uint8_t
+db_unwind_exec_read_byte(struct unwind_state *state)
+{
+	uint8_t insn;
+
+	/* Read the unwind instruction */
+	insn = (*state->insn) >> (state->byte * 8);
+
+	/* Update the location of the next instruction */
+	if (state->byte == 0) {
+		state->byte = 3;
+		state->insn++;
+		state->entries--;
+	} else
+		state->byte--;
+
+	return insn;
+}
+
+/* Executes the next instruction on the list */
+static int
+db_unwind_exec_insn(struct unwind_state *state)
+{
+	unsigned int insn;
+	uint32_t *vsp = (uint32_t *)state->registers[SP];
+	int update_vsp = 0;
+
+	/* This should never happen */
+	if (state->entries == 0)
+		return 1;
+
+	/* Read the next instruction */
+	insn = db_unwind_exec_read_byte(state);
+
+	if ((insn & INSN_VSP_MASK) == INSN_VSP_INC) {
+		state->registers[SP] += ((insn & INSN_VSP_SIZE_MASK) << 2) + 4;
+
+	} else if ((insn & INSN_VSP_MASK) == INSN_VSP_DEC) {
+		state->registers[SP] -= ((insn & INSN_VSP_SIZE_MASK) << 2) + 4;
+
+	} else if ((insn & INSN_STD_MASK) == INSN_POP_MASKED) {
+		unsigned int mask, reg;
+
+		/* Load the mask */
+		mask = db_unwind_exec_read_byte(state);
+		mask |= (insn & INSN_STD_DATA_MASK) << 8;
+
+		/* We have a refuse to unwind instruction */
+		if (mask == 0)
+			return 1;
+
+		/* Update SP */
+		update_vsp = 1;
+
+		/* Load the registers */
+		for (reg = 4; mask && reg < 16; mask >>= 1, reg++) {
+			if (mask & 1) {
+				state->registers[reg] = *vsp++;
+				state->update_mask |= 1 << reg;
+
+				/* If we have updated SP kep its value */
+				if (reg == SP)
+					update_vsp = 0;
+			}
+		}
+
+	} else if ((insn & INSN_STD_MASK) == INSN_VSP_REG &&
+	    ((insn & INSN_STD_DATA_MASK) != 13) &&
+	    ((insn & INSN_STD_DATA_MASK) != 15)) {
+		/* sp = register */
+		state->registers[SP] =
+		    state->registers[insn & INSN_STD_DATA_MASK];
+
+	} else if ((insn & INSN_STD_MASK) == INSN_POP_COUNT) {
+		unsigned int count, reg;
+
+		/* Read how many registers to load */
+		count = insn & INSN_POP_COUNT_MASK;
+
+		/* Update sp */
+		update_vsp = 1;
+
+		/* Pop the registers */
+		for (reg = 4; reg <= 4 + count; reg++) {
+			state->registers[reg] = *vsp++;
+			state->update_mask |= 1 << reg;
+		}
+
+		/* Check if we are in the pop r14 version */
+		if ((insn & INSN_POP_TYPE_MASK) != 0) {
+			state->registers[14] = *vsp++;
+		}
+
+	} else if (insn == INSN_FINISH) {
+		/* Stop processing */
+		state->entries = 0;
+
+	} else if ((insn & INSN_VSP_LARGE_INC_MASK) == INSN_VSP_LARGE_INC) {
+		unsigned int uleb128;
+
+		/* Read the increment value */
+		uleb128 = db_unwind_exec_read_byte(state);
+
+		state->registers[SP] += 0x204 + (uleb128 << 2);
+
+	} else {
+		/* We hit a new instruction that needs to be implemented */
+		db_printf("Unhandled instruction %.2x\n", insn);
+		return 1;
+	}
+
+	if (update_vsp) {
+		state->registers[SP] = (uint32_t)vsp;
+	}
+
+#if 0
+	db_printf("fp = %08x, sp = %08x, lr = %08x, pc = %08x\n",
+	    state->registers[FP], state->registers[SP], state->registers[LR],
+	    state->registers[PC]);
+#endif
+
+	return 0;
+}
+
+/* Performs the unwind of a function */
+static int
+db_unwind_tab(struct unwind_state *state)
+{
+	uint32_t entry;
+
+	/* Set PC to a known value */
+	state->registers[PC] = 0;
+
+	/* Read the personality */
+	entry = *state->insn & ENTRY_MASK;
+
+	if (entry == ENTRY_ARM_SU16) {
+		state->byte = 2;
+		state->entries = 1;
+	} else if (entry == ENTRY_ARM_LU16) {
+		state->byte = 1;
+		state->entries = ((*state->insn >> 16) & 0xFF) + 1;
+	} else {
+		db_printf("Unknown entry: %x\n", entry);
+		return 1;
+	}
+
+	while (state->entries > 0) {
+		if (db_unwind_exec_insn(state) != 0)
+			return 1;
+	}
+
+	/*
+	 * The program counter was not updated, load it from the link register.
+	 */
+	if (state->registers[PC] == 0)
+		state->registers[PC] = state->registers[LR];
+
+	return 0;
+}
+
+static void
+db_stack_trace_cmd(struct unwind_state *state)
+{
+	struct unwind_idx *index;
+	const char *name;
+	db_expr_t value;
+	db_expr_t offset;
+	c_db_sym_t sym;
+	u_int reg, i;
+	char *sep;
+
+	while (1) {
+		/* Reset the mask of updated registers */
+		state->update_mask = 0;
+
+		/* The pc value is correct and will be overwritten, save it */
+		state->start_pc = state->registers[PC];
+
+		/* Find the item to run */
+		index = db_find_index(state->start_pc);
+
+		if (index->insn == EXIDX_CANTUNWIND) {
+			printf("Unable to unwind\n");
+			break;
+		} else if (index->insn & (1 << 31)) {
+			/* The data is within the instruction */
+			state->insn = &index->insn;
+		} else {
+			/* We have a prel31 offset to the unwind table */
+			uint32_t prel31_tbl = db_expand_prel31(index->insn);
+
+			state->insn = (uint32_t *)((uintptr_t)&index->insn +
+			    prel31_tbl);
+		}
+
+		/* Run the unwind function */
+		if (db_unwind_tab(state) != 0)
+			break;
+
+		/* This is not a kernel address, stop processing */
+		if (state->registers[PC] < VM_MIN_KERNEL_ADDRESS)
+			break;
+
+		/* Print the frame details */
+		sym = db_search_symbol(state->start_pc, DB_STGY_ANY, &offset);
+		if (sym == C_DB_SYM_NULL) {
+			value = 0;
+			name = "(null)";
+		} else
+			db_symbol_values(sym, &name, &value);
+		db_printf("%s() at ", name);
+		db_printsym(state->start_pc, DB_STGY_PROC);
+		db_printf("\n");
+		db_printf("\t pc = 0x%08x  lr = 0x%08x (", state->start_pc,
+		    state->registers[LR]);
+		db_printsym(state->registers[LR], DB_STGY_PROC);
+		db_printf(")\n");
+		db_printf("\t sp = 0x%08x  fp = 0x%08x",
+		    state->registers[SP], state->registers[FP]);
+
+		/* Don't print the registers we have already printed */
+		state->update_mask &= ~((1 << SP) | (1 << FP) | (1 << LR) |
+		    (1 << PC));
+		sep = "\n\t";
+		for (i = 0, reg = 0; state->update_mask != 0;
+		    state->update_mask >>= 1, reg++) {
+			if ((state->update_mask & 1) != 0) {
+				db_printf("%s%sr%d = 0x%08x", sep,
+				    (reg < 10) ? " " : "", reg,
+				    state->registers[reg]);
+				i++;
+				if (i == 2) {
+					sep = "\n\t";
+					i = 0;
+				} else
+					sep = " ";
+				
+			}
+		}
+		db_printf("\n");
+	}
+}
+#endif
+
 /*
  * APCS stack frames are awkward beasts, so I don't think even trying to use
  * a structure to represent them is a good idea.
@@ -78,6 +467,7 @@ __FBSDID("$FreeBSD$");
  * fields are actually present.
  */
 
+#ifndef __ARM_EABI__	/* The frame format is differend in AAPCS */
 static void
 db_stack_trace_cmd(db_expr_t addr, db_expr_t count, boolean_t kernel_only)
 {
@@ -171,6 +561,7 @@ db_stack_trace_cmd(db_expr_t addr, db_expr_t count, boolean_t kernel_only)
 		}
 	}
 }
+#endif
 
 /* XXX stubs */
 void
@@ -193,11 +584,24 @@ db_md_set_watchpoint(db_expr_t addr, db_expr_t size)
 int
 db_trace_thread(struct thread *thr, int count)
 {
+#ifdef __ARM_EABI__
+	struct unwind_state state;
+#endif
 	struct pcb *ctx;
 
 	if (thr != curthread) {
 		ctx = kdb_thr_ctx(thr);
+
+#ifdef __ARM_EABI__
+		state.registers[FP] = ctx->un_32.pcb32_r11;
+		state.registers[SP] = ctx->un_32.pcb32_sp;
+		state.registers[LR] = ctx->un_32.pcb32_lr;
+		state.registers[PC] = ctx->un_32.pcb32_pc;
+
+		db_stack_trace_cmd(&state);
+#else
 		db_stack_trace_cmd(ctx->un_32.pcb32_r11, -1, TRUE);
+#endif
 	} else
 		db_trace_self();
 	return (0);
@@ -206,8 +610,20 @@ db_trace_thread(struct thread *thr, int count)
 void
 db_trace_self(void)
 {
+#ifdef __ARM_EABI__
+	struct unwind_state state;
+	register uint32_t sp __asm__ ("sp");
+
+	state.registers[FP] = (uint32_t)__builtin_frame_address(0);
+	state.registers[SP] = (uint32_t)sp;
+	state.registers[LR] = (uint32_t)__builtin_return_address(0);
+	state.registers[PC] = (uint32_t)db_trace_self;
+
+	db_stack_trace_cmd(&state);
+#else
 	db_addr_t addr;
 
 	addr = (db_addr_t)__builtin_frame_address(0);
 	db_stack_trace_cmd(addr, -1, FALSE);
+#endif
 }
diff --git a/sys/arm/arm/intr.c b/sys/arm/arm/intr.c
index e79e44c..062b10a 100644
--- a/sys/arm/arm/intr.c
+++ b/sys/arm/arm/intr.c
@@ -50,17 +50,34 @@ __FBSDID("$FreeBSD$");
 #include <machine/intr.h>
 #include <machine/cpu.h>
 
+#define	INTRNAME_LEN	(MAXCOMLEN + 1)
+
 typedef void (*mask_fn)(void *);
 
 static struct intr_event *intr_events[NIRQ];
-static int intrcnt_tab[NIRQ];
-static int intrcnt_index = 0;
-static int last_printed = 0;
 
 void	arm_handler_execute(struct trapframe *, int);
 
 void (*arm_post_filter)(void *) = NULL;
 
+/*
+ * Pre-format intrnames into an array of fixed-size strings containing spaces.
+ * This allows us to avoid the need for an intermediate table of indices into
+ * the names and counts arrays, while still meeting the requirements and
+ * assumptions of vmstat(8) and the kdb "show intrcnt" command, the two
+ * consumers of this data.
+ */
+void
+arm_intrnames_init(void)
+{
+	int i;
+
+	for (i = 0; i < NIRQ; ++i) {
+		snprintf(&intrnames[i * INTRNAME_LEN], INTRNAME_LEN, "%-*s",
+		    INTRNAME_LEN - 1, "");
+	}
+}
+
 void
 arm_setup_irqhandler(const char *name, driver_filter_t *filt,
     void (*hand)(void*), void *arg, int irq, int flags, void **cookiep)
@@ -78,14 +95,8 @@ arm_setup_irqhandler(const char *name, driver_filter_t *filt,
 		if (error)
 			return;
 		intr_events[irq] = event;
-		last_printed +=
-		    snprintf(intrnames + last_printed,
-		    MAXCOMLEN + 1,
-		    "irq%d: %s", irq, name);
-		last_printed++;
-		intrcnt_tab[irq] = intrcnt_index;
-		intrcnt_index++;
-		
+		snprintf(&intrnames[irq * INTRNAME_LEN], INTRNAME_LEN, 
+		    "irq%d: %-*s", irq, INTRNAME_LEN - 1, name);
 	}
 	intr_event_add_handler(event, name, filt, hand, arg,
 	    intr_priority(flags), flags, cookiep);
@@ -122,7 +133,7 @@ arm_handler_execute(struct trapframe *frame, int irqnb)
 	PCPU_INC(cnt.v_intr);
 	i = -1;
 	while ((i = arm_get_next_irq(i)) != -1) {
-		intrcnt[intrcnt_tab[i]]++;
+		intrcnt[i]++;
 		event = intr_events[i];
 		if (intr_event_handle(event, frame) != 0) {
 			/* XXX: Log stray IRQs */
diff --git a/sys/arm/arm/locore.S b/sys/arm/arm/locore.S
index ed7af25..00a61e7 100644
--- a/sys/arm/arm/locore.S
+++ b/sys/arm/arm/locore.S
@@ -484,12 +484,29 @@ ENTRY_NP(abort)
 
 ENTRY_NP(sigcode)
 	mov	r0, sp
+
+	/*
+	 * Call the sigreturn system call.
+	 * 
+	 * We have to load r7 manually rather than using
+	 * "ldr r7, =SYS_sigreturn" to ensure the value of szsigcode is
+	 * correct. Using the alternative places esigcode at the address
+	 * of the data rather than the address one past the data.
+	 */
+
+	ldr	r7, [pc, #12]	/* Load SYS_sigreturn */
 	swi	SYS_sigreturn
 
 	/* Well if that failed we better exit quick ! */
 
+	ldr	r7, [pc, #8]	/* Load SYS_exit */
 	swi	SYS_exit
-	b	. - 8
+
+	/* Branch back to retry SYS_sigreturn */
+	b	. - 16
+
+	.word	SYS_sigreturn
+	.word	SYS_exit
 
 	.align	0
 	.global _C_LABEL(esigcode)
diff --git a/sys/arm/arm/machdep.c b/sys/arm/arm/machdep.c
index a1b5a8f..5c4d92c 100644
--- a/sys/arm/arm/machdep.c
+++ b/sys/arm/arm/machdep.c
@@ -1474,6 +1474,7 @@ initarm(struct arm_boot_params *abp)
 
 	init_proc0(kernelstack.pv_va);
 
+	arm_intrnames_init();
 	arm_vector_init(ARM_VECTORS_HIGH, ARM_VEC_ALL);
 	arm_dump_avail_init(memsize, sizeof(dump_avail) / sizeof(dump_avail[0]));
 	pmap_bootstrap(freemempos, pmap_bootstrap_lastaddr, &kernel_l1pt);
diff --git a/sys/arm/arm/swtch.S b/sys/arm/arm/swtch.S
index e93b948..4c422e8 100644
--- a/sys/arm/arm/swtch.S
+++ b/sys/arm/arm/swtch.S
@@ -79,6 +79,7 @@
  */
 
 #include "assym.s"
+#include "opt_sched.h"
 
 #include <machine/asm.h>
 #include <machine/asmacros.h>
@@ -428,6 +429,7 @@ ENTRY(cpu_switch)
 
 	/* Release the old thread */
 	str	r6, [r4, #TD_LOCK]
+#if defined(SCHED_ULE) && defined(SMP)
 	ldr	r6, .Lblocked_lock
 	GET_CURTHREAD_PTR(r3)
 
@@ -435,6 +437,7 @@ ENTRY(cpu_switch)
 	ldr	r4, [r3, #TD_LOCK]
 	cmp	r4, r6
 	beq	1b
+#endif
 	
 	/* XXXSCW: Safe to re-enable FIQs here */
 
diff --git a/sys/arm/arm/trap.c b/sys/arm/arm/trap.c
index e737dc8..bdc4ccc 100644
--- a/sys/arm/arm/trap.c
+++ b/sys/arm/arm/trap.c
@@ -866,7 +866,11 @@ cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa)
 	register_t *ap;
 	int error;
 
+#ifdef __ARM_EABI__
+	sa->code = td->td_frame->tf_r7;
+#else
 	sa->code = sa->insn & 0x000fffff;
+#endif
 	ap = &td->td_frame->tf_r0;
 	if (sa->code == SYS_syscall) {
 		sa->code = *ap++;
@@ -905,16 +909,18 @@ syscall(struct thread *td, trapframe_t *frame)
 	struct syscall_args sa;
 	int error;
 
+#ifndef __ARM_EABI__
 	sa.insn = *(uint32_t *)(frame->tf_pc - INSN_SIZE);
 	switch (sa.insn & SWI_OS_MASK) {
 	case 0: /* XXX: we need our own one. */
-		sa.nap = 4;
 		break;
 	default:
 		call_trapsignal(td, SIGILL, 0);
 		userret(td, frame);
 		return;
 	}
+#endif
+	sa.nap = 4;
 
 	error = syscallenter(td, &sa);
 	KASSERT(error != 0 || td->td_ar == NULL,
diff --git a/sys/arm/arm/vm_machdep.c b/sys/arm/arm/vm_machdep.c
index cc87bbe..22a209b 100644
--- a/sys/arm/arm/vm_machdep.c
+++ b/sys/arm/arm/vm_machdep.c
@@ -398,6 +398,12 @@ cpu_thread_alloc(struct thread *td)
 	    PAGE_SIZE) - 1;
 	td->td_frame = (struct trapframe *)
 	    ((u_int)td->td_kstack + USPACE_SVC_STACK_TOP - sizeof(struct pcb)) - 1;
+	/*
+	 * Ensure td_frame is aligned to an 8 byte boundary as it will be
+	 * placed into the stack pointer which must be 8 byte aligned in
+	 * the ARM EABI.
+	 */
+	td->td_frame = (struct trapframe *)((u_int)td->td_frame & ~7);
 #ifdef __XSCALE__
 #ifndef CPU_XSCALE_CORE3
 	pmap_use_minicache(td->td_kstack, td->td_kstack_pages * PAGE_SIZE);
diff --git a/sys/arm/broadcom/bcm2835/bcm2835_fb.c b/sys/arm/broadcom/bcm2835/bcm2835_fb.c
index 35343c8..0f1e81c 100644
--- a/sys/arm/broadcom/bcm2835/bcm2835_fb.c
+++ b/sys/arm/broadcom/bcm2835/bcm2835_fb.c
@@ -136,6 +136,7 @@ struct video_adapter_softc {
 	int		console;
 
 	intptr_t	fb_addr;
+	intptr_t	fb_paddr;
 	unsigned int	fb_size;
 
 	unsigned int	height;
@@ -222,6 +223,7 @@ bcm_fb_init(void *arg)
 			fb_config->screen_size);
 
 		va_sc->fb_addr = (intptr_t)pmap_mapdev(fb_config->base, fb_config->screen_size);
+		va_sc->fb_paddr = fb_config->base;
 		va_sc->fb_size = fb_config->screen_size;
 		va_sc->depth = fb_config->bpp;
 		va_sc->stride = fb_config->pitch;
@@ -795,7 +797,7 @@ bcmfb_mmap(video_adapter_t *adp, vm_ooffset_t offset, vm_paddr_t *paddr,
 	 * framebuffer, since it shouldn't be touched
 	 */
 	if (offset < sc->stride*sc->height) {
-		*paddr = sc->fb_addr + offset;
+		*paddr = sc->fb_paddr + offset;
 		return (0);
 	}
 
@@ -805,6 +807,27 @@ bcmfb_mmap(video_adapter_t *adp, vm_ooffset_t offset, vm_paddr_t *paddr,
 static int
 bcmfb_ioctl(video_adapter_t *adp, u_long cmd, caddr_t data)
 {
+	struct video_adapter_softc *sc;
+	struct fbtype *fb;
+
+	sc = (struct video_adapter_softc *)adp;
+
+	switch (cmd) {
+	case FBIOGTYPE:
+		fb = (struct fbtype *)data;
+		fb->fb_type = FBTYPE_PCIMISC;
+		fb->fb_height = sc->height;
+		fb->fb_width = sc->width;
+		fb->fb_depth = sc->depth;
+		if (sc->depth <= 1 || sc->depth > 8)
+			fb->fb_cmsize = 0;
+		else
+			fb->fb_cmsize = 1 << sc->depth;
+		fb->fb_size = sc->fb_size;
+		break;
+	default:
+		return (fb_commonioctl(adp, cmd, data));
+	}
 
 	return (0);
 }
diff --git a/sys/arm/broadcom/bcm2835/bcm2835_machdep.c b/sys/arm/broadcom/bcm2835/bcm2835_machdep.c
index 7308c0f..7eff61c 100644
--- a/sys/arm/broadcom/bcm2835/bcm2835_machdep.c
+++ b/sys/arm/broadcom/bcm2835/bcm2835_machdep.c
@@ -82,11 +82,6 @@ initarm_late_init(void)
 	pcell_t cells[2];
 	int len;
 
-	/*
-	 * It seems there is no way to let syscons framework know
-	 * that framebuffer resolution has changed. So just try
-	 * to fetch data from FDT and go with defaults if failed
-	 */
 	system = OF_finddevice("/system");
 	if (system != 0) {
 		len = OF_getprop(system, "linux,serial", &cells, sizeof(cells));
diff --git a/sys/arm/conf/BEAGLEBONE b/sys/arm/conf/BEAGLEBONE
index 1481d92..6336c0c 100644
--- a/sys/arm/conf/BEAGLEBONE
+++ b/sys/arm/conf/BEAGLEBONE
@@ -37,6 +37,7 @@ options 	MSDOSFS			#MSDOS Filesystem
 options 	CD9660			#ISO 9660 Filesystem
 options 	PROCFS			#Process filesystem (requires PSEUDOFS)
 options 	PSEUDOFS		#Pseudo-filesystem framework
+options 	TMPFS			#Efficient memory filesystem
 options 	COMPAT_43		#Compatible with BSD 4.3 [KEEP THIS!]
 options 	SCSI_DELAY=5000		#Delay (in ms) before probing SCSI
 options 	KTRACE			#ktrace(1) support
@@ -61,9 +62,9 @@ options 	WITNESS_SKIPSPIN	#Don't run witness on spinlocks for speed
 #options 	DIAGNOSTIC
 
 # NFS support
-#options 	NFSCL
+options 	NFSCL
 #options	NFSD
-#options	NFSLOCKD
+options 	NFSLOCKD
 
 # Uncomment this for NFS root
 #options 	NFS_ROOT		#NFS usable as /, requires NFSCL
@@ -124,4 +125,3 @@ device		axe			# ASIX Electronics USB Ethernet
 options         FDT
 options         FDT_DTB_STATIC
 makeoptions     FDT_DTS_FILE=beaglebone.dts
-
diff --git a/sys/arm/conf/CUBIEBOARD b/sys/arm/conf/CUBIEBOARD
new file mode 100644
index 0000000..82f484b
--- /dev/null
+++ b/sys/arm/conf/CUBIEBOARD
@@ -0,0 +1,134 @@
+# CUBIEBOARD -- Custom configuration for the CUBIEBOARD ARM development
+# platform, check out http://www.cubieboard.org
+#
+# For more information on this file, please read the handbook section on
+# Kernel Configuration Files:
+#
+#    http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html
+#
+# The handbook is also available locally in /usr/share/doc/handbook
+# if you've installed the doc distribution, otherwise always see the
+# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the
+# latest information.
+#
+# An exhaustive list of options and more detailed explanations of the
+# device lines is also present in the ../../conf/NOTES and NOTES files.
+# If you are in doubt as to the purpose or necessity of a line, check first
+# in NOTES.
+#
+# $FreeBSD$
+
+ident		CUBIEBOARD
+
+include		"../allwinner/std.a10"
+
+makeoptions	MODULES_OVERRIDE=""
+makeoptions	WITHOUT_MODULES="ahc"
+
+options 	HZ=100
+options 	SCHED_4BSD		#4BSD scheduler
+options 	INET			#InterNETworking
+options 	INET6			#IPv6 communications protocols
+options 	FFS			#Berkeley Fast Filesystem
+options 	SOFTUPDATES		#Enable FFS soft updates support
+options 	UFS_ACL			#Support for access control lists
+options 	UFS_DIRHASH		#Improve performance on big directories
+options 	MSDOSFS			#MSDOS Filesystem
+options 	CD9660			#ISO 9660 Filesystem
+options 	PROCFS			#Process filesystem (requires PSEUDOFS)
+options 	PSEUDOFS		#Pseudo-filesystem framework
+options 	COMPAT_43		#Compatible with BSD 4.3 [KEEP THIS!]
+options 	SCSI_DELAY=5000		#Delay (in ms) before probing SCSI
+options 	KTRACE			#ktrace(1) support
+options 	SYSVSHM			#SYSV-style shared memory
+options 	SYSVMSG			#SYSV-style message queues
+options 	SYSVSEM			#SYSV-style semaphores
+options 	_KPOSIX_PRIORITY_SCHEDULING #Posix P1003_1B real-time extensions
+options 	KBD_INSTALL_CDEV	# install a CDEV entry in /dev
+options 	PREEMPTION
+options 	FREEBSD_BOOT_LOADER
+
+# Debugging
+makeoptions	DEBUG=-g		#Build kernel with gdb(1) debug symbols
+options 	BREAK_TO_DEBUGGER
+#options 	VERBOSE_SYSINIT		#Enable verbose sysinit messages
+options 	KDB
+options 	DDB			#Enable the kernel debugger
+options 	INVARIANTS		#Enable calls of extra sanity checking
+options 	INVARIANT_SUPPORT	#Extra sanity checks of internal structures, required by INVARIANTS
+options 	WITNESS			#Enable checks to detect deadlocks and cycles
+options 	WITNESS_SKIPSPIN	#Don't run witness on spinlocks for speed
+#options 	DIAGNOSTIC
+
+# NFS support
+#options 	NFSCL
+#options 	NFSSERVER		#Network Filesystem Server
+#options 	NFSCLIENT		#Network Filesystem Client
+
+# Uncomment this for NFS root
+#options 	NFS_ROOT		#NFS usable as /, requires NFSCLIENT
+#options 	BOOTP_NFSROOT
+#options 	BOOTP_COMPAT
+#options 	BOOTP
+#options 	BOOTP_NFSV3
+#options 	BOOTP_WIRED_TO=cpsw0
+
+# MMC/SD/SDIO card slot support
+#device		mmc			# mmc/sd bus
+#device		mmcsd			# mmc/sd flash cards
+
+# Boot device is 2nd slice on MMC/SD card
+options		ROOTDEVNAME=\"ufs:/dev/da0s2\"
+
+# ATA controllers
+#device		ahci		# AHCI-compatible SATA controllers
+#device		ata		# Legacy ATA/SATA controllers
+#options	ATA_CAM		# Handle legacy controllers with CAM
+#options	ATA_STATIC_ID	# Static device numbering
+
+# Console and misc
+#device		uart
+#device		uart_ns8250
+device		pty
+device		snp
+device		md
+device		random			# Entropy device
+
+# I2C support
+#device		iicbus
+#device		iic
+
+# GPIO
+#device		gpio
+
+device		scbus			# SCSI bus (required for SCSI)
+device		da			# Direct Access (disks)
+device		pass
+
+# USB support
+device		usb
+options 	USB_DEBUG
+#options 	USB_REQ_DEBUG
+#options 	USB_VERBOSE
+#device		uhci
+#device		ohci
+device		ehci
+
+device		umass
+
+# Ethernet
+device		loop
+device		ether
+device		mii
+device		smscphy
+#device		cpsw
+device		bpf
+
+# USB ethernet support, requires miibus
+device		miibus
+
+# Flattened Device Tree
+options		FDT
+options		FDT_DTB_STATIC
+makeoptions	FDT_DTS_FILE=cubieboard.dts
+
diff --git a/sys/arm/conf/DOCKSTAR b/sys/arm/conf/DOCKSTAR
index 3dd83fc..25c0121 100644
--- a/sys/arm/conf/DOCKSTAR
+++ b/sys/arm/conf/DOCKSTAR
@@ -5,7 +5,7 @@
 #
 
 ident		DOCKSTAR
-include		"../mv/kirkwood/std.sheevaplug"
+include		"../mv/kirkwood/std.db88f6xxx"
 
 options 	SOC_MV_KIRKWOOD
 makeoptions	MODULES_OVERRIDE=""
diff --git a/sys/arm/conf/DREAMPLUG-1001 b/sys/arm/conf/DREAMPLUG-1001
new file mode 100644
index 0000000..442cde0
--- /dev/null
+++ b/sys/arm/conf/DREAMPLUG-1001
@@ -0,0 +1,183 @@
+# Kernel config for GlobalScale Technologies DreamPlug version 1001.
+#
+# This is for units that are version 10, revision 01, with NOR SPI flash.
+# These units are identified with the number "1001" on the S/N label.
+#
+# For more information on this file, please read the handbook section on
+# Kernel Configuration Files:
+#
+#    http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html
+#
+# The handbook is also available locally in /usr/share/doc/handbook
+# if you've installed the doc distribution, otherwise always see the
+# FreeBSD World Wide Web server (http://www.FreeBSD.org/) for the
+# latest information.
+#
+# An exhaustive list of options and more detailed explanations of the
+# device lines is also present in the ../../conf/NOTES and NOTES files. 
+# If you are in doubt as to the purpose or necessity of a line, check first 
+# in NOTES.
+#
+# $FreeBSD$
+#
+
+ident		DREAMPLUG-1001
+
+include		"../mv/kirkwood/std.db88f6xxx"
+
+makeoptions	FDT_DTS_FILE=dreamplug-1001.dts
+
+makeoptions	MODULES_OVERRIDE=""
+
+options 	SOC_MV_KIRKWOOD
+
+options 	SCHED_4BSD		#4BSD scheduler
+options 	INET			#InterNETworking
+options 	INET6			#IPv6 communications protocols
+options 	SOFTUPDATES
+options 	CD9660			#ISO 9660 filesystem
+options 	FFS			#Berkeley Fast Filesystem
+options 	MSDOSFS			#MS DOS File System (FAT, FAT32)
+options 	NULLFS			#NULL filesystem
+options 	TMPFS			#Efficient memory filesystem
+options 	SYSVSHM			#SYSV-style shared memory
+options 	SYSVMSG			#SYSV-style message queues
+options 	SYSVSEM			#SYSV-style semaphores
+options 	_KPOSIX_PRIORITY_SCHEDULING #Posix P1003_1B real-time extensions
+options 	GEOM_ELI		# Disk encryption.
+options 	GEOM_LABEL		# Providers labelization.
+options 	GEOM_PART_GPT		# GPT partitioning
+
+# Flattened Device Tree
+device		fdt
+options 	FDT
+options 	FDT_DTB_STATIC
+
+# Misc pseudo devices
+device		bpf			#Required for DHCP
+device  	faith			#IPv6-to-IPv4 relaying (translation)
+device  	firmware		#firmware(9) required for USB wlan
+device  	gif			#IPv6 and IPv4 tunneling
+device		loop			#Network loopback
+device  	md			#Memory/malloc disk
+device		pty			#BSD-style compatibility pseudo ttys
+device		random			#Entropy device
+device  	tun			#Packet tunnel.
+device		ether			#Required for all ethernet devices
+device  	vlan			#802.1Q VLAN support
+device		wlan			#802.11 WLAN support
+
+# cam support for umass and ahci
+device		scbus
+device		pass
+device		da
+device 		cd
+
+# Serial ports
+device		uart
+
+# Networking
+device		mge			# Marvell Gigabit Ethernet controller
+device		mii
+device		e1000phy
+
+# USB
+options 	USB_HOST_ALIGN=32	# Align DMA to cacheline
+#options	USB_DEBUG       	# Compile in USB debug support
+device  	usb  			# Basic usb support			
+device  	ehci 			# USB host controller
+device  	umass			# Mass storage
+device  	uhid 			# Human-interface devices
+device  	rum  			# Ralink Technology RT2501USB wireless NICs
+device  	uath 			# Atheros AR5523 wireless NICs
+device  	ural 			# Ralink Technology RT2500USB wireless NICs
+device  	zyd  			# ZyDAS zb1211/zb1211b wireless NICs
+device  	urtw 			# Realtek RTL8187B/L USB
+device  	upgt 			# Conexant/Intersil PrismGT SoftMAC USB
+device  	u3g  			# USB-based 3G modems (Option, Huawei, Sierra)
+
+# I2C (TWSI)
+device		iic
+device		iicbus
+
+# SATA
+device		mvs
+device		ahci
+
+# Sound
+device 		sound
+device 		snd_uaudio
+
+#crypto
+device  	cesa			# Marvell security engine
+device  	crypto
+device  	cryptodev
+
+# IPSec
+device  	enc
+options  	IPSEC
+options  	IPSEC_NAT_T
+options  	TCP_SIGNATURE           #include support for RFC 2385
+
+# IPFW
+options  	IPFIREWALL
+options  	IPFIREWALL_DEFAULT_TO_ACCEPT
+options  	IPFIREWALL_VERBOSE
+options  	IPFIREWALL_VERBOSE_LIMIT=100
+options  	IPFIREWALL_NAT
+options  	LIBALIAS
+options  	DUMMYNET
+options  	IPDIVERT
+
+#PF 
+device  	pf
+device  	pflog
+device  	pfsync
+
+# ALTQ, required for PF
+options  	ALTQ      	      # Basic ALTQ support
+options  	ALTQ_CBQ  	      # Class Based Queueing
+options  	ALTQ_RED  	      # Random Early Detection
+options  	ALTQ_RIO  	      # RED In/Out
+options  	ALTQ_HFSC 	      # Hierarchical Packet Scheduler
+options  	ALTQ_CDNR 	      # Traffic conditioner
+options  	ALTQ_PRIQ 	      # Priority Queueing
+options  	ALTQ_NOPCC	      # Required if the TSC is unusable
+#options 	ALTQ_DEBUG
+
+# Debugging
+makeoptions 	DEBUG=-g		#Build kernel with gdb(1) debug symbols
+options 	BREAK_TO_DEBUGGER
+options 	ALT_BREAK_TO_DEBUGGER
+options 	DDB
+options 	KDB
+options 	DIAGNOSTIC
+options 	INVARIANTS		#Enable calls of extra sanity checking
+options 	INVARIANT_SUPPORT	#Extra sanity checks of internal structures, required by INVARIANTS
+#options 	WITNESS			#Enable checks to detect deadlocks and cycles
+#options 	WITNESS_SKIPSPIN	#Don't run witness on spinlocks for speed
+#options 	WITNESS_KDB
+
+# Enable these options for nfs root configured via BOOTP.
+options 	NFSCL			#Network Filesystem Client
+options 	NFSLOCKD		#Network Lock Manager
+#options 	NFS_ROOT		#NFS usable as /, requires NFSCLIENT
+#options 	BOOTP
+#options 	BOOTP_NFSROOT
+#options 	BOOTP_NFSV3
+#options 	BOOTP_WIRED_TO=mge0
+
+# If not using BOOTP, use something like one of these...
+#options 	ROOTDEVNAME=\"ufs:/dev/da1a\"
+options 	ROOTDEVNAME=\"ufs:/dev/da1s1a\"
+#options 	ROOTDEVNAME=\"ufs:/dev/da1p10\"
+#options 	ROOTDEVNAME=\"nfs:192.168.0.254/dreamplug\"
+
+# To use this configuration with the (rare) model 1001N (nand flash),
+# create a kernel config file that looks like this:
+#
+# include DREAMPLUG-1001
+# nomakeoptions FDT_DTS_FILE
+# makeoptions	FDT_DTS_FILE=dreamplug-1001N.dts
+# device 	nand
+
diff --git a/sys/arm/conf/NOTES b/sys/arm/conf/NOTES
index 1d4f897..fa36275 100644
--- a/sys/arm/conf/NOTES
+++ b/sys/arm/conf/NOTES
@@ -19,7 +19,6 @@ files	"../econa/files.econa"
 files	"../mv/files.mv"
 files	"../mv/discovery/files.db78xxx"
 files	"../mv/kirkwood/files.kirkwood"
-files	"../mv/kirkwood/files.sheevaplug"
 files	"../mv/orion/files.db88f5xxx"
 files	"../mv/orion/files.ts7800"
 files	"../s3c2xx0/files.s3c2xx0"
diff --git a/sys/arm/conf/SHEEVAPLUG b/sys/arm/conf/SHEEVAPLUG
index b7de8f6..c6bd901 100644
--- a/sys/arm/conf/SHEEVAPLUG
+++ b/sys/arm/conf/SHEEVAPLUG
@@ -5,7 +5,7 @@
 #
 
 ident		SHEEVAPLUG
-include		"../mv/kirkwood/std.sheevaplug"
+include		"../mv/kirkwood/std.db88f6xxx"
 
 options 	SOC_MV_KIRKWOOD
 makeoptions	MODULES_OVERRIDE=""
diff --git a/sys/arm/include/atomic.h b/sys/arm/include/atomic.h
index 4862453..82b7d1b 100644
--- a/sys/arm/include/atomic.h
+++ b/sys/arm/include/atomic.h
@@ -677,6 +677,17 @@ atomic_readandclear_32(volatile u_int32_t *p)
 #define atomic_store_rel_long	atomic_store_long
 #define atomic_load_acq_32	atomic_load_32
 #define atomic_load_acq_long	atomic_load_long
+#define atomic_add_acq_long		atomic_add_long
+#define atomic_add_rel_long		atomic_add_long
+#define atomic_subtract_acq_long	atomic_subtract_long
+#define atomic_subtract_rel_long	atomic_subtract_long
+#define atomic_clear_acq_long		atomic_clear_long
+#define atomic_clear_rel_long		atomic_clear_long
+#define atomic_set_acq_long		atomic_set_long
+#define atomic_set_rel_long		atomic_set_long
+#define atomic_cmpset_acq_long		atomic_cmpset_long
+#define atomic_cmpset_rel_long		atomic_cmpset_long
+#define atomic_load_acq_long		atomic_load_long
 #undef __with_interrupts_disabled
 
 static __inline void
@@ -758,25 +769,13 @@ atomic_store_long(volatile u_long *dst, u_long src)
 	*dst = src;
 }
 
-#define atomic_add_acq_long		atomic_add_long
-#define atomic_add_rel_long		atomic_add_long
-#define atomic_subtract_acq_long	atomic_subtract_long
-#define atomic_subtract_rel_long	atomic_subtract_long
-#define atomic_clear_acq_long		atomic_clear_long
-#define atomic_clear_rel_long		atomic_clear_long
-#define atomic_set_acq_long		atomic_set_long
-#define atomic_set_rel_long		atomic_set_long
-#define atomic_cmpset_acq_long		atomic_cmpset_long
-#define atomic_cmpset_rel_long		atomic_cmpset_long
-#define atomic_load_acq_long		atomic_load_long
-
 #define atomic_clear_ptr		atomic_clear_32
 #define atomic_set_ptr			atomic_set_32
 #define atomic_cmpset_ptr		atomic_cmpset_32
 #define atomic_cmpset_rel_ptr		atomic_cmpset_rel_32
 #define atomic_cmpset_acq_ptr		atomic_cmpset_acq_32
 #define atomic_store_ptr		atomic_store_32
-#define atomic_store_rel_ptr		atomic_store_ptr
+#define atomic_store_rel_ptr		atomic_store_rel_32
 
 #define atomic_add_int			atomic_add_32
 #define atomic_add_acq_int		atomic_add_acq_32
diff --git a/sys/arm/include/intr.h b/sys/arm/include/intr.h
index 45e36e9..9392969 100644
--- a/sys/arm/include/intr.h
+++ b/sys/arm/include/intr.h
@@ -70,6 +70,7 @@
 int arm_get_next_irq(int);
 void arm_mask_irq(uintptr_t);
 void arm_unmask_irq(uintptr_t);
+void arm_intrnames_init(void);
 void arm_setup_irqhandler(const char *, int (*)(void*), void (*)(void*),
     void *, int, int, void **);
 int arm_remove_irqhandler(int, void *);
diff --git a/sys/arm/include/proc.h b/sys/arm/include/proc.h
index c1bce00..597a5ad 100644
--- a/sys/arm/include/proc.h
+++ b/sys/arm/include/proc.h
@@ -60,7 +60,11 @@ struct mdproc {
 	void	*md_sigtramp;
 };
 
+#ifdef __ARM_EABI__
+#define	KINFO_PROC_SIZE 816
+#else
 #define	KINFO_PROC_SIZE 792
+#endif
 
 #define MAXARGS	8
 struct syscall_args {
@@ -69,7 +73,9 @@ struct syscall_args {
 	register_t args[MAXARGS];
 	int narg;
 	u_int nap;
+#ifndef __ARM_EABI__
 	u_int32_t insn;
+#endif
 };
 
 #endif /* !_MACHINE_PROC_H_ */
diff --git a/sys/arm/include/vmparam.h b/sys/arm/include/vmparam.h
index eac46cc..267ba6e 100644
--- a/sys/arm/include/vmparam.h
+++ b/sys/arm/include/vmparam.h
@@ -134,12 +134,15 @@
 #endif
 
 #define VM_MAX_KERNEL_ADDRESS	0xffffffff
+
 /*
  * Virtual size (bytes) for various kernel submaps.
  */
-
 #ifndef VM_KMEM_SIZE
-#define VM_KMEM_SIZE            (12*1024*1024)
+#define VM_KMEM_SIZE		(12*1024*1024)
+#endif
+#ifndef VM_KMEM_SIZE_SCALE
+#define VM_KMEM_SIZE_SCALE	(2)
 #endif
 
 #define MAXTSIZ 	(16*1024*1024)
diff --git a/sys/arm/mv/kirkwood/files.sheevaplug b/sys/arm/mv/kirkwood/files.sheevaplug
deleted file mode 100644
index f56c2ab..0000000
--- a/sys/arm/mv/kirkwood/files.sheevaplug
+++ /dev/null
@@ -1,4 +0,0 @@
-# $FreeBSD$
-
-include "arm/mv/kirkwood/files.kirkwood"
-arm/mv/kirkwood/sheevaplug.c	standard
diff --git a/sys/arm/mv/kirkwood/std.sheevaplug b/sys/arm/mv/kirkwood/std.sheevaplug
deleted file mode 100644
index 44c977e..0000000
--- a/sys/arm/mv/kirkwood/std.sheevaplug
+++ /dev/null
@@ -1,5 +0,0 @@
-# $FreeBSD$
-
-include	"../mv/std.mv"
-include "../mv/kirkwood/std.kirkwood"
-files	"../mv/kirkwood/files.sheevaplug"
diff --git a/sys/arm/mv/mv_machdep.c b/sys/arm/mv/mv_machdep.c
index ae74fa3..fba222d 100644
--- a/sys/arm/mv/mv_machdep.c
+++ b/sys/arm/mv/mv_machdep.c
@@ -279,6 +279,25 @@ out:
 }
 
 /*
+ * Supply a default do-nothing implementation of fdt_pci_devmap() via a weak
+ * alias.  Many Marvell platforms don't support a PCI interface, but to support
+ * those that do, we end up with a reference to this function below, in
+ * platform_devmap_init().  If "device pci" appears in the kernel config, the
+ * real implementation of this function in dev/fdt/fdt_pci.c overrides the weak
+ * alias defined here.
+ */
+int mv_default_fdt_pci_devmap(phandle_t node, struct pmap_devmap *devmap,
+    vm_offset_t io_va, vm_offset_t mem_va);
+int
+mv_default_fdt_pci_devmap(phandle_t node, struct pmap_devmap *devmap,
+    vm_offset_t io_va, vm_offset_t mem_va)
+{
+
+	return (0);
+}
+__weak_reference(mv_default_fdt_pci_devmap, fdt_pci_devmap);
+
+/*
  * XXX: When device entry in devmap has pd_size smaller than section size,
  * system will freeze during initialization
  */
diff --git a/sys/arm/ti/am335x/am335x_scm_padconf.c b/sys/arm/ti/am335x/am335x_scm_padconf.c
index 4ed63a3..6bb89f5 100644
--- a/sys/arm/ti/am335x/am335x_scm_padconf.c
+++ b/sys/arm/ti/am335x/am335x_scm_padconf.c
@@ -86,127 +86,117 @@ const struct ti_scm_padstate ti_padstate_devmap[] = {
 };
 
 const struct ti_scm_padconf ti_padconf_devmap[] = {
-	_PIN(0x800, "GPMC_AD0",		 32, 7,"gpmc_ad0", "mmc1_dat0", NULL, NULL, NULL, NULL, NULL, "gpio1_0"),
-	_PIN(0x804, "GPMC_AD1",		 33, 7,"gpmc_ad1", "mmc1_dat1", NULL, NULL, NULL, NULL, NULL, "gpio1_1"),
-	_PIN(0x808, "GPMC_AD2",		 34, 7,"gpmc_ad2", "mmc1_dat2", NULL, NULL, NULL, NULL, NULL, "gpio1_2"),
-	_PIN(0x80C, "GPMC_AD3",		 35, 7,"gpmc_ad3", "mmc1_dat3", NULL, NULL, NULL, NULL, NULL, "gpio1_3"),
-	_PIN(0x810, "GPMC_AD4",		 36, 7,"gpmc_ad4", "mmc1_dat4", NULL, NULL, NULL, NULL, NULL, "gpio1_4"),
-	_PIN(0x814, "GPMC_AD5",		 37, 7,"gpmc_ad5", "mmc1_dat5", NULL, NULL, NULL, NULL, NULL, "gpio1_5"),
-	_PIN(0x818, "GPMC_AD6",		 38, 7,"gpmc_ad6", "mmc1_dat6", NULL, NULL, NULL, NULL, NULL, "gpio1_6"),
-	_PIN(0x81C, "GPMC_AD7",		 39, 7,"gpmc_ad7", "mmc1_dat7", NULL, NULL, NULL, NULL, NULL, "gpio1_7"),
-#if 0 /* Incomplete Entries - fill with data from table 2-7 in datasheet */
-	_PIN(0x820, "gpmc_ad8",		0, 0, "gpmc_ad8", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x824, "gpmc_ad9",		0, 0, "gpmc_ad9", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x828, "gpmc_ad10",	0, 0, "gpmc_ad10", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x82C, "gpmc_ad11",	0, 0, "gpmc_ad11", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x830, "gpmc_ad12",	0, 0, "gpmc_ad12", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x834, "gpmc_ad13",	0, 0, "gpmc_ad13", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x838, "gpmc_ad14",	0, 0, "gpmc_ad14", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x83C, "gpmc_ad15",	0, 0, "gpmc_ad15", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x840, "gpmc_a0",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x844, "gpmc_a1",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x848, "gpmc_a2",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x84C, "gpmc_a3",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x850, "gpmc_a4",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-#endif
-	_PIN(0x854, "GPMC_A5",		 53, 7, "gpmc_a5", "gmii2_txd0", "rgmii2_td0", "rmii2_txd0", "gpmc_a21", "pr1_mii1_rxd3", "eQEP1B_in", "gpio1_21"),
-	_PIN(0x858, "GPMC_A6",		 54, 7, "gpmc_a6", "gmii2_txclk", "rgmii2_tclk", "mmc2_dat4", "gpmc_a22", "pr1_mii1_rxd2", "eQEP1_index", "gpio1_22"),
-	_PIN(0x85C, "GPMC_A7",		 55, 7, "gpmc_a7", "gmii2_rxclk", "rgmii2_rclk", "mmc2_dat5", "gpmc_a23", "pr1_mii1_rxd1", "eQEP1_strobe", "gpio1_23"),
-	_PIN(0x860, "GPMC_A8",		 56, 7, "gpmc_a8", "gmii2_rxd3", "rgmii2_rd3", "mmc2_dat6", "gpmc_a24", "pr1_mii1_rxd0", "mcasp0_aclkx", "gpio1_24"),
-#if 0
-	_PIN(0x864, "gpmc_a9",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x868, "gpmc_a10",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x86C, "gpmc_a11",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x870, "gpmc_wait0",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x874, "gpmc_wpn",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x878, "gpmc_be1n",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x87c, "gpmc_csn0",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x880, "gpmc_csn1",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x884, "gpmc_csn2",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x888, "gpmc_csn3",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x88c, "gpmc_clk",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x890, "gpmc_advn_ale",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x894, "gpmc_oen_ren",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x898, "gpmc_wen",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x89c, "gpmc_be0n_cle",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8a0, "lcd_data0",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8a4, "lcd_data1",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8a8, "lcd_data2",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8ac, "lcd_data3",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8b0, "lcd_data4",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8b4, "lcd_data5",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8b8, "lcd_data6",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8bc, "lcd_data7",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8c0, "lcd_data8",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8c4, "lcd_data9",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8c8, "lcd_data10",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8cc, "lcd_data11",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8d0, "lcd_data12",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8d4, "lcd_data13",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8d8, "lcd_data14",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8dc, "lcd_data15",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8e0, "lcd_vsync",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8e4, "lcd_hsync",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8e8, "lcd_pclk",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x8ec, "lcd_ac_bias_en",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-#endif
-	_PIN(0x8f0, "MMC0_DAT3",	 90, 7, "mmc0_dat3", "gpmc_a20", "uart4_ctsn", "timer5", "uart1_dcdn", "pr1_pru0_pru_r30_8", "pr1_pru0_pru_r31_8", "gpio2_26"),
-	_PIN(0x8f4, "MMC0_DAT2",	 91, 7, "mmc0_dat2", "gpmc_a21", "uart4_rtsn", "timer6", "uart1_dsrn", "pr1_pru0_pru_r30_9", "pr1_pru0_pru_r31_9", "gpio2_27"),
-	_PIN(0x8f8, "MMC0_DAT1",	 92, 7, "mmc0_dat1", "gpmc_a22", "uart5_ctsn", "uart3_rxd", "uart1_dtrn", "pr1_pru0_pru_r30_10", "pr1_pru0_pru_r31_10", "gpio2_28"),
-	_PIN(0x8fc, "MMC0_DAT0",	 93, 7, "mmc0_dat0", "gpmc_a23", "uart5_rtsn", "uart3_txd", "uart1_rin", "pr1_pru0_pru_r30_11", "pr1_pru0_pru_r31_11", "gpio2_29"),
-	_PIN(0x900, "MMC0_CLK",		 94, 7, "mmc0_clk", "gpmc_a24", "uart3_ctsn", "uart2_rxd", "dcan1_tx", "pr1_pru0_pru_r30_12", "pr1_pru0_pru_r31_12", "gpio2_30"),
-	_PIN(0x904, "MMC0_CMD",		 95, 7, "mmc0_cmd", "gpmc_a25", "uart3_rtsn", "uart2_txd", "dcan1_rx", "pr1_pru0_pru_r30_13", "pr1_pru0_pru_r31_13", "gpio2_31"),
-	_PIN(0x908, "MII1_COL",		 96, 7, "gmii1_col", "rmii2_refclk", "spi1_sclk", "uart5_rxd", "mcasp1_axr2", "mmc2_dat3", "mcasp0_axr2", "gpio3_0"),
-	_PIN(0x90c, "MII1_CRS",		 97, 7, "gmii1_crs", "rmii1_crs_dv", "spi1_d0", "I2C1_SDA", "mcasp1_aclkx", "uart5_ctsn", "uart2_rxd", "gpio3_1"),
-	_PIN(0x910, "MII1_RX_ER",	 98, 7, "gmii1_rxerr", "rmii1_rxerr", "spi1_d1", "I2C1_SCL", "mcasp1_fsx", "uart5_rtsn", "uart2_txd", "gpio3_2"),
-	_PIN(0x914, "MII1_TX_EN",	 99, 7, "gmii1_txen", "rmii1_txen", "rgmii1_tctl", "timer4", "mcasp1_axr0", "eQEP0_index", "mmc2_cmd", "gpio3_3"),
+	_PIN(0x800, "GPMC_AD0",		32, 7,"gpmc_ad0", "mmc1_dat0", NULL, NULL, NULL, NULL, NULL, "gpio1_0"),
+	_PIN(0x804, "GPMC_AD1",		33, 7,"gpmc_ad1", "mmc1_dat1", NULL, NULL, NULL, NULL, NULL, "gpio1_1"),
+	_PIN(0x808, "GPMC_AD2",		34, 7,"gpmc_ad2", "mmc1_dat2", NULL, NULL, NULL, NULL, NULL, "gpio1_2"),
+	_PIN(0x80C, "GPMC_AD3",		35, 7,"gpmc_ad3", "mmc1_dat3", NULL, NULL, NULL, NULL, NULL, "gpio1_3"),
+	_PIN(0x810, "GPMC_AD4",		36, 7,"gpmc_ad4", "mmc1_dat4", NULL, NULL, NULL, NULL, NULL, "gpio1_4"),
+	_PIN(0x814, "GPMC_AD5",		37, 7,"gpmc_ad5", "mmc1_dat5", NULL, NULL, NULL, NULL, NULL, "gpio1_5"),
+	_PIN(0x818, "GPMC_AD6",		38, 7,"gpmc_ad6", "mmc1_dat6", NULL, NULL, NULL, NULL, NULL, "gpio1_6"),
+	_PIN(0x81C, "GPMC_AD7",		39, 7,"gpmc_ad7", "mmc1_dat7", NULL, NULL, NULL, NULL, NULL, "gpio1_7"),
+	_PIN(0x820, "GPMC_AD8",		22, 7, "gpmc_ad8", "lcd_data23", "mmc1_dat0", "mmc2_dat4", "ehrpwm2A", NULL, NULL, "gpio0_22"),
+	_PIN(0x824, "GPMC_AD9",		23, 7, "gpmc_ad9", "lcd_data22", "mmc1_dat1", "mmc2_dat5", "ehrpwm2B", NULL, NULL, "gpio0_23"),
+	_PIN(0x828, "GPMC_AD10",	26, 7, "gpmc_ad10", "lcd_data21", "mmc1_dat2", "mmc2_dat6", "ehrpwm2_tripzone_in", NULL, NULL, "gpio0_26"),
+	_PIN(0x82C, "GPMC_AD11",	27, 7, "gpmc_ad11", "lcd_data20", "mmc1_dat3", "mmc2_dat7", "ehrpwm0_synco", NULL, NULL, "gpio0_27"),
+	_PIN(0x830, "GPMC_AD12",	44, 7, "gpmc_ad12", "lcd_data19", "mmc1_dat4", "mmc2_dat0", "eQEP2A_in", "pr1_mii0_txd2", "pr1_pru0_pru_r30_14", "gpio1_12"),
+	_PIN(0x834, "GPMC_AD13",	45, 7, "gpmc_ad13", "lcd_data18", "mmc1_dat5", "mmc2_dat1", "eQEP2B_in", "pr1_mii0_txd1", "pr1_pru0_pru_r30_15", "gpio1_13"),
+	_PIN(0x838, "GPMC_AD14",	46, 7, "gpmc_ad14", "lcd_data17", "mmc1_dat6", "mmc2_dat2", "eQEP2_index", "pr1_mii0_txd0", "pr1_pru0_pru_r31_14", "gpio1_14"),
+	_PIN(0x83C, "GPMC_AD15",	47, 7, "gpmc_ad15", "lcd_data16", "mmc1_dat7", "mmc2_dat3", "eQEP2_strobe", "pr1_ecap0_ecap_capin_apwm_o", "pr1_pru0_pru_r31_15", "gpio1_15"),
+	_PIN(0x840, "GPMC_A0",		48, 7, "gpmc_a0", "gmii2_txen", "rgmii2_tctl", "rmii2_txen", "gpmc_a16", "pr1_mii_mt1_clk", "ehrpwm1_tripzone_input", "gpio1_16"),
+	_PIN(0x844, "GPMC_A1",		49, 7, "gpmc_a1", "gmii2_rxdv", "rgmii2_rctl", "mmc2_dat0", "gpmc_a17", "pr1_mii1_txd3", "ehrpwm0_synco", "gpio1_17"),
+	_PIN(0x848, "GPMC_A2",		50, 7, "gpmc_a2", "gmii2_txd3", "rgmii2_td3", "mmc2_dat1", "gpmc_a18", "pr1_mii1_txd2", "ehrpwm1A", "gpio1_18"),
+	_PIN(0x84C, "GPMC_A3",		51, 7, "gpmc_a3", "gmii2_txd2", "rgmii2_td2", "mmc2_dat2", "gpmc_a19", "pr1_mii1_txd1", "ehrpwm1B", "gpio1_19"),
+	_PIN(0x850, "GPMC_A4",		52, 7, "gpmc_a4", "gmii2_txd1", "rgmii2_td1", "rmii2_tdx1", "gpmc_a20", "pr1_mii1_txd0", "eQEP1A_in", "gpio1_20"),
+	_PIN(0x854, "GPMC_A5",		53, 7, "gpmc_a5", "gmii2_txd0", "rgmii2_td0", "rmii2_txd0", "gpmc_a21", "pr1_mii1_rxd3", "eQEP1B_in", "gpio1_21"),
+	_PIN(0x858, "GPMC_A6",		54, 7, "gpmc_a6", "gmii2_txclk", "rgmii2_tclk", "mmc2_dat4", "gpmc_a22", "pr1_mii1_rxd2", "eQEP1_index", "gpio1_22"),
+	_PIN(0x85C, "GPMC_A7",		55, 7, "gpmc_a7", "gmii2_rxclk", "rgmii2_rclk", "mmc2_dat5", "gpmc_a23", "pr1_mii1_rxd1", "eQEP1_strobe", "gpio1_23"),
+	_PIN(0x860, "GPMC_A8",		56, 7, "gpmc_a8", "gmii2_rxd3", "rgmii2_rd3", "mmc2_dat6", "gpmc_a24", "pr1_mii1_rxd0", "mcasp0_aclkx", "gpio1_24"),
+	_PIN(0x864, "GPMC_A9",		57, 7, "gmpc_a9", "gmii2_rxd2", "rgmii2_rd2", "mmc2_dat7 / rmii2_crs_dv", "gpmc_a25", "pr1_mii_mr1_clk", "mcasp0_fsx", "gpio1_25"),
+	_PIN(0x868, "GPMC_A10",		58, 7, "gmpc_a10", "gmii2_rxd1", "rgmii2_rd1", "rmii2_rxd1", "gpmc_a26", "pr1_mii1_rxdv", "mcasp0_arx0", "gpio1_26"),
+	_PIN(0x86C, "GPMC_A11",		59, 7, "gmpc_a11", "gmii2_rxd0", "rgmii2_rd0", "rmii2_rxd0", "gpmc_a27", "pr1_mii1_rxer", "mcasp0_axr1", "gpio1_27"),
+	_PIN(0x870, "GPMC_WAIT0",	30, 7, "gpmc_wait0", "gmii2_crs", "gpmc_csn4", "rmii2_crs_dv", "mmc1_sdcd", "pr1_mii1_col", "uart4_rxd", "gpio0_30"),
+	_PIN(0x874, "GPMC_WPn",		31, 7, "gpmc_wpn", "gmii2_rxerr", "gpmc_csn5", "rmii2_rxerr", "mmc2_sdcd", "pr1_mii1_txen", "uart4_txd", "gpio0_31"),
+	_PIN(0x878, "GPMC_BEn1",	60, 7, "gpmc_be1n", "gmii2_col", "gmpc_csn6","mmc2_dat3", "gpmc_dir", "pr1_mii1_rxlink", "mcasp0_aclkr", "gpio1_28"),
+	_PIN(0x87c, "GPMC_CSn0",	61, 7, "gpmc_csn0", NULL, NULL, NULL, NULL, NULL, NULL, "gpio1_29"),
+	_PIN(0x880, "GPMC_CSn1",	62, 7, "gpmc_csn1", "gpmc_clk", "mmc1_clk", "pr1_edio_data_in6", "pr1_edio_data_out6", "pr1_pru1_pru_r30_12", "pr1_pru1_pru_r31_12", "gpio1_30"),
+	_PIN(0x884, "GPMC_CSn2",	63, 7, "gpmc_csn2", "gpmc_be1n", "mmc1_cmd", "pr1_edio_data_in7", "pr1_edio_data_out7", "pr1_pru1_pru_r30_13", "pr1_pru1_pru_r31_13", "gpio1_31"),
+	_PIN(0x888, "GPMC_CSn3",	64, 7, "gpmc_csn3", "gpmc_a3", "rmii2_crs_dv", "mmc2_cmd", "pr1_mii0_crs", "pr1_mdio_data", "EMU4", "gpio2_0"),
+	_PIN(0x88c, "GPMC_CLK",		65, 7, "gpmc_clk", "lcd_memory_clk", "gpmc_wait1", "mmc2_clk", "pr1_mii1_crs", "pr1_mdio_mdclk", "mcasp0_fsr", "gpio2_1"),
+	_PIN(0x890, "GPMC_ADVn_ALE",	66, 7, "gpmc_advn_ale", NULL, "timer4", NULL, NULL, NULL, NULL, "gpio2_2"),
+	_PIN(0x894, "GPMC_OEn_REn",	67, 7, "gpmc_oen_ren", NULL, "timer7", NULL, NULL, NULL, NULL, "gpio2_3"),
+	_PIN(0x898, "GPMC_WEn",		68, 7, "gpmc_wen", NULL, "timer6", NULL, NULL, NULL, NULL, "gpio2_4"),
+	_PIN(0x89c, "GPMC_BEn0_CLE",	67, 7, "gpmc_ben0_cle", NULL, "timer5", NULL, NULL, NULL, NULL, "gpio2_5"),
+	_PIN(0x8a0, "LCD_DATA0",	68, 7, "lcd_data0", "gpmc_a0", "pr1_mii_mt0_clk", "ehrpwm2A", NULL, "pr1_pru1_pru_r30_0", "pr1_pru1_pru_r31_0", "gpio2_6"),
+	_PIN(0x8a4, "LCD_DATA1",	69, 7, "lcd_data1", "gpmc_a1", "pr1_mii0_txen", "ehrpwm2B", NULL, "pr1_pru1_pru_r30_1", "pr1_pru1_pru_r31_1", "gpio2_7"),
+	_PIN(0x8a8, "LCD_DATA2",	70, 7, "lcd_data2", "gpmc_a2", "pr1_mii0_txd3", "ehrpwm2_tripzone_input", NULL, "pr1_pru1_pru_r30_2", "pr1_pru1_pru_r31_2", "gpio2_8"),
+	_PIN(0x8ac, "LCD_DATA3",	71, 7, "lcd_data3", "gpmc_a3", "pr1_mii0_txd2", "ehrpwm0_synco", NULL, "pr1_pru1_pru_r30_3", "pr1_pru1_pru_r31_3", "gpio2_9"),
+	_PIN(0x8b0, "LCD_DATA4",	72, 7, "lcd_data4", "gpmc_a4", "pr1_mii0_txd1", "eQEP2A_in", NULL, "pr1_pru1_pru_r30_4", "pr1_pru1_pru_r31_4", "gpio2_10"),
+	_PIN(0x8b4, "LCD_DATA5",	73, 7, "lcd_data5", "gpmc_a5", "pr1_mii0_txd0", "eQEP2B_in", NULL, "pr1_pru1_pru_r30_5", "pr1_pru1_pru_r31_5", "gpio2_11"),
+	_PIN(0x8b8, "LCD_DATA6",	74, 7, "lcd_data6", "gpmc_a6", "pr1_edio_data_in6", "eQEP2_index", "pr1_edio_data_out6", "pr1_pru1_pru_r30_6", "pr1_pru1_pru_r31_6", "gpio2_12"),
+	_PIN(0x8bc, "LCD_DATA7",	75, 7, "lcd_data7", "gpmc_a7", "pr1_edio_data_in7", "eQEP2_strobe", "pr1_edio_data_out7", "pr1_pru1_pru_r30_7", "pr1_pru1_pru_r31_7", "gpio2_13"),
+	_PIN(0x8c0, "LCD_DATA8",	76, 7, "lcd_data8", "gpmc_a12", "ehrpwm1_tripzone_input", "mcasp0_aclkx", "uart5_txd", "pr1_mii0_rxd3", "uart2_ctsn", "gpio2_14"),
+	_PIN(0x8c4, "LCD_DATA9",	76, 7, "lcd_data9", "gpmc_a13", "ehrpwm0_synco", "mcasp0_fsx", "uart5_rxd", "pr1_mii0_rxd2", "uart2_rtsn", "gpio2_15"),
+	_PIN(0x8c8, "LCD_DATA10",	77, 7, "lcd_data10", "gpmc_a14", "ehrpwm1A", "mcasp0_axr0", NULL, "pr1_mii0_rxd1", "uart3_ctsn", "gpio2_16"),
+	_PIN(0x8cc, "LCD_DATA11",	78, 7, "lcd_data11", "gpmc_a15", "ehrpwm1B", "mcasp0_ahclkr", "mcasp0_axr2", "pr1_mii0_rxd0", "uart3_rtsn", "gpio2_17"),
+	_PIN(0x8d0, "LCD_DATA12",	8, 7, "lcd_data12", "gpmc_a16", "eQEP1A_in", "mcasp0_aclkr", "mcasp0_axr2", "pr1_mii0_rxlink", "uart4_ctsn", "gpio0_8"),
+	_PIN(0x8d4, "LCD_DATA13",	9, 7, "lcd_data13", "gpmc_a17", "eQEP1B_in", "mcasp0_fsr", "mcasp0_axr3", "pr1_mii0_rxer", "uart4_rtsn", "gpio0_9"),
+	_PIN(0x8d8, "LCD_DATA14",	10, 7, "lcd_data14", "gpmc_a18", "eQEP1_index", "mcasp0_axr1", "uart5_rxd", "pr1_mii_mr0_clk", "uart5_ctsn", "gpio0_10"),
+	_PIN(0x8dc, "LCD_DATA15",	11, 7, "lcd_data15", "gpmc_a19", "eQEP1_strobe", "mcasp0_ahclkx", "mcasp0_axr3", "pr1_mii0_rxdv", "uart5_rtsn", "gpio0_11"),
+	_PIN(0x8e0, "LCD_VSYNC",	86, 7, "lcd_vsync", "gpmc_a8", "gpmc_a1", "pr1_edio_data_in2", "pr1_edio_data_out2", "pr1_pru1_pru_r30_8", "pr1_pru1_pru_r31_8", "gpio2_22"),
+	_PIN(0x8e4, "LCD_HSYNC",	87, 7, "lcd_hsync", "gmpc_a9", "gpmc_a2", "pr1_edio_data_in3", "pr1_edio_data_out3", "pr1_pru1_pru_r30_9", "pr1_pru1_pru_r31_9", "gpio2_23"),
+	_PIN(0x8e8, "LCD_PCLK",		88, 7, "lcd_pclk", "gpmc_a10", "pr1_mii0_crs", "pr1_edio_data_in4", "pr1_edio_data_out4", "pr1_pru1_pru_r30_10", "pr1_pru1_pru_r31_10", "gpio2_24"),
+	_PIN(0x8ec, "LCD_AC_BIAS_EN",	89, 7, "lcd_ac_bias_en", "gpmc_a11", "pr1_mii1_crs", "pr1_edio_data_in5", "pr1_edio_data_out5", "pr1_pru1_pru_r30_11", "pr1_pru1_pru_r31_11", "gpio2_25"),
+	_PIN(0x8f0, "MMC0_DAT3",	90, 7, "mmc0_dat3", "gpmc_a20", "uart4_ctsn", "timer5", "uart1_dcdn", "pr1_pru0_pru_r30_8", "pr1_pru0_pru_r31_8", "gpio2_26"),
+	_PIN(0x8f4, "MMC0_DAT2",	91, 7, "mmc0_dat2", "gpmc_a21", "uart4_rtsn", "timer6", "uart1_dsrn", "pr1_pru0_pru_r30_9", "pr1_pru0_pru_r31_9", "gpio2_27"),
+	_PIN(0x8f8, "MMC0_DAT1",	92, 7, "mmc0_dat1", "gpmc_a22", "uart5_ctsn", "uart3_rxd", "uart1_dtrn", "pr1_pru0_pru_r30_10", "pr1_pru0_pru_r31_10", "gpio2_28"),
+	_PIN(0x8fc, "MMC0_DAT0",	93, 7, "mmc0_dat0", "gpmc_a23", "uart5_rtsn", "uart3_txd", "uart1_rin", "pr1_pru0_pru_r30_11", "pr1_pru0_pru_r31_11", "gpio2_29"),
+	_PIN(0x900, "MMC0_CLK",		94, 7, "mmc0_clk", "gpmc_a24", "uart3_ctsn", "uart2_rxd", "dcan1_tx", "pr1_pru0_pru_r30_12", "pr1_pru0_pru_r31_12", "gpio2_30"),
+	_PIN(0x904, "MMC0_CMD",		95, 7, "mmc0_cmd", "gpmc_a25", "uart3_rtsn", "uart2_txd", "dcan1_rx", "pr1_pru0_pru_r30_13", "pr1_pru0_pru_r31_13", "gpio2_31"),
+	_PIN(0x908, "MII1_COL",		96, 7, "gmii1_col", "rmii2_refclk", "spi1_sclk", "uart5_rxd", "mcasp1_axr2", "mmc2_dat3", "mcasp0_axr2", "gpio3_0"),
+	_PIN(0x90c, "MII1_CRS",		97, 7, "gmii1_crs", "rmii1_crs_dv", "spi1_d0", "I2C1_SDA", "mcasp1_aclkx", "uart5_ctsn", "uart2_rxd", "gpio3_1"),
+	_PIN(0x910, "MII1_RX_ER",	98, 7, "gmii1_rxerr", "rmii1_rxerr", "spi1_d1", "I2C1_SCL", "mcasp1_fsx", "uart5_rtsn", "uart2_txd", "gpio3_2"),
+	_PIN(0x914, "MII1_TX_EN",	99, 7, "gmii1_txen", "rmii1_txen", "rgmii1_tctl", "timer4", "mcasp1_axr0", "eQEP0_index", "mmc2_cmd", "gpio3_3"),
 	_PIN(0x918, "MII1_RX_DV",	100, 7, "gmii1_rxdv", "cd_memory_clk", "rgmii1_rctl", "uart5_txd", "mcasp1_aclkx", "mmc2_dat0", "mcasp0_aclkr", "gpio3_4"),
-	_PIN(0x91c, "MII1_TXD3",	 16, 7, "gmii1_txd3", "dcan0_tx", "rgmii1_td3", "uart4_rxd", "mcasp1_fsx", "mmc2_dat1", "mcasp0_fsr", "gpio0_16"),
-	_PIN(0x920, "MII1_TXD2",	 17, 7, "gmii1_txd2", "dcan0_rx", "rgmii1_td2", "uart4_txd", "mcasp1_axr0", "mmc2_dat2", "mcasp0_ahclkx", "gpio0_17"),
-	_PIN(0x924, "MII1_TXD1",	 21, 7, "gmii1_txd1", "rmii1_txd1", "rgmii1_td1", "mcasp1_fsr", "mcasp1_axr1", "eQEP0A_in", "mmc1_cmd", "gpio0_21"),
-	_PIN(0x928, "MII1_TXD0",	 28, 7, "gmii1_txd0", "rmii1_txd0", "rgmii1_td0", "mcasp1_axr2", "mcasp1_aclkr", "eQEP0B_in", "mmc1_clk", "gpio0_28"),
+	_PIN(0x91c, "MII1_TXD3",	16, 7, "gmii1_txd3", "dcan0_tx", "rgmii1_td3", "uart4_rxd", "mcasp1_fsx", "mmc2_dat1", "mcasp0_fsr", "gpio0_16"),
+	_PIN(0x920, "MII1_TXD2",	17, 7, "gmii1_txd2", "dcan0_rx", "rgmii1_td2", "uart4_txd", "mcasp1_axr0", "mmc2_dat2", "mcasp0_ahclkx", "gpio0_17"),
+	_PIN(0x924, "MII1_TXD1",	21, 7, "gmii1_txd1", "rmii1_txd1", "rgmii1_td1", "mcasp1_fsr", "mcasp1_axr1", "eQEP0A_in", "mmc1_cmd", "gpio0_21"),
+	_PIN(0x928, "MII1_TXD0",	28, 7, "gmii1_txd0", "rmii1_txd0", "rgmii1_td0", "mcasp1_axr2", "mcasp1_aclkr", "eQEP0B_in", "mmc1_clk", "gpio0_28"),
 	_PIN(0x92c, "MII1_TX_CLK",	105, 7, "gmii1_txclk", "uart2_rxd", "rgmii1_tclk", "mmc0_dat7", "mmc1_dat0", "uart1_dcdn", "mcasp0_aclkx", "gpio3_9"),
 	_PIN(0x930, "MII1_RX_CLK",	106, 7, "gmii1_rxclk", "uart2_txd", "rgmii1_rclk", "mmc0_dat6", "mmc1_dat1", "uart1_dsrn", "mcasp0_fsx", "gpio3_10"),
-	_PIN(0x934, "MII1_RXD3",	 82, 7, "gmii1_rxd3", "uart3_rxd", "rgmii1_rd3", "mmc0_dat5", "mmc1_dat2", "uart1_dtrn", "mcasp0_axr0", "gpio2_18"),
-	_PIN(0x938, "MII1_RXD2",	 83, 7, "gmii1_rxd2", "uart3_txd", "rgmii1_rd2", "mmc0_dat4", "mmc1_dat3", "uart1_rin", "mcasp0_axr1", "gpio2_19"),
-	_PIN(0x93c, "MII1_RXD1",	 84, 7, "gmii1_rxd1", "rmii1_rxd1", "rgmii1_rd1", "mcasp1_axr3", "mcasp1_fsr", "eQEP0_strobe", "mmc2_clk", "gpio2_20"),
-	_PIN(0x940, "MII1_RXD0",	 85, 7, "gmii1_rxd0", "rmii1_rxd0", "rgmii1_rd0", "mcasp1_ahclkx", "mcasp1_ahclkr", "mcasp1_aclkr", "mcasp0_axr3", "gpio2_21"),
-	_PIN(0x944, "RMII1_REF_CLK",	 29, 7, "rmii1_refclk", "xdma_event_intr2", "spi1_cs0", "uart5_txd", "mcasp1_axr3", "mmc0_pow", "mcasp1_ahclkx", "gpio0_29"),
-	_PIN(0x948, "MDIO",		  0, 7, "mdio_data", "timer6", "uart5_rxd", "uart3_ctsn", "mmc0_sdcd","mmc1_cmd", "mmc2_cmd","gpio0_0"),
-	_PIN(0x94c, "MDC",		  1, 7, "mdio_clk", "timer5", "uart5_txd", "uart3_rtsn", "mmc0_sdwp", "mmc1_clk", "mmc2_clk", "gpio0_1"),
-#if 0 /* Incomplete Entries - fill with data from table 2-7 in datasheet */
-	_PIN(0x950, "spi0_sclk",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x954, "spi0_d0",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-#endif
-	_PIN(0x958, "spi0_d1",		  4, 7, "spi0_d1", "mmc1_sdwp", "I2C1_SDA", "ehrpwm0_tripzone_input", "pr1_uart0_rxd", "pr1_edio_data_in0", "pr1_edio_data_out0", "gpio0_4"),
-	_PIN(0x95c, "spi0_cs0",		  5, 7, "spi0_cs0", "mmc2_sdwp", "I2C1_SCL", "ehrpwm0_synci", "pr1_uart0_txd", "pr1_edio_data_in1", "pr1_edio_data_out1", "gpio0_5"),
-#if 0
-	_PIN(0x960, "spi0_cs1",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x964, "ecap0_in_pwm0_out",0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x968, "uart0_ctsn",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x96c, "uart0_rtsn",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x970, "uart0_rxd",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x974, "uart0_txd",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-#endif
-	_PIN(0x978, "uart1_ctsn",	 12, 7, "uart1_ctsn", "timer6_mux1", "dcan0_tx", "I2C2_SDA", "spi1_cs0", "pr1_uart0_cts_n", "pr1_edc_latch0_in", "gpio0_12"),
-	_PIN(0x97c, "uart1_rtsn",	 13, 7, "uart1_rtsn", "timer5_mux1", "dcan0_rx", "I2C2_SCL", "spi1_cs1", "pr1_uart0_rts_n	", "pr1_edc_latch1_in", "gpio0_13"),
-#if 0
-	_PIN(0x980, "uart1_rxd",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x984, "uart1_txd",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-#endif
+	_PIN(0x934, "MII1_RXD3",	82, 7, "gmii1_rxd3", "uart3_rxd", "rgmii1_rd3", "mmc0_dat5", "mmc1_dat2", "uart1_dtrn", "mcasp0_axr0", "gpio2_18"),
+	_PIN(0x938, "MII1_RXD2",	83, 7, "gmii1_rxd2", "uart3_txd", "rgmii1_rd2", "mmc0_dat4", "mmc1_dat3", "uart1_rin", "mcasp0_axr1", "gpio2_19"),
+	_PIN(0x93c, "MII1_RXD1",	84, 7, "gmii1_rxd1", "rmii1_rxd1", "rgmii1_rd1", "mcasp1_axr3", "mcasp1_fsr", "eQEP0_strobe", "mmc2_clk", "gpio2_20"),
+	_PIN(0x940, "MII1_RXD0",	85, 7, "gmii1_rxd0", "rmii1_rxd0", "rgmii1_rd0", "mcasp1_ahclkx", "mcasp1_ahclkr", "mcasp1_aclkr", "mcasp0_axr3", "gpio2_21"),
+	_PIN(0x944, "RMII1_REF_CLK",	29, 7, "rmii1_refclk", "xdma_event_intr2", "spi1_cs0", "uart5_txd", "mcasp1_axr3", "mmc0_pow", "mcasp1_ahclkx", "gpio0_29"),
+	_PIN(0x948, "MDIO",		0, 7, "mdio_data", "timer6", "uart5_rxd", "uart3_ctsn", "mmc0_sdcd","mmc1_cmd", "mmc2_cmd","gpio0_0"),
+	_PIN(0x94c, "MDC",		1, 7, "mdio_clk", "timer5", "uart5_txd", "uart3_rtsn", "mmc0_sdwp", "mmc1_clk", "mmc2_clk", "gpio0_1"),
+	_PIN(0x950, "SPI0_SCLK",	2, 7, "spi0_sclk", "uart2_rxd", "I2C2_SDA", "ehrpwm0A", "pr1_uart0_cts_n", "pr1_edio_sof", "EMU2", "gpio0_2"),
+	_PIN(0x954, "SPI0_D0",		3, 7, "spi0_d0", "uart2_txd", "i2C2_SCL", "ehrpwm0B", "pr1_uart0_rts_n", "pr1_edio_latch_in", "EMU3", "gpio0_3"),
+	_PIN(0x958, "SPIO_D1",		4, 7, "spi0_d1", "mmc1_sdwp", "I2C1_SDA", "ehrpwm0_tripzone_input", "pr1_uart0_rxd", "pr1_edio_data_in0", "pr1_edio_data_out0", "gpio0_4"),
+	_PIN(0x95c, "SPI0_CS0",		5, 7, "spi0_cs0", "mmc2_sdwp", "I2C1_SCL", "ehrpwm0_synci", "pr1_uart0_txd", "pr1_edio_data_in1", "pr1_edio_data_out1", "gpio0_5"),
+	_PIN(0x960, "SPI0_CS1",		6, 7, "spi0_cs1", "uart3_rxd", "eCAP1_in_PWM1_out", "mcc0_pow", "xdm_event_intr2", "mmc0_sdcd", "EMU4", "gpio0_6"),
+	_PIN(0x964, "ECAP0_IN_PWM0_OUT",7, 7, "eCAP0_in_PWM0_out", "uart3_txd", "spi1_cs1", "pr1_ecap0_ecap_capin_apwm_o", "spi1_sclk", "mmc0_sdwp", "xdma_event_intr2", "gpio0_7"),
+	_PIN(0x968, "UART0_CTSn",	40, 7, "uart0_ctsn", "uart4_rxd", "dcan1_tx", "I2C1_SDA", "spi1_d0", "timer7", "pr1_edc_sync0_out", "gpio1_8"),
+	_PIN(0x96c, "UART0_RTSn",	41, 7, "uart0_rtsn", "uart4_txd", "dcan1_rx", "I2C1_SCL", "spi1_d1", "spi1_cs0", "pr1_edc_sync1_out", "gpio1_9"),
+	_PIN(0x970, "UART0_rxd",	42, 7, "uart0_rxd", "spi1_cs0", "dcan0_tx", "I2C2_SDA", "eCAP2_in_PWM2_out", "pr1_pru1_pru_r30_14", "pr1_pru1_pru_r31_14", "gpio1_10"),
+	_PIN(0x974, "UART0_txd",	43, 7, "uart0_txd", "spi1_cs1", "dcan0_rx", "I2C2_SCL", "eCAP1_in_PWM1_out", "pr1_pru1_pru_r30_15", "pr1_pru1_pru_r31_15", "gpio1_11"),
+	_PIN(0x978, "UART1_CTSn",	12, 7, "uart1_ctsn", "timer6_mux1", "dcan0_tx", "I2C2_SDA", "spi1_cs0", "pr1_uart0_cts_n", "pr1_edc_latch0_in", "gpio0_12"),
+	_PIN(0x97c, "UART1_RTSn",	13, 7, "uart1_rtsn", "timer5_mux1", "dcan0_rx", "I2C2_SCL", "spi1_cs1", "pr1_uart0_rts_n	", "pr1_edc_latch1_in", "gpio0_13"),
+	_PIN(0x980, "UART1_RXD",	14, 7, "uart1_rxd", "mmc1_sdwp", "dcan1_tx", "I2C1_SDA", NULL, "pr1_uart0_rxd", "pr1_pru1_pru_r31_16", "gpio0_14"),
+	_PIN(0x984, "UART1_TXD",	15, 7, "uart1_txd", "mmc2_sdwp", "dcan1_rx", "I2C1_SCL", NULL, "pr1_uart0_txd", "pr1_pru0_pru_r31_16", "gpio0_15"),
 	_PIN(0x988, "I2C0_SDA",		101, 7, "I2C0_SDA", "timer4", "uart2_ctsn", "eCAP2_in_PWM2_out", NULL, NULL, NULL, "gpio3_5"),
 	_PIN(0x98c, "I2C0_SCL",		102, 7, "I2C0_SCL", "timer7", "uart2_rtsn", "eCAP1_in_PWM1_out", NULL, NULL, NULL, "gpio3_6"),
+	_PIN(0x990, "MCASP0_ACLKX",	110, 7, "mcasp0_aclkx", "ehrpwm0A", NULL, "spi1_sclk", "mmc0_sdcd", "pr1_pru0_pru_r30_0", "pr1_pru0_pru_r31_0", "gpio3_14"),
+	_PIN(0x994, "MCASP0_FSX",	111, 7, "mcasp0_fsx", "ehrpwm0B", NULL, "spi1_d0", "mmc1_sdcd", "pr1_pru0_pru_r30_1", "pr1_pru0_pru_r31_1", "gpio3_15"),
+	_PIN(0x998, "MCASP0_AXR0",	112, 7, "mcasp0_axr0", "ehrpwm0_tripzone_input", NULL, "spi1_d1", "mmc2_sdcd", "pr1_pru0_pru_r30_2", "pr1_pru0_pru_r31_2", "gpio3_16"),
+	_PIN(0x99c, "MCASP0_AHCLKR",	113, 7, "mcasp0_ahclkr", "ehrpwm0_synci", "mcasp0_axr2", "spi1_cs0", "eCAP2_in_PWM2_out", "pr1_pru0_pru_r30_3", "pr1_pru0_pru_r31_3", "gpio3_17"),
+	_PIN(0x9a0, "MCASP0_ACLKR",	114, 7, "mcasp0_aclkr", "eQEP0A_in", "mcasp0_axr2", "mcasp1_aclkx", "mmc0_sdwp", "pr1_pru0_pru_r30_4", "pr1_pru0_pru_r31_4", "gpio3_18"),
+	_PIN(0x9a4, "MCASP0_FSR",	115, 7, "mcasp0_fsr", "eQEP0B_in", "mcasp0_axr3", "mcasp1_fsx", "EMU2", "pr1_pru0_pru_r30_5", "pr1_pru0_pru_r31_5", "gpio3_19"),
+	_PIN(0x9a8, "MCASP0_AXR1",	116, 7, "mcasp0_axr1", "eQEP0_index", NULL, "mcasp1_axr0", "EMU3", "pr1_pru0_pru_r30_6", "pr1_pru0_pru_r31_6", "gpio3_20"),
+	_PIN(0x9ac, "MCASP0_AHCLKX",	117, 7, "mcasp0_ahclkx", "eQEP0_strobe", "mcasp0_axr3", "mcasp1_axr1", "EMU4", "pr1_pru0_pru_r30_7", "pr1_pru0_pru_r31_7", "gpio3_21"),
+	_PIN(0x9b0, "XDMA_EVENT_INTR0",	19, 7, "xdma_event_intr0", NULL, "timer4", "clkout1", "spi1_cs1", "pr1_pru1_pru_r31_16", "EMU2", "gpio0_19"),
+	_PIN(0x9b4, "XDMA_EVENT_INTR1",	20, 7, "xdma_event_intr1", NULL, "tclkin", "clkout2", "timer7", "pr1_pru0_pru_r31_16", "EMU3", "gpio0_20"),
 #if 0
-	_PIN(0x990, "mcasp0_aclkx",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x994, "mcasp0_fsx",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x998, "mcasp0_axr0",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x99c, "mcasp0_ahclkr",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x9a0, "mcasp0_aclkr",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x9a4, "mcasp0_fsr",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x9a8, "mcasp0_axr1",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x9ac, "mcasp0_ahclkx",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x9b0, "xdma_event_intr0",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x9b4, "xdma_event_intr1",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
 	_PIN(0x9b8, "nresetin_out",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
 	_PIN(0x9bc, "porz",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
 	_PIN(0x9c0, "nnmi",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
@@ -218,8 +208,10 @@ const struct ti_scm_padconf ti_padconf_devmap[] = {
 	_PIN(0x9d8, "tdo",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
 	_PIN(0x9dc, "tck",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
 	_PIN(0x9e0, "ntrst",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x9e4, "emu0",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0x9e8, "emu1",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
+#endif
+	_PIN(0x9e4, "EMU0",		103, 7, "EMU0", NULL, NULL, NULL, NULL, NULL, NULL, "gpio3_7"),
+	_PIN(0x9e8, "EMU1",		104, 0, "EMU1", NULL, NULL, NULL, NULL, NULL, NULL, "gpio3_8"),
+#if 0
 	_PIN(0x9ec, "osc1_in",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
 	_PIN(0x9f0, "osc1_out",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
 	_PIN(0x9f4, "osc1_vss",		0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
@@ -228,17 +220,17 @@ const struct ti_scm_padconf ti_padconf_devmap[] = {
 	_PIN(0xa00, "ext_wakeup",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
 	_PIN(0xa04, "enz_kaldo_1p8v",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
 #endif
-	_PIN(0xa08, "USB0_DM",		  0, 0, "USB0_DM", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0xa0c, "USB0_DP",		  0, 0, "USB0_DP", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0xa10, "USB0_CE",		  0, 0, "USB0_CE", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0xa14, "USB0_ID",		  0, 0, "USB0_ID", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0xa18, "USB0_VBUS",	  0, 0, "USB0_VBUS", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0xa1c, "USB0_DRVVBUS",	 18, 7, "USB0_DRVVBUS", NULL, NULL, NULL, NULL, NULL, NULL, "gpio0_18"),
-	_PIN(0xa20, "USB1_DM",		  0, 0, "USB1_DM", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0xa24, "USB1_DP",		  0, 0, "USB1_DP", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0xa28, "USB1_CE",		  0, 0, "USB1_CE", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0xa2c, "USB1_ID",		  0, 0, "USB1_ID", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
-	_PIN(0xa30, "USB1_VBUS",	  0, 0, "USB1_VBUS", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
+	_PIN(0xa08, "USB0_DM",		0, 0, "USB0_DM", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
+	_PIN(0xa0c, "USB0_DP",		0, 0, "USB0_DP", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
+	_PIN(0xa10, "USB0_CE",		0, 0, "USB0_CE", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
+	_PIN(0xa14, "USB0_ID",		0, 0, "USB0_ID", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
+	_PIN(0xa18, "USB0_VBUS",	0, 0, "USB0_VBUS", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
+	_PIN(0xa1c, "USB0_DRVVBUS",	18, 7, "USB0_DRVVBUS", NULL, NULL, NULL, NULL, NULL, NULL, "gpio0_18"),
+	_PIN(0xa20, "USB1_DM",		0, 0, "USB1_DM", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
+	_PIN(0xa24, "USB1_DP",		0, 0, "USB1_DP", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
+	_PIN(0xa28, "USB1_CE",		0, 0, "USB1_CE", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
+	_PIN(0xa2c, "USB1_ID",		0, 0, "USB1_ID", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
+	_PIN(0xa30, "USB1_VBUS",	0, 0, "USB1_VBUS", NULL, NULL, NULL, NULL, NULL, NULL, NULL),
 	_PIN(0xa34, "USB1_DRVVBUS",	109, 7, "USB1_DRVVBUS", NULL, NULL, NULL, NULL, NULL, NULL, "gpio3_13"),
 #if 0
 	_PIN(0xa38, "ddr_resetn",	0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
diff --git a/sys/arm/ti/cpsw/if_cpsw.c b/sys/arm/ti/cpsw/if_cpsw.c
index a47c131..93702df 100644
--- a/sys/arm/ti/cpsw/if_cpsw.c
+++ b/sys/arm/ti/cpsw/if_cpsw.c
@@ -25,8 +25,22 @@
  */
 
 /*
- * TI 3 Port Switch Ethernet (CPSW) Driver
- * Found in TI8148, AM335x SoCs
+ * TI Common Platform Ethernet Switch (CPSW) Driver
+ * Found in TI8148 "DaVinci" and AM335x "Sitara" SoCs.
+ *
+ * This controller is documented in the AM335x Technical Reference
+ * Manual, in the TMS320DM814x DaVinci Digital Video Processors TRM
+ * and in the TMS320C6452 3 Port Switch Ethernet Subsystem TRM.
+ *
+ * It is basically a single Ethernet port (port 0) wired internally to
+ * a 3-port store-and-forward switch connected to two independent
+ * "sliver" controllers (port 1 and port 2).  You can operate the
+ * controller in a variety of different ways by suitably configuring
+ * the slivers and the Address Lookup Engine (ALE) that routes packets
+ * between the ports.
+ *
+ * This code was developed and tested on a BeagleBone with
+ * an AM335x SoC.
  */
 
 #include <sys/cdefs.h>
@@ -76,44 +90,82 @@ __FBSDID("$FreeBSD$");
 
 #include "miibus_if.h"
 
-static int cpsw_probe(device_t dev);
-static int cpsw_attach(device_t dev);
-static int cpsw_detach(device_t dev);
-static int cpsw_shutdown(device_t dev);
-static int cpsw_suspend(device_t dev);
-static int cpsw_resume(device_t dev);
-
-static int cpsw_miibus_readreg(device_t dev, int phy, int reg);
-static int cpsw_miibus_writereg(device_t dev, int phy, int reg, int value);
-
-static int cpsw_ifmedia_upd(struct ifnet *ifp);
-static void cpsw_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr);
-
-static void cpsw_init(void *arg);
-static void cpsw_init_locked(void *arg);
-static void cpsw_start(struct ifnet *ifp);
-static void cpsw_start_locked(struct ifnet *ifp);
-static void cpsw_stop_locked(struct cpsw_softc *sc);
-static int cpsw_ioctl(struct ifnet *ifp, u_long command, caddr_t data);
-static int cpsw_init_slot_lists(struct cpsw_softc *sc);
-static void cpsw_free_slot(struct cpsw_softc *sc, struct cpsw_slot *slot);
-static void cpsw_fill_rx_queue_locked(struct cpsw_softc *sc);
-static void cpsw_tx_watchdog(struct cpsw_softc *sc);
-
-static void cpsw_intr_rx_thresh(void *arg);
+/* Device probe/attach/detach. */
+static int cpsw_probe(device_t);
+static void cpsw_init_slots(struct cpsw_softc *);
+static int cpsw_attach(device_t);
+static void cpsw_free_slot(struct cpsw_softc *, struct cpsw_slot *);
+static int cpsw_detach(device_t);
+
+/* Device Init/shutdown. */
+static void cpsw_init(void *);
+static void cpsw_init_locked(void *);
+static int cpsw_shutdown(device_t);
+static void cpsw_shutdown_locked(struct cpsw_softc *);
+
+/* Device Suspend/Resume. */
+static int cpsw_suspend(device_t);
+static int cpsw_resume(device_t);
+
+/* Ioctl. */
+static int cpsw_ioctl(struct ifnet *, u_long command, caddr_t data);
+
+static int cpsw_miibus_readreg(device_t, int phy, int reg);
+static int cpsw_miibus_writereg(device_t, int phy, int reg, int value);
+
+/* Send/Receive packets. */
 static void cpsw_intr_rx(void *arg);
-static void cpsw_intr_rx_locked(void *arg);
-static void cpsw_intr_tx(void *arg);
-static void cpsw_intr_tx_locked(void *arg);
-static void cpsw_intr_misc(void *arg);
-
-static void cpsw_ale_read_entry(struct cpsw_softc *sc, uint16_t idx, uint32_t *ale_entry);
-static void cpsw_ale_write_entry(struct cpsw_softc *sc, uint16_t idx, uint32_t *ale_entry);
-static int cpsw_ale_uc_entry_set(struct cpsw_softc *sc, uint8_t port, uint8_t *mac);
-static int cpsw_ale_mc_entry_set(struct cpsw_softc *sc, uint8_t portmap, uint8_t *mac);
-#ifdef CPSW_DEBUG
-static void cpsw_ale_dump_table(struct cpsw_softc *sc);
-#endif
+static struct mbuf *cpsw_rx_dequeue(struct cpsw_softc *);
+static void cpsw_rx_enqueue(struct cpsw_softc *);
+static void cpsw_start(struct ifnet *);
+static void cpsw_tx_enqueue(struct cpsw_softc *);
+static int cpsw_tx_dequeue(struct cpsw_softc *);
+
+/* Misc interrupts and watchdog. */
+static void cpsw_intr_rx_thresh(void *);
+static void cpsw_intr_misc(void *);
+static void cpsw_tick(void *);
+static void cpsw_ifmedia_sts(struct ifnet *, struct ifmediareq *);
+static int cpsw_ifmedia_upd(struct ifnet *);
+static void cpsw_tx_watchdog(struct cpsw_softc *);
+
+/* ALE support */
+static void cpsw_ale_read_entry(struct cpsw_softc *, uint16_t idx, uint32_t *ale_entry);
+static void cpsw_ale_write_entry(struct cpsw_softc *, uint16_t idx, uint32_t *ale_entry);
+static int cpsw_ale_mc_entry_set(struct cpsw_softc *, uint8_t portmap, uint8_t *mac);
+static int cpsw_ale_update_addresses(struct cpsw_softc *, int purge);
+static void cpsw_ale_dump_table(struct cpsw_softc *);
+
+/* Statistics and sysctls. */
+static void cpsw_add_sysctls(struct cpsw_softc *);
+static void cpsw_stats_collect(struct cpsw_softc *);
+static int cpsw_stats_sysctl(SYSCTL_HANDLER_ARGS);
+
+/*
+ * Arbitrary limit on number of segments in an mbuf to be transmitted.
+ * Packets with more segments than this will be defragmented before
+ * they are queued.
+ */
+#define CPSW_TXFRAGS 8
+
+
+/*
+ * TODO: The CPSW subsystem (CPSW_SS) can drive two independent PHYs
+ * as separate Ethernet ports.  To properly support this, we should
+ * break this into two separate devices: a CPSW_SS device that owns
+ * the interrupts and actually talks to the CPSW hardware, and a
+ * separate CPSW Ethernet child device for each Ethernet port.  The RX
+ * interrupt, for example, would be part of CPSW_SS; it would receive
+ * a packet, note the input port, and then dispatch it to the child
+ * device's interface queue.  Similarly for transmit.
+ *
+ * It's not clear to me whether the device tree should be restructured
+ * with a cpsw_ss node and two child nodes.  That would allow specifying
+ * MAC addresses for each port, for example, but might be overkill.
+ *
+ * Unfortunately, I don't have hardware right now that supports two
+ * Ethernet ports via CPSW.
+ */
 
 static device_method_t cpsw_methods[] = {
 	/* Device interface */
@@ -137,7 +189,6 @@ static driver_t cpsw_driver = {
 
 static devclass_t cpsw_devclass;
 
-
 DRIVER_MODULE(cpsw, simplebus, cpsw_driver, cpsw_devclass, 0, 0);
 DRIVER_MODULE(miibus, cpsw, miibus_driver, miibus_devclass, 0, 0);
 MODULE_DEPEND(cpsw, ether, 1, 1, 1);
@@ -152,40 +203,109 @@ static struct resource_spec res_spec[] = {
 	{ -1, 0 }
 };
 
-static struct {
-	driver_intr_t *handler;
-	char * description;
-} cpsw_intrs[CPSW_INTR_COUNT + 1] = {
-	{ cpsw_intr_rx_thresh, "CPSW RX threshold interrupt" },
-	{ cpsw_intr_rx,	"CPSW RX interrupt" },
-	{ cpsw_intr_tx,	"CPSW TX interrupt" },
-	{ cpsw_intr_misc, "CPSW misc interrupt" },
+/* Number of entries here must match size of stats
+ * array in struct cpsw_softc. */
+static struct cpsw_stat {
+	int	reg;
+	char *oid;
+} cpsw_stat_sysctls[CPSW_SYSCTL_COUNT] = {
+	{0x00, "GoodRxFrames"},
+	{0x04, "BroadcastRxFrames"},
+	{0x08, "MulticastRxFrames"},
+	{0x0C, "PauseRxFrames"},
+	{0x10, "RxCrcErrors"},
+	{0x14, "RxAlignErrors"},
+	{0x18, "OversizeRxFrames"},
+	{0x1c, "RxJabbers"},
+	{0x20, "ShortRxFrames"},
+	{0x24, "RxFragments"},
+	{0x30, "RxOctets"},
+	{0x34, "GoodTxFrames"},
+	{0x38, "BroadcastTxFrames"},
+	{0x3c, "MulticastTxFrames"},
+	{0x40, "PauseTxFrames"},
+	{0x44, "DeferredTxFrames"},
+	{0x48, "CollisionsTxFrames"},
+	{0x4c, "SingleCollisionTxFrames"},
+	{0x50, "MultipleCollisionTxFrames"},
+	{0x54, "ExcessiveCollisions"},
+	{0x58, "LateCollisions"},
+	{0x5c, "TxUnderrun"},
+	{0x60, "CarrierSenseErrors"},
+	{0x64, "TxOctets"},
+	{0x68, "RxTx64OctetFrames"},
+	{0x6c, "RxTx65to127OctetFrames"},
+	{0x70, "RxTx128to255OctetFrames"},
+	{0x74, "RxTx256to511OctetFrames"},
+	{0x78, "RxTx512to1024OctetFrames"},
+	{0x7c, "RxTx1024upOctetFrames"},
+	{0x80, "NetOctets"},
+	{0x84, "RxStartOfFrameOverruns"},
+	{0x88, "RxMiddleOfFrameOverruns"},
+	{0x8c, "RxDmaOverruns"}
 };
 
-/* Locking macros */
+/*
+ * Basic debug support.
+ */
+
+#define IF_DEBUG(sc)  if (sc->cpsw_if_flags & IFF_DEBUG)
+
+static void
+cpsw_debugf_head(const char *funcname)
+{
+	int t = (int)(time_second % (24 * 60 * 60));
+
+	printf("%02d:%02d:%02d %s ", t / (60 * 60), (t / 60) % 60, t % 60, funcname);
+}
+
+#include <machine/stdarg.h>
+static void
+cpsw_debugf(const char *fmt, ...)
+{
+	va_list ap;
+
+	va_start(ap, fmt);
+	vprintf(fmt, ap);
+	va_end(ap);
+	printf("\n");
+
+}
+
+#define CPSW_DEBUGF(a) do {					\
+	IF_DEBUG(sc) {						\
+		cpsw_debugf_head(__func__);			\
+		cpsw_debugf a;					\
+	}							\
+} while (0)
+
+
+/*
+ * Locking macros
+ */
 #define CPSW_TX_LOCK(sc) do {					\
-		mtx_assert(&(sc)->rx_lock, MA_NOTOWNED);		\
-		mtx_lock(&(sc)->tx_lock);				\
+		mtx_assert(&(sc)->rx.lock, MA_NOTOWNED);		\
+		mtx_lock(&(sc)->tx.lock);				\
 } while (0)
 
-#define CPSW_TX_UNLOCK(sc)	mtx_unlock(&(sc)->tx_lock)
-#define CPSW_TX_LOCK_ASSERT(sc)	mtx_assert(&(sc)->tx_lock, MA_OWNED)
+#define CPSW_TX_UNLOCK(sc)	mtx_unlock(&(sc)->tx.lock)
+#define CPSW_TX_LOCK_ASSERT(sc)	mtx_assert(&(sc)->tx.lock, MA_OWNED)
 
 #define CPSW_RX_LOCK(sc) do {					\
-		mtx_assert(&(sc)->tx_lock, MA_NOTOWNED);		\
-		mtx_lock(&(sc)->rx_lock);				\
+		mtx_assert(&(sc)->tx.lock, MA_NOTOWNED);		\
+		mtx_lock(&(sc)->rx.lock);				\
 } while (0)
 
-#define CPSW_RX_UNLOCK(sc)		mtx_unlock(&(sc)->rx_lock)
-#define CPSW_RX_LOCK_ASSERT(sc)	mtx_assert(&(sc)->rx_lock, MA_OWNED)
+#define CPSW_RX_UNLOCK(sc)		mtx_unlock(&(sc)->rx.lock)
+#define CPSW_RX_LOCK_ASSERT(sc)	mtx_assert(&(sc)->rx.lock, MA_OWNED)
 
 #define CPSW_GLOBAL_LOCK(sc) do {					\
-		if ((mtx_owned(&(sc)->tx_lock) ? 1 : 0) !=	\
-		    (mtx_owned(&(sc)->rx_lock) ? 1 : 0)) {		\
+		if ((mtx_owned(&(sc)->tx.lock) ? 1 : 0) !=	\
+		    (mtx_owned(&(sc)->rx.lock) ? 1 : 0)) {		\
 			panic("cpsw deadlock possibility detection!");	\
 		}							\
-		mtx_lock(&(sc)->tx_lock);				\
-		mtx_lock(&(sc)->rx_lock);				\
+		mtx_lock(&(sc)->tx.lock);				\
+		mtx_lock(&(sc)->rx.lock);				\
 } while (0)
 
 #define CPSW_GLOBAL_UNLOCK(sc) do {					\
@@ -198,35 +318,128 @@ static struct {
 		CPSW_RX_LOCK_ASSERT(sc);				\
 } while (0)
 
-
-#include <machine/stdarg.h>
+/*
+ * Read/Write macros
+ */
+#define	cpsw_read_4(sc, reg)		bus_read_4(sc->res[0], reg)
+#define	cpsw_write_4(sc, reg, val)	bus_write_4(sc->res[0], reg, val)
+
+#define	cpsw_cpdma_bd_offset(i)	(CPSW_CPPI_RAM_OFFSET + ((i)*16))
+
+#define	cpsw_cpdma_bd_paddr(sc, slot)				\
+	(slot->bd_offset + vtophys(rman_get_start(sc->res[0])))
+#define	cpsw_cpdma_read_bd(sc, slot, val)				\
+	bus_read_region_4(sc->res[0], slot->bd_offset, (uint32_t *) val, 4)
+#define	cpsw_cpdma_write_bd(sc, slot, val)				\
+	bus_write_region_4(sc->res[0], slot->bd_offset, (uint32_t *) val, 4)
+#define	cpsw_cpdma_write_bd_next(sc, slot, next_slot)			\
+	cpsw_write_4(sc, slot->bd_offset, cpsw_cpdma_bd_paddr(sc, next_slot))
+#define	cpsw_cpdma_read_bd_flags(sc, slot)		\
+	bus_read_2(sc->res[0], slot->bd_offset + 14)
+#define	cpsw_write_hdp_slot(sc, queue, slot)				\
+	cpsw_write_4(sc, (queue)->hdp_offset, cpsw_cpdma_bd_paddr(sc, slot))
+#define	CP_OFFSET (CPSW_CPDMA_TX_CP(0) - CPSW_CPDMA_TX_HDP(0))
+#define	cpsw_read_cp(sc, queue)				\
+	cpsw_read_4(sc, (queue)->hdp_offset + CP_OFFSET) 
+#define	cpsw_write_cp(sc, queue, val)				\
+	cpsw_write_4(sc, (queue)->hdp_offset + CP_OFFSET, (val))
+#define	cpsw_write_cp_slot(sc, queue, slot)		\
+	cpsw_write_cp(sc, queue, cpsw_cpdma_bd_paddr(sc, slot))
+
+#if 0
+/* XXX temporary function versions for debugging. */
 static void
-cpsw_debugf_head(const char *funcname)
+cpsw_write_hdp_slotX(struct cpsw_softc *sc, struct cpsw_queue *queue, struct cpsw_slot *slot)
 {
-	int t = (int)(time_second % (24 * 60 * 60));
+	uint32_t reg = queue->hdp_offset;
+	uint32_t v = cpsw_cpdma_bd_paddr(sc, slot);
+	CPSW_DEBUGF(("HDP <=== 0x%08x (was 0x%08x)", v, cpsw_read_4(sc, reg)));
+	cpsw_write_4(sc, reg, v);
+}
 
-	printf("%02d:%02d:%02d %s ", t / (60 * 60), (t / 60) % 60, t % 60, funcname);
+static void
+cpsw_write_cp_slotX(struct cpsw_softc *sc, struct cpsw_queue *queue, struct cpsw_slot *slot)
+{
+	uint32_t v = cpsw_cpdma_bd_paddr(sc, slot);
+	CPSW_DEBUGF(("CP <=== 0x%08x (expecting 0x%08x)", v, cpsw_read_cp(sc, queue)));
+	cpsw_write_cp(sc, queue, v);
 }
+#endif
 
+/*
+ * Expanded dump routines for verbose debugging.
+ */
 static void
-cpsw_debugf(const char *fmt, ...)
+cpsw_dump_slot(struct cpsw_softc *sc, struct cpsw_slot *slot)
 {
-	va_list ap;
+	static const char *flags[] = {"SOP", "EOP", "Owner", "EOQ",
+	    "TDownCmplt", "PassCRC", "Long", "Short", "MacCtl", "Overrun",
+	    "PktErr1", "PortEn/PktErr0", "RxVlanEncap", "Port2", "Port1",
+	    "Port0"};
+	struct cpsw_cpdma_bd bd;
+	const char *sep;
+	int i;
 
-	va_start(ap, fmt);
-	vprintf(fmt, ap);
-	va_end(ap);
+	cpsw_cpdma_read_bd(sc, slot, &bd);
+	printf("BD Addr: 0x%08x   Next: 0x%08x\n", cpsw_cpdma_bd_paddr(sc, slot), bd.next);
+	printf("  BufPtr: 0x%08x   BufLen: 0x%08x\n", bd.bufptr, bd.buflen);
+	printf("  BufOff: 0x%08x   PktLen: 0x%08x\n", bd.bufoff, bd.pktlen);
+	printf("  Flags: ");
+	sep = "";
+	for (i = 0; i < 16; ++i) {
+		if (bd.flags & (1 << (15 - i))) {
+			printf("%s%s", sep, flags[i]);
+			sep = ",";
+		}
+	}
 	printf("\n");
+	if (slot->mbuf) {
+		printf("  Ether:  %14D\n",
+		    (char *)(slot->mbuf->m_hdr.mh_data), " ");
+		printf("  Packet: %16D\n",
+		    (char *)(slot->mbuf->m_hdr.mh_data) + 14, " ");
+	}
+}
 
+#define CPSW_DUMP_SLOT(cs, slot) do {				\
+	IF_DEBUG(sc) {						\
+		cpsw_dump_slot(sc, slot);			\
+	}							\
+} while (0)
+
+
+static void
+cpsw_dump_queue(struct cpsw_softc *sc, struct cpsw_slots *q)
+{
+	struct cpsw_slot *slot;
+	int i = 0;
+	int others = 0;
+
+	STAILQ_FOREACH(slot, q, next) {
+		if (i > 4)
+			++others;
+		else
+			cpsw_dump_slot(sc, slot);
+		++i;
+	}
+	if (others)
+		printf(" ... and %d more.\n", others);
+	printf("\n");
 }
 
-#define CPSW_DEBUGF(a) do {						\
-		if (sc->cpsw_if_flags & IFF_DEBUG) {			\
-			cpsw_debugf_head(__func__);			\
-			cpsw_debugf a;					\
-		}							\
+#define CPSW_DUMP_QUEUE(sc, q) do {				\
+	IF_DEBUG(sc) {						\
+		cpsw_dump_queue(sc, q);				\
+	}							\
 } while (0)
 
+
+/*
+ *
+ * Device Probe, Attach, Detach.
+ *
+ */
+
 static int
 cpsw_probe(device_t dev)
 {
@@ -238,18 +451,107 @@ cpsw_probe(device_t dev)
 	return (BUS_PROBE_DEFAULT);
 }
 
+
+static void
+cpsw_init_slots(struct cpsw_softc *sc)
+{
+	struct cpsw_slot *slot;
+	int i;
+
+	STAILQ_INIT(&sc->avail);
+
+	/* Put the slot descriptors onto the global avail list. */
+	for (i = 0; i < sizeof(sc->_slots) / sizeof(sc->_slots[0]); i++) {
+		slot = &sc->_slots[i];
+		slot->bd_offset = cpsw_cpdma_bd_offset(i);
+		STAILQ_INSERT_TAIL(&sc->avail, slot, next);
+	}
+}
+
+/*
+ * bind an interrupt, add the relevant info to sc->interrupts
+ */
+static int
+cpsw_attach_interrupt(struct cpsw_softc *sc, struct resource *res, driver_intr_t *handler, const char *description)
+{
+	void **pcookie;
+	int error;
+
+	sc->interrupts[sc->interrupt_count].res = res;
+	sc->interrupts[sc->interrupt_count].description = description;
+	pcookie = &sc->interrupts[sc->interrupt_count].ih_cookie;
+
+	error = bus_setup_intr(sc->dev, res, INTR_TYPE_NET | INTR_MPSAFE,
+	    NULL, *handler, sc, pcookie);
+	if (error)
+		device_printf(sc->dev,
+		    "could not setup %s\n", description);
+	else
+		++sc->interrupt_count;
+	return (error);
+}
+
+/*
+ * teardown everything in sc->interrupts.
+ */
+static void
+cpsw_detach_interrupts(struct cpsw_softc *sc)
+{
+	int error;
+	int i;
+
+	for (i = 0; i < sizeof(sc->interrupts) / sizeof(sc->interrupts[0]); ++i) {
+		if (!sc->interrupts[i].ih_cookie)
+			continue;
+		error = bus_teardown_intr(sc->dev,
+		    sc->interrupts[i].res, sc->interrupts[i].ih_cookie);
+		if (error)
+			device_printf(sc->dev, "could not release %s\n",
+			    sc->interrupts[i].description);
+		sc->interrupts[i].ih_cookie = NULL;
+	}
+}
+
+static int
+cpsw_add_slots(struct cpsw_softc *sc, struct cpsw_queue *queue, int requested)
+{
+	const int max_slots = sizeof(sc->_slots) / sizeof(sc->_slots[0]);
+	struct cpsw_slot *slot;
+	int i;
+
+	if (requested < 0)
+		requested = max_slots;
+
+	for (i = 0; i < requested; ++i) {
+		slot = STAILQ_FIRST(&sc->avail);
+		if (slot == NULL)
+			return (0);
+		if (bus_dmamap_create(sc->mbuf_dtag, 0, &slot->dmamap)) {
+			if_printf(sc->ifp, "failed to create dmamap\n");
+			return (ENOMEM);
+		}
+		STAILQ_REMOVE_HEAD(&sc->avail, next);
+		STAILQ_INSERT_TAIL(&queue->avail, slot, next);
+		++queue->avail_queue_len;
+		++queue->queue_slots;
+	}
+	return (0);
+}
+
 static int
 cpsw_attach(device_t dev)
 {
+	bus_dma_segment_t segs[1];
 	struct cpsw_softc *sc = device_get_softc(dev);
 	struct mii_softc *miisc;
 	struct ifnet *ifp;
 	void *phy_sc;
-	int i, error, phy;
+	int error, phy, nsegs;
 	uint32_t reg;
 
 	CPSW_DEBUGF((""));
 
+	getbinuptime(&sc->attach_uptime);
 	sc->dev = dev;
 	sc->node = ofw_bus_get_node(dev);
 
@@ -259,10 +561,10 @@ cpsw_attach(device_t dev)
 		return (ENXIO);
 	}
 	/* Initialize mutexes */
-	mtx_init(&sc->tx_lock, device_get_nameunit(dev),
-		"cpsw TX lock", MTX_DEF);
-	mtx_init(&sc->rx_lock, device_get_nameunit(dev),
-		"cpsw RX lock", MTX_DEF);
+	mtx_init(&sc->tx.lock, device_get_nameunit(dev),
+	    "cpsw TX lock", MTX_DEF);
+	mtx_init(&sc->rx.lock, device_get_nameunit(dev),
+	    "cpsw RX lock", MTX_DEF);
 
 	/* Allocate IO and IRQ resources */
 	error = bus_alloc_resources(dev, res_spec, sc->res);
@@ -272,11 +574,11 @@ cpsw_attach(device_t dev)
 		return (ENXIO);
 	}
 
-	reg = cpsw_read_4(CPSW_SS_IDVER);
-	device_printf(dev, "Version %d.%d (%d)\n", (reg >> 8 & 0x7),
+	reg = cpsw_read_4(sc, CPSW_SS_IDVER);
+	device_printf(dev, "CPSW SS Version %d.%d (%d)\n", (reg >> 8 & 0x7),
 		reg & 0xFF, (reg >> 11) & 0x1F);
 
-	//cpsw_add_sysctls(sc); TODO
+	cpsw_add_sysctls(sc);
 
 	/* Allocate a busdma tag and DMA safe memory for mbufs. */
 	error = bus_dma_tag_create(
@@ -285,22 +587,14 @@ cpsw_attach(device_t dev)
 		BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 		BUS_SPACE_MAXADDR,		/* highaddr */
 		NULL, NULL,			/* filtfunc, filtfuncarg */
-		MCLBYTES, 1,			/* maxsize, nsegments */
+		MCLBYTES, CPSW_TXFRAGS,		/* maxsize, nsegments */
 		MCLBYTES, 0,			/* maxsegsz, flags */
 		NULL, NULL,			/* lockfunc, lockfuncarg */
 		&sc->mbuf_dtag);		/* dmatag */
 	if (error) {
 		device_printf(dev, "bus_dma_tag_create failed\n");
 		cpsw_detach(dev);
-		return (ENOMEM);
-	}
-
-	/* Initialize the tx_avail and rx_avail lists. */
-	error = cpsw_init_slot_lists(sc);
-	if (error) {
-		device_printf(dev, "failed to allocate dmamaps\n");
-		cpsw_detach(dev);
-		return (ENOMEM);
+		return (error);
 	}
 
 	/* Allocate network interface */
@@ -311,6 +605,16 @@ cpsw_attach(device_t dev)
 		return (ENOMEM);
 	}
 
+	/* Allocate the null mbuf and pre-sync it. */
+	sc->null_mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
+	memset(sc->null_mbuf->m_hdr.mh_data, 0, sc->null_mbuf->m_ext.ext_size);
+	bus_dmamap_create(sc->mbuf_dtag, 0, &sc->null_mbuf_dmamap);
+	bus_dmamap_load_mbuf_sg(sc->mbuf_dtag, sc->null_mbuf_dmamap,
+	    sc->null_mbuf, segs, &nsegs, BUS_DMA_NOWAIT);
+	bus_dmamap_sync(sc->mbuf_dtag, sc->null_mbuf_dmamap,
+	    BUS_DMASYNC_PREWRITE);
+	sc->null_mbuf_paddr = segs[0].ds_addr;
+
 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 	ifp->if_softc = sc;
 	ifp->if_flags = IFF_SIMPLEX | IFF_MULTICAST | IFF_BROADCAST;
@@ -321,11 +625,32 @@ cpsw_attach(device_t dev)
 	ifp->if_start = cpsw_start;
 	ifp->if_ioctl = cpsw_ioctl;
 
-	ifp->if_snd.ifq_drv_maxlen = CPSW_MAX_TX_BUFFERS - 1;
+	cpsw_init_slots(sc);
+
+	/* Allocate slots to TX and RX queues. */
+	STAILQ_INIT(&sc->rx.avail);
+	STAILQ_INIT(&sc->rx.active);
+	STAILQ_INIT(&sc->tx.avail);
+	STAILQ_INIT(&sc->tx.active);
+	// For now:  128 slots to TX, rest to RX.
+	// XXX TODO: start with 32/64 and grow dynamically based on demand.
+	if (cpsw_add_slots(sc, &sc->tx, 128) || cpsw_add_slots(sc, &sc->rx, -1)) {
+		device_printf(dev, "failed to allocate dmamaps\n");
+		cpsw_detach(dev);
+		return (ENOMEM);
+	}
+	device_printf(dev, "Initial queue size TX=%d RX=%d\n",
+	    sc->tx.queue_slots, sc->rx.queue_slots);
+
+	ifp->if_snd.ifq_drv_maxlen = sc->tx.queue_slots;
 	IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
 	IFQ_SET_READY(&ifp->if_snd);
 
+	sc->tx.hdp_offset = CPSW_CPDMA_TX_HDP(0);
+	sc->rx.hdp_offset = CPSW_CPDMA_RX_HDP(0);
+
 	/* Get high part of MAC address from control module (mac_id0_hi) */
+	/* TODO: Get MAC ID1 as well as MAC ID0. */
 	ti_scm_reg_read_4(0x634, &reg);
 	sc->mac_addr[0] = reg & 0xFF;
 	sc->mac_addr[1] = (reg >>  8) & 0xFF;
@@ -338,11 +663,14 @@ cpsw_attach(device_t dev)
 	sc->mac_addr[5] = (reg >>  8) & 0xFF;
 
 	ether_ifattach(ifp, sc->mac_addr);
-	callout_init(&sc->wd_callout, 0);
+	callout_init(&sc->watchdog.callout, 0);
 
 	/* Initialze MDIO - ENABLE, PREAMBLE=0, FAULTENB, CLKDIV=0xFF */
 	/* TODO Calculate MDCLK=CLK/(CLKDIV+1) */
-	cpsw_write_4(MDIOCONTROL, 1 << 30 | 1 << 18 | 0xFF);
+	cpsw_write_4(sc, MDIOCONTROL, 1 << 30 | 1 << 18 | 0xFF);
+
+	/* Clear ALE */
+	cpsw_write_4(sc, CPSW_ALE_CONTROL, 1 << 30);
 
 	/* Attach PHY(s) */
 	error = mii_attach(dev, &sc->miibus, ifp, cpsw_ifmedia_upd,
@@ -358,25 +686,38 @@ cpsw_attach(device_t dev)
 	miisc = LIST_FIRST(&sc->mii->mii_phys);
 
 	/* Select PHY and enable interrupts */
-	cpsw_write_4(MDIOUSERPHYSEL0, 1 << 6 | (miisc->mii_phy & 0x1F));
-
-	/* Attach interrupt handlers */
-	for (i = 1; i <= CPSW_INTR_COUNT; ++i) {
-		error = bus_setup_intr(dev, sc->res[i],
-		    INTR_TYPE_NET | INTR_MPSAFE,
-		    NULL, *cpsw_intrs[i - 1].handler,
-		    sc, &sc->ih_cookie[i - 1]);
-		if (error) {
-			device_printf(dev, "could not setup %s\n",
-			    cpsw_intrs[i].description);
-			cpsw_detach(dev);
-			return (error);
-		}
+	cpsw_write_4(sc, MDIOUSERPHYSEL0, 1 << 6 | (miisc->mii_phy & 0x1F));
+	
+	/* Note: We don't use sc->res[3] (TX interrupt) */
+	if (cpsw_attach_interrupt(sc, sc->res[1],
+		cpsw_intr_rx_thresh, "CPSW RX threshold interrupt") ||
+	    cpsw_attach_interrupt(sc, sc->res[2],
+		cpsw_intr_rx, "CPSW RX interrupt") ||
+	    cpsw_attach_interrupt(sc, sc->res[4],
+		cpsw_intr_misc, "CPSW misc interrupt")) {
+		cpsw_detach(dev);
+		return (ENXIO);
 	}
 
 	return (0);
 }
 
+static void
+cpsw_free_slot(struct cpsw_softc *sc, struct cpsw_slot *slot)
+{
+	int error;
+
+	if (slot->dmamap) {
+		error = bus_dmamap_destroy(sc->mbuf_dtag, slot->dmamap);
+		KASSERT(error == 0, ("Mapping still active"));
+		slot->dmamap = NULL;
+	}
+	if (slot->mbuf) {
+		m_freem(slot->mbuf);
+		slot->mbuf = NULL;
+	}
+}
+
 static int
 cpsw_detach(device_t dev)
 {
@@ -389,31 +730,20 @@ cpsw_detach(device_t dev)
 	if (device_is_attached(dev)) {
 		ether_ifdetach(sc->ifp);
 		CPSW_GLOBAL_LOCK(sc);
-		cpsw_stop_locked(sc);
+		cpsw_shutdown_locked(sc);
 		CPSW_GLOBAL_UNLOCK(sc);
-		callout_drain(&sc->wd_callout);
+		callout_drain(&sc->watchdog.callout);
 	}
 
 	bus_generic_detach(dev);
 	device_delete_child(dev, sc->miibus);
 
 	/* Stop and release all interrupts */
-	for (i = 0; i < CPSW_INTR_COUNT; ++i) {
-		if (!sc->ih_cookie[i])
-			continue;
-
-		error = bus_teardown_intr(dev, sc->res[1 + i], sc->ih_cookie[i]);
-		if (error)
-			device_printf(dev, "could not release %s\n",
-			    cpsw_intrs[i + 1].description);
-	}
+	cpsw_detach_interrupts(sc);
 
 	/* Free dmamaps and mbufs */
-	for (i = 0; i < CPSW_MAX_TX_BUFFERS; i++) {
-		cpsw_free_slot(sc, &sc->_tx_slots[i]);
-	}
-	for (i = 0; i < CPSW_MAX_RX_BUFFERS; i++) {
-		cpsw_free_slot(sc, &sc->_rx_slots[i]);
+	for (i = 0; i < sizeof(sc->_slots) / sizeof(sc->_slots[0]); ++i) {
+		cpsw_free_slot(sc, &sc->_slots[i]);
 	}
 
 	/* Free DMA tag */
@@ -424,409 +754,323 @@ cpsw_detach(device_t dev)
 	bus_release_resources(dev, res_spec, sc->res);
 
 	/* Destroy mutexes */
-	mtx_destroy(&sc->rx_lock);
-	mtx_destroy(&sc->tx_lock);
+	mtx_destroy(&sc->rx.lock);
+	mtx_destroy(&sc->tx.lock);
 
 	return (0);
 }
 
-static int
-cpsw_suspend(device_t dev)
+/*
+ *
+ * Init/Shutdown.
+ *
+ */
+
+static void
+cpsw_reset(struct cpsw_softc *sc)
 {
-	struct cpsw_softc *sc = device_get_softc(dev);
+	int i;
 
-	CPSW_DEBUGF((""));
-	CPSW_GLOBAL_LOCK(sc);
-	cpsw_stop_locked(sc);
-	CPSW_GLOBAL_UNLOCK(sc);
-	return (0);
-}
+	/* Reset RMII/RGMII wrapper. */
+	cpsw_write_4(sc, CPSW_WR_SOFT_RESET, 1);
+	while (cpsw_read_4(sc, CPSW_WR_SOFT_RESET) & 1)
+		;
 
-static int
-cpsw_resume(device_t dev)
-{
-	/* XXX TODO XXX */
-	device_printf(dev, "%s\n", __FUNCTION__);
-	return (0);
+	/* Disable TX and RX interrupts for all cores. */
+	for (i = 0; i < 3; ++i) {
+		cpsw_write_4(sc, CPSW_WR_C_RX_THRESH_EN(i), 0x00);
+		cpsw_write_4(sc, CPSW_WR_C_TX_EN(i), 0x00);
+		cpsw_write_4(sc, CPSW_WR_C_RX_EN(i), 0x00);
+		cpsw_write_4(sc, CPSW_WR_C_MISC_EN(i), 0x00);
+	}
+
+	/* Reset CPSW subsystem. */
+	cpsw_write_4(sc, CPSW_SS_SOFT_RESET, 1);
+	while (cpsw_read_4(sc, CPSW_SS_SOFT_RESET) & 1)
+		;
+
+	/* Reset Sliver port 1 and 2 */
+	for (i = 0; i < 2; i++) {
+		/* Reset */
+		cpsw_write_4(sc, CPSW_SL_SOFT_RESET(i), 1);
+		while (cpsw_read_4(sc, CPSW_SL_SOFT_RESET(i)) & 1)
+			;
+	}
+
+	/* Reset DMA controller. */
+	cpsw_write_4(sc, CPSW_CPDMA_SOFT_RESET, 1);
+	while (cpsw_read_4(sc, CPSW_CPDMA_SOFT_RESET) & 1)
+		;
+
+	/* Disable TX & RX DMA */
+	cpsw_write_4(sc, CPSW_CPDMA_TX_CONTROL, 0);
+	cpsw_write_4(sc, CPSW_CPDMA_RX_CONTROL, 0);
+
+	/* Clear all queues. */
+	for (i = 0; i < 8; i++) {
+		cpsw_write_4(sc, CPSW_CPDMA_TX_HDP(i), 0);
+		cpsw_write_4(sc, CPSW_CPDMA_RX_HDP(i), 0);
+		cpsw_write_4(sc, CPSW_CPDMA_TX_CP(i), 0);
+		cpsw_write_4(sc, CPSW_CPDMA_RX_CP(i), 0);
+	}
+
+	/* Clear all interrupt Masks */
+	cpsw_write_4(sc, CPSW_CPDMA_RX_INTMASK_CLEAR, 0xFFFFFFFF);
+	cpsw_write_4(sc, CPSW_CPDMA_TX_INTMASK_CLEAR, 0xFFFFFFFF);
 }
 
-static int
-cpsw_shutdown(device_t dev)
+static void
+cpsw_init(void *arg)
 {
-	struct cpsw_softc *sc = device_get_softc(dev);
+	struct cpsw_softc *sc = arg;
 
 	CPSW_DEBUGF((""));
 	CPSW_GLOBAL_LOCK(sc);
-	cpsw_stop_locked(sc);
+	cpsw_init_locked(arg);
 	CPSW_GLOBAL_UNLOCK(sc);
-	return (0);
 }
 
-static int
-cpsw_miibus_ready(struct cpsw_softc *sc)
+static void
+cpsw_init_locked(void *arg)
 {
-	uint32_t r, retries = CPSW_MIIBUS_RETRIES;
+	struct ifnet *ifp;
+	struct cpsw_softc *sc = arg;
+	struct cpsw_slot *slot;
+	uint32_t i;
 
-	while (--retries) {
-		r = cpsw_read_4(MDIOUSERACCESS0);
-		if ((r & 1 << 31) == 0)
-			return 1;
-		DELAY(CPSW_MIIBUS_DELAY);
-	}
-	return 0;
-}
+	CPSW_DEBUGF((""));
+	ifp = sc->ifp;
+	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
+		return;
 
-static int
-cpsw_miibus_readreg(device_t dev, int phy, int reg)
-{
-	struct cpsw_softc *sc = device_get_softc(dev);
-	uint32_t cmd, r;
+	getbinuptime(&sc->init_uptime);
 
-	if (!cpsw_miibus_ready(sc)) {
-		device_printf(dev, "MDIO not ready to read\n");
-		return 0;
-	}
+	/* Reset the controller. */
+	cpsw_reset(sc);
 
-	/* Set GO, reg, phy */
-	cmd = 1 << 31 | (reg & 0x1F) << 21 | (phy & 0x1F) << 16;
-	cpsw_write_4(MDIOUSERACCESS0, cmd);
+	/* Enable ALE */
+	cpsw_write_4(sc, CPSW_ALE_CONTROL, 1 << 31 | 1 << 4);
 
-	if (!cpsw_miibus_ready(sc)) {
-		device_printf(dev, "MDIO timed out during read\n");
-		return 0;
-	}
-
-	r = cpsw_read_4(MDIOUSERACCESS0);
-	if((r & 1 << 29) == 0) {
-		device_printf(dev, "Failed to read from PHY.\n");
-		r = 0;
+	/* Init Sliver port 1 and 2 */
+	for (i = 0; i < 2; i++) {
+		/* Set Slave Mapping */
+		cpsw_write_4(sc, CPSW_SL_RX_PRI_MAP(i), 0x76543210);
+		cpsw_write_4(sc, CPSW_PORT_P_TX_PRI_MAP(i + 1), 0x33221100);
+		cpsw_write_4(sc, CPSW_SL_RX_MAXLEN(i), 0x5f2);
+		/* Set MACCONTROL for ports 0,1: IFCTL_B(16), IFCTL_A(15),
+		   GMII_EN(5), FULLDUPLEX(1) */
+		/* TODO: Docs claim that IFCTL_B and IFCTL_A do the same thing? */
+		/* Huh?  Docs call bit 0 "Loopback" some places, "FullDuplex" others. */
+		cpsw_write_4(sc, CPSW_SL_MACCONTROL(i), 1 << 15 | 1 << 5 | 1);
 	}
-	return (r & 0xFFFF);
-}
-
-static int
-cpsw_miibus_writereg(device_t dev, int phy, int reg, int value)
-{
-	struct cpsw_softc *sc = device_get_softc(dev);
-	uint32_t cmd;
 
-	if (!cpsw_miibus_ready(sc)) {
-		device_printf(dev, "MDIO not ready to write\n");
-		return 0;
-	}
+	/* Set Host Port Mapping */
+	cpsw_write_4(sc, CPSW_PORT_P0_CPDMA_TX_PRI_MAP, 0x76543210);
+	cpsw_write_4(sc, CPSW_PORT_P0_CPDMA_RX_CH_MAP, 0);
 
-	/* Set GO, WRITE, reg, phy, and value */
-	cmd = 3 << 30 | (reg & 0x1F) << 21 | (phy & 0x1F) << 16
-	    | (value & 0xFFFF);
-	cpsw_write_4(MDIOUSERACCESS0, cmd);
+	/* Initialize ALE: all ports set to forwarding(3), initialize addrs */
+	for (i = 0; i < 3; i++)
+		cpsw_write_4(sc, CPSW_ALE_PORTCTL(i), 3);
+	cpsw_ale_update_addresses(sc, 1);
 
-	if (!cpsw_miibus_ready(sc)) {
-		device_printf(dev, "MDIO timed out during write\n");
-		return 0;
-	}
+	cpsw_write_4(sc, CPSW_SS_PTYPE, 0);
 
-	if((cpsw_read_4(MDIOUSERACCESS0) & (1 << 29)) == 0)
-		device_printf(dev, "Failed to write to PHY.\n");
+	/* Enable statistics for ports 0, 1 and 2 */
+	cpsw_write_4(sc, CPSW_SS_STAT_PORT_EN, 7);
 
-	return 0;
-}
+	/* Experiment:  Turn off flow control */
+	/* This seems to fix the watchdog resets that have plagued
+	   earlier versions of this driver; I'm not yet sure if there
+	   are negative effects yet. */
+	cpsw_write_4(sc, CPSW_SS_FLOW_CONTROL, 0);
 
-static int
-cpsw_init_slot_lists(struct cpsw_softc *sc)
-{
-	int i;
+	/* Make IP hdr aligned with 4 */
+	cpsw_write_4(sc, CPSW_CPDMA_RX_BUFFER_OFFSET, 2);
 
-	STAILQ_INIT(&sc->rx_active);
-	STAILQ_INIT(&sc->rx_avail);
-	STAILQ_INIT(&sc->tx_active);
-	STAILQ_INIT(&sc->tx_avail);
-
-	/* Put the slot descriptors onto the avail lists. */
-	for (i = 0; i < CPSW_MAX_TX_BUFFERS; i++) {
-		struct cpsw_slot *slot = &sc->_tx_slots[i];
-		slot->index = i;
-		/* XXX TODO: Remove this from here; allocate dmamaps lazily
-		   in the encap routine to reduce memory usage. */
-		if (bus_dmamap_create(sc->mbuf_dtag, 0, &slot->dmamap)) {
-			if_printf(sc->ifp, "failed to create dmamap for tx mbuf\n");
-			return (ENOMEM);
-		}
-		STAILQ_INSERT_TAIL(&sc->tx_avail, slot, next);
-	}
+	/* Initialize RX Buffer Descriptors */
+	cpsw_write_4(sc, CPSW_CPDMA_RX_FREEBUFFER(0), 0);
 
-	for (i = 0; i < CPSW_MAX_RX_BUFFERS; i++) {
-		struct cpsw_slot *slot = &sc->_rx_slots[i];
-		slot->index = i;
-		if (bus_dmamap_create(sc->mbuf_dtag, 0, &slot->dmamap)) {
-			if_printf(sc->ifp, "failed to create dmamap for rx mbuf\n");
-			return (ENOMEM);
-		}
-		STAILQ_INSERT_TAIL(&sc->rx_avail, slot, next);
-	}
+	/* Enable TX & RX DMA */
+	cpsw_write_4(sc, CPSW_CPDMA_TX_CONTROL, 1);
+	cpsw_write_4(sc, CPSW_CPDMA_RX_CONTROL, 1);
 
-	return (0);
-}
+	/* Enable Interrupts for core 0 */
+	cpsw_write_4(sc, CPSW_WR_C_RX_THRESH_EN(0), 0xFF);
+	cpsw_write_4(sc, CPSW_WR_C_RX_EN(0), 0xFF);
+	cpsw_write_4(sc, CPSW_WR_C_MISC_EN(0), 0x3F);
 
-static void
-cpsw_free_slot(struct cpsw_softc *sc, struct cpsw_slot *slot)
-{
-	int error;
+	/* Enable host Error Interrupt */
+	cpsw_write_4(sc, CPSW_CPDMA_DMA_INTMASK_SET, 3);
 
-	if (slot->dmamap) {
-		error = bus_dmamap_destroy(sc->mbuf_dtag, slot->dmamap);
-		KASSERT(error == 0, ("Mapping still active"));
-		slot->dmamap = NULL;
-	}
-	if (slot->mbuf) {
-		m_freem(slot->mbuf);
-		slot->mbuf = NULL;
-	}
-}
+	/* Enable interrupts for RX Channel 0 */
+	cpsw_write_4(sc, CPSW_CPDMA_RX_INTMASK_SET, 1);
 
-/*
- * Pad the packet to the minimum length for Ethernet.
- * (CPSW hardware doesn't do this for us.)
- */
-static int
-cpsw_pad(struct mbuf *m)
-{
-	int padlen = ETHER_MIN_LEN - m->m_pkthdr.len;
-	struct mbuf *last, *n;
+	/* Initialze MDIO - ENABLE, PREAMBLE=0, FAULTENB, CLKDIV=0xFF */
+	/* TODO Calculate MDCLK=CLK/(CLKDIV+1) */
+	cpsw_write_4(sc, MDIOCONTROL, 1 << 30 | 1 << 18 | 0xFF);
 
-	if (padlen <= 0)
-		return (0);
+	/* Select MII in GMII_SEL, Internal Delay mode */
+	//ti_scm_reg_write_4(0x650, 0);
 
-	/* If there's only the packet-header and we can pad there, use it. */
-	if (m->m_pkthdr.len == m->m_len && M_WRITABLE(m) &&
-	    M_TRAILINGSPACE(m) >= padlen) {
-		last = m;
-	} else {
-		/*
-		 * Walk packet chain to find last mbuf. We will either
-		 * pad there, or append a new mbuf and pad it.
-		 */
-		for (last = m; last->m_next != NULL; last = last->m_next)
-			;
-		if (!(M_WRITABLE(last) && M_TRAILINGSPACE(last) >= padlen)) {
-			/* Allocate new empty mbuf, pad it. Compact later. */
-			MGET(n, M_NOWAIT, MT_DATA);
-			if (n == NULL)
-				return (ENOBUFS);
-			n->m_len = 0;
-			last->m_next = n;
-			last = n;
-		}
-	}
+	/* Initialize active queues. */
+	slot = STAILQ_FIRST(&sc->tx.active);
+	if (slot != NULL)
+		cpsw_write_hdp_slot(sc, &sc->tx, slot);
+	slot = STAILQ_FIRST(&sc->rx.active);
+	if (slot != NULL)
+		cpsw_write_hdp_slot(sc, &sc->rx, slot);
+	cpsw_rx_enqueue(sc);
 
-	/* Now zero the pad area. */
-	memset(mtod(last, caddr_t) + last->m_len, 0, padlen);
-	last->m_len += padlen;
-	m->m_pkthdr.len += padlen;
+	/* Activate network interface */
+	sc->rx.running = 1;
+	sc->tx.running = 1;
+	sc->watchdog.timer = 0;
+	callout_reset(&sc->watchdog.callout, hz, cpsw_tick, sc);
+	sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
+	sc->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 
-	return (0);
 }
 
-static void
-cpsw_start(struct ifnet *ifp)
+static int
+cpsw_shutdown(device_t dev)
 {
-	struct cpsw_softc *sc = ifp->if_softc;
+	struct cpsw_softc *sc = device_get_softc(dev);
 
-	CPSW_TX_LOCK(sc);
-	cpsw_start_locked(ifp);
-	CPSW_TX_UNLOCK(sc);
+	CPSW_DEBUGF((""));
+	CPSW_GLOBAL_LOCK(sc);
+	cpsw_shutdown_locked(sc);
+	CPSW_GLOBAL_UNLOCK(sc);
+	return (0);
 }
 
 static void
-cpsw_start_locked(struct ifnet *ifp)
+cpsw_rx_teardown_locked(struct cpsw_softc *sc)
 {
-	bus_dma_segment_t seg[1];
-	struct cpsw_cpdma_bd bd;
-	struct cpsw_softc *sc = ifp->if_softc;
-	struct cpsw_queue newslots = STAILQ_HEAD_INITIALIZER(newslots);
-	struct cpsw_slot *slot, *prev_slot = NULL, *first_new_slot;
-	struct mbuf *m0, *mtmp;
-	int error, nsegs, enqueued = 0;
-
-	CPSW_TX_LOCK_ASSERT(sc);
-
-	if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
-	    IFF_DRV_RUNNING)
-		return;
+	struct mbuf *received, *next;
+	int i = 0;
 
-	/* Pull pending packets from IF queue and prep them for DMA. */
+	CPSW_DEBUGF(("starting RX teardown"));
+	cpsw_write_4(sc, CPSW_CPDMA_RX_TEARDOWN, 0);
 	for (;;) {
-		slot = STAILQ_FIRST(&sc->tx_avail);
-		if (slot == NULL) {
-			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
-			break;
+		received = cpsw_rx_dequeue(sc);
+		CPSW_GLOBAL_UNLOCK(sc);
+		while (received != NULL) {
+			next = received->m_nextpkt;
+			received->m_nextpkt = NULL;
+			(*sc->ifp->if_input)(sc->ifp, received);
+			received = next;
 		}
-
-		IF_DEQUEUE(&ifp->if_snd, m0);
-		if (m0 == NULL)
-			break;
-
-		if ((error = cpsw_pad(m0))) {
-			if_printf(ifp,
-			    "%s: Dropping packet; could not pad\n", __func__);
-			m_freem(m0);
-			continue;
+		CPSW_GLOBAL_LOCK(sc);
+		if (!sc->rx.running) {
+			CPSW_DEBUGF(("finished RX teardown (%d retries)", i));
+			return;
 		}
-
-		/* TODO: don't defragment here, queue each
-		   packet fragment as a separate entry. */
-		mtmp = m_defrag(m0, M_NOWAIT);
-		if (mtmp)
-			m0 = mtmp;
-
-		slot->mbuf = m0;
-		/* Create mapping in DMA memory */
-		error = bus_dmamap_load_mbuf_sg(sc->mbuf_dtag, slot->dmamap,
-		    m0, seg, &nsegs, BUS_DMA_NOWAIT);
-		KASSERT(nsegs == 1, ("More than one segment (nsegs=%d)", nsegs));
-		KASSERT(error == 0, ("DMA error (error=%d)", error));
-		if (error != 0 || nsegs != 1) {
-			if_printf(ifp,
-			    "%s: Can't load packet for DMA (nsegs=%d, error=%d), dropping packet\n",
-			    __func__, nsegs, error);
-			bus_dmamap_unload(sc->mbuf_dtag, slot->dmamap);
-			m_freem(m0);
-			break;
+		if (++i > 10) {
+			if_printf(sc->ifp, "Unable to cleanly shutdown receiver\n");
+			return;
 		}
-		bus_dmamap_sync(sc->mbuf_dtag, slot->dmamap,
-				BUS_DMASYNC_PREWRITE);
-
-		if (prev_slot != NULL)
-			cpsw_cpdma_write_txbd_next(prev_slot->index,
-			    cpsw_cpdma_txbd_paddr(slot->index));
-		bd.next = 0;
-		bd.bufptr = seg->ds_addr;
-		bd.bufoff = 0;
-		bd.buflen = seg->ds_len;
-		bd.pktlen = seg->ds_len;
-		bd.flags = 7 << 13;	/* Set OWNERSHIP, SOP, EOP */
-		cpsw_cpdma_write_txbd(slot->index, &bd);
-		++enqueued;
-
-		prev_slot = slot;
-		STAILQ_REMOVE_HEAD(&sc->tx_avail, next);
-		STAILQ_INSERT_TAIL(&newslots, slot, next);
-		BPF_MTAP(ifp, m0);
+		DELAY(10);
 	}
+}
 
-	if (STAILQ_EMPTY(&newslots))
-		return;
+static void
+cpsw_tx_teardown_locked(struct cpsw_softc *sc)
+{
+	int i = 0;
 
-	/* Attach the list of new buffers to the hardware TX queue. */
-	prev_slot = STAILQ_LAST(&sc->tx_active, cpsw_slot, next);
-	first_new_slot = STAILQ_FIRST(&newslots);
-	STAILQ_CONCAT(&sc->tx_active, &newslots);
-	if (prev_slot == NULL) {
-		/* Start the TX queue fresh. */
-		cpsw_write_4(CPSW_CPDMA_TX_HDP(0),
-		    cpsw_cpdma_txbd_paddr(first_new_slot->index));
-	} else {
-		/* Add buffers to end of current queue. */
-		cpsw_cpdma_write_txbd_next(prev_slot->index,
-		   cpsw_cpdma_txbd_paddr(first_new_slot->index));
-		/* If underrun, restart queue. */
-		if (cpsw_cpdma_read_txbd_flags(prev_slot->index) & CPDMA_BD_EOQ)
-			cpsw_write_4(CPSW_CPDMA_TX_HDP(0),
-			    cpsw_cpdma_txbd_paddr(first_new_slot->index));
-	}
-	sc->tx_enqueues += enqueued;
-	sc->tx_queued += enqueued;
-	if (sc->tx_queued > sc->tx_max_queued) {
-		sc->tx_max_queued = sc->tx_queued;
-		CPSW_DEBUGF(("New TX high water mark %d", sc->tx_queued));
+	CPSW_DEBUGF(("starting TX teardown"));
+	cpsw_write_4(sc, CPSW_CPDMA_TX_TEARDOWN, 0);
+	cpsw_tx_dequeue(sc);
+	while (sc->tx.running && ++i < 10) {
+		DELAY(10);
+		cpsw_tx_dequeue(sc);
 	}
+	if (sc->tx.running)
+		if_printf(sc->ifp, "Unable to cleanly shutdown transmitter\n");
+	CPSW_DEBUGF(("finished TX teardown (%d retries, %d idle buffers)",
+	    i, sc->tx.active_queue_len));
 }
 
 static void
-cpsw_stop_locked(struct cpsw_softc *sc)
+cpsw_shutdown_locked(struct cpsw_softc *sc)
 {
 	struct ifnet *ifp;
-	int i;
 
 	CPSW_DEBUGF((""));
-
 	CPSW_GLOBAL_LOCK_ASSERT(sc);
-
 	ifp = sc->ifp;
 
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 		return;
 
 	/* Disable interface */
-	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
-
-	/* Stop tick engine */
-	callout_stop(&sc->wd_callout);
+	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+	ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 
-	/* Wait for hardware to clear pending ops. */
-	CPSW_GLOBAL_UNLOCK(sc);
-	CPSW_DEBUGF(("starting RX and TX teardown"));
-	cpsw_write_4(CPSW_CPDMA_RX_TEARDOWN, 0);
-	cpsw_write_4(CPSW_CPDMA_TX_TEARDOWN, 0);
-	i = 0;
-	cpsw_intr_rx(sc); // Try clearing without delay.
-	cpsw_intr_tx(sc);
-	while (sc->rx_running || sc->tx_running) {
-		DELAY(10);
-		cpsw_intr_rx(sc);
-		cpsw_intr_tx(sc);
-		++i;
-	}
-	CPSW_DEBUGF(("finished RX and TX teardown (%d tries)", i));
-	CPSW_GLOBAL_LOCK(sc);
+	/* Stop ticker */
+	callout_stop(&sc->watchdog.callout);
 
-	/* All slots are now available */
-	STAILQ_CONCAT(&sc->rx_avail, &sc->rx_active);
-	STAILQ_CONCAT(&sc->tx_avail, &sc->tx_active);
-	CPSW_DEBUGF(("%d buffers dropped at TX reset", sc->tx_queued));
-	sc->tx_queued = 0;
+	/* Tear down the RX/TX queues. */
+	cpsw_rx_teardown_locked(sc);
+	cpsw_tx_teardown_locked(sc);
 
-	/* Reset writer */
-	cpsw_write_4(CPSW_WR_SOFT_RESET, 1);
-	while (cpsw_read_4(CPSW_WR_SOFT_RESET) & 1)
-		;
+	/* Capture stats before we reset controller. */
+	cpsw_stats_collect(sc);
 
-	/* Reset SS */
-	cpsw_write_4(CPSW_SS_SOFT_RESET, 1);
-	while (cpsw_read_4(CPSW_SS_SOFT_RESET) & 1)
-		;
+	cpsw_reset(sc);
+}
 
-	/* Reset Sliver port 1 and 2 */
-	for (i = 0; i < 2; i++) {
-		/* Reset */
-		cpsw_write_4(CPSW_SL_SOFT_RESET(i), 1);
-		while (cpsw_read_4(CPSW_SL_SOFT_RESET(i)) & 1)
-			;
-	}
+/*
+ *  Suspend/Resume.
+ */
 
-	/* Reset CPDMA */
-	cpsw_write_4(CPSW_CPDMA_SOFT_RESET, 1);
-	while (cpsw_read_4(CPSW_CPDMA_SOFT_RESET) & 1)
-		;
+static int
+cpsw_suspend(device_t dev)
+{
+	struct cpsw_softc *sc = device_get_softc(dev);
 
-	/* Disable TX & RX DMA */
-	cpsw_write_4(CPSW_CPDMA_TX_CONTROL, 0);
-	cpsw_write_4(CPSW_CPDMA_RX_CONTROL, 0);
+	CPSW_DEBUGF((""));
+	CPSW_GLOBAL_LOCK(sc);
+	cpsw_shutdown_locked(sc);
+	CPSW_GLOBAL_UNLOCK(sc);
+	return (0);
+}
 
-	/* Disable TX and RX interrupts for all cores. */
-	for (i = 0; i < 3; ++i) {
-		cpsw_write_4(CPSW_WR_C_TX_EN(i), 0x00);
-		cpsw_write_4(CPSW_WR_C_RX_EN(i), 0x00);
-		cpsw_write_4(CPSW_WR_C_MISC_EN(i), 0x00);
-	}
+static int
+cpsw_resume(device_t dev)
+{
+	struct cpsw_softc *sc = device_get_softc(dev);
 
-	/* Clear all interrupt Masks */
-	cpsw_write_4(CPSW_CPDMA_RX_INTMASK_CLEAR, 0xFFFFFFFF);
-	cpsw_write_4(CPSW_CPDMA_TX_INTMASK_CLEAR, 0xFFFFFFFF);
+	CPSW_DEBUGF(("UNIMPLEMENTED"));
+	return (0);
 }
 
+/*
+ *
+ *  IOCTL
+ *
+ */
+
 static void
 cpsw_set_promisc(struct cpsw_softc *sc, int set)
 {
+	/*
+	 * Enabling promiscuous mode requires two bits of work: First,
+	 * ALE_BYPASS needs to be enabled.  That disables the ALE
+	 * forwarding logic and causes every packet to be sent to the
+	 * host port.  That makes us promiscuous wrt received packets.
+	 *
+	 * With ALE forwarding disabled, the transmitter needs to set
+	 * an explicit output port on every packet to route it to the
+	 * correct egress.  This should be doable for systems such as
+	 * BeagleBone where only one egress port is actually wired to
+	 * a PHY.  If you have both egress ports wired up, life gets a
+	 * lot more interesting.
+	 *
+	 * Hmmm.... NetBSD driver uses ALE_BYPASS always and doesn't
+	 * seem to set explicit egress ports.  Does that mean they
+	 * are always promiscuous?
+	 */
 	if (set) {
 		printf("Promiscuous mode unimplemented\n");
 	}
@@ -848,7 +1092,6 @@ cpsw_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 	int error;
 	uint32_t changed;
 
-	CPSW_DEBUGF(("command=0x%lx", command));
 	error = 0;
 
 	switch (command) {
@@ -865,118 +1108,175 @@ cpsw_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 					cpsw_set_allmulti(sc,
 					    ifp->if_flags & IFF_ALLMULTI);
 			} else {
-				CPSW_DEBUGF(("SIOCSIFFLAGS: UP but not RUNNING"));
+				CPSW_DEBUGF(("SIOCSIFFLAGS: UP but not RUNNING; starting up"));
 				cpsw_init_locked(sc);
 			}
 		} else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
-			CPSW_DEBUGF(("SIOCSIFFLAGS: not UP but RUNNING"));
-			cpsw_stop_locked(sc);
+			CPSW_DEBUGF(("SIOCSIFFLAGS: not UP but RUNNING; shutting down"));
+			cpsw_shutdown_locked(sc);
 		}
 
 		sc->cpsw_if_flags = ifp->if_flags;
 		CPSW_GLOBAL_UNLOCK(sc);
 		break;
 	case SIOCADDMULTI:
-		CPSW_DEBUGF(("SIOCADDMULTI unimplemented"));
+		cpsw_ale_update_addresses(sc, 0);
 		break;
 	case SIOCDELMULTI:
-		CPSW_DEBUGF(("SIOCDELMULTI unimplemented"));
+		/* Ugh.  DELMULTI doesn't provide the specific address
+		   being removed, so the best we can do is remove
+		   everything and rebuild it all. */
+		cpsw_ale_update_addresses(sc, 1);
 		break;
 	case SIOCGIFMEDIA:
 	case SIOCSIFMEDIA:
 		error = ifmedia_ioctl(ifp, ifr, &sc->mii->mii_media, command);
 		break;
 	default:
-		CPSW_DEBUGF(("ether ioctl"));
 		error = ether_ioctl(ifp, command, data);
 	}
 	return (error);
 }
 
-static void
-cpsw_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
+/*
+ *
+ * MIIBUS
+ *
+ */
+static int
+cpsw_miibus_ready(struct cpsw_softc *sc)
 {
-	struct cpsw_softc *sc = ifp->if_softc;
-	struct mii_data *mii;
+	uint32_t r, retries = CPSW_MIIBUS_RETRIES;
 
-	CPSW_DEBUGF((""));
-	CPSW_TX_LOCK(sc);
+	while (--retries) {
+		r = cpsw_read_4(sc, MDIOUSERACCESS0);
+		if ((r & 1 << 31) == 0)
+			return 1;
+		DELAY(CPSW_MIIBUS_DELAY);
+	}
+	return 0;
+}
 
-	mii = sc->mii;
-	mii_pollstat(mii);
+static int
+cpsw_miibus_readreg(device_t dev, int phy, int reg)
+{
+	struct cpsw_softc *sc = device_get_softc(dev);
+	uint32_t cmd, r;
 
-	ifmr->ifm_active = mii->mii_media_active;
-	ifmr->ifm_status = mii->mii_media_status;
+	if (!cpsw_miibus_ready(sc)) {
+		device_printf(dev, "MDIO not ready to read\n");
+		return 0;
+	}
 
-	CPSW_TX_UNLOCK(sc);
-}
+	/* Set GO, reg, phy */
+	cmd = 1 << 31 | (reg & 0x1F) << 21 | (phy & 0x1F) << 16;
+	cpsw_write_4(sc, MDIOUSERACCESS0, cmd);
 
+	if (!cpsw_miibus_ready(sc)) {
+		device_printf(dev, "MDIO timed out during read\n");
+		return 0;
+	}
+
+	r = cpsw_read_4(sc, MDIOUSERACCESS0);
+	if((r & 1 << 29) == 0) {
+		device_printf(dev, "Failed to read from PHY.\n");
+		r = 0;
+	}
+	return (r & 0xFFFF);
+}
 
 static int
-cpsw_ifmedia_upd(struct ifnet *ifp)
+cpsw_miibus_writereg(device_t dev, int phy, int reg, int value)
 {
-	struct cpsw_softc *sc = ifp->if_softc;
+	struct cpsw_softc *sc = device_get_softc(dev);
+	uint32_t cmd;
 
-	CPSW_DEBUGF((""));
-	if (ifp->if_flags & IFF_UP) {
-		CPSW_GLOBAL_LOCK(sc);
-		sc->cpsw_media_status = sc->mii->mii_media.ifm_media;
-		mii_mediachg(sc->mii);
-		cpsw_init_locked(sc);
-		CPSW_GLOBAL_UNLOCK(sc);
+	if (!cpsw_miibus_ready(sc)) {
+		device_printf(dev, "MDIO not ready to write\n");
+		return 0;
 	}
 
-	return (0);
-}
+	/* Set GO, WRITE, reg, phy, and value */
+	cmd = 3 << 30 | (reg & 0x1F) << 21 | (phy & 0x1F) << 16
+	    | (value & 0xFFFF);
+	cpsw_write_4(sc, MDIOUSERACCESS0, cmd);
 
-static void
-cpsw_intr_rx_thresh(void *arg)
-{
-	struct cpsw_softc *sc = arg;
-	CPSW_DEBUGF((""));
+	if (!cpsw_miibus_ready(sc)) {
+		device_printf(dev, "MDIO timed out during write\n");
+		return 0;
+	}
+
+	if((cpsw_read_4(sc, MDIOUSERACCESS0) & (1 << 29)) == 0)
+		device_printf(dev, "Failed to write to PHY.\n");
+
+	return 0;
 }
 
+/*
+ *
+ * Transmit/Receive Packets.
+ *
+ */
+
+
 static void
 cpsw_intr_rx(void *arg)
 {
 	struct cpsw_softc *sc = arg;
+	struct mbuf *received, *next;
 
 	CPSW_RX_LOCK(sc);
-	cpsw_intr_rx_locked(arg);
-	cpsw_write_4(CPSW_CPDMA_CPDMA_EOI_VECTOR, 1);
+	received = cpsw_rx_dequeue(sc);
+	cpsw_rx_enqueue(sc);
+	cpsw_write_4(sc, CPSW_CPDMA_CPDMA_EOI_VECTOR, 1);
 	CPSW_RX_UNLOCK(sc);
+	
+	while (received != NULL) {
+		next = received->m_nextpkt;
+		received->m_nextpkt = NULL;
+		(*sc->ifp->if_input)(sc->ifp, received);
+		received = next;
+	}
 }
 
-static void
-cpsw_intr_rx_locked(void *arg)
+static struct mbuf *
+cpsw_rx_dequeue(struct cpsw_softc *sc)
 {
-	struct cpsw_softc *sc = arg;
 	struct cpsw_cpdma_bd bd;
-	struct cpsw_slot *slot, *last_slot = NULL;
+	struct cpsw_slot *slot;
 	struct ifnet *ifp;
+	struct mbuf *mb_head, *mb_tail;
+	int removed = 0;
 
 	ifp = sc->ifp;
-	if (!sc->rx_running)
-		return;
+	mb_head = mb_tail = NULL;
 
 	/* Pull completed packets off hardware RX queue. */
-	slot = STAILQ_FIRST(&sc->rx_active);
-	while (slot != NULL) {
-		cpsw_cpdma_read_rxbd(slot->index, &bd);
+	while ((slot = STAILQ_FIRST(&sc->rx.active)) != NULL) {
+		cpsw_cpdma_read_bd(sc, slot, &bd);
 		if (bd.flags & CPDMA_BD_OWNER)
 			break; /* Still in use by hardware */
 
+		CPSW_DEBUGF(("Removing received packet from RX queue"));
+		++removed;
+		STAILQ_REMOVE_HEAD(&sc->rx.active, next);
+		STAILQ_INSERT_TAIL(&sc->rx.avail, slot, next);
+
+		bus_dmamap_sync(sc->mbuf_dtag, slot->dmamap, BUS_DMASYNC_POSTREAD);
+		bus_dmamap_unload(sc->mbuf_dtag, slot->dmamap);
+
 		if (bd.flags & CPDMA_BD_TDOWNCMPLT) {
 			CPSW_DEBUGF(("RX teardown in progress"));
-			cpsw_write_4(CPSW_CPDMA_RX_CP(0), 0xfffffffc);
-			sc->rx_running = 0;
-			return;
+			m_freem(slot->mbuf);
+			slot->mbuf = NULL;
+			cpsw_write_cp(sc, &sc->rx, 0xfffffffc);
+			sc->rx.running = 0;
+			break;
 		}
 
-		bus_dmamap_sync(sc->mbuf_dtag, slot->dmamap, BUS_DMASYNC_POSTREAD);
-		bus_dmamap_unload(sc->mbuf_dtag, slot->dmamap);
+		cpsw_write_cp_slot(sc, &sc->rx, slot);
 
-		/* Fill mbuf */
+		/* Set up mbuf */
 		/* TODO: track SOP/EOP bits to assemble a full mbuf
 		   out of received fragments. */
 		slot->mbuf->m_hdr.mh_data += bd.bufoff;
@@ -984,6 +1284,7 @@ cpsw_intr_rx_locked(void *arg)
 		slot->mbuf->m_pkthdr.len = bd.pktlen - 4;
 		slot->mbuf->m_flags |= M_PKTHDR;
 		slot->mbuf->m_pkthdr.rcvif = ifp;
+		slot->mbuf->m_nextpkt = NULL;
 
 		if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) {
 			/* check for valid CRC by looking into pkt_err[5:4] */
@@ -994,64 +1295,60 @@ cpsw_intr_rx_locked(void *arg)
 			}
 		}
 
-		/* Handover packet */
-		CPSW_RX_UNLOCK(sc);
-		(*ifp->if_input)(ifp, slot->mbuf);
+		/* Add mbuf to packet list to be returned. */
+		if (mb_tail) {
+			mb_tail->m_nextpkt = slot->mbuf;
+		} else {
+			mb_head = slot->mbuf;
+		}
+		mb_tail = slot->mbuf;
 		slot->mbuf = NULL;
-		CPSW_RX_LOCK(sc);
-
-		last_slot = slot;
-		STAILQ_REMOVE_HEAD(&sc->rx_active, next);
-		STAILQ_INSERT_TAIL(&sc->rx_avail, slot, next);
-		slot = STAILQ_FIRST(&sc->rx_active);
 	}
 
-	/* Tell hardware last slot we processed. */
-	if (last_slot)
-		cpsw_write_4(CPSW_CPDMA_RX_CP(0),
-		    cpsw_cpdma_rxbd_paddr(last_slot->index));
-
-	/* Repopulate hardware RX queue. */
-	cpsw_fill_rx_queue_locked(sc);
+	if (removed != 0) {
+		sc->rx.queue_removes += removed;
+		sc->rx.active_queue_len -= removed;
+		sc->rx.avail_queue_len += removed;
+		if (sc->rx.avail_queue_len > sc->rx.max_avail_queue_len)
+			sc->rx.max_avail_queue_len = sc->rx.avail_queue_len;
+	}
+	return (mb_head);
 }
 
 static void
-cpsw_fill_rx_queue_locked(struct cpsw_softc *sc)
+cpsw_rx_enqueue(struct cpsw_softc *sc)
 {
 	bus_dma_segment_t seg[1];
-	struct cpsw_queue tmpqueue = STAILQ_HEAD_INITIALIZER(tmpqueue);
 	struct cpsw_cpdma_bd bd;
-	struct cpsw_slot *slot, *prev_slot, *next_slot;
-	int error, nsegs;
+	struct ifnet *ifp = sc->ifp;
+	struct cpsw_slots tmpqueue = STAILQ_HEAD_INITIALIZER(tmpqueue);
+	struct cpsw_slot *slot, *prev_slot = NULL;
+	struct cpsw_slot *last_old_slot, *first_new_slot;
+	int error, nsegs, added = 0;
 
-	/* Try to allocate new mbufs. */
-	STAILQ_FOREACH(slot, &sc->rx_avail, next) {
-		if (slot->mbuf != NULL)
-			continue;
-		slot->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
+	/* Register new mbufs with hardware. */
+	while ((slot = STAILQ_FIRST(&sc->rx.avail)) != NULL) {
 		if (slot->mbuf == NULL) {
-			if_printf(sc->ifp, "Unable to fill RX queue\n");
-			break;
+			slot->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
+			if (slot->mbuf == NULL) {
+				if_printf(sc->ifp, "Unable to fill RX queue\n");
+				break;
+			}
+			slot->mbuf->m_len =
+			    slot->mbuf->m_pkthdr.len =
+			    slot->mbuf->m_ext.ext_size;
 		}
-		slot->mbuf->m_len = slot->mbuf->m_pkthdr.len = slot->mbuf->m_ext.ext_size;
-	}
-
-	/* Register new mbufs with hardware. */
-	prev_slot = NULL;
-	while (!STAILQ_EMPTY(&sc->rx_avail)) {
-		slot = STAILQ_FIRST(&sc->rx_avail);
-		if (slot->mbuf == NULL)
-			break;
 
 		error = bus_dmamap_load_mbuf_sg(sc->mbuf_dtag, slot->dmamap,
 		    slot->mbuf, seg, &nsegs, BUS_DMA_NOWAIT);
 
 		KASSERT(nsegs == 1, ("More than one segment (nsegs=%d)", nsegs));
 		KASSERT(error == 0, ("DMA error (error=%d)", error));
-		if (nsegs != 1 || error) {
-			if_printf(sc->ifp,
+		if (error != 0 || nsegs != 1) {
+			if_printf(ifp,
 			    "%s: Can't prep RX buf for DMA (nsegs=%d, error=%d)\n",
 			    __func__, nsegs, error);
+			bus_dmamap_unload(sc->mbuf_dtag, slot->dmamap);
 			m_freem(slot->mbuf);
 			slot->mbuf = NULL;
 			break;
@@ -1062,410 +1359,563 @@ cpsw_fill_rx_queue_locked(struct cpsw_softc *sc)
 		/* Create and submit new rx descriptor*/
 		bd.next = 0;
 		bd.bufptr = seg->ds_addr;
-		bd.buflen = MCLBYTES-1;
-		bd.bufoff = 2; /* make IP hdr aligned with 4 */
-		bd.pktlen = 0;
+		bd.bufoff = 0;
+		bd.buflen = MCLBYTES - 1;
+		bd.pktlen = bd.buflen;
 		bd.flags = CPDMA_BD_OWNER;
-		cpsw_cpdma_write_rxbd(slot->index, &bd);
+		cpsw_cpdma_write_bd(sc, slot, &bd);
+		++added;
 
-		if (prev_slot) {
-			cpsw_cpdma_write_rxbd_next(prev_slot->index,
-			    cpsw_cpdma_rxbd_paddr(slot->index));
-		}
+		if (prev_slot != NULL)
+			cpsw_cpdma_write_bd_next(sc, prev_slot, slot);
 		prev_slot = slot;
-		STAILQ_REMOVE_HEAD(&sc->rx_avail, next);
+		STAILQ_REMOVE_HEAD(&sc->rx.avail, next);
+		sc->rx.avail_queue_len--;
 		STAILQ_INSERT_TAIL(&tmpqueue, slot, next);
 	}
 
+	if (added == 0)
+		return;
+
+	CPSW_DEBUGF(("Adding %d buffers to RX queue", added));
+
 	/* Link new entries to hardware RX queue. */
-	prev_slot = STAILQ_LAST(&sc->rx_active, cpsw_slot, next);
-	next_slot = STAILQ_FIRST(&tmpqueue);
-	if (next_slot == NULL) {
+	last_old_slot = STAILQ_LAST(&sc->rx.active, cpsw_slot, next);
+	first_new_slot = STAILQ_FIRST(&tmpqueue);
+	STAILQ_CONCAT(&sc->rx.active, &tmpqueue);
+	if (first_new_slot == NULL) {
 		return;
-	} else if (prev_slot == NULL) {
-		/* Start a fresh RX queue. */
-		cpsw_write_4(CPSW_CPDMA_RX_HDP(0),
-		    cpsw_cpdma_rxbd_paddr(next_slot->index));
+	} else if (last_old_slot == NULL) {
+		/* Start a fresh queue. */
+		cpsw_write_hdp_slot(sc, &sc->rx, first_new_slot);
 	} else {
 		/* Add buffers to end of current queue. */
-		cpsw_cpdma_write_rxbd_next(prev_slot->index,
-		    cpsw_cpdma_rxbd_paddr(next_slot->index));
+		cpsw_cpdma_write_bd_next(sc, last_old_slot, first_new_slot);
 		/* If underrun, restart queue. */
-		if (cpsw_cpdma_read_rxbd_flags(prev_slot->index) & CPDMA_BD_EOQ) {
-			cpsw_write_4(CPSW_CPDMA_RX_HDP(0),
-			    cpsw_cpdma_rxbd_paddr(next_slot->index));
+		if (cpsw_cpdma_read_bd_flags(sc, last_old_slot) & CPDMA_BD_EOQ) {
+			cpsw_write_hdp_slot(sc, &sc->rx, first_new_slot);
 		}
 	}
-	STAILQ_CONCAT(&sc->rx_active, &tmpqueue);
+	sc->rx.queue_adds += added;
+	sc->rx.active_queue_len += added;
+	if (sc->rx.active_queue_len > sc->rx.max_active_queue_len) {
+		sc->rx.max_active_queue_len = sc->rx.active_queue_len;
+	}
 }
 
 static void
-cpsw_intr_tx(void *arg)
+cpsw_start(struct ifnet *ifp)
 {
-	struct cpsw_softc *sc = arg;
+	struct cpsw_softc *sc = ifp->if_softc;
+
 	CPSW_TX_LOCK(sc);
-	cpsw_intr_tx_locked(arg);
-	cpsw_write_4(CPSW_CPDMA_CPDMA_EOI_VECTOR, 2);
+	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && sc->tx.running) {
+		cpsw_tx_enqueue(sc);
+		cpsw_tx_dequeue(sc);
+	}
 	CPSW_TX_UNLOCK(sc);
 }
 
 static void
-cpsw_intr_tx_locked(void *arg)
+cpsw_tx_enqueue(struct cpsw_softc *sc)
 {
-	struct cpsw_softc *sc = arg;
-	struct cpsw_slot *slot, *last_slot = NULL;
-	uint32_t flags, last_flags = 0, retires = 0;
+	bus_dma_segment_t segs[CPSW_TXFRAGS];
+	struct cpsw_cpdma_bd bd;
+	struct cpsw_slots tmpqueue = STAILQ_HEAD_INITIALIZER(tmpqueue);
+	struct cpsw_slot *slot, *prev_slot = NULL;
+	struct cpsw_slot *last_old_slot, *first_new_slot;
+	struct mbuf *m0;
+	int error, nsegs, seg, added = 0, padlen;
+
+	/* Pull pending packets from IF queue and prep them for DMA. */
+	while ((slot = STAILQ_FIRST(&sc->tx.avail)) != NULL) {
+		IF_DEQUEUE(&sc->ifp->if_snd, m0);
+		if (m0 == NULL)
+			break;
+
+		slot->mbuf = m0;
+		padlen = ETHER_MIN_LEN - slot->mbuf->m_pkthdr.len;
+		if (padlen < 0)
+			padlen = 0;
+
+		/* Create mapping in DMA memory */
+		error = bus_dmamap_load_mbuf_sg(sc->mbuf_dtag, slot->dmamap,
+		    slot->mbuf, segs, &nsegs, BUS_DMA_NOWAIT);
+		/* If the packet is too fragmented, try to simplify. */
+		if (error == EFBIG ||
+		    (error == 0 &&
+			nsegs + (padlen > 0 ? 1 : 0) > sc->tx.avail_queue_len)) {
+			bus_dmamap_unload(sc->mbuf_dtag, slot->dmamap);
+			if (padlen > 0) /* May as well add padding. */
+				m_append(slot->mbuf, padlen,
+				    sc->null_mbuf->m_hdr.mh_data);
+			m0 = m_defrag(slot->mbuf, M_NOWAIT);
+			if (m0 == NULL) {
+				if_printf(sc->ifp,
+				    "Can't defragment packet; dropping\n");
+				m_freem(slot->mbuf);
+			} else {
+				CPSW_DEBUGF(("Requeueing defragmented packet"));
+				IF_PREPEND(&sc->ifp->if_snd, m0);
+			}
+			slot->mbuf = NULL;
+			continue;
+		}
+		if (error != 0) {
+			if_printf(sc->ifp,
+			    "%s: Can't setup DMA (error=%d), dropping packet\n",
+			    __func__, error);
+			bus_dmamap_unload(sc->mbuf_dtag, slot->dmamap);
+			m_freem(slot->mbuf);
+			slot->mbuf = NULL;
+			break;
+		}
+
+		bus_dmamap_sync(sc->mbuf_dtag, slot->dmamap,
+				BUS_DMASYNC_PREWRITE);
+
+
+		CPSW_DEBUGF(("Queueing TX packet: %d segments + %d pad bytes",
+			nsegs, padlen));
+
+		/* If there is only one segment, the for() loop
+		 * gets skipped and the single buffer gets set up
+		 * as both SOP and EOP. */
+		/* Start by setting up the first buffer */
+		bd.next = 0;
+		bd.bufptr = segs[0].ds_addr;
+		bd.bufoff = 0;
+		bd.buflen = segs[0].ds_len;
+		bd.pktlen = m_length(slot->mbuf, NULL) + padlen;
+		bd.flags =  CPDMA_BD_SOP | CPDMA_BD_OWNER;
+		for (seg = 1; seg < nsegs; ++seg) {
+			/* Save the previous buffer (which isn't EOP) */
+			cpsw_cpdma_write_bd(sc, slot, &bd);
+			if (prev_slot != NULL)
+				cpsw_cpdma_write_bd_next(sc, prev_slot, slot);
+			prev_slot = slot;
+			STAILQ_REMOVE_HEAD(&sc->tx.avail, next);
+			sc->tx.avail_queue_len--;
+			STAILQ_INSERT_TAIL(&tmpqueue, slot, next);
+			++added;
+			slot = STAILQ_FIRST(&sc->tx.avail);
+
+			/* Setup next buffer (which isn't SOP) */
+			bd.next = 0;
+			bd.bufptr = segs[seg].ds_addr;
+			bd.bufoff = 0;
+			bd.buflen = segs[seg].ds_len;
+			bd.pktlen = 0;
+			bd.flags = CPDMA_BD_OWNER;
+		}
+		/* Save the final buffer. */
+		if (padlen <= 0)
+			bd.flags |= CPDMA_BD_EOP;
+		cpsw_cpdma_write_bd(sc, slot, &bd);
+		if (prev_slot != NULL)
+			cpsw_cpdma_write_bd_next(sc, prev_slot, slot);
+		prev_slot = slot;
+		STAILQ_REMOVE_HEAD(&sc->tx.avail, next);
+		sc->tx.avail_queue_len--;
+		STAILQ_INSERT_TAIL(&tmpqueue, slot, next);
+		++added;
+
+		if (padlen > 0) {
+			slot = STAILQ_FIRST(&sc->tx.avail);
+			STAILQ_REMOVE_HEAD(&sc->tx.avail, next);
+			sc->tx.avail_queue_len--;
+			STAILQ_INSERT_TAIL(&tmpqueue, slot, next);
+			++added;
+
+			/* Setup buffer of null pad bytes (definitely EOP) */
+			cpsw_cpdma_write_bd_next(sc, prev_slot, slot);
+			prev_slot = slot;
+			bd.next = 0;
+			bd.bufptr = sc->null_mbuf_paddr;
+			bd.bufoff = 0;
+			bd.buflen = padlen;
+			bd.pktlen = 0;
+			bd.flags = CPDMA_BD_EOP | CPDMA_BD_OWNER;
+			cpsw_cpdma_write_bd(sc, slot, &bd);
+			++nsegs;
+		}
 
-	if (!sc->tx_running)
+		if (nsegs > sc->tx.longest_chain)
+			sc->tx.longest_chain = nsegs;
+
+		// TODO: Should we defer the BPF tap until
+		// after all packets are queued?
+		BPF_MTAP(sc->ifp, m0);
+	}
+
+	/* Attach the list of new buffers to the hardware TX queue. */
+	last_old_slot = STAILQ_LAST(&sc->tx.active, cpsw_slot, next);
+	first_new_slot = STAILQ_FIRST(&tmpqueue);
+	STAILQ_CONCAT(&sc->tx.active, &tmpqueue);
+	if (first_new_slot == NULL) {
 		return;
+	} else if (last_old_slot == NULL) {
+		/* Start a fresh queue. */
+		cpsw_write_hdp_slot(sc, &sc->tx, first_new_slot);
+	} else {
+		/* Add buffers to end of current queue. */
+		cpsw_cpdma_write_bd_next(sc, last_old_slot, first_new_slot);
+		/* If underrun, restart queue. */
+		if (cpsw_cpdma_read_bd_flags(sc, last_old_slot) & CPDMA_BD_EOQ) {
+			cpsw_write_hdp_slot(sc, &sc->tx, first_new_slot);
+		}
+	}
+	sc->tx.queue_adds += added;
+	sc->tx.active_queue_len += added;
+	if (sc->tx.active_queue_len > sc->tx.max_active_queue_len) {
+		sc->tx.max_active_queue_len = sc->tx.active_queue_len;
+	}
+}
+
+static int
+cpsw_tx_dequeue(struct cpsw_softc *sc)
+{
+	struct cpsw_slot *slot, *last_removed_slot = NULL;
+	uint32_t flags, removed = 0;
 
-	slot = STAILQ_FIRST(&sc->tx_active);
-	if (slot == NULL &&
-	    cpsw_read_4(CPSW_CPDMA_TX_CP(0)) == 0xfffffffc) {
+	slot = STAILQ_FIRST(&sc->tx.active);
+	if (slot == NULL && cpsw_read_cp(sc, &sc->tx) == 0xfffffffc) {
 		CPSW_DEBUGF(("TX teardown of an empty queue"));
-		cpsw_write_4(CPSW_CPDMA_TX_CP(0), 0xfffffffc);
-		sc->tx_running = 0;
-		return;
+		cpsw_write_cp(sc, &sc->tx, 0xfffffffc);
+		sc->tx.running = 0;
+		return (0);
 	}
 
-	/* Pull completed segments off the hardware TX queue. */
+	/* Pull completed buffers off the hardware TX queue. */
 	while (slot != NULL) {
-		flags = cpsw_cpdma_read_txbd_flags(slot->index);
+		flags = cpsw_cpdma_read_bd_flags(sc, slot);
 		if (flags & CPDMA_BD_OWNER)
-			break; /* Hardware is still using this. */
-
-		if (flags & CPDMA_BD_TDOWNCMPLT) {
-			CPSW_DEBUGF(("TX teardown in progress"));
-			cpsw_write_4(CPSW_CPDMA_TX_CP(0), 0xfffffffc);
-			sc->tx_running = 0;
-			return;
-		}
+			break; /* Hardware is still using this packet. */
 
-		/* Release dmamap, free mbuf. */
-		bus_dmamap_sync(sc->mbuf_dtag, slot->dmamap,
-		    BUS_DMASYNC_POSTWRITE);
+		CPSW_DEBUGF(("TX removing completed packet"));
+		bus_dmamap_sync(sc->mbuf_dtag, slot->dmamap, BUS_DMASYNC_POSTWRITE);
 		bus_dmamap_unload(sc->mbuf_dtag, slot->dmamap);
 		m_freem(slot->mbuf);
 		slot->mbuf = NULL;
 
-		STAILQ_REMOVE_HEAD(&sc->tx_active, next);
-		STAILQ_INSERT_TAIL(&sc->tx_avail, slot, next);
-		sc->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+		/* Dequeue any additional buffers used by this packet. */
+		while (slot != NULL && slot->mbuf == NULL) {
+			STAILQ_REMOVE_HEAD(&sc->tx.active, next);
+			STAILQ_INSERT_TAIL(&sc->tx.avail, slot, next);
+			++removed;
+			last_removed_slot = slot;
+			slot = STAILQ_FIRST(&sc->tx.active);
+		}
 
-		last_slot = slot;
-		last_flags = flags;
-		++retires;
-		slot = STAILQ_FIRST(&sc->tx_active);
+		/* TearDown complete is only marked on the SOP for the packet. */
+		if (flags & CPDMA_BD_TDOWNCMPLT) {
+			CPSW_DEBUGF(("TX teardown in progress"));
+			cpsw_write_cp(sc, &sc->tx, 0xfffffffc);
+			// TODO: Increment a count of dropped TX packets
+			sc->tx.running = 0;
+			break;
+		}
 	}
 
-	if (retires != 0) {
-		/* Tell hardware the last item we dequeued. */
-		cpsw_write_4(CPSW_CPDMA_TX_CP(0),
-		     cpsw_cpdma_txbd_paddr(last_slot->index));
-		sc->tx_retires += retires;
-		sc->tx_queued -= retires;
+	if (removed != 0) {
+		cpsw_write_cp_slot(sc, &sc->tx, last_removed_slot);
+		sc->tx.queue_removes += removed;
+		sc->tx.active_queue_len -= removed;
+		sc->tx.avail_queue_len += removed;
+		if (sc->tx.avail_queue_len > sc->tx.max_avail_queue_len)
+			sc->tx.max_avail_queue_len = sc->tx.avail_queue_len;
 	}
+	return (removed);
 }
 
+/*
+ *
+ * Miscellaneous interrupts.
+ *
+ */
+
 static void
-cpsw_intr_misc(void *arg)
+cpsw_intr_rx_thresh(void *arg)
 {
 	struct cpsw_softc *sc = arg;
-	uint32_t stat = cpsw_read_4(CPSW_WR_C_MISC_STAT(0));
+	uint32_t stat = cpsw_read_4(sc, CPSW_WR_C_RX_THRESH_STAT(0));
 
 	CPSW_DEBUGF(("stat=%x", stat));
-	/* EOI_RX_PULSE */
-	cpsw_write_4(CPSW_CPDMA_CPDMA_EOI_VECTOR, 3);
+	cpsw_write_4(sc, CPSW_CPDMA_CPDMA_EOI_VECTOR, 0);
 }
 
 static void
-cpsw_tick(void *msc)
+cpsw_intr_misc_host_error(struct cpsw_softc *sc)
 {
-	struct cpsw_softc *sc = msc;
+	uint32_t intstat;
+	uint32_t dmastat;
+	int txerr, rxerr, txchan, rxchan;
+
+	printf("\n\n");
+	device_printf(sc->dev,
+	    "HOST ERROR:  PROGRAMMING ERROR DETECTED BY HARDWARE\n");
+	printf("\n\n");
+	intstat = cpsw_read_4(sc, CPSW_CPDMA_DMA_INTSTAT_MASKED);
+	device_printf(sc->dev, "CPSW_CPDMA_DMA_INTSTAT_MASKED=0x%x\n", intstat);
+	dmastat = cpsw_read_4(sc, CPSW_CPDMA_DMASTATUS);
+	device_printf(sc->dev, "CPSW_CPDMA_DMASTATUS=0x%x\n", dmastat);
+
+	txerr = (dmastat >> 20) & 15;
+	txchan = (dmastat >> 16) & 7;
+	rxerr = (dmastat >> 12) & 15;
+	rxchan = (dmastat >> 8) & 7;
+
+	switch (txerr) {
+	case 0: break;
+	case 1:	printf("SOP error on TX channel %d\n", txchan);
+		break;
+	case 2:	printf("Ownership bit not set on SOP buffer on TX channel %d\n", txchan);
+		break;
+	case 3:	printf("Zero Next Buffer but not EOP on TX channel %d\n", txchan);
+		break;
+	case 4:	printf("Zero Buffer Pointer on TX channel %d\n", txchan);
+		break;
+	case 5:	printf("Zero Buffer Length on TX channel %d\n", txchan);
+		break;
+	case 6:	printf("Packet length error on TX channel %d\n", txchan);
+		break;
+	default: printf("Unknown error on TX channel %d\n", txchan);
+		break;
+	}
 
-	/* Check for TX timeout */
-	cpsw_tx_watchdog(sc);
+	if (txerr != 0) {
+		printf("CPSW_CPDMA_TX%d_HDP=0x%x\n",
+		    txchan, cpsw_read_4(sc, CPSW_CPDMA_TX_HDP(txchan)));
+		printf("CPSW_CPDMA_TX%d_CP=0x%x\n",
+		    txchan, cpsw_read_4(sc, CPSW_CPDMA_TX_CP(txchan)));
+		cpsw_dump_queue(sc, &sc->tx.active);
+	}
 
-	mii_tick(sc->mii);
+	switch (rxerr) {
+	case 0: break;
+	case 2:	printf("Ownership bit not set on RX channel %d\n", rxchan);
+		break;
+	case 4:	printf("Zero Buffer Pointer on RX channel %d\n", rxchan);
+		break;
+	case 5:	printf("Zero Buffer Length on RX channel %d\n", rxchan);
+		break;
+	case 6:	printf("Buffer offset too big on RX channel %d\n", rxchan);
+		break;
+	default: printf("Unknown RX error on RX channel %d\n", rxchan);
+		break;
+	}
 
-	/* Check for media type change */
-	if(sc->cpsw_media_status != sc->mii->mii_media.ifm_media) {
-		printf("%s: media type changed (ifm_media=%x)\n", __func__, 
-			sc->mii->mii_media.ifm_media);
-		cpsw_ifmedia_upd(sc->ifp);
+	if (rxerr != 0) {
+		printf("CPSW_CPDMA_RX%d_HDP=0x%x\n",
+		    rxchan, cpsw_read_4(sc,CPSW_CPDMA_RX_HDP(rxchan)));
+		printf("CPSW_CPDMA_RX%d_CP=0x%x\n",
+		    rxchan, cpsw_read_4(sc, CPSW_CPDMA_RX_CP(rxchan)));
+		cpsw_dump_queue(sc, &sc->rx.active);
 	}
 
-	/* Schedule another timeout one second from now */
-	callout_reset(&sc->wd_callout, hz, cpsw_tick, sc);
-}
+	printf("\nALE Table\n");
+	cpsw_ale_dump_table(sc);
 
-static void
-cpsw_tx_watchdog(struct cpsw_softc *sc)
-{
-	struct ifnet *ifp = sc->ifp;
+	// XXX do something useful here??
+	panic("CPSW HOST ERROR INTERRUPT");
 
-	CPSW_GLOBAL_LOCK(sc);
-	if (sc->tx_retires > sc->tx_retires_at_last_tick) {
-		sc->tx_wd_timer = 0;  /* Stuff got sent. */
-	} else if (sc->tx_queued == 0) {
-		sc->tx_wd_timer = 0; /* Nothing to send. */
-	} else {
-		/* There was something to send but we didn't. */
-		++sc->tx_wd_timer;
-		if (sc->tx_wd_timer > 3) {
-			sc->tx_wd_timer = 0;
-			ifp->if_oerrors++;
-			if_printf(ifp, "watchdog timeout\n");
-			cpsw_stop_locked(sc);
-			cpsw_init_locked(sc);
-			CPSW_DEBUGF(("watchdog reset completed\n"));
-		}
-	}
-	sc->tx_retires_at_last_tick = sc->tx_retires;
-	CPSW_GLOBAL_UNLOCK(sc);
+	// Suppress this interrupt in the future.
+	cpsw_write_4(sc, CPSW_CPDMA_DMA_INTMASK_CLEAR, intstat);
+	printf("XXX HOST ERROR INTERRUPT SUPPRESSED\n");
+	// The watchdog will probably reset the controller
+	// in a little while.  It will probably fail again.
 }
 
 static void
-cpsw_init(void *arg)
+cpsw_intr_misc(void *arg)
 {
 	struct cpsw_softc *sc = arg;
-
-	CPSW_DEBUGF((""));
-	CPSW_GLOBAL_LOCK(sc);
-	cpsw_init_locked(arg);
-	CPSW_GLOBAL_UNLOCK(sc);
+	uint32_t stat = cpsw_read_4(sc, CPSW_WR_C_MISC_STAT(0));
+
+	if (stat & 16)
+		CPSW_DEBUGF(("Time sync event interrupt unimplemented"));
+	if (stat & 8)
+		cpsw_stats_collect(sc);
+	if (stat & 4)
+		cpsw_intr_misc_host_error(sc);
+	if (stat & 2)
+		CPSW_DEBUGF(("MDIO link change interrupt unimplemented"));
+	if (stat & 1)
+		CPSW_DEBUGF(("MDIO operation completed interrupt unimplemented"));
+	cpsw_write_4(sc, CPSW_CPDMA_CPDMA_EOI_VECTOR, 3);
 }
 
-int once = 1;
+/*
+ *
+ * Periodic Checks and Watchdog.
+ *
+ */
 
 static void
-cpsw_init_locked(void *arg)
+cpsw_tick(void *msc)
 {
-	struct ifnet *ifp;
-	struct cpsw_softc *sc = arg;
-	uint8_t  broadcast_address[6] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
-	uint32_t i;
-
-	CPSW_DEBUGF((""));
-	ifp = sc->ifp;
-	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
-		return;
-
-	/* Reset writer */
-	cpsw_write_4(CPSW_WR_SOFT_RESET, 1);
-	while (cpsw_read_4(CPSW_WR_SOFT_RESET) & 1)
-		;
-
-	/* Reset SS */
-	cpsw_write_4(CPSW_SS_SOFT_RESET, 1);
-	while (cpsw_read_4(CPSW_SS_SOFT_RESET) & 1)
-		;
+	struct cpsw_softc *sc = msc;
 
-	/* Clear table (30) and enable ALE(31) */
-	if (once)
-		cpsw_write_4(CPSW_ALE_CONTROL, 3 << 30);
-	else
-		cpsw_write_4(CPSW_ALE_CONTROL, 1 << 31);
-	once = 0; // FIXME
+	/* Check for TX timeout */
+	cpsw_tx_watchdog(sc);
 
-	/* Reset and init Sliver port 1 and 2 */
-	for (i = 0; i < 2; i++) {
-		/* Reset */
-		cpsw_write_4(CPSW_SL_SOFT_RESET(i), 1);
-		while (cpsw_read_4(CPSW_SL_SOFT_RESET(i)) & 1)
-			;
-		/* Set Slave Mapping */
-		cpsw_write_4(CPSW_SL_RX_PRI_MAP(i), 0x76543210);
-		cpsw_write_4(CPSW_PORT_P_TX_PRI_MAP(i + 1), 0x33221100);
-		cpsw_write_4(CPSW_SL_RX_MAXLEN(i), 0x5f2);
-		/* Set MAC Address */
-		cpsw_write_4(CPSW_PORT_P_SA_HI(i + 1),
-			sc->mac_addr[3] << 24 |
-			sc->mac_addr[2] << 16 |
-			sc->mac_addr[1] << 8 |
-			sc->mac_addr[0]);
-		cpsw_write_4(CPSW_PORT_P_SA_LO(i+1),
-			sc->mac_addr[5] << 8 |
-			sc->mac_addr[4]);
-
-		/* Set MACCONTROL for ports 0,1: FULLDUPLEX(1), GMII_EN(5),
-		   IFCTL_A(15), IFCTL_B(16) FIXME */
-		cpsw_write_4(CPSW_SL_MACCONTROL(i), 1 << 15 | 1 << 5 | 1);
-
-		/* Set ALE port to forwarding(3) */
-		cpsw_write_4(CPSW_ALE_PORTCTL(i + 1), 3);
+	/* Check for media type change */
+	mii_tick(sc->mii);
+	if(sc->cpsw_media_status != sc->mii->mii_media.ifm_media) {
+		printf("%s: media type changed (ifm_media=%x)\n", __func__, 
+			sc->mii->mii_media.ifm_media);
+		cpsw_ifmedia_upd(sc->ifp);
 	}
 
-	/* Set Host Port Mapping */
-	cpsw_write_4(CPSW_PORT_P0_CPDMA_TX_PRI_MAP, 0x76543210);
-	cpsw_write_4(CPSW_PORT_P0_CPDMA_RX_CH_MAP, 0);
-
-	/* Set ALE port to forwarding(3)*/
-	cpsw_write_4(CPSW_ALE_PORTCTL(0), 3);
-
-	/* Add own MAC address and broadcast to ALE */
-	cpsw_ale_uc_entry_set(sc, 0, sc->mac_addr);
-	cpsw_ale_mc_entry_set(sc, 7, broadcast_address);
-
-	cpsw_write_4(CPSW_SS_PTYPE, 0);
-	/* Enable statistics for ports 0, 1 and 2 */
-	cpsw_write_4(CPSW_SS_STAT_PORT_EN, 7);
-
-	/* Reset CPDMA */
-	cpsw_write_4(CPSW_CPDMA_SOFT_RESET, 1);
-	while (cpsw_read_4(CPSW_CPDMA_SOFT_RESET) & 1)
-		;
-
-	/* Make IP hdr aligned with 4 */
-	cpsw_write_4(CPSW_CPDMA_RX_BUFFER_OFFSET, 2);
+	/* Schedule another timeout one second from now */
+	callout_reset(&sc->watchdog.callout, hz, cpsw_tick, sc);
+}
 
-	for (i = 0; i < 8; i++) {
-		cpsw_write_4(CPSW_CPDMA_TX_HDP(i), 0);
-		cpsw_write_4(CPSW_CPDMA_RX_HDP(i), 0);
-		cpsw_write_4(CPSW_CPDMA_TX_CP(i), 0);
-		cpsw_write_4(CPSW_CPDMA_RX_CP(i), 0);
-	}
+static void
+cpsw_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
+{
+	struct cpsw_softc *sc = ifp->if_softc;
+	struct mii_data *mii;
 
-	/* Initialize RX Buffer Descriptors */
-	cpsw_write_4(CPSW_CPDMA_RX_FREEBUFFER(0), 0);
-	cpsw_fill_rx_queue_locked(sc);
+	CPSW_DEBUGF((""));
+	CPSW_TX_LOCK(sc);
 
-	/* Clear all interrupt Masks */
-	cpsw_write_4(CPSW_CPDMA_RX_INTMASK_CLEAR, 0xFFFFFFFF);
-	cpsw_write_4(CPSW_CPDMA_TX_INTMASK_CLEAR, 0xFFFFFFFF);
+	mii = sc->mii;
+	mii_pollstat(mii);
 
-	/* Enable TX & RX DMA */
-	cpsw_write_4(CPSW_CPDMA_TX_CONTROL, 1);
-	cpsw_write_4(CPSW_CPDMA_RX_CONTROL, 1);
+	ifmr->ifm_active = mii->mii_media_active;
+	ifmr->ifm_status = mii->mii_media_status;
 
-	/* Enable TX and RX interrupt receive for core 0 */
-	cpsw_write_4(CPSW_WR_C_TX_EN(0), 0xFF);
-	cpsw_write_4(CPSW_WR_C_RX_EN(0), 0xFF);
-	//cpsw_write_4(CPSW_WR_C_MISC_EN(0), 0x3F);
+	CPSW_TX_UNLOCK(sc);
+}
 
-	/* Enable host Error Interrupt */
-	cpsw_write_4(CPSW_CPDMA_DMA_INTMASK_SET, 1);
+static int
+cpsw_ifmedia_upd(struct ifnet *ifp)
+{
+	struct cpsw_softc *sc = ifp->if_softc;
 
-	/* Enable interrupts for TX and RX Channel 0 */
-	cpsw_write_4(CPSW_CPDMA_TX_INTMASK_SET, 1);
-	cpsw_write_4(CPSW_CPDMA_RX_INTMASK_SET, 1);
+	CPSW_DEBUGF((""));
+	if (ifp->if_flags & IFF_UP) {
+		CPSW_GLOBAL_LOCK(sc);
+		sc->cpsw_media_status = sc->mii->mii_media.ifm_media;
+		mii_mediachg(sc->mii);
+		cpsw_init_locked(sc);
+		CPSW_GLOBAL_UNLOCK(sc);
+	}
 
-	/* Ack stalled irqs */
-	cpsw_write_4(CPSW_CPDMA_CPDMA_EOI_VECTOR, 0);
-	cpsw_write_4(CPSW_CPDMA_CPDMA_EOI_VECTOR, 1);
-	cpsw_write_4(CPSW_CPDMA_CPDMA_EOI_VECTOR, 2);
-	cpsw_write_4(CPSW_CPDMA_CPDMA_EOI_VECTOR, 3);
+	return (0);
+}
 
-	/* Initialze MDIO - ENABLE, PREAMBLE=0, FAULTENB, CLKDIV=0xFF */
-	/* TODO Calculate MDCLK=CLK/(CLKDIV+1) */
-	cpsw_write_4(MDIOCONTROL, 1 << 30 | 1 << 18 | 0xFF);
+static void
+cpsw_tx_watchdog_full_reset(struct cpsw_softc *sc)
+{
+	cpsw_debugf_head("CPSW watchdog");
+	if_printf(sc->ifp, "watchdog timeout\n");
+	cpsw_shutdown_locked(sc);
+	cpsw_init_locked(sc);
+}
 
-	/* Select MII in GMII_SEL, Internal Delay mode */
-	//ti_scm_reg_write_4(0x650, 0);
+static void
+cpsw_tx_watchdog(struct cpsw_softc *sc)
+{
+	struct ifnet *ifp = sc->ifp;
 
-	/* Activate network interface */
-	sc->rx_running = 1;
-	sc->tx_running = 1;
-	sc->tx_wd_timer = 0;
-	callout_reset(&sc->wd_callout, hz, cpsw_tick, sc);
-	sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
-	sc->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+	CPSW_GLOBAL_LOCK(sc);
+	if (sc->tx.active_queue_len == 0 || (ifp->if_flags & IFF_UP) == 0 ||
+	    (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || !sc->tx.running) {
+		sc->watchdog.timer = 0; /* Nothing to do. */
+	} else if (sc->tx.queue_removes > sc->tx.queue_removes_at_last_tick) {
+		sc->watchdog.timer = 0;  /* Stuff done while we weren't looking. */
+	} else if (cpsw_tx_dequeue(sc) > 0) {
+		sc->watchdog.timer = 0;  /* We just did something. */
+	} else {
+		/* There was something to do but it didn't get done. */
+		++sc->watchdog.timer;
+		if (sc->watchdog.timer > 2) {
+			sc->watchdog.timer = 0;
+			++ifp->if_oerrors;
+			++sc->watchdog.resets;
+			cpsw_tx_watchdog_full_reset(sc);
+		}
+	}
+	sc->tx.queue_removes_at_last_tick = sc->tx.queue_removes;
+	CPSW_GLOBAL_UNLOCK(sc);
 }
 
+/*
+ *
+ * ALE support routines.
+ *
+ */
+
 static void
 cpsw_ale_read_entry(struct cpsw_softc *sc, uint16_t idx, uint32_t *ale_entry)
 {
-	cpsw_write_4(CPSW_ALE_TBLCTL, idx & 1023);
-	ale_entry[0] = cpsw_read_4(CPSW_ALE_TBLW0);
-	ale_entry[1] = cpsw_read_4(CPSW_ALE_TBLW1);
-	ale_entry[2] = cpsw_read_4(CPSW_ALE_TBLW2);
+	cpsw_write_4(sc, CPSW_ALE_TBLCTL, idx & 1023);
+	ale_entry[0] = cpsw_read_4(sc, CPSW_ALE_TBLW0);
+	ale_entry[1] = cpsw_read_4(sc, CPSW_ALE_TBLW1);
+	ale_entry[2] = cpsw_read_4(sc, CPSW_ALE_TBLW2);
 }
 
 static void
 cpsw_ale_write_entry(struct cpsw_softc *sc, uint16_t idx, uint32_t *ale_entry)
 {
-	cpsw_write_4(CPSW_ALE_TBLW0, ale_entry[0]);
-	cpsw_write_4(CPSW_ALE_TBLW1, ale_entry[1]);
-	cpsw_write_4(CPSW_ALE_TBLW2, ale_entry[2]);
-	cpsw_write_4(CPSW_ALE_TBLCTL, 1 << 31 | (idx & 1023));
+	cpsw_write_4(sc, CPSW_ALE_TBLW0, ale_entry[0]);
+	cpsw_write_4(sc, CPSW_ALE_TBLW1, ale_entry[1]);
+	cpsw_write_4(sc, CPSW_ALE_TBLW2, ale_entry[2]);
+	cpsw_write_4(sc, CPSW_ALE_TBLCTL, 1 << 31 | (idx & 1023));
 }
 
 static int
-cpsw_ale_find_entry_by_mac(struct cpsw_softc *sc, uint8_t *mac)
+cpsw_ale_remove_all_mc_entries(struct cpsw_softc *sc)
 {
 	int i;
 	uint32_t ale_entry[3];
-	for (i = 0; i < CPSW_MAX_ALE_ENTRIES; i++) {
+
+	/* First two entries are link address and broadcast. */
+	for (i = 2; i < CPSW_MAX_ALE_ENTRIES; i++) {
 		cpsw_ale_read_entry(sc, i, ale_entry);
-		if ((((ale_entry[1] >> 8) & 0xFF) == mac[0]) &&
-		    (((ale_entry[1] >> 0) & 0xFF) == mac[1]) &&
-		    (((ale_entry[0] >>24) & 0xFF) == mac[2]) &&
-		    (((ale_entry[0] >>16) & 0xFF) == mac[3]) &&
-		    (((ale_entry[0] >> 8) & 0xFF) == mac[4]) &&
-		    (((ale_entry[0] >> 0) & 0xFF) == mac[5])) {
-			return (i);
+		if (((ale_entry[1] >> 28) & 3) == 1 && /* Address entry */
+		    ((ale_entry[1] >> 8) & 1) == 1) { /* MCast link addr */
+			ale_entry[0] = ale_entry[1] = ale_entry[2] = 0;
+			cpsw_ale_write_entry(sc, i, ale_entry);
 		}
 	}
 	return CPSW_MAX_ALE_ENTRIES;
 }
 
 static int
-cpsw_ale_find_free_entry(struct cpsw_softc *sc)
+cpsw_ale_mc_entry_set(struct cpsw_softc *sc, uint8_t portmap, uint8_t *mac)
 {
-	int i;
+	int free_index = -1, matching_index = -1, i;
 	uint32_t ale_entry[3];
+
+	/* Find a matching entry or a free entry. */
 	for (i = 0; i < CPSW_MAX_ALE_ENTRIES; i++) {
 		cpsw_ale_read_entry(sc, i, ale_entry);
+
 		/* Entry Type[61:60] is 0 for free entry */ 
-		if (((ale_entry[1] >> 28) & 3) == 0) {
-			return i;
+		if (free_index < 0 && ((ale_entry[1] >> 28) & 3) == 0) {
+			free_index = i;
 		}
-	}
-	return CPSW_MAX_ALE_ENTRIES;
-}
-
-
-static int
-cpsw_ale_uc_entry_set(struct cpsw_softc *sc, uint8_t port, uint8_t *mac)
-{
-	int i;
-	uint32_t ale_entry[3];
 
-	if ((i = cpsw_ale_find_entry_by_mac(sc, mac)) == CPSW_MAX_ALE_ENTRIES) {
-		i = cpsw_ale_find_free_entry(sc);
+		if ((((ale_entry[1] >> 8) & 0xFF) == mac[0]) &&
+		    (((ale_entry[1] >> 0) & 0xFF) == mac[1]) &&
+		    (((ale_entry[0] >>24) & 0xFF) == mac[2]) &&
+		    (((ale_entry[0] >>16) & 0xFF) == mac[3]) &&
+		    (((ale_entry[0] >> 8) & 0xFF) == mac[4]) &&
+		    (((ale_entry[0] >> 0) & 0xFF) == mac[5])) {
+			matching_index = i;
+			break;
+		}
 	}
 
-	if (i == CPSW_MAX_ALE_ENTRIES)
-		return (ENOMEM);
-
-	/* Set MAC address */
-	ale_entry[0] = mac[2] << 24 | mac[3] << 16 | mac[4] << 8 | mac[5];
-	ale_entry[1] = mac[0] << 8 | mac[1];
-
-	/* Entry type[61:60] is addr entry(1) */
-	ale_entry[1] |= 0x10 << 24;
-
-	/* Set portmask [67:66] */
-	ale_entry[2] = (port & 3) << 2;
-
-	cpsw_ale_write_entry(sc, i, ale_entry);
-
-	return 0;
-}
-
-static int
-cpsw_ale_mc_entry_set(struct cpsw_softc *sc, uint8_t portmap, uint8_t *mac)
-{
-	int i;
-	uint32_t ale_entry[3];
-
-	if ((i = cpsw_ale_find_entry_by_mac(sc, mac)) == CPSW_MAX_ALE_ENTRIES) {
-		i = cpsw_ale_find_free_entry(sc);
+	if (matching_index < 0) {
+		if (free_index < 0)
+			return (ENOMEM);
+		i = free_index;
 	}
 
-	if (i == CPSW_MAX_ALE_ENTRIES)
-		return (ENOMEM);
-
 	/* Set MAC address */
 	ale_entry[0] = mac[2] << 24 | mac[3] << 16 | mac[4] << 8 | mac[5];
 	ale_entry[1] = mac[0] << 8 | mac[1];
@@ -1481,7 +1931,6 @@ cpsw_ale_mc_entry_set(struct cpsw_softc *sc, uint8_t portmap, uint8_t *mac)
 	return 0;
 }
 
-#ifdef CPSW_DEBUG
 static void
 cpsw_ale_dump_table(struct cpsw_softc *sc) {
 	int i;
@@ -1504,5 +1953,232 @@ cpsw_ale_dump_table(struct cpsw_softc *sc) {
 			printf("\n");
 		}
 	}
+	printf("\n");
+}
+
+static int
+cpsw_ale_update_addresses(struct cpsw_softc *sc, int purge)
+{
+	uint8_t *mac;
+	uint32_t ale_entry[3];
+	struct ifnet *ifp = sc->ifp;
+	struct ifmultiaddr *ifma;
+	int i;
+
+	/* Route incoming packets for our MAC address to Port 0 (host). */
+	/* For simplicity, keep this entry at table index 0 in the ALE. */
+        if_addr_rlock(ifp);
+	mac = LLADDR((struct sockaddr_dl *)ifp->if_addr->ifa_addr);
+	ale_entry[0] = mac[2] << 24 | mac[3] << 16 | mac[4] << 8 | mac[5];
+	ale_entry[1] = 0x10 << 24 | mac[0] << 8 | mac[1]; /* addr entry + mac */
+	ale_entry[2] = 0; /* port = 0 */
+	cpsw_ale_write_entry(sc, 0, ale_entry);
+
+	/* Set outgoing MAC Address for Ports 1 and 2. */
+	for (i = 1; i < 3; ++i) {
+		cpsw_write_4(sc, CPSW_PORT_P_SA_HI(i),
+		    mac[3] << 24 | mac[2] << 16 | mac[1] << 8 | mac[0]);
+		cpsw_write_4(sc, CPSW_PORT_P_SA_LO(i),
+		    mac[5] << 8 | mac[4]);
+	}
+        if_addr_runlock(ifp);
+
+	/* Keep the broadcast address at table entry 1. */
+	ale_entry[0] = 0xffffffff; /* Lower 32 bits of MAC */
+	ale_entry[1] = 0xd000ffff; /* FW (3 << 30), Addr entry (1 << 24), upper 16 bits of Mac */ 
+	ale_entry[2] = 0x0000001c; /* Forward to all ports */
+	cpsw_ale_write_entry(sc, 1, ale_entry);
+
+	/* SIOCDELMULTI doesn't specify the particular address
+	   being removed, so we have to remove all and rebuild. */
+	if (purge)
+		cpsw_ale_remove_all_mc_entries(sc);
+
+        /* Set other multicast addrs desired. */
+        if_maddr_rlock(ifp);
+        TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
+                if (ifma->ifma_addr->sa_family != AF_LINK)
+                        continue;
+		cpsw_ale_mc_entry_set(sc, 7,
+		    LLADDR((struct sockaddr_dl *)ifma->ifma_addr));
+        }
+        if_maddr_runlock(ifp);
+
+	return (0);
+}
+
+/*
+ *
+ * Statistics and Sysctls.
+ *
+ */
+
+#if 0
+static void
+cpsw_stats_dump(struct cpsw_softc *sc)
+{
+	int i;
+	uint32_t r;
+
+	for (i = 0; i < CPSW_SYSCTL_COUNT; ++i) {
+		r = cpsw_read_4(sc, CPSW_STATS_OFFSET +
+		    cpsw_stat_sysctls[i].reg);
+		CPSW_DEBUGF(("%s: %ju + %u = %ju", cpsw_stat_sysctls[i].oid,
+			     (intmax_t)sc->shadow_stats[i], r,
+			     (intmax_t)sc->shadow_stats[i] + r));
+	}
 }
 #endif
+
+static void
+cpsw_stats_collect(struct cpsw_softc *sc)
+{
+	int i;
+	uint32_t r;
+
+	CPSW_DEBUGF(("Controller shadow statistics updated."));
+
+	for (i = 0; i < CPSW_SYSCTL_COUNT; ++i) {
+		r = cpsw_read_4(sc, CPSW_STATS_OFFSET +
+		    cpsw_stat_sysctls[i].reg);
+		sc->shadow_stats[i] += r;
+		cpsw_write_4(sc, CPSW_STATS_OFFSET + cpsw_stat_sysctls[i].reg, r);
+	}
+}
+
+static int
+cpsw_stats_sysctl(SYSCTL_HANDLER_ARGS)
+{
+	struct cpsw_softc *sc;
+	struct cpsw_stat *stat;
+	uint64_t result;
+
+	sc = (struct cpsw_softc *)arg1;
+	stat = &cpsw_stat_sysctls[oidp->oid_number];
+	result = sc->shadow_stats[oidp->oid_number];
+	result += cpsw_read_4(sc, CPSW_STATS_OFFSET + stat->reg);
+	return (sysctl_handle_64(oidp, &result, 0, req));
+}
+
+static int
+cpsw_stat_attached(SYSCTL_HANDLER_ARGS)
+{
+	struct cpsw_softc *sc;
+	struct bintime t;
+	unsigned result;
+
+	sc = (struct cpsw_softc *)arg1;
+	getbinuptime(&t);
+	bintime_sub(&t, &sc->attach_uptime);
+	result = t.sec;
+	return (sysctl_handle_int(oidp, &result, 0, req));
+}
+
+static int
+cpsw_stat_uptime(SYSCTL_HANDLER_ARGS)
+{
+	struct cpsw_softc *sc;
+	struct bintime t;
+	unsigned result;
+
+	sc = (struct cpsw_softc *)arg1;
+	if (sc->ifp->if_drv_flags & IFF_DRV_RUNNING) {
+		getbinuptime(&t);
+		bintime_sub(&t, &sc->init_uptime);
+		result = t.sec;
+	} else
+		result = 0;
+	return (sysctl_handle_int(oidp, &result, 0, req));
+}
+
+static void
+cpsw_add_queue_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *node, struct cpsw_queue *queue)
+{
+	struct sysctl_oid_list *parent;
+
+	parent = SYSCTL_CHILDREN(node);
+	SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "totalBuffers",
+	    CTLFLAG_RD, &queue->queue_slots, 0,
+	    "Total buffers currently assigned to this queue");
+	SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "activeBuffers",
+	    CTLFLAG_RD, &queue->active_queue_len, 0,
+	    "Buffers currently registered with hardware controller");
+	SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "maxActiveBuffers",
+	    CTLFLAG_RD, &queue->max_active_queue_len, 0,
+	    "Max value of activeBuffers since last driver reset");
+	SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "availBuffers",
+	    CTLFLAG_RD, &queue->avail_queue_len, 0,
+	    "Buffers allocated to this queue but not currently "
+	    "registered with hardware controller");
+	SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "maxAvailBuffers",
+	    CTLFLAG_RD, &queue->max_avail_queue_len, 0,
+	    "Max value of availBuffers since last driver reset");
+	SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "totalEnqueued",
+	    CTLFLAG_RD, &queue->queue_adds, 0,
+	    "Total buffers added to queue");
+	SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "totalDequeued",
+	    CTLFLAG_RD, &queue->queue_removes, 0,
+	    "Total buffers removed from queue");
+	SYSCTL_ADD_UINT(ctx, parent, OID_AUTO, "longestChain",
+	    CTLFLAG_RD, &queue->longest_chain, 0,
+	    "Max buffers used for a single packet");
+}
+
+static void
+cpsw_add_watchdog_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *node, struct cpsw_softc *sc)
+{
+	struct sysctl_oid_list *parent;
+
+	parent = SYSCTL_CHILDREN(node);
+	SYSCTL_ADD_INT(ctx, parent, OID_AUTO, "resets",
+	    CTLFLAG_RD, &sc->watchdog.resets, 0,
+	    "Total number of watchdog resets");
+}
+
+static void
+cpsw_add_sysctls(struct cpsw_softc *sc)
+{
+	struct sysctl_ctx_list *ctx;
+	struct sysctl_oid *stats_node, *queue_node, *node;
+	struct sysctl_oid_list *parent, *stats_parent, *queue_parent;
+	int i;
+
+	ctx = device_get_sysctl_ctx(sc->dev);
+	parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
+
+	SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "attachedSecs",
+	    CTLTYPE_UINT | CTLFLAG_RD, sc, 0, cpsw_stat_attached, "IU",
+	    "Time since driver attach");
+
+	SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "uptime",
+	    CTLTYPE_UINT | CTLFLAG_RD, sc, 0, cpsw_stat_uptime, "IU",
+	    "Seconds since driver init");
+
+	stats_node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "stats",
+				     CTLFLAG_RD, NULL, "CPSW Statistics");
+	stats_parent = SYSCTL_CHILDREN(stats_node);
+	for (i = 0; i < CPSW_SYSCTL_COUNT; ++i) {
+		SYSCTL_ADD_PROC(ctx, stats_parent, i,
+				cpsw_stat_sysctls[i].oid,
+				CTLTYPE_U64 | CTLFLAG_RD, sc, 0,
+				cpsw_stats_sysctl, "IU",
+				cpsw_stat_sysctls[i].oid);
+	}
+
+	queue_node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "queue",
+	    CTLFLAG_RD, NULL, "CPSW Queue Statistics");
+	queue_parent = SYSCTL_CHILDREN(queue_node);
+
+	node = SYSCTL_ADD_NODE(ctx, queue_parent, OID_AUTO, "tx",
+	    CTLFLAG_RD, NULL, "TX Queue Statistics");
+	cpsw_add_queue_sysctls(ctx, node, &sc->tx);
+
+	node = SYSCTL_ADD_NODE(ctx, queue_parent, OID_AUTO, "rx",
+	    CTLFLAG_RD, NULL, "RX Queue Statistics");
+	cpsw_add_queue_sysctls(ctx, node, &sc->rx);
+
+	node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "watchdog",
+	    CTLFLAG_RD, NULL, "Watchdog Statistics");
+	cpsw_add_watchdog_sysctls(ctx, node, sc);
+}
+
diff --git a/sys/arm/ti/cpsw/if_cpswreg.h b/sys/arm/ti/cpsw/if_cpswreg.h
index dab4f36..46f8417 100644
--- a/sys/arm/ti/cpsw/if_cpswreg.h
+++ b/sys/arm/ti/cpsw/if_cpswreg.h
@@ -34,8 +34,11 @@
 #define CPSW_SS_SOFT_RESET		(CPSW_SS_OFFSET + 0x08)
 #define CPSW_SS_STAT_PORT_EN		(CPSW_SS_OFFSET + 0x0C)
 #define CPSW_SS_PTYPE			(CPSW_SS_OFFSET + 0x10)
+#define	CPSW_SS_FLOW_CONTROL		(CPSW_SS_OFFSET + 0x24)
 
 #define CPSW_PORT_OFFSET		0x0100
+#define	CPSW_PORT_P_MAX_BLKS(p)		(CPSW_PORT_OFFSET + 0x08 + ((p) * 0x100))
+#define	CPSW_PORT_P_BLK_CNT(p)		(CPSW_PORT_OFFSET + 0x0C + ((p) * 0x100))
 #define CPSW_PORT_P_TX_PRI_MAP(p)	(CPSW_PORT_OFFSET + 0x118 + ((p-1) * 0x100))
 #define CPSW_PORT_P0_CPDMA_TX_PRI_MAP	(CPSW_PORT_OFFSET + 0x01C)
 #define CPSW_PORT_P0_CPDMA_RX_CH_MAP	(CPSW_PORT_OFFSET + 0x020)
@@ -84,10 +87,14 @@
 #define CPSW_ALE_TBLW0			(CPSW_ALE_OFFSET + 0x3C)
 #define CPSW_ALE_PORTCTL(p)		(CPSW_ALE_OFFSET + 0x40 + ((p) * 0x04))
 
+/* SL1 is at 0x0D80, SL2 is at 0x0DC0 */
 #define CPSW_SL_OFFSET			0x0D80
 #define CPSW_SL_MACCONTROL(p)		(CPSW_SL_OFFSET + (0x40 * (p)) + 0x04)
+#define CPSW_SL_MACSTATUS(p)		(CPSW_SL_OFFSET + (0x40 * (p)) + 0x08)
 #define CPSW_SL_SOFT_RESET(p)		(CPSW_SL_OFFSET + (0x40 * (p)) + 0x0C)
 #define CPSW_SL_RX_MAXLEN(p)		(CPSW_SL_OFFSET + (0x40 * (p)) + 0x10)
+#define CPSW_SL_RX_PAUSE(p)		(CPSW_SL_OFFSET + (0x40 * (p)) + 0x18)
+#define CPSW_SL_TX_PAUSE(p)		(CPSW_SL_OFFSET + (0x40 * (p)) + 0x1C)
 #define CPSW_SL_RX_PRI_MAP(p)		(CPSW_SL_OFFSET + (0x40 * (p)) + 0x24)
 
 #define MDIO_OFFSET			0x1000
@@ -109,5 +116,22 @@
 #define CPSW_WR_C_MISC_STAT(p)		(CPSW_WR_OFFSET + (0x10 * (p)) + 0x4C)
 
 #define CPSW_CPPI_RAM_OFFSET		0x2000
+#define	CPSW_CPPI_RAM_SIZE		0x2000
+
+#define CPDMA_BD_SOP		(1<<15)
+#define CPDMA_BD_EOP		(1<<14)
+#define CPDMA_BD_OWNER		(1<<13)
+#define CPDMA_BD_EOQ		(1<<12)
+#define CPDMA_BD_TDOWNCMPLT	(1<<11)
+#define CPDMA_BD_PKT_ERR_MASK	(3<< 4)
+
+struct cpsw_cpdma_bd {
+	volatile uint32_t next;
+	volatile uint32_t bufptr;
+	volatile uint16_t buflen;
+	volatile uint16_t bufoff;
+	volatile uint16_t pktlen;
+	volatile uint16_t flags;
+};
 
 #endif /*_IF_CPSWREG_H */
diff --git a/sys/arm/ti/cpsw/if_cpswvar.h b/sys/arm/ti/cpsw/if_cpswvar.h
index 35024c6..2aebb0f 100644
--- a/sys/arm/ti/cpsw/if_cpswvar.h
+++ b/sys/arm/ti/cpsw/if_cpswvar.h
@@ -35,105 +35,90 @@
 #define CPSW_MIIBUS_RETRIES	5
 #define CPSW_MIIBUS_DELAY	1000
 
-#define CPSW_MAX_TX_BUFFERS	128
-#define CPSW_MAX_RX_BUFFERS	128
 #define CPSW_MAX_ALE_ENTRIES	1024
 
+#define CPSW_SYSCTL_COUNT 34
+
 struct cpsw_slot {
+	uint32_t bd_offset;  /* Offset of corresponding BD within CPPI RAM. */
 	bus_dmamap_t dmamap;
 	struct mbuf *mbuf;
-	int index;
 	STAILQ_ENTRY(cpsw_slot) next;
 };
-STAILQ_HEAD(cpsw_queue, cpsw_slot);
+STAILQ_HEAD(cpsw_slots, cpsw_slot);
+
+struct cpsw_queue {
+	struct mtx	lock;
+	int		running;
+	struct cpsw_slots active;
+	struct cpsw_slots avail;
+	uint32_t	queue_adds; /* total bufs added */
+	uint32_t	queue_removes; /* total bufs removed */
+	uint32_t	queue_removes_at_last_tick; /* Used by watchdog */
+	int		queue_slots;
+	int		active_queue_len;
+	int		max_active_queue_len;
+	int		avail_queue_len;
+	int		max_avail_queue_len;
+	int		longest_chain; /* Largest # segments in a single packet. */
+	int		hdp_offset;
+};
 
 struct cpsw_softc {
 	struct ifnet	*ifp;
 	phandle_t	node;
 	device_t	dev;
+	struct bintime	attach_uptime; /* system uptime when attach happened. */
+	struct bintime	init_uptime; /* system uptime when init happened. */
+
+	/* TODO: We should set up a child structure for each port;
+	   store mac, phy information, etc, in that structure. */
 	uint8_t		mac_addr[ETHER_ADDR_LEN];
+
 	device_t	miibus;
 	struct mii_data	*mii;
-	struct mtx	tx_lock;			/* transmitter lock */
-	struct mtx	rx_lock;			/* receiver lock */
-	struct resource	*res[1 + CPSW_INTR_COUNT];	/* resources */
-	void		*ih_cookie[CPSW_INTR_COUNT];	/* interrupt handlers cookies */
+	/* We expect 1 memory resource and 4 interrupts from the device tree. */
+	struct resource	*res[1 + CPSW_INTR_COUNT];
+
+	/* Interrupts get recorded here as we initialize them. */
+	/* Interrupt teardown just walks this list. */
+	struct {
+		struct resource *res;
+		void		*ih_cookie;
+		const char *description;
+	} interrupts[CPSW_INTR_COUNT];
+	int		interrupt_count;
 
 	uint32_t	cpsw_if_flags;
 	int		cpsw_media_status;
 
-	struct callout	wd_callout;
-	int		tx_wd_timer;
+	struct {
+		int resets;
+		int timer;
+		struct callout	callout;
+	} watchdog;
 
 	bus_dma_tag_t	mbuf_dtag;
 
-	/* RX buffer tracking */
-	int		rx_running;
-	struct cpsw_queue rx_active;
-	struct cpsw_queue rx_avail;
-	struct cpsw_slot _rx_slots[CPSW_MAX_RX_BUFFERS];
-
-	/* TX buffer tracking. */
-	int		tx_running;
-	struct cpsw_queue tx_active;
-	struct cpsw_queue tx_avail;
-	struct cpsw_slot _tx_slots[CPSW_MAX_TX_BUFFERS];
-
-	/* Statistics */
-	uint32_t	tx_enqueues; /* total TX bufs added to queue */
-	uint32_t	tx_retires; /* total TX bufs removed from queue */
-	uint32_t	tx_retires_at_last_tick; /* used for watchdog */
-	/* Note:  tx_queued != tx_enqueues - tx_retires
-	   At driver reset, packets can be discarded
-	   from TX queue without being retired. */
-	int		tx_queued; /* Current bufs in TX queue */
-	int		tx_max_queued;
+	/* An mbuf full of nulls for TX padding. */
+	bus_dmamap_t null_mbuf_dmamap;
+	struct mbuf *null_mbuf;
+	bus_addr_t null_mbuf_paddr;
+
+	/* RX and TX buffer tracking */
+	struct cpsw_queue rx, tx;
+
+	/* 64-bit versions of 32-bit hardware statistics counters */
+	uint64_t shadow_stats[CPSW_SYSCTL_COUNT];
+
+	/* CPPI STATERAM has 512 slots for building TX/RX queues. */
+	/* TODO: Size here supposedly varies with different versions
+	   of the controller.  Check DaVinci specs and find a good
+	   way to adjust this.  One option is to have a separate
+	   Device Tree parameter for number slots; another option
+	   is to calculate it from the memory size in the device tree. */
+	struct cpsw_slot _slots[CPSW_CPPI_RAM_SIZE / sizeof(struct cpsw_cpdma_bd)];
+	struct cpsw_slots avail;
 };
 
-#define CPDMA_BD_SOP		(1<<15)
-#define CPDMA_BD_EOP		(1<<14)
-#define CPDMA_BD_OWNER		(1<<13)
-#define CPDMA_BD_EOQ		(1<<12)
-#define CPDMA_BD_TDOWNCMPLT	(1<<11)
-#define CPDMA_BD_PKT_ERR_MASK	(3<< 4)
-
-struct cpsw_cpdma_bd {
-	volatile uint32_t next;
-	volatile uint32_t bufptr;
-	volatile uint16_t buflen;
-	volatile uint16_t bufoff;
-	volatile uint16_t pktlen;
-	volatile uint16_t flags;
-};
-
-/* Read/Write macros */
-#define cpsw_read_4(reg)		bus_read_4(sc->res[0], reg)
-#define cpsw_write_4(reg, val)		bus_write_4(sc->res[0], reg, val)
-
-#define cpsw_cpdma_txbd_offset(i)	\
-	(CPSW_CPPI_RAM_OFFSET + ((i)*16))
-#define cpsw_cpdma_txbd_paddr(i)	(cpsw_cpdma_txbd_offset(i) + \
-	vtophys(rman_get_start(sc->res[0])))
-#define cpsw_cpdma_read_txbd(i, val)	\
-	bus_read_region_4(sc->res[0], cpsw_cpdma_txbd_offset(i), (uint32_t *) val, 4)
-#define cpsw_cpdma_write_txbd(i, val)	\
-	bus_write_region_4(sc->res[0], cpsw_cpdma_txbd_offset(i), (uint32_t *) val, 4)
-#define cpsw_cpdma_write_txbd_next(i, val) \
-	bus_write_4(sc->res[0], cpsw_cpdma_txbd_offset(i), val)
-#define cpsw_cpdma_read_txbd_flags(i) \
-	bus_read_2(sc->res[0], cpsw_cpdma_txbd_offset(i)+14)
-
-#define cpsw_cpdma_rxbd_offset(i)	\
-	(CPSW_CPPI_RAM_OFFSET + ((CPSW_MAX_TX_BUFFERS + (i))*16))
-#define cpsw_cpdma_rxbd_paddr(i)	(cpsw_cpdma_rxbd_offset(i) + \
-	vtophys(rman_get_start(sc->res[0])))
-#define cpsw_cpdma_read_rxbd(i, val)	\
-	bus_read_region_4(sc->res[0], cpsw_cpdma_rxbd_offset(i), (uint32_t *) val, 4)
-#define cpsw_cpdma_write_rxbd(i, val)	\
-	bus_write_region_4(sc->res[0], cpsw_cpdma_rxbd_offset(i), (uint32_t *) val, 4)
-#define cpsw_cpdma_write_rxbd_next(i, val) \
-	bus_write_4(sc->res[0], cpsw_cpdma_rxbd_offset(i), val)
-#define cpsw_cpdma_read_rxbd_flags(i) \
-	bus_read_2(sc->res[0], cpsw_cpdma_rxbd_offset(i)+14)
-
 #endif /*_IF_CPSWVAR_H */
diff --git a/sys/arm/ti/ti_scm.c b/sys/arm/ti/ti_scm.c
index ce31f6e..58849c2 100644
--- a/sys/arm/ti/ti_scm.c
+++ b/sys/arm/ti/ti_scm.c
@@ -155,8 +155,10 @@ ti_scm_padconf_set_internal(struct ti_scm_softc *sc,
 	}
 
 	/* couldn't find the mux mode */
-	if (mode >= 8)
+	if (mode >= 8) {
+		printf("Invalid mode \"%s\"\n", muxmode);
 		return (EINVAL);
+	}
 
 	/* set the mux mode */
 	reg_val |= (uint16_t)(mode & ti_scm_dev.padconf_muxmode_mask);
@@ -391,13 +393,16 @@ ti_scm_padconf_init_from_fdt(struct ti_scm_softc *sc)
 				while (padstates->state != NULL) {
 					if (strcmp(padstates->state, padstate) == 0) {
 						err = ti_scm_padconf_set_internal(sc,
-							padconf, muxname, padstates->reg);
+						    padconf, muxname, padstates->reg);
 					}
 					padstates++;
 				}
 				if (err)
-					device_printf(sc->sc_dev, "err: failed to configure"
-						"pin \"%s\"\n", padconf->ballname);
+					device_printf(sc->sc_dev,
+					    "err: failed to configure "
+					    "pin \"%s\" as \"%s\"\n",
+					    padconf->ballname,
+					    muxname);
 			}
 			padconf++;
 		}
diff --git a/sys/boot/arm/uboot/Makefile b/sys/boot/arm/uboot/Makefile
index 1974218..c3c4bd8 100644
--- a/sys/boot/arm/uboot/Makefile
+++ b/sys/boot/arm/uboot/Makefile
@@ -112,8 +112,8 @@ CFLAGS+=	-I${.CURDIR}/../../../../lib/libstand/
 # clang doesn't understand %D as a specifier to printf
 NO_WERROR.clang=
 
-DPADD=		${LIBFICL} ${LIBUBOOT} ${LIBFDT} ${LIBSTAND}
-LDADD=		${LIBFICL} ${LIBUBOOT} ${LIBFDT} -lstand
+DPADD=		${LIBFICL} ${LIBUBOOT} ${LIBFDT} ${LIBSTAND} ${LIBGCC}
+LDADD=		${LIBFICL} ${LIBUBOOT} ${LIBFDT} -lstand -lgcc
 
 vers.c:	${.CURDIR}/../../common/newvers.sh ${.CURDIR}/version
 	sh ${.CURDIR}/../../common/newvers.sh ${.CURDIR}/version ${NEWVERSWHAT}
diff --git a/sys/boot/fdt/dts/beaglebone.dts b/sys/boot/fdt/dts/beaglebone.dts
index 2c68021..3e2dc80 100644
--- a/sys/boot/fdt/dts/beaglebone.dts
+++ b/sys/boot/fdt/dts/beaglebone.dts
@@ -91,7 +91,58 @@
 				"MMC0_DAT0", "mmc0_dat0", "input_pullup",
 				"MMC0_DAT1", "mmc0_dat1", "input_pullup",
 				"MMC0_DAT2", "mmc0_dat2", "input_pullup",
-				"MMC0_DAT3", "mmc0_dat3", "input_pullup";
+				"MMC0_DAT3", "mmc0_dat3", "input_pullup",
+				/* GPIO */
+				"ECAP0_IN_PWM0_OUT", "gpio0_7", "input_pulldown",
+				"GPMC_AD10", "gpio0_26", "input_pulldown",
+				"GPMC_AD11", "gpio0_27", "input_pulldown",
+				"GPMC_AD0", "gpio1_0", "input_pulldown",
+				"GPMC_AD1", "gpio1_1", "input_pulldown",
+				"GPMC_AD2", "gpio1_2", "input_pulldown",
+				"GPMC_AD3", "gpio1_3", "input_pulldown",
+				"GPMC_AD4", "gpio1_4", "input_pulldown",
+				"GPMC_AD5", "gpio1_5", "input_pulldown",
+				"GPMC_AD6", "gpio1_6", "input_pulldown",
+				"GPMC_AD7", "gpio1_7", "input_pulldown",
+				"GPMC_AD12", "gpio1_12", "input_pulldown",
+				"GPMC_AD13", "gpio1_13", "input_pulldown",
+				"GPMC_AD14", "gpio1_14", "input_pulldown",
+				"GPMC_AD15", "gpio1_15", "input_pulldown",
+				"GPMC_A0", "gpio1_16", "input_pulldown",
+				"GPMC_A1", "gpio1_17", "input_pulldown",
+				"GPMC_A5", "gpio1_21", "output", /* User LED 1 */
+				"GPMC_A6", "gpio1_22", "output", /* User LED 2 */
+				"GPMC_A7", "gpio1_23", "output", /* User LED 3 */
+				"GPMC_A8", "gpio1_24", "output", /* User LED 4 */
+				"GPMC_BEn1", "gpio1_28", "input_pulldown",
+				"GPMC_CSn0", "gpio1_29", "input_pulldown",
+				"GPMC_CSn1", "gpio1_30", "input_pulldown",
+				"GPMC_CSn2", "gpio1_31", "input_pulldown",
+				"GPMC_CLK", "gpio2_1", "input_pulldown",
+				"LCD_DATA0", "gpio2_6", "input_pulldown", 
+				"LCD_DATA1", "gpio2_7", "input_pulldown", 
+				"LCD_DATA2", "gpio2_8", "input_pulldown", 
+				"LCD_DATA3", "gpio2_9", "input_pulldown", 
+				"LCD_DATA4", "gpio2_10", "input_pulldown", 
+				"LCD_DATA5", "gpio2_11", "input_pulldown", 
+				"LCD_DATA6", "gpio2_12", "input_pulldown", 
+				"LCD_DATA7", "gpio2_13", "input_pulldown", 
+				"LCD_VSYNC", "gpio2_22", "input_pulldown",
+				"LCD_HSYNC", "gpio2_23", "input_pulldown",
+				"LCD_PCLK", "gpio2_24", "input_pulldown",
+				"LCD_AC_BIAS_EN", "gpio2_25", "input_pulldown",
+				"MCASP0_FSR", "gpio3_19", "input_pulldown",
+				"MCASP0_AHCLKX", "gpio3_21", "input_pulldown",
+				/* TIMERs */
+				"GPMC_ADVn_ALE", "timer4", "output",
+				"GPMC_BEn0_CLE", "timer5", "output",
+				"GPMC_WEn", "timer6", "output",
+				"GPMC_OEn_REn", "timer7", "output",
+				/* PWM */
+				"GPMC_A2", "ehrpwm1A", "output",
+				"GPMC_A3", "ehrpwm1B", "output",
+				"GPMC_AD8", "ehrpwm2A", "output",
+				"GPMC_AD9", "ehrpwm2B", "output";
 		};
 
 		prcm@44E00000 {
@@ -125,7 +176,7 @@
 				0x4804C000 0x1000
 				0x481AC000 0x1000
 				0x481AE000 0x1000 >;
-			interrupts = < 17 19 21 23 >;
+			interrupts = < 96 97 98 99 32 33 62 63 >;
 			interrupt-parent = <&AINTC>;
 		};
 
@@ -161,7 +212,7 @@
 			#address-cells = <1>;
 			#size-cells = <1>;
 			compatible = "ti,cpsw";
-			reg = <0x4A100000 0x3000>;
+			reg = <0x4A100000 0x4000>;
 			interrupts = <40 41 42 43>;
 			interrupt-parent = <&AINTC>;
 			phy-handle = <&phy0>;
diff --git a/sys/boot/fdt/dts/cubieboard.dts b/sys/boot/fdt/dts/cubieboard.dts
new file mode 100644
index 0000000..c4dd2a8
--- /dev/null
+++ b/sys/boot/fdt/dts/cubieboard.dts
@@ -0,0 +1,111 @@
+/*-
+ * Copyright (c) 2012 Ganbold Tsagaankhuu <ganbold@gmail.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ * 
+ * $FreeBSD$
+ */
+
+/dts-v1/;
+
+/ {
+	model = "Cubietech Cubieboard";
+	compatible = "cubietech,a10-cubieboard", "allwinner,sun4i-a10";
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	interrupt-parent = <&AINTC>;
+
+	memory {
+		device_type = "memory";
+		reg = < 0x40000000 0x20000000 >;	/* 512MB RAM */
+	};
+
+	aliases {
+		soc = &SOC;
+		UART0 = &UART0;
+	};
+
+	SOC: a10 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "simple-bus";
+		ranges;
+		bus-frequency = <0>;
+
+		AINTC: interrupt-controller@01c20400 {
+			compatible = "allwinner,sun4i-ic";
+			interrupt-controller;
+			#address-cells = <0>;
+			#interrupt-cells = <1>;
+			reg =   < 0x01c20400 0x400 >;
+		};
+
+		ccm@01c20000 {
+			compatible = "allwinner,sun4i-ccm";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			reg = < 0x01c20000 0x400 >;
+		};
+
+		timer@01c20c00 {
+			compatible = "allwinner,sun4i-timer";
+			reg = <0x01c20c00 0x90>;
+			interrupts = < 22 >;
+			interrupt-parent = <&AINTC>;
+			clock-frequency = < 24000000 >;
+		};
+
+		usb1: usb@01c1c000 {
+			compatible = "allwinner,usb-ehci", "usb-ehci";
+			reg = <0x01c1c000 0x1000>;
+			interrupts = < 40 >;
+			interrupt-parent = <&AINTC>;
+		};
+
+		sata@01c18000 {
+			compatible = "allwinner,ahci";
+			reg = <0x01c18000 0x1000>;
+			interrupts = <56>;
+			interrupt-parent = <&AINTC>;
+		};
+
+		UART0: serial@01c28000 {
+			status = "okay";
+			compatible = "ns16550";
+			reg = <0x01c28000 0x400>;
+			reg-shift = <2>;
+			interrupts = <1>;
+			interrupt-parent = <&AINTC>;
+			current-speed = <115200>;
+			clock-frequency = < 24000000 >;
+		};
+	};
+
+	chosen {
+		bootargs = "-v";
+		stdin = "UART0";
+		stdout = "UART0";
+	};
+};
+
diff --git a/sys/boot/fdt/dts/dreamplug-1001.dts b/sys/boot/fdt/dts/dreamplug-1001.dts
new file mode 100644
index 0000000..92ca1b0
--- /dev/null
+++ b/sys/boot/fdt/dts/dreamplug-1001.dts
@@ -0,0 +1,319 @@
+/*
+ * Copyright (c) 2013 Ian Lepore
+ * Copyright (c) 2010 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software substantially based on work developed by Semihalf 
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * GlobalScale Technologies DreamPlug Device Tree Source.
+ *
+ * This source is for version 10 revision 01 units with NOR SPI flash.
+ * These units are marked "1001" on the serial number label.
+ *
+ * $FreeBSD$
+ */
+
+/dts-v1/;
+
+/ {
+	model = "GlobalScale Technologies Dreamplug v1001";
+	compatible = "globalscale,dreamplug-003-ds2001", "globalscale,dreamplug", "marvell,kirkwood-88f6281", "marvell,kirkwood";
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	aliases {
+		ethernet0 = &enet0;
+		ethernet1 = &enet1;
+		mpp = &MPP;
+		serial0 = &serial0;
+		serial1 = &serial1;
+		soc = &SOC;
+		sram = &SRAM;
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu@0 {
+			device_type = "cpu";
+			compatible = "ARM,88FR131";
+			reg = <0x0>;
+			d-cache-line-size = <32>;	// 32 bytes
+			i-cache-line-size = <32>;	// 32 bytes
+			d-cache-size = <0x4000>;	// L1, 16K
+			i-cache-size = <0x4000>;	// L1, 16K
+			timebase-frequency = <0>;
+			bus-frequency = <0>;
+			clock-frequency = <0>;
+		};
+
+	};
+
+	memory {
+		device_type = "memory";
+		reg = <0x0 0x20000000>;		// 512M at 0x0
+	};
+
+	localbus@0 {
+		#address-cells = <2>;
+		#size-cells = <1>;
+		compatible = "mrvl,lbc";
+		bank-count = <1>;
+
+		/* This reflects CPU decode windows setup. */
+		ranges = <0x0 0x1e 0xfa000000 0x00100000>;
+
+		nor@0,0 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "cfi-flash";
+			reg = <0x0 0x0 0x00100000>;
+			bank-width = <2>;
+			device-width = <1>;
+		};
+	};
+
+	SOC: soc88f6281@f1000000 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "simple-bus";
+		ranges = <0x0 0xf1000000 0x00100000>;
+		bus-frequency = <0>;
+
+		PIC: pic@20200 {
+			interrupt-controller;
+			#address-cells = <0>;
+			#interrupt-cells = <1>;
+			reg = <0x20200 0x3c>;
+			compatible = "mrvl,pic";
+		};
+
+		timer@20300 {
+			compatible = "mrvl,timer";
+			reg = <0x20300 0x30>;
+			interrupts = <1>;
+			interrupt-parent = <&PIC>;
+			mrvl,has-wdt;
+		};
+
+		MPP: mpp@10000 {
+			#pin-cells = <2>;
+			compatible = "mrvl,mpp";
+			reg = <0x10000 0x34>;
+			pin-count = <50>;
+			pin-map = <
+				 0 2		/* MPP[ 0]:  SPI_SCn */
+				 1 2		/* MPP[ 1]:  SPI_MOSI */
+				 2 2		/* MPP[ 2]:  SPI_SCK */
+				 3 2		/* MPP[ 3]:  SPI_MISO */
+				 4 1		/* MPP[ 4]: NF_IO[6] */
+				 5 1		/* MPP[ 5]: NF_IO[7] */
+				 6 1		/* MPP[ 6]: SYSRST_OUTn */
+				 7 0		/* MPP[ 7]: GPO[7] */
+				 8 1		/* MPP[ 8]: TW_SDA */
+				 9 1		/* MPP[ 9]: TW_SCK */
+				10 3		/* MPP[10]: UA0_TXD */
+				11 3		/* MPP[11]: US0_RXD */
+				12 1		/* MPP[12]: SD_CLK */
+				13 1		/* MPP[13]: SD_CMD */
+				14 1		/* MPP[14]: SD_D[0] */
+				15 1		/* MPP[15]: SD_D[1] */
+				16 1		/* MPP[16]: SD_D[2] */
+				17 1		/* MPP[17]: SD_D[3] */
+				18 1		/* MPP[18]: NF_IO[0] */
+				19 1		/* MPP[19]: NF_IO[1] */
+				20 3		/* MPP[20]: GE1[ 0] */
+				21 3		/* MPP[21]: GE1[ 1] */
+				22 3		/* MPP[22]: GE1[ 2] */
+				23 3		/* MPP[23]: GE1[ 3] */
+				24 3		/* MPP[24]: GE1[ 4] */
+				25 3		/* MPP[25]: GE1[ 5] */
+				26 3		/* MPP[26]: GE1[ 6] */
+				27 3		/* MPP[27]: GE1[ 7] */
+				28 3		/* MPP[28]: GE1[ 8] */
+				29 3		/* MPP[29]: GE1[ 9] */
+				30 3		/* MPP[30]: GE1[10] */
+				31 3		/* MPP[31]: GE1[11] */
+				32 3		/* MPP[32]: GE1[12] */
+				33 3		/* MPP[33]: GE1[13] */
+				34 3		/* MPP[34]: GE1[14] */
+				35 3		/* MPP[35]: GE1[15] */
+				36 0		/* MPP[36]: GPIO[36] */
+				37 0		/* MPP[37]: GPIO[37] */
+				38 0		/* MPP[38]: GPIO[38] */
+				39 0		/* MPP[39]: GPIO[39] */
+				40 2		/* MPP[40]: TDM_SPI_SCK */
+				41 2		/* MPP[41]: TDM_SPI_MISO */
+				42 2		/* MPP[42]: TDM_SPI_MOSI */
+				43 0		/* MPP[43]: GPIO[43] */
+				44 0		/* MPP[44]: GPIO[44] */
+				45 0		/* MPP[45]: GPIO[45] */
+				46 0		/* MPP[46]: GPIO[46] */
+				47 0		/* MPP[47]: GPIO[47] */
+				48 0		/* MPP[48]: GPIO[48] */
+				49 0		/* MPP[49]: GPIO[49] */
+				>;
+		};
+
+		GPIO: gpio@10100 {
+			#gpio-cells = <3>;
+			compatible = "mrvl,gpio";
+			reg = <0x10100 0x20>;
+			gpio-controller;
+			interrupts = <35 36 37 38 39 40 41>;
+			interrupt-parent = <&PIC>;
+			pin-count = <50>;
+		};
+
+		gpioled@0 {
+			compatible = "mrvl,gpioled";
+
+			gpios = <&GPIO 47 2 0	/* GPIO[47] BT LED: OUT */
+				 &GPIO 48 2 0	/* GPIO[48] WLAN LED: OUT */
+				 &GPIO 49 2 0>;	/* GPIO[49] WLAN AP LED: OUT */
+		};
+
+		rtc@10300 {
+			compatible = "mrvl,rtc";
+			reg = <0x10300 0x08>;
+		};
+
+		twsi@11000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "mrvl,twsi";
+			reg = <0x11000 0x20>;
+			interrupts = <43>;
+			interrupt-parent = <&PIC>;
+		};
+
+		enet0: ethernet@72000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			model = "V2";
+			compatible = "mrvl,ge";
+			reg = <0x72000 0x2000>;
+			ranges = <0x0 0x72000 0x2000>;
+			local-mac-address = [ 00 00 00 00 00 00 ];
+			interrupts = <12 13 14 11 46>;
+			interrupt-parent = <&PIC>;
+			phy-handle = <&phy0>;
+
+			mdio@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				compatible = "mrvl,mdio";
+
+				phy0: ethernet-phy@0 {
+					reg = <0x0>;
+				};
+
+				phy1: ethernet-phy@1 {
+					reg = <0x1>;
+				};
+			};
+		};
+
+		enet1: ethernet@76000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			model = "V2";
+			compatible = "mrvl,ge";
+			reg = <0x76000 0x02000>;
+			ranges = <0x0 0x76000 0x2000>;
+			local-mac-address = [ 00 00 00 00 00 00 ];
+			interrupts = <16 17 18 15 47>;
+			interrupt-parent = <&PIC>;
+			phy-handle = <&phy1>;
+		};
+
+		serial0: serial@12000 {
+			compatible = "ns16550";
+			reg = <0x12000 0x20>;
+			reg-shift = <2>;
+			clock-frequency = <0>;
+			interrupts = <33>;
+			interrupt-parent = <&PIC>;
+		};
+
+		serial1: serial@12100 {
+			compatible = "ns16550";
+			reg = <0x12100 0x20>;
+			reg-shift = <2>;
+			clock-frequency = <0>;
+			interrupts = <34>;
+			interrupt-parent = <&PIC>;
+		};
+
+		crypto@30000 {
+			compatible = "mrvl,cesa";
+			reg = <0x30000 0x10000>;
+			interrupts = <22>;
+			interrupt-parent = <&PIC>;
+
+			sram-handle = <&SRAM>;
+		};
+
+		usb@50000 {
+			compatible = "mrvl,usb-ehci", "usb-ehci";
+			reg = <0x50000 0x1000>;
+			interrupts = <48 19>;
+			interrupt-parent = <&PIC>;
+		};
+
+		xor@60000 {
+			compatible = "mrvl,xor";
+			reg = <0x60000 0x1000>;
+			interrupts = <5 6 7 8>;
+			interrupt-parent = <&PIC>;
+		};
+
+		sata@80000 {
+			compatible = "mrvl,sata";
+			reg = <0x80000 0x6000>;
+			interrupts = <21>;
+			interrupt-parent = <&PIC>;
+		};
+
+		sdio@90000 {
+			compatible = "mrvl,sdio";
+			reg = <0x90000 0x134>;
+			interrupts = <28>;
+			interrupt-parent = <&PIC>;
+		};
+	};
+
+	SRAM: sram@fd000000 {
+		compatible = "mrvl,cesa-sram";
+		reg = <0xfd000000 0x00100000>;
+	};
+
+	chosen {
+		stdin = "serial0";
+		stdout = "serial0";
+	};
+
+};
diff --git a/sys/boot/fdt/dts/dreamplug-1001N.dts b/sys/boot/fdt/dts/dreamplug-1001N.dts
new file mode 100644
index 0000000..230a65f
--- /dev/null
+++ b/sys/boot/fdt/dts/dreamplug-1001N.dts
@@ -0,0 +1,340 @@
+/*
+ * Copyright (c) 2013 Ian Lepore
+ * Copyright (c) 2010 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software substantially based on work developed by Semihalf 
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * GlobalScale Technologies DreamPlug Device Tree Source.
+ *
+ * This source is for version 10 revision 01 units with NAND flash.
+ * These units are marked "1001N" on the serial number label.
+ *
+ * $FreeBSD$
+ */
+
+/dts-v1/;
+
+/ {
+	model = "GlobalScale Technologies Dreamplug v1001N";
+	compatible = "globalscale,dreamplug-003-ds2001", "globalscale,dreamplug", "marvell,kirkwood-88f6281", "marvell,kirkwood";
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	aliases {
+		ethernet0 = &enet0;
+		ethernet1 = &enet1;
+		mpp = &MPP;
+		serial0 = &serial0;
+		serial1 = &serial1;
+		soc = &SOC;
+		sram = &SRAM;
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu@0 {
+			device_type = "cpu";
+			compatible = "ARM,88FR131";
+			reg = <0x0>;
+			d-cache-line-size = <32>;	// 32 bytes
+			i-cache-line-size = <32>;	// 32 bytes
+			d-cache-size = <0x4000>;	// L1, 16K
+			i-cache-size = <0x4000>;	// L1, 16K
+			timebase-frequency = <0>;
+			bus-frequency = <0>;
+			clock-frequency = <0>;
+		};
+
+	};
+
+	memory {
+		device_type = "memory";
+		reg = <0x0 0x20000000>;		// 512M at 0x0
+	};
+
+	localbus@0 {
+		#address-cells = <2>;
+		#size-cells = <1>;
+		compatible = "mrvl,lbc";
+		bank-count = <1>;
+
+		/* This reflects CPU decode windows setup. */
+		ranges = <0x0 0x2f 0xf9300000 0x00100000>;
+
+		nand@0,0 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			compatible = "mrvl,nfc";
+			reg = <0x0 0x0 0x00100000>;
+			bank-width = <2>;
+			device-width = <1>;
+
+			// Slice info reported by builtin linux when it boots...
+			//[   11.161328] 0x00000000-0x00100000 : "u-boot"
+			//[   11.167431] 0x00100000-0x00500000 : "uImage"
+			//[   11.173471] 0x00500000-0x20000000 : "root"
+
+			slice@0 {
+				reg = <0x0 0x100000>;
+				label = "u-boot";
+				read-only;
+			};
+
+			slice@200000 {
+				reg = <0x100000 0x40000>;
+				label = "uImage";
+			};
+
+			slice@500000 {
+				reg = <0x500000 0x1FB00000>;
+				label = "root";
+			};
+		};
+	};
+
+	SOC: soc88f6281@f1000000 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "simple-bus";
+		ranges = <0x0 0xf1000000 0x00100000>;
+		bus-frequency = <0>;
+
+		PIC: pic@20200 {
+			interrupt-controller;
+			#address-cells = <0>;
+			#interrupt-cells = <1>;
+			reg = <0x20200 0x3c>;
+			compatible = "mrvl,pic";
+		};
+
+		timer@20300 {
+			compatible = "mrvl,timer";
+			reg = <0x20300 0x30>;
+			interrupts = <1>;
+			interrupt-parent = <&PIC>;
+			mrvl,has-wdt;
+		};
+
+		MPP: mpp@10000 {
+			#pin-cells = <2>;
+			compatible = "mrvl,mpp";
+			reg = <0x10000 0x34>;
+			pin-count = <50>;
+			pin-map = <
+				 0 1		/* MPP[ 0]: NF_IO[2] */
+				 1 1		/* MPP[ 1]: NF_IO[3] */
+				 2 1		/* MPP[ 2]: NF_IO[4] */
+				 3 1		/* MPP[ 3]: NF_IO[5] */
+				 4 1		/* MPP[ 4]: NF_IO[6] */
+				 5 1		/* MPP[ 5]: NF_IO[7] */
+				 6 1		/* MPP[ 6]: SYSRST_OUTn */
+				 7 0		/* MPP[ 7]: GPO[7] */
+				 8 1		/* MPP[ 8]: TW_SDA */
+				 9 1		/* MPP[ 9]: TW_SCK */
+				10 3		/* MPP[10]: UA0_TXD */
+				11 3		/* MPP[11]: US0_RXD */
+				12 1		/* MPP[12]: SD_CLK */
+				13 1		/* MPP[13]: SD_CMD */
+				14 1		/* MPP[14]: SD_D[0] */
+				15 1		/* MPP[15]: SD_D[1] */
+				16 1		/* MPP[16]: SD_D[2] */
+				17 1		/* MPP[17]: SD_D[3] */
+				18 1		/* MPP[18]: NF_IO[0] */
+				19 1		/* MPP[19]: NF_IO[1] */
+				20 3		/* MPP[20]: GE1[ 0] */
+				21 3		/* MPP[21]: GE1[ 1] */
+				22 3		/* MPP[22]: GE1[ 2] */
+				23 3		/* MPP[23]: GE1[ 3] */
+				24 3		/* MPP[24]: GE1[ 4] */
+				25 3		/* MPP[25]: GE1[ 5] */
+				26 3		/* MPP[26]: GE1[ 6] */
+				27 3		/* MPP[27]: GE1[ 7] */
+				28 3		/* MPP[28]: GE1[ 8] */
+				29 3		/* MPP[29]: GE1[ 9] */
+				30 3		/* MPP[30]: GE1[10] */
+				31 3		/* MPP[31]: GE1[11] */
+				32 3		/* MPP[32]: GE1[12] */
+				33 3		/* MPP[33]: GE1[13] */
+				34 3		/* MPP[34]: GE1[14] */
+				35 3		/* MPP[35]: GE1[15] */
+				36 0		/* MPP[36]: GPIO[36] */
+				37 0		/* MPP[37]: GPIO[37] */
+				38 0		/* MPP[38]: GPIO[38] */
+				39 0		/* MPP[39]: GPIO[39] */
+				40 2		/* MPP[40]: TDM_SPI_SCK */
+				41 2		/* MPP[41]: TDM_SPI_MISO */
+				42 2		/* MPP[42]: TDM_SPI_MOSI */
+				43 0		/* MPP[43]: GPIO[43] */
+				44 0		/* MPP[44]: GPIO[44] */
+				45 0		/* MPP[45]: GPIO[45] */
+				46 0		/* MPP[46]: GPIO[46] */
+				47 0		/* MPP[47]: GPIO[47] */
+				48 0		/* MPP[48]: GPIO[48] */
+				49 0		/* MPP[49]: GPIO[49] */
+				>;
+		};
+
+		GPIO: gpio@10100 {
+			#gpio-cells = <3>;
+			compatible = "mrvl,gpio";
+			reg = <0x10100 0x20>;
+			gpio-controller;
+			interrupts = <35 36 37 38 39 40 41>;
+			interrupt-parent = <&PIC>;
+			pin-count = <50>;
+		};
+
+		gpioled@0 {
+			compatible = "mrvl,gpioled";
+
+			gpios = <&GPIO 47 2 0	/* GPIO[47] BT LED: OUT */
+				 &GPIO 48 2 0	/* GPIO[48] WLAN LED: OUT */
+				 &GPIO 49 2 0>;	/* GPIO[49] WLAN AP LED: OUT */
+		};
+
+		rtc@10300 {
+			compatible = "mrvl,rtc";
+			reg = <0x10300 0x08>;
+		};
+
+		twsi@11000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "mrvl,twsi";
+			reg = <0x11000 0x20>;
+			interrupts = <43>;
+			interrupt-parent = <&PIC>;
+		};
+
+		enet0: ethernet@72000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			model = "V2";
+			compatible = "mrvl,ge";
+			reg = <0x72000 0x2000>;
+			ranges = <0x0 0x72000 0x2000>;
+			local-mac-address = [ 00 00 00 00 00 00 ];
+			interrupts = <12 13 14 11 46>;
+			interrupt-parent = <&PIC>;
+			phy-handle = <&phy0>;
+
+			mdio@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				compatible = "mrvl,mdio";
+
+				phy0: ethernet-phy@0 {
+					reg = <0x0>;
+				};
+
+				phy1: ethernet-phy@1 {
+					reg = <0x1>;
+				};
+			};
+		};
+
+		enet1: ethernet@76000 {
+			#address-cells = <1>;
+			#size-cells = <1>;
+			model = "V2";
+			compatible = "mrvl,ge";
+			reg = <0x76000 0x02000>;
+			ranges = <0x0 0x76000 0x2000>;
+			local-mac-address = [ 00 00 00 00 00 00 ];
+			interrupts = <16 17 18 15 47>;
+			interrupt-parent = <&PIC>;
+			phy-handle = <&phy1>;
+		};
+
+		serial0: serial@12000 {
+			compatible = "ns16550";
+			reg = <0x12000 0x20>;
+			reg-shift = <2>;
+			clock-frequency = <0>;
+			interrupts = <33>;
+			interrupt-parent = <&PIC>;
+		};
+
+		serial1: serial@12100 {
+			compatible = "ns16550";
+			reg = <0x12100 0x20>;
+			reg-shift = <2>;
+			clock-frequency = <0>;
+			interrupts = <34>;
+			interrupt-parent = <&PIC>;
+		};
+
+		crypto@30000 {
+			compatible = "mrvl,cesa";
+			reg = <0x30000 0x10000>;
+			interrupts = <22>;
+			interrupt-parent = <&PIC>;
+
+			sram-handle = <&SRAM>;
+		};
+
+		usb@50000 {
+			compatible = "mrvl,usb-ehci", "usb-ehci";
+			reg = <0x50000 0x1000>;
+			interrupts = <48 19>;
+			interrupt-parent = <&PIC>;
+		};
+
+		xor@60000 {
+			compatible = "mrvl,xor";
+			reg = <0x60000 0x1000>;
+			interrupts = <5 6 7 8>;
+			interrupt-parent = <&PIC>;
+		};
+
+		sata@80000 {
+			compatible = "mrvl,sata";
+			reg = <0x80000 0x6000>;
+			interrupts = <21>;
+			interrupt-parent = <&PIC>;
+		};
+
+		sdio@90000 {
+			compatible = "mrvl,sdio";
+			reg = <0x90000 0x134>;
+			interrupts = <28>;
+			interrupt-parent = <&PIC>;
+		};
+	};
+
+	SRAM: sram@fd000000 {
+		compatible = "mrvl,cesa-sram";
+		reg = <0xfd000000 0x00100000>;
+	};
+
+	chosen {
+		stdin = "serial0";
+		stdout = "serial0";
+	};
+
+};
diff --git a/sys/boot/i386/efi/Makefile b/sys/boot/i386/efi/Makefile
index 95c219d..25df59b 100644
--- a/sys/boot/i386/efi/Makefile
+++ b/sys/boot/i386/efi/Makefile
@@ -36,7 +36,7 @@ FILES=	loader.efi
 FILESMODE_loader.efi=	${BINMODE}
 
 LDSCRIPT=	${.CURDIR}/ldscript.i386
-LDFLAGS=	-Wl,-T${LDSCRIPT} -shared -symbolic
+LDFLAGS=	-Wl,-T${LDSCRIPT} -Wl,-Bsymbolic -shared
 
 ${PROG}:	${LDSCRIPT}
 
diff --git a/sys/boot/i386/libi386/bootinfo64.c b/sys/boot/i386/libi386/bootinfo64.c
index 0c6d077..617414e 100644
--- a/sys/boot/i386/libi386/bootinfo64.c
+++ b/sys/boot/i386/libi386/bootinfo64.c
@@ -134,7 +134,8 @@ bi_checkcpu(void)
 {
     char *cpu_vendor;
     int vendor[3];
-    int eflags, regs[4];
+    int eflags;
+    unsigned int regs[4];
 
     /* Check for presence of "cpuid". */
     eflags = read_eflags();
diff --git a/sys/boot/i386/libi386/comconsole.c b/sys/boot/i386/libi386/comconsole.c
index 3822597..e2710c1 100644
--- a/sys/boot/i386/libi386/comconsole.c
+++ b/sys/boot/i386/libi386/comconsole.c
@@ -50,7 +50,6 @@ static int	comc_init(int arg);
 static void	comc_putchar(int c);
 static int	comc_getchar(void);
 static int	comc_getspeed(void);
-static void	set_hw_console_hint(void);
 static int	comc_ischar(void);
 static int	comc_parseint(const char *string);
 static uint32_t comc_parse_pcidev(const char *string);
@@ -202,27 +201,14 @@ comc_port_set(struct env_var *ev, int flags, const void *value)
     }
 
     if ((comconsole.c_flags & (C_ACTIVEIN | C_ACTIVEOUT)) != 0 &&
-	comc_port != port) {
+	comc_port != port)
 	comc_setup(comc_curspeed, port);
-	set_hw_console_hint();
-    }
 
     env_setenv(ev->ev_name, flags | EV_NOHOOK, value, NULL, NULL);
 
     return (CMD_OK);
 }
 
-static void
-set_hw_console_hint(void)
-{
-	char intbuf[64];
-
-	unsetenv("hw.uart.console");
-	sprintf(intbuf, "io:%d,br:%d", comc_port, comc_curspeed);
-	env_setenv("hw.uart.console", EV_VOLATILE, intbuf,
-	    env_noset, env_nounset);
-}
-
 /*
  * Input: bus:dev:func[:bar]. If bar is not specified, it is 0x10.
  * Output: bar[24:16] bus[15:8] dev[7:3] func[2:0]
@@ -288,7 +274,6 @@ comc_pcidev_handle(uint32_t locator)
 		   comc_port_set, env_nounset);
 
 	comc_setup(comc_curspeed, port);
-	set_hw_console_hint();
 	comc_locator = locator;
 
 	return (CMD_OK);
@@ -318,8 +303,10 @@ static void
 comc_setup(int speed, int port)
 {
     static int TRY_COUNT = 1000000;
+    char intbuf[64];
     int tries;
 
+    unsetenv("hw.uart.console");
     comc_curspeed = speed;
     comc_port = port;
 
@@ -334,9 +321,11 @@ comc_setup(int speed, int port)
         inb(comc_port + com_data);
     while (inb(comc_port + com_lsr) & LSR_RXRDY && ++tries < TRY_COUNT);
 
-    if (tries < TRY_COUNT)
+    if (tries < TRY_COUNT) {
 	comconsole.c_flags |= (C_PRESENTIN | C_PRESENTOUT);
-    else
+	sprintf(intbuf, "io:%d,br:%d", comc_port, comc_curspeed);
+	env_setenv("hw.uart.console", EV_VOLATILE, intbuf, NULL, NULL);
+    } else
 	comconsole.c_flags &= ~(C_PRESENTIN | C_PRESENTOUT);
 }
 
diff --git a/sys/boot/i386/libi386/devicename.c b/sys/boot/i386/libi386/devicename.c
index 15889d3..c7705d7 100644
--- a/sys/boot/i386/libi386/devicename.c
+++ b/sys/boot/i386/libi386/devicename.c
@@ -128,7 +128,7 @@ i386_parsedev(struct i386_devdesc **dev, const char *devspec, const char **path)
 		goto fail;
 	    }
 	} else {
-		cp = np;
+		cp = (char *)np;
 	}
 	if (*cp && (*cp != ':')) {
 	    err = EINVAL;
diff --git a/sys/boot/i386/libi386/pxe.c b/sys/boot/i386/libi386/pxe.c
index e4ec5c0..49814dd 100644
--- a/sys/boot/i386/libi386/pxe.c
+++ b/sys/boot/i386/libi386/pxe.c
@@ -88,6 +88,12 @@ static int	pxe_netif_get(struct iodesc *desc, void *pkt, size_t len,
 static int	pxe_netif_put(struct iodesc *desc, void *pkt, size_t len);
 static void	pxe_netif_end(struct netif *nif);
 
+#ifdef OLD_NFSV2
+int nfs_getrootfh(struct iodesc*, char*, u_char*);
+#else
+int nfs_getrootfh(struct iodesc*, char*, uint32_t*, u_char*);
+#endif
+
 extern struct netif_stats	pxe_st[];
 extern u_int16_t		__bangpxeseg;
 extern u_int16_t		__bangpxeoff;
diff --git a/sys/boot/sparc64/loader/main.c b/sys/boot/sparc64/loader/main.c
index fa9d068..5a0badf 100644
--- a/sys/boot/sparc64/loader/main.c
+++ b/sys/boot/sparc64/loader/main.c
@@ -142,6 +142,10 @@ static vm_offset_t heapva;
 static char bootpath[64];
 static phandle_t root;
 
+#ifdef LOADER_ZFS_SUPPORT
+static struct zfs_devdesc zfs_currdev;
+#endif
+
 /*
  * Machine dependent structures that the machine independent
  * loader part uses.
@@ -732,7 +736,6 @@ static void
 sparc64_zfs_probe(void)
 {
 	struct vtoc8 vtoc;
-	struct zfs_devdesc zfs_currdev;
 	char alias[64], devname[sizeof(alias) + sizeof(":x") - 1];
 	char type[sizeof("device_type")];
 	char *bdev, *dev, *odev;
@@ -805,9 +808,6 @@ sparc64_zfs_probe(void)
 		zfs_currdev.root_guid = 0;
 		zfs_currdev.d_dev = &zfs_dev;
 		zfs_currdev.d_type = zfs_currdev.d_dev->dv_type;
-		(void)strncpy(bootpath, zfs_fmtdev(&zfs_currdev),
-		    sizeof(bootpath) - 1);
-		bootpath[sizeof(bootpath) - 1] = '\0';
 	}
 }
 #endif /* LOADER_ZFS_SUPPORT */
@@ -878,10 +878,14 @@ main(int (*openfirm)(void *))
 		if ((*dp)->dv_init != 0)
 			(*dp)->dv_init();
 
-	/*
-	 * Now that sparc64_zfs_probe() might have altered bootpath,
-	 * export it.
-	 */
+#ifdef LOADER_ZFS_SUPPORT
+	if (zfs_currdev.pool_guid != 0) {
+		(void)strncpy(bootpath, zfs_fmtdev(&zfs_currdev),
+		    sizeof(bootpath) - 1);
+		bootpath[sizeof(bootpath) - 1] = '\0';
+	}
+#endif
+
 	env_setenv("currdev", EV_VOLATILE, bootpath,
 	    ofw_setcurrdev, env_nounset);
 	env_setenv("loaddev", EV_VOLATILE, bootpath,
diff --git a/sys/boot/uboot/common/metadata.c b/sys/boot/uboot/common/metadata.c
index 74497a9..bbc1893 100644
--- a/sys/boot/uboot/common/metadata.c
+++ b/sys/boot/uboot/common/metadata.c
@@ -369,12 +369,15 @@ md_load(char *args, vm_offset_t *modulep)
 	/* Convert addresses to the final VA */
 	*modulep -= __elfN(relocation_offset);
 
-	for (i = 0; i < sizeof mdt / sizeof mdt[0]; i++) {
-		md = file_findmetadata(kfp, mdt[i]);
-		if (md) {
-			bcopy(md->md_data, &vaddr, sizeof vaddr);
-			vaddr -= __elfN(relocation_offset);
-			bcopy(&vaddr, md->md_data, sizeof vaddr);
+	/* Do relocation fixup on metadata of each module. */
+	for (xp = file_findfile(NULL, NULL); xp != NULL; xp = xp->f_next) {
+		for (i = 0; i < sizeof mdt / sizeof mdt[0]; i++) {
+			md = file_findmetadata(xp, mdt[i]);
+			if (md) {
+				bcopy(md->md_data, &vaddr, sizeof vaddr);
+				vaddr -= __elfN(relocation_offset);
+				bcopy(&vaddr, md->md_data, sizeof vaddr);
+			}
 		}
 	}
 
diff --git a/sys/boot/usb/Makefile b/sys/boot/usb/Makefile
new file mode 100644
index 0000000..b71b10d
--- /dev/null
+++ b/sys/boot/usb/Makefile
@@ -0,0 +1,150 @@
+#
+# $FreeBSD$
+#
+# Copyright (c) 2013 Hans Petter Selasky. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+T=${.CURDIR}/tools
+S=${.CURDIR}/../..
+
+.PATH: \
+	${.CURDIR} \
+	${S}/dev/usb \
+	${S}/dev/usb/controller \
+	${S}/dev/usb/serial \
+	${S}/dev/usb/storage \
+	${S}/dev/usb/template
+
+LIB=		usbboot
+INTERNALLIB=
+OBJCOPY?=	objcopy
+SYSCC?=		cc
+
+CFLAGS+=	-DBOOTPROG=\"usbloader\"
+CFLAGS+=	-DUSB_GLOBAL_INCLUDE_FILE="\"bsd_global.h\""
+CFLAGS+=	-ffunction-sections -fdata-sections
+CFLAGS+=	-ffreestanding
+CFLAGS+=	-Wformat -Wall
+CFLAGS+=	-I ${S}
+CFLAGS+=	-I ${T}
+CFLAGS+=	-I ${.CURDIR}
+CFLAGS+=	-g
+
+.if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64"
+CFLAGS+=	-march=i386
+CFLAGS+=	-mpreferred-stack-boundary=2
+.endif
+.if ${MACHINE_CPUARCH} == "amd64"
+CFLAGS+=	-m32
+.endif
+
+#
+# Single threaded BSD kernel
+#
+SRCS+=	bsd_kernel.c
+
+#
+# BUSSPACE implementation
+#
+SRCS+=	bsd_busspace.c
+
+#
+# BUSDMA implementation
+#
+SRCS+=	usb_busdma_loader.c
+
+#
+# USB controller drivers
+#
+SRCS+=	at91dci.c
+SRCS+=	atmegadci.c
+SRCS+=	avr32dci.c
+SRCS+=	dwc_otg.c
+SRCS+=	ehci.c
+SRCS+=	musb_otg.c
+SRCS+=	ohci.c
+SRCS+=	uhci.c
+SRCS+=	uss820dci.c
+SRCS+=	xhci.c
+SRCS+=	usb_controller.c
+
+CFLAGS += -DUSB_PROBE_LIST="\"xhci\", \"ehci\", \"uhci\", \"ohci\""
+
+#
+# USB core and templates
+#
+SRCS+=	usb_core.c
+SRCS+=	usb_debug.c
+SRCS+=	usb_device.c
+SRCS+=	usb_dynamic.c
+SRCS+=	usb_error.c
+SRCS+=	usb_handle_request.c
+SRCS+=	usb_hid.c
+SRCS+=	usb_hub.c
+SRCS+=	usb_lookup.c
+SRCS+=	usb_msctest.c
+SRCS+=	usb_parse.c
+SRCS+=	usb_request.c
+SRCS+=	usb_transfer.c
+SRCS+=	usb_util.c
+SRCS+=	usb_template.c
+SRCS+=	usb_template_cdce.c
+SRCS+=	usb_template_msc.c
+SRCS+=	usb_template_mtp.c
+SRCS+=	usb_template_modem.c
+SRCS+=	usb_template_mouse.c
+SRCS+=	usb_template_kbd.c
+SRCS+=	usb_template_audio.c
+SRCS+=	sysinit_data.c
+SRCS+=	sysuninit_data.c
+
+CLEANFILES+= sysinit
+CLEANFILES+= sysinit.bin
+CLEANFILES+= sysinit_data.c
+CLEANFILES+= sysuninit_data.c
+
+CLEANFILES+= ${SRCS:C/\.c/.osys/g}
+
+.include <bsd.lib.mk>
+
+#
+# SYSINIT() and SYSUNINIT() handling
+#
+sysinit: ${T}/sysinit.c
+	${SYSCC} -Wall -o ${.TARGET} ${.ALLSRC}
+
+sysinit_data.c: sysinit.bin sysinit
+	${.OBJDIR}/sysinit -i sysinit.bin -o ${.TARGET} -k sysinit -s sysinit_data
+
+sysuninit_data.c: sysinit.bin sysinit
+	${.OBJDIR}/sysinit -i sysinit.bin -o ${.TARGET} -R -k sysuninit -s sysuninit_data
+
+.for F in ${OBJS}
+${F}sys: ${F}
+	${OBJCOPY} -j ".debug.sysinit" -O binary ${F} ${.TARGET}
+	[ -f ${.TARGET} ] || touch ${.TARGET}
+.endfor
+
+sysinit.bin: ${OBJS:C/\.o/.osys/g:C/sysinit_data.osys//g:C/sysuninit_data.osys//g}
+	cat ${.ALLSRC} > sysinit.bin
diff --git a/sys/boot/usb/Makefile.test b/sys/boot/usb/Makefile.test
new file mode 100644
index 0000000..a366d36
--- /dev/null
+++ b/sys/boot/usb/Makefile.test
@@ -0,0 +1,61 @@
+#
+# $FreeBSD$
+#
+# Copyright (c) 2013 Hans Petter Selasky. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+
+#
+# USB test application
+#
+
+.PATH: ${.CURDIR}
+
+PROG=	usbloader
+MAN=
+SRCS= 
+
+CFLAGS+= -Wall
+CFLAGS+= -g
+
+.if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64"
+CFLAGS+=	-march=i386
+CFLAGS+=	-mpreferred-stack-boundary=2
+.endif
+.if ${MACHINE_CPUARCH} == "amd64"
+CFLAGS+=	-m32
+.endif
+
+LDFLAGS+= -Wl,--gc-sections
+
+SRCS+=	bsd_usbloader_test.c
+
+LDADD+=	libusbboot.a
+DPADD+= libusbboot.a
+
+all: libusbboot.a
+
+.include <bsd.prog.mk>
+
+libusbboot.a:
+	make -f Makefile
diff --git a/sys/boot/usb/bsd_busspace.c b/sys/boot/usb/bsd_busspace.c
new file mode 100644
index 0000000..c9ba09f
--- /dev/null
+++ b/sys/boot/usb/bsd_busspace.c
@@ -0,0 +1,207 @@
+/* $FreeBSD$ */
+/*-
+ * Copyright (c) 2013 Hans Petter Selasky. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <bsd_kernel.h>
+
+struct burst {
+	uint32_t dw0;
+	uint32_t dw1;
+	uint32_t dw2;
+	uint32_t dw3;
+	uint32_t dw4;
+	uint32_t dw5;
+	uint32_t dw6;
+	uint32_t dw7;
+};
+
+void
+bus_space_read_multi_1(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t offset, uint8_t *datap, bus_size_t count)
+{
+	while (count--) {
+		*datap++ = bus_space_read_1(t, h, offset);
+	}
+}
+
+void
+bus_space_read_multi_2(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t offset, uint16_t *datap, bus_size_t count)
+{
+	while (count--) {
+		*datap++ = bus_space_read_2(t, h, offset);
+	}
+}
+
+void
+bus_space_read_multi_4(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t offset, uint32_t *datap, bus_size_t count)
+{
+	h += offset;
+
+	while (count--) {
+		*datap++ = *((volatile uint32_t *)h);
+	}
+}
+
+void
+bus_space_write_multi_1(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t offset, uint8_t *datap, bus_size_t count)
+{
+	while (count--) {
+		uint8_t temp = *datap++;
+
+		bus_space_write_1(t, h, offset, temp);
+	}
+}
+
+void
+bus_space_write_multi_2(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t offset, uint16_t *datap, bus_size_t count)
+{
+	while (count--) {
+		uint16_t temp = *datap++;
+
+		bus_space_write_2(t, h, offset, temp);
+	}
+}
+
+void
+bus_space_write_multi_4(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t offset, uint32_t *datap, bus_size_t count)
+{
+	h += offset;
+
+	while (count--) {
+		*((volatile uint32_t *)h) = *datap++;
+	}
+}
+
+void
+bus_space_write_1(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t offset, uint8_t data)
+{
+	*((volatile uint8_t *)(h + offset)) = data;
+}
+
+void
+bus_space_write_2(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t offset, uint16_t data)
+{
+	*((volatile uint16_t *)(h + offset)) = data;
+}
+
+void
+bus_space_write_4(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t offset, uint32_t data)
+{
+	*((volatile uint32_t *)(h + offset)) = data;
+}
+
+uint8_t
+bus_space_read_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset)
+{
+	return (*((volatile uint8_t *)(h + offset)));
+}
+
+uint16_t
+bus_space_read_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset)
+{
+	return (*((volatile uint16_t *)(h + offset)));
+}
+
+uint32_t
+bus_space_read_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset)
+{
+	return (*((volatile uint32_t *)(h + offset)));
+}
+
+void
+bus_space_read_region_1(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t offset, uint8_t *datap, bus_size_t count)
+{
+	h += offset;
+
+	while (count--) {
+		*datap++ = *((volatile uint8_t *)h);
+		h += 1;
+	}
+}
+
+void
+bus_space_write_region_1(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t offset, uint8_t *datap, bus_size_t count)
+{
+	h += offset;
+
+	while (count--) {
+		*((volatile uint8_t *)h) = *datap++;
+		h += 1;
+	}
+}
+
+void
+bus_space_read_region_4(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t offset, uint32_t *datap, bus_size_t count)
+{
+	enum { BURST = sizeof(struct burst) / 4 };
+
+	h += offset;
+
+	while (count >= BURST) {
+		*(struct burst *)datap = *((/* volatile */ struct burst *)h);
+
+		h += BURST * 4;
+		datap += BURST;
+		count -= BURST;
+	}
+
+	while (count--) {
+		*datap++ = *((volatile uint32_t *)h);
+		h += 4;
+	}
+}
+
+void
+bus_space_write_region_4(bus_space_tag_t t, bus_space_handle_t h,
+    bus_size_t offset, uint32_t *datap, bus_size_t count)
+{
+	enum { BURST = sizeof(struct burst) / 4 };
+
+	h += offset;
+
+	while (count >= BURST) {
+		*((/* volatile */ struct burst *)h) = *(struct burst *)datap;
+
+		h += BURST * 4;
+		datap += BURST;
+		count -= BURST;
+	}
+
+	while (count--) {
+		*((volatile uint32_t *)h) = *datap++;
+		h += 4;
+	}
+}
diff --git a/sys/boot/usb/bsd_global.h b/sys/boot/usb/bsd_global.h
new file mode 100644
index 0000000..9fe1c8c
--- /dev/null
+++ b/sys/boot/usb/bsd_global.h
@@ -0,0 +1,63 @@
+/* $FreeBSD$ */
+/*-
+ * Copyright (c) 2013 Hans Petter Selasky. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _BSD_GLOBAL_H_
+#define	_BSD_GLOBAL_H_
+
+#include <bsd_kernel.h>
+
+#define	USB_DEBUG_VAR usb_debug
+#include <dev/usb/usb_freebsd_loader.h>
+#include <dev/usb/usb_endian.h>
+#include <dev/usb/usb.h>
+#include <dev/usb/usbdi.h>
+#include <dev/usb/usb_core.h>
+#include <dev/usb/usb_debug.h>
+#include <dev/usb/usb_process.h>
+#include <dev/usb/usb_busdma.h>
+#include <dev/usb/usb_dynamic.h>
+#include <dev/usb/usb_device.h>
+#include <dev/usb/usb_hub.h>
+#include <dev/usb/usb_controller.h>
+#include <dev/usb/usb_bus.h>
+#include <dev/usb/usbdi_util.h>
+#include <dev/usb/usb_cdc.h>
+#include <dev/usb/usb_dev.h>
+#include <dev/usb/usb_mbuf.h>
+#include <dev/usb/usb_msctest.h>
+#include <dev/usb/usb_pci.h>
+#include <dev/usb/usb_pf.h>
+#include <dev/usb/usb_request.h>
+#include <dev/usb/usb_util.h>
+#include <dev/usb/usb_transfer.h>
+#include <dev/usb/usb_compat_linux.h>
+#include <dev/usb/usbhid.h>
+#include <dev/usb/usb_ioctl.h>
+#include <dev/usb/usb_generic.h>
+#include <dev/usb/quirk/usb_quirk.h>
+#include <dev/usb/template/usb_template.h>
+
+#endif					/* _BSD_GLOBAL_H_ */
diff --git a/sys/boot/usb/bsd_kernel.c b/sys/boot/usb/bsd_kernel.c
new file mode 100644
index 0000000..5f24c2d
--- /dev/null
+++ b/sys/boot/usb/bsd_kernel.c
@@ -0,0 +1,1269 @@
+/* $FreeBSD$ */
+/*-
+ * Copyright (c) 2013 Hans Petter Selasky. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <bsd_global.h>
+
+static struct usb_process usb_process[USB_PROC_MAX];
+static device_t usb_pci_root;
+
+/*------------------------------------------------------------------------*
+ * Implementation of mutex API
+ *------------------------------------------------------------------------*/
+
+struct mtx Giant;
+
+static void
+mtx_system_init(void *arg)
+{
+	mtx_init(&Giant, "Giant", NULL, MTX_DEF | MTX_RECURSE);
+}
+SYSINIT(mtx_system_init, SI_SUB_LOCK, SI_ORDER_MIDDLE, mtx_system_init, NULL);
+
+void
+mtx_init(struct mtx *mtx, const char *name, const char *type, int opt)
+{
+	mtx->owned = 0;
+	mtx->parent = mtx;
+}
+
+void
+mtx_lock(struct mtx *mtx)
+{
+	mtx = mtx->parent;
+	mtx->owned++;
+}
+
+void
+mtx_unlock(struct mtx *mtx)
+{
+	mtx = mtx->parent;
+	mtx->owned--;
+}
+
+int
+mtx_owned(struct mtx *mtx)
+{
+	mtx = mtx->parent;
+	return (mtx->owned != 0);
+}
+
+void
+mtx_destroy(struct mtx *mtx)
+{
+	/* NOP */
+}
+
+/*------------------------------------------------------------------------*
+ * Implementation of shared/exclusive mutex API
+ *------------------------------------------------------------------------*/
+
+void
+sx_init_flags(struct sx *sx, const char *name, int flags)
+{
+	sx->owned = 0;
+}
+
+void
+sx_destroy(struct sx *sx)
+{
+	/* NOP */
+}
+
+void
+sx_xlock(struct sx *sx)
+{
+	sx->owned++;
+}
+
+void
+sx_xunlock(struct sx *sx)
+{
+	sx->owned--;
+}
+
+int
+sx_xlocked(struct sx *sx)
+{
+	return (sx->owned != 0);
+}
+
+/*------------------------------------------------------------------------*
+ * Implementaiton of condition variable API
+ *------------------------------------------------------------------------*/
+
+void
+cv_init(struct cv *cv, const char *desc)
+{
+	cv->sleeping = 0;
+}
+
+void
+cv_destroy(struct cv *cv)
+{
+	/* NOP */
+}
+
+void
+cv_wait(struct cv *cv, struct mtx *mtx)
+{
+	cv_timedwait(cv, mtx, -1);
+}
+
+int
+cv_timedwait(struct cv *cv, struct mtx *mtx, int timo)
+{
+	int start = ticks;
+	int delta;
+
+	if (cv->sleeping)
+		return (EWOULDBLOCK);	/* not allowed */
+
+	cv->sleeping = 1;
+
+	while (cv->sleeping) {
+		if (timo >= 0) {
+			delta = ticks - start;
+			if (delta >= timo || delta < 0)
+				break;
+		}
+		mtx_unlock(mtx);
+
+		usb_idle();
+
+		mtx_lock(mtx);
+	}
+
+	if (cv->sleeping) {
+		cv->sleeping = 0;
+		return (EWOULDBLOCK);	/* not allowed */
+	}
+	return (0);
+}
+
+void
+cv_signal(struct cv *cv)
+{
+	cv->sleeping = 0;
+}
+
+void
+cv_broadcast(struct cv *cv)
+{
+	cv->sleeping = 0;
+}
+
+/*------------------------------------------------------------------------*
+ * Implementation of callout API
+ *------------------------------------------------------------------------*/
+
+static void callout_proc_msg(struct usb_proc_msg *);
+
+volatile int ticks = 0;
+
+static LIST_HEAD(, callout) head_callout = LIST_HEAD_INITIALIZER(&head_callout);
+
+static struct mtx mtx_callout;
+static struct usb_proc_msg callout_msg[2];
+
+static void
+callout_system_init(void *arg)
+{
+	mtx_init(&mtx_callout, "callout-mtx", NULL, MTX_DEF | MTX_RECURSE);
+
+	callout_msg[0].pm_callback = &callout_proc_msg;
+	callout_msg[1].pm_callback = &callout_proc_msg;
+}
+SYSINIT(callout_system_init, SI_SUB_LOCK, SI_ORDER_MIDDLE, callout_system_init, NULL);
+
+static void
+callout_callback(struct callout *c)
+{
+	mtx_lock(c->mtx);
+
+	mtx_lock(&mtx_callout);
+	if (c->entry.le_prev != NULL) {
+		LIST_REMOVE(c, entry);
+		c->entry.le_prev = NULL;
+	}
+	mtx_unlock(&mtx_callout);
+
+	if (c->func)
+		(c->func) (c->arg);
+
+	if (!(c->flags & CALLOUT_RETURNUNLOCKED))
+		mtx_unlock(c->mtx);
+}
+
+void
+callout_process(int timeout)
+{
+	ticks += timeout;
+	usb_proc_msignal(usb_process + 2, &callout_msg[0], &callout_msg[1]);
+}
+
+static void
+callout_proc_msg(struct usb_proc_msg *pmsg)
+{
+	struct callout *c;
+	int delta;
+
+repeat:
+	mtx_lock(&mtx_callout);
+
+	LIST_FOREACH(c, &head_callout, entry) {
+
+		delta = c->timeout - ticks;
+		if (delta < 0) {
+			mtx_unlock(&mtx_callout);
+
+			callout_callback(c);
+
+			goto repeat;
+		}
+	}
+	mtx_unlock(&mtx_callout);
+}
+
+void
+callout_init_mtx(struct callout *c, struct mtx *mtx, int flags)
+{
+	memset(c, 0, sizeof(*c));
+
+	if (mtx == NULL)
+		mtx = &Giant;
+
+	c->mtx = mtx;
+	c->flags = (flags & CALLOUT_RETURNUNLOCKED);
+}
+
+void
+callout_reset(struct callout *c, int to_ticks,
+    void (*func) (void *), void *arg)
+{
+	callout_stop(c);
+
+	c->func = func;
+	c->arg = arg;
+	c->timeout = ticks + to_ticks;
+
+	mtx_lock(&mtx_callout);
+	LIST_INSERT_HEAD(&head_callout, c, entry);
+	mtx_unlock(&mtx_callout);
+}
+
+void
+callout_stop(struct callout *c)
+{
+	mtx_lock(&mtx_callout);
+
+	if (c->entry.le_prev != NULL) {
+		LIST_REMOVE(c, entry);
+		c->entry.le_prev = NULL;
+	}
+	mtx_unlock(&mtx_callout);
+
+	c->func = NULL;
+	c->arg = NULL;
+}
+
+void
+callout_drain(struct callout *c)
+{
+	if (c->mtx == NULL)
+		return;			/* not initialised */
+
+	mtx_lock(c->mtx);
+	callout_stop(c);
+	mtx_unlock(c->mtx);
+}
+
+int
+callout_pending(struct callout *c)
+{
+	int retval;
+
+	mtx_lock(&mtx_callout);
+	retval = (c->entry.le_prev != NULL);
+	mtx_unlock(&mtx_callout);
+
+	return (retval);
+}
+
+/*------------------------------------------------------------------------*
+ * Implementation of device API
+ *------------------------------------------------------------------------*/
+
+static const char unknown_string[] = { "unknown" };
+
+static TAILQ_HEAD(, module_data) module_head =
+    TAILQ_HEAD_INITIALIZER(module_head);
+
+static uint8_t
+devclass_equal(const char *a, const char *b)
+{
+	char ta, tb;
+
+	if (a == b)
+		return (1);
+
+	while (1) {
+		ta = *a;
+		tb = *b;
+		if (ta != tb)
+			return (0);
+		if (ta == 0)
+			break;
+		a++;
+		b++;
+	}
+	return (1);
+}
+
+int
+bus_generic_resume(device_t dev)
+{
+	return (0);
+}
+
+int
+bus_generic_shutdown(device_t dev)
+{
+	return (0);
+}
+
+int
+bus_generic_suspend(device_t dev)
+{
+	return (0);
+}
+
+int
+bus_generic_print_child(device_t dev, device_t child)
+{
+	return (0);
+}
+
+void
+bus_generic_driver_added(device_t dev, driver_t *driver)
+{
+	return;
+}
+
+device_t
+device_get_parent(device_t dev)
+{
+	return (dev ? dev->dev_parent : NULL);
+}
+
+void
+device_set_interrupt(device_t dev, intr_fn_t *fn, void *arg)
+{
+	dev->dev_irq_fn = fn;
+	dev->dev_irq_arg = arg;
+}
+
+void
+device_run_interrupts(device_t parent)
+{
+	device_t child;
+
+	if (parent == NULL)
+		return;
+
+	TAILQ_FOREACH(child, &parent->dev_children, dev_link) {
+		if (child->dev_irq_fn != NULL)
+			(child->dev_irq_fn) (child->dev_irq_arg);
+	}
+}
+
+void
+device_set_ivars(device_t dev, void *ivars)
+{
+	dev->dev_aux = ivars;
+}
+
+void   *
+device_get_ivars(device_t dev)
+{
+	return (dev ? dev->dev_aux : NULL);
+}
+
+int
+device_get_unit(device_t dev)
+{
+	return (dev ? dev->dev_unit : 0);
+}
+
+int
+bus_generic_detach(device_t dev)
+{
+	device_t child;
+	int error;
+
+	if (!dev->dev_attached)
+		return (EBUSY);
+
+	TAILQ_FOREACH(child, &dev->dev_children, dev_link) {
+		if ((error = device_detach(child)) != 0)
+			return (error);
+	}
+	return (0);
+}
+
+const char *
+device_get_nameunit(device_t dev)
+{
+	if (dev && dev->dev_nameunit[0])
+		return (dev->dev_nameunit);
+
+	return (unknown_string);
+}
+
+static uint8_t
+devclass_create(devclass_t *dc_pp)
+{
+	if (dc_pp == NULL) {
+		return (1);
+	}
+	if (dc_pp[0] == NULL) {
+		dc_pp[0] = malloc(sizeof(**(dc_pp)),
+		    M_DEVBUF, M_WAITOK | M_ZERO);
+
+		if (dc_pp[0] == NULL) {
+			return (1);
+		}
+	}
+	return (0);
+}
+
+static const struct module_data *
+devclass_find_create(const char *classname)
+{
+	const struct module_data *mod;
+
+	TAILQ_FOREACH(mod, &module_head, entry) {
+		if (devclass_equal(mod->mod_name, classname)) {
+			if (devclass_create(mod->devclass_pp)) {
+				continue;
+			}
+			return (mod);
+		}
+	}
+	return (NULL);
+}
+
+static uint8_t
+devclass_add_device(const struct module_data *mod, device_t dev)
+{
+	device_t *pp_dev;
+	device_t *end;
+	uint8_t unit;
+
+	pp_dev = mod->devclass_pp[0]->dev_list;
+	end = pp_dev + DEVCLASS_MAXUNIT;
+	unit = 0;
+
+	while (pp_dev != end) {
+		if (*pp_dev == NULL) {
+			*pp_dev = dev;
+			dev->dev_unit = unit;
+			dev->dev_module = mod;
+			snprintf(dev->dev_nameunit,
+			    sizeof(dev->dev_nameunit),
+			    "%s%d", device_get_name(dev), unit);
+			return (0);
+		}
+		pp_dev++;
+		unit++;
+	}
+	DPRINTF("Could not add device to devclass.\n");
+	return (1);
+}
+
+static void
+devclass_delete_device(const struct module_data *mod, device_t dev)
+{
+	if (mod == NULL) {
+		return;
+	}
+	mod->devclass_pp[0]->dev_list[dev->dev_unit] = NULL;
+	dev->dev_module = NULL;
+}
+
+static device_t
+make_device(device_t parent, const char *name)
+{
+	device_t dev = NULL;
+	const struct module_data *mod = NULL;
+
+	if (name) {
+
+		mod = devclass_find_create(name);
+
+		if (!mod) {
+
+			DPRINTF("%s:%d:%s: can't find device "
+			    "class %s\n", __FILE__, __LINE__,
+			    __FUNCTION__, name);
+
+			goto done;
+		}
+	}
+	dev = malloc(sizeof(*dev),
+	    M_DEVBUF, M_WAITOK | M_ZERO);
+
+	if (dev == NULL)
+		goto done;
+
+	dev->dev_parent = parent;
+	TAILQ_INIT(&dev->dev_children);
+
+	if (name) {
+		dev->dev_fixed_class = 1;
+		if (devclass_add_device(mod, dev)) {
+			goto error;
+		}
+	}
+done:
+	return (dev);
+
+error:
+	if (dev) {
+		free(dev, M_DEVBUF);
+	}
+	return (NULL);
+}
+
+device_t
+device_add_child(device_t dev, const char *name, int unit)
+{
+	device_t child;
+
+	if (unit != -1) {
+		device_printf(dev, "Unit is not -1\n");
+	}
+	child = make_device(dev, name);
+	if (child == NULL) {
+		device_printf(dev, "Could not add child '%s'\n", name);
+		goto done;
+	}
+	if (dev == NULL) {
+		/* no parent */
+		goto done;
+	}
+	TAILQ_INSERT_TAIL(&dev->dev_children, child, dev_link);
+done:
+	return (child);
+}
+
+int
+device_delete_child(device_t dev, device_t child)
+{
+	int error = 0;
+	device_t grandchild;
+
+	/* remove children first */
+
+	while ((grandchild = TAILQ_FIRST(&child->dev_children))) {
+		error = device_delete_child(child, grandchild);
+		if (error) {
+			device_printf(dev, "Error deleting child!\n");
+			goto done;
+		}
+	}
+
+	error = device_detach(child);
+
+	if (error)
+		goto done;
+
+	devclass_delete_device(child->dev_module, child);
+
+	if (dev != NULL) {
+		/* remove child from parent */
+		TAILQ_REMOVE(&dev->dev_children, child, dev_link);
+	}
+	free(child, M_DEVBUF);
+
+done:
+	return (error);
+}
+
+int
+device_delete_children(device_t dev)
+{
+	device_t child;
+	int error = 0;
+
+	while ((child = TAILQ_FIRST(&dev->dev_children))) {
+		error = device_delete_child(dev, child);
+		if (error) {
+			device_printf(dev, "Error deleting child!\n");
+			break;
+		}
+	}
+	return (error);
+}
+
+void
+device_quiet(device_t dev)
+{
+	dev->dev_quiet = 1;
+}
+
+const char *
+device_get_desc(device_t dev)
+{
+	if (dev)
+		return &(dev->dev_desc[0]);
+	return (unknown_string);
+}
+
+static int
+default_method(void)
+{
+	/* do nothing */
+	DPRINTF("Default method called\n");
+	return (0);
+}
+
+void   *
+device_get_method(device_t dev, const char *what)
+{
+	const struct device_method *mtod;
+
+	mtod = dev->dev_module->driver->methods;
+	while (mtod->func != NULL) {
+		if (devclass_equal(mtod->desc, what)) {
+			return (mtod->func);
+		}
+		mtod++;
+	}
+	return ((void *)&default_method);
+}
+
+const char *
+device_get_name(device_t dev)
+{
+	if (dev == NULL)
+		return (unknown_string);
+
+	return (dev->dev_module->driver->name);
+}
+
+static int
+device_allocate_softc(device_t dev)
+{
+	const struct module_data *mod;
+
+	mod = dev->dev_module;
+
+	if ((dev->dev_softc_alloc == 0) &&
+	    (mod->driver->size != 0)) {
+		dev->dev_sc = malloc(mod->driver->size,
+		    M_DEVBUF, M_WAITOK | M_ZERO);
+
+		if (dev->dev_sc == NULL)
+			return (ENOMEM);
+
+		dev->dev_softc_alloc = 1;
+	}
+	return (0);
+}
+
+int
+device_probe_and_attach(device_t dev)
+{
+	const struct module_data *mod;
+	const char *bus_name_parent;
+
+	bus_name_parent = device_get_name(device_get_parent(dev));
+
+	if (dev->dev_attached)
+		return (0);		/* fail-safe */
+
+	if (dev->dev_fixed_class) {
+
+		mod = dev->dev_module;
+
+		if (DEVICE_PROBE(dev) <= 0) {
+
+			if (device_allocate_softc(dev) == 0) {
+
+				if (DEVICE_ATTACH(dev) == 0) {
+					/* success */
+					dev->dev_attached = 1;
+					return (0);
+				}
+			}
+		}
+		device_detach(dev);
+
+		goto error;
+	}
+	/*
+         * Else find a module for our device, if any
+         */
+
+	TAILQ_FOREACH(mod, &module_head, entry) {
+		if (devclass_equal(mod->bus_name, bus_name_parent)) {
+			if (devclass_create(mod->devclass_pp)) {
+				continue;
+			}
+			if (devclass_add_device(mod, dev)) {
+				continue;
+			}
+			if (DEVICE_PROBE(dev) <= 0) {
+
+				if (device_allocate_softc(dev) == 0) {
+
+					if (DEVICE_ATTACH(dev) == 0) {
+						/* success */
+						dev->dev_attached = 1;
+						return (0);
+					}
+				}
+			}
+			/* else try next driver */
+
+			device_detach(dev);
+		}
+	}
+
+error:
+	return (ENODEV);
+}
+
+int
+device_detach(device_t dev)
+{
+	const struct module_data *mod = dev->dev_module;
+	int error;
+
+	if (dev->dev_attached) {
+
+		error = DEVICE_DETACH(dev);
+		if (error) {
+			return error;
+		}
+		dev->dev_attached = 0;
+	}
+	device_set_softc(dev, NULL);
+
+	if (dev->dev_fixed_class == 0)
+		devclass_delete_device(mod, dev);
+
+	return (0);
+}
+
+void
+device_set_softc(device_t dev, void *softc)
+{
+	if (dev->dev_softc_alloc) {
+		free(dev->dev_sc, M_DEVBUF);
+		dev->dev_sc = NULL;
+	}
+	dev->dev_sc = softc;
+	dev->dev_softc_alloc = 0;
+}
+
+void   *
+device_get_softc(device_t dev)
+{
+	if (dev == NULL)
+		return (NULL);
+
+	return (dev->dev_sc);
+}
+
+int
+device_is_attached(device_t dev)
+{
+	return (dev->dev_attached);
+}
+
+void
+device_set_desc(device_t dev, const char *desc)
+{
+	snprintf(dev->dev_desc, sizeof(dev->dev_desc), "%s", desc);
+}
+
+void
+device_set_desc_copy(device_t dev, const char *desc)
+{
+	device_set_desc(dev, desc);
+}
+
+void   *
+devclass_get_softc(devclass_t dc, int unit)
+{
+	return (device_get_softc(devclass_get_device(dc, unit)));
+}
+
+int
+devclass_get_maxunit(devclass_t dc)
+{
+	int max_unit = 0;
+
+	if (dc) {
+		max_unit = DEVCLASS_MAXUNIT;
+		while (max_unit--) {
+			if (dc->dev_list[max_unit]) {
+				break;
+			}
+		}
+		max_unit++;
+	}
+	return (max_unit);
+}
+
+device_t
+devclass_get_device(devclass_t dc, int unit)
+{
+	return (((unit < 0) || (unit >= DEVCLASS_MAXUNIT) || (dc == NULL)) ?
+	    NULL : dc->dev_list[unit]);
+}
+
+devclass_t
+devclass_find(const char *classname)
+{
+	const struct module_data *mod;
+
+	TAILQ_FOREACH(mod, &module_head, entry) {
+		if (devclass_equal(mod->mod_name, classname))
+			return (mod->devclass_pp[0]);
+	}
+	return (NULL);
+}
+
+void
+module_register(void *data)
+{
+	struct module_data *mdata = data;
+
+	TAILQ_INSERT_TAIL(&module_head, mdata, entry);
+}
+
+/*------------------------------------------------------------------------*
+ * System startup
+ *------------------------------------------------------------------------*/
+
+static void
+sysinit_run(const void **ppdata)
+{
+	const struct sysinit *psys;
+
+	while ((psys = *ppdata) != NULL) {
+		(psys->func) (psys->data);
+		ppdata++;
+	}
+}
+
+/*------------------------------------------------------------------------*
+ * USB process API
+ *------------------------------------------------------------------------*/
+
+static int usb_do_process(struct usb_process *);
+static int usb_proc_level = -1;
+static struct mtx usb_proc_mtx;
+
+void
+usb_idle(void)
+{
+	int old_level = usb_proc_level;
+	int old_giant = Giant.owned;
+	int worked;
+
+	device_run_interrupts(usb_pci_root);
+
+	do {
+		worked = 0;
+		Giant.owned = 0;
+
+		while (++usb_proc_level < USB_PROC_MAX)
+			worked |= usb_do_process(usb_process + usb_proc_level);
+
+		usb_proc_level = old_level;
+		Giant.owned = old_giant;
+
+	} while (worked);
+}
+
+void
+usb_init(void)
+{
+	sysinit_run(sysinit_data);
+}
+
+void
+usb_uninit(void)
+{
+	sysinit_run(sysuninit_data);
+}
+
+static void
+usb_process_init_sub(struct usb_process *up)
+{
+	TAILQ_INIT(&up->up_qhead);
+
+	cv_init(&up->up_cv, "-");
+	cv_init(&up->up_drain, "usbdrain");
+
+	up->up_mtx = &usb_proc_mtx;
+}
+
+static void
+usb_process_init(void *arg)
+{
+	uint8_t x;
+
+	mtx_init(&usb_proc_mtx, "usb-proc-mtx", NULL, MTX_DEF | MTX_RECURSE);
+
+	for (x = 0; x != USB_PROC_MAX; x++)
+		usb_process_init_sub(&usb_process[x]);
+
+}
+SYSINIT(usb_process_init, SI_SUB_LOCK, SI_ORDER_MIDDLE, usb_process_init, NULL);
+
+static int
+usb_do_process(struct usb_process *up)
+{
+	struct usb_proc_msg *pm;
+	int worked = 0;
+
+	mtx_lock(&usb_proc_mtx);
+
+repeat:
+	pm = TAILQ_FIRST(&up->up_qhead);
+
+	if (pm != NULL) {
+
+		worked = 1;
+
+		(pm->pm_callback) (pm);
+
+		if (pm == TAILQ_FIRST(&up->up_qhead)) {
+			/* nothing changed */
+			TAILQ_REMOVE(&up->up_qhead, pm, pm_qentry);
+			pm->pm_qentry.tqe_prev = NULL;
+		}
+		goto repeat;
+	}
+	mtx_unlock(&usb_proc_mtx);
+
+	return (worked);
+}
+
+int
+usb_proc_create(struct usb_process *up, struct mtx *p_mtx,
+    const char *pmesg, uint8_t prio)
+{
+#define	USB_PROC_OFFSET(a,b) \
+  ((int)(((long)&((struct usb_bus *)0)->a) - \
+	 ((long)&((struct usb_bus *)0)->b)))
+
+	/* figure out which process we are creating */
+	switch ((int)((long)up - (long)p_mtx)) {
+	case USB_PROC_OFFSET(giant_callback_proc, bus_mtx):
+		up->up_ptr = (void *)(usb_process + 2);
+		break;
+	case USB_PROC_OFFSET(non_giant_callback_proc, bus_mtx):
+		up->up_ptr = (void *)(usb_process + 2);
+		break;
+	case USB_PROC_OFFSET(explore_proc, bus_mtx):
+		up->up_ptr = (void *)(usb_process + 0);
+		break;
+	case USB_PROC_OFFSET(control_xfer_proc, bus_mtx):
+		up->up_ptr = (void *)(usb_process + 1);
+		break;
+	default:
+		up->up_ptr = (void *)(usb_process + 1);
+		break;
+	}
+	return (0);			/* success */
+}
+
+void
+usb_proc_free(struct usb_process *up)
+{
+	/* NOP */
+}
+
+void   *
+usb_proc_msignal(struct usb_process *up, void *_pm0, void *_pm1)
+{
+	struct usb_proc_msg *pm0 = _pm0;
+	struct usb_proc_msg *pm1 = _pm1;
+	struct usb_proc_msg *pm2;
+	usb_size_t d;
+	uint8_t t;
+
+	/* find the correct parent */
+	while (up->up_ptr != NULL)
+		up = (struct usb_process *)up->up_ptr;
+
+	t = 0;
+
+	if (pm0->pm_qentry.tqe_prev) {
+		t |= 1;
+	}
+	if (pm1->pm_qentry.tqe_prev) {
+		t |= 2;
+	}
+	if (t == 0) {
+		/*
+		 * No entries are queued. Queue "pm0" and use the existing
+		 * message number.
+		 */
+		pm2 = pm0;
+	} else if (t == 1) {
+		/* Check if we need to increment the message number. */
+		if (pm0->pm_num == up->up_msg_num) {
+			up->up_msg_num++;
+		}
+		pm2 = pm1;
+	} else if (t == 2) {
+		/* Check if we need to increment the message number. */
+		if (pm1->pm_num == up->up_msg_num) {
+			up->up_msg_num++;
+		}
+		pm2 = pm0;
+	} else if (t == 3) {
+		/*
+		 * Both entries are queued. Re-queue the entry closest to
+		 * the end.
+		 */
+		d = (pm1->pm_num - pm0->pm_num);
+
+		/* Check sign after subtraction */
+		if (d & 0x80000000) {
+			pm2 = pm0;
+		} else {
+			pm2 = pm1;
+		}
+
+		TAILQ_REMOVE(&up->up_qhead, pm2, pm_qentry);
+	} else {
+		pm2 = NULL;		/* panic - should not happen */
+	}
+
+	/* Put message last on queue */
+
+	pm2->pm_num = up->up_msg_num;
+	TAILQ_INSERT_TAIL(&up->up_qhead, pm2, pm_qentry);
+
+	return (pm2);
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_proc_is_gone
+ *
+ * Return values:
+ *    0: USB process is running
+ * Else: USB process is tearing down
+ *------------------------------------------------------------------------*/
+uint8_t
+usb_proc_is_gone(struct usb_process *up)
+{
+	return (0);
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_proc_mwait
+ *
+ * This function will return when the USB process message pointed to
+ * by "pm" is no longer on a queue. This function must be called
+ * having "usb_proc_mtx" locked.
+ *------------------------------------------------------------------------*/
+void
+usb_proc_mwait(struct usb_process *up, void *_pm0, void *_pm1)
+{
+	struct usb_proc_msg *pm0 = _pm0;
+	struct usb_proc_msg *pm1 = _pm1;
+
+	/* find the correct parent */
+	while (up->up_ptr != NULL)
+		up = (struct usb_process *)up->up_ptr;
+
+	/* Just remove the messages from the queue. */
+	if (pm0->pm_qentry.tqe_prev) {
+		TAILQ_REMOVE(&up->up_qhead, pm0, pm_qentry);
+		pm0->pm_qentry.tqe_prev = NULL;
+	}
+	if (pm1->pm_qentry.tqe_prev) {
+		TAILQ_REMOVE(&up->up_qhead, pm1, pm_qentry);
+		pm1->pm_qentry.tqe_prev = NULL;
+	}
+}
+
+/*------------------------------------------------------------------------*
+ * SYSTEM attach
+ *------------------------------------------------------------------------*/
+
+static device_method_t pci_methods[] = {
+	DEVMETHOD_END
+};
+
+static driver_t pci_driver = {
+	.name = "pci",
+	.methods = pci_methods,
+};
+
+static devclass_t pci_devclass;
+
+DRIVER_MODULE(pci, pci, pci_driver, pci_devclass, 0, 0);
+
+static const char *usb_pci_devices[] = {
+#ifdef USB_PROBE_LIST
+	USB_PROBE_LIST
+#endif
+};
+
+#define	USB_PCI_USB_MAX	(sizeof(usb_pci_devices) / sizeof(void *))
+
+static device_t usb_pci_dev[USB_PCI_USB_MAX];
+
+static void
+usb_pci_mod_load(void *arg)
+{
+	uint32_t x;
+
+	usb_pci_root = device_add_child(NULL, "pci", -1);
+	if (usb_pci_root == NULL)
+		return;
+
+	for (x = 0; x != USB_PCI_USB_MAX; x++) {
+		usb_pci_dev[x] = device_add_child(usb_pci_root, usb_pci_devices[x], -1);
+		if (usb_pci_dev[x] == NULL)
+			continue;
+		if (device_probe_and_attach(usb_pci_dev[x])) {
+			device_printf(usb_pci_dev[x],
+			    "WARNING: Probe and attach failed!\n");
+		}
+	}
+}
+SYSINIT(usb_pci_mod_load, SI_SUB_RUN_SCHEDULER, SI_ORDER_MIDDLE, usb_pci_mod_load, 0);
+
+static void
+usb_pci_mod_unload(void *arg)
+{
+	uint32_t x;
+
+	for (x = 0; x != USB_PCI_USB_MAX; x++) {
+		if (usb_pci_dev[x]) {
+			device_detach(usb_pci_dev[x]);
+			device_delete_child(usb_pci_root, usb_pci_dev[x]);
+		}
+	}
+	if (usb_pci_root)
+		device_delete_child(NULL, usb_pci_root);
+}
+SYSUNINIT(usb_pci_mod_unload, SI_SUB_RUN_SCHEDULER, SI_ORDER_MIDDLE, usb_pci_mod_unload, 0);
+
+/*------------------------------------------------------------------------*
+ * MALLOC API
+ *------------------------------------------------------------------------*/
+
+#define	USB_POOL_ALIGN 8
+
+static uint8_t usb_pool[USB_POOL_SIZE] __aligned(USB_POOL_ALIGN);
+static uint32_t usb_pool_rem = USB_POOL_SIZE;
+static uint32_t usb_pool_entries;
+
+struct malloc_hdr {
+	TAILQ_ENTRY(malloc_hdr) entry;
+	uint32_t size;
+} __aligned(USB_POOL_ALIGN);
+
+static TAILQ_HEAD(, malloc_hdr) malloc_head =
+	TAILQ_HEAD_INITIALIZER(malloc_head);
+
+void   *
+usb_malloc(unsigned long size)
+{
+	struct malloc_hdr *hdr;
+
+	size = (size + USB_POOL_ALIGN - 1) & ~(USB_POOL_ALIGN - 1);
+	size += sizeof(struct malloc_hdr);
+
+	TAILQ_FOREACH(hdr, &malloc_head, entry) {
+		if (hdr->size == size)
+			break;
+	}
+
+	if (hdr) {
+		printf("MALLOC: Entries = %d; Remainder = %d; Size = %d\n",
+		    (int)usb_pool_entries, (int)usb_pool_rem, (int)size);
+
+		TAILQ_REMOVE(&malloc_head, hdr, entry);
+		memset(hdr + 1, 0, hdr->size - sizeof(*hdr));
+		return (hdr + 1);
+	}
+	if (usb_pool_rem >= size) {
+		hdr = (void *)(usb_pool + USB_POOL_SIZE - usb_pool_rem);
+		hdr->size = size;
+
+		usb_pool_rem -= size;
+		usb_pool_entries++;
+
+		printf("MALLOC: Entries = %d; Remainder = %d; Size = %d\n",
+		    (int)usb_pool_entries, (int)usb_pool_rem, (int)size);
+
+		memset(hdr + 1, 0, hdr->size - sizeof(*hdr));
+		return (hdr + 1);
+	}
+	return (NULL);
+}
+
+void
+usb_free(void *arg)
+{
+	struct malloc_hdr *hdr;
+
+	if (arg == NULL)
+		return;
+
+	hdr = arg;
+	hdr--;
+
+	TAILQ_INSERT_TAIL(&malloc_head, hdr, entry);
+}
+
+char   *
+usb_strdup(const char *str)
+{
+	char *tmp;
+	int len;
+
+	len = 1 + strlen(str);
+
+	tmp = usb_malloc(len);
+	if (tmp == NULL)
+		return (NULL);
+
+	memcpy(tmp, str, len);
+	return (tmp);
+}
diff --git a/sys/boot/usb/bsd_kernel.h b/sys/boot/usb/bsd_kernel.h
new file mode 100644
index 0000000..4c94721
--- /dev/null
+++ b/sys/boot/usb/bsd_kernel.h
@@ -0,0 +1,458 @@
+/* $FreeBSD$ */
+/*-
+ * Copyright (c) 2011 Hans Petter Selasky. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _BSD_KERNEL_H_
+#define	_BSD_KERNEL_H_
+
+#define	_KERNEL
+#define	__FreeBSD_version 1000000
+
+#include <sys/cdefs.h>
+#include <sys/queue.h>
+#include <sys/errno.h>
+
+#define	isalpha(x) (((x) >= 'a' && (x) <= 'z') || ((x) >= 'A' && (x) <= 'Z'))
+#define	isdigit(x) ((x) >= '0' && (x) <= '9')
+#define	panic(...) do { printf("USB PANIC: " __VA_ARGS__); while (1) ; } while (0)
+#define	M_USB 0
+#define	M_USBDEV 0
+#define	USB_PROC_MAX 3
+#define	SYSCTL_DECL(...)
+#define	SYSCTL_NODE(name,...) struct { } name __used
+#define	SYSCTL_INT(...)
+#define	TUNABLE_INT(...)
+#define	MALLOC_DECLARE(...)
+#define	MALLOC_DEFINE(...)
+#define	EVENTHANDLER_DECLARE(...)
+#define	EVENTHANDLER_INVOKE(...)
+#define	KASSERT(...)
+#define	SCHEDULER_STOPPED(x) (0)
+#define	PI_SWI(...) (0)
+#define	UNIQ_NAME(x) x
+#define	UNIQ_NAME_STR(x) #x
+#define	DEVCLASS_MAXUNIT 32
+#define	MOD_LOAD 1
+#define	MOD_UNLOAD 2
+#define	DEVMETHOD(what,func) { #what, (void *)&func }
+#define	DEVMETHOD_END {0,0}
+#define	DRIVER_MODULE(name, busname, driver, devclass, evh, arg)	\
+  static struct module_data bsd_##name##_##busname##_driver_mod = {	\
+	evh, arg, #busname, #name, #busname "/" #name,			\
+	&driver, &devclass, { 0, 0 } };					\
+SYSINIT(bsd_##name##_##busname##_driver_mod, SI_SUB_DRIVERS,		\
+  SI_ORDER_MIDDLE, module_register,					\
+  &bsd_##name##_##busname##_driver_mod)
+#define	SYSINIT(uniq, subs, order, _func, _data)	\
+const struct sysinit UNIQ_NAME(sysinit_##uniq) = {	\
+	.func = (_func),				\
+	.data = __DECONST(void *, _data)		\
+};							\
+SYSINIT_ENTRY(uniq##_entry, "sysinit", (subs),		\
+    (order), "const struct sysinit",			\
+    UNIQ_NAME_STR(sysinit_##uniq), "SYSINIT")
+
+#define	SYSUNINIT(uniq, subs, order, _func, _data)	\
+const struct sysinit UNIQ_NAME(sysuninit_##uniq) = {	\
+	.func = (_func),				\
+	.data = __DECONST(void *, _data)		\
+};							\
+SYSINIT_ENTRY(uniq##_entry, "sysuninit", (subs),	\
+    (order), "const struct sysuninit",			\
+    UNIQ_NAME_STR(sysuninit_##uniq), "SYSUNINIT")
+#define	MODULE_DEPEND(...)
+#define	MODULE_VERSION(...)
+#define	NULL ((void *)0)
+#define	BUS_SPACE_BARRIER_READ 0x01
+#define	BUS_SPACE_BARRIER_WRITE 0x02
+#define	hz 1000
+#define	PAGE_SIZE 4096
+#define	MIN(a,b) (((a) < (b)) ? (a) : (b))
+#define	MAX(a,b) (((a) > (b)) ? (a) : (b))
+#define	MTX_DEF 0
+#define	MTX_RECURSE 0
+#define	SX_DUPOK 0
+#define	SX_NOWITNESS 0
+#define	WITNESS_WARN(...)
+#define	cold 0
+#define	BUS_PROBE_GENERIC 0
+#define	CALLOUT_RETURNUNLOCKED 0x1
+#define	va_list __builtin_va_list
+#define	va_size(type) __builtin_va_size(type)
+#define	va_start(ap, last) __builtin_va_start(ap, last)
+#define	va_end(ap) __builtin_va_end(ap)
+#define	va_arg(ap, type) __builtin_va_arg((ap), type)
+#define	DEVICE_ATTACH(dev, ...) \
+  (((device_attach_t *)(device_get_method(dev, "device_attach")))(dev,## __VA_ARGS__))
+#define	DEVICE_DETACH(dev, ...) \
+  (((device_detach_t *)(device_get_method(dev, "device_detach")))(dev,## __VA_ARGS__))
+#define	DEVICE_PROBE(dev, ...) \
+  (((device_probe_t *)(device_get_method(dev, "device_probe")))(dev,## __VA_ARGS__))
+#define	DEVICE_RESUME(dev, ...) \
+  (((device_resume_t *)(device_get_method(dev, "device_resume")))(dev,## __VA_ARGS__))
+#define	DEVICE_SHUTDOWN(dev, ...) \
+  (((device_shutdown_t *)(device_get_method(dev, "device_shutdown")))(dev,## __VA_ARGS__))
+#define	DEVICE_SUSPEND(dev, ...) \
+  (((device_suspend_t *)(device_get_method(dev, "device_suspend")))(dev,## __VA_ARGS__))
+#define	USB_HANDLE_REQUEST(dev, ...) \
+  (((usb_handle_request_t *)(device_get_method(dev, "usb_handle_request")))(dev,## __VA_ARGS__))
+#define	USB_TAKE_CONTROLLER(dev, ...) \
+  (((usb_take_controller_t *)(device_get_method(dev, "usb_take_controller")))(dev,## __VA_ARGS__))
+
+enum {
+	SI_SUB_DUMMY = 0x0000000,
+	SI_SUB_LOCK = 0x1B00000,
+	SI_SUB_KLD = 0x2000000,
+	SI_SUB_DRIVERS = 0x3100000,
+	SI_SUB_PSEUDO = 0x7000000,
+	SI_SUB_KICK_SCHEDULER = 0xa000000,
+	SI_SUB_RUN_SCHEDULER = 0xfffffff
+};
+
+enum {
+	SI_ORDER_FIRST = 0x0000000,
+	SI_ORDER_SECOND = 0x0000001,
+	SI_ORDER_THIRD = 0x0000002,
+	SI_ORDER_FOURTH = 0x0000003,
+	SI_ORDER_MIDDLE = 0x1000000,
+	SI_ORDER_ANY = 0xfffffff	/* last */
+};
+
+struct uio;
+struct thread;
+struct malloc_type;
+struct usb_process;
+
+typedef unsigned char uint8_t;
+typedef signed char int8_t;
+
+typedef unsigned short uint16_t;
+typedef signed short int16_t;
+
+typedef unsigned int uint32_t;
+typedef signed int int32_t;
+
+typedef unsigned long long uint64_t;
+typedef signed long long int64_t;
+
+typedef unsigned long bus_addr_t;
+typedef unsigned long bus_size_t;
+
+typedef unsigned long size_t;
+typedef unsigned long u_long;
+
+typedef void *bus_dmamap_t;
+typedef void *bus_dma_tag_t;
+
+typedef void *bus_space_tag_t;
+typedef uint8_t *bus_space_handle_t;
+
+typedef uint16_t uid_t;
+typedef uint16_t gid_t;
+typedef uint16_t mode_t;
+
+typedef uint8_t *caddr_t;
+typedef unsigned long __uintptr_t;
+typedef unsigned long uintptr_t;
+
+/* SYSINIT API */
+
+#include <sysinit.h>
+
+struct sysinit {
+	void    (*func) (void *arg);
+	void   *data;
+};
+
+/* MUTEX API */
+
+struct mtx {
+	int	owned;
+	struct mtx *parent;
+};
+
+#define	mtx_assert(...) do { } while (0)
+void	mtx_init(struct mtx *, const char *, const char *, int);
+void	mtx_lock(struct mtx *);
+void	mtx_unlock(struct mtx *);
+int	mtx_owned(struct mtx *);
+void	mtx_destroy(struct mtx *);
+
+extern struct mtx Giant;
+
+/* SX API */
+
+struct sx {
+	int	owned;
+};
+
+#define	sx_assert(...) do { } while (0)
+#define	sx_init(...) sx_init_flags(__VA_ARGS__, 0)
+void	sx_init_flags(struct sx *, const char *, int);
+void	sx_destroy(struct sx *);
+void	sx_xlock(struct sx *);
+void	sx_xunlock(struct sx *);
+int	sx_xlocked(struct sx *);
+
+/* CONDVAR API */
+
+struct cv {
+	int	sleeping;
+};
+
+void	cv_init(struct cv *, const char *desc);
+void	cv_destroy(struct cv *);
+void	cv_wait(struct cv *, struct mtx *);
+int	cv_timedwait(struct cv *, struct mtx *, int);
+void	cv_signal(struct cv *);
+void	cv_broadcast(struct cv *);
+
+/* CALLOUT API */
+
+typedef void callout_fn_t (void *);
+
+extern volatile int ticks;
+
+struct callout {
+	LIST_ENTRY(callout) entry;
+	callout_fn_t *func;
+	void   *arg;
+	struct mtx *mtx;
+	int	flags;
+	int	timeout;
+};
+
+void	callout_init_mtx(struct callout *, struct mtx *, int);
+void	callout_reset(struct callout *, int, callout_fn_t *, void *);
+void	callout_stop(struct callout *);
+void	callout_drain(struct callout *);
+int	callout_pending(struct callout *);
+void	callout_process(int timeout);
+
+/* DEVICE API */
+
+struct driver;
+struct devclass;
+struct device;
+struct module;
+struct module_data;
+
+typedef struct driver driver_t;
+typedef struct devclass *devclass_t;
+typedef struct device *device_t;
+typedef void (intr_fn_t)(void *arg);
+
+typedef int device_attach_t (device_t dev);
+typedef int device_detach_t (device_t dev);
+typedef int device_resume_t (device_t dev);
+typedef int device_shutdown_t (device_t dev);
+typedef int device_probe_t (device_t dev);
+typedef int device_suspend_t (device_t dev);
+
+typedef int bus_child_location_str_t (device_t parent, device_t child, char *buf, size_t buflen);
+typedef int bus_child_pnpinfo_str_t (device_t parent, device_t child, char *buf, size_t buflen);
+typedef void bus_driver_added_t (device_t dev, driver_t *driver);
+
+struct device_method {
+	const char *desc;
+	void   *const func;
+};
+
+typedef struct device_method device_method_t;
+
+struct device {
+	TAILQ_HEAD(device_list, device) dev_children;
+	TAILQ_ENTRY(device) dev_link;
+
+	struct device *dev_parent;
+	const struct module_data *dev_module;
+	void   *dev_sc;
+	void   *dev_aux;
+	intr_fn_t *dev_irq_fn;
+	void   *dev_irq_arg;
+
+	uint16_t dev_unit;
+
+	char	dev_nameunit[64];
+	char	dev_desc[64];
+
+	uint8_t	dev_res_alloc:1;
+	uint8_t	dev_quiet:1;
+	uint8_t	dev_softc_set:1;
+	uint8_t	dev_softc_alloc:1;
+	uint8_t	dev_attached:1;
+	uint8_t	dev_fixed_class:1;
+	uint8_t	dev_unit_manual:1;
+};
+
+struct devclass {
+	device_t dev_list[DEVCLASS_MAXUNIT];
+};
+
+struct driver {
+	const char *name;
+	const struct device_method *methods;
+	uint32_t size;
+};
+
+struct module_data {
+	int     (*callback) (struct module *, int, void *arg);
+	void   *arg;
+	const char *bus_name;
+	const char *mod_name;
+	const char *long_name;
+	const struct driver *driver;
+	struct devclass **devclass_pp;
+	TAILQ_ENTRY(module_data) entry;
+};
+
+device_t device_get_parent(device_t dev);
+void   *device_get_method(device_t dev, const char *what);
+const char *device_get_name(device_t dev);
+const char *device_get_nameunit(device_t dev);
+
+#define	device_printf(dev, fmt,...) \
+	printf("%s: " fmt, device_get_nameunit(dev),## __VA_ARGS__)
+device_t device_add_child(device_t dev, const char *name, int unit);
+void	device_quiet(device_t dev);
+void	device_set_interrupt(device_t dev, intr_fn_t *fn, void *arg);
+void	device_run_interrupts(device_t parent);
+void	device_set_ivars(device_t dev, void *ivars);
+void   *device_get_ivars(device_t dev);
+const char *device_get_desc(device_t dev);
+int	device_probe_and_attach(device_t dev);
+int	device_detach(device_t dev);
+void   *device_get_softc(device_t dev);
+void	device_set_softc(device_t dev, void *softc);
+int	device_delete_child(device_t dev, device_t child);
+int	device_delete_children(device_t dev);
+int	device_is_attached(device_t dev);
+void	device_set_desc(device_t dev, const char *desc);
+void	device_set_desc_copy(device_t dev, const char *desc);
+int	device_get_unit(device_t dev);
+void   *devclass_get_softc(devclass_t dc, int unit);
+int	devclass_get_maxunit(devclass_t dc);
+device_t devclass_get_device(devclass_t dc, int unit);
+devclass_t devclass_find(const char *classname);
+
+#define	bus_get_dma_tag(...) (NULL)
+int	bus_generic_detach(device_t dev);
+int	bus_generic_resume(device_t dev);
+int	bus_generic_shutdown(device_t dev);
+int	bus_generic_suspend(device_t dev);
+int	bus_generic_print_child(device_t dev, device_t child);
+void	bus_generic_driver_added(device_t dev, driver_t *driver);
+
+/* BUS SPACE API */
+
+void	bus_space_write_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset, uint8_t data);
+void	bus_space_write_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset, uint16_t data);
+void	bus_space_write_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset, uint32_t data);
+
+uint8_t	bus_space_read_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset);
+uint16_t bus_space_read_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset);
+uint32_t bus_space_read_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset);
+
+void	bus_space_read_multi_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset, uint8_t *datap, bus_size_t count);
+void	bus_space_read_multi_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset, uint16_t *datap, bus_size_t count);
+void	bus_space_read_multi_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset, uint32_t *datap, bus_size_t count);
+
+void	bus_space_write_multi_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset, uint8_t *datap, bus_size_t count);
+void	bus_space_write_multi_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset, uint16_t *datap, bus_size_t count);
+void	bus_space_write_multi_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset, uint32_t *datap, bus_size_t count);
+
+void	bus_space_read_region_1(bus_space_tag_t space, bus_space_handle_t handle, bus_size_t offset, uint8_t *datap, bus_size_t count);
+void	bus_space_write_region_1(bus_space_tag_t space, bus_space_handle_t handle, bus_size_t offset, uint8_t *datap, bus_size_t count);
+void	bus_space_read_region_4(bus_space_tag_t space, bus_space_handle_t handle, bus_size_t offset, uint32_t *datap, bus_size_t count);
+void	bus_space_write_region_4(bus_space_tag_t space, bus_space_handle_t handle, bus_size_t offset, uint32_t *datap, bus_size_t count);
+
+void	bus_space_barrier(bus_space_tag_t space, bus_space_handle_t handle, bus_size_t offset, bus_size_t length, int flags);
+
+void	module_register(void *);
+
+/* LIB-C */
+
+void   *memset(void *, int, size_t len);
+void   *memcpy(void *, const void *, size_t len);
+int	printf(const char *,...) __printflike(1, 2);
+int	snprintf(char *restrict str, size_t size, const char *restrict format,...) __printflike(3, 4);
+size_t	strlen(const char *s);
+
+/* MALLOC API */
+
+#define	malloc(s,x,f) usb_malloc(s)
+void   *usb_malloc(size_t);
+
+#define	free(p,x) usb_free(p)
+void	usb_free(void *);
+
+#define	strdup(p,x) usb_strdup(p)
+char   *usb_strdup(const char *str);
+
+/* ENDIANNESS */
+
+/* Assume little endian */
+
+#define	htole64(x) ((uint64_t)(x))
+#define	le64toh(x) ((uint64_t)(x))
+
+#define	htole32(x) ((uint32_t)(x))
+#define	le32toh(x) ((uint32_t)(x))
+
+#define	htole16(x) ((uint16_t)(x))
+#define	le16toh(x) ((uint16_t)(x))
+
+#define	be32toh(x) ((uint32_t)(x))
+#define	htobe32(x) ((uint32_t)(x))
+
+/* USB */
+
+typedef int usb_handle_request_t (device_t dev, const void *req, void **pptr, uint16_t *plen, uint16_t offset, uint8_t *pstate);
+typedef int usb_take_controller_t (device_t dev);
+
+void	usb_idle(void);
+void	usb_init(void);
+void	usb_uninit(void);
+
+/* set some defaults */
+
+#ifndef USB_POOL_SIZE
+#define	USB_POOL_SIZE (1024*1024)	/* 1 MByte */
+#endif
+
+int	pause(const char *, int);
+void	DELAY(unsigned int);
+
+/* OTHER */
+
+struct selinfo {
+};
+
+/* SYSTEM STARTUP API */
+
+extern const void *sysinit_data[];
+extern const void *sysuninit_data[];
+
+#endif					/* _BSD_KERNEL_H_ */
diff --git a/sys/boot/usb/bsd_usbloader_test.c b/sys/boot/usb/bsd_usbloader_test.c
new file mode 100644
index 0000000..929e2e9
--- /dev/null
+++ b/sys/boot/usb/bsd_usbloader_test.c
@@ -0,0 +1,80 @@
+/* $FreeBSD$ */
+/*-
+ * Copyright (c) 2013 Hans Petter Selasky. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <time.h>
+
+extern int usleep(int);
+extern void callout_process(int);
+extern void usb_idle(void);
+extern void usb_init(void);
+extern void usb_uninit(void);
+
+#define	hz 1000
+
+void
+DELAY(unsigned int delay)
+{
+	usleep(delay);
+}
+
+int
+pause(const char *what, int timeout)
+{
+	if (timeout == 0)
+		timeout = 1;
+
+	usleep((1000000 / hz) * timeout);
+
+	return (0);
+}
+
+int
+main(int argc, char **argv)
+{
+	uint32_t time;
+
+	usb_init();
+
+	time = 0;
+
+	while (1) {
+
+		usb_idle();
+
+		usleep(1000);
+
+		if (++time >= (1000 / hz)) {
+			time = 0;
+			callout_process(1);
+		}
+	}
+
+	usb_uninit();
+
+	return (0);
+}
diff --git a/sys/boot/usb/tools/sysinit.c b/sys/boot/usb/tools/sysinit.c
new file mode 100644
index 0000000..b968fe0
--- /dev/null
+++ b/sys/boot/usb/tools/sysinit.c
@@ -0,0 +1,331 @@
+/* $FreeBSD$ */
+/*-
+ * Copyright (c) 2013 Hans Petter Selasky. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * This utility sorts sysinit structure entries in binary format and
+ * prints out the result in C-format.
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <err.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <sysexits.h>
+#include "sysinit.h"
+
+static int opt_R;
+static const char *input_f;
+static const char *output_f;
+static const char *struct_name;
+static const char *keyword;
+static struct sysinit_data **start;
+static struct sysinit_data **stop;
+
+static int input_file = -1;
+static int output_file = -1;
+
+static uint8_t *input_ptr;
+static uint32_t input_len;
+
+static uint32_t endian32;
+
+static char scratch_buf[4096];
+
+static int success;
+
+static void do_sysinit(void);
+
+/* the following function converts the numbers into host endian format */
+
+static uint32_t
+read32(uint32_t val)
+{
+	uint32_t temp;
+	uint32_t endian;
+
+	endian = endian32;
+	temp = 0;
+
+	while (val) {
+		temp |= (val & 0xF) << ((endian & 0xF) * 4);
+		endian >>= 4;
+		val >>= 4;
+	}
+	return (temp);
+}
+
+static void
+do_write(int fd, const char *buf)
+{
+	int len = strlen(buf);
+
+	if (write(fd, buf, len) != len)
+		err(EX_SOFTWARE, "Could not write to output file");
+}
+
+static void *
+do_malloc(int size)
+{
+	void *ptr;
+
+	ptr = malloc(size);
+	if (ptr == NULL)
+		errx(EX_SOFTWARE, "Could not allocate memory");
+	return (ptr);
+}
+
+static void
+usage(void)
+{
+	errx(EX_USAGE, "sysinit -i sysinit.bin -o sysinit_data.c \\\n"
+	    "\t" "-k sysinit -s sysinit_data [ -R (reverse)]");
+}
+
+static void
+cleanup(void)
+{
+	if (output_file >= 0)
+		close(output_file);
+	if (input_file >= 0)
+		close(input_file);
+	if (success == 0) {
+		if (output_f)
+			unlink(output_f);
+	}
+}
+
+static int
+compare(const void *_pa, const void *_pb)
+{
+	const struct sysinit_data * const *pa = _pa;
+	const struct sysinit_data * const *pb = _pb;
+
+	if ((*pa)->dw_msb_value > (*pb)->dw_msb_value)
+		return (1);
+
+	if ((*pa)->dw_msb_value < (*pb)->dw_msb_value)
+		return (-1);
+
+	if ((*pa)->dw_lsb_value > (*pb)->dw_lsb_value)
+		return (1);
+
+	if ((*pa)->dw_lsb_value < (*pb)->dw_lsb_value)
+		return (-1);
+
+	return (0);	/* equal */
+}
+
+static int
+compare_R(const void *_pa, const void *_pb)
+{
+	const struct sysinit_data * const *pa = _pa;
+	const struct sysinit_data * const *pb = _pb;
+
+	if ((*pa)->dw_msb_value > (*pb)->dw_msb_value)
+		return (-1);
+
+	if ((*pa)->dw_msb_value < (*pb)->dw_msb_value)
+		return (1);
+
+	if ((*pa)->dw_lsb_value > (*pb)->dw_lsb_value)
+		return (-1);
+
+	if ((*pa)->dw_lsb_value < (*pb)->dw_lsb_value)
+		return (1);
+
+	return (0);	/* equal */
+}
+
+int
+main(int argc, char **argv)
+{
+	struct sysinit_data **sipp;
+	int c;
+	int entries;
+	off_t off;
+
+	while ((c = getopt(argc, argv, "k:s:i:o:Rh")) != -1) {
+		switch (c) {
+		case 'i':
+			input_f = optarg;
+			break;
+		case 'o':
+			output_f = optarg;
+			break;
+		case 'R':
+			opt_R = 1;
+			break;
+		case 'k':
+			keyword = optarg;
+			break;
+		case 's':
+			struct_name = optarg;
+			break;
+		default:
+			usage();
+		}
+	}
+
+	if (input_f == NULL || output_f == NULL ||
+	    struct_name == NULL || keyword == NULL)
+		usage();
+
+	atexit(&cleanup);
+
+	cleanup();
+
+	input_file = open(input_f, O_RDONLY);
+	if (input_file < 0)
+		err(EX_SOFTWARE, "Could not open input file: %s", input_f);
+
+	output_file = open(output_f, O_TRUNC | O_CREAT | O_RDWR, 0600);
+	if (output_file < 0)
+		err(EX_SOFTWARE, "Could not open output file: %s", output_f);
+
+	off = lseek(input_file, 0, SEEK_END);
+
+	input_ptr = do_malloc(off);
+	input_len = off;
+
+	if (input_len % (uint32_t)sizeof(struct sysinit_data)) {
+		errx(EX_SOFTWARE, "Input file size is not divisible by %u",
+		    (unsigned int)sizeof(struct sysinit_data));
+	}
+	off = lseek(input_file, 0, SEEK_SET);
+	if (off < 0)
+		err(EX_SOFTWARE, "Could not seek to start of input file");
+
+	if (read(input_file, input_ptr, input_len) != input_len)
+		err(EX_SOFTWARE, "Could not read input file");
+
+	entries = input_len / (uint32_t)sizeof(struct sysinit_data);
+
+	start = do_malloc(sizeof(void *) * entries);
+	stop = start + entries;
+
+	for (c = 0; c != entries; c++)
+		start[c] = &((struct sysinit_data *)input_ptr)[c];
+
+	if (start != stop)
+		endian32 = (*start)->dw_endian32;
+
+	/* switch all fields to host endian order */
+	for (sipp = start; sipp < stop; sipp++) {
+		(*sipp)->dw_lsb_value = read32((*sipp)->dw_lsb_value);
+		(*sipp)->dw_msb_value = read32((*sipp)->dw_msb_value);
+		(*sipp)->dw_file_line = read32((*sipp)->dw_file_line);
+	}
+
+	if (opt_R == 0) {
+		/* sort entries, rising numerical order */
+		qsort(start, entries, sizeof(void *), &compare);
+	} else {
+		/* sort entries, falling numerical order */
+		qsort(start, entries, sizeof(void *), &compare_R);
+	}
+
+	/* safe all strings */
+	for (sipp = start; sipp < stop; sipp++) {
+		(*sipp)->b_keyword_name[sizeof((*sipp)->b_keyword_name) - 1] = 0;
+		(*sipp)->b_global_type[sizeof((*sipp)->b_global_type) - 1] = 0;
+		(*sipp)->b_global_name[sizeof((*sipp)->b_global_name) - 1] = 0;
+		(*sipp)->b_file_name[sizeof((*sipp)->b_file_name) - 1] = 0;
+		(*sipp)->b_debug_info[sizeof((*sipp)->b_debug_info) - 1] = 0;
+	}
+
+	if (strcmp(keyword, "sysinit") == 0)
+		do_sysinit();
+	else if (strcmp(keyword, "sysuninit") == 0)
+		do_sysinit();
+	else
+		errx(EX_USAGE, "Unknown keyword '%s'", keyword);
+
+	success = 1;
+
+	return (0);
+}
+
+static void
+do_sysinit(void)
+{
+	struct sysinit_data **sipp;
+	int c;
+
+	snprintf(scratch_buf, sizeof(scratch_buf),
+	    "/*\n"
+	    " * This file was automatically generated.\n"
+	    " * Please do not edit.\n"
+	    " */\n\n");
+
+	/* write out externals */
+	for (c = 0, sipp = start; sipp < stop; c++, sipp++) {
+		if (strcmp((const char *)(*sipp)->b_keyword_name, keyword))
+			continue;
+		if ((*sipp)->dw_msb_value == 0)
+			continue;
+
+		snprintf(scratch_buf, sizeof(scratch_buf),
+		    "/* #%04u: %s entry at %s:%u */\n",
+		    c, (*sipp)->b_debug_info, (*sipp)->b_file_name,
+		    (unsigned int)(*sipp)->dw_file_line);
+
+		do_write(output_file, scratch_buf);
+
+		snprintf(scratch_buf, sizeof(scratch_buf),
+		    "extern %s %s;\n\n", (*sipp)->b_global_type,
+		    (*sipp)->b_global_name);
+
+		do_write(output_file, scratch_buf);
+	}
+
+	snprintf(scratch_buf, sizeof(scratch_buf),
+	    "const void *%s[] = {\n", struct_name);
+
+	do_write(output_file, scratch_buf);
+
+	/* write out actual table */
+	for (c = 0, sipp = start; sipp < stop; c++, sipp++) {
+		if (strcmp((const char *)(*sipp)->b_keyword_name, keyword))
+			continue;
+		if ((*sipp)->dw_msb_value == 0)
+			continue;
+
+		snprintf(scratch_buf, sizeof(scratch_buf),
+		    "\t&%s, /* #%04u */\n",
+		    (*sipp)->b_global_name, (unsigned int)c);
+
+		do_write(output_file, scratch_buf);
+	}
+
+	snprintf(scratch_buf, sizeof(scratch_buf),
+	    "\t(const void *)0\n"
+	    "};\n");
+
+	do_write(output_file, scratch_buf);
+}
diff --git a/sys/boot/usb/tools/sysinit.h b/sys/boot/usb/tools/sysinit.h
new file mode 100644
index 0000000..a7a450e
--- /dev/null
+++ b/sys/boot/usb/tools/sysinit.h
@@ -0,0 +1,57 @@
+/* $FreeBSD$ */
+/*-
+ * Copyright (c) 2013 Hans Petter Selasky. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _SYSINIT_H_
+#define	_SYSINIT_H_
+
+struct sysinit_data {
+	uint8_t	b_keyword_name[64];
+	uint8_t	b_debug_info[128];
+	uint8_t	b_global_type[128];
+	uint8_t	b_global_name[128];
+	uint8_t	b_file_name[256];
+	uint32_t dw_endian32;
+	uint32_t dw_msb_value;		/* must be non-zero, else entry is skipped */
+	uint32_t dw_lsb_value;
+	uint32_t dw_file_line;
+}	__attribute__((__packed__));
+
+#define	SYSINIT_ENTRY(uniq, keyword, msb, lsb, g_type, g_name, debug)	\
+	static const struct sysinit_data sysinit_##uniq			\
+	__attribute__((__section__(".debug.sysinit"),			\
+		__used__, __aligned__(1))) = {				\
+	.b_keyword_name = { keyword },					\
+	.b_debug_info = { debug },					\
+	.b_global_type = { g_type },					\
+	.b_global_name = { g_name },					\
+	.b_file_name = { __FILE__ },					\
+	.dw_endian32 = 0x76543210U,					\
+	.dw_msb_value = (msb),						\
+	.dw_lsb_value = (lsb),						\
+	.dw_file_line = __LINE__					\
+}
+
+#endif				/* _SYSINIT_H_ */
diff --git a/sys/boot/usb/usb_busdma_loader.c b/sys/boot/usb/usb_busdma_loader.c
new file mode 100644
index 0000000..c7d1ecb
--- /dev/null
+++ b/sys/boot/usb/usb_busdma_loader.c
@@ -0,0 +1,619 @@
+/* $FreeBSD$ */
+/*-
+ * Copyright (c) 2013 Hans Petter Selasky. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <bsd_global.h>
+
+#if USB_HAVE_BUSDMA
+static void	usb_pc_common_mem_cb(struct usb_page_cache *pc,
+		    void *vaddr, uint32_t length);
+#endif
+
+/*------------------------------------------------------------------------*
+ *  usbd_get_page - lookup DMA-able memory for the given offset
+ *
+ * NOTE: Only call this function when the "page_cache" structure has
+ * been properly initialized !
+ *------------------------------------------------------------------------*/
+void
+usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
+    struct usb_page_search *res)
+{
+#if USB_HAVE_BUSDMA
+	struct usb_page *page;
+
+	if (pc->page_start) {
+
+		/* Case 1 - something has been loaded into DMA */
+
+		if (pc->buffer) {
+
+			/* Case 1a - Kernel Virtual Address */
+
+			res->buffer = USB_ADD_BYTES(pc->buffer, offset);
+		}
+		offset += pc->page_offset_buf;
+
+		/* compute destination page */
+
+		page = pc->page_start;
+
+		if (pc->ismultiseg) {
+
+			page += (offset / USB_PAGE_SIZE);
+
+			offset %= USB_PAGE_SIZE;
+
+			res->length = USB_PAGE_SIZE - offset;
+			res->physaddr = page->physaddr + offset;
+		} else {
+			res->length = (usb_size_t)-1;
+			res->physaddr = page->physaddr + offset;
+		}
+		if (!pc->buffer) {
+
+			/* Case 1b - Non Kernel Virtual Address */
+
+			res->buffer = USB_ADD_BYTES(page->buffer, offset);
+		}
+		return;
+	}
+#endif
+	/* Case 2 - Plain PIO */
+
+	res->buffer = USB_ADD_BYTES(pc->buffer, offset);
+	res->length = (usb_size_t)-1;
+#if USB_HAVE_BUSDMA
+	res->physaddr = 0;
+#endif
+}
+
+/*------------------------------------------------------------------------*
+ *  usbd_copy_in - copy directly to DMA-able memory
+ *------------------------------------------------------------------------*/
+void
+usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
+    const void *ptr, usb_frlength_t len)
+{
+	struct usb_page_search buf_res;
+
+	while (len != 0) {
+
+		usbd_get_page(cache, offset, &buf_res);
+
+		if (buf_res.length > len) {
+			buf_res.length = len;
+		}
+		memcpy(buf_res.buffer, ptr, buf_res.length);
+
+		offset += buf_res.length;
+		len -= buf_res.length;
+		ptr = USB_ADD_BYTES(ptr, buf_res.length);
+	}
+}
+
+/*------------------------------------------------------------------------*
+ *  usbd_copy_out - copy directly from DMA-able memory
+ *------------------------------------------------------------------------*/
+void
+usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
+    void *ptr, usb_frlength_t len)
+{
+	struct usb_page_search res;
+
+	while (len != 0) {
+
+		usbd_get_page(cache, offset, &res);
+
+		if (res.length > len) {
+			res.length = len;
+		}
+		memcpy(ptr, res.buffer, res.length);
+
+		offset += res.length;
+		len -= res.length;
+		ptr = USB_ADD_BYTES(ptr, res.length);
+	}
+}
+
+/*------------------------------------------------------------------------*
+ *  usbd_frame_zero - zero DMA-able memory
+ *------------------------------------------------------------------------*/
+void
+usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset,
+    usb_frlength_t len)
+{
+	struct usb_page_search res;
+
+	while (len != 0) {
+
+		usbd_get_page(cache, offset, &res);
+
+		if (res.length > len) {
+			res.length = len;
+		}
+		memset(res.buffer, 0, res.length);
+
+		offset += res.length;
+		len -= res.length;
+	}
+}
+
+#if USB_HAVE_BUSDMA
+
+/*------------------------------------------------------------------------*
+ *	usb_pc_common_mem_cb - BUS-DMA callback function
+ *------------------------------------------------------------------------*/
+static void
+usb_pc_common_mem_cb(struct usb_page_cache *pc,
+    void *vaddr, uint32_t length)
+{
+	struct usb_page *pg;
+	usb_size_t rem;
+	bus_size_t off;
+	bus_addr_t phys = (uintptr_t)vaddr;	/* XXX */
+	uint32_t nseg;
+
+	if (length == 0)
+		nseg = 1;
+	else
+		nseg = ((length + USB_PAGE_SIZE - 1) / USB_PAGE_SIZE);
+
+	pg = pc->page_start;
+	pg->physaddr = phys & ~(USB_PAGE_SIZE - 1);
+	rem = phys & (USB_PAGE_SIZE - 1);
+	pc->page_offset_buf = rem;
+	pc->page_offset_end += rem;
+	length += rem;
+
+	for (off = USB_PAGE_SIZE; off < length; off += USB_PAGE_SIZE) {
+		pg++;
+		pg->physaddr = (phys + off) & ~(USB_PAGE_SIZE - 1);
+	}
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_pc_alloc_mem - allocate DMA'able memory
+ *
+ * Returns:
+ *    0: Success
+ * Else: Failure
+ *------------------------------------------------------------------------*/
+uint8_t
+usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,
+    usb_size_t size, usb_size_t align)
+{
+	void *ptr;
+	uint32_t rem;
+
+	/* allocate zeroed memory */
+
+	if (align != 1) {
+		ptr = usb_malloc(size + align);
+		if (ptr == NULL)
+			goto error;
+
+		rem = (-((uintptr_t)ptr)) & (align - 1);
+	} else {
+		ptr = usb_malloc(size);
+		if (ptr == NULL)
+			goto error;
+		rem = 0;
+	}
+
+	/* setup page cache */
+	pc->buffer = ((uint8_t *)ptr) + rem;
+	pc->page_start = pg;
+	pc->page_offset_buf = 0;
+	pc->page_offset_end = size;
+	pc->map = NULL;
+	pc->tag = ptr;
+	pc->ismultiseg = (align == 1);
+
+	/* compute physical address */
+ 	usb_pc_common_mem_cb(pc, ptr, size);
+
+	usb_pc_cpu_flush(pc);
+ 	return (0);
+
+error:
+	/* reset most of the page cache */
+	pc->buffer = NULL;
+	pc->page_start = NULL;
+	pc->page_offset_buf = 0;
+	pc->page_offset_end = 0;
+	pc->map = NULL;
+	pc->tag = NULL;
+	return (1);
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_pc_free_mem - free DMA memory
+ *
+ * This function is NULL safe.
+ *------------------------------------------------------------------------*/
+void
+usb_pc_free_mem(struct usb_page_cache *pc)
+{
+	if (pc != NULL && pc->buffer != NULL) {
+		usb_free(pc->tag);
+		pc->buffer = NULL;
+	}
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_pc_load_mem - load virtual memory into DMA
+ *
+ * Return values:
+ * 0: Success
+ * Else: Error
+ *------------------------------------------------------------------------*/
+uint8_t
+usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)
+{
+	/* setup page cache */
+	pc->page_offset_buf = 0;
+	pc->page_offset_end = size;
+	pc->ismultiseg = 1;
+
+	mtx_assert(pc->tag_parent->mtx, MA_OWNED);
+
+	if (size > 0) {
+		/* compute physical address */
+		usb_pc_common_mem_cb(pc, pc->buffer, size);
+	}
+	if (sync == 0) {
+		/*
+		 * Call callback so that refcount is decremented
+		 * properly:
+		 */
+		pc->tag_parent->dma_error = 0;
+		(pc->tag_parent->func) (pc->tag_parent);
+	}
+	return (0);
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_pc_cpu_invalidate - invalidate CPU cache
+ *------------------------------------------------------------------------*/
+void
+usb_pc_cpu_invalidate(struct usb_page_cache *pc)
+{
+	if (pc->page_offset_end == pc->page_offset_buf) {
+		/* nothing has been loaded into this page cache! */
+		return;
+	}
+	/* NOP */
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_pc_cpu_flush - flush CPU cache
+ *------------------------------------------------------------------------*/
+void
+usb_pc_cpu_flush(struct usb_page_cache *pc)
+{
+	if (pc->page_offset_end == pc->page_offset_buf) {
+		/* nothing has been loaded into this page cache! */
+		return;
+	}
+	/* NOP */
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_pc_dmamap_create - create a DMA map
+ *
+ * Returns:
+ *    0: Success
+ * Else: Failure
+ *------------------------------------------------------------------------*/
+uint8_t
+usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
+{
+	return (0);	/* NOP, success */
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_pc_dmamap_destroy
+ *
+ * This function is NULL safe.
+ *------------------------------------------------------------------------*/
+void
+usb_pc_dmamap_destroy(struct usb_page_cache *pc)
+{
+	/* NOP */
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_dma_tag_setup - initialise USB DMA tags
+ *------------------------------------------------------------------------*/
+void
+usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
+    struct usb_dma_tag *udt, bus_dma_tag_t dmat,
+    struct mtx *mtx, usb_dma_callback_t *func,
+    uint8_t ndmabits, uint8_t nudt)
+{
+	memset(udpt, 0, sizeof(*udpt));
+
+	/* sanity checking */
+	if ((nudt == 0) ||
+	    (ndmabits == 0) ||
+	    (mtx == NULL)) {
+		/* something is corrupt */
+		return;
+	}
+	/* initialise condition variable */
+	cv_init(udpt->cv, "USB DMA CV");
+
+	/* store some information */
+	udpt->mtx = mtx;
+	udpt->func = func;
+	udpt->tag = dmat;
+	udpt->utag_first = udt;
+	udpt->utag_max = nudt;
+	udpt->dma_bits = ndmabits;
+
+	while (nudt--) {
+		memset(udt, 0, sizeof(*udt));
+		udt->tag_parent = udpt;
+		udt++;
+	}
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_bus_tag_unsetup - factored out code
+ *------------------------------------------------------------------------*/
+void
+usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt)
+{
+	struct usb_dma_tag *udt;
+	uint8_t nudt;
+
+	udt = udpt->utag_first;
+	nudt = udpt->utag_max;
+
+	while (nudt--) {
+		udt->align = 0;
+		udt++;
+	}
+
+	if (udpt->utag_max) {
+		/* destroy the condition variable */
+		cv_destroy(udpt->cv);
+	}
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_bdma_work_loop
+ *
+ * This function handles loading of virtual buffers into DMA and is
+ * only called when "dma_refcount" is zero.
+ *------------------------------------------------------------------------*/
+void
+usb_bdma_work_loop(struct usb_xfer_queue *pq)
+{
+	struct usb_xfer_root *info;
+	struct usb_xfer *xfer;
+	usb_frcount_t nframes;
+
+	xfer = pq->curr;
+	info = xfer->xroot;
+
+	mtx_assert(info->xfer_mtx, MA_OWNED);
+
+	if (xfer->error) {
+		/* some error happened */
+		USB_BUS_LOCK(info->bus);
+		usbd_transfer_done(xfer, 0);
+		USB_BUS_UNLOCK(info->bus);
+		return;
+	}
+	if (!xfer->flags_int.bdma_setup) {
+		struct usb_page *pg;
+		usb_frlength_t frlength_0;
+		uint8_t isread;
+
+		xfer->flags_int.bdma_setup = 1;
+
+		/* reset BUS-DMA load state */
+
+		info->dma_error = 0;
+
+		if (xfer->flags_int.isochronous_xfr) {
+			/* only one frame buffer */
+			nframes = 1;
+			frlength_0 = xfer->sumlen;
+		} else {
+			/* can be multiple frame buffers */
+			nframes = xfer->nframes;
+			frlength_0 = xfer->frlengths[0];
+		}
+
+		/*
+		 * Set DMA direction first. This is needed to
+		 * select the correct cache invalidate and cache
+		 * flush operations.
+		 */
+		isread = USB_GET_DATA_ISREAD(xfer);
+		pg = xfer->dma_page_ptr;
+
+		if (xfer->flags_int.control_xfr &&
+		    xfer->flags_int.control_hdr) {
+			/* special case */
+			if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
+				/* The device controller writes to memory */
+				xfer->frbuffers[0].isread = 1;
+			} else {
+				/* The host controller reads from memory */
+				xfer->frbuffers[0].isread = 0;
+			}
+		} else {
+			/* default case */
+			xfer->frbuffers[0].isread = isread;
+		}
+
+		/*
+		 * Setup the "page_start" pointer which points to an array of
+		 * USB pages where information about the physical address of a
+		 * page will be stored. Also initialise the "isread" field of
+		 * the USB page caches.
+		 */
+		xfer->frbuffers[0].page_start = pg;
+
+		info->dma_nframes = nframes;
+		info->dma_currframe = 0;
+		info->dma_frlength_0 = frlength_0;
+
+		pg += (frlength_0 / USB_PAGE_SIZE);
+		pg += 2;
+
+		while (--nframes > 0) {
+			xfer->frbuffers[nframes].isread = isread;
+			xfer->frbuffers[nframes].page_start = pg;
+
+			pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
+			pg += 2;
+		}
+
+	}
+	if (info->dma_error) {
+		USB_BUS_LOCK(info->bus);
+		usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
+		USB_BUS_UNLOCK(info->bus);
+		return;
+	}
+	if (info->dma_currframe != info->dma_nframes) {
+
+		if (info->dma_currframe == 0) {
+			/* special case */
+			usb_pc_load_mem(xfer->frbuffers,
+			    info->dma_frlength_0, 0);
+		} else {
+			/* default case */
+			nframes = info->dma_currframe;
+			usb_pc_load_mem(xfer->frbuffers + nframes,
+			    xfer->frlengths[nframes], 0);
+		}
+
+		/* advance frame index */
+		info->dma_currframe++;
+
+		return;
+	}
+	/* go ahead */
+	usb_bdma_pre_sync(xfer);
+
+	/* start loading next USB transfer, if any */
+	usb_command_wrapper(pq, NULL);
+
+	/* finally start the hardware */
+	usbd_pipe_enter(xfer);
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_bdma_done_event
+ *
+ * This function is called when the BUS-DMA has loaded virtual memory
+ * into DMA, if any.
+ *------------------------------------------------------------------------*/
+void
+usb_bdma_done_event(struct usb_dma_parent_tag *udpt)
+{
+	struct usb_xfer_root *info;
+
+	info = USB_DMATAG_TO_XROOT(udpt);
+
+	mtx_assert(info->xfer_mtx, MA_OWNED);
+
+	/* copy error */
+	info->dma_error = udpt->dma_error;
+
+	/* enter workloop again */
+	usb_command_wrapper(&info->dma_q,
+	    info->dma_q.curr);
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_bdma_pre_sync
+ *
+ * This function handles DMA synchronisation that must be done before
+ * an USB transfer is started.
+ *------------------------------------------------------------------------*/
+void
+usb_bdma_pre_sync(struct usb_xfer *xfer)
+{
+	struct usb_page_cache *pc;
+	usb_frcount_t nframes;
+
+	if (xfer->flags_int.isochronous_xfr) {
+		/* only one frame buffer */
+		nframes = 1;
+	} else {
+		/* can be multiple frame buffers */
+		nframes = xfer->nframes;
+	}
+
+	pc = xfer->frbuffers;
+
+	while (nframes--) {
+
+		if (pc->isread) {
+			usb_pc_cpu_invalidate(pc);
+		} else {
+			usb_pc_cpu_flush(pc);
+		}
+		pc++;
+	}
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_bdma_post_sync
+ *
+ * This function handles DMA synchronisation that must be done after
+ * an USB transfer is complete.
+ *------------------------------------------------------------------------*/
+void
+usb_bdma_post_sync(struct usb_xfer *xfer)
+{
+	struct usb_page_cache *pc;
+	usb_frcount_t nframes;
+
+	if (xfer->flags_int.isochronous_xfr) {
+		/* only one frame buffer */
+		nframes = 1;
+	} else {
+		/* can be multiple frame buffers */
+		nframes = xfer->nframes;
+	}
+
+	pc = xfer->frbuffers;
+
+	while (nframes--) {
+		if (pc->isread) {
+			usb_pc_cpu_invalidate(pc);
+		}
+		pc++;
+	}
+}
+#endif
diff --git a/sys/cam/ctl/ctl.c b/sys/cam/ctl/ctl.c
index ac5eba7..768332e 100644
--- a/sys/cam/ctl/ctl.c
+++ b/sys/cam/ctl/ctl.c
@@ -2257,11 +2257,31 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 				 */
 				mtx_unlock(&softc->ctl_lock);
 
-				if (cmd == CTL_ENABLE_PORT)
+				if (cmd == CTL_ENABLE_PORT) {
+					struct ctl_lun *lun;
+
+					STAILQ_FOREACH(lun, &softc->lun_list,
+						       links) {
+						fe->lun_enable(fe->targ_lun_arg,
+						    lun->target,
+						    lun->lun);
+					}
+
 					ctl_frontend_online(fe);
-				else if (cmd == CTL_DISABLE_PORT)
+				} else if (cmd == CTL_DISABLE_PORT) {
+					struct ctl_lun *lun;
+
 					ctl_frontend_offline(fe);
 
+					STAILQ_FOREACH(lun, &softc->lun_list,
+						       links) {
+						fe->lun_disable(
+						    fe->targ_lun_arg,
+						    lun->target,
+						    lun->lun);
+					}
+				}
+
 				mtx_lock(&softc->ctl_lock);
 
 				if (cmd == CTL_SET_PORT_WWNS)
diff --git a/sys/cam/ctl/scsi_ctl.c b/sys/cam/ctl/scsi_ctl.c
index 59357123..5dc20e5 100644
--- a/sys/cam/ctl/scsi_ctl.c
+++ b/sys/cam/ctl/scsi_ctl.c
@@ -73,6 +73,7 @@ __FBSDID("$FreeBSD$");
 #include <cam/ctl/ctl_error.h>
 
 typedef enum {
+	CTLFE_CCB_DEFAULT	= 0x00,
 	CTLFE_CCB_WAITING 	= 0x01
 } ctlfe_ccb_types;
 
@@ -304,10 +305,7 @@ ctlfeasync(void *callback_arg, uint32_t code, struct cam_path *path, void *arg)
 	case AC_PATH_REGISTERED: {
 		struct ctl_frontend *fe;
 		struct ctlfe_softc *bus_softc;
-		struct ctlfe_lun_softc *lun_softc;
-		struct cam_path *path;
 		struct ccb_pathinq *cpi;
-		cam_status status;
 		int retval;
 
 		cpi = (struct ccb_pathinq *)arg;
@@ -330,7 +328,6 @@ ctlfeasync(void *callback_arg, uint32_t code, struct cam_path *path, void *arg)
 						  M_NOWAIT | M_ZERO);
 			if (ccb == NULL) {
 				printf("%s: unable to malloc CCB!\n", __func__);
-				xpt_free_path(path);
 				return;
 			}
 			xpt_setup_ccb(&ccb->ccb_h, cpi->ccb_h.path,
@@ -448,44 +445,31 @@ ctlfeasync(void *callback_arg, uint32_t code, struct cam_path *path, void *arg)
 			mtx_unlock(&ctlfe_list_mtx);
 		}
 
-		status = xpt_create_path(&path, /*periph*/ NULL,
-					 bus_softc->path_id,CAM_TARGET_WILDCARD,
-					 CAM_LUN_WILDCARD);
-		if (status != CAM_REQ_CMP) {
-			printf("%s: unable to create path for wildcard "
-			       "periph\n", __func__);
-			break;
-		}
-		lun_softc = malloc(sizeof(*lun_softc), M_CTLFE,
-				   M_NOWAIT | M_ZERO);
-		if (lun_softc == NULL) {
-			xpt_print(path, "%s: unable to allocate softc for "
-				  "wildcard periph\n", __func__);
-			xpt_free_path(path);
-			break;
+		break;
+	}
+	case AC_PATH_DEREGISTERED: {
+		struct ctlfe_softc *softc = NULL;
+
+		mtx_lock(&ctlfe_list_mtx);
+		STAILQ_FOREACH(softc, &ctlfe_softc_list, links) {
+			if (softc->path_id == xpt_path_path_id(path)) {
+				STAILQ_REMOVE(&ctlfe_softc_list, softc,
+						ctlfe_softc, links);
+				break;
+			}
 		}
+		mtx_unlock(&ctlfe_list_mtx);
 
-		lun_softc->parent_softc = bus_softc;
-		lun_softc->flags |= CTLFE_LUN_WILDCARD;
-
-		status = cam_periph_alloc(ctlferegister,
-					  ctlfeoninvalidate,
-					  ctlfecleanup,
-					  ctlfestart,
-					  "ctl",
-					  CAM_PERIPH_BIO,
-					  path,
-					  ctlfeasync,
-					  0,
-					  lun_softc);
-
-		xpt_free_path(path);
-
+		if (softc != NULL) {
+			/*
+			 * XXX KDM are we certain at this point that there
+			 * are no outstanding commands for this frontend?
+			 */
+			ctl_frontend_deregister(&softc->fe);
+			free(softc, M_CTLFE);
+		}
 		break;
 	}
-	case AC_PATH_DEREGISTERED:
-		/* ctl_frontend_deregister() */
-		break;
 	case AC_CONTRACT: {
 		struct ac_contract *ac;
 
@@ -699,11 +683,14 @@ ctlfecleanup(struct cam_periph *periph)
 	softc = (struct ctlfe_lun_softc *)periph->softc;
 	bus_softc = softc->parent_softc;
 
-	STAILQ_REMOVE(&bus_softc->lun_softc_list, softc, ctlfe_lun_softc,links);
+	STAILQ_REMOVE(&bus_softc->lun_softc_list, softc, ctlfe_lun_softc, links);
 	
 	/*
 	 * XXX KDM is there anything else that needs to be done here?
 	 */
+
+	callout_stop(&softc->dma_callout);
+
 	free(softc, M_CTLFE);
 }
 
@@ -717,6 +704,8 @@ ctlfestart(struct cam_periph *periph, union ccb *start_ccb)
 
 	softc->ccbs_alloced++;
 
+	start_ccb->ccb_h.ccb_type = CTLFE_CCB_DEFAULT;
+
 	ccb_h = TAILQ_FIRST(&softc->work_queue);
 	if (periph->immediate_priority <= periph->pinfo.priority) {
 		panic("shouldn't get to the CCB waiting case!");
@@ -857,8 +846,6 @@ ctlfestart(struct cam_periph *periph, union ccb *start_ccb)
 
 			/*
 			 * Datamove call, we need to setup the S/G list. 
-			 * If we pass in a S/G list, the isp(4) driver at
-			 * least expects physical/bus addresses.
 			 */
 
 			cmd_info = (struct ctlfe_lun_cmd_info *)
@@ -933,12 +920,13 @@ ctlfestart(struct cam_periph *periph, union ccb *start_ccb)
 				int *ti;
 
 				/*
-				 * XXX KDM this is a temporary hack.  The
-				 * isp(4) driver can't deal with S/G lists
-				 * with virtual pointers, so we need to
-				 * go through and send down one virtual
-				 * pointer at a time.
+				 * If we have more S/G list pointers than
+				 * will fit in the available storage in the
+				 * cmd_info structure inside the ctl_io header,
+				 * then we need to send down the pointers
+				 * one element at a time.
 				 */
+
 				sglist = (struct ctl_sg_entry *)
 					io->scsiio.kern_data_ptr;
 				ti = &cmd_info->cur_transfer_index;
@@ -1405,13 +1393,15 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb)
 				break;
 			default:
 				/*
-				 * XXX KDM the isp(4) driver doesn't really
-				 * seem to send errors back for data
-				 * transfers that I can tell.  There is one
-				 * case where it'll send CAM_REQ_CMP_ERR,
-				 * but probably not that many more cases.
-				 * So set a generic data phase error here,
-				 * like the SXP driver sets.
+				 * XXX KDM we probably need to figure out a
+				 * standard set of errors that the SIM
+				 * drivers should return in the event of a
+				 * data transfer failure.  A data phase
+				 * error will at least point the user to a
+				 * data transfer error of some sort.
+				 * Hopefully the SIM printed out some
+				 * additional information to give the user
+				 * a clue what happened.
 				 */
 				io->io_hdr.port_status = 0xbad1;
 				ctl_set_data_phase_error(&io->scsiio);
@@ -1689,20 +1679,22 @@ ctlfe_onoffline(void *arg, int online)
 
 	sim = bus_softc->sim;
 
-	CAM_SIM_LOCK(sim);
+	mtx_assert(sim->mtx, MA_OWNED);
+
 	status = xpt_create_path(&path, /*periph*/ NULL, bus_softc->path_id,
 		CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
 	if (status != CAM_REQ_CMP) {
 		printf("%s: unable to create path!\n", __func__);
-		CAM_SIM_UNLOCK(sim);
 		return;
 	}
-	CAM_SIM_UNLOCK(sim);
-
-	ccb = (union ccb *)malloc(sizeof(*ccb), M_TEMP, M_WAITOK | M_ZERO);
+	ccb = (union ccb *)malloc(sizeof(*ccb), M_TEMP, M_NOWAIT | M_ZERO);
+	if (ccb == NULL) {
+		printf("%s: unable to malloc CCB!\n", __func__);
+		xpt_free_path(path);
+		return;
+	}
 	xpt_setup_ccb(&ccb->ccb_h, path, CAM_PRIORITY_NONE);
 
-
 	/*
 	 * Copan WWN format:
 	 *
@@ -1720,11 +1712,9 @@ ctlfe_onoffline(void *arg, int online)
 
 		ccb->ccb_h.func_code = XPT_GET_SIM_KNOB;
 
-		CAM_SIM_LOCK(sim);
 
 		xpt_action(ccb);
 
-		CAM_SIM_UNLOCK(sim);
 
 		if ((ccb->knob.xport_specific.valid & KNOB_VALID_ADDRESS) != 0){
 #ifdef RANDOM_WWNN
@@ -1822,9 +1812,6 @@ ctlfe_onoffline(void *arg, int online)
 	else
 		ccb->knob.xport_specific.fc.role = KNOB_ROLE_NONE;
 
-
-	CAM_SIM_LOCK(sim);
-
 	xpt_action(ccb);
 
 	if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
@@ -1841,8 +1828,6 @@ ctlfe_onoffline(void *arg, int online)
 
 	xpt_free_path(path);
 
-	CAM_SIM_UNLOCK(sim);
-
 	free(ccb, M_TEMP);
 
 	return;
@@ -1851,13 +1836,111 @@ ctlfe_onoffline(void *arg, int online)
 static void
 ctlfe_online(void *arg)
 {
+	struct ctlfe_softc *bus_softc;
+	struct cam_path *path;
+	cam_status status;
+	struct ctlfe_lun_softc *lun_softc;
+	struct cam_sim *sim;
+
+	bus_softc = (struct ctlfe_softc *)arg;
+	sim = bus_softc->sim;
+
+	CAM_SIM_LOCK(sim);
+
+	/*
+	 * Create the wildcard LUN before bringing the port online.
+	 */
+	status = xpt_create_path(&path, /*periph*/ NULL,
+				 bus_softc->path_id, CAM_TARGET_WILDCARD,
+				 CAM_LUN_WILDCARD);
+	if (status != CAM_REQ_CMP) {
+		printf("%s: unable to create path for wildcard periph\n",
+				__func__);
+		CAM_SIM_UNLOCK(sim);
+		return;
+	}
+
+	lun_softc = malloc(sizeof(*lun_softc), M_CTLFE,
+			M_NOWAIT | M_ZERO);
+	if (lun_softc == NULL) {
+		xpt_print(path, "%s: unable to allocate softc for "
+				"wildcard periph\n", __func__);
+		xpt_free_path(path);
+		CAM_SIM_UNLOCK(sim);
+		return;
+	}
+
+	lun_softc->parent_softc = bus_softc;
+	lun_softc->flags |= CTLFE_LUN_WILDCARD;
+
+	STAILQ_INSERT_TAIL(&bus_softc->lun_softc_list, lun_softc, links);
+
+
+	status = cam_periph_alloc(ctlferegister,
+				  ctlfeoninvalidate,
+				  ctlfecleanup,
+				  ctlfestart,
+				  "ctl",
+				  CAM_PERIPH_BIO,
+				  path,
+				  ctlfeasync,
+				  0,
+				  lun_softc);
+
+	if ((status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
+		const struct cam_status_entry *entry;
+
+		entry = cam_fetch_status_entry(status);
+
+		printf("%s: CAM error %s (%#x) returned from "
+		       "cam_periph_alloc()\n", __func__, (entry != NULL) ?
+		       entry->status_text : "Unknown", status);
+	}
+
+	xpt_free_path(path);
+
 	ctlfe_onoffline(arg, /*online*/ 1);
+
+	CAM_SIM_UNLOCK(sim);
 }
 
 static void
 ctlfe_offline(void *arg)
 {
+	struct ctlfe_softc *bus_softc;
+	struct cam_path *path;
+	cam_status status;
+	struct cam_periph *periph;
+	struct cam_sim *sim;
+
+	bus_softc = (struct ctlfe_softc *)arg;
+	sim = bus_softc->sim;
+
+	CAM_SIM_LOCK(sim);
+
 	ctlfe_onoffline(arg, /*online*/ 0);
+
+	/*
+	 * Disable the wildcard LUN for this port now that we have taken
+	 * the port offline.
+	 */
+	status = xpt_create_path(&path, /*periph*/ NULL,
+				 bus_softc->path_id, CAM_TARGET_WILDCARD,
+				 CAM_LUN_WILDCARD);
+	if (status != CAM_REQ_CMP) {
+		CAM_SIM_UNLOCK(sim);
+		printf("%s: unable to create path for wildcard periph\n",
+		       __func__);
+		return;
+	}
+
+
+	if ((periph = cam_periph_find(path, "ctl")) != NULL)
+		cam_periph_invalidate(periph);
+
+	xpt_free_path(path);
+
+	CAM_SIM_UNLOCK(sim);
 }
 
 static int
@@ -1890,21 +1973,17 @@ ctlfe_lun_enable(void *arg, struct ctl_id targ_id, int lun_id)
 	bus_softc = (struct ctlfe_softc *)arg;
 	sim = bus_softc->sim;
 
-	CAM_SIM_LOCK(sim);
-
-	status = xpt_create_path(&path, /*periph*/ NULL, bus_softc->path_id,
-				 targ_id.id, lun_id);
+	status = xpt_create_path_unlocked(&path, /*periph*/ NULL,
+					  bus_softc->path_id,
+					  targ_id.id, lun_id);
 	/* XXX KDM need some way to return status to CTL here? */
 	if (status != CAM_REQ_CMP) {
 		printf("%s: could not create path, status %#x\n", __func__,
 		       status);
-		CAM_SIM_UNLOCK(sim);
 		return (1);
 	}
-	CAM_SIM_UNLOCK(sim);
 
 	softc = malloc(sizeof(*softc), M_CTLFE, M_WAITOK | M_ZERO);
-
 	CAM_SIM_LOCK(sim);
 	periph = cam_periph_find(path, "ctl");
 	if (periph != NULL) {
@@ -1937,23 +2016,20 @@ ctlfe_lun_enable(void *arg, struct ctl_id targ_id, int lun_id)
 }
 
 /*
- * XXX KDM we disable LUN removal here.  The problem is that the isp(4)
- * driver doesn't currently handle LUN removal properly.  We need to keep 
- * enough state here at the peripheral level even after LUNs have been
- * removed inside CTL.
- *
- * Once the isp(4) driver is fixed, this can be re-enabled.
+ * This will get called when the user removes a LUN to disable that LUN
+ * on every bus that is attached to CTL.  
  */
 static int
 ctlfe_lun_disable(void *arg, struct ctl_id targ_id, int lun_id)
 {
-#ifdef NOTYET
 	struct ctlfe_softc *softc;
 	struct ctlfe_lun_softc *lun_softc;
+	struct cam_sim *sim;
 
 	softc = (struct ctlfe_softc *)arg;
+	sim = softc->sim;
 
-	mtx_lock(softc->sim->mtx);
+	CAM_SIM_LOCK(sim);
 	STAILQ_FOREACH(lun_softc, &softc->lun_softc_list, links) {
 		struct cam_path *path;
 
@@ -1965,7 +2041,7 @@ ctlfe_lun_disable(void *arg, struct ctl_id targ_id, int lun_id)
 		}
 	}
 	if (lun_softc == NULL) {
-		mtx_unlock(softc->sim->mtx);
+		CAM_SIM_UNLOCK(sim);
 		printf("%s: can't find target %d lun %d\n", __func__,
 		       targ_id.id, lun_id);
 		return (1);
@@ -1973,8 +2049,7 @@ ctlfe_lun_disable(void *arg, struct ctl_id targ_id, int lun_id)
 
 	cam_periph_invalidate(lun_softc->periph);
 
-	mtx_unlock(softc->sim->mtx);
-#endif
+	CAM_SIM_UNLOCK(sim);
 
 	return (0);
 }
@@ -2139,7 +2214,7 @@ ctlfe_datamove_done(union ctl_io *io)
 
 	sim = xpt_path_sim(ccb->ccb_h.path);
 
-	mtx_lock(sim->mtx);
+	CAM_SIM_LOCK(sim);
 
 	periph = xpt_path_periph(ccb->ccb_h.path);
 
@@ -2186,7 +2261,7 @@ ctlfe_datamove_done(union ctl_io *io)
 			xpt_schedule(periph, /*priority*/ 1);
 	}
 
-	mtx_unlock(sim->mtx);
+	CAM_SIM_UNLOCK(sim);
 }
 
 static void
diff --git a/sys/cam/scsi/scsi_all.c b/sys/cam/scsi/scsi_all.c
index a37ae19..9dac9c0 100644
--- a/sys/cam/scsi/scsi_all.c
+++ b/sys/cam/scsi/scsi_all.c
@@ -3071,16 +3071,15 @@ scsi_error_action(struct ccb_scsiio *csio, struct scsi_inquiry_data *inq_data,
 					  SSQ_PRINT_SENSE;
 			}
 		}
-		if ((action & SS_MASK) >= SS_START &&
-		    (sense_flags & SF_NO_RECOVERY)) {
-			action &= ~SS_MASK;
-			action |= SS_FAIL;
-		} else if ((action & SS_MASK) == SS_RETRY &&
-		    (sense_flags & SF_NO_RETRY)) {
-			action &= ~SS_MASK;
-			action |= SS_FAIL;
-		}
-
+	}
+	if ((action & SS_MASK) >= SS_START &&
+	    (sense_flags & SF_NO_RECOVERY)) {
+		action &= ~SS_MASK;
+		action |= SS_FAIL;
+	} else if ((action & SS_MASK) == SS_RETRY &&
+	    (sense_flags & SF_NO_RETRY)) {
+		action &= ~SS_MASK;
+		action |= SS_FAIL;
 	}
 	if ((sense_flags & SF_PRINT_ALWAYS) != 0)
 		action |= SSQ_PRINT_SENSE;
@@ -3140,7 +3139,7 @@ scsi_cdb_string(u_int8_t *cdb_ptr, char *cdb_string, size_t len)
 	*cdb_string = '\0';
 	for (i = 0; i < cdb_len; i++)
 		snprintf(cdb_string + strlen(cdb_string),
-			 len - strlen(cdb_string), "%x ", cdb_ptr[i]);
+			 len - strlen(cdb_string), "%02hhx ", cdb_ptr[i]);
 
 	return(cdb_string);
 }
diff --git a/sys/cam/scsi/scsi_cd.c b/sys/cam/scsi/scsi_cd.c
index d5d7156..a7c4c5b 100644
--- a/sys/cam/scsi/scsi_cd.c
+++ b/sys/cam/scsi/scsi_cd.c
@@ -581,7 +581,7 @@ cdasync(void *callback_arg, u_int32_t code,
 		if (softc->state == CD_STATE_NORMAL && !softc->tur) {
 			if (cam_periph_acquire(periph) == CAM_REQ_CMP) {
 				softc->tur = 1;
-				xpt_schedule(periph, CAM_PRIORITY_DEV);
+				xpt_schedule(periph, CAM_PRIORITY_NORMAL);
 			}
 		}
 		/* FALLTHROUGH */
@@ -1612,9 +1612,11 @@ cdstart(struct cam_periph *periph, union ccb *start_ccb)
 
 			xpt_action(start_ccb);
 		}
-		if (bp != NULL || softc->tur) {
+		if (bp != NULL || softc->tur ||
+		    periph->immediate_priority != CAM_PRIORITY_NONE) {
 			/* Have more work to do, so ensure we stay scheduled */
-			xpt_schedule(periph, CAM_PRIORITY_NORMAL);
+			xpt_schedule(periph, min(CAM_PRIORITY_NORMAL,
+			    periph->immediate_priority));
 		}
 		break;
 	}
@@ -3293,10 +3295,11 @@ cdmediapoll(void *arg)
 	if (softc->flags & CD_FLAG_CHANGER)
 		return;
 
-	if (softc->state == CD_STATE_NORMAL && !softc->tur) {
+	if (softc->state == CD_STATE_NORMAL && !softc->tur &&
+	    softc->outstanding_cmds == 0) {
 		if (cam_periph_acquire(periph) == CAM_REQ_CMP) {
 			softc->tur = 1;
-			xpt_schedule(periph, CAM_PRIORITY_DEV);
+			xpt_schedule(periph, CAM_PRIORITY_NORMAL);
 		}
 	}
 	/* Queue us up again */
diff --git a/sys/cam/scsi/scsi_da.c b/sys/cam/scsi/scsi_da.c
index adda9c9..7854215 100644
--- a/sys/cam/scsi/scsi_da.c
+++ b/sys/cam/scsi/scsi_da.c
@@ -863,6 +863,8 @@ static	void		daasync(void *callback_arg, u_int32_t code,
 static	void		dasysctlinit(void *context, int pending);
 static	int		dacmdsizesysctl(SYSCTL_HANDLER_ARGS);
 static	int		dadeletemethodsysctl(SYSCTL_HANDLER_ARGS);
+static	int		dadeletemethodset(struct da_softc *softc,
+					  da_delete_methods delete_method);
 static	periph_ctor_t	daregister;
 static	periph_dtor_t	dacleanup;
 static	periph_start_t	dastart;
@@ -1079,6 +1081,9 @@ daschedule(struct cam_periph *periph)
 	struct da_softc *softc = (struct da_softc *)periph->softc;
 	uint32_t prio;
 
+	if (softc->state != DA_STATE_NORMAL)
+		return;
+
 	/* Check if cam_periph_getccb() was called. */
 	prio = periph->immediate_priority;
 
@@ -1423,10 +1428,10 @@ daasync(void *callback_arg, u_int32_t code,
 	}
 	case AC_SCSI_AEN:
 		softc = (struct da_softc *)periph->softc;
-		if (softc->state == DA_STATE_NORMAL && !softc->tur) {
+		if (!softc->tur) {
 			if (cam_periph_acquire(periph) == CAM_REQ_CMP) {
 				softc->tur = 1;
-				xpt_schedule(periph, CAM_PRIORITY_DEV);
+				daschedule(periph);
 			}
 		}
 		/* FALLTHROUGH */
@@ -1489,7 +1494,7 @@ dasysctlinit(void *context, int pending)
 	 */
 	SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
 		OID_AUTO, "delete_method", CTLTYPE_STRING | CTLFLAG_RW,
-		&softc->delete_method, 0, dadeletemethodsysctl, "A",
+		softc, 0, dadeletemethodsysctl, "A",
 		"BIO_DELETE execution method");
 	SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
 		OID_AUTO, "minimum_cmd_size", CTLTYPE_INT | CTLFLAG_RW,
@@ -1566,14 +1571,33 @@ dacmdsizesysctl(SYSCTL_HANDLER_ARGS)
 }
 
 static int
+dadeletemethodset(struct da_softc *softc, da_delete_methods delete_method)
+{
+
+	if (delete_method < 0 || delete_method > DA_DELETE_MAX)
+		return (EINVAL);
+
+	softc->delete_method = delete_method;
+
+	if (softc->delete_method > DA_DELETE_DISABLE)
+		softc->disk->d_flags |= DISKFLAG_CANDELETE;
+	else
+		softc->disk->d_flags &= ~DISKFLAG_CANDELETE;
+
+	return (0);
+}
+
+static int
 dadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
 {
 	char buf[16];
-	int error;
 	const char *p;
-	int i, value;
+	struct da_softc *softc;
+	int i, error, value;
 
-	value = *(int *)arg1;
+	softc = (struct da_softc *)arg1;
+
+	value = softc->delete_method;
 	if (value < 0 || value > DA_DELETE_MAX)
 		p = "UNKNOWN";
 	else
@@ -1585,8 +1609,7 @@ dadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
 	for (i = 0; i <= DA_DELETE_MAX; i++) {
 		if (strcmp(buf, da_delete_method_names[i]) != 0)
 			continue;
-		*(int *)arg1 = i;
-		return (0);
+		return dadeletemethodset(softc, i);
 	}
 	return (EINVAL);
 }
@@ -2014,7 +2037,6 @@ out:
 	}
 	case DA_STATE_PROBE:
 	{
-		struct ccb_scsiio *csio;
 		struct scsi_read_capacity_data *rcap;
 
 		rcap = (struct scsi_read_capacity_data *)
@@ -2024,9 +2046,8 @@ out:
 			/* da_free_periph??? */
 			break;
 		}
-		csio = &start_ccb->csio;
-		scsi_read_capacity(csio,
-				   /*retries*/4,
+		scsi_read_capacity(&start_ccb->csio,
+				   /*retries*/da_retry_count,
 				   dadone,
 				   MSG_SIMPLE_Q_TAG,
 				   rcap,
@@ -2039,7 +2060,6 @@ out:
 	}
 	case DA_STATE_PROBE2:
 	{
-		struct ccb_scsiio *csio;
 		struct scsi_read_capacity_data_long *rcaplong;
 
 		rcaplong = (struct scsi_read_capacity_data_long *)
@@ -2049,9 +2069,8 @@ out:
 			/* da_free_periph??? */
 			break;
 		}
-		csio = &start_ccb->csio;
-		scsi_read_capacity_16(csio,
-				      /*retries*/ 4,
+		scsi_read_capacity_16(&start_ccb->csio,
+				      /*retries*/ da_retry_count,
 				      /*cbfcnp*/ dadone,
 				      /*tag_action*/ MSG_SIMPLE_Q_TAG,
 				      /*lba*/ 0,
@@ -2060,7 +2079,7 @@ out:
 				      /*rcap_buf*/ (uint8_t *)rcaplong,
 				      /*rcap_buf_len*/ sizeof(*rcaplong),
 				      /*sense_len*/ SSD_FULL_SIZE,
-				      /*timeout*/ 60000);
+				      /*timeout*/ da_default_timeout * 1000);
 		start_ccb->ccb_h.ccb_bp = NULL;
 		start_ccb->ccb_h.ccb_state = DA_CCB_PROBE2;
 		xpt_action(start_ccb);	
@@ -2086,24 +2105,24 @@ cmd6workaround(union ccb *ccb)
 		if (softc->delete_method == DA_DELETE_UNMAP) {
 			xpt_print(ccb->ccb_h.path, "UNMAP is not supported, "
 			    "switching to WRITE SAME(16) with UNMAP.\n");
-			softc->delete_method = DA_DELETE_WS16;
+			dadeletemethodset(softc, DA_DELETE_WS16);
 		} else if (softc->delete_method == DA_DELETE_WS16) {
 			xpt_print(ccb->ccb_h.path,
 			    "WRITE SAME(16) with UNMAP is not supported, "
 			    "disabling BIO_DELETE.\n");
-			softc->delete_method = DA_DELETE_DISABLE;
+			dadeletemethodset(softc, DA_DELETE_DISABLE);
 		} else if (softc->delete_method == DA_DELETE_WS10) {
 			xpt_print(ccb->ccb_h.path,
 			    "WRITE SAME(10) with UNMAP is not supported, "
 			    "disabling BIO_DELETE.\n");
-			softc->delete_method = DA_DELETE_DISABLE;
+			dadeletemethodset(softc, DA_DELETE_DISABLE);
 		} else if (softc->delete_method == DA_DELETE_ZERO) {
 			xpt_print(ccb->ccb_h.path,
 			    "WRITE SAME(10) is not supported, "
 			    "disabling BIO_DELETE.\n");
-			softc->delete_method = DA_DELETE_DISABLE;
+			dadeletemethodset(softc, DA_DELETE_DISABLE);
 		} else
-			softc->delete_method = DA_DELETE_DISABLE;
+			dadeletemethodset(softc, DA_DELETE_DISABLE);
 		while ((bp = bioq_takefirst(&softc->delete_run_queue))
 		    != NULL)
 			bioq_disksort(&softc->delete_queue, bp);
@@ -2152,6 +2171,7 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 	struct da_softc *softc;
 	struct ccb_scsiio *csio;
 	u_int32_t  priority;
+	da_ccb_state state;
 
 	softc = (struct da_softc *)periph->softc;
 	priority = done_ccb->ccb_h.pinfo.priority;
@@ -2159,7 +2179,8 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone\n"));
 
 	csio = &done_ccb->csio;
-	switch (csio->ccb_h.ccb_state & DA_CCB_TYPE_MASK) {
+	state = csio->ccb_h.ccb_state & DA_CCB_TYPE_MASK;
+	switch (state) {
 	case DA_CCB_BUFFER_IO:
 	case DA_CCB_DELETE:
 	{
@@ -2257,8 +2278,7 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 				  softc->outstanding_cmds);
 		}
 
-		if ((csio->ccb_h.ccb_state & DA_CCB_TYPE_MASK) ==
-		    DA_CCB_DELETE) {
+		if (state == DA_CCB_DELETE) {
 			while ((bp1 = bioq_takefirst(&softc->delete_run_queue))
 			    != NULL) {
 				bp1->bio_resid = bp->bio_resid;
@@ -2284,7 +2304,7 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 
 		rdcap = NULL;
 		rcaplong = NULL;
-		if (softc->state == DA_STATE_PROBE)
+		if (state == DA_CCB_PROBE)
 			rdcap =(struct scsi_read_capacity_data *)csio->data_ptr;
 		else
 			rcaplong = (struct scsi_read_capacity_data_long *)
@@ -2297,7 +2317,7 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 			u_int lbppbe;	/* LB per physical block exponent. */
 			u_int lalba;	/* Lowest aligned LBA. */
 
-			if (softc->state == DA_STATE_PROBE) {
+			if (state == DA_CCB_PROBE) {
 				block_size = scsi_4btoul(rdcap->length);
 				maxsector = scsi_4btoul(rdcap->addr);
 				lbppbe = 0;
@@ -2349,7 +2369,7 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 					  rcaplong, sizeof(*rcaplong));
 				if ((lalba & SRC16_LBPME_A)
 				 && softc->delete_method == DA_DELETE_NONE)
-					softc->delete_method = DA_DELETE_UNMAP;
+					dadeletemethodset(softc, DA_DELETE_UNMAP);
 				dp = &softc->params;
 				snprintf(announce_buf, sizeof(announce_buf),
 				        "%juMB (%ju %u byte sectors: %dH %dS/T "
@@ -2410,7 +2430,7 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 				 * If we tried READ CAPACITY(16) and failed,
 				 * fallback to READ CAPACITY(10).
 				 */
-				if ((softc->state == DA_STATE_PROBE2) &&
+				if ((state == DA_CCB_PROBE2) &&
 				    (softc->flags & DA_FLAG_CAN_RC16) &&
 				    (((csio->ccb_h.status & CAM_STATUS_MASK) ==
 					CAM_REQ_INVALID) ||
@@ -2490,7 +2510,8 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 		 * operation.
 		 */
 		xpt_release_ccb(done_ccb);
-		softc->state = DA_STATE_NORMAL;	
+		softc->state = DA_STATE_NORMAL;
+		daschedule(periph);
 		wakeup(&softc->disk->d_mediasize);
 		if ((softc->flags & DA_FLAG_PROBED) == 0) {
 			softc->flags |= DA_FLAG_PROBED;
@@ -2615,7 +2636,7 @@ damediapoll(void *arg)
 	struct cam_periph *periph = arg;
 	struct da_softc *softc = periph->softc;
 
-	if (softc->state == DA_STATE_NORMAL && !softc->tur) {
+	if (!softc->tur && softc->outstanding_cmds == 0) {
 		if (cam_periph_acquire(periph) == CAM_REQ_CMP) {
 			softc->tur = 1;
 			daschedule(periph);
diff --git a/sys/cam/scsi/scsi_enc_ses.c b/sys/cam/scsi/scsi_enc_ses.c
index 79e96db..c4a8c49 100644
--- a/sys/cam/scsi/scsi_enc_ses.c
+++ b/sys/cam/scsi/scsi_enc_ses.c
@@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 
+#include <sys/ctype.h>
 #include <sys/errno.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
@@ -1053,10 +1054,11 @@ ses_set_physpath(enc_softc_t *enc, enc_element_t *elm,
 {
 	struct ccb_dev_advinfo cdai;
 	ses_setphyspath_callback_args_t args;
-	int ret;
+	int i, ret;
 	struct sbuf sb;
 	uint8_t *devid, *elmaddr;
 	ses_element_t *elmpriv;
+	const char *c;
 
 	ret = EIO;
 	devid = NULL;
@@ -1099,7 +1101,13 @@ ses_set_physpath(enc_softc_t *enc, enc_element_t *elm,
 	elmpriv = elm->elm_private;
 	if (elmpriv->descr != NULL && elmpriv->descr_len > 0) {
 		sbuf_cat(&sb, "/elmdesc@");
-		sbuf_bcat(&sb, elmpriv->descr, elmpriv->descr_len);
+		for (i = 0, c = elmpriv->descr; i < elmpriv->descr_len;
+		    i++, c++) {
+			if (!isprint(*c) || isspace(*c) || *c == '/')
+				sbuf_putc(&sb, '_');
+			else
+				sbuf_putc(&sb, *c);
+		}
 	}
 	sbuf_finish(&sb);
 
diff --git a/sys/cddl/compat/opensolaris/sys/kmem.h b/sys/cddl/compat/opensolaris/sys/kmem.h
index 428badf..ba8b42d 100644
--- a/sys/cddl/compat/opensolaris/sys/kmem.h
+++ b/sys/cddl/compat/opensolaris/sys/kmem.h
@@ -45,7 +45,6 @@ MALLOC_DECLARE(M_SOLARIS);
 #define	KM_SLEEP		M_WAITOK
 #define	KM_PUSHPAGE		M_WAITOK
 #define	KM_NOSLEEP		M_NOWAIT
-#define	KM_ZERO			M_ZERO
 #define	KM_NODEBUG		M_NODUMP
 #define	KMC_NODEBUG		UMA_ZONE_NODUMP
 #define	KMC_NOTOUCH		0
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt.c
index ef3d0f4..885af95 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt.c
@@ -89,12 +89,13 @@ ddt_object_destroy(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
 	spa_t *spa = ddt->ddt_spa;
 	objset_t *os = ddt->ddt_os;
 	uint64_t *objectp = &ddt->ddt_object[type][class];
+	uint64_t count;
 	char name[DDT_NAMELEN];
 
 	ddt_object_name(ddt, type, class, name);
 
 	ASSERT(*objectp != 0);
-	ASSERT(ddt_object_count(ddt, type, class) == 0);
+	VERIFY(ddt_object_count(ddt, type, class, &count) == 0 && count == 0);
 	ASSERT(ddt_histogram_empty(&ddt->ddt_histogram[type][class]));
 	VERIFY(zap_remove(os, DMU_POOL_DIRECTORY_OBJECT, name, tx) == 0);
 	VERIFY(zap_remove(os, spa->spa_ddt_stat_object, name, tx) == 0);
@@ -109,6 +110,7 @@ ddt_object_load(ddt_t *ddt, enum ddt_type type, enum ddt_class class)
 {
 	ddt_object_t *ddo = &ddt->ddt_object_stats[type][class];
 	dmu_object_info_t doi;
+	uint64_t count;
 	char name[DDT_NAMELEN];
 	int error;
 
@@ -129,7 +131,11 @@ ddt_object_load(ddt_t *ddt, enum ddt_type type, enum ddt_class class)
 	 */
 	VERIFY(ddt_object_info(ddt, type, class, &doi) == 0);
 
-	ddo->ddo_count = ddt_object_count(ddt, type, class);
+	error = ddt_object_count(ddt, type, class, &count);
+	if (error)
+		return error;
+
+	ddo->ddo_count = count;
 	ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9;
 	ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size;
 
@@ -143,6 +149,7 @@ ddt_object_sync(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
 {
 	ddt_object_t *ddo = &ddt->ddt_object_stats[type][class];
 	dmu_object_info_t doi;
+	uint64_t count;
 	char name[DDT_NAMELEN];
 
 	ddt_object_name(ddt, type, class, name);
@@ -155,8 +162,9 @@ ddt_object_sync(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
 	 * Cache DDT statistics; this is the only time they'll change.
 	 */
 	VERIFY(ddt_object_info(ddt, type, class, &doi) == 0);
+	VERIFY(ddt_object_count(ddt, type, class, &count) == 0);
 
-	ddo->ddo_count = ddt_object_count(ddt, type, class);
+	ddo->ddo_count = count;
 	ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9;
 	ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size;
 }
@@ -213,13 +221,13 @@ ddt_object_walk(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
 	    ddt->ddt_object[type][class], dde, walk));
 }
 
-uint64_t
-ddt_object_count(ddt_t *ddt, enum ddt_type type, enum ddt_class class)
+int
+ddt_object_count(ddt_t *ddt, enum ddt_type type, enum ddt_class class, uint64_t *count)
 {
 	ASSERT(ddt_object_exists(ddt, type, class));
 
 	return (ddt_ops[type]->ddt_op_count(ddt->ddt_os,
-	    ddt->ddt_object[type][class]));
+	    ddt->ddt_object[type][class], count));
 }
 
 int
@@ -1079,11 +1087,13 @@ ddt_sync_table(ddt_t *ddt, dmu_tx_t *tx, uint64_t txg)
 	}
 
 	for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
-		uint64_t count = 0;
+		uint64_t add, count = 0;
 		for (enum ddt_class class = 0; class < DDT_CLASSES; class++) {
 			if (ddt_object_exists(ddt, type, class)) {
 				ddt_object_sync(ddt, type, class, tx);
-				count += ddt_object_count(ddt, type, class);
+				VERIFY(ddt_object_count(ddt, type, class,
+				    &add) == 0);
+				count += add;
 			}
 		}
 		for (enum ddt_class class = 0; class < DDT_CLASSES; class++) {
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt_zap.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt_zap.c
index 6812aa3..f42b4dd 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt_zap.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt_zap.c
@@ -133,14 +133,11 @@ ddt_zap_walk(objset_t *os, uint64_t object, ddt_entry_t *dde, uint64_t *walk)
 	return (error);
 }
 
-static uint64_t
-ddt_zap_count(objset_t *os, uint64_t object)
+static int
+ddt_zap_count(objset_t *os, uint64_t object, uint64_t *count)
 {
-	uint64_t count = 0;
-
-	VERIFY(zap_count(os, object, &count) == 0);
 
-	return (count);
+	return (zap_count(os, object, count));
 }
 
 const ddt_ops_t ddt_zap_ops = {
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c
index 4392ea8..8da3f0e 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c
@@ -3792,9 +3792,8 @@ spa_generate_rootconf(const char *name)
 	/*
 	 * Multi-vdev root pool configuration discovery is not supported yet.
 	 */
-	nchildren = 0;
-	VERIFY(nvlist_lookup_uint64(best_cfg, ZPOOL_CONFIG_VDEV_CHILDREN,
-	    &nchildren) == 0);
+	nchildren = 1;
+	nvlist_lookup_uint64(best_cfg, ZPOOL_CONFIG_VDEV_CHILDREN, &nchildren);
 	holes = NULL;
 	nvlist_lookup_uint64_array(best_cfg, ZPOOL_CONFIG_HOLE_ARRAY,
 	    &holes, &nholes);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/ddt.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/ddt.h
index 405622b..5d9747b 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/ddt.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/ddt.h
@@ -163,7 +163,7 @@ typedef struct ddt_ops {
 	    dmu_tx_t *tx);
 	int (*ddt_op_walk)(objset_t *os, uint64_t object, ddt_entry_t *dde,
 	    uint64_t *walk);
-	uint64_t (*ddt_op_count)(objset_t *os, uint64_t object);
+	int (*ddt_op_count)(objset_t *os, uint64_t object, uint64_t *count);
 } ddt_ops_t;
 
 #define	DDT_NAMELEN	80
@@ -172,8 +172,8 @@ extern void ddt_object_name(ddt_t *ddt, enum ddt_type type,
     enum ddt_class cls, char *name);
 extern int ddt_object_walk(ddt_t *ddt, enum ddt_type type,
     enum ddt_class cls, uint64_t *walk, ddt_entry_t *dde);
-extern uint64_t ddt_object_count(ddt_t *ddt, enum ddt_type type,
-    enum ddt_class cls);
+extern int ddt_object_count(ddt_t *ddt, enum ddt_type type,
+    enum ddt_class class, uint64_t *count);
 extern int ddt_object_info(ddt_t *ddt, enum ddt_type type,
     enum ddt_class cls, dmu_object_info_t *);
 extern boolean_t ddt_object_exists(ddt_t *ddt, enum ddt_type type,
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c
index dac619b..ce0579c 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c
@@ -21,7 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Portions Copyright 2011 Martin Matuska <mm@FreeBSD.org>
- * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2013 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -33,7 +33,76 @@
 #include <sys/callb.h>
 
 /*
- * Pool-wide transaction groups.
+ * ZFS Transaction Groups
+ * ----------------------
+ *
+ * ZFS transaction groups are, as the name implies, groups of transactions
+ * that act on persistent state. ZFS asserts consistency at the granularity of
+ * these transaction groups. Each successive transaction group (txg) is
+ * assigned a 64-bit consecutive identifier. There are three active
+ * transaction group states: open, quiescing, or syncing. At any given time,
+ * there may be an active txg associated with each state; each active txg may
+ * either be processing, or blocked waiting to enter the next state. There may
+ * be up to three active txgs, and there is always a txg in the open state
+ * (though it may be blocked waiting to enter the quiescing state). In broad
+ * strokes, transactions — operations that change in-memory structures — are
+ * accepted into the txg in the open state, and are completed while the txg is
+ * in the open or quiescing states. The accumulated changes are written to
+ * disk in the syncing state.
+ *
+ * Open
+ *
+ * When a new txg becomes active, it first enters the open state. New
+ * transactions — updates to in-memory structures — are assigned to the
+ * currently open txg. There is always a txg in the open state so that ZFS can
+ * accept new changes (though the txg may refuse new changes if it has hit
+ * some limit). ZFS advances the open txg to the next state for a variety of
+ * reasons such as it hitting a time or size threshold, or the execution of an
+ * administrative action that must be completed in the syncing state.
+ *
+ * Quiescing
+ *
+ * After a txg exits the open state, it enters the quiescing state. The
+ * quiescing state is intended to provide a buffer between accepting new
+ * transactions in the open state and writing them out to stable storage in
+ * the syncing state. While quiescing, transactions can continue their
+ * operation without delaying either of the other states. Typically, a txg is
+ * in the quiescing state very briefly since the operations are bounded by
+ * software latencies rather than, say, slower I/O latencies. After all
+ * transactions complete, the txg is ready to enter the next state.
+ *
+ * Syncing
+ *
+ * In the syncing state, the in-memory state built up during the open and (to
+ * a lesser degree) the quiescing states is written to stable storage. The
+ * process of writing out modified data can, in turn modify more data. For
+ * example when we write new blocks, we need to allocate space for them; those
+ * allocations modify metadata (space maps)... which themselves must be
+ * written to stable storage. During the sync state, ZFS iterates, writing out
+ * data until it converges and all in-memory changes have been written out.
+ * The first such pass is the largest as it encompasses all the modified user
+ * data (as opposed to filesystem metadata). Subsequent passes typically have
+ * far less data to write as they consist exclusively of filesystem metadata.
+ *
+ * To ensure convergence, after a certain number of passes ZFS begins
+ * overwriting locations on stable storage that had been allocated earlier in
+ * the syncing state (and subsequently freed). ZFS usually allocates new
+ * blocks to optimize for large, continuous, writes. For the syncing state to
+ * converge however it must complete a pass where no new blocks are allocated
+ * since each allocation requires a modification of persistent metadata.
+ * Further, to hasten convergence, after a prescribed number of passes, ZFS
+ * also defers frees, and stops compressing.
+ *
+ * In addition to writing out user data, we must also execute synctasks during
+ * the syncing context. A synctask is the mechanism by which some
+ * administrative activities work such as creating and destroying snapshots or
+ * datasets. Note that when a synctask is initiated it enters the open txg,
+ * and ZFS then pushes that txg as quickly as possible to completion of the
+ * syncing state in order to reduce the latency of the administrative
+ * activity. To complete the syncing state, ZFS writes out a new uberblock,
+ * the root of the tree of blocks that comprise all state stored on the ZFS
+ * pool. Finally, if there is a quiesced txg waiting, we signal that it can
+ * now transition to the syncing state.
  */
 
 static void txg_sync_thread(void *arg);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
index e4157df..6fa9cea 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
@@ -5536,6 +5536,7 @@ static moduledata_t zfs_mod = {
 	0
 };
 DECLARE_MODULE(zfsctrl, zfs_mod, SI_SUB_VFS, SI_ORDER_ANY);
+MODULE_VERSION(zfsctrl, 1);
 MODULE_DEPEND(zfsctrl, opensolaris, 1, 1, 1);
 MODULE_DEPEND(zfsctrl, krpc, 1, 1, 1);
 MODULE_DEPEND(zfsctrl, acl_nfs4, 1, 1, 1);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
index 72c22dc..7f3f7a1 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
@@ -2025,6 +2025,8 @@ zfs_vget(vfs_t *vfsp, ino_t ino, int flags, vnode_t **vpp)
 		err = zfs_vnode_lock(*vpp, flags);
 	if (err != 0)
 		*vpp = NULL;
+	else
+		(*vpp)->v_hash = ino;
 	return (err);
 }
 
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
index 354deaa..9438266 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
@@ -323,7 +323,7 @@ zfs_ioctl(vnode_t *vp, u_long com, intptr_t data, int flag, cred_t *cred,
 }
 
 static vm_page_t
-page_lookup(vnode_t *vp, int64_t start, int64_t off, int64_t nbytes)
+page_busy(vnode_t *vp, int64_t start, int64_t off, int64_t nbytes)
 {
 	vm_object_t obj;
 	vm_page_t pp;
@@ -332,32 +332,30 @@ page_lookup(vnode_t *vp, int64_t start, int64_t off, int64_t nbytes)
 	VM_OBJECT_LOCK_ASSERT(obj, MA_OWNED);
 
 	for (;;) {
-		pp = vm_radix_lookup(&obj->rtree, OFF_TO_IDX(start),
-		    VM_RADIX_ANY);
-		if (pp != NULL) {
-			if (vm_page_is_valid(pp, (vm_offset_t)off, nbytes)) {
-				if ((pp->oflags & VPO_BUSY) != 0) {
-					/*
-					 * Reference the page before unlocking
-					 * and sleeping so that the page
-					 * daemon is less likely to reclaim it.
-					 */
-					vm_page_reference(pp);
-					vm_page_sleep(pp, "zfsmwb");
-					continue;
-				}
-				vm_page_busy(pp);
-				vm_page_undirty(pp);
-			} else {
-				if (obj->cached_page_count != 0 &&
-				    (pp->flags & PG_CACHED) != 0) {
-					mtx_lock(&vm_page_queue_free_mtx);
-					if (pp->object == obj)
-						vm_page_cache_free(pp);
-					mtx_unlock(&vm_page_queue_free_mtx);
-				}
-				pp = NULL;
+		if ((pp = vm_page_lookup(obj, OFF_TO_IDX(start))) != NULL &&
+		    pp->valid) {
+			if ((pp->oflags & VPO_BUSY) != 0) {
+				/*
+				 * Reference the page before unlocking and
+				 * sleeping so that the page daemon is less
+				 * likely to reclaim it.
+				 */
+				vm_page_reference(pp);
+				vm_page_sleep(pp, "zfsmwb");
+				continue;
 			}
+		} else {
+			pp = vm_page_alloc(obj, OFF_TO_IDX(start),
+			    VM_ALLOC_SYSTEM | VM_ALLOC_IFCACHED |
+			    VM_ALLOC_NOBUSY);
+		}
+
+		if (pp != NULL) {
+			ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL);
+			vm_object_pip_add(obj, 1);
+			vm_page_io_start(pp);
+			pmap_remove_write(pp);
+			vm_page_clear_dirty(pp, off, nbytes);
 		}
 		break;
 	}
@@ -365,10 +363,55 @@ page_lookup(vnode_t *vp, int64_t start, int64_t off, int64_t nbytes)
 }
 
 static void
-page_unlock(vm_page_t pp)
+page_unbusy(vm_page_t pp)
 {
 
-	vm_page_wakeup(pp);
+	vm_page_io_finish(pp);
+	vm_object_pip_subtract(pp->object, 1);
+}
+
+static vm_page_t
+page_hold(vnode_t *vp, int64_t start)
+{
+	vm_object_t obj;
+	vm_page_t pp;
+
+	obj = vp->v_object;
+	VM_OBJECT_LOCK_ASSERT(obj, MA_OWNED);
+
+	for (;;) {
+		if ((pp = vm_page_lookup(obj, OFF_TO_IDX(start))) != NULL &&
+		    pp->valid) {
+			if ((pp->oflags & VPO_BUSY) != 0) {
+				/*
+				 * Reference the page before unlocking and
+				 * sleeping so that the page daemon is less
+				 * likely to reclaim it.
+				 */
+				vm_page_reference(pp);
+				vm_page_sleep(pp, "zfsmwb");
+				continue;
+			}
+
+			ASSERT3U(pp->valid, ==, VM_PAGE_BITS_ALL);
+			vm_page_lock(pp);
+			vm_page_hold(pp);
+			vm_page_unlock(pp);
+
+		} else
+			pp = NULL;
+		break;
+	}
+	return (pp);
+}
+
+static void
+page_unhold(vm_page_t pp)
+{
+
+	vm_page_lock(pp);
+	vm_page_unhold(pp);
+	vm_page_unlock(pp);
 }
 
 static caddr_t
@@ -399,6 +442,7 @@ update_pages(vnode_t *vp, int64_t start, int len, objset_t *os, uint64_t oid,
 {
 	vm_object_t obj;
 	struct sf_buf *sf;
+	caddr_t va;
 	int off;
 
 	ASSERT(vp->v_mount != NULL);
@@ -409,27 +453,44 @@ update_pages(vnode_t *vp, int64_t start, int len, objset_t *os, uint64_t oid,
 	VM_OBJECT_LOCK(obj);
 	for (start &= PAGEMASK; len > 0; start += PAGESIZE) {
 		vm_page_t pp;
-		int nbytes = MIN(PAGESIZE - off, len);
+		int nbytes = imin(PAGESIZE - off, len);
+
+		if (segflg == UIO_NOCOPY) {
+			pp = vm_page_lookup(obj, OFF_TO_IDX(start));
+			KASSERT(pp != NULL,
+			    ("zfs update_pages: NULL page in putpages case"));
+			KASSERT(off == 0,
+			    ("zfs update_pages: unaligned data in putpages case"));
+			KASSERT(pp->valid == VM_PAGE_BITS_ALL,
+			    ("zfs update_pages: invalid page in putpages case"));
+			KASSERT(pp->busy > 0,
+			    ("zfs update_pages: unbusy page in putpages case"));
+			KASSERT(!pmap_page_is_write_mapped(pp),
+			    ("zfs update_pages: writable page in putpages case"));
+			VM_OBJECT_UNLOCK(obj);
 
-		if ((pp = page_lookup(vp, start, off, nbytes)) != NULL) {
-			caddr_t va;
+			va = zfs_map_page(pp, &sf);
+			(void) dmu_write(os, oid, start, nbytes, va, tx);
+			zfs_unmap_page(sf);
 
+			VM_OBJECT_LOCK(obj);
+			vm_page_undirty(pp);
+		} else if ((pp = page_busy(vp, start, off, nbytes)) != NULL) {
 			VM_OBJECT_UNLOCK(obj);
+
 			va = zfs_map_page(pp, &sf);
-			if (segflg == UIO_NOCOPY) {
-				(void) dmu_write(os, oid, start+off, nbytes,
-				    va+off, tx);
-			} else {
-				(void) dmu_read(os, oid, start+off, nbytes,
-				    va+off, DMU_READ_PREFETCH);
-			}
+			(void) dmu_read(os, oid, start+off, nbytes,
+			    va+off, DMU_READ_PREFETCH);;
 			zfs_unmap_page(sf);
+
 			VM_OBJECT_LOCK(obj);
-			page_unlock(pp);
+			page_unbusy(pp);
 		}
 		len -= nbytes;
 		off = 0;
 	}
+	if (segflg != UIO_NOCOPY)
+		vm_object_pip_wakeupn(obj, 0);
 	VM_OBJECT_UNLOCK(obj);
 }
 
@@ -531,7 +592,7 @@ mappedread(vnode_t *vp, int nbytes, uio_t *uio)
 		vm_page_t pp;
 		uint64_t bytes = MIN(PAGESIZE - off, len);
 
-		if (pp = page_lookup(vp, start, off, bytes)) {
+		if (pp = page_hold(vp, start)) {
 			struct sf_buf *sf;
 			caddr_t va;
 
@@ -540,7 +601,7 @@ mappedread(vnode_t *vp, int nbytes, uio_t *uio)
 			error = uiomove(va + off, bytes, UIO_READ, uio);
 			zfs_unmap_page(sf);
 			VM_OBJECT_LOCK(obj);
-			page_unlock(pp);
+			page_unhold(pp);
 		} else {
 			VM_OBJECT_UNLOCK(obj);
 			error = dmu_read_uio(os, zp->z_id, uio, bytes);
diff --git a/sys/cddl/dev/dtrace/powerpc/dtrace_asm.S b/sys/cddl/dev/dtrace/powerpc/dtrace_asm.S
index 9ff83db..8537e3c 100644
--- a/sys/cddl/dev/dtrace/powerpc/dtrace_asm.S
+++ b/sys/cddl/dev/dtrace/powerpc/dtrace_asm.S
@@ -19,6 +19,8 @@
  *
  * CDDL HEADER END
  *
+ * Portions Copyright 2012,2013 Justin Hibbits <jhibbits@freebsd.org>
+ *
  * $FreeBSD$
  */
 /*
@@ -114,48 +116,6 @@ ASENTRY_NOPROF(dtrace_fulword)
 END(dtrace_fulword)
 
 /*
-uint8_t
-dtrace_fuword8_nocheck(void *addr)
-*/
-ASENTRY_NOPROF(dtrace_fuword8_nocheck)
-	lbz	%r3,0(%r3)
-	blr
-END(dtrace_fuword8_nocheck)
-
-/*
-uint16_t
-dtrace_fuword16_nocheck(void *addr)
-*/
-ASENTRY_NOPROF(dtrace_fuword16_nocheck)
-	lhz	%r3,0(%r3)
-	blr
-END(dtrace_fuword16_nocheck)
-
-/*
-uint32_t
-dtrace_fuword32_nocheck(void *addr)
-*/
-ASENTRY_NOPROF(dtrace_fuword32_nocheck)
-	lwz	%r3,0(%r3)
-	blr
-END(dtrace_fuword32_nocheck)
-
-/*
-uint64_t
-dtrace_fuword64_nocheck(void *addr)
-*/
-ASENTRY_NOPROF(dtrace_fuword64_nocheck)
-#if defined(__powerpc64__)
-	ld	%r3,0(%r3)
-#else
-	lwz	%r5,0(%r3)
-	lwz	%r4,4(%r3)
-	mr	%r3,%r5
-#endif
-	blr
-END(dtrace_fuword64_nocheck)
-
-/*
 XXX: unoptimized
 void
 dtrace_copy(uintptr_t src, uintptr_t dest, size_t size)
diff --git a/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c b/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c
index a697816..efbca6f 100644
--- a/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c
+++ b/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c
@@ -19,6 +19,8 @@
  *
  * CDDL HEADER END
  *
+ * Portions Copyright 2012,2013 Justin Hibbits <jhibbits@freebsd.org>
+ *
  * $FreeBSD$
  */
 /*
@@ -45,11 +47,6 @@
 
 #include "regset.h"
 
-uint8_t dtrace_fuword8_nocheck(void *);
-uint16_t dtrace_fuword16_nocheck(void *);
-uint32_t dtrace_fuword32_nocheck(void *);
-uint64_t dtrace_fuword64_nocheck(void *);
-
 /* Offset to the LR Save word (ppc32) */
 #define RETURN_OFFSET	4
 #define RETURN_OFFSET64	8
@@ -462,31 +459,63 @@ dtrace_copyin(uintptr_t uaddr, uintptr_t kaddr, size_t size,
     volatile uint16_t *flags)
 {
 	if (dtrace_copycheck(uaddr, kaddr, size))
-		dtrace_copy(uaddr, kaddr, size);
+		if (copyin((const void *)uaddr, (void *)kaddr, size)) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
+		}
 }
 
 void
 dtrace_copyout(uintptr_t kaddr, uintptr_t uaddr, size_t size,
     volatile uint16_t *flags)
 {
-	if (dtrace_copycheck(uaddr, kaddr, size))
-		dtrace_copy(kaddr, uaddr, size);
+	if (dtrace_copycheck(uaddr, kaddr, size)) {
+		if (copyout((const void *)kaddr, (void *)uaddr, size)) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
+		}
+	}
 }
 
 void
 dtrace_copyinstr(uintptr_t uaddr, uintptr_t kaddr, size_t size,
     volatile uint16_t *flags)
 {
-	if (dtrace_copycheck(uaddr, kaddr, size))
-		dtrace_copystr(uaddr, kaddr, size, flags);
+	size_t actual;
+	int    error;
+
+	if (dtrace_copycheck(uaddr, kaddr, size)) {
+		error = copyinstr((const void *)uaddr, (void *)kaddr,
+		    size, &actual);
+		
+		/* ENAMETOOLONG is not a fault condition. */
+		if (error && error != ENAMETOOLONG) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
+		}
+	}
 }
 
+/*
+ * The bulk of this function could be replaced to match dtrace_copyinstr() 
+ * if we ever implement a copyoutstr().
+ */
 void
 dtrace_copyoutstr(uintptr_t kaddr, uintptr_t uaddr, size_t size,
     volatile uint16_t *flags)
 {
-	if (dtrace_copycheck(uaddr, kaddr, size))
-		dtrace_copystr(kaddr, uaddr, size, flags);
+	size_t len;
+
+	if (dtrace_copycheck(uaddr, kaddr, size)) {
+		len = strlen((const char *)kaddr);
+		if (len > size)
+			len = size;
+
+		if (copyout((const void *)kaddr, (void *)uaddr, len)) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
+		}
+	}
 }
 
 uint8_t
@@ -497,18 +526,21 @@ dtrace_fuword8(void *uaddr)
 		cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
 		return (0);
 	}
-	return (dtrace_fuword8_nocheck(uaddr));
+	return (fubyte(uaddr));
 }
 
 uint16_t
 dtrace_fuword16(void *uaddr)
 {
-	if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) {
-		DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
-		cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
-		return (0);
+	uint16_t ret = 0;
+
+	if (dtrace_copycheck((uintptr_t)uaddr, (uintptr_t)&ret, sizeof(ret))) {
+		if (copyin((const void *)uaddr, (void *)&ret, sizeof(ret))) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
+		}
 	}
-	return (dtrace_fuword16_nocheck(uaddr));
+	return ret;
 }
 
 uint32_t
@@ -519,16 +551,19 @@ dtrace_fuword32(void *uaddr)
 		cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
 		return (0);
 	}
-	return (dtrace_fuword32_nocheck(uaddr));
+	return (fuword32(uaddr));
 }
 
 uint64_t
 dtrace_fuword64(void *uaddr)
 {
-	if ((uintptr_t)uaddr > VM_MAXUSER_ADDRESS) {
-		DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
-		cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
-		return (0);
+	uint64_t ret = 0;
+
+	if (dtrace_copycheck((uintptr_t)uaddr, (uintptr_t)&ret, sizeof(ret))) {
+		if (copyin((const void *)uaddr, (void *)&ret, sizeof(ret))) {
+			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
+			cpu_core[curcpu].cpuc_dtrace_illval = (uintptr_t)uaddr;
+		}
 	}
-	return (dtrace_fuword64_nocheck(uaddr));
+	return ret;
 }
diff --git a/sys/compat/linprocfs/linprocfs.c b/sys/compat/linprocfs/linprocfs.c
index 31cda7d..d86da04 100644
--- a/sys/compat/linprocfs/linprocfs.c
+++ b/sys/compat/linprocfs/linprocfs.c
@@ -101,13 +101,9 @@ __FBSDID("$FreeBSD$");
 #include <compat/freebsd32/freebsd32_util.h>
 #endif
 
-#ifdef COMPAT_LINUX32				/* XXX */
-#include <machine/../linux32/linux.h>
-#else
-#include <machine/../linux/linux.h>
-#endif
 #include <compat/linux/linux_ioctl.h>
 #include <compat/linux/linux_mib.h>
+#include <compat/linux/linux_misc.h>
 #include <compat/linux/linux_util.h>
 #include <fs/pseudofs/pseudofs.h>
 #include <fs/procfs/procfs.h>
diff --git a/sys/compat/linux/linux_emul.c b/sys/compat/linux/linux_emul.c
index 6a1098d..61156ba 100644
--- a/sys/compat/linux/linux_emul.c
+++ b/sys/compat/linux/linux_emul.c
@@ -58,6 +58,7 @@ __FBSDID("$FreeBSD$");
 #include <compat/linux/linux_dtrace.h>
 #include <compat/linux/linux_emul.h>
 #include <compat/linux/linux_futex.h>
+#include <compat/linux/linux_misc.h>
 
 /**
  * Special DTrace provider for the linuxulator.
diff --git a/sys/compat/linux/linux_file.c b/sys/compat/linux/linux_file.c
index df13277..0318a5c 100644
--- a/sys/compat/linux/linux_file.c
+++ b/sys/compat/linux/linux_file.c
@@ -66,6 +66,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/../linux/linux.h>
 #include <machine/../linux/linux_proto.h>
 #endif
+#include <compat/linux/linux_misc.h>
 #include <compat/linux/linux_util.h>
 #include <compat/linux/linux_file.h>
 
diff --git a/sys/compat/linux/linux_file.h b/sys/compat/linux/linux_file.h
index d0d95ac..2d3106f 100644
--- a/sys/compat/linux/linux_file.h
+++ b/sys/compat/linux/linux_file.h
@@ -35,4 +35,23 @@
 #define	LINUX_AT_REMOVEDIR		0x200
 #define	LINUX_AT_SYMLINK_FOLLOW		0x400
 
+/*
+ * posix_fadvise advice
+ */
+#define	LINUX_POSIX_FADV_NORMAL		0
+#define	LINUX_POSIX_FADV_RANDOM		1
+#define	LINUX_POSIX_FADV_SEQUENTIAL    	2
+#define	LINUX_POSIX_FADV_WILLNEED      	3
+#define	LINUX_POSIX_FADV_DONTNEED      	4
+#define	LINUX_POSIX_FADV_NOREUSE       	5
+
+/*
+ * mount flags
+ */
+#define	LINUX_MS_RDONLY		0x0001
+#define	LINUX_MS_NOSUID		0x0002
+#define	LINUX_MS_NODEV		0x0004
+#define	LINUX_MS_NOEXEC		0x0008
+#define	LINUX_MS_REMOUNT	0x0020
+
 #endif	/* !_LINUX_FILE_H_ */
diff --git a/sys/compat/linux/linux_fork.c b/sys/compat/linux/linux_fork.c
index f71063c..2103636 100644
--- a/sys/compat/linux/linux_fork.c
+++ b/sys/compat/linux/linux_fork.c
@@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$");
 #include <compat/linux/linux_dtrace.h>
 #include <compat/linux/linux_signal.h>
 #include <compat/linux/linux_emul.h>
+#include <compat/linux/linux_misc.h>
 
 /* DTrace init */
 LIN_SDT_PROVIDER_DECLARE(LINUX_DTRACE);
diff --git a/sys/compat/linux/linux_futex.c b/sys/compat/linux/linux_futex.c
index c87fd00..c1531d0 100644
--- a/sys/compat/linux/linux_futex.c
+++ b/sys/compat/linux/linux_futex.c
@@ -1203,7 +1203,7 @@ release_futexes(struct proc *p)
 
 		if (entry != pending)
 			if (handle_futex_death(p,
-			    (uint32_t *)entry + futex_offset, pi)) {
+			    (uint32_t *)((caddr_t)entry + futex_offset), pi)) {
 				LIN_SDT_PROBE0(futex, release_futexes, return);
 				return;
 			}
@@ -1222,7 +1222,7 @@ release_futexes(struct proc *p)
 	}
 
 	if (pending)
-		handle_futex_death(p, (uint32_t *)pending + futex_offset, pip);
+		handle_futex_death(p, (uint32_t *)((caddr_t)pending + futex_offset), pip);
 
 	LIN_SDT_PROBE0(futex, release_futexes, return);
 }
diff --git a/sys/compat/linux/linux_ioctl.h b/sys/compat/linux/linux_ioctl.h
index c075231..aa7650b 100644
--- a/sys/compat/linux/linux_ioctl.h
+++ b/sys/compat/linux/linux_ioctl.h
@@ -748,4 +748,20 @@ int		 linux_ifname(struct ifnet *, char *, size_t);
 #define	FBSD_LUSB_MAX			0xffff
 #define	FBSD_LUSB_MIN			0xffe0
 
+/*
+ * Pluggable ioctl handlers
+ */
+struct linux_ioctl_args;
+struct thread;
+
+typedef int linux_ioctl_function_t(struct thread *, struct linux_ioctl_args *);
+
+struct linux_ioctl_handler {
+	linux_ioctl_function_t *func;
+	int	low, high;
+};
+
+int	linux_ioctl_register_handler(struct linux_ioctl_handler *h);
+int	linux_ioctl_unregister_handler(struct linux_ioctl_handler *h);
+
 #endif /* !_LINUX_IOCTL_H_ */
diff --git a/sys/compat/linux/linux_ipc.h b/sys/compat/linux/linux_ipc.h
index 4c01342..f1531ba 100644
--- a/sys/compat/linux/linux_ipc.h
+++ b/sys/compat/linux/linux_ipc.h
@@ -32,6 +32,48 @@
 #define _LINUX_IPC_H_
 
 /*
+ * SystemV IPC defines
+ */
+#define	LINUX_SEMOP		1
+#define	LINUX_SEMGET		2
+#define	LINUX_SEMCTL		3
+#define	LINUX_MSGSND		11
+#define	LINUX_MSGRCV		12
+#define	LINUX_MSGGET		13
+#define	LINUX_MSGCTL		14
+#define	LINUX_SHMAT		21
+#define	LINUX_SHMDT		22
+#define	LINUX_SHMGET		23
+#define	LINUX_SHMCTL		24
+
+#define	LINUX_IPC_RMID		0
+#define	LINUX_IPC_SET		1
+#define	LINUX_IPC_STAT		2
+#define	LINUX_IPC_INFO		3
+
+#define	LINUX_MSG_INFO	12
+
+#define	LINUX_SHM_LOCK		11
+#define	LINUX_SHM_UNLOCK	12
+#define	LINUX_SHM_STAT		13
+#define	LINUX_SHM_INFO		14
+
+#define	LINUX_SHM_RDONLY	0x1000
+#define	LINUX_SHM_RND		0x2000
+#define	LINUX_SHM_REMAP		0x4000
+
+/* semctl commands */
+#define	LINUX_GETPID		11
+#define	LINUX_GETVAL		12
+#define	LINUX_GETALL		13
+#define	LINUX_GETNCNT		14
+#define	LINUX_GETZCNT		15
+#define	LINUX_SETVAL		16
+#define	LINUX_SETALL		17
+#define	LINUX_SEM_STAT		18
+#define	LINUX_SEM_INFO		19
+
+/*
  * Version flags for semctl, msgctl, and shmctl commands
  * These are passed as bitflags or-ed with the actual command
  */
@@ -135,8 +177,6 @@ int linux_shmctl(struct thread *, struct linux_shmctl_args *);
 int linux_shmdt(struct thread *, struct linux_shmdt_args *);
 int linux_shmget(struct thread *, struct linux_shmget_args *);
 
-#define	LINUX_MSG_INFO	12
-
 #endif	/* __i386__ || __amd64__ */
 
 #endif /* _LINUX_IPC_H_ */
diff --git a/sys/compat/linux/linux_mib.c b/sys/compat/linux/linux_mib.c
index 0856662..323a38f 100644
--- a/sys/compat/linux/linux_mib.c
+++ b/sys/compat/linux/linux_mib.c
@@ -52,6 +52,7 @@ __FBSDID("$FreeBSD$");
 #endif
 #include <compat/linux/linux_dtrace.h>
 #include <compat/linux/linux_mib.h>
+#include <compat/linux/linux_misc.h>
 
 /* DTrace init */
 LIN_SDT_PROVIDER_DECLARE(LINUX_DTRACE);
diff --git a/sys/compat/linux/linux_misc.h b/sys/compat/linux/linux_misc.h
index b771825..154d78f 100644
--- a/sys/compat/linux/linux_misc.h
+++ b/sys/compat/linux/linux_misc.h
@@ -31,6 +31,14 @@
 #ifndef _LINUX_MISC_H_
 #define	_LINUX_MISC_H_
 
+/*
+ * Miscellaneous
+ */
+#define	LINUX_NAME_MAX		255
+#define	LINUX_MAX_UTSNAME	65
+
+#define	LINUX_CTL_MAXNAME	10
+
 /* defines for prctl */
 #define	LINUX_PR_SET_PDEATHSIG  1	/* Second arg is a signal. */
 #define	LINUX_PR_GET_PDEATHSIG  2	/*
@@ -67,11 +75,51 @@ extern const char *linux_platform;
 #define	__LINUX_NPXCW__		0x37f
 #endif
 
+#define	LINUX_CLONE_VM			0x00000100
+#define	LINUX_CLONE_FS			0x00000200
+#define	LINUX_CLONE_FILES		0x00000400
+#define	LINUX_CLONE_SIGHAND		0x00000800
+#define	LINUX_CLONE_PID			0x00001000	/* No longer exist in Linux */
+#define	LINUX_CLONE_VFORK		0x00004000
+#define	LINUX_CLONE_PARENT		0x00008000
+#define	LINUX_CLONE_THREAD		0x00010000
+#define	LINUX_CLONE_SETTLS		0x00080000
+#define	LINUX_CLONE_PARENT_SETTID	0x00100000
+#define	LINUX_CLONE_CHILD_CLEARTID	0x00200000
+#define	LINUX_CLONE_CHILD_SETTID	0x01000000
+
+#define	LINUX_THREADING_FLAGS					\
+	(LINUX_CLONE_VM | LINUX_CLONE_FS | LINUX_CLONE_FILES |	\
+	LINUX_CLONE_SIGHAND | LINUX_CLONE_THREAD)
+
+/* Scheduling policies */
+#define	LINUX_SCHED_OTHER	0
+#define	LINUX_SCHED_FIFO	1
+#define	LINUX_SCHED_RR		2
+
+struct l_new_utsname {
+	char	sysname[LINUX_MAX_UTSNAME];
+	char	nodename[LINUX_MAX_UTSNAME];
+	char	release[LINUX_MAX_UTSNAME];
+	char	version[LINUX_MAX_UTSNAME];
+	char	machine[LINUX_MAX_UTSNAME];
+	char	domainname[LINUX_MAX_UTSNAME];
+};
+
+#define	LINUX_CLOCK_REALTIME		0
+#define	LINUX_CLOCK_MONOTONIC		1
+#define	LINUX_CLOCK_PROCESS_CPUTIME_ID	2
+#define	LINUX_CLOCK_THREAD_CPUTIME_ID	3
+#define	LINUX_CLOCK_REALTIME_HR		4
+#define	LINUX_CLOCK_MONOTONIC_HR	5
+
 extern int stclohz;
 
 #define __WCLONE 0x80000000
 
 int linux_common_wait(struct thread *td, int pid, int *status,
 			int options, struct rusage *ru);
+int linux_set_upcall_kse(struct thread *td, register_t stack);
+int linux_set_cloned_tls(struct thread *td, void *desc);
 
 #endif	/* _LINUX_MISC_H_ */
diff --git a/sys/compat/linux/linux_signal.h b/sys/compat/linux/linux_signal.h
index 939009f..426cf43 100644
--- a/sys/compat/linux/linux_signal.h
+++ b/sys/compat/linux/linux_signal.h
@@ -33,6 +33,9 @@
 
 #define	LINUX_SI_TKILL		-6;
 
+extern int bsd_to_linux_signal[];
+extern int linux_to_bsd_signal[];
+
 void linux_to_bsd_sigset(l_sigset_t *, sigset_t *);
 void bsd_to_linux_sigset(sigset_t *, l_sigset_t *);
 int linux_do_sigaction(struct thread *, int, l_sigaction_t *, l_sigaction_t *);
diff --git a/sys/compat/linux/linux_socket.c b/sys/compat/linux/linux_socket.c
index 7271a18..cd288f8 100644
--- a/sys/compat/linux/linux_socket.c
+++ b/sys/compat/linux/linux_socket.c
@@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$");
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
 #include <netinet/ip.h>
+#include <netinet/tcp.h>
 #ifdef INET6
 #include <netinet/ip6.h>
 #include <netinet6/ip6_var.h>
@@ -326,6 +327,27 @@ linux_to_bsd_so_sockopt(int opt)
 }
 
 static int
+linux_to_bsd_tcp_sockopt(int opt)
+{
+
+	switch (opt) {
+	case LINUX_TCP_NODELAY:
+		return (TCP_NODELAY);
+	case LINUX_TCP_MAXSEG:
+		return (TCP_MAXSEG);
+	case LINUX_TCP_KEEPIDLE:
+		return (TCP_KEEPIDLE);
+	case LINUX_TCP_KEEPINTVL:
+		return (TCP_KEEPINTVL);
+	case LINUX_TCP_KEEPCNT:
+		return (TCP_KEEPCNT);
+	case LINUX_TCP_MD5SIG:
+		return (TCP_MD5SIG);
+	}
+	return (-1);
+}
+
+static int
 linux_to_bsd_msg_flags(int flags)
 {
 	int ret_flags = 0;
@@ -1496,8 +1518,7 @@ linux_setsockopt(struct thread *td, struct linux_setsockopt_args *args)
 		name = linux_to_bsd_ip_sockopt(args->optname);
 		break;
 	case IPPROTO_TCP:
-		/* Linux TCP option values match BSD's */
-		name = args->optname;
+		name = linux_to_bsd_tcp_sockopt(args->optname);
 		break;
 	default:
 		name = -1;
@@ -1591,8 +1612,7 @@ linux_getsockopt(struct thread *td, struct linux_getsockopt_args *args)
 		name = linux_to_bsd_ip_sockopt(args->optname);
 		break;
 	case IPPROTO_TCP:
-		/* Linux TCP option values match BSD's */
-		name = args->optname;
+		name = linux_to_bsd_tcp_sockopt(args->optname);
 		break;
 	default:
 		name = -1;
diff --git a/sys/compat/linux/linux_socket.h b/sys/compat/linux/linux_socket.h
index 1fe53ed..e6efadb 100644
--- a/sys/compat/linux/linux_socket.h
+++ b/sys/compat/linux/linux_socket.h
@@ -116,4 +116,44 @@ struct l_ucred {
 	uint32_t	gid;
 };
 
+/* Operations for socketcall */
+
+#define	LINUX_SOCKET 		1
+#define	LINUX_BIND		2
+#define	LINUX_CONNECT 		3
+#define	LINUX_LISTEN 		4
+#define	LINUX_ACCEPT 		5
+#define	LINUX_GETSOCKNAME	6
+#define	LINUX_GETPEERNAME	7
+#define	LINUX_SOCKETPAIR	8
+#define	LINUX_SEND		9
+#define	LINUX_RECV		10
+#define	LINUX_SENDTO 		11
+#define	LINUX_RECVFROM 		12
+#define	LINUX_SHUTDOWN 		13
+#define	LINUX_SETSOCKOPT	14
+#define	LINUX_GETSOCKOPT	15
+#define	LINUX_SENDMSG		16
+#define	LINUX_RECVMSG		17
+#define	LINUX_ACCEPT4		18
+
+/* Socket options */
+#define	LINUX_IP_TOS		1
+#define	LINUX_IP_TTL		2
+#define	LINUX_IP_HDRINCL	3
+#define	LINUX_IP_OPTIONS	4
+
+#define	LINUX_IP_MULTICAST_IF		32
+#define	LINUX_IP_MULTICAST_TTL		33
+#define	LINUX_IP_MULTICAST_LOOP		34
+#define	LINUX_IP_ADD_MEMBERSHIP		35
+#define	LINUX_IP_DROP_MEMBERSHIP	36
+
+#define	LINUX_TCP_NODELAY	1
+#define	LINUX_TCP_MAXSEG	2
+#define	LINUX_TCP_KEEPIDLE	4
+#define	LINUX_TCP_KEEPINTVL	5
+#define	LINUX_TCP_KEEPCNT	6
+#define	LINUX_TCP_MD5SIG	14
+
 #endif /* _LINUX_SOCKET_H_ */
diff --git a/sys/compat/linux/linux_sysctl.c b/sys/compat/linux/linux_sysctl.c
index b2c10e1..decd8f8 100644
--- a/sys/compat/linux/linux_sysctl.c
+++ b/sys/compat/linux/linux_sysctl.c
@@ -52,6 +52,7 @@ __FBSDID("$FreeBSD$");
 #endif
 
 #include <compat/linux/linux_dtrace.h>
+#include <compat/linux/linux_misc.h>
 #include <compat/linux/linux_util.h>
 
 #define	LINUX_CTL_KERN		1
diff --git a/sys/compat/linux/linux_time.c b/sys/compat/linux/linux_time.c
index 1359025..e03af00 100644
--- a/sys/compat/linux/linux_time.c
+++ b/sys/compat/linux/linux_time.c
@@ -60,6 +60,7 @@ __KERNEL_RCSID(0, "$NetBSD: linux_time.c,v 1.14 2006/05/14 03:40:54 christos Exp
 #endif
 
 #include <compat/linux/linux_dtrace.h>
+#include <compat/linux/linux_misc.h>
 
 /* DTrace init */
 LIN_SDT_PROVIDER_DECLARE(LINUX_DTRACE);
diff --git a/sys/conf/Makefile.arm b/sys/conf/Makefile.arm
index 4145d60..63b451d 100644
--- a/sys/conf/Makefile.arm
+++ b/sys/conf/Makefile.arm
@@ -39,8 +39,12 @@ SYSTEM_DEP:= ${SYSTEM_DEP:$S/conf/ldscript.$M=ldscript.$M}
 STRIP_FLAGS = -S
 .endif
 
+CFLAGS += -mno-thumb-interwork
+
 .if empty(DDB_ENABLED)
 CFLAGS += -mno-apcs-frame
+.elif defined(WITH_ARM_EABI)
+CFLAGS += -funwind-tables
 .endif
 
 SYSTEM_LD_ = ${LD} -Bdynamic -T ldscript.$M.noheader ${LDFLAGS} \
diff --git a/sys/conf/NOTES b/sys/conf/NOTES
index 216d2f5..ff245ed 100644
--- a/sys/conf/NOTES
+++ b/sys/conf/NOTES
@@ -446,6 +446,8 @@ options 	KTRACE_REQUEST_POOL=101
 # KTR is a kernel tracing facility imported from BSD/OS.  It is
 # enabled with the KTR option.  KTR_ENTRIES defines the number of
 # entries in the circular trace buffer; it may be an arbitrary number.
+# KTR_BOOT_ENTRIES defines the number of entries during the early boot,
+# before malloc(9) is functional.
 # KTR_COMPILE defines the mask of events to compile into the kernel as
 # defined by the KTR_* constants in <sys/ktr.h>.  KTR_MASK defines the
 # initial value of the ktr_mask variable which determines at runtime
@@ -459,7 +461,8 @@ options 	KTRACE_REQUEST_POOL=101
 # if KTR_VERBOSE is not defined.  See ktr(4) and ktrdump(8) for details.
 #
 options 	KTR
-options 	KTR_ENTRIES=1024
+options 	KTR_BOOT_ENTRIES=1024
+options 	KTR_ENTRIES=(128 * 1024)
 options 	KTR_COMPILE=(KTR_INTR|KTR_PROC)
 options 	KTR_MASK=KTR_INTR
 options 	KTR_CPUMASK=0x3
diff --git a/sys/conf/files b/sys/conf/files
index 04718e8..2a3db4f 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -283,6 +283,7 @@ contrib/altq/altq/altq_rio.c		optional altq
 contrib/altq/altq/altq_rmclass.c	optional altq
 contrib/altq/altq/altq_subr.c		optional altq
 contrib/dev/acpica/components/debugger/dbcmds.c		optional acpi acpi_debug
+contrib/dev/acpica/components/debugger/dbconvert.c	optional acpi acpi_debug
 contrib/dev/acpica/components/debugger/dbdisply.c	optional acpi acpi_debug
 contrib/dev/acpica/components/debugger/dbexec.c		optional acpi acpi_debug
 contrib/dev/acpica/components/debugger/dbfileio.c	optional acpi acpi_debug
@@ -324,6 +325,7 @@ contrib/dev/acpica/components/events/evgpe.c		optional acpi
 contrib/dev/acpica/components/events/evgpeblk.c		optional acpi
 contrib/dev/acpica/components/events/evgpeinit.c	optional acpi
 contrib/dev/acpica/components/events/evgpeutil.c	optional acpi
+contrib/dev/acpica/components/events/evhandler.c	optional acpi
 contrib/dev/acpica/components/events/evmisc.c		optional acpi
 contrib/dev/acpica/components/events/evregion.c		optional acpi
 contrib/dev/acpica/components/events/evrgnini.c		optional acpi
@@ -376,6 +378,7 @@ contrib/dev/acpica/components/namespace/nsnames.c	optional acpi
 contrib/dev/acpica/components/namespace/nsobject.c	optional acpi
 contrib/dev/acpica/components/namespace/nsparse.c	optional acpi
 contrib/dev/acpica/components/namespace/nspredef.c	optional acpi
+contrib/dev/acpica/components/namespace/nsprepkg.c	optional acpi
 contrib/dev/acpica/components/namespace/nsrepair.c	optional acpi
 contrib/dev/acpica/components/namespace/nsrepair2.c	optional acpi
 contrib/dev/acpica/components/namespace/nssearch.c	optional acpi
@@ -386,7 +389,9 @@ contrib/dev/acpica/components/namespace/nsxfname.c	optional acpi
 contrib/dev/acpica/components/namespace/nsxfobj.c	optional acpi
 contrib/dev/acpica/components/parser/psargs.c		optional acpi
 contrib/dev/acpica/components/parser/psloop.c		optional acpi
+contrib/dev/acpica/components/parser/psobject.c		optional acpi
 contrib/dev/acpica/components/parser/psopcode.c		optional acpi
+contrib/dev/acpica/components/parser/psopinfo.c		optional acpi
 contrib/dev/acpica/components/parser/psparse.c		optional acpi
 contrib/dev/acpica/components/parser/psscope.c		optional acpi
 contrib/dev/acpica/components/parser/pstree.c		optional acpi
@@ -397,6 +402,7 @@ contrib/dev/acpica/components/resources/rsaddr.c	optional acpi
 contrib/dev/acpica/components/resources/rscalc.c	optional acpi
 contrib/dev/acpica/components/resources/rscreate.c	optional acpi
 contrib/dev/acpica/components/resources/rsdump.c	optional acpi
+contrib/dev/acpica/components/resources/rsdumpinfo.c	optional acpi
 contrib/dev/acpica/components/resources/rsinfo.c	optional acpi
 contrib/dev/acpica/components/resources/rsio.c		optional acpi
 contrib/dev/acpica/components/resources/rsirq.c		optional acpi
@@ -431,8 +437,10 @@ contrib/dev/acpica/components/utilities/utmisc.c	optional acpi
 contrib/dev/acpica/components/utilities/utmutex.c	optional acpi
 contrib/dev/acpica/components/utilities/utobject.c	optional acpi
 contrib/dev/acpica/components/utilities/utosi.c		optional acpi
+contrib/dev/acpica/components/utilities/utownerid.c	optional acpi
 contrib/dev/acpica/components/utilities/utresrc.c	optional acpi
 contrib/dev/acpica/components/utilities/utstate.c	optional acpi
+contrib/dev/acpica/components/utilities/utstring.c	optional acpi
 contrib/dev/acpica/components/utilities/utxface.c	optional acpi
 contrib/dev/acpica/components/utilities/utxferror.c	optional acpi
 contrib/dev/acpica/components/utilities/utxfinit.c	optional acpi
@@ -638,9 +646,12 @@ dev/aic7xxx/aic7xxx_pci.c	optional ahc pci
 dev/alc/if_alc.c		optional alc pci
 dev/ale/if_ale.c		optional ale pci
 dev/altera/avgen/altera_avgen.c		optional altera_avgen
+dev/altera/avgen/altera_avgen_fdt.c	optional altera_avgen fdt
+dev/altera/avgen/altera_avgen_nexus.c	optional altera_avgen
 dev/altera/sdcard/altera_sdcard.c	optional altera_sdcard
 dev/altera/sdcard/altera_sdcard_disk.c	optional altera_sdcard
 dev/altera/sdcard/altera_sdcard_io.c	optional altera_sdcard
+dev/altera/sdcard/altera_sdcard_fdt.c	optional altera_sdcard fdt
 dev/altera/sdcard/altera_sdcard_nexus.c	optional altera_sdcard
 dev/amr/amr.c			optional amr
 dev/amr/amr_cam.c		optional amrp amr
@@ -1415,6 +1426,7 @@ dev/iscsi/initiator/isc_soc.c	optional iscsi_initiator scbus
 dev/iscsi/initiator/isc_sm.c	optional iscsi_initiator scbus
 dev/iscsi/initiator/isc_subr.c	optional iscsi_initiator scbus
 dev/isf/isf.c			optional isf
+dev/isf/isf_fdt.c		optional isf fdt
 dev/isf/isf_nexus.c		optional isf
 dev/isp/isp.c			optional isp
 dev/isp/isp_freebsd.c		optional isp
@@ -2276,15 +2288,6 @@ dev/utopia/idtphy.c		optional utopia
 dev/utopia/suni.c		optional utopia
 dev/utopia/utopia.c		optional utopia
 dev/vge/if_vge.c		optional vge
-#
-# virtio support
-#
-dev/virtio/pci/virtio_pci.c    optional vtnet
-dev/virtio/virtio.c            optional vtnet
-dev/virtio/virtqueue.c         optional vtnet
-dev/virtio/network/if_vtnet.c  optional vtnet
-dev/virtio/virtio_bus_if.m     optional vtnet
-dev/virtio/virtio_if.m         optional vtnet
 
 dev/vkbd/vkbd.c			optional vkbd
 dev/vr/if_vr.c			optional vr pci
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index b6a474e..131a1e7 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -190,8 +190,7 @@ dev/fdc/fdc_acpi.c		optional	fdc
 dev/fdc/fdc_isa.c		optional	fdc isa
 dev/fdc/fdc_pccard.c		optional	fdc pccard
 dev/hpt27xx/os_bsd.c		optional	hpt27xx
-dev/hpt27xx/osm_bsd.c		optional	hpt27xx \
-	compile-with "${NORMAL_C} ${NO_WFORMAT_SECURITY}"
+dev/hpt27xx/osm_bsd.c		optional	hpt27xx
 dev/hpt27xx/hpt27xx_config.c	optional	hpt27xx
 dev/hptmv/entry.c		optional	hptmv
 dev/hptmv/mv.c			optional	hptmv
@@ -390,6 +389,15 @@ dev/isci/scil/scif_sas_task_request.c				optional isci
 dev/isci/scil/scif_sas_task_request_state_handlers.c		optional isci
 dev/isci/scil/scif_sas_task_request_states.c			optional isci
 dev/isci/scil/scif_sas_timer.c					optional isci
+dev/virtio/virtio.c			optional	virtio
+dev/virtio/virtqueue.c			optional	virtio
+dev/virtio/virtio_bus_if.m		optional	virtio
+dev/virtio/virtio_if.m			optional	virtio
+dev/virtio/pci/virtio_pci.c		optional	virtio_pci virtio pci
+dev/virtio/network/if_vtnet.c		optional	vtnet virtio
+dev/virtio/block/virtio_blk.c		optional	virtio_blk virtio
+dev/virtio/balloon/virtio_balloon.c	optional	virtio_balloon virtio
+dev/virtio/scsi/virtio_scsi.c		optional	virtio_scsi virtio scbus
 isa/syscons_isa.c		optional	sc
 isa/vga_isa.c			optional	vga
 kern/kern_clocksource.c		standard
@@ -455,6 +463,11 @@ libkern/memset.c		standard
 compat/x86bios/x86bios.c	optional x86bios | atkbd | dpms | vesa
 contrib/x86emu/x86emu.c		optional x86bios | atkbd | dpms | vesa
 #
+# bvm console
+#
+dev/bvm/bvm_console.c		optional	bvmconsole
+dev/bvm/bvm_dbg.c		optional	bvmdebug
+#
 # x86 shared code between IA32, AMD64 and PC98 architectures
 #
 x86/acpica/OsdEnvironment.c	optional	acpi
diff --git a/sys/conf/files.arm b/sys/conf/files.arm
index 428757a..14d6c4b 100644
--- a/sys/conf/files.arm
+++ b/sys/conf/files.arm
@@ -74,6 +74,8 @@ kern/subr_busdma_bufalloc.c	standard
 kern/subr_dummy_vdso_tc.c	standard
 libkern/arm/divsi3.S		standard
 libkern/arm/ffs.S		standard
+libkern/arm/ldivmod.S		standard
+libkern/arm/ldivmod_helper.c	standard
 libkern/arm/muldi3.c		standard
 libkern/ashldi3.c		standard
 libkern/ashrdi3.c		standard
diff --git a/sys/conf/files.i386 b/sys/conf/files.i386
index d4df01d..fd536af 100644
--- a/sys/conf/files.i386
+++ b/sys/conf/files.i386
@@ -181,8 +181,7 @@ dev/glxiic/glxiic.c		optional glxiic
 dev/glxsb/glxsb.c		optional glxsb
 dev/glxsb/glxsb_hash.c		optional glxsb
 dev/hpt27xx/os_bsd.c		optional hpt27xx
-dev/hpt27xx/osm_bsd.c		optional hpt27xx \
-	compile-with "${NORMAL_C} ${NO_WFORMAT_SECURITY}"
+dev/hpt27xx/osm_bsd.c		optional hpt27xx
 dev/hpt27xx/hpt27xx_config.c	optional hpt27xx
 dev/hptmv/entry.c		optional hptmv
 dev/hptmv/mv.c			optional hptmv
@@ -371,6 +370,15 @@ dev/isci/scil/scif_sas_task_request.c				optional isci
 dev/isci/scil/scif_sas_task_request_state_handlers.c		optional isci
 dev/isci/scil/scif_sas_task_request_states.c			optional isci
 dev/isci/scil/scif_sas_timer.c					optional isci
+dev/virtio/virtio.c			optional	virtio
+dev/virtio/virtqueue.c			optional	virtio
+dev/virtio/virtio_bus_if.m		optional	virtio
+dev/virtio/virtio_if.m			optional	virtio
+dev/virtio/pci/virtio_pci.c		optional	virtio_pci virtio pci
+dev/virtio/network/if_vtnet.c		optional	vtnet virtio
+dev/virtio/block/virtio_blk.c		optional	virtio_blk virtio
+dev/virtio/balloon/virtio_balloon.c	optional	virtio_balloon virtio
+dev/virtio/scsi/virtio_scsi.c		optional	virtio_scsi virtio scbus
 i386/acpica/acpi_machdep.c	optional acpi
 acpi_wakecode.o			optional acpi				\
 	dependency	"$S/i386/acpica/acpi_wakecode.S assym.s"	\
diff --git a/sys/conf/files.mips b/sys/conf/files.mips
index 8c7a530..2dfc5dd 100644
--- a/sys/conf/files.mips
+++ b/sys/conf/files.mips
@@ -7,6 +7,7 @@
 # Arch dependent files
 mips/mips/autoconf.c			standard
 mips/mips/bus_space_generic.c		standard
+mips/mips/bus_space_fdt.c		optional	fdt
 mips/mips/busdma_machdep.c		standard
 mips/mips/cache.c			standard
 mips/mips/cache_mipsNN.c		standard
diff --git a/sys/conf/ldscript.arm b/sys/conf/ldscript.arm
index 3b8422c..d681c87 100644
--- a/sys/conf/ldscript.arm
+++ b/sys/conf/ldscript.arm
@@ -56,6 +56,18 @@ SECTIONS
   .init          : { *(.init)	} =0x9090
   .plt      : { *(.plt)	}
 
+  _extab_start = .;
+  PROVIDE(extab_start = .);
+  .ARM.extab : { *(.ARM.extab) }
+  _extab.end = .;
+  PROVIDE(extab_end = .);
+
+  _exidx_start = .;
+  PROVIDE(exidx_start = .);
+  .ARM.exidx : { *(.ARM.exidx) }
+  _exidx_end = .;
+  PROVIDE(exidx_end = .);
+
   /* Adjust the address for the data segment.  We want to adjust up to
      the same address within the page on the next page up.  */
   . = ALIGN(0x1000) + (. & (0x1000 - 1)) ; 
diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh
index 3cb211b..1e5e24b 100644
--- a/sys/conf/newvers.sh
+++ b/sys/conf/newvers.sh
@@ -86,6 +86,7 @@ fi
 touch version
 v=`cat version` u=${USER:-root} d=`pwd` h=${HOSTNAME:-`hostname`} t=`date`
 i=`${MAKE:-make} -V KERN_IDENT`
+compiler_v=$($(${MAKE:-make} -V CC) -v 2>&1 | grep 'version')
 
 for dir in /bin /usr/bin /usr/local/bin; do
 	if [ -x "${dir}/svnversion" ] && [ -z ${svnversion} ] ; then
@@ -159,6 +160,7 @@ $COPYRIGHT
 
 char sccs[sizeof(SCCSSTR) > 128 ? sizeof(SCCSSTR) : 128] = SCCSSTR;
 char version[sizeof(VERSTR) > 256 ? sizeof(VERSTR) : 256] = VERSTR;
+char compiler_version[] = "${compiler_v}";
 char ostype[] = "${TYPE}";
 char osrelease[sizeof(RELSTR) > 32 ? sizeof(RELSTR) : 32] = RELSTR;
 int osreldate = ${RELDATE};
diff --git a/sys/conf/options b/sys/conf/options
index 1fb3a2a..5722ac4 100644
--- a/sys/conf/options
+++ b/sys/conf/options
@@ -669,6 +669,7 @@ KTR_ALQ			opt_ktr.h
 KTR_MASK		opt_ktr.h
 KTR_CPUMASK		opt_ktr.h
 KTR_COMPILE		opt_global.h
+KTR_BOOT_ENTRIES	opt_global.h
 KTR_ENTRIES		opt_global.h
 KTR_VERBOSE		opt_ktr.h
 WITNESS			opt_global.h
@@ -775,6 +776,8 @@ ATH_ENABLE_11N		opt_ath.h
 ATH_ENABLE_DFS		opt_ath.h
 ATH_EEPROM_FIRMWARE	opt_ath.h
 ATH_ENABLE_RADIOTAP_VENDOR_EXT	opt_ath.h
+ATH_DEBUG_ALQ		opt_ath.h
+ATH_KTR_INTR_DEBUG	opt_ath.h
 
 # options for the Atheros hal
 AH_SUPPORT_AR5416	opt_ah.h
@@ -794,7 +797,7 @@ AH_NEED_DESC_SWAP	opt_ah.h
 AH_USE_INIPDGAIN	opt_ah.h
 AH_MAXCHAN		opt_ah.h
 AH_RXCFG_SDMAMW_4BYTES	opt_ah.h
-
+AH_INTERRUPT_DEBUGGING	opt_ah.h
 # AR5416 and later interrupt mitigation
 # XXX do not use this for AR9130
 AH_AR5416_INTERRUPT_MITIGATION	opt_ah.h
diff --git a/sys/contrib/dev/acpica/changes.txt b/sys/contrib/dev/acpica/changes.txt
index 6374788..5848335 100644
--- a/sys/contrib/dev/acpica/changes.txt
+++ b/sys/contrib/dev/acpica/changes.txt
@@ -1,8 +1,154 @@
 ----------------------------------------
-14 November 2012. Summary of changes for version 20121114:
+17 January 2013. Summary of changes for version 20130117:
+
+
+1) ACPICA Kernel-resident Subsystem:
+
+Updated the AcpiGetSleepTypeData interface: Allow the \_Sx methods to 
+return either 1 or 2 integers. Although the ACPI spec defines the \_Sx 
+objects to return a package containing one integer, most BIOS code returns 
+two integers and the previous code reflects that. However, we also need to 
+support BIOS code that actually implements to the ACPI spec, and this 
+change reflects this.
+
+Fixed two issues with the ACPI_DEBUG_PRINT macros:
+1) Added the ACPI_DO_WHILE macro to the main DEBUG_PRINT helper macro for 
+C compilers that require this support.
+2) Renamed the internal ACPI_DEBUG macro to ACPI_DO_DEBUG_PRINT since 
+ACPI_DEBUG is already used by many of the various hosts.
+
+Updated all ACPICA copyrights and signons to 2013. Added the 2013 
+copyright to all module headers and signons, including the standard Linux 
+header. This affects virtually every file in the ACPICA core subsystem, 
+iASL compiler, all ACPICA utilities, and the test suites.
 
-This release is available at https://www.acpica.org/downloads
-The ACPI 5.0 specification is available at www.acpi.info
+Example Code and Data Size: These are the sizes for the OS-independent 
+acpica.lib produced by the Microsoft Visual C++ 9.0 32-bit compiler. The 
+debug version of the code includes the debug output trace mechanism and 
+has a much larger code and data size.
+
+  Previous Release:
+    Non-Debug Version:  94.5K Code, 25.5K Data, 120.0K Total
+    Debug Version:     182.2K Code, 74.9K Data, 257.1K Total
+  Current Release:
+    Non-Debug Version:  94.5K Code, 25.4K Data, 119.9K Total
+    Debug Version:     182.3K Code, 75.0K Data, 257.3K Total
+
+
+2) iASL Compiler/Disassembler and Tools:
+
+Generic Unix OSL: Use a buffer to eliminate multiple vfprintf()s and 
+prevent a possible fault on some hosts. Some C libraries modify the arg 
+pointer parameter to vfprintf making it difficult to call it twice in the 
+AcpiOsVprintf function. Use a local buffer to workaround this issue. This 
+does not affect the Windows OSL since the Win C library does not modify 
+the arg pointer. Chao Guan, Bob Moore.
+
+iASL: Fixed a possible infinite loop when the maximum error count is 
+reached. If an output file other than the .AML file is specified (such as 
+a listing file), and the maximum number of errors is reached, do not 
+attempt to flush data to the output file(s) as the compiler is aborting. 
+This can cause an infinite loop as the max error count code essentially 
+keeps calling itself.
+
+iASL/Disassembler: Added an option (-in) to ignore NOOP opcodes/operators. 
+Implemented for both the compiler and the disassembler. Often, the NOOP 
+opcode is used as padding for packages that are changed dynamically by the 
+BIOS. When disassembled and recompiled, these NOOPs will cause syntax 
+errors. This option causes the disassembler to ignore all NOOP opcodes 
+(0xA3), and it also causes the compiler to ignore all ASL source code NOOP 
+statements as well.
+
+Debugger: Enhanced the Sleep command to execute all sleep states. This 
+change allows Sleep to be invoked with no arguments and causes the 
+debugger to execute all of the sleep states, 0-5, automatically.
+
+----------------------------------------
+20 December 2012. Summary of changes for version 20121220:
+
+1) ACPICA Kernel-resident Subsystem:
+
+Implemented a new interface, AcpiWalkResourceBuffer. This interface is an 
+alternate entry point for AcpiWalkResources and improves the usability of 
+the resource manager by accepting as input a buffer containing the output 
+of either a _CRS, _PRS, or _AEI method. The key functionality is that the 
+input buffer is not deleted by this interface so that it can be used by 
+the host later. See the ACPICA reference for details.
+
+Interpreter: Add a warning if a 64-bit constant appears in a 32-bit table 
+(DSDT version < 2). The constant will be truncated and this warning 
+reflects that behavior.
+
+Resource Manager: Add support for the new ACPI 5.0 wake bit in the IRQ, 
+ExtendedInterrupt, and GpioInt descriptors. This change adds support to 
+both get and set the new wake bit in these descriptors, separately from 
+the existing share bit. Reported by Aaron Lu.
+
+Interpreter: Fix Store() when an implicit conversion is not possible. For 
+example, in the cases such as a store of a string to an existing package 
+object, implement the store as a CopyObject(). This is a small departure 
+from the ACPI specification which states that the control method should be 
+aborted in this case. However, the ASLTS suite depends on this behavior.
+
+Performance improvement for the various FUNCTION_TRACE and DEBUG_PRINT 
+macros: check if debug output is currently enabled as soon as possible to 
+minimize performance impact if debug is in fact not enabled.
+
+Source code restructuring: Cleanup to improve modularity. The following 
+new files have been added: dbconvert.c, evhandler.c, nsprepkg.c, 
+psopinfo.c, psobject.c, rsdumpinfo.c, utstring.c, and utownerid.c. 
+Associated makefiles and project files have been updated.
+
+Changed an exception code for LoadTable operator. For the case where one 
+of the input strings is too long, change the returned exception code from 
+AE_BAD_PARAMETER to AE_AML_STRING_LIMIT.
+
+Fixed a possible memory leak in dispatcher error path. On error, delete 
+the mutex object created during method mutex creation. Reported by 
+tim.gardner@canonical.com.
+
+Example Code and Data Size: These are the sizes for the OS-independent 
+acpica.lib produced by the Microsoft Visual C++ 9.0 32-bit compiler. The 
+debug version of the code includes the debug output trace mechanism and 
+has a much larger code and data size.
+
+  Previous Release:
+    Non-Debug Version:  94.3K Code, 25.3K Data, 119.6K Total
+    Debug Version:     175.5K Code, 74.5K Data, 250.0K Total
+  Current Release:
+    Non-Debug Version:  94.5K Code, 25.5K Data, 120.0K Total
+    Debug Version:     182.2K Code, 74.9K Data, 257.1K Total
+
+
+2) iASL Compiler/Disassembler and Tools:
+
+iASL: Disallow a method call as argument to the ObjectType ASL operator. 
+This change tracks an errata to the ACPI 5.0 document. The AML grammar 
+will not allow the interpreter to differentiate between a method and a 
+method invocation when these are used as an argument to the ObjectType 
+operator. The ACPI specification change is to disallow a method invocation 
+(UserTerm) for the ObjectType operator.
+
+Finish support for the TPM2 and CSRT tables in the headers, table 
+compiler, and disassembler.
+
+Unix user-space OSL: Fix a problem with WaitSemaphore where the timeout 
+always expires immediately if the semaphore is not available. The original 
+code was using a relative-time timeout, but sem_timedwait requires the use 
+of an absolute time.
+
+iASL: Added a remark if the Timer() operator is used within a 32-bit 
+table. This operator returns a 64-bit time value that will be truncated 
+within a 32-bit table.
+
+iASL Source code restructuring: Cleanup to improve modularity. The 
+following new files have been added: aslhex.c, aslxref.c, aslnamesp.c, 
+aslmethod.c, and aslfileio.c. Associated makefiles and project files have 
+been updated.
+
+
+----------------------------------------
+14 November 2012. Summary of changes for version 20121114:
 
 1) ACPICA Kernel-resident Subsystem:
 
diff --git a/sys/contrib/dev/acpica/common/adfile.c b/sys/contrib/dev/acpica/common/adfile.c
index 67d7b79..5cde806 100644
--- a/sys/contrib/dev/acpica/common/adfile.c
+++ b/sys/contrib/dev/acpica/common/adfile.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/common/adisasm.c b/sys/contrib/dev/acpica/common/adisasm.c
index c63d9b5..42befa8 100644
--- a/sys/contrib/dev/acpica/common/adisasm.c
+++ b/sys/contrib/dev/acpica/common/adisasm.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -69,11 +69,11 @@ extern int                  AslCompilerdebug;
 #endif
 
 ACPI_STATUS
-LsDisplayNamespace (
+NsDisplayNamespace (
     void);
 
 void
-LsSetupNsList (
+NsSetupNamespaceListing (
     void                    *Handle);
 
 
@@ -429,8 +429,8 @@ AdAmlDisassemble (
         {
             AcpiOsPrintf ("/**** Before second load\n");
 
-            LsSetupNsList (File);
-            LsDisplayNamespace ();
+            NsSetupNamespaceListing (File);
+            NsDisplayNamespace ();
             AcpiOsPrintf ("*****/\n");
         }
 
@@ -498,8 +498,8 @@ AdAmlDisassemble (
             if (AslCompilerdebug)
             {
                 AcpiOsPrintf ("/**** After second load and resource conversion\n");
-                LsSetupNsList (File);
-                LsDisplayNamespace ();
+                NsSetupNamespaceListing (File);
+                NsDisplayNamespace ();
                 AcpiOsPrintf ("*****/\n");
 
                 AcpiDmDumpTree (AcpiGbl_ParseOpRoot);
@@ -549,8 +549,8 @@ Cleanup:
     {
         if (AslCompilerdebug) /* Display final namespace, with transforms */
         {
-            LsSetupNsList (File);
-            LsDisplayNamespace ();
+            NsSetupNamespaceListing (File);
+            NsDisplayNamespace ();
         }
 
         fclose (File);
diff --git a/sys/contrib/dev/acpica/common/adwalk.c b/sys/contrib/dev/acpica/common/adwalk.c
index af86c91..f801f10 100644
--- a/sys/contrib/dev/acpica/common/adwalk.c
+++ b/sys/contrib/dev/acpica/common/adwalk.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/common/ahpredef.c b/sys/contrib/dev/acpica/common/ahpredef.c
index 858a714..4583b58 100644
--- a/sys/contrib/dev/acpica/common/ahpredef.c
+++ b/sys/contrib/dev/acpica/common/ahpredef.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/common/dmextern.c b/sys/contrib/dev/acpica/common/dmextern.c
index f5d676d..67f13f7 100644
--- a/sys/contrib/dev/acpica/common/dmextern.c
+++ b/sys/contrib/dev/acpica/common/dmextern.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/common/dmrestag.c b/sys/contrib/dev/acpica/common/dmrestag.c
index 1e9bd88..54f4282 100644
--- a/sys/contrib/dev/acpica/common/dmrestag.c
+++ b/sys/contrib/dev/acpica/common/dmrestag.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -89,7 +89,7 @@ AcpiDmAddResourceToNamespace (
     UINT32                  Length,
     UINT32                  Offset,
     UINT8                   ResourceIndex,
-    void                    *Context);
+    void                    **Context);
 
 static void
 AcpiDmAddResourcesToNamespace (
@@ -993,7 +993,7 @@ AcpiDmAddResourcesToNamespace (
      */
     AcpiUtWalkAmlResources (NULL, (UINT8 *) NextOp->Named.Data,
         (ACPI_SIZE) NextOp->Common.Value.Integer,
-        AcpiDmAddResourceToNamespace, BufferNode);
+        AcpiDmAddResourceToNamespace, (void **) BufferNode);
 }
 
 
@@ -1019,7 +1019,7 @@ AcpiDmAddResourceToNamespace (
     UINT32                  Length,
     UINT32                  Offset,
     UINT8                   ResourceIndex,
-    void                    *Context)
+    void                    **Context)
 {
     ACPI_STATUS             Status;
     ACPI_GENERIC_STATE      ScopeInfo;
diff --git a/sys/contrib/dev/acpica/common/dmtable.c b/sys/contrib/dev/acpica/common/dmtable.c
index 5d2717c..e569b63 100644
--- a/sys/contrib/dev/acpica/common/dmtable.c
+++ b/sys/contrib/dev/acpica/common/dmtable.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -281,7 +281,7 @@ ACPI_DMTABLE_DATA    AcpiDmTableData[] =
     {ACPI_SIG_BGRT, AcpiDmTableInfoBgrt,    NULL,           NULL,           TemplateBgrt,   "Boot Graphics Resource Table"},
     {ACPI_SIG_BOOT, AcpiDmTableInfoBoot,    NULL,           NULL,           TemplateBoot,   "Simple Boot Flag Table"},
     {ACPI_SIG_CPEP, NULL,                   AcpiDmDumpCpep, DtCompileCpep,  TemplateCpep,   "Corrected Platform Error Polling table"},
-    {ACPI_SIG_CSRT, NULL,                   AcpiDmDumpCsrt, NULL,           NULL,           "Core System Resource Table"},
+    {ACPI_SIG_CSRT, NULL,                   AcpiDmDumpCsrt, DtCompileCsrt,  TemplateCsrt,   "Core System Resource Table"},
     {ACPI_SIG_DBG2, NULL,                   AcpiDmDumpDbg2, NULL,           NULL,           "Debug Port table type 2"},
     {ACPI_SIG_DBGP, AcpiDmTableInfoDbgp,    NULL,           NULL,           TemplateDbgp,   "Debug Port table"},
     {ACPI_SIG_DMAR, NULL,                   AcpiDmDumpDmar, DtCompileDmar,  TemplateDmar,   "DMA Remapping table"},
@@ -310,6 +310,7 @@ ACPI_DMTABLE_DATA    AcpiDmTableData[] =
     {ACPI_SIG_SPMI, AcpiDmTableInfoSpmi,    NULL,           NULL,           TemplateSpmi,   "Server Platform Management Interface table"},
     {ACPI_SIG_SRAT, NULL,                   AcpiDmDumpSrat, DtCompileSrat,  TemplateSrat,   "System Resource Affinity Table"},
     {ACPI_SIG_TCPA, AcpiDmTableInfoTcpa,    NULL,           NULL,           TemplateTcpa,   "Trusted Computing Platform Alliance table"},
+    {ACPI_SIG_TPM2, AcpiDmTableInfoTpm2,    NULL,           NULL,           TemplateTpm2,   "Trusted Platform Module hardware interface table"},
     {ACPI_SIG_UEFI, AcpiDmTableInfoUefi,    NULL,           DtCompileUefi,  TemplateUefi,   "UEFI Boot Optimization Table"},
     {ACPI_SIG_WAET, AcpiDmTableInfoWaet,    NULL,           NULL,           TemplateWaet,   "Windows ACPI Emulated Devices Table"},
     {ACPI_SIG_WDAT, NULL,                   AcpiDmDumpWdat, DtCompileWdat,  TemplateWdat,   "Watchdog Action Table"},
diff --git a/sys/contrib/dev/acpica/common/dmtbdump.c b/sys/contrib/dev/acpica/common/dmtbdump.c
index 2a79b25..ac5ee90 100644
--- a/sys/contrib/dev/acpica/common/dmtbdump.c
+++ b/sys/contrib/dev/acpica/common/dmtbdump.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -614,6 +614,7 @@ AcpiDmDumpCsrt (
 {
     ACPI_STATUS             Status;
     ACPI_CSRT_GROUP         *SubTable;
+    ACPI_CSRT_SHARED_INFO   *SharedInfoTable;
     ACPI_CSRT_DESCRIPTOR    *SubSubTable;
     UINT32                  Length = Table->Length;
     UINT32                  Offset = sizeof (ACPI_TABLE_CSRT);
@@ -629,6 +630,8 @@ AcpiDmDumpCsrt (
     SubTable = ACPI_ADD_PTR (ACPI_CSRT_GROUP, Table, Offset);
     while (Offset < Table->Length)
     {
+        /* Resource group subtable */
+
         AcpiOsPrintf ("\n");
         Status = AcpiDmDumpTable (Length, Offset, SubTable,
                     SubTable->Length, AcpiDmTableInfoCsrt0);
@@ -637,15 +640,23 @@ AcpiDmDumpCsrt (
             return;
         }
 
+        /* Shared info subtable (One per resource group) */
+
         SubOffset = sizeof (ACPI_CSRT_GROUP);
+        SharedInfoTable = ACPI_ADD_PTR (ACPI_CSRT_SHARED_INFO, Table,
+            Offset + SubOffset);
 
-        /* Shared resource group info buffer */
+        AcpiOsPrintf ("\n");
+        Status = AcpiDmDumpTable (Length, Offset + SubOffset, SharedInfoTable,
+                    sizeof (ACPI_CSRT_SHARED_INFO), AcpiDmTableInfoCsrt1);
+        if (ACPI_FAILURE (Status))
+        {
+            return;
+        }
 
-        AcpiDmDumpBuffer (SubTable, SubOffset, SubTable->InfoLength,
-            Offset+SubOffset, "Shared Data");
-        SubOffset += SubTable->InfoLength;
+        SubOffset += SubTable->SharedInfoLength;
 
-        /* Sub-Sub-tables (Resource Descriptors) */
+        /* Sub-Subtables (Resource Descriptors) */
 
         SubSubTable = ACPI_ADD_PTR (ACPI_CSRT_DESCRIPTOR, Table,
             Offset + SubOffset);
@@ -655,7 +666,7 @@ AcpiDmDumpCsrt (
         {
             AcpiOsPrintf ("\n");
             Status = AcpiDmDumpTable (Length, Offset + SubOffset, SubSubTable,
-                        SubSubTable->Length, AcpiDmTableInfoCsrt1);
+                        SubSubTable->Length, AcpiDmTableInfoCsrt2);
             if (ACPI_FAILURE (Status))
             {
                 return;
@@ -671,7 +682,7 @@ AcpiDmDumpCsrt (
                 Offset + SubOffset + SubSubOffset, "ResourceInfo");
             SubSubOffset += InfoLength;
 
-            /* Point to next sub-sub-table */
+            /* Point to next sub-subtable */
 
             SubOffset += SubSubTable->Length;
             SubSubTable = ACPI_ADD_PTR (ACPI_CSRT_DESCRIPTOR, SubSubTable,
diff --git a/sys/contrib/dev/acpica/common/dmtbinfo.c b/sys/contrib/dev/acpica/common/dmtbinfo.c
index 21fddec..54bd5c2 100644
--- a/sys/contrib/dev/acpica/common/dmtbinfo.c
+++ b/sys/contrib/dev/acpica/common/dmtbinfo.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -111,6 +111,7 @@
 #define ACPI_SPMI_OFFSET(f)             (UINT16) ACPI_OFFSET (ACPI_TABLE_SPMI,f)
 #define ACPI_SRAT_OFFSET(f)             (UINT16) ACPI_OFFSET (ACPI_TABLE_SRAT,f)
 #define ACPI_TCPA_OFFSET(f)             (UINT16) ACPI_OFFSET (ACPI_TABLE_TCPA,f)
+#define ACPI_TPM2_OFFSET(f)             (UINT16) ACPI_OFFSET (ACPI_TABLE_TPM2,f)
 #define ACPI_UEFI_OFFSET(f)             (UINT16) ACPI_OFFSET (ACPI_TABLE_UEFI,f)
 #define ACPI_WAET_OFFSET(f)             (UINT16) ACPI_OFFSET (ACPI_TABLE_WAET,f)
 #define ACPI_WDAT_OFFSET(f)             (UINT16) ACPI_OFFSET (ACPI_TABLE_WDAT,f)
@@ -128,7 +129,8 @@
 #define ACPI_ASF4_OFFSET(f)             (UINT16) ACPI_OFFSET (ACPI_ASF_ADDRESS,f)
 #define ACPI_CPEP0_OFFSET(f)            (UINT16) ACPI_OFFSET (ACPI_CPEP_POLLING,f)
 #define ACPI_CSRT0_OFFSET(f)            (UINT16) ACPI_OFFSET (ACPI_CSRT_GROUP,f)
-#define ACPI_CSRT1_OFFSET(f)            (UINT16) ACPI_OFFSET (ACPI_CSRT_DESCRIPTOR,f)
+#define ACPI_CSRT1_OFFSET(f)            (UINT16) ACPI_OFFSET (ACPI_CSRT_SHARED_INFO,f)
+#define ACPI_CSRT2_OFFSET(f)            (UINT16) ACPI_OFFSET (ACPI_CSRT_DESCRIPTOR,f)
 #define ACPI_DBG20_OFFSET(f)            (UINT16) ACPI_OFFSET (ACPI_DBG2_DEVICE,f)
 #define ACPI_DMARS_OFFSET(f)            (UINT16) ACPI_OFFSET (ACPI_DMAR_DEVICE_SCOPE,f)
 #define ACPI_DMAR0_OFFSET(f)            (UINT16) ACPI_OFFSET (ACPI_DMAR_HARDWARE_UNIT,f)
@@ -662,18 +664,38 @@ ACPI_DMTABLE_INFO           AcpiDmTableInfoCsrt0[] =
     {ACPI_DMT_UINT16,   ACPI_CSRT0_OFFSET (SubdeviceId),            "Subdevice ID", 0},
     {ACPI_DMT_UINT16,   ACPI_CSRT0_OFFSET (Revision),               "Revision", 0},
     {ACPI_DMT_UINT16,   ACPI_CSRT0_OFFSET (Reserved),               "Reserved", 0},
-    {ACPI_DMT_UINT32,   ACPI_CSRT0_OFFSET (InfoLength),             "InfoLength", 0},
+    {ACPI_DMT_UINT32,   ACPI_CSRT0_OFFSET (SharedInfoLength),       "Shared Info Length", 0},
     ACPI_DMT_TERMINATOR
 };
 
-/* Resource Descriptor subtable */
+/* Shared Info subtable */
 
 ACPI_DMTABLE_INFO           AcpiDmTableInfoCsrt1[] =
 {
-    {ACPI_DMT_UINT32,   ACPI_CSRT1_OFFSET (Length),                 "Length", 0},
-    {ACPI_DMT_UINT16,   ACPI_CSRT1_OFFSET (Type),                   "Type", 0},
-    {ACPI_DMT_UINT16,   ACPI_CSRT1_OFFSET (Subtype),                "Subtype", 0},
-    {ACPI_DMT_UINT32,   ACPI_CSRT1_OFFSET (Uid),                    "UID", 0},
+    {ACPI_DMT_UINT16,   ACPI_CSRT1_OFFSET (MajorVersion),           "Major Version", 0},
+    {ACPI_DMT_UINT16,   ACPI_CSRT1_OFFSET (MinorVersion),           "Minor Version", 0},
+    {ACPI_DMT_UINT32,   ACPI_CSRT1_OFFSET (MmioBaseLow),            "MMIO Base Address Low", 0},
+    {ACPI_DMT_UINT32,   ACPI_CSRT1_OFFSET (MmioBaseHigh),           "MMIO Base Address High", 0},
+    {ACPI_DMT_UINT32,   ACPI_CSRT1_OFFSET (GsiInterrupt),           "GSI Interrupt", 0},
+    {ACPI_DMT_UINT8,    ACPI_CSRT1_OFFSET (InterruptPolarity),      "Interrupt Polarity", 0},
+    {ACPI_DMT_UINT8,    ACPI_CSRT1_OFFSET (InterruptMode),          "Interrupt Mode", 0},
+    {ACPI_DMT_UINT8,    ACPI_CSRT1_OFFSET (NumChannels),            "Num Channels", 0},
+    {ACPI_DMT_UINT8,    ACPI_CSRT1_OFFSET (DmaAddressWidth),        "DMA Address Width", 0},
+    {ACPI_DMT_UINT16,   ACPI_CSRT1_OFFSET (BaseRequestLine),        "Base Request Line", 0},
+    {ACPI_DMT_UINT16,   ACPI_CSRT1_OFFSET (NumHandshakeSignals),    "Num Handshake Signals", 0},
+    {ACPI_DMT_UINT32,   ACPI_CSRT1_OFFSET (MaxBlockSize),           "Max Block Size", 0},
+    ACPI_DMT_TERMINATOR
+};
+
+
+/* Resource Descriptor subtable */
+
+ACPI_DMTABLE_INFO           AcpiDmTableInfoCsrt2[] =
+{
+    {ACPI_DMT_UINT32,   ACPI_CSRT2_OFFSET (Length),                 "Length", 0},
+    {ACPI_DMT_UINT16,   ACPI_CSRT2_OFFSET (Type),                   "Type", 0},
+    {ACPI_DMT_UINT16,   ACPI_CSRT2_OFFSET (Subtype),                "Subtype", 0},
+    {ACPI_DMT_UINT32,   ACPI_CSRT2_OFFSET (Uid),                    "UID", 0},
     ACPI_DMT_TERMINATOR
 };
 
@@ -1984,6 +2006,21 @@ ACPI_DMTABLE_INFO           AcpiDmTableInfoTcpa[] =
 
 /*******************************************************************************
  *
+ * TPM2 - Trusted Platform Module (TPM) 2.0 Hardware Interface Table
+ *
+ ******************************************************************************/
+
+ACPI_DMTABLE_INFO           AcpiDmTableInfoTpm2[] =
+{
+    {ACPI_DMT_UINT32,   ACPI_TPM2_OFFSET (Flags),                   "Flags", 0},
+    {ACPI_DMT_UINT64,   ACPI_TPM2_OFFSET (ControlAddress),          "Control Address", 0},
+    {ACPI_DMT_UINT32,   ACPI_TPM2_OFFSET (StartMethod),             "Start Method", 0},
+    ACPI_DMT_TERMINATOR
+};
+
+
+/*******************************************************************************
+ *
  * UEFI - UEFI Boot optimization Table
  *
  ******************************************************************************/
diff --git a/sys/contrib/dev/acpica/common/getopt.c b/sys/contrib/dev/acpica/common/getopt.c
index 3855fa5..6bedff6 100644
--- a/sys/contrib/dev/acpica/common/getopt.c
+++ b/sys/contrib/dev/acpica/common/getopt.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslanalyze.c b/sys/contrib/dev/acpica/compiler/aslanalyze.c
index ea4ee35..a42526c 100644
--- a/sys/contrib/dev/acpica/compiler/aslanalyze.c
+++ b/sys/contrib/dev/acpica/compiler/aslanalyze.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslbtypes.c b/sys/contrib/dev/acpica/compiler/aslbtypes.c
index d7140d2..aac3870 100644
--- a/sys/contrib/dev/acpica/compiler/aslbtypes.c
+++ b/sys/contrib/dev/acpica/compiler/aslbtypes.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslcodegen.c b/sys/contrib/dev/acpica/compiler/aslcodegen.c
index 571e5dd..13ba665 100644
--- a/sys/contrib/dev/acpica/compiler/aslcodegen.c
+++ b/sys/contrib/dev/acpica/compiler/aslcodegen.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslcompile.c b/sys/contrib/dev/acpica/compiler/aslcompile.c
index 4ca98f6..6d4c466 100644
--- a/sys/contrib/dev/acpica/compiler/aslcompile.c
+++ b/sys/contrib/dev/acpica/compiler/aslcompile.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -617,7 +617,7 @@ CmDoCompile (
     /* Namespace cross-reference */
 
     AslGbl_NamespaceEvent = UtBeginEvent ("Cross reference parse tree and Namespace");
-    Status = LkCrossReferenceNamespace ();
+    Status = XfCrossReferenceNamespace ();
     if (ACPI_FAILURE (Status))
     {
         goto ErrorExit;
@@ -639,8 +639,8 @@ CmDoCompile (
 
     DbgPrint (ASL_DEBUG_OUTPUT, "\nSemantic analysis - Method analysis\n\n");
     TrWalkParseTree (RootNode, ASL_WALK_VISIT_TWICE,
-        AnMethodAnalysisWalkBegin,
-        AnMethodAnalysisWalkEnd, &AnalysisWalkInfo);
+        MtMethodAnalysisWalkBegin,
+        MtMethodAnalysisWalkEnd, &AnalysisWalkInfo);
     UtEndEvent (Event);
 
     /* Semantic error checking part two - typing of method returns */
@@ -719,11 +719,11 @@ CmDoOutputFiles (
     /* Create listings and hex files */
 
     LsDoListings ();
-    LsDoHexOutput ();
+    HxDoHexOutput ();
 
     /* Dump the namespace to the .nsp file if requested */
 
-    (void) LsDisplayNamespace ();
+    (void) NsDisplayNamespace ();
 }
 
 
@@ -766,12 +766,12 @@ CmDumpAllEvents (
 
             Delta = (UINT32) (Event->EndTime - Event->StartTime);
 
-            USec = Delta / 10;
-            MSec = Delta / 10000;
+            USec = Delta / ACPI_100NSEC_PER_USEC;
+            MSec = Delta / ACPI_100NSEC_PER_MSEC;
 
             /* Round milliseconds up */
 
-            if ((USec - (MSec * 1000)) >= 500)
+            if ((USec - (MSec * ACPI_USEC_PER_MSEC)) >= 500)
             {
                 MSec++;
             }
diff --git a/sys/contrib/dev/acpica/compiler/aslcompiler.h b/sys/contrib/dev/acpica/compiler/aslcompiler.h
index 72c21ab..d772e8f 100644
--- a/sys/contrib/dev/acpica/compiler/aslcompiler.h
+++ b/sys/contrib/dev/acpica/compiler/aslcompiler.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -188,19 +188,23 @@ AnOperandTypecheckWalkEnd (
     void                    *Context);
 
 ACPI_STATUS
-AnMethodAnalysisWalkBegin (
+AnMethodTypingWalkEnd (
     ACPI_PARSE_OBJECT       *Op,
     UINT32                  Level,
     void                    *Context);
 
+
+/*
+ * aslmethod - Control method analysis walk
+ */
 ACPI_STATUS
-AnMethodAnalysisWalkEnd (
+MtMethodAnalysisWalkBegin (
     ACPI_PARSE_OBJECT       *Op,
     UINT32                  Level,
     void                    *Context);
 
 ACPI_STATUS
-AnMethodTypingWalkEnd (
+MtMethodAnalysisWalkEnd (
     ACPI_PARSE_OBJECT       *Op,
     UINT32                  Level,
     void                    *Context);
@@ -342,6 +346,12 @@ LsDoListings (
     void);
 
 void
+LsDumpAsciiInComment (
+    UINT32                  FileId,
+    UINT32                  Count,
+    UINT8                   *Buffer);
+
+void
 LsWriteNodeToAsmListing (
     ACPI_PARSE_OBJECT       *Op);
 
@@ -351,13 +361,18 @@ LsWriteNode (
     UINT32                  FileId);
 
 void
-LsDoHexOutput (
+LsDumpParseTree (
     void);
 
+
+/*
+ * aslhex - generate all "hex" output files (C, ASM, ASL)
+ */
 void
-LsDumpParseTree (
+HxDoHexOutput (
     void);
 
+
 /*
  * aslfold - constant folding
  */
@@ -689,26 +704,34 @@ LdLoadNamespace (
 
 
 /*
- * asllookup - namespace cross reference
+ * asllookup - namespace lookup functions
  */
-ACPI_STATUS
-LkCrossReferenceNamespace (
-    void);
-
 void
 LkFindUnreferencedObjects (
     void);
 
+
+/*
+ * aslnamesp - namespace output file generation
+ */
 ACPI_STATUS
-LsDisplayNamespace (
+NsDisplayNamespace (
     void);
 
 void
-LsSetupNsList (
+NsSetupNamespaceListing (
     void                    *Handle);
 
 
 /*
+ * aslxref - namespace cross reference
+ */
+ACPI_STATUS
+XfCrossReferenceNamespace (
+    void);
+
+
+/*
  * aslutils - common compiler utilites
  */
 void
diff --git a/sys/contrib/dev/acpica/compiler/aslcompiler.l b/sys/contrib/dev/acpica/compiler/aslcompiler.l
index 5b9ff12..5b2d9ce 100644
--- a/sys/contrib/dev/acpica/compiler/aslcompiler.l
+++ b/sys/contrib/dev/acpica/compiler/aslcompiler.l
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -201,7 +201,7 @@ NamePathTail                [.]{NameSeg}
 "Mutex"                     { count (2); return (PARSEOP_MUTEX); }
 "Name"                      { count (2); return (PARSEOP_NAME); }
 "NAnd"                      { count (3); return (PARSEOP_NAND); }
-"Noop"                      { count (3); return (PARSEOP_NOOP); }
+"Noop"                      { if (!AcpiGbl_IgnoreNoopOperator) {count (3); return (PARSEOP_NOOP);} }
 "NOr"                       { count (3); return (PARSEOP_NOR); }
 "Not"                       { count (3); return (PARSEOP_NOT); }
 "Notify"                    { count (3); return (PARSEOP_NOTIFY); }
diff --git a/sys/contrib/dev/acpica/compiler/aslcompiler.y b/sys/contrib/dev/acpica/compiler/aslcompiler.y
index b6453ac..42ff850 100644
--- a/sys/contrib/dev/acpica/compiler/aslcompiler.y
+++ b/sys/contrib/dev/acpica/compiler/aslcompiler.y
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -661,6 +661,7 @@ void *                      AslLocalAllocate (unsigned int Size);
 /* Types */
 
 %type <n> SuperName
+%type <n> ObjectTypeName
 %type <n> ArgTerm
 %type <n> LocalTerm
 %type <n> DebugTerm
@@ -1991,7 +1992,7 @@ NotTerm
 
 ObjectTypeTerm
     : PARSEOP_OBJECTTYPE '('        {$<n>$ = TrCreateLeafNode (PARSEOP_OBJECTTYPE);}
-        SuperName
+        ObjectTypeName
         ')'                         {$$ = TrLinkChildren ($<n>3,1,$4);}
     | PARSEOP_OBJECTTYPE '('
         error ')'                   {$$ = AslDoError(); yyclearin;}
@@ -2439,6 +2440,18 @@ SuperName
     | LocalTerm                     {}
     | DebugTerm                     {}
     | Type6Opcode                   {}
+
+/* For ObjectType: SuperName except for UserTerm (method invocation) */
+
+ObjectTypeName
+    : NameString                    {}
+    | ArgTerm                       {}
+    | LocalTerm                     {}
+    | DebugTerm                     {}
+    | RefOfTerm                     {}
+    | DerefOfTerm                   {}
+    | IndexTerm                     {}
+
 /*    | UserTerm                      {} */  /* Caused reduce/reduce with Type6Opcode->UserTerm */
     ;
 
diff --git a/sys/contrib/dev/acpica/compiler/asldefine.h b/sys/contrib/dev/acpica/compiler/asldefine.h
index 0ee3c69..7c8bd66 100644
--- a/sys/contrib/dev/acpica/compiler/asldefine.h
+++ b/sys/contrib/dev/acpica/compiler/asldefine.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslerror.c b/sys/contrib/dev/acpica/compiler/aslerror.c
index 41ae967..f362589 100644
--- a/sys/contrib/dev/acpica/compiler/aslerror.c
+++ b/sys/contrib/dev/acpica/compiler/aslerror.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -660,7 +660,6 @@ AslCommonError (
 
         Gbl_SourceLine = 0;
         Gbl_NextError = Gbl_ErrorLog;
-        CmDoOutputFiles ();
         CmCleanupAndExit ();
         exit(1);
     }
diff --git a/sys/contrib/dev/acpica/compiler/aslfileio.c b/sys/contrib/dev/acpica/compiler/aslfileio.c
new file mode 100644
index 0000000..660b82f
--- /dev/null
+++ b/sys/contrib/dev/acpica/compiler/aslfileio.c
@@ -0,0 +1,393 @@
+/******************************************************************************
+ *
+ * Module Name: aslfileio - File I/O support
+ *
+ *****************************************************************************/
+
+/*
+ * Copyright (C) 2000 - 2013, Intel Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ * 3. Neither the names of the above-listed copyright holders nor the names
+ *    of any contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ */
+
+#include <contrib/dev/acpica/compiler/aslcompiler.h>
+
+#define _COMPONENT          ACPI_COMPILER
+        ACPI_MODULE_NAME    ("aslfileio")
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AslAbort
+ *
+ * PARAMETERS:  None
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Dump the error log and abort the compiler. Used for serious
+ *              I/O errors.
+ *
+ ******************************************************************************/
+
+void
+AslAbort (
+    void)
+{
+
+    AePrintErrorLog (ASL_FILE_STDERR);
+    if (Gbl_DebugFlag)
+    {
+        /* Print error summary to stdout also */
+
+        AePrintErrorLog (ASL_FILE_STDOUT);
+    }
+
+    exit (1);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    FlFileError
+ *
+ * PARAMETERS:  FileId              - Index into file info array
+ *              ErrorId             - Index into error message array
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Decode errno to an error message and add the entire error
+ *              to the error log.
+ *
+ ******************************************************************************/
+
+void
+FlFileError (
+    UINT32                  FileId,
+    UINT8                   ErrorId)
+{
+
+    sprintf (MsgBuffer, "\"%s\" (%s)", Gbl_Files[FileId].Filename,
+        strerror (errno));
+    AslCommonError (ASL_ERROR, ErrorId, 0, 0, 0, 0, NULL, MsgBuffer);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    FlOpenFile
+ *
+ * PARAMETERS:  FileId              - Index into file info array
+ *              Filename            - file pathname to open
+ *              Mode                - Open mode for fopen
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Open a file.
+ *              NOTE: Aborts compiler on any error.
+ *
+ ******************************************************************************/
+
+void
+FlOpenFile (
+    UINT32                  FileId,
+    char                    *Filename,
+    char                    *Mode)
+{
+    FILE                    *File;
+
+
+    File = fopen (Filename, Mode);
+    if (!File)
+    {
+        FlFileError (FileId, ASL_MSG_OPEN);
+        AslAbort ();
+    }
+
+    Gbl_Files[FileId].Filename = Filename;
+    Gbl_Files[FileId].Handle   = File;
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    FlGetFileSize
+ *
+ * PARAMETERS:  FileId              - Index into file info array
+ *
+ * RETURN:      File Size
+ *
+ * DESCRIPTION: Get current file size. Uses seek-to-EOF. File must be open.
+ *
+ ******************************************************************************/
+
+UINT32
+FlGetFileSize (
+    UINT32                  FileId)
+{
+    FILE                    *fp;
+    UINT32                  FileSize;
+    long                    Offset;
+
+
+    fp = Gbl_Files[FileId].Handle;
+    Offset = ftell (fp);
+
+    fseek (fp, 0, SEEK_END);
+    FileSize = (UINT32) ftell (fp);
+
+    /* Restore file pointer */
+
+    fseek (fp, Offset, SEEK_SET);
+    return (FileSize);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    FlReadFile
+ *
+ * PARAMETERS:  FileId              - Index into file info array
+ *              Buffer              - Where to place the data
+ *              Length              - Amount to read
+ *
+ * RETURN:      Status. AE_ERROR indicates EOF.
+ *
+ * DESCRIPTION: Read data from an open file.
+ *              NOTE: Aborts compiler on any error.
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+FlReadFile (
+    UINT32                  FileId,
+    void                    *Buffer,
+    UINT32                  Length)
+{
+    UINT32                  Actual;
+
+
+    /* Read and check for error */
+
+    Actual = fread (Buffer, 1, Length, Gbl_Files[FileId].Handle);
+    if (Actual < Length)
+    {
+        if (feof (Gbl_Files[FileId].Handle))
+        {
+            /* End-of-file, just return error */
+
+            return (AE_ERROR);
+        }
+
+        FlFileError (FileId, ASL_MSG_READ);
+        AslAbort ();
+    }
+
+    return (AE_OK);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    FlWriteFile
+ *
+ * PARAMETERS:  FileId              - Index into file info array
+ *              Buffer              - Data to write
+ *              Length              - Amount of data to write
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Write data to an open file.
+ *              NOTE: Aborts compiler on any error.
+ *
+ ******************************************************************************/
+
+void
+FlWriteFile (
+    UINT32                  FileId,
+    void                    *Buffer,
+    UINT32                  Length)
+{
+    UINT32                  Actual;
+
+
+    /* Write and check for error */
+
+    Actual = fwrite ((char *) Buffer, 1, Length, Gbl_Files[FileId].Handle);
+    if (Actual != Length)
+    {
+        FlFileError (FileId, ASL_MSG_WRITE);
+        AslAbort ();
+    }
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    FlPrintFile
+ *
+ * PARAMETERS:  FileId              - Index into file info array
+ *              Format              - Printf format string
+ *              ...                 - Printf arguments
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Formatted write to an open file.
+ *              NOTE: Aborts compiler on any error.
+ *
+ ******************************************************************************/
+
+void
+FlPrintFile (
+    UINT32                  FileId,
+    char                    *Format,
+    ...)
+{
+    INT32                   Actual;
+    va_list                 Args;
+
+
+    va_start (Args, Format);
+
+    Actual = vfprintf (Gbl_Files[FileId].Handle, Format, Args);
+    va_end (Args);
+
+    if (Actual == -1)
+    {
+        FlFileError (FileId, ASL_MSG_WRITE);
+        AslAbort ();
+    }
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    FlSeekFile
+ *
+ * PARAMETERS:  FileId              - Index into file info array
+ *              Offset              - Absolute byte offset in file
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Seek to absolute offset.
+ *              NOTE: Aborts compiler on any error.
+ *
+ ******************************************************************************/
+
+void
+FlSeekFile (
+    UINT32                  FileId,
+    long                    Offset)
+{
+    int                     Error;
+
+
+    Error = fseek (Gbl_Files[FileId].Handle, Offset, SEEK_SET);
+    if (Error)
+    {
+        FlFileError (FileId, ASL_MSG_SEEK);
+        AslAbort ();
+    }
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    FlCloseFile
+ *
+ * PARAMETERS:  FileId              - Index into file info array
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Close an open file. Aborts compiler on error
+ *
+ ******************************************************************************/
+
+void
+FlCloseFile (
+    UINT32                  FileId)
+{
+    int                     Error;
+
+
+    if (!Gbl_Files[FileId].Handle)
+    {
+        return;
+    }
+
+    Error = fclose (Gbl_Files[FileId].Handle);
+    if (Error)
+    {
+        FlFileError (FileId, ASL_MSG_CLOSE);
+        AslAbort ();
+    }
+
+    Gbl_Files[FileId].Handle = NULL;
+    return;
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    FlDeleteFile
+ *
+ * PARAMETERS:  FileId              - Index into file info array
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Delete a file.
+ *
+ ******************************************************************************/
+
+void
+FlDeleteFile (
+    UINT32                  FileId)
+{
+    ASL_FILE_INFO           *Info = &Gbl_Files[FileId];
+
+
+    if (!Info->Filename)
+    {
+        return;
+    }
+
+    if (remove (Info->Filename))
+    {
+        printf ("%s (%s file) ",
+            Info->Filename, Info->Description);
+        perror ("Could not delete");
+    }
+
+    Info->Filename = NULL;
+    return;
+}
diff --git a/sys/contrib/dev/acpica/compiler/aslfiles.c b/sys/contrib/dev/acpica/compiler/aslfiles.c
index 39f64b7..a525fc2 100644
--- a/sys/contrib/dev/acpica/compiler/aslfiles.c
+++ b/sys/contrib/dev/acpica/compiler/aslfiles.c
@@ -1,11 +1,11 @@
 /******************************************************************************
  *
- * Module Name: aslfiles - file I/O suppoert
+ * Module Name: aslfiles - File support functions
  *
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -64,352 +64,6 @@ FlParseInputPathname (
 
 /*******************************************************************************
  *
- * FUNCTION:    AslAbort
- *
- * PARAMETERS:  None
- *
- * RETURN:      None
- *
- * DESCRIPTION: Dump the error log and abort the compiler. Used for serious
- *              I/O errors
- *
- ******************************************************************************/
-
-void
-AslAbort (
-    void)
-{
-
-    AePrintErrorLog (ASL_FILE_STDERR);
-    if (Gbl_DebugFlag)
-    {
-        /* Print error summary to stdout also */
-
-        AePrintErrorLog (ASL_FILE_STDOUT);
-    }
-
-    exit (1);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    FlFileError
- *
- * PARAMETERS:  FileId              - Index into file info array
- *              ErrorId             - Index into error message array
- *
- * RETURN:      None
- *
- * DESCRIPTION: Decode errno to an error message and add the entire error
- *              to the error log.
- *
- ******************************************************************************/
-
-void
-FlFileError (
-    UINT32                  FileId,
-    UINT8                   ErrorId)
-{
-
-    sprintf (MsgBuffer, "\"%s\" (%s)", Gbl_Files[FileId].Filename,
-        strerror (errno));
-    AslCommonError (ASL_ERROR, ErrorId, 0, 0, 0, 0, NULL, MsgBuffer);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    FlOpenFile
- *
- * PARAMETERS:  FileId              - Index into file info array
- *              Filename            - file pathname to open
- *              Mode                - Open mode for fopen
- *
- * RETURN:      None
- *
- * DESCRIPTION: Open a file.
- *              NOTE: Aborts compiler on any error.
- *
- ******************************************************************************/
-
-void
-FlOpenFile (
-    UINT32                  FileId,
-    char                    *Filename,
-    char                    *Mode)
-{
-    FILE                    *File;
-
-
-    File = fopen (Filename, Mode);
-    if (!File)
-    {
-        FlFileError (FileId, ASL_MSG_OPEN);
-        AslAbort ();
-    }
-
-    Gbl_Files[FileId].Filename = Filename;
-    Gbl_Files[FileId].Handle   = File;
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    FlGetFileSize
- *
- * PARAMETERS:  FileId              - Index into file info array
- *
- * RETURN:      File Size
- *
- * DESCRIPTION: Get current file size. Uses seek-to-EOF. File must be open.
- *
- ******************************************************************************/
-
-UINT32
-FlGetFileSize (
-    UINT32                  FileId)
-{
-    FILE                    *fp;
-    UINT32                  FileSize;
-    long                    Offset;
-
-
-    fp = Gbl_Files[FileId].Handle;
-    Offset = ftell (fp);
-
-    fseek (fp, 0, SEEK_END);
-    FileSize = (UINT32) ftell (fp);
-
-    /* Restore file pointer */
-
-    fseek (fp, Offset, SEEK_SET);
-    return (FileSize);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    FlReadFile
- *
- * PARAMETERS:  FileId              - Index into file info array
- *              Buffer              - Where to place the data
- *              Length              - Amount to read
- *
- * RETURN:      Status. AE_ERROR indicates EOF.
- *
- * DESCRIPTION: Read data from an open file.
- *              NOTE: Aborts compiler on any error.
- *
- ******************************************************************************/
-
-ACPI_STATUS
-FlReadFile (
-    UINT32                  FileId,
-    void                    *Buffer,
-    UINT32                  Length)
-{
-    UINT32                  Actual;
-
-
-    /* Read and check for error */
-
-    Actual = fread (Buffer, 1, Length, Gbl_Files[FileId].Handle);
-    if (Actual < Length)
-    {
-        if (feof (Gbl_Files[FileId].Handle))
-        {
-            /* End-of-file, just return error */
-
-            return (AE_ERROR);
-        }
-
-        FlFileError (FileId, ASL_MSG_READ);
-        AslAbort ();
-    }
-
-    return (AE_OK);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    FlWriteFile
- *
- * PARAMETERS:  FileId              - Index into file info array
- *              Buffer              - Data to write
- *              Length              - Amount of data to write
- *
- * RETURN:      None
- *
- * DESCRIPTION: Write data to an open file.
- *              NOTE: Aborts compiler on any error.
- *
- ******************************************************************************/
-
-void
-FlWriteFile (
-    UINT32                  FileId,
-    void                    *Buffer,
-    UINT32                  Length)
-{
-    UINT32                  Actual;
-
-
-    /* Write and check for error */
-
-    Actual = fwrite ((char *) Buffer, 1, Length, Gbl_Files[FileId].Handle);
-    if (Actual != Length)
-    {
-        FlFileError (FileId, ASL_MSG_WRITE);
-        AslAbort ();
-    }
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    FlPrintFile
- *
- * PARAMETERS:  FileId              - Index into file info array
- *              Format              - Printf format string
- *              ...                 - Printf arguments
- *
- * RETURN:      None
- *
- * DESCRIPTION: Formatted write to an open file.
- *              NOTE: Aborts compiler on any error.
- *
- ******************************************************************************/
-
-void
-FlPrintFile (
-    UINT32                  FileId,
-    char                    *Format,
-    ...)
-{
-    INT32                   Actual;
-    va_list                 Args;
-
-
-    va_start (Args, Format);
-
-    Actual = vfprintf (Gbl_Files[FileId].Handle, Format, Args);
-    va_end (Args);
-
-    if (Actual == -1)
-    {
-        FlFileError (FileId, ASL_MSG_WRITE);
-        AslAbort ();
-    }
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    FlSeekFile
- *
- * PARAMETERS:  FileId              - Index into file info array
- *              Offset              - Absolute byte offset in file
- *
- * RETURN:      None
- *
- * DESCRIPTION: Seek to absolute offset
- *              NOTE: Aborts compiler on any error.
- *
- ******************************************************************************/
-
-void
-FlSeekFile (
-    UINT32                  FileId,
-    long                    Offset)
-{
-    int                     Error;
-
-
-    Error = fseek (Gbl_Files[FileId].Handle, Offset, SEEK_SET);
-    if (Error)
-    {
-        FlFileError (FileId, ASL_MSG_SEEK);
-        AslAbort ();
-    }
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    FlCloseFile
- *
- * PARAMETERS:  FileId              - Index into file info array
- *
- * RETURN:      None
- *
- * DESCRIPTION: Close an open file. Aborts compiler on error
- *
- ******************************************************************************/
-
-void
-FlCloseFile (
-    UINT32                  FileId)
-{
-    int                     Error;
-
-
-    if (!Gbl_Files[FileId].Handle)
-    {
-        return;
-    }
-
-    Error = fclose (Gbl_Files[FileId].Handle);
-    if (Error)
-    {
-        FlFileError (FileId, ASL_MSG_CLOSE);
-        AslAbort ();
-    }
-
-    Gbl_Files[FileId].Handle = NULL;
-    return;
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    FlDeleteFile
- *
- * PARAMETERS:  FileId              - Index into file info array
- *
- * RETURN:      None
- *
- * DESCRIPTION: Delete a file.
- *
- ******************************************************************************/
-
-void
-FlDeleteFile (
-    UINT32                  FileId)
-{
-    ASL_FILE_INFO           *Info = &Gbl_Files[FileId];
-
-
-    if (!Info->Filename)
-    {
-        return;
-    }
-
-    if (remove (Info->Filename))
-    {
-        printf ("%s (%s file) ",
-            Info->Filename, Info->Description);
-        perror ("Could not delete");
-    }
-
-    Info->Filename = NULL;
-    return;
-}
-
-
-/*******************************************************************************
- *
  * FUNCTION:    FlSetLineNumber
  *
  * PARAMETERS:  Op        - Parse node for the LINE asl statement
diff --git a/sys/contrib/dev/acpica/compiler/aslfold.c b/sys/contrib/dev/acpica/compiler/aslfold.c
index e5b6520..c01bce4 100644
--- a/sys/contrib/dev/acpica/compiler/aslfold.c
+++ b/sys/contrib/dev/acpica/compiler/aslfold.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslglobal.h b/sys/contrib/dev/acpica/compiler/aslglobal.h
index 101b23d..1b1f67c 100644
--- a/sys/contrib/dev/acpica/compiler/aslglobal.h
+++ b/sys/contrib/dev/acpica/compiler/aslglobal.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslhex.c b/sys/contrib/dev/acpica/compiler/aslhex.c
new file mode 100644
index 0000000..c6b310e
--- /dev/null
+++ b/sys/contrib/dev/acpica/compiler/aslhex.c
@@ -0,0 +1,401 @@
+/******************************************************************************
+ *
+ * Module Name: aslhex - ASCII hex output file generation (C, ASM, and ASL)
+ *
+ *****************************************************************************/
+
+/*
+ * Copyright (C) 2000 - 2013, Intel Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ * 3. Neither the names of the above-listed copyright holders nor the names
+ *    of any contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ */
+
+
+#include <contrib/dev/acpica/compiler/aslcompiler.h>
+
+#define _COMPONENT          ACPI_COMPILER
+        ACPI_MODULE_NAME    ("ashex")
+
+/*
+ * This module emits ASCII hex output files in either C, ASM, or ASL format
+ */
+
+
+/* Local prototypes */
+
+static void
+HxDoHexOutputC (
+    void);
+
+static void
+HxDoHexOutputAsl (
+    void);
+
+static void
+HxDoHexOutputAsm (
+    void);
+
+static UINT32
+HxReadAmlOutputFile (
+    UINT8                   *Buffer);
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    HxDoHexOutput
+ *
+ * PARAMETERS:  None
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Create the hex output file. Note: data is obtained by reading
+ *              the entire AML output file that was previously generated.
+ *
+ ******************************************************************************/
+
+void
+HxDoHexOutput (
+    void)
+{
+
+    switch (Gbl_HexOutputFlag)
+    {
+    case HEX_OUTPUT_C:
+
+        HxDoHexOutputC ();
+        break;
+
+    case HEX_OUTPUT_ASM:
+
+        HxDoHexOutputAsm ();
+        break;
+
+    case HEX_OUTPUT_ASL:
+
+        HxDoHexOutputAsl ();
+        break;
+
+    default:
+        /* No other output types supported */
+        break;
+    }
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    HxReadAmlOutputFile
+ *
+ * PARAMETERS:  Buffer              - Where to return data
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Read a line of the AML output prior to formatting the data
+ *
+ ******************************************************************************/
+
+static UINT32
+HxReadAmlOutputFile (
+    UINT8                   *Buffer)
+{
+    UINT32                  Actual;
+
+
+    Actual = fread (Buffer, 1, HEX_TABLE_LINE_SIZE,
+        Gbl_Files[ASL_FILE_AML_OUTPUT].Handle);
+
+    if (ferror (Gbl_Files[ASL_FILE_AML_OUTPUT].Handle))
+    {
+        FlFileError (ASL_FILE_AML_OUTPUT, ASL_MSG_READ);
+        AslAbort ();
+    }
+
+    return (Actual);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    HxDoHexOutputC
+ *
+ * PARAMETERS:  None
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Create the hex output file. This is the same data as the AML
+ *              output file, but formatted into hex/ascii bytes suitable for
+ *              inclusion into a C source file.
+ *
+ ******************************************************************************/
+
+static void
+HxDoHexOutputC (
+    void)
+{
+    UINT8                   FileData[HEX_TABLE_LINE_SIZE];
+    UINT32                  LineLength;
+    UINT32                  Offset = 0;
+    UINT32                  AmlFileSize;
+    UINT32                  i;
+
+
+    /* Get AML size, seek back to start */
+
+    AmlFileSize = FlGetFileSize (ASL_FILE_AML_OUTPUT);
+    FlSeekFile (ASL_FILE_AML_OUTPUT, 0);
+
+    FlPrintFile (ASL_FILE_HEX_OUTPUT, " * C source code output\n");
+    FlPrintFile (ASL_FILE_HEX_OUTPUT, " * AML code block contains 0x%X bytes\n *\n */\n",
+        AmlFileSize);
+    FlPrintFile (ASL_FILE_HEX_OUTPUT, "unsigned char AmlCode[] =\n{\n");
+
+    while (Offset < AmlFileSize)
+    {
+        /* Read enough bytes needed for one output line */
+
+        LineLength = HxReadAmlOutputFile (FileData);
+        if (!LineLength)
+        {
+            break;
+        }
+
+        FlPrintFile (ASL_FILE_HEX_OUTPUT, "    ");
+
+        for (i = 0; i < LineLength; i++)
+        {
+            /*
+             * Print each hex byte.
+             * Add a comma until the very last byte of the AML file
+             * (Some C compilers complain about a trailing comma)
+             */
+            FlPrintFile (ASL_FILE_HEX_OUTPUT, "0x%2.2X", FileData[i]);
+            if ((Offset + i + 1) < AmlFileSize)
+            {
+                FlPrintFile (ASL_FILE_HEX_OUTPUT, ",");
+            }
+            else
+            {
+                FlPrintFile (ASL_FILE_HEX_OUTPUT, " ");
+            }
+        }
+
+        /* Add fill spaces if needed for last line */
+
+        if (LineLength < HEX_TABLE_LINE_SIZE)
+        {
+            FlPrintFile (ASL_FILE_HEX_OUTPUT, "%*s",
+                5 * (HEX_TABLE_LINE_SIZE - LineLength), " ");
+        }
+
+        /* Emit the offset and ascii dump for the entire line */
+
+        FlPrintFile (ASL_FILE_HEX_OUTPUT, "  /* %8.8X", Offset);
+        LsDumpAsciiInComment (ASL_FILE_HEX_OUTPUT, LineLength, FileData);
+        FlPrintFile (ASL_FILE_HEX_OUTPUT, "%*s*/\n",
+            HEX_TABLE_LINE_SIZE - LineLength + 1, " ");
+
+        Offset += LineLength;
+    }
+
+    FlPrintFile (ASL_FILE_HEX_OUTPUT, "};\n");
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    HxDoHexOutputAsl
+ *
+ * PARAMETERS:  None
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Create the hex output file. This is the same data as the AML
+ *              output file, but formatted into hex/ascii bytes suitable for
+ *              inclusion into a C source file.
+ *
+ ******************************************************************************/
+
+static void
+HxDoHexOutputAsl (
+    void)
+{
+    UINT8                   FileData[HEX_TABLE_LINE_SIZE];
+    UINT32                  LineLength;
+    UINT32                  Offset = 0;
+    UINT32                  AmlFileSize;
+    UINT32                  i;
+
+
+    /* Get AML size, seek back to start */
+
+    AmlFileSize = FlGetFileSize (ASL_FILE_AML_OUTPUT);
+    FlSeekFile (ASL_FILE_AML_OUTPUT, 0);
+
+    FlPrintFile (ASL_FILE_HEX_OUTPUT, " * ASL source code output\n");
+    FlPrintFile (ASL_FILE_HEX_OUTPUT, " * AML code block contains 0x%X bytes\n *\n */\n",
+        AmlFileSize);
+    FlPrintFile (ASL_FILE_HEX_OUTPUT, "    Name (BUF1, Buffer()\n    {\n");
+
+    while (Offset < AmlFileSize)
+    {
+        /* Read enough bytes needed for one output line */
+
+        LineLength = HxReadAmlOutputFile (FileData);
+        if (!LineLength)
+        {
+            break;
+        }
+
+        FlPrintFile (ASL_FILE_HEX_OUTPUT, "        ");
+
+        for (i = 0; i < LineLength; i++)
+        {
+            /*
+             * Print each hex byte.
+             * Add a comma until the very last byte of the AML file
+             * (Some C compilers complain about a trailing comma)
+             */
+            FlPrintFile (ASL_FILE_HEX_OUTPUT, "0x%2.2X", FileData[i]);
+            if ((Offset + i + 1) < AmlFileSize)
+            {
+                FlPrintFile (ASL_FILE_HEX_OUTPUT, ",");
+            }
+            else
+            {
+                FlPrintFile (ASL_FILE_HEX_OUTPUT, " ");
+            }
+        }
+
+        /* Add fill spaces if needed for last line */
+
+        if (LineLength < HEX_TABLE_LINE_SIZE)
+        {
+            FlPrintFile (ASL_FILE_HEX_OUTPUT, "%*s",
+                5 * (HEX_TABLE_LINE_SIZE - LineLength), " ");
+        }
+
+        /* Emit the offset and ascii dump for the entire line */
+
+        FlPrintFile (ASL_FILE_HEX_OUTPUT, "  /* %8.8X", Offset);
+        LsDumpAsciiInComment (ASL_FILE_HEX_OUTPUT, LineLength, FileData);
+        FlPrintFile (ASL_FILE_HEX_OUTPUT, "%*s*/\n",
+            HEX_TABLE_LINE_SIZE - LineLength + 1, " ");
+
+        Offset += LineLength;
+    }
+
+    FlPrintFile (ASL_FILE_HEX_OUTPUT, "    })\n");
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    HxDoHexOutputAsm
+ *
+ * PARAMETERS:  None
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Create the hex output file. This is the same data as the AML
+ *              output file, but formatted into hex/ascii bytes suitable for
+ *              inclusion into a ASM source file.
+ *
+ ******************************************************************************/
+
+static void
+HxDoHexOutputAsm (
+    void)
+{
+    UINT8                   FileData[HEX_TABLE_LINE_SIZE];
+    UINT32                  LineLength;
+    UINT32                  Offset = 0;
+    UINT32                  AmlFileSize;
+    UINT32                  i;
+
+
+    /* Get AML size, seek back to start */
+
+    AmlFileSize = FlGetFileSize (ASL_FILE_AML_OUTPUT);
+    FlSeekFile (ASL_FILE_AML_OUTPUT, 0);
+
+    FlPrintFile (ASL_FILE_HEX_OUTPUT, "; Assembly code source output\n");
+    FlPrintFile (ASL_FILE_HEX_OUTPUT, "; AML code block contains 0x%X bytes\n;\n",
+        AmlFileSize);
+
+    while (Offset < AmlFileSize)
+    {
+        /* Read enough bytes needed for one output line */
+
+        LineLength = HxReadAmlOutputFile (FileData);
+        if (!LineLength)
+        {
+            break;
+        }
+
+        FlPrintFile (ASL_FILE_HEX_OUTPUT, "  db  ");
+
+        for (i = 0; i < LineLength; i++)
+        {
+            /*
+             * Print each hex byte.
+             * Add a comma until the last byte of the line
+             */
+            FlPrintFile (ASL_FILE_HEX_OUTPUT, "0%2.2Xh", FileData[i]);
+            if ((i + 1) < LineLength)
+            {
+                FlPrintFile (ASL_FILE_HEX_OUTPUT, ",");
+            }
+        }
+
+        FlPrintFile (ASL_FILE_HEX_OUTPUT, " ");
+
+        /* Add fill spaces if needed for last line */
+
+        if (LineLength < HEX_TABLE_LINE_SIZE)
+        {
+            FlPrintFile (ASL_FILE_HEX_OUTPUT, "%*s",
+                5 * (HEX_TABLE_LINE_SIZE - LineLength), " ");
+        }
+
+        /* Emit the offset and ascii dump for the entire line */
+
+        FlPrintFile (ASL_FILE_HEX_OUTPUT, "  ; %8.8X", Offset);
+        LsDumpAsciiInComment (ASL_FILE_HEX_OUTPUT, LineLength, FileData);
+        FlPrintFile (ASL_FILE_HEX_OUTPUT, "\n");
+
+        Offset += LineLength;
+    }
+
+    FlPrintFile (ASL_FILE_HEX_OUTPUT, "\n");
+}
diff --git a/sys/contrib/dev/acpica/compiler/asllength.c b/sys/contrib/dev/acpica/compiler/asllength.c
index ef113f9..57835a6 100644
--- a/sys/contrib/dev/acpica/compiler/asllength.c
+++ b/sys/contrib/dev/acpica/compiler/asllength.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/asllisting.c b/sys/contrib/dev/acpica/compiler/asllisting.c
index 9a85fd7..49c43fb 100644
--- a/sys/contrib/dev/acpica/compiler/asllisting.c
+++ b/sys/contrib/dev/acpica/compiler/asllisting.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -59,12 +59,6 @@ LsDumpAscii (
     UINT32                  Count,
     UINT8                   *Buffer);
 
-static void
-LsDumpAsciiInComment (
-    UINT32                  FileId,
-    UINT32                  Count,
-    UINT8                   *Buffer);
-
 static ACPI_STATUS
 LsAmlListingWalk (
     ACPI_PARSE_OBJECT       *Op,
@@ -117,27 +111,57 @@ LsWriteNodeToListing (
     ACPI_PARSE_OBJECT       *Op,
     UINT32                  FileId);
 
-static void
-LsDoHexOutputC (
-    void);
-
-static void
-LsDoHexOutputAsm (
-    void);
-
-static void
-LsDoHexOutputAsl (
-    void);
-
 static ACPI_STATUS
 LsTreeWriteWalk (
     ACPI_PARSE_OBJECT       *Op,
     UINT32                  Level,
     void                    *Context);
 
-static UINT32
-LsReadAmlOutputFile (
-    UINT8                   *Buffer);
+#define ASL_LISTING_LINE_PREFIX         ":  "
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    LsDoListings
+ *
+ * PARAMETERS:  None
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Generate all requested listing files.
+ *
+ ******************************************************************************/
+
+void
+LsDoListings (
+    void)
+{
+
+    if (Gbl_C_OutputFlag)
+    {
+        LsGenerateListing (ASL_FILE_C_SOURCE_OUTPUT);
+    }
+
+    if (Gbl_ListingFlag)
+    {
+        LsGenerateListing (ASL_FILE_LISTING_OUTPUT);
+    }
+
+    if (Gbl_AsmOutputFlag)
+    {
+        LsGenerateListing (ASL_FILE_ASM_SOURCE_OUTPUT);
+    }
+
+    if (Gbl_C_IncludeOutputFlag)
+    {
+        LsGenerateListing (ASL_FILE_C_INCLUDE_OUTPUT);
+    }
+
+    if (Gbl_AsmIncludeOutputFlag)
+    {
+        LsGenerateListing (ASL_FILE_ASM_INCLUDE_OUTPUT);
+    }
+}
 
 
 /*******************************************************************************
@@ -147,7 +171,7 @@ LsReadAmlOutputFile (
  * PARAMETERS:  ASL_WALK_CALLBACK
  *
  *
- * RETURN:      None.
+ * RETURN:      None
  *
  * DESCRIPTION: Dump entire parse tree, for compiler debug only
  *
@@ -196,7 +220,7 @@ LsDumpParseTree (
  *              Count           - Number of bytes to convert
  *              Buffer          - Buffer of bytes to convert
  *
- * RETURN:      None.
+ * RETURN:      None
  *
  * DESCRIPTION: Convert hex bytes to ascii
  *
@@ -239,13 +263,13 @@ LsDumpAscii (
  *              Count           - Number of bytes to convert
  *              Buffer          - Buffer of bytes to convert
  *
- * RETURN:      None.
+ * RETURN:      None
  *
  * DESCRIPTION: Convert hex bytes to ascii
  *
  ******************************************************************************/
 
-static void
+void
 LsDumpAsciiInComment (
     UINT32                  FileId,
     UINT32                  Count,
@@ -374,50 +398,6 @@ LsGenerateListing (
 
 /*******************************************************************************
  *
- * FUNCTION:    LsDoListings
- *
- * PARAMETERS:  None.
- *
- * RETURN:      None
- *
- * DESCRIPTION: Generate all requested listing files.
- *
- ******************************************************************************/
-
-void
-LsDoListings (
-    void)
-{
-
-    if (Gbl_C_OutputFlag)
-    {
-        LsGenerateListing (ASL_FILE_C_SOURCE_OUTPUT);
-    }
-
-    if (Gbl_ListingFlag)
-    {
-        LsGenerateListing (ASL_FILE_LISTING_OUTPUT);
-    }
-
-    if (Gbl_AsmOutputFlag)
-    {
-        LsGenerateListing (ASL_FILE_ASM_SOURCE_OUTPUT);
-    }
-
-    if (Gbl_C_IncludeOutputFlag)
-    {
-        LsGenerateListing (ASL_FILE_C_INCLUDE_OUTPUT);
-    }
-
-    if (Gbl_AsmIncludeOutputFlag)
-    {
-        LsGenerateListing (ASL_FILE_ASM_INCLUDE_OUTPUT);
-    }
-}
-
-
-/*******************************************************************************
- *
  * FUNCTION:    LsPushNode
  *
  * PARAMETERS:  Filename        - Pointer to the include filename
@@ -687,7 +667,8 @@ LsWriteListingHexBytes (
             {
             case ASL_FILE_LISTING_OUTPUT:
 
-                FlPrintFile (FileId, "%8.8X....", Gbl_CurrentAmlOffset);
+                FlPrintFile (FileId, "%8.8X%s", Gbl_CurrentAmlOffset,
+                    ASL_LISTING_LINE_PREFIX);
                 break;
 
             case ASL_FILE_ASM_SOURCE_OUTPUT:
@@ -747,6 +728,24 @@ LsWriteOneSourceLine (
     Gbl_SourceLine++;
     Gbl_ListingNode->LineNumber++;
 
+    /* Ignore lines that are completely blank (but count the line above) */
+
+    if (FlReadFile (ASL_FILE_SOURCE_OUTPUT, &FileByte, 1) != AE_OK)
+    {
+        return (0);
+    }
+    if (FileByte == '\n')
+    {
+        return (1);
+    }
+
+    /*
+     * This is a non-empty line, we will print the entire line with
+     * the line number and possibly other prefixes and transforms.
+     */
+
+    /* Line prefixes for special files, C and ASM output */
+
     if (FileId == ASL_FILE_C_SOURCE_OUTPUT)
     {
         FlPrintFile (FileId, "     *");
@@ -762,19 +761,21 @@ LsWriteOneSourceLine (
          * This file contains "include" statements, print the current
          * filename and line number within the current file
          */
-        FlPrintFile (FileId, "%12s %5d....",
-                    Gbl_ListingNode->Filename, Gbl_ListingNode->LineNumber);
+        FlPrintFile (FileId, "%12s %5d%s",
+            Gbl_ListingNode->Filename, Gbl_ListingNode->LineNumber,
+            ASL_LISTING_LINE_PREFIX);
     }
     else
     {
         /* No include files, just print the line number */
 
-        FlPrintFile (FileId, "%8d....", Gbl_SourceLine);
+        FlPrintFile (FileId, "%8u%s", Gbl_SourceLine,
+            ASL_LISTING_LINE_PREFIX);
     }
 
-    /* Read one line (up to a newline or EOF) */
+    /* Read the rest of this line (up to a newline or EOF) */
 
-    while (FlReadFile (ASL_FILE_SOURCE_OUTPUT, &FileByte, 1) == AE_OK)
+    do
     {
         if (FileId == ASL_FILE_C_SOURCE_OUTPUT)
         {
@@ -788,13 +789,15 @@ LsWriteOneSourceLine (
         if (FileByte == '\n')
         {
             /*
+             * This line has been completed.
              * Check if an error occurred on this source line during the compile.
              * If so, we print the error message after the source line.
              */
             LsCheckException (Gbl_SourceLine, FileId);
             return (1);
         }
-    }
+
+    } while (FlReadFile (ASL_FILE_SOURCE_OUTPUT, &FileByte, 1) == AE_OK);
 
     /* EOF on the input file was reached */
 
@@ -1241,332 +1244,3 @@ LsWriteNodeToListing (
         break;
     }
 }
-
-
-/*******************************************************************************
- *
- * FUNCTION:    LsDoHexOutput
- *
- * PARAMETERS:  None
- *
- * RETURN:      None.
- *
- * DESCRIPTION: Create the hex output file.
- *
- ******************************************************************************/
-
-void
-LsDoHexOutput (
-    void)
-{
-
-    switch (Gbl_HexOutputFlag)
-    {
-    case HEX_OUTPUT_C:
-
-        LsDoHexOutputC ();
-        break;
-
-    case HEX_OUTPUT_ASM:
-
-        LsDoHexOutputAsm ();
-        break;
-
-    case HEX_OUTPUT_ASL:
-
-        LsDoHexOutputAsl ();
-        break;
-
-    default:
-        /* No other output types supported */
-        break;
-    }
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    LsReadAmlOutputFile
- *
- * PARAMETERS:  Buffer              - Where to return data
- *
- * RETURN:      None.
- *
- * DESCRIPTION: Read a line of the AML output prior to formatting the data
- *
- ******************************************************************************/
-
-static UINT32
-LsReadAmlOutputFile (
-    UINT8                   *Buffer)
-{
-    UINT32                  Actual;
-
-
-    Actual = fread (Buffer, 1, HEX_TABLE_LINE_SIZE,
-        Gbl_Files[ASL_FILE_AML_OUTPUT].Handle);
-
-    if (ferror (Gbl_Files[ASL_FILE_AML_OUTPUT].Handle))
-    {
-        FlFileError (ASL_FILE_AML_OUTPUT, ASL_MSG_READ);
-        AslAbort ();
-    }
-
-    return (Actual);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    LsDoHexOutputC
- *
- * PARAMETERS:  None
- *
- * RETURN:      None.
- *
- * DESCRIPTION: Create the hex output file. This is the same data as the AML
- *              output file, but formatted into hex/ascii bytes suitable for
- *              inclusion into a C source file.
- *
- ******************************************************************************/
-
-static void
-LsDoHexOutputC (
-    void)
-{
-    UINT8                   FileData[HEX_TABLE_LINE_SIZE];
-    UINT32                  LineLength;
-    UINT32                  Offset = 0;
-    UINT32                  AmlFileSize;
-    UINT32                  i;
-
-
-    /* Get AML size, seek back to start */
-
-    AmlFileSize = FlGetFileSize (ASL_FILE_AML_OUTPUT);
-    FlSeekFile (ASL_FILE_AML_OUTPUT, 0);
-
-    FlPrintFile (ASL_FILE_HEX_OUTPUT, " * C source code output\n");
-    FlPrintFile (ASL_FILE_HEX_OUTPUT, " * AML code block contains 0x%X bytes\n *\n */\n",
-        AmlFileSize);
-    FlPrintFile (ASL_FILE_HEX_OUTPUT, "unsigned char AmlCode[] =\n{\n");
-
-    while (Offset < AmlFileSize)
-    {
-        /* Read enough bytes needed for one output line */
-
-        LineLength = LsReadAmlOutputFile (FileData);
-        if (!LineLength)
-        {
-            break;
-        }
-
-        FlPrintFile (ASL_FILE_HEX_OUTPUT, "    ");
-
-        for (i = 0; i < LineLength; i++)
-        {
-            /*
-             * Print each hex byte.
-             * Add a comma until the very last byte of the AML file
-             * (Some C compilers complain about a trailing comma)
-             */
-            FlPrintFile (ASL_FILE_HEX_OUTPUT, "0x%2.2X", FileData[i]);
-            if ((Offset + i + 1) < AmlFileSize)
-            {
-                FlPrintFile (ASL_FILE_HEX_OUTPUT, ",");
-            }
-            else
-            {
-                FlPrintFile (ASL_FILE_HEX_OUTPUT, " ");
-            }
-        }
-
-        /* Add fill spaces if needed for last line */
-
-        if (LineLength < HEX_TABLE_LINE_SIZE)
-        {
-            FlPrintFile (ASL_FILE_HEX_OUTPUT, "%*s",
-                5 * (HEX_TABLE_LINE_SIZE - LineLength), " ");
-        }
-
-        /* Emit the offset and ascii dump for the entire line */
-
-        FlPrintFile (ASL_FILE_HEX_OUTPUT, "  /* %8.8X", Offset);
-        LsDumpAsciiInComment (ASL_FILE_HEX_OUTPUT, LineLength, FileData);
-        FlPrintFile (ASL_FILE_HEX_OUTPUT, "%*s*/\n",
-            HEX_TABLE_LINE_SIZE - LineLength + 1, " ");
-
-        Offset += LineLength;
-    }
-
-    FlPrintFile (ASL_FILE_HEX_OUTPUT, "};\n");
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    LsDoHexOutputAsl
- *
- * PARAMETERS:  None
- *
- * RETURN:      None.
- *
- * DESCRIPTION: Create the hex output file. This is the same data as the AML
- *              output file, but formatted into hex/ascii bytes suitable for
- *              inclusion into a C source file.
- *
- ******************************************************************************/
-
-static void
-LsDoHexOutputAsl (
-    void)
-{
-    UINT8                   FileData[HEX_TABLE_LINE_SIZE];
-    UINT32                  LineLength;
-    UINT32                  Offset = 0;
-    UINT32                  AmlFileSize;
-    UINT32                  i;
-
-
-    /* Get AML size, seek back to start */
-
-    AmlFileSize = FlGetFileSize (ASL_FILE_AML_OUTPUT);
-    FlSeekFile (ASL_FILE_AML_OUTPUT, 0);
-
-    FlPrintFile (ASL_FILE_HEX_OUTPUT, " * ASL source code output\n");
-    FlPrintFile (ASL_FILE_HEX_OUTPUT, " * AML code block contains 0x%X bytes\n *\n */\n",
-        AmlFileSize);
-    FlPrintFile (ASL_FILE_HEX_OUTPUT, "    Name (BUF1, Buffer()\n    {\n");
-
-    while (Offset < AmlFileSize)
-    {
-        /* Read enough bytes needed for one output line */
-
-        LineLength = LsReadAmlOutputFile (FileData);
-        if (!LineLength)
-        {
-            break;
-        }
-
-        FlPrintFile (ASL_FILE_HEX_OUTPUT, "        ");
-
-        for (i = 0; i < LineLength; i++)
-        {
-            /*
-             * Print each hex byte.
-             * Add a comma until the very last byte of the AML file
-             * (Some C compilers complain about a trailing comma)
-             */
-            FlPrintFile (ASL_FILE_HEX_OUTPUT, "0x%2.2X", FileData[i]);
-            if ((Offset + i + 1) < AmlFileSize)
-            {
-                FlPrintFile (ASL_FILE_HEX_OUTPUT, ",");
-            }
-            else
-            {
-                FlPrintFile (ASL_FILE_HEX_OUTPUT, " ");
-            }
-        }
-
-        /* Add fill spaces if needed for last line */
-
-        if (LineLength < HEX_TABLE_LINE_SIZE)
-        {
-            FlPrintFile (ASL_FILE_HEX_OUTPUT, "%*s",
-                5 * (HEX_TABLE_LINE_SIZE - LineLength), " ");
-        }
-
-        /* Emit the offset and ascii dump for the entire line */
-
-        FlPrintFile (ASL_FILE_HEX_OUTPUT, "  /* %8.8X", Offset);
-        LsDumpAsciiInComment (ASL_FILE_HEX_OUTPUT, LineLength, FileData);
-        FlPrintFile (ASL_FILE_HEX_OUTPUT, "%*s*/\n",
-            HEX_TABLE_LINE_SIZE - LineLength + 1, " ");
-
-        Offset += LineLength;
-    }
-
-    FlPrintFile (ASL_FILE_HEX_OUTPUT, "    })\n");
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    LsDoHexOutputAsm
- *
- * PARAMETERS:  None
- *
- * RETURN:      None.
- *
- * DESCRIPTION: Create the hex output file. This is the same data as the AML
- *              output file, but formatted into hex/ascii bytes suitable for
- *              inclusion into a ASM source file.
- *
- ******************************************************************************/
-
-static void
-LsDoHexOutputAsm (
-    void)
-{
-    UINT8                   FileData[HEX_TABLE_LINE_SIZE];
-    UINT32                  LineLength;
-    UINT32                  Offset = 0;
-    UINT32                  AmlFileSize;
-    UINT32                  i;
-
-
-    /* Get AML size, seek back to start */
-
-    AmlFileSize = FlGetFileSize (ASL_FILE_AML_OUTPUT);
-    FlSeekFile (ASL_FILE_AML_OUTPUT, 0);
-
-    FlPrintFile (ASL_FILE_HEX_OUTPUT, "; Assembly code source output\n");
-    FlPrintFile (ASL_FILE_HEX_OUTPUT, "; AML code block contains 0x%X bytes\n;\n",
-        AmlFileSize);
-
-    while (Offset < AmlFileSize)
-    {
-        /* Read enough bytes needed for one output line */
-
-        LineLength = LsReadAmlOutputFile (FileData);
-        if (!LineLength)
-        {
-            break;
-        }
-
-        FlPrintFile (ASL_FILE_HEX_OUTPUT, "  db  ");
-
-        for (i = 0; i < LineLength; i++)
-        {
-            /*
-             * Print each hex byte.
-             * Add a comma until the last byte of the line
-             */
-            FlPrintFile (ASL_FILE_HEX_OUTPUT, "0%2.2Xh", FileData[i]);
-            if ((i + 1) < LineLength)
-            {
-                FlPrintFile (ASL_FILE_HEX_OUTPUT, ",");
-            }
-        }
-
-        FlPrintFile (ASL_FILE_HEX_OUTPUT, " ");
-
-        /* Add fill spaces if needed for last line */
-
-        if (LineLength < HEX_TABLE_LINE_SIZE)
-        {
-            FlPrintFile (ASL_FILE_HEX_OUTPUT, "%*s",
-                5 * (HEX_TABLE_LINE_SIZE - LineLength), " ");
-        }
-
-        /* Emit the offset and ascii dump for the entire line */
-
-        FlPrintFile (ASL_FILE_HEX_OUTPUT, "  ; %8.8X", Offset);
-        LsDumpAsciiInComment (ASL_FILE_HEX_OUTPUT, LineLength, FileData);
-        FlPrintFile (ASL_FILE_HEX_OUTPUT, "\n");
-
-        Offset += LineLength;
-    }
-
-    FlPrintFile (ASL_FILE_HEX_OUTPUT, "\n");
-}
diff --git a/sys/contrib/dev/acpica/compiler/aslload.c b/sys/contrib/dev/acpica/compiler/aslload.c
index 907dc14..813e603 100644
--- a/sys/contrib/dev/acpica/compiler/aslload.c
+++ b/sys/contrib/dev/acpica/compiler/aslload.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/asllookup.c b/sys/contrib/dev/acpica/compiler/asllookup.c
index 9af193b..67952a6 100644
--- a/sys/contrib/dev/acpica/compiler/asllookup.c
+++ b/sys/contrib/dev/acpica/compiler/asllookup.c
@@ -1,11 +1,11 @@
 /******************************************************************************
  *
- * Module Name: asllookup- Namespace lookup
+ * Module Name: asllookup- Namespace lookup functions
  *
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -44,7 +44,6 @@
 
 #include <contrib/dev/acpica/compiler/aslcompiler.h>
 #include "aslcompiler.y.h"
-
 #include <contrib/dev/acpica/include/acparser.h>
 #include <contrib/dev/acpica/include/amlcode.h>
 #include <contrib/dev/acpica/include/acnamesp.h>
@@ -57,57 +56,12 @@
 /* Local prototypes */
 
 static ACPI_STATUS
-LsCompareOneNamespaceObject (
-    ACPI_HANDLE             ObjHandle,
-    UINT32                  Level,
-    void                    *Context,
-    void                    **ReturnValue);
-
-static ACPI_STATUS
-LsDoOneNamespaceObject (
-    ACPI_HANDLE             ObjHandle,
-    UINT32                  Level,
-    void                    *Context,
-    void                    **ReturnValue);
-
-static BOOLEAN
-LkObjectExists (
-    char                    *Name);
-
-static void
-LkCheckFieldRange (
-    ACPI_PARSE_OBJECT       *Op,
-    UINT32                  RegionBitLength,
-    UINT32                  FieldBitOffset,
-    UINT32                  FieldBitLength,
-    UINT32                  AccessBitWidth);
-
-static ACPI_STATUS
-LkNamespaceLocateBegin (
-    ACPI_PARSE_OBJECT       *Op,
-    UINT32                  Level,
-    void                    *Context);
-
-static ACPI_STATUS
-LkNamespaceLocateEnd (
-    ACPI_PARSE_OBJECT       *Op,
-    UINT32                  Level,
-    void                    *Context);
-
-static ACPI_STATUS
 LkIsObjectUsed (
     ACPI_HANDLE             ObjHandle,
     UINT32                  Level,
     void                    *Context,
     void                    **ReturnValue);
 
-static ACPI_STATUS
-LsDoOnePathname (
-    ACPI_HANDLE             ObjHandle,
-    UINT32                  Level,
-    void                    *Context,
-    void                    **ReturnValue);
-
 static ACPI_PARSE_OBJECT *
 LkGetNameOp (
     ACPI_PARSE_OBJECT       *Op);
@@ -115,480 +69,28 @@ LkGetNameOp (
 
 /*******************************************************************************
  *
- * FUNCTION:    LsDoOneNamespaceObject
- *
- * PARAMETERS:  ACPI_WALK_CALLBACK
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Dump a namespace object to the namespace output file.
- *              Called during the walk of the namespace to dump all objects.
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-LsDoOneNamespaceObject (
-    ACPI_HANDLE             ObjHandle,
-    UINT32                  Level,
-    void                    *Context,
-    void                    **ReturnValue)
-{
-    ACPI_NAMESPACE_NODE     *Node = (ACPI_NAMESPACE_NODE *) ObjHandle;
-    ACPI_OPERAND_OBJECT     *ObjDesc;
-    ACPI_PARSE_OBJECT       *Op;
-
-
-    Gbl_NumNamespaceObjects++;
-
-    FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "%5u  [%u]  %*s %4.4s - %s",
-        Gbl_NumNamespaceObjects, Level, (Level * 3), " ",
-        &Node->Name,
-        AcpiUtGetTypeName (Node->Type));
-
-    Op = Node->Op;
-    ObjDesc = ACPI_CAST_PTR (ACPI_OPERAND_OBJECT, Node->Object);
-
-    if (!Op)
-    {
-        FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "\n");
-        return (AE_OK);
-    }
-
-
-    if ((ObjDesc) &&
-        (ACPI_GET_DESCRIPTOR_TYPE (ObjDesc) == ACPI_DESC_TYPE_OPERAND))
-    {
-        switch (Node->Type)
-        {
-        case ACPI_TYPE_INTEGER:
-
-            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
-                "       [Initial Value   0x%8.8X%8.8X]",
-                ACPI_FORMAT_UINT64 (ObjDesc->Integer.Value));
-            break;
-
-
-        case ACPI_TYPE_STRING:
-
-            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
-                "        [Initial Value   \"%s\"]",
-                ObjDesc->String.Pointer);
-            break;
-
-        default:
-            /* Nothing to do for other types */
-            break;
-        }
-
-    }
-    else
-    {
-        switch (Node->Type)
-        {
-        case ACPI_TYPE_INTEGER:
-
-            if (Op->Asl.ParseOpcode == PARSEOP_NAME)
-            {
-                Op = Op->Asl.Child;
-            }
-            if ((Op->Asl.ParseOpcode == PARSEOP_NAMESEG)  ||
-                (Op->Asl.ParseOpcode == PARSEOP_NAMESTRING))
-            {
-                Op = Op->Asl.Next;
-            }
-            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
-                "       [Initial Value   0x%8.8X%8.8X]",
-                ACPI_FORMAT_UINT64 (Op->Asl.Value.Integer));
-            break;
-
-
-        case ACPI_TYPE_STRING:
-
-            if (Op->Asl.ParseOpcode == PARSEOP_NAME)
-            {
-                Op = Op->Asl.Child;
-            }
-            if ((Op->Asl.ParseOpcode == PARSEOP_NAMESEG)  ||
-                (Op->Asl.ParseOpcode == PARSEOP_NAMESTRING))
-            {
-                Op = Op->Asl.Next;
-            }
-            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
-                "        [Initial Value   \"%s\"]",
-                Op->Asl.Value.String);
-            break;
-
-
-        case ACPI_TYPE_LOCAL_REGION_FIELD:
-
-            if ((Op->Asl.ParseOpcode == PARSEOP_NAMESEG)  ||
-                (Op->Asl.ParseOpcode == PARSEOP_NAMESTRING))
-            {
-                Op = Op->Asl.Child;
-            }
-            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
-                "   [Offset 0x%04X   Length 0x%04X bits]",
-                Op->Asl.Parent->Asl.ExtraValue, (UINT32) Op->Asl.Value.Integer);
-            break;
-
-
-        case ACPI_TYPE_BUFFER_FIELD:
-
-            switch (Op->Asl.ParseOpcode)
-            {
-            case PARSEOP_CREATEBYTEFIELD:
-                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "   [BYTE  ( 8 bit)]");
-                break;
-
-            case PARSEOP_CREATEDWORDFIELD:
-                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "   [DWORD (32 bit)]");
-                break;
-
-            case PARSEOP_CREATEQWORDFIELD:
-                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "   [QWORD (64 bit)]");
-                break;
-
-            case PARSEOP_CREATEWORDFIELD:
-                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "   [WORD  (16 bit)]");
-                break;
-
-            case PARSEOP_CREATEBITFIELD:
-                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "   [BIT   ( 1 bit)]");
-                break;
-
-            case PARSEOP_CREATEFIELD:
-                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "   [Arbitrary Bit Field]");
-                break;
-
-            default:
-                break;
-
-            }
-            break;
-
-
-        case ACPI_TYPE_PACKAGE:
-
-            if (Op->Asl.ParseOpcode == PARSEOP_NAME)
-            {
-                Op = Op->Asl.Child;
-            }
-            if ((Op->Asl.ParseOpcode == PARSEOP_NAMESEG)  ||
-                (Op->Asl.ParseOpcode == PARSEOP_NAMESTRING))
-            {
-                Op = Op->Asl.Next;
-            }
-            Op = Op->Asl.Child;
-
-            if ((Op->Asl.ParseOpcode == PARSEOP_BYTECONST) ||
-                (Op->Asl.ParseOpcode == PARSEOP_RAW_DATA))
-            {
-                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
-                    "       [Initial Length  0x%.2X elements]",
-                    Op->Asl.Value.Integer);
-            }
-            break;
-
-
-        case ACPI_TYPE_BUFFER:
-
-            if (Op->Asl.ParseOpcode == PARSEOP_NAME)
-            {
-                Op = Op->Asl.Child;
-            }
-            if ((Op->Asl.ParseOpcode == PARSEOP_NAMESEG)  ||
-                (Op->Asl.ParseOpcode == PARSEOP_NAMESTRING))
-            {
-                Op = Op->Asl.Next;
-            }
-            Op = Op->Asl.Child;
-
-            if (Op && (Op->Asl.ParseOpcode == PARSEOP_INTEGER))
-            {
-                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
-                    "        [Initial Length  0x%.2X bytes]",
-                    Op->Asl.Value.Integer);
-            }
-            break;
-
-
-        case ACPI_TYPE_METHOD:
-
-            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
-                "        [Code Length     0x%.4X bytes]",
-                Op->Asl.AmlSubtreeLength);
-            break;
-
-
-        case ACPI_TYPE_LOCAL_RESOURCE:
-
-            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
-                "  [Desc Offset     0x%.4X Bytes]", Node->Value);
-            break;
-
-
-        case ACPI_TYPE_LOCAL_RESOURCE_FIELD:
-
-            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
-                "   [Field Offset    0x%.4X Bits 0x%.4X Bytes] ",
-                Node->Value, Node->Value / 8);
-
-            if (Node->Flags & ANOBJ_IS_REFERENCED)
-            {
-                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
-                    "Referenced");
-            }
-            else
-            {
-                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
-                    "Name not referenced");
-            }
-            break;
-
-
-        default:
-            /* Nothing to do for other types */
-            break;
-        }
-    }
-
-    FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "\n");
-    return (AE_OK);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    LsSetupNsList
+ * FUNCTION:    LkFindUnreferencedObjects
  *
- * PARAMETERS:  Handle          - local file handle
+ * PARAMETERS:  None
  *
  * RETURN:      None
  *
- * DESCRIPTION: Set the namespace output file to the input handle
+ * DESCRIPTION: Namespace walk to find objects that are not referenced in any
+ *              way. Must be called after the namespace has been cross
+ *              referenced.
  *
  ******************************************************************************/
 
 void
-LsSetupNsList (
-    void                    *Handle)
-{
-
-    Gbl_NsOutputFlag = TRUE;
-    Gbl_Files[ASL_FILE_NAMESPACE_OUTPUT].Handle = Handle;
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    LsDoOnePathname
- *
- * PARAMETERS:  ACPI_WALK_CALLBACK
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Print the full pathname for a namespace node.
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-LsDoOnePathname (
-    ACPI_HANDLE             ObjHandle,
-    UINT32                  Level,
-    void                    *Context,
-    void                    **ReturnValue)
-{
-    ACPI_NAMESPACE_NODE     *Node = (ACPI_NAMESPACE_NODE *) ObjHandle;
-    ACPI_STATUS             Status;
-    ACPI_BUFFER             TargetPath;
-
-
-    TargetPath.Length = ACPI_ALLOCATE_LOCAL_BUFFER;
-    Status = AcpiNsHandleToPathname (Node, &TargetPath);
-    if (ACPI_FAILURE (Status))
-    {
-        return (Status);
-    }
-
-    FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "%s\n", TargetPath.Pointer);
-    ACPI_FREE (TargetPath.Pointer);
-
-    return (AE_OK);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    LsDisplayNamespace
- *
- * PARAMETERS:  None
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Walk the namespace an display information about each node
- *              in the tree. Information is written to the optional
- *              namespace output file.
- *
- ******************************************************************************/
-
-ACPI_STATUS
-LsDisplayNamespace (
+LkFindUnreferencedObjects (
     void)
 {
-    ACPI_STATUS             Status;
-
-
-    if (!Gbl_NsOutputFlag)
-    {
-        return (AE_OK);
-    }
-
-    Gbl_NumNamespaceObjects = 0;
-
-    /* File header */
-
-    FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "Contents of ACPI Namespace\n\n");
-    FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "Count  Depth    Name - Type\n\n");
-
-    /* Walk entire namespace from the root */
-
-    Status = AcpiNsWalkNamespace (ACPI_TYPE_ANY, ACPI_ROOT_OBJECT,
-                ACPI_UINT32_MAX, FALSE, LsDoOneNamespaceObject, NULL,
-                NULL, NULL);
-
-    /* Print the full pathname for each namespace node */
-
-    FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "\nNamespace pathnames\n\n");
-
-    Status = AcpiNsWalkNamespace (ACPI_TYPE_ANY, ACPI_ROOT_OBJECT,
-                ACPI_UINT32_MAX, FALSE, LsDoOnePathname, NULL,
-                NULL, NULL);
-
-    return (Status);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    LsCompareOneNamespaceObject
- *
- * PARAMETERS:  ACPI_WALK_CALLBACK
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Compare name of one object.
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-LsCompareOneNamespaceObject (
-    ACPI_HANDLE             ObjHandle,
-    UINT32                  Level,
-    void                    *Context,
-    void                    **ReturnValue)
-{
-    ACPI_NAMESPACE_NODE     *Node = (ACPI_NAMESPACE_NODE *) ObjHandle;
-
-
-    /* Simply check the name */
-
-    if (*((UINT32 *) (Context)) == Node->Name.Integer)
-    {
-        /* Abort walk if we found one instance */
-
-        return (AE_CTRL_TRUE);
-    }
-
-    return (AE_OK);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    LkObjectExists
- *
- * PARAMETERS:  Name            - 4 char ACPI name
- *
- * RETURN:      TRUE if name exists in namespace
- *
- * DESCRIPTION: Walk the namespace to find an object
- *
- ******************************************************************************/
-
-static BOOLEAN
-LkObjectExists (
-    char                    *Name)
-{
-    ACPI_STATUS             Status;
-
 
     /* Walk entire namespace from the supplied root */
 
-    Status = AcpiNsWalkNamespace (ACPI_TYPE_ANY, ACPI_ROOT_OBJECT,
-                ACPI_UINT32_MAX, FALSE, LsCompareOneNamespaceObject, NULL,
-                Name, NULL);
-    if (Status == AE_CTRL_TRUE)
-    {
-        /* At least one instance of the name was found */
-
-        return (TRUE);
-    }
-
-    return (FALSE);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    LkGetNameOp
- *
- * PARAMETERS:  Op              - Current Op
- *
- * RETURN:      NameOp associated with the input op
- *
- * DESCRIPTION: Find the name declaration op associated with the operator
- *
- ******************************************************************************/
-
-static ACPI_PARSE_OBJECT *
-LkGetNameOp (
-    ACPI_PARSE_OBJECT       *Op)
-{
-    const ACPI_OPCODE_INFO  *OpInfo;
-    ACPI_PARSE_OBJECT       *NameOp = Op;
-
-
-    OpInfo = AcpiPsGetOpcodeInfo (Op->Asl.AmlOpcode);
-
-
-    /* Get the NamePath from the appropriate place */
-
-    if (OpInfo->Flags & AML_NAMED)
-    {
-        /* For nearly all NAMED operators, the name reference is the first child */
-
-        NameOp = Op->Asl.Child;
-        if (Op->Asl.AmlOpcode == AML_ALIAS_OP)
-        {
-            /*
-             * ALIAS is the only oddball opcode, the name declaration
-             * (alias name) is the second operand
-             */
-            NameOp = Op->Asl.Child->Asl.Next;
-        }
-    }
-    else if (OpInfo->Flags & AML_CREATE)
-    {
-        /* Name must appear as the last parameter */
-
-        NameOp = Op->Asl.Child;
-        while (!(NameOp->Asl.CompileFlags & NODE_IS_NAME_DECLARATION))
-        {
-            NameOp = NameOp->Asl.Next;
-        }
-    }
-
-    return (NameOp);
+    (void) AcpiNsWalkNamespace (ACPI_TYPE_ANY, ACPI_ROOT_OBJECT,
+                ACPI_UINT32_MAX, FALSE, LkIsObjectUsed, NULL,
+                NULL, NULL);
 }
 
 
@@ -660,232 +162,26 @@ LkIsObjectUsed (
 
 /*******************************************************************************
  *
- * FUNCTION:    LkFindUnreferencedObjects
- *
- * PARAMETERS:  None
- *
- * RETURN:      None
- *
- * DESCRIPTION: Namespace walk to find objects that are not referenced in any
- *              way. Must be called after the namespace has been cross
- *              referenced.
- *
- ******************************************************************************/
-
-void
-LkFindUnreferencedObjects (
-    void)
-{
-
-    /* Walk entire namespace from the supplied root */
-
-    (void) AcpiNsWalkNamespace (ACPI_TYPE_ANY, ACPI_ROOT_OBJECT,
-                ACPI_UINT32_MAX, FALSE, LkIsObjectUsed, NULL,
-                NULL, NULL);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    LkCrossReferenceNamespace
- *
- * PARAMETERS:  None
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Perform a cross reference check of the parse tree against the
- *              namespace. Every named referenced within the parse tree
- *              should be get resolved with a namespace lookup. If not, the
- *              original reference in the ASL code is invalid -- i.e., refers
- *              to a non-existent object.
- *
- * NOTE:  The ASL "External" operator causes the name to be inserted into the
- *        namespace so that references to the external name will be resolved
- *        correctly here.
- *
- ******************************************************************************/
-
-ACPI_STATUS
-LkCrossReferenceNamespace (
-    void)
-{
-    ACPI_WALK_STATE         *WalkState;
-
-
-    DbgPrint (ASL_DEBUG_OUTPUT, "\nCross referencing namespace\n\n");
-
-    /*
-     * Create a new walk state for use when looking up names
-     * within the namespace (Passed as context to the callbacks)
-     */
-    WalkState = AcpiDsCreateWalkState (0, NULL, NULL, NULL);
-    if (!WalkState)
-    {
-        return (AE_NO_MEMORY);
-    }
-
-    /* Walk the entire parse tree */
-
-    TrWalkParseTree (RootNode, ASL_WALK_VISIT_TWICE, LkNamespaceLocateBegin,
-                        LkNamespaceLocateEnd, WalkState);
-    return (AE_OK);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    LkCheckFieldRange
- *
- * PARAMETERS:  RegionBitLength     - Length of entire parent region
- *              FieldBitOffset      - Start of the field unit (within region)
- *              FieldBitLength      - Entire length of field unit
- *              AccessBitWidth      - Access width of the field unit
- *
- * RETURN:      None
- *
- * DESCRIPTION: Check one field unit to make sure it fits in the parent
- *              op region.
- *
- * Note: AccessBitWidth must be either 8,16,32, or 64
- *
- ******************************************************************************/
-
-static void
-LkCheckFieldRange (
-    ACPI_PARSE_OBJECT       *Op,
-    UINT32                  RegionBitLength,
-    UINT32                  FieldBitOffset,
-    UINT32                  FieldBitLength,
-    UINT32                  AccessBitWidth)
-{
-    UINT32                  FieldEndBitOffset;
-
-
-    /*
-     * Check each field unit against the region size. The entire
-     * field unit (start offset plus length) must fit within the
-     * region.
-     */
-    FieldEndBitOffset = FieldBitOffset + FieldBitLength;
-
-    if (FieldEndBitOffset > RegionBitLength)
-    {
-        /* Field definition itself is beyond the end-of-region */
-
-        AslError (ASL_ERROR, ASL_MSG_FIELD_UNIT_OFFSET, Op, NULL);
-        return;
-    }
-
-    /*
-     * Now check that the field plus AccessWidth doesn't go beyond
-     * the end-of-region. Assumes AccessBitWidth is a power of 2
-     */
-    FieldEndBitOffset = ACPI_ROUND_UP (FieldEndBitOffset, AccessBitWidth);
-
-    if (FieldEndBitOffset > RegionBitLength)
-    {
-        /* Field definition combined with the access is beyond EOR */
-
-        AslError (ASL_ERROR, ASL_MSG_FIELD_UNIT_ACCESS_WIDTH, Op, NULL);
-    }
-}
-
-/*******************************************************************************
- *
- * FUNCTION:    LkNamespaceLocateBegin
- *
- * PARAMETERS:  ASL_WALK_CALLBACK
+ * FUNCTION:    LkGetNameOp
  *
- * RETURN:      Status
+ * PARAMETERS:  Op              - Current Op
  *
- * DESCRIPTION: Descending callback used during cross-reference. For named
- *              object references, attempt to locate the name in the
- *              namespace.
+ * RETURN:      NameOp associated with the input op
  *
- * NOTE: ASL references to named fields within resource descriptors are
- *       resolved to integer values here. Therefore, this step is an
- *       important part of the code generation. We don't know that the
- *       name refers to a resource descriptor until now.
+ * DESCRIPTION: Find the name declaration op associated with the operator
  *
  ******************************************************************************/
 
-static ACPI_STATUS
-LkNamespaceLocateBegin (
-    ACPI_PARSE_OBJECT       *Op,
-    UINT32                  Level,
-    void                    *Context)
+static ACPI_PARSE_OBJECT *
+LkGetNameOp (
+    ACPI_PARSE_OBJECT       *Op)
 {
-    ACPI_WALK_STATE         *WalkState = (ACPI_WALK_STATE *) Context;
-    ACPI_NAMESPACE_NODE     *Node;
-    ACPI_STATUS             Status;
-    ACPI_OBJECT_TYPE        ObjectType;
-    char                    *Path;
-    UINT8                   PassedArgs;
-    ACPI_PARSE_OBJECT       *NextOp;
-    ACPI_PARSE_OBJECT       *OwningOp;
-    ACPI_PARSE_OBJECT       *SpaceIdOp;
-    UINT32                  MinimumLength;
-    UINT32                  Offset;
-    UINT32                  FieldBitLength;
-    UINT32                  TagBitLength;
-    UINT8                   Message = 0;
     const ACPI_OPCODE_INFO  *OpInfo;
-    UINT32                  Flags;
-
-
-    ACPI_FUNCTION_TRACE_PTR (LkNamespaceLocateBegin, Op);
-
-    /*
-     * If this node is the actual declaration of a name
-     * [such as the XXXX name in "Method (XXXX)"],
-     * we are not interested in it here. We only care about names that are
-     * references to other objects within the namespace and the parent objects
-     * of name declarations
-     */
-    if (Op->Asl.CompileFlags & NODE_IS_NAME_DECLARATION)
-    {
-        return (AE_OK);
-    }
+    ACPI_PARSE_OBJECT       *NameOp = Op;
 
-    /* We are only interested in opcodes that have an associated name */
 
     OpInfo = AcpiPsGetOpcodeInfo (Op->Asl.AmlOpcode);
 
-    if ((!(OpInfo->Flags & AML_NAMED)) &&
-        (!(OpInfo->Flags & AML_CREATE)) &&
-        (Op->Asl.ParseOpcode != PARSEOP_NAMESTRING) &&
-        (Op->Asl.ParseOpcode != PARSEOP_NAMESEG)    &&
-        (Op->Asl.ParseOpcode != PARSEOP_METHODCALL))
-    {
-        return (AE_OK);
-    }
-
-    /*
-     * One special case: CondRefOf operator - we don't care if the name exists
-     * or not at this point, just ignore it, the point of the operator is to
-     * determine if the name exists at runtime.
-     */
-    if ((Op->Asl.Parent) &&
-        (Op->Asl.Parent->Asl.ParseOpcode == PARSEOP_CONDREFOF))
-    {
-        return (AE_OK);
-    }
-
-    /*
-     * We must enable the "search-to-root" for single NameSegs, but
-     * we have to be very careful about opening up scopes
-     */
-    Flags = ACPI_NS_SEARCH_PARENT;
-    if ((Op->Asl.ParseOpcode == PARSEOP_NAMESTRING) ||
-        (Op->Asl.ParseOpcode == PARSEOP_NAMESEG)    ||
-        (Op->Asl.ParseOpcode == PARSEOP_METHODCALL))
-    {
-        /*
-         * These are name references, do not push the scope stack
-         * for them.
-         */
-        Flags |= ACPI_NS_DONT_OPEN_SCOPE;
-    }
 
     /* Get the NamePath from the appropriate place */
 
@@ -893,510 +189,26 @@ LkNamespaceLocateBegin (
     {
         /* For nearly all NAMED operators, the name reference is the first child */
 
-        Path = Op->Asl.Child->Asl.Value.String;
+        NameOp = Op->Asl.Child;
         if (Op->Asl.AmlOpcode == AML_ALIAS_OP)
         {
             /*
              * ALIAS is the only oddball opcode, the name declaration
              * (alias name) is the second operand
              */
-            Path = Op->Asl.Child->Asl.Next->Asl.Value.String;
+            NameOp = Op->Asl.Child->Asl.Next;
         }
     }
     else if (OpInfo->Flags & AML_CREATE)
     {
         /* Name must appear as the last parameter */
 
-        NextOp = Op->Asl.Child;
-        while (!(NextOp->Asl.CompileFlags & NODE_IS_NAME_DECLARATION))
-        {
-            NextOp = NextOp->Asl.Next;
-        }
-        Path = NextOp->Asl.Value.String;
-    }
-    else
-    {
-        Path = Op->Asl.Value.String;
-    }
-
-    ObjectType = AslMapNamedOpcodeToDataType (Op->Asl.AmlOpcode);
-    ACPI_DEBUG_PRINT ((ACPI_DB_DISPATCH,
-        "Type=%s\n", AcpiUtGetTypeName (ObjectType)));
-
-    /*
-     * Lookup the name in the namespace. Name must exist at this point, or it
-     * is an invalid reference.
-     *
-     * The namespace is also used as a lookup table for references to resource
-     * descriptors and the fields within them.
-     */
-    Gbl_NsLookupCount++;
-
-    Status = AcpiNsLookup (WalkState->ScopeInfo, Path, ObjectType,
-                ACPI_IMODE_EXECUTE, Flags, WalkState, &(Node));
-    if (ACPI_FAILURE (Status))
-    {
-        if (Status == AE_NOT_FOUND)
-        {
-            /*
-             * We didn't find the name reference by path -- we can qualify this
-             * a little better before we print an error message
-             */
-            if (strlen (Path) == ACPI_NAME_SIZE)
-            {
-                /* A simple, one-segment ACPI name */
-
-                if (LkObjectExists (Path))
-                {
-                    /*
-                     * There exists such a name, but we couldn't get to it
-                     * from this scope
-                     */
-                    AslError (ASL_ERROR, ASL_MSG_NOT_REACHABLE, Op,
-                        Op->Asl.ExternalName);
-                }
-                else
-                {
-                    /* The name doesn't exist, period */
-
-                    AslError (ASL_ERROR, ASL_MSG_NOT_EXIST,
-                        Op, Op->Asl.ExternalName);
-                }
-            }
-            else
-            {
-                /* Check for a fully qualified path */
-
-                if (Path[0] == AML_ROOT_PREFIX)
-                {
-                    /* Gave full path, the object does not exist */
-
-                    AslError (ASL_ERROR, ASL_MSG_NOT_EXIST, Op,
-                        Op->Asl.ExternalName);
-                }
-                else
-                {
-                    /*
-                     * We can't tell whether it doesn't exist or just
-                     * can't be reached.
-                     */
-                    AslError (ASL_ERROR, ASL_MSG_NOT_FOUND, Op,
-                        Op->Asl.ExternalName);
-                }
-            }
-
-            Status = AE_OK;
-        }
-        return (Status);
-    }
-
-    /* Check for a reference vs. name declaration */
-
-    if (!(OpInfo->Flags & AML_NAMED) &&
-        !(OpInfo->Flags & AML_CREATE))
-    {
-        /* This node has been referenced, mark it for reference check */
-
-        Node->Flags |= ANOBJ_IS_REFERENCED;
-    }
-
-    /* Attempt to optimize the NamePath */
-
-    OptOptimizeNamePath (Op, OpInfo->Flags, WalkState, Path, Node);
-
-    /*
-     * 1) Dereference an alias (A name reference that is an alias)
-     *    Aliases are not nested, the alias always points to the final object
-     */
-    if ((Op->Asl.ParseOpcode != PARSEOP_ALIAS) &&
-        (Node->Type == ACPI_TYPE_LOCAL_ALIAS))
-    {
-        /* This node points back to the original PARSEOP_ALIAS */
-
-        NextOp = Node->Op;
-
-        /* The first child is the alias target op */
-
-        NextOp = NextOp->Asl.Child;
-
-        /* That in turn points back to original target alias node */
-
-        if (NextOp->Asl.Node)
-        {
-            Node = NextOp->Asl.Node;
-        }
-
-        /* Else - forward reference to alias, will be resolved later */
-    }
-
-    /* 2) Check for a reference to a resource descriptor */
-
-    if ((Node->Type == ACPI_TYPE_LOCAL_RESOURCE_FIELD) ||
-        (Node->Type == ACPI_TYPE_LOCAL_RESOURCE))
-    {
-        /*
-         * This was a reference to a field within a resource descriptor.
-         * Extract the associated field offset (either a bit or byte
-         * offset depending on the field type) and change the named
-         * reference into an integer for AML code generation
-         */
-        Offset = Node->Value;
-        TagBitLength = Node->Length;
-
-        /*
-         * If a field is being created, generate the length (in bits) of
-         * the field. Note: Opcodes other than CreateXxxField and Index
-         * can come through here. For other opcodes, we just need to
-         * convert the resource tag reference to an integer offset.
-         */
-        switch (Op->Asl.Parent->Asl.AmlOpcode)
-        {
-        case AML_CREATE_FIELD_OP: /* Variable "Length" field, in bits */
-            /*
-             * We know the length operand is an integer constant because
-             * we know that it contains a reference to a resource
-             * descriptor tag.
-             */
-            FieldBitLength = (UINT32) Op->Asl.Next->Asl.Value.Integer;
-            break;
-
-        case AML_CREATE_BIT_FIELD_OP:
-            FieldBitLength = 1;
-            break;
-
-        case AML_CREATE_BYTE_FIELD_OP:
-        case AML_INDEX_OP:
-            FieldBitLength = 8;
-            break;
-
-        case AML_CREATE_WORD_FIELD_OP:
-            FieldBitLength = 16;
-            break;
-
-        case AML_CREATE_DWORD_FIELD_OP:
-            FieldBitLength = 32;
-            break;
-
-        case AML_CREATE_QWORD_FIELD_OP:
-            FieldBitLength = 64;
-            break;
-
-        default:
-            FieldBitLength = 0;
-            break;
-        }
-
-        /* Check the field length against the length of the resource tag */
-
-        if (FieldBitLength)
-        {
-            if (TagBitLength < FieldBitLength)
-            {
-                Message = ASL_MSG_TAG_SMALLER;
-            }
-            else if (TagBitLength > FieldBitLength)
-            {
-                Message = ASL_MSG_TAG_LARGER;
-            }
-
-            if (Message)
-            {
-                sprintf (MsgBuffer, "Size mismatch, Tag: %u bit%s, Field: %u bit%s",
-                    TagBitLength, (TagBitLength > 1) ? "s" : "",
-                    FieldBitLength, (FieldBitLength > 1) ? "s" : "");
-
-                AslError (ASL_WARNING, Message, Op, MsgBuffer);
-            }
-        }
-
-        /* Convert the BitOffset to a ByteOffset for certain opcodes */
-
-        switch (Op->Asl.Parent->Asl.AmlOpcode)
-        {
-        case AML_CREATE_BYTE_FIELD_OP:
-        case AML_CREATE_WORD_FIELD_OP:
-        case AML_CREATE_DWORD_FIELD_OP:
-        case AML_CREATE_QWORD_FIELD_OP:
-        case AML_INDEX_OP:
-
-            Offset = ACPI_DIV_8 (Offset);
-            break;
-
-        default:
-            break;
-        }
-
-        /* Now convert this node to an integer whose value is the field offset */
-
-        Op->Asl.AmlLength = 0;
-        Op->Asl.ParseOpcode = PARSEOP_INTEGER;
-        Op->Asl.Value.Integer = (UINT64) Offset;
-        Op->Asl.CompileFlags |= NODE_IS_RESOURCE_FIELD;
-
-        OpcGenerateAmlOpcode (Op);
-    }
-
-    /* 3) Check for a method invocation */
-
-    else if ((((Op->Asl.ParseOpcode == PARSEOP_NAMESTRING) || (Op->Asl.ParseOpcode == PARSEOP_NAMESEG)) &&
-                (Node->Type == ACPI_TYPE_METHOD) &&
-                (Op->Asl.Parent) &&
-                (Op->Asl.Parent->Asl.ParseOpcode != PARSEOP_METHOD))   ||
-
-                (Op->Asl.ParseOpcode == PARSEOP_METHODCALL))
-    {
-
-        /*
-         * A reference to a method within one of these opcodes is not an
-         * invocation of the method, it is simply a reference to the method.
-         */
-        if ((Op->Asl.Parent) &&
-           ((Op->Asl.Parent->Asl.ParseOpcode == PARSEOP_REFOF)      ||
-            (Op->Asl.Parent->Asl.ParseOpcode == PARSEOP_DEREFOF)    ||
-            (Op->Asl.Parent->Asl.ParseOpcode == PARSEOP_OBJECTTYPE)))
-        {
-            return (AE_OK);
-        }
-        /*
-         * There are two types of method invocation:
-         * 1) Invocation with arguments -- the parser recognizes this
-         *    as a METHODCALL.
-         * 2) Invocation with no arguments --the parser cannot determine that
-         *    this is a method invocation, therefore we have to figure it out
-         *    here.
-         */
-        if (Node->Type != ACPI_TYPE_METHOD)
-        {
-            sprintf (MsgBuffer, "%s is a %s",
-                    Op->Asl.ExternalName, AcpiUtGetTypeName (Node->Type));
-
-            AslError (ASL_ERROR, ASL_MSG_NOT_METHOD, Op, MsgBuffer);
-            return (AE_OK);
-        }
-
-        /* Save the method node in the caller's op */
-
-        Op->Asl.Node = Node;
-        if (Op->Asl.Parent->Asl.ParseOpcode == PARSEOP_CONDREFOF)
-        {
-            return (AE_OK);
-        }
-
-        /*
-         * This is a method invocation, with or without arguments.
-         * Count the number of arguments, each appears as a child
-         * under the parent node
-         */
-        Op->Asl.ParseOpcode = PARSEOP_METHODCALL;
-        UtSetParseOpName (Op);
-
-        PassedArgs = 0;
-        NextOp     = Op->Asl.Child;
-
-        while (NextOp)
-        {
-            PassedArgs++;
-            NextOp = NextOp->Asl.Next;
-        }
-
-        if (Node->Value != ASL_EXTERNAL_METHOD)
-        {
-            /*
-             * Check the parsed arguments with the number expected by the
-             * method declaration itself
-             */
-            if (PassedArgs != Node->Value)
-            {
-                sprintf (MsgBuffer, "%s requires %u", Op->Asl.ExternalName,
-                            Node->Value);
-
-                if (PassedArgs < Node->Value)
-                {
-                    AslError (ASL_ERROR, ASL_MSG_ARG_COUNT_LO, Op, MsgBuffer);
-                }
-                else
-                {
-                    AslError (ASL_ERROR, ASL_MSG_ARG_COUNT_HI, Op, MsgBuffer);
-                }
-            }
-        }
-    }
-
-    /* 4) Check for an ASL Field definition */
-
-    else if ((Op->Asl.Parent) &&
-            ((Op->Asl.Parent->Asl.ParseOpcode == PARSEOP_FIELD)     ||
-             (Op->Asl.Parent->Asl.ParseOpcode == PARSEOP_BANKFIELD)))
-    {
-        /*
-         * Offset checking for fields. If the parent operation region has a
-         * constant length (known at compile time), we can check fields
-         * defined in that region against the region length. This will catch
-         * fields and field units that cannot possibly fit within the region.
-         *
-         * Note: Index fields do not directly reference an operation region,
-         * thus they are not included in this check.
-         */
-        if (Op == Op->Asl.Parent->Asl.Child)
-        {
-            /*
-             * This is the first child of the field node, which is
-             * the name of the region. Get the parse node for the
-             * region -- which contains the length of the region.
-             */
-            OwningOp = Node->Op;
-            Op->Asl.Parent->Asl.ExtraValue =
-                ACPI_MUL_8 ((UINT32) OwningOp->Asl.Value.Integer);
-
-            /* Examine the field access width */
-
-            switch ((UINT8) Op->Asl.Parent->Asl.Value.Integer)
-            {
-            case AML_FIELD_ACCESS_ANY:
-            case AML_FIELD_ACCESS_BYTE:
-            case AML_FIELD_ACCESS_BUFFER:
-            default:
-                MinimumLength = 1;
-                break;
-
-            case AML_FIELD_ACCESS_WORD:
-                MinimumLength = 2;
-                break;
-
-            case AML_FIELD_ACCESS_DWORD:
-                MinimumLength = 4;
-                break;
-
-            case AML_FIELD_ACCESS_QWORD:
-                MinimumLength = 8;
-                break;
-            }
-
-            /*
-             * Is the region at least as big as the access width?
-             * Note: DataTableRegions have 0 length
-             */
-            if (((UINT32) OwningOp->Asl.Value.Integer) &&
-                ((UINT32) OwningOp->Asl.Value.Integer < MinimumLength))
-            {
-                AslError (ASL_ERROR, ASL_MSG_FIELD_ACCESS_WIDTH, Op, NULL);
-            }
-
-            /*
-             * Check EC/CMOS/SMBUS fields to make sure that the correct
-             * access type is used (BYTE for EC/CMOS, BUFFER for SMBUS)
-             */
-            SpaceIdOp = OwningOp->Asl.Child->Asl.Next;
-            switch ((UINT32) SpaceIdOp->Asl.Value.Integer)
-            {
-            case ACPI_ADR_SPACE_EC:
-            case ACPI_ADR_SPACE_CMOS:
-            case ACPI_ADR_SPACE_GPIO:
-
-                if ((UINT8) Op->Asl.Parent->Asl.Value.Integer != AML_FIELD_ACCESS_BYTE)
-                {
-                    AslError (ASL_ERROR, ASL_MSG_REGION_BYTE_ACCESS, Op, NULL);
-                }
-                break;
-
-            case ACPI_ADR_SPACE_SMBUS:
-            case ACPI_ADR_SPACE_IPMI:
-            case ACPI_ADR_SPACE_GSBUS:
-
-                if ((UINT8) Op->Asl.Parent->Asl.Value.Integer != AML_FIELD_ACCESS_BUFFER)
-                {
-                    AslError (ASL_ERROR, ASL_MSG_REGION_BUFFER_ACCESS, Op, NULL);
-                }
-                break;
-
-            default:
-
-                /* Nothing to do for other address spaces */
-                break;
-            }
-        }
-        else
+        NameOp = Op->Asl.Child;
+        while (!(NameOp->Asl.CompileFlags & NODE_IS_NAME_DECLARATION))
         {
-            /*
-             * This is one element of the field list. Check to make sure
-             * that it does not go beyond the end of the parent operation region.
-             *
-             * In the code below:
-             *    Op->Asl.Parent->Asl.ExtraValue      - Region Length (bits)
-             *    Op->Asl.ExtraValue                  - Field start offset (bits)
-             *    Op->Asl.Child->Asl.Value.Integer32  - Field length (bits)
-             *    Op->Asl.Child->Asl.ExtraValue       - Field access width (bits)
-             */
-            if (Op->Asl.Parent->Asl.ExtraValue && Op->Asl.Child)
-            {
-                LkCheckFieldRange (Op,
-                            Op->Asl.Parent->Asl.ExtraValue,
-                            Op->Asl.ExtraValue,
-                            (UINT32) Op->Asl.Child->Asl.Value.Integer,
-                            Op->Asl.Child->Asl.ExtraValue);
-            }
+            NameOp = NameOp->Asl.Next;
         }
     }
 
-    Op->Asl.Node = Node;
-    return (Status);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    LkNamespaceLocateEnd
- *
- * PARAMETERS:  ASL_WALK_CALLBACK
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Ascending callback used during cross reference. We only
- *              need to worry about scope management here.
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-LkNamespaceLocateEnd (
-    ACPI_PARSE_OBJECT       *Op,
-    UINT32                  Level,
-    void                    *Context)
-{
-    ACPI_WALK_STATE         *WalkState = (ACPI_WALK_STATE *) Context;
-    const ACPI_OPCODE_INFO  *OpInfo;
-
-
-    ACPI_FUNCTION_TRACE (LkNamespaceLocateEnd);
-
-
-    /* We are only interested in opcodes that have an associated name */
-
-    OpInfo = AcpiPsGetOpcodeInfo (Op->Asl.AmlOpcode);
-    if (!(OpInfo->Flags & AML_NAMED))
-    {
-        return (AE_OK);
-    }
-
-    /* Not interested in name references, we did not open a scope for them */
-
-    if ((Op->Asl.ParseOpcode == PARSEOP_NAMESTRING) ||
-        (Op->Asl.ParseOpcode == PARSEOP_NAMESEG)    ||
-        (Op->Asl.ParseOpcode == PARSEOP_METHODCALL))
-    {
-        return (AE_OK);
-    }
-
-    /* Pop the scope stack if necessary */
-
-    if (AcpiNsOpensScope (AslMapNamedOpcodeToDataType (Op->Asl.AmlOpcode)))
-    {
-
-        ACPI_DEBUG_PRINT ((ACPI_DB_DISPATCH,
-            "%s: Popping scope for Op %p\n",
-            AcpiUtGetTypeName (OpInfo->ObjectType), Op));
-
-        (void) AcpiDsScopeStackPop (WalkState);
-    }
-
-    return (AE_OK);
+    return (NameOp);
 }
diff --git a/sys/contrib/dev/acpica/compiler/aslmain.c b/sys/contrib/dev/acpica/compiler/aslmain.c
index 3b6d45b..1a05b9f 100644
--- a/sys/contrib/dev/acpica/compiler/aslmain.c
+++ b/sys/contrib/dev/acpica/compiler/aslmain.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -153,6 +153,7 @@ Options (
     ACPI_OPTION ("-oi",             "Disable integer optimization to Zero/One/Ones");
     ACPI_OPTION ("-on",             "Disable named reference string optimization");
     ACPI_OPTION ("-cr",             "Disable Resource Descriptor error checking");
+    ACPI_OPTION ("-in",             "Ignore NoOp operators");
     ACPI_OPTION ("-r <revision>",   "Override table header Revision (1-255)");
 
     printf ("\nASL Listing Files:\n");
@@ -172,6 +173,7 @@ Options (
     ACPI_OPTION ("",                "(Obtain DSDT from current system if no input file)");
     ACPI_OPTION ("-e  [f1,f2]",     "Include ACPI table(s) for external symbol resolution");
     ACPI_OPTION ("-g",              "Get ACPI tables and write to files (*.dat)");
+    ACPI_OPTION ("-in",             "Ignore NoOp opcodes");
     ACPI_OPTION ("-vt",             "Dump binary table data in hex format within output file");
 
     printf ("\nHelp:\n");
@@ -622,6 +624,13 @@ AslDoOptions (
             Gbl_C_IncludeOutputFlag = TRUE;
             break;
 
+        case 'n':
+
+            /* Compiler/Disassembler: Ignore the NOOP operator */
+
+            AcpiGbl_IgnoreNoopOperator = TRUE;
+            break;
+
         default:
             printf ("Unknown option: -i%s\n", AcpiGbl_Optarg);
             return (-1);
diff --git a/sys/contrib/dev/acpica/compiler/aslmap.c b/sys/contrib/dev/acpica/compiler/aslmap.c
index d9f3325..0d2e636 100644
--- a/sys/contrib/dev/acpica/compiler/aslmap.c
+++ b/sys/contrib/dev/acpica/compiler/aslmap.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslmessages.h b/sys/contrib/dev/acpica/compiler/aslmessages.h
index 07758ef..be7a3f9 100644
--- a/sys/contrib/dev/acpica/compiler/aslmessages.h
+++ b/sys/contrib/dev/acpica/compiler/aslmessages.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -190,6 +190,7 @@ typedef enum
     ASL_MSG_TAG_SMALLER,
     ASL_MSG_TIMEOUT,
     ASL_MSG_TOO_MANY_TEMPS,
+    ASL_MSG_TRUNCATION,
     ASL_MSG_UNKNOWN_RESERVED_NAME,
     ASL_MSG_UNREACHABLE_CODE,
     ASL_MSG_UNSUPPORTED,
@@ -274,7 +275,7 @@ char                        *AslMessages [] = {
 /*    ASL_MSG_HID_SUFFIX */                 "_HID suffix must be all hex digits",
 /*    ASL_MSG_INCLUDE_FILE_OPEN */          "Could not open include file",
 /*    ASL_MSG_INPUT_FILE_OPEN */            "Could not open input file",
-/*    ASL_MSG_INTEGER_LENGTH */             "64-bit integer in 32-bit table, truncating",
+/*    ASL_MSG_INTEGER_LENGTH */             "64-bit integer in 32-bit table, truncating (DSDT version < 2)",
 /*    ASL_MSG_INTEGER_OPTIMIZATION */       "Integer optimized to single-byte AML opcode",
 /*    ASL_MSG_INTERRUPT_LIST */             "Too many interrupts (16 max)",
 /*    ASL_MSG_INTERRUPT_NUMBER */           "Invalid interrupt number (must be 0-15)",
@@ -366,6 +367,7 @@ char                        *AslMessages [] = {
 /*    ASL_MSG_TAG_SMALLER */                "ResourceTag smaller than Field",
 /*    ASL_MSG_TIMEOUT */                    "Result is not used, possible operator timeout will be missed",
 /*    ASL_MSG_TOO_MANY_TEMPS */             "Method requires too many temporary variables (_T_x)",
+/*    ASL_MSG_TRUNCATION */                 "64-bit return value will be truncated to 32 bits (DSDT version < 2)",
 /*    ASL_MSG_UNKNOWN_RESERVED_NAME */      "Unknown reserved name",
 /*    ASL_MSG_UNREACHABLE_CODE */           "Statement is unreachable",
 /*    ASL_MSG_UNSUPPORTED */                "Unsupported feature",
diff --git a/sys/contrib/dev/acpica/compiler/aslmethod.c b/sys/contrib/dev/acpica/compiler/aslmethod.c
new file mode 100644
index 0000000..f104d44
--- /dev/null
+++ b/sys/contrib/dev/acpica/compiler/aslmethod.c
@@ -0,0 +1,619 @@
+/******************************************************************************
+ *
+ * Module Name: aslmethod.c - Control method analysis walk
+ *
+ *****************************************************************************/
+
+/*
+ * Copyright (C) 2000 - 2013, Intel Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ * 3. Neither the names of the above-listed copyright holders nor the names
+ *    of any contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ */
+
+
+#include <contrib/dev/acpica/compiler/aslcompiler.h>
+#include "aslcompiler.y.h"
+
+
+#define _COMPONENT          ACPI_COMPILER
+        ACPI_MODULE_NAME    ("aslmethod")
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    MtMethodAnalysisWalkBegin
+ *
+ * PARAMETERS:  ASL_WALK_CALLBACK
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Descending callback for the analysis walk. Check methods for:
+ *              1) Initialized local variables
+ *              2) Valid arguments
+ *              3) Return types
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+MtMethodAnalysisWalkBegin (
+    ACPI_PARSE_OBJECT       *Op,
+    UINT32                  Level,
+    void                    *Context)
+{
+    ASL_ANALYSIS_WALK_INFO  *WalkInfo = (ASL_ANALYSIS_WALK_INFO *) Context;
+    ASL_METHOD_INFO         *MethodInfo = WalkInfo->MethodStack;
+    ACPI_PARSE_OBJECT       *Next;
+    UINT32                  RegisterNumber;
+    UINT32                  i;
+    char                    LocalName[] = "Local0";
+    char                    ArgName[] = "Arg0";
+    ACPI_PARSE_OBJECT       *ArgNode;
+    ACPI_PARSE_OBJECT       *NextType;
+    ACPI_PARSE_OBJECT       *NextParamType;
+    UINT8                   ActualArgs = 0;
+
+
+    switch (Op->Asl.ParseOpcode)
+    {
+    case PARSEOP_METHOD:
+
+        TotalMethods++;
+
+        /* Create and init method info */
+
+        MethodInfo       = UtLocalCalloc (sizeof (ASL_METHOD_INFO));
+        MethodInfo->Next = WalkInfo->MethodStack;
+        MethodInfo->Op = Op;
+
+        WalkInfo->MethodStack = MethodInfo;
+
+        /* Get the name node, ignored here */
+
+        Next = Op->Asl.Child;
+
+        /* Get the NumArguments node */
+
+        Next = Next->Asl.Next;
+        MethodInfo->NumArguments = (UINT8)
+            (((UINT8) Next->Asl.Value.Integer) & 0x07);
+
+        /* Get the SerializeRule and SyncLevel nodes, ignored here */
+
+        Next = Next->Asl.Next;
+        Next = Next->Asl.Next;
+        ArgNode = Next;
+
+        /* Get the ReturnType node */
+
+        Next = Next->Asl.Next;
+
+        NextType = Next->Asl.Child;
+        while (NextType)
+        {
+            /* Get and map each of the ReturnTypes */
+
+            MethodInfo->ValidReturnTypes |= AnMapObjTypeToBtype (NextType);
+            NextType->Asl.ParseOpcode = PARSEOP_DEFAULT_ARG;
+            NextType = NextType->Asl.Next;
+        }
+
+        /* Get the ParameterType node */
+
+        Next = Next->Asl.Next;
+
+        NextType = Next->Asl.Child;
+        while (NextType)
+        {
+            if (NextType->Asl.ParseOpcode == PARSEOP_DEFAULT_ARG)
+            {
+                NextParamType = NextType->Asl.Child;
+                while (NextParamType)
+                {
+                    MethodInfo->ValidArgTypes[ActualArgs] |= AnMapObjTypeToBtype (NextParamType);
+                    NextParamType->Asl.ParseOpcode = PARSEOP_DEFAULT_ARG;
+                    NextParamType = NextParamType->Asl.Next;
+                }
+            }
+            else
+            {
+                MethodInfo->ValidArgTypes[ActualArgs] =
+                    AnMapObjTypeToBtype (NextType);
+                NextType->Asl.ParseOpcode = PARSEOP_DEFAULT_ARG;
+                ActualArgs++;
+            }
+
+            NextType = NextType->Asl.Next;
+        }
+
+        if ((MethodInfo->NumArguments) &&
+            (MethodInfo->NumArguments != ActualArgs))
+        {
+            /* error: Param list did not match number of args */
+        }
+
+        /* Allow numarguments == 0 for Function() */
+
+        if ((!MethodInfo->NumArguments) && (ActualArgs))
+        {
+            MethodInfo->NumArguments = ActualArgs;
+            ArgNode->Asl.Value.Integer |= ActualArgs;
+        }
+
+        /*
+         * Actual arguments are initialized at method entry.
+         * All other ArgX "registers" can be used as locals, so we
+         * track their initialization.
+         */
+        for (i = 0; i < MethodInfo->NumArguments; i++)
+        {
+            MethodInfo->ArgInitialized[i] = TRUE;
+        }
+        break;
+
+
+    case PARSEOP_METHODCALL:
+
+        if (MethodInfo &&
+           (Op->Asl.Node == MethodInfo->Op->Asl.Node))
+        {
+            AslError (ASL_REMARK, ASL_MSG_RECURSION, Op, Op->Asl.ExternalName);
+        }
+        break;
+
+
+    case PARSEOP_LOCAL0:
+    case PARSEOP_LOCAL1:
+    case PARSEOP_LOCAL2:
+    case PARSEOP_LOCAL3:
+    case PARSEOP_LOCAL4:
+    case PARSEOP_LOCAL5:
+    case PARSEOP_LOCAL6:
+    case PARSEOP_LOCAL7:
+
+        if (!MethodInfo)
+        {
+            /*
+             * Local was used outside a control method, or there was an error
+             * in the method declaration.
+             */
+            AslError (ASL_REMARK, ASL_MSG_LOCAL_OUTSIDE_METHOD, Op, Op->Asl.ExternalName);
+            return (AE_ERROR);
+        }
+
+        RegisterNumber = (Op->Asl.AmlOpcode & 0x000F);
+
+        /*
+         * If the local is being used as a target, mark the local
+         * initialized
+         */
+        if (Op->Asl.CompileFlags & NODE_IS_TARGET)
+        {
+            MethodInfo->LocalInitialized[RegisterNumber] = TRUE;
+        }
+
+        /*
+         * Otherwise, this is a reference, check if the local
+         * has been previously initialized.
+         *
+         * The only operator that accepts an uninitialized value is ObjectType()
+         */
+        else if ((!MethodInfo->LocalInitialized[RegisterNumber]) &&
+                 (Op->Asl.Parent->Asl.ParseOpcode != PARSEOP_OBJECTTYPE))
+        {
+            LocalName[strlen (LocalName) -1] = (char) (RegisterNumber + 0x30);
+            AslError (ASL_ERROR, ASL_MSG_LOCAL_INIT, Op, LocalName);
+        }
+        break;
+
+
+    case PARSEOP_ARG0:
+    case PARSEOP_ARG1:
+    case PARSEOP_ARG2:
+    case PARSEOP_ARG3:
+    case PARSEOP_ARG4:
+    case PARSEOP_ARG5:
+    case PARSEOP_ARG6:
+
+        if (!MethodInfo)
+        {
+            /*
+             * Arg was used outside a control method, or there was an error
+             * in the method declaration.
+             */
+            AslError (ASL_REMARK, ASL_MSG_LOCAL_OUTSIDE_METHOD, Op, Op->Asl.ExternalName);
+            return (AE_ERROR);
+        }
+
+        RegisterNumber = (Op->Asl.AmlOpcode & 0x000F) - 8;
+        ArgName[strlen (ArgName) -1] = (char) (RegisterNumber + 0x30);
+
+        /*
+         * If the Arg is being used as a target, mark the local
+         * initialized
+         */
+        if (Op->Asl.CompileFlags & NODE_IS_TARGET)
+        {
+            MethodInfo->ArgInitialized[RegisterNumber] = TRUE;
+        }
+
+        /*
+         * Otherwise, this is a reference, check if the Arg
+         * has been previously initialized.
+         *
+         * The only operator that accepts an uninitialized value is ObjectType()
+         */
+        else if ((!MethodInfo->ArgInitialized[RegisterNumber]) &&
+                 (Op->Asl.Parent->Asl.ParseOpcode != PARSEOP_OBJECTTYPE))
+        {
+            AslError (ASL_ERROR, ASL_MSG_ARG_INIT, Op, ArgName);
+        }
+
+        /* Flag this arg if it is not a "real" argument to the method */
+
+        if (RegisterNumber >= MethodInfo->NumArguments)
+        {
+            AslError (ASL_REMARK, ASL_MSG_NOT_PARAMETER, Op, ArgName);
+        }
+        break;
+
+
+    case PARSEOP_RETURN:
+
+        if (!MethodInfo)
+        {
+            /*
+             * Probably was an error in the method declaration,
+             * no additional error here
+             */
+            ACPI_WARNING ((AE_INFO, "%p, No parent method", Op));
+            return (AE_ERROR);
+        }
+
+        /*
+         * A child indicates a possible return value. A simple Return or
+         * Return() is marked with NODE_IS_NULL_RETURN by the parser so
+         * that it is not counted as a "real" return-with-value, although
+         * the AML code that is actually emitted is Return(0). The AML
+         * definition of Return has a required parameter, so we are
+         * forced to convert a null return to Return(0).
+         */
+        if ((Op->Asl.Child) &&
+            (Op->Asl.Child->Asl.ParseOpcode != PARSEOP_DEFAULT_ARG) &&
+            (!(Op->Asl.Child->Asl.CompileFlags & NODE_IS_NULL_RETURN)))
+        {
+            MethodInfo->NumReturnWithValue++;
+        }
+        else
+        {
+            MethodInfo->NumReturnNoValue++;
+        }
+        break;
+
+
+    case PARSEOP_BREAK:
+    case PARSEOP_CONTINUE:
+
+        Next = Op->Asl.Parent;
+        while (Next)
+        {
+            if (Next->Asl.ParseOpcode == PARSEOP_WHILE)
+            {
+                break;
+            }
+            Next = Next->Asl.Parent;
+        }
+
+        if (!Next)
+        {
+            AslError (ASL_ERROR, ASL_MSG_NO_WHILE, Op, NULL);
+        }
+        break;
+
+
+    case PARSEOP_STALL:
+
+        /* We can range check if the argument is an integer */
+
+        if ((Op->Asl.Child->Asl.ParseOpcode == PARSEOP_INTEGER) &&
+            (Op->Asl.Child->Asl.Value.Integer > ACPI_UINT8_MAX))
+        {
+            AslError (ASL_ERROR, ASL_MSG_INVALID_TIME, Op, NULL);
+        }
+        break;
+
+
+    case PARSEOP_DEVICE:
+    case PARSEOP_EVENT:
+    case PARSEOP_MUTEX:
+    case PARSEOP_OPERATIONREGION:
+    case PARSEOP_POWERRESOURCE:
+    case PARSEOP_PROCESSOR:
+    case PARSEOP_THERMALZONE:
+
+        /*
+         * The first operand is a name to be created in the namespace.
+         * Check against the reserved list.
+         */
+        i = ApCheckForPredefinedName (Op, Op->Asl.NameSeg);
+        if (i < ACPI_VALID_RESERVED_NAME_MAX)
+        {
+            AslError (ASL_ERROR, ASL_MSG_RESERVED_USE, Op, Op->Asl.ExternalName);
+        }
+        break;
+
+
+    case PARSEOP_NAME:
+
+        /* Typecheck any predefined names statically defined with Name() */
+
+        ApCheckForPredefinedObject (Op, Op->Asl.NameSeg);
+
+        /* Special typechecking for _HID */
+
+        if (!ACPI_STRCMP (METHOD_NAME__HID, Op->Asl.NameSeg))
+        {
+            Next = Op->Asl.Child->Asl.Next;
+            AnCheckId (Next, ASL_TYPE_HID);
+        }
+
+        /* Special typechecking for _CID */
+
+        else if (!ACPI_STRCMP (METHOD_NAME__CID, Op->Asl.NameSeg))
+        {
+            Next = Op->Asl.Child->Asl.Next;
+
+            if ((Next->Asl.ParseOpcode == PARSEOP_PACKAGE) ||
+                (Next->Asl.ParseOpcode == PARSEOP_VAR_PACKAGE))
+            {
+                Next = Next->Asl.Child;
+                while (Next)
+                {
+                    AnCheckId (Next, ASL_TYPE_CID);
+                    Next = Next->Asl.Next;
+                }
+            }
+            else
+            {
+                AnCheckId (Next, ASL_TYPE_CID);
+            }
+        }
+        break;
+
+
+    default:
+        break;
+    }
+
+    return (AE_OK);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    MtMethodAnalysisWalkEnd
+ *
+ * PARAMETERS:  ASL_WALK_CALLBACK
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Ascending callback for analysis walk. Complete method
+ *              return analysis.
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+MtMethodAnalysisWalkEnd (
+    ACPI_PARSE_OBJECT       *Op,
+    UINT32                  Level,
+    void                    *Context)
+{
+    ASL_ANALYSIS_WALK_INFO  *WalkInfo = (ASL_ANALYSIS_WALK_INFO *) Context;
+    ASL_METHOD_INFO         *MethodInfo = WalkInfo->MethodStack;
+
+
+    switch (Op->Asl.ParseOpcode)
+    {
+    case PARSEOP_METHOD:
+    case PARSEOP_RETURN:
+        if (!MethodInfo)
+        {
+            printf ("No method info for method! [%s]\n", Op->Asl.Namepath);
+            AslError (ASL_ERROR, ASL_MSG_COMPILER_INTERNAL, Op,
+                "No method info for this method");
+
+            CmCleanupAndExit ();
+            return (AE_AML_INTERNAL);
+        }
+        break;
+
+    default:
+        break;
+    }
+
+    switch (Op->Asl.ParseOpcode)
+    {
+    case PARSEOP_METHOD:
+
+        WalkInfo->MethodStack = MethodInfo->Next;
+
+        /*
+         * Check if there is no return statement at the end of the
+         * method AND we can actually get there -- i.e., the execution
+         * of the method can possibly terminate without a return statement.
+         */
+        if ((!AnLastStatementIsReturn (Op)) &&
+            (!(Op->Asl.CompileFlags & NODE_HAS_NO_EXIT)))
+        {
+            /*
+             * No return statement, and execution can possibly exit
+             * via this path. This is equivalent to Return ()
+             */
+            MethodInfo->NumReturnNoValue++;
+        }
+
+        /*
+         * Check for case where some return statements have a return value
+         * and some do not. Exit without a return statement is a return with
+         * no value
+         */
+        if (MethodInfo->NumReturnNoValue &&
+            MethodInfo->NumReturnWithValue)
+        {
+            AslError (ASL_WARNING, ASL_MSG_RETURN_TYPES, Op,
+                Op->Asl.ExternalName);
+        }
+
+        /*
+         * If there are any RETURN() statements with no value, or there is a
+         * control path that allows the method to exit without a return value,
+         * we mark the method as a method that does not return a value. This
+         * knowledge can be used to check method invocations that expect a
+         * returned value.
+         */
+        if (MethodInfo->NumReturnNoValue)
+        {
+            if (MethodInfo->NumReturnWithValue)
+            {
+                Op->Asl.CompileFlags |= NODE_METHOD_SOME_NO_RETVAL;
+            }
+            else
+            {
+                Op->Asl.CompileFlags |= NODE_METHOD_NO_RETVAL;
+            }
+        }
+
+        /*
+         * Check predefined method names for correct return behavior
+         * and correct number of arguments. Also, some special checks
+         * For GPE and _REG methods.
+         */
+        if (ApCheckForPredefinedMethod (Op, MethodInfo))
+        {
+            /* Special check for two names like _L01 and _E01 in same scope */
+
+            ApCheckForGpeNameConflict (Op);
+
+            /*
+             * Special check for _REG: Must have an operation region definition
+             * within the same scope!
+             */
+            ApCheckRegMethod (Op);
+        }
+
+        ACPI_FREE (MethodInfo);
+        break;
+
+
+    case PARSEOP_NAME:
+
+         /* Special check for two names like _L01 and _E01 in same scope */
+
+        ApCheckForGpeNameConflict (Op);
+        break;
+
+
+    case PARSEOP_RETURN:
+
+        /*
+         * If the parent is a predefined method name, attempt to typecheck
+         * the return value. Only static types can be validated.
+         */
+        ApCheckPredefinedReturnValue (Op, MethodInfo);
+
+        /*
+         * The parent block does not "exit" and continue execution -- the
+         * method is terminated here with the Return() statement.
+         */
+        Op->Asl.Parent->Asl.CompileFlags |= NODE_HAS_NO_EXIT;
+
+        /* Used in the "typing" pass later */
+
+        Op->Asl.ParentMethod = MethodInfo->Op;
+
+        /*
+         * If there is a peer node after the return statement, then this
+         * node is unreachable code -- i.e., it won't be executed because of
+         * the preceding Return() statement.
+         */
+        if (Op->Asl.Next)
+        {
+            AslError (ASL_WARNING, ASL_MSG_UNREACHABLE_CODE, Op->Asl.Next, NULL);
+        }
+        break;
+
+
+    case PARSEOP_IF:
+
+        if ((Op->Asl.CompileFlags & NODE_HAS_NO_EXIT) &&
+            (Op->Asl.Next) &&
+            (Op->Asl.Next->Asl.ParseOpcode == PARSEOP_ELSE))
+        {
+            /*
+             * This IF has a corresponding ELSE. The IF block has no exit,
+             * (it contains an unconditional Return)
+             * mark the ELSE block to remember this fact.
+             */
+            Op->Asl.Next->Asl.CompileFlags |= NODE_IF_HAS_NO_EXIT;
+        }
+        break;
+
+
+    case PARSEOP_ELSE:
+
+        if ((Op->Asl.CompileFlags & NODE_HAS_NO_EXIT) &&
+            (Op->Asl.CompileFlags & NODE_IF_HAS_NO_EXIT))
+        {
+            /*
+             * This ELSE block has no exit and the corresponding IF block
+             * has no exit either. Therefore, the parent node has no exit.
+             */
+            Op->Asl.Parent->Asl.CompileFlags |= NODE_HAS_NO_EXIT;
+        }
+        break;
+
+
+    default:
+
+        if ((Op->Asl.CompileFlags & NODE_HAS_NO_EXIT) &&
+            (Op->Asl.Parent))
+        {
+            /* If this node has no exit, then the parent has no exit either */
+
+            Op->Asl.Parent->Asl.CompileFlags |= NODE_HAS_NO_EXIT;
+        }
+        break;
+    }
+
+    return (AE_OK);
+}
diff --git a/sys/contrib/dev/acpica/compiler/aslnamesp.c b/sys/contrib/dev/acpica/compiler/aslnamesp.c
new file mode 100644
index 0000000..5eb3b64
--- /dev/null
+++ b/sys/contrib/dev/acpica/compiler/aslnamesp.c
@@ -0,0 +1,422 @@
+/******************************************************************************
+ *
+ * Module Name: aslnamesp - Namespace output file generation
+ *
+ *****************************************************************************/
+
+/*
+ * Copyright (C) 2000 - 2013, Intel Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ * 3. Neither the names of the above-listed copyright holders nor the names
+ *    of any contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ */
+
+
+#include <contrib/dev/acpica/compiler/aslcompiler.h>
+#include "aslcompiler.y.h"
+#include <contrib/dev/acpica/include/acnamesp.h>
+
+
+#define _COMPONENT          ACPI_COMPILER
+        ACPI_MODULE_NAME    ("aslnamesp")
+
+/* Local prototypes */
+
+static ACPI_STATUS
+NsDoOneNamespaceObject (
+    ACPI_HANDLE             ObjHandle,
+    UINT32                  Level,
+    void                    *Context,
+    void                    **ReturnValue);
+
+static ACPI_STATUS
+NsDoOnePathname (
+    ACPI_HANDLE             ObjHandle,
+    UINT32                  Level,
+    void                    *Context,
+    void                    **ReturnValue);
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    NsSetupNamespaceListing
+ *
+ * PARAMETERS:  Handle          - local file handle
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Set the namespace output file to the input handle
+ *
+ ******************************************************************************/
+
+void
+NsSetupNamespaceListing (
+    void                    *Handle)
+{
+
+    Gbl_NsOutputFlag = TRUE;
+    Gbl_Files[ASL_FILE_NAMESPACE_OUTPUT].Handle = Handle;
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    NsDisplayNamespace
+ *
+ * PARAMETERS:  None
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Walk the namespace an display information about each node
+ *              in the tree. Information is written to the optional
+ *              namespace output file.
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+NsDisplayNamespace (
+    void)
+{
+    ACPI_STATUS             Status;
+
+
+    if (!Gbl_NsOutputFlag)
+    {
+        return (AE_OK);
+    }
+
+    Gbl_NumNamespaceObjects = 0;
+
+    /* File header */
+
+    FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "Contents of ACPI Namespace\n\n");
+    FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "Count  Depth    Name - Type\n\n");
+
+    /* Walk entire namespace from the root */
+
+    Status = AcpiNsWalkNamespace (ACPI_TYPE_ANY, ACPI_ROOT_OBJECT,
+                ACPI_UINT32_MAX, FALSE, NsDoOneNamespaceObject, NULL,
+                NULL, NULL);
+
+    /* Print the full pathname for each namespace node */
+
+    FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "\nNamespace pathnames\n\n");
+
+    Status = AcpiNsWalkNamespace (ACPI_TYPE_ANY, ACPI_ROOT_OBJECT,
+                ACPI_UINT32_MAX, FALSE, NsDoOnePathname, NULL,
+                NULL, NULL);
+
+    return (Status);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    NsDoOneNamespaceObject
+ *
+ * PARAMETERS:  ACPI_WALK_CALLBACK
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Dump a namespace object to the namespace output file.
+ *              Called during the walk of the namespace to dump all objects.
+ *
+ ******************************************************************************/
+
+static ACPI_STATUS
+NsDoOneNamespaceObject (
+    ACPI_HANDLE             ObjHandle,
+    UINT32                  Level,
+    void                    *Context,
+    void                    **ReturnValue)
+{
+    ACPI_NAMESPACE_NODE     *Node = (ACPI_NAMESPACE_NODE *) ObjHandle;
+    ACPI_OPERAND_OBJECT     *ObjDesc;
+    ACPI_PARSE_OBJECT       *Op;
+
+
+    Gbl_NumNamespaceObjects++;
+
+    FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "%5u  [%u]  %*s %4.4s - %s",
+        Gbl_NumNamespaceObjects, Level, (Level * 3), " ",
+        &Node->Name,
+        AcpiUtGetTypeName (Node->Type));
+
+    Op = Node->Op;
+    ObjDesc = ACPI_CAST_PTR (ACPI_OPERAND_OBJECT, Node->Object);
+
+    if (!Op)
+    {
+        FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "\n");
+        return (AE_OK);
+    }
+
+
+    if ((ObjDesc) &&
+        (ACPI_GET_DESCRIPTOR_TYPE (ObjDesc) == ACPI_DESC_TYPE_OPERAND))
+    {
+        switch (Node->Type)
+        {
+        case ACPI_TYPE_INTEGER:
+
+            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
+                "       [Initial Value   0x%8.8X%8.8X]",
+                ACPI_FORMAT_UINT64 (ObjDesc->Integer.Value));
+            break;
+
+
+        case ACPI_TYPE_STRING:
+
+            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
+                "        [Initial Value   \"%s\"]",
+                ObjDesc->String.Pointer);
+            break;
+
+        default:
+            /* Nothing to do for other types */
+            break;
+        }
+
+    }
+    else
+    {
+        switch (Node->Type)
+        {
+        case ACPI_TYPE_INTEGER:
+
+            if (Op->Asl.ParseOpcode == PARSEOP_NAME)
+            {
+                Op = Op->Asl.Child;
+            }
+            if ((Op->Asl.ParseOpcode == PARSEOP_NAMESEG)  ||
+                (Op->Asl.ParseOpcode == PARSEOP_NAMESTRING))
+            {
+                Op = Op->Asl.Next;
+            }
+            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
+                "       [Initial Value   0x%8.8X%8.8X]",
+                ACPI_FORMAT_UINT64 (Op->Asl.Value.Integer));
+            break;
+
+
+        case ACPI_TYPE_STRING:
+
+            if (Op->Asl.ParseOpcode == PARSEOP_NAME)
+            {
+                Op = Op->Asl.Child;
+            }
+            if ((Op->Asl.ParseOpcode == PARSEOP_NAMESEG)  ||
+                (Op->Asl.ParseOpcode == PARSEOP_NAMESTRING))
+            {
+                Op = Op->Asl.Next;
+            }
+            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
+                "        [Initial Value   \"%s\"]",
+                Op->Asl.Value.String);
+            break;
+
+
+        case ACPI_TYPE_LOCAL_REGION_FIELD:
+
+            if ((Op->Asl.ParseOpcode == PARSEOP_NAMESEG)  ||
+                (Op->Asl.ParseOpcode == PARSEOP_NAMESTRING))
+            {
+                Op = Op->Asl.Child;
+            }
+            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
+                "   [Offset 0x%04X   Length 0x%04X bits]",
+                Op->Asl.Parent->Asl.ExtraValue, (UINT32) Op->Asl.Value.Integer);
+            break;
+
+
+        case ACPI_TYPE_BUFFER_FIELD:
+
+            switch (Op->Asl.ParseOpcode)
+            {
+            case PARSEOP_CREATEBYTEFIELD:
+                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "   [BYTE  ( 8 bit)]");
+                break;
+
+            case PARSEOP_CREATEDWORDFIELD:
+                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "   [DWORD (32 bit)]");
+                break;
+
+            case PARSEOP_CREATEQWORDFIELD:
+                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "   [QWORD (64 bit)]");
+                break;
+
+            case PARSEOP_CREATEWORDFIELD:
+                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "   [WORD  (16 bit)]");
+                break;
+
+            case PARSEOP_CREATEBITFIELD:
+                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "   [BIT   ( 1 bit)]");
+                break;
+
+            case PARSEOP_CREATEFIELD:
+                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "   [Arbitrary Bit Field]");
+                break;
+
+            default:
+                break;
+
+            }
+            break;
+
+
+        case ACPI_TYPE_PACKAGE:
+
+            if (Op->Asl.ParseOpcode == PARSEOP_NAME)
+            {
+                Op = Op->Asl.Child;
+            }
+            if ((Op->Asl.ParseOpcode == PARSEOP_NAMESEG)  ||
+                (Op->Asl.ParseOpcode == PARSEOP_NAMESTRING))
+            {
+                Op = Op->Asl.Next;
+            }
+            Op = Op->Asl.Child;
+
+            if ((Op->Asl.ParseOpcode == PARSEOP_BYTECONST) ||
+                (Op->Asl.ParseOpcode == PARSEOP_RAW_DATA))
+            {
+                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
+                    "       [Initial Length  0x%.2X elements]",
+                    Op->Asl.Value.Integer);
+            }
+            break;
+
+
+        case ACPI_TYPE_BUFFER:
+
+            if (Op->Asl.ParseOpcode == PARSEOP_NAME)
+            {
+                Op = Op->Asl.Child;
+            }
+            if ((Op->Asl.ParseOpcode == PARSEOP_NAMESEG)  ||
+                (Op->Asl.ParseOpcode == PARSEOP_NAMESTRING))
+            {
+                Op = Op->Asl.Next;
+            }
+            Op = Op->Asl.Child;
+
+            if (Op && (Op->Asl.ParseOpcode == PARSEOP_INTEGER))
+            {
+                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
+                    "        [Initial Length  0x%.2X bytes]",
+                    Op->Asl.Value.Integer);
+            }
+            break;
+
+
+        case ACPI_TYPE_METHOD:
+
+            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
+                "        [Code Length     0x%.4X bytes]",
+                Op->Asl.AmlSubtreeLength);
+            break;
+
+
+        case ACPI_TYPE_LOCAL_RESOURCE:
+
+            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
+                "  [Desc Offset     0x%.4X Bytes]", Node->Value);
+            break;
+
+
+        case ACPI_TYPE_LOCAL_RESOURCE_FIELD:
+
+            FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
+                "   [Field Offset    0x%.4X Bits 0x%.4X Bytes] ",
+                Node->Value, Node->Value / 8);
+
+            if (Node->Flags & ANOBJ_IS_REFERENCED)
+            {
+                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
+                    "Referenced");
+            }
+            else
+            {
+                FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT,
+                    "Name not referenced");
+            }
+            break;
+
+
+        default:
+            /* Nothing to do for other types */
+            break;
+        }
+    }
+
+    FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "\n");
+    return (AE_OK);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    NsDoOnePathname
+ *
+ * PARAMETERS:  ACPI_WALK_CALLBACK
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Print the full pathname for a namespace node.
+ *
+ ******************************************************************************/
+
+static ACPI_STATUS
+NsDoOnePathname (
+    ACPI_HANDLE             ObjHandle,
+    UINT32                  Level,
+    void                    *Context,
+    void                    **ReturnValue)
+{
+    ACPI_NAMESPACE_NODE     *Node = (ACPI_NAMESPACE_NODE *) ObjHandle;
+    ACPI_STATUS             Status;
+    ACPI_BUFFER             TargetPath;
+
+
+    TargetPath.Length = ACPI_ALLOCATE_LOCAL_BUFFER;
+    Status = AcpiNsHandleToPathname (Node, &TargetPath);
+    if (ACPI_FAILURE (Status))
+    {
+        return (Status);
+    }
+
+    FlPrintFile (ASL_FILE_NAMESPACE_OUTPUT, "%s\n", TargetPath.Pointer);
+    ACPI_FREE (TargetPath.Pointer);
+
+    return (AE_OK);
+}
diff --git a/sys/contrib/dev/acpica/compiler/aslopcodes.c b/sys/contrib/dev/acpica/compiler/aslopcodes.c
index 9f4ab42..ca358dc 100644
--- a/sys/contrib/dev/acpica/compiler/aslopcodes.c
+++ b/sys/contrib/dev/acpica/compiler/aslopcodes.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -802,6 +802,14 @@ OpcGenerateAmlOpcode (
         Op->Asl.Child->Asl.Next->Asl.ParseOpcode = PARSEOP_DEFAULT_ARG;
         break;
 
+    case PARSEOP_TIMER:
+
+        if (AcpiGbl_IntegerBitWidth == 32)
+        {
+            AslError (ASL_REMARK, ASL_MSG_TRUNCATION, Op, NULL);
+        }
+        break;
+
     default:
         /* Nothing to do for other opcodes */
         break;
diff --git a/sys/contrib/dev/acpica/compiler/asloperands.c b/sys/contrib/dev/acpica/compiler/asloperands.c
index 2427265..f773f6f 100644
--- a/sys/contrib/dev/acpica/compiler/asloperands.c
+++ b/sys/contrib/dev/acpica/compiler/asloperands.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslopt.c b/sys/contrib/dev/acpica/compiler/aslopt.c
index 244e46c..3471006 100644
--- a/sys/contrib/dev/acpica/compiler/aslopt.c
+++ b/sys/contrib/dev/acpica/compiler/aslopt.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -305,7 +305,7 @@ OptBuildShortestPath (
 
     for (i = 0; i < NumCarats; i++)
     {
-        NewPathExternal[i] = '^';
+        NewPathExternal[i] = AML_PARENT_PREFIX;
     }
 
     /*
@@ -329,7 +329,7 @@ OptBuildShortestPath (
 
         if (Op->Asl.AmlOpcode == AML_SCOPE_OP)
         {
-            NewPathExternal[i] = '^';
+            NewPathExternal[i] = AML_PARENT_PREFIX;
             i++;
             ACPI_DEBUG_PRINT_RAW ((ACPI_DB_OPTIMIZATIONS, "(EXTRA ^)"));
         }
@@ -448,7 +448,7 @@ OptOptimizeNameDeclaration (
 
     if (((CurrentNode == AcpiGbl_RootNode) ||
         (Op->Common.Parent->Asl.ParseOpcode == PARSEOP_DEFINITIONBLOCK)) &&
-            (AmlNameString[0] == '\\'))
+            (ACPI_IS_ROOT_PREFIX (AmlNameString[0])))
     {
         /*
          * The current scope is the root, and the namepath has a root prefix
diff --git a/sys/contrib/dev/acpica/compiler/aslpredef.c b/sys/contrib/dev/acpica/compiler/aslpredef.c
index a5ebb83..dea0c83 100644
--- a/sys/contrib/dev/acpica/compiler/aslpredef.c
+++ b/sys/contrib/dev/acpica/compiler/aslpredef.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -66,6 +66,7 @@ ApCheckForSpecialName (
 
 static void
 ApCheckObjectType (
+    const char              *PredefinedName,
     ACPI_PARSE_OBJECT       *Op,
     UINT32                  ExpectedBtypes);
 
@@ -383,7 +384,8 @@ ApCheckPredefinedReturnValue (
 
             /* Static data return object - check against expected type */
 
-            ApCheckObjectType (ReturnValueOp,
+            ApCheckObjectType (PredefinedNames[Index].Info.Name,
+                ReturnValueOp,
                 PredefinedNames[Index].Info.ExpectedBtypes);
             break;
 
@@ -482,7 +484,8 @@ ApCheckForPredefinedObject (
 
         /* Typecheck the actual object, it is the next argument */
 
-        ApCheckObjectType (Op->Asl.Child->Asl.Next,
+        ApCheckObjectType (PredefinedNames[Index].Info.Name,
+            Op->Asl.Child->Asl.Next,
             PredefinedNames[Index].Info.ExpectedBtypes);
         return;
     }
@@ -640,7 +643,8 @@ ApCheckForSpecialName (
  *
  * FUNCTION:    ApCheckObjectType
  *
- * PARAMETERS:  Op              - Current parse node
+ * PARAMETERS:  PredefinedName  - Name of the predefined object we are checking
+ *              Op              - Current parse node
  *              ExpectedBtypes  - Bitmap of expected return type(s)
  *
  * RETURN:      None
@@ -653,6 +657,7 @@ ApCheckForSpecialName (
 
 static void
 ApCheckObjectType (
+    const char              *PredefinedName,
     ACPI_PARSE_OBJECT       *Op,
     UINT32                  ExpectedBtypes)
 {
@@ -701,8 +706,8 @@ TypeErrorExit:
 
     ApGetExpectedTypes (StringBuffer, ExpectedBtypes);
 
-    sprintf (MsgBuffer, "found %s, requires %s",
-        UtGetOpName (Op->Asl.ParseOpcode), StringBuffer);
+    sprintf (MsgBuffer, "%s: found %s, requires %s",
+        PredefinedName, UtGetOpName (Op->Asl.ParseOpcode), StringBuffer);
 
     AslError (ASL_ERROR, ASL_MSG_RESERVED_OPERAND_TYPE, Op,
         MsgBuffer);
diff --git a/sys/contrib/dev/acpica/compiler/aslresource.c b/sys/contrib/dev/acpica/compiler/aslresource.c
index e0e582a..d809b73 100644
--- a/sys/contrib/dev/acpica/compiler/aslresource.c
+++ b/sys/contrib/dev/acpica/compiler/aslresource.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslrestype1.c b/sys/contrib/dev/acpica/compiler/aslrestype1.c
index 3de2421..21a8705 100644
--- a/sys/contrib/dev/acpica/compiler/aslrestype1.c
+++ b/sys/contrib/dev/acpica/compiler/aslrestype1.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslrestype1i.c b/sys/contrib/dev/acpica/compiler/aslrestype1i.c
index 77bdd91..3ec92aa 100644
--- a/sys/contrib/dev/acpica/compiler/aslrestype1i.c
+++ b/sys/contrib/dev/acpica/compiler/aslrestype1i.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslrestype2.c b/sys/contrib/dev/acpica/compiler/aslrestype2.c
index 4784f86..942d45e 100644
--- a/sys/contrib/dev/acpica/compiler/aslrestype2.c
+++ b/sys/contrib/dev/acpica/compiler/aslrestype2.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslrestype2d.c b/sys/contrib/dev/acpica/compiler/aslrestype2d.c
index 7836461..035b6ec 100644
--- a/sys/contrib/dev/acpica/compiler/aslrestype2d.c
+++ b/sys/contrib/dev/acpica/compiler/aslrestype2d.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslrestype2e.c b/sys/contrib/dev/acpica/compiler/aslrestype2e.c
index fe2f0d8..5621762 100644
--- a/sys/contrib/dev/acpica/compiler/aslrestype2e.c
+++ b/sys/contrib/dev/acpica/compiler/aslrestype2e.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslrestype2q.c b/sys/contrib/dev/acpica/compiler/aslrestype2q.c
index 70cfbd5..3a002a4 100644
--- a/sys/contrib/dev/acpica/compiler/aslrestype2q.c
+++ b/sys/contrib/dev/acpica/compiler/aslrestype2q.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslrestype2s.c b/sys/contrib/dev/acpica/compiler/aslrestype2s.c
index fa30b58..42d51a9 100644
--- a/sys/contrib/dev/acpica/compiler/aslrestype2s.c
+++ b/sys/contrib/dev/acpica/compiler/aslrestype2s.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslrestype2w.c b/sys/contrib/dev/acpica/compiler/aslrestype2w.c
index 613b7b3..be388b7 100644
--- a/sys/contrib/dev/acpica/compiler/aslrestype2w.c
+++ b/sys/contrib/dev/acpica/compiler/aslrestype2w.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslstartup.c b/sys/contrib/dev/acpica/compiler/aslstartup.c
index 3d50a21..8deb894 100644
--- a/sys/contrib/dev/acpica/compiler/aslstartup.c
+++ b/sys/contrib/dev/acpica/compiler/aslstartup.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -341,7 +341,7 @@ AslDoOneFile (
         /* TBD: Handle additional output files for disassembler */
 
         Status = FlOpenMiscOutputFiles (Gbl_OutputFilenamePrefix);
-        LsDisplayNamespace ();
+        NsDisplayNamespace ();
 #endif
 
         /* Shutdown compiler and ACPICA subsystem */
diff --git a/sys/contrib/dev/acpica/compiler/aslstubs.c b/sys/contrib/dev/acpica/compiler/aslstubs.c
index 8711441..9ac9698 100644
--- a/sys/contrib/dev/acpica/compiler/aslstubs.c
+++ b/sys/contrib/dev/acpica/compiler/aslstubs.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslsupport.l b/sys/contrib/dev/acpica/compiler/aslsupport.l
index daab402..90af049 100644
--- a/sys/contrib/dev/acpica/compiler/aslsupport.l
+++ b/sys/contrib/dev/acpica/compiler/aslsupport.l
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/asltransform.c b/sys/contrib/dev/acpica/compiler/asltransform.c
index 7c8bfcd..7a29446 100644
--- a/sys/contrib/dev/acpica/compiler/asltransform.c
+++ b/sys/contrib/dev/acpica/compiler/asltransform.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/asltree.c b/sys/contrib/dev/acpica/compiler/asltree.c
index 86e7f2d..8fa532b 100644
--- a/sys/contrib/dev/acpica/compiler/asltree.c
+++ b/sys/contrib/dev/acpica/compiler/asltree.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/asltypes.h b/sys/contrib/dev/acpica/compiler/asltypes.h
index 0e3b16a..538911f 100644
--- a/sys/contrib/dev/acpica/compiler/asltypes.h
+++ b/sys/contrib/dev/acpica/compiler/asltypes.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslutils.c b/sys/contrib/dev/acpica/compiler/aslutils.c
index 32ab1c9..1a4a27d 100644
--- a/sys/contrib/dev/acpica/compiler/aslutils.c
+++ b/sys/contrib/dev/acpica/compiler/aslutils.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -826,7 +826,8 @@ UtAttachNameseg (
         /* No dots in the namepath, there is only a single nameseg. */
         /* Handle prefixes */
 
-        while ((*Name == '\\') || (*Name == '^'))
+        while (ACPI_IS_ROOT_PREFIX (*Name) ||
+               ACPI_IS_PARENT_PREFIX (*Name))
         {
             Name++;
         }
diff --git a/sys/contrib/dev/acpica/compiler/asluuid.c b/sys/contrib/dev/acpica/compiler/asluuid.c
index 485db6b..ed884dc 100644
--- a/sys/contrib/dev/acpica/compiler/asluuid.c
+++ b/sys/contrib/dev/acpica/compiler/asluuid.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/aslwalks.c b/sys/contrib/dev/acpica/compiler/aslwalks.c
index 77bff82..6613cac 100644
--- a/sys/contrib/dev/acpica/compiler/aslwalks.c
+++ b/sys/contrib/dev/acpica/compiler/aslwalks.c
@@ -1,11 +1,11 @@
 /******************************************************************************
  *
- * Module Name: aslwalks.c - major analytical parse tree walks
+ * Module Name: aslwalks.c - Miscellaneous analytical parse tree walks
  *
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -54,575 +54,6 @@
 
 /*******************************************************************************
  *
- * FUNCTION:    AnMethodAnalysisWalkBegin
- *
- * PARAMETERS:  ASL_WALK_CALLBACK
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Descending callback for the analysis walk. Check methods for:
- *              1) Initialized local variables
- *              2) Valid arguments
- *              3) Return types
- *
- ******************************************************************************/
-
-ACPI_STATUS
-AnMethodAnalysisWalkBegin (
-    ACPI_PARSE_OBJECT       *Op,
-    UINT32                  Level,
-    void                    *Context)
-{
-    ASL_ANALYSIS_WALK_INFO  *WalkInfo = (ASL_ANALYSIS_WALK_INFO *) Context;
-    ASL_METHOD_INFO         *MethodInfo = WalkInfo->MethodStack;
-    ACPI_PARSE_OBJECT       *Next;
-    UINT32                  RegisterNumber;
-    UINT32                  i;
-    char                    LocalName[] = "Local0";
-    char                    ArgName[] = "Arg0";
-    ACPI_PARSE_OBJECT       *ArgNode;
-    ACPI_PARSE_OBJECT       *NextType;
-    ACPI_PARSE_OBJECT       *NextParamType;
-    UINT8                   ActualArgs = 0;
-
-
-    switch (Op->Asl.ParseOpcode)
-    {
-    case PARSEOP_METHOD:
-
-        TotalMethods++;
-
-        /* Create and init method info */
-
-        MethodInfo       = UtLocalCalloc (sizeof (ASL_METHOD_INFO));
-        MethodInfo->Next = WalkInfo->MethodStack;
-        MethodInfo->Op = Op;
-
-        WalkInfo->MethodStack = MethodInfo;
-
-        /* Get the name node, ignored here */
-
-        Next = Op->Asl.Child;
-
-        /* Get the NumArguments node */
-
-        Next = Next->Asl.Next;
-        MethodInfo->NumArguments = (UINT8)
-            (((UINT8) Next->Asl.Value.Integer) & 0x07);
-
-        /* Get the SerializeRule and SyncLevel nodes, ignored here */
-
-        Next = Next->Asl.Next;
-        Next = Next->Asl.Next;
-        ArgNode = Next;
-
-        /* Get the ReturnType node */
-
-        Next = Next->Asl.Next;
-
-        NextType = Next->Asl.Child;
-        while (NextType)
-        {
-            /* Get and map each of the ReturnTypes */
-
-            MethodInfo->ValidReturnTypes |= AnMapObjTypeToBtype (NextType);
-            NextType->Asl.ParseOpcode = PARSEOP_DEFAULT_ARG;
-            NextType = NextType->Asl.Next;
-        }
-
-        /* Get the ParameterType node */
-
-        Next = Next->Asl.Next;
-
-        NextType = Next->Asl.Child;
-        while (NextType)
-        {
-            if (NextType->Asl.ParseOpcode == PARSEOP_DEFAULT_ARG)
-            {
-                NextParamType = NextType->Asl.Child;
-                while (NextParamType)
-                {
-                    MethodInfo->ValidArgTypes[ActualArgs] |= AnMapObjTypeToBtype (NextParamType);
-                    NextParamType->Asl.ParseOpcode = PARSEOP_DEFAULT_ARG;
-                    NextParamType = NextParamType->Asl.Next;
-                }
-            }
-            else
-            {
-                MethodInfo->ValidArgTypes[ActualArgs] =
-                    AnMapObjTypeToBtype (NextType);
-                NextType->Asl.ParseOpcode = PARSEOP_DEFAULT_ARG;
-                ActualArgs++;
-            }
-
-            NextType = NextType->Asl.Next;
-        }
-
-        if ((MethodInfo->NumArguments) &&
-            (MethodInfo->NumArguments != ActualArgs))
-        {
-            /* error: Param list did not match number of args */
-        }
-
-        /* Allow numarguments == 0 for Function() */
-
-        if ((!MethodInfo->NumArguments) && (ActualArgs))
-        {
-            MethodInfo->NumArguments = ActualArgs;
-            ArgNode->Asl.Value.Integer |= ActualArgs;
-        }
-
-        /*
-         * Actual arguments are initialized at method entry.
-         * All other ArgX "registers" can be used as locals, so we
-         * track their initialization.
-         */
-        for (i = 0; i < MethodInfo->NumArguments; i++)
-        {
-            MethodInfo->ArgInitialized[i] = TRUE;
-        }
-        break;
-
-
-    case PARSEOP_METHODCALL:
-
-        if (MethodInfo &&
-           (Op->Asl.Node == MethodInfo->Op->Asl.Node))
-        {
-            AslError (ASL_REMARK, ASL_MSG_RECURSION, Op, Op->Asl.ExternalName);
-        }
-        break;
-
-
-    case PARSEOP_LOCAL0:
-    case PARSEOP_LOCAL1:
-    case PARSEOP_LOCAL2:
-    case PARSEOP_LOCAL3:
-    case PARSEOP_LOCAL4:
-    case PARSEOP_LOCAL5:
-    case PARSEOP_LOCAL6:
-    case PARSEOP_LOCAL7:
-
-        if (!MethodInfo)
-        {
-            /*
-             * Local was used outside a control method, or there was an error
-             * in the method declaration.
-             */
-            AslError (ASL_REMARK, ASL_MSG_LOCAL_OUTSIDE_METHOD, Op, Op->Asl.ExternalName);
-            return (AE_ERROR);
-        }
-
-        RegisterNumber = (Op->Asl.AmlOpcode & 0x000F);
-
-        /*
-         * If the local is being used as a target, mark the local
-         * initialized
-         */
-        if (Op->Asl.CompileFlags & NODE_IS_TARGET)
-        {
-            MethodInfo->LocalInitialized[RegisterNumber] = TRUE;
-        }
-
-        /*
-         * Otherwise, this is a reference, check if the local
-         * has been previously initialized.
-         *
-         * The only operator that accepts an uninitialized value is ObjectType()
-         */
-        else if ((!MethodInfo->LocalInitialized[RegisterNumber]) &&
-                 (Op->Asl.Parent->Asl.ParseOpcode != PARSEOP_OBJECTTYPE))
-        {
-            LocalName[strlen (LocalName) -1] = (char) (RegisterNumber + 0x30);
-            AslError (ASL_ERROR, ASL_MSG_LOCAL_INIT, Op, LocalName);
-        }
-        break;
-
-
-    case PARSEOP_ARG0:
-    case PARSEOP_ARG1:
-    case PARSEOP_ARG2:
-    case PARSEOP_ARG3:
-    case PARSEOP_ARG4:
-    case PARSEOP_ARG5:
-    case PARSEOP_ARG6:
-
-        if (!MethodInfo)
-        {
-            /*
-             * Arg was used outside a control method, or there was an error
-             * in the method declaration.
-             */
-            AslError (ASL_REMARK, ASL_MSG_LOCAL_OUTSIDE_METHOD, Op, Op->Asl.ExternalName);
-            return (AE_ERROR);
-        }
-
-        RegisterNumber = (Op->Asl.AmlOpcode & 0x000F) - 8;
-        ArgName[strlen (ArgName) -1] = (char) (RegisterNumber + 0x30);
-
-        /*
-         * If the Arg is being used as a target, mark the local
-         * initialized
-         */
-        if (Op->Asl.CompileFlags & NODE_IS_TARGET)
-        {
-            MethodInfo->ArgInitialized[RegisterNumber] = TRUE;
-        }
-
-        /*
-         * Otherwise, this is a reference, check if the Arg
-         * has been previously initialized.
-         *
-         * The only operator that accepts an uninitialized value is ObjectType()
-         */
-        else if ((!MethodInfo->ArgInitialized[RegisterNumber]) &&
-                 (Op->Asl.Parent->Asl.ParseOpcode != PARSEOP_OBJECTTYPE))
-        {
-            AslError (ASL_ERROR, ASL_MSG_ARG_INIT, Op, ArgName);
-        }
-
-        /* Flag this arg if it is not a "real" argument to the method */
-
-        if (RegisterNumber >= MethodInfo->NumArguments)
-        {
-            AslError (ASL_REMARK, ASL_MSG_NOT_PARAMETER, Op, ArgName);
-        }
-        break;
-
-
-    case PARSEOP_RETURN:
-
-        if (!MethodInfo)
-        {
-            /*
-             * Probably was an error in the method declaration,
-             * no additional error here
-             */
-            ACPI_WARNING ((AE_INFO, "%p, No parent method", Op));
-            return (AE_ERROR);
-        }
-
-        /*
-         * A child indicates a possible return value. A simple Return or
-         * Return() is marked with NODE_IS_NULL_RETURN by the parser so
-         * that it is not counted as a "real" return-with-value, although
-         * the AML code that is actually emitted is Return(0). The AML
-         * definition of Return has a required parameter, so we are
-         * forced to convert a null return to Return(0).
-         */
-        if ((Op->Asl.Child) &&
-            (Op->Asl.Child->Asl.ParseOpcode != PARSEOP_DEFAULT_ARG) &&
-            (!(Op->Asl.Child->Asl.CompileFlags & NODE_IS_NULL_RETURN)))
-        {
-            MethodInfo->NumReturnWithValue++;
-        }
-        else
-        {
-            MethodInfo->NumReturnNoValue++;
-        }
-        break;
-
-
-    case PARSEOP_BREAK:
-    case PARSEOP_CONTINUE:
-
-        Next = Op->Asl.Parent;
-        while (Next)
-        {
-            if (Next->Asl.ParseOpcode == PARSEOP_WHILE)
-            {
-                break;
-            }
-            Next = Next->Asl.Parent;
-        }
-
-        if (!Next)
-        {
-            AslError (ASL_ERROR, ASL_MSG_NO_WHILE, Op, NULL);
-        }
-        break;
-
-
-    case PARSEOP_STALL:
-
-        /* We can range check if the argument is an integer */
-
-        if ((Op->Asl.Child->Asl.ParseOpcode == PARSEOP_INTEGER) &&
-            (Op->Asl.Child->Asl.Value.Integer > ACPI_UINT8_MAX))
-        {
-            AslError (ASL_ERROR, ASL_MSG_INVALID_TIME, Op, NULL);
-        }
-        break;
-
-
-    case PARSEOP_DEVICE:
-    case PARSEOP_EVENT:
-    case PARSEOP_MUTEX:
-    case PARSEOP_OPERATIONREGION:
-    case PARSEOP_POWERRESOURCE:
-    case PARSEOP_PROCESSOR:
-    case PARSEOP_THERMALZONE:
-
-        /*
-         * The first operand is a name to be created in the namespace.
-         * Check against the reserved list.
-         */
-        i = ApCheckForPredefinedName (Op, Op->Asl.NameSeg);
-        if (i < ACPI_VALID_RESERVED_NAME_MAX)
-        {
-            AslError (ASL_ERROR, ASL_MSG_RESERVED_USE, Op, Op->Asl.ExternalName);
-        }
-        break;
-
-
-    case PARSEOP_NAME:
-
-        /* Typecheck any predefined names statically defined with Name() */
-
-        ApCheckForPredefinedObject (Op, Op->Asl.NameSeg);
-
-        /* Special typechecking for _HID */
-
-        if (!ACPI_STRCMP (METHOD_NAME__HID, Op->Asl.NameSeg))
-        {
-            Next = Op->Asl.Child->Asl.Next;
-            AnCheckId (Next, ASL_TYPE_HID);
-        }
-
-        /* Special typechecking for _CID */
-
-        else if (!ACPI_STRCMP (METHOD_NAME__CID, Op->Asl.NameSeg))
-        {
-            Next = Op->Asl.Child->Asl.Next;
-
-            if ((Next->Asl.ParseOpcode == PARSEOP_PACKAGE) ||
-                (Next->Asl.ParseOpcode == PARSEOP_VAR_PACKAGE))
-            {
-                Next = Next->Asl.Child;
-                while (Next)
-                {
-                    AnCheckId (Next, ASL_TYPE_CID);
-                    Next = Next->Asl.Next;
-                }
-            }
-            else
-            {
-                AnCheckId (Next, ASL_TYPE_CID);
-            }
-        }
-        break;
-
-
-    default:
-        break;
-    }
-
-    return (AE_OK);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AnMethodAnalysisWalkEnd
- *
- * PARAMETERS:  ASL_WALK_CALLBACK
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Ascending callback for analysis walk. Complete method
- *              return analysis.
- *
- ******************************************************************************/
-
-ACPI_STATUS
-AnMethodAnalysisWalkEnd (
-    ACPI_PARSE_OBJECT       *Op,
-    UINT32                  Level,
-    void                    *Context)
-{
-    ASL_ANALYSIS_WALK_INFO  *WalkInfo = (ASL_ANALYSIS_WALK_INFO *) Context;
-    ASL_METHOD_INFO         *MethodInfo = WalkInfo->MethodStack;
-
-
-    switch (Op->Asl.ParseOpcode)
-    {
-    case PARSEOP_METHOD:
-    case PARSEOP_RETURN:
-        if (!MethodInfo)
-        {
-            printf ("No method info for method! [%s]\n", Op->Asl.Namepath);
-            AslError (ASL_ERROR, ASL_MSG_COMPILER_INTERNAL, Op,
-                "No method info for this method");
-
-            CmCleanupAndExit ();
-            return (AE_AML_INTERNAL);
-        }
-        break;
-
-    default:
-        break;
-    }
-
-    switch (Op->Asl.ParseOpcode)
-    {
-    case PARSEOP_METHOD:
-
-        WalkInfo->MethodStack = MethodInfo->Next;
-
-        /*
-         * Check if there is no return statement at the end of the
-         * method AND we can actually get there -- i.e., the execution
-         * of the method can possibly terminate without a return statement.
-         */
-        if ((!AnLastStatementIsReturn (Op)) &&
-            (!(Op->Asl.CompileFlags & NODE_HAS_NO_EXIT)))
-        {
-            /*
-             * No return statement, and execution can possibly exit
-             * via this path. This is equivalent to Return ()
-             */
-            MethodInfo->NumReturnNoValue++;
-        }
-
-        /*
-         * Check for case where some return statements have a return value
-         * and some do not. Exit without a return statement is a return with
-         * no value
-         */
-        if (MethodInfo->NumReturnNoValue &&
-            MethodInfo->NumReturnWithValue)
-        {
-            AslError (ASL_WARNING, ASL_MSG_RETURN_TYPES, Op,
-                Op->Asl.ExternalName);
-        }
-
-        /*
-         * If there are any RETURN() statements with no value, or there is a
-         * control path that allows the method to exit without a return value,
-         * we mark the method as a method that does not return a value. This
-         * knowledge can be used to check method invocations that expect a
-         * returned value.
-         */
-        if (MethodInfo->NumReturnNoValue)
-        {
-            if (MethodInfo->NumReturnWithValue)
-            {
-                Op->Asl.CompileFlags |= NODE_METHOD_SOME_NO_RETVAL;
-            }
-            else
-            {
-                Op->Asl.CompileFlags |= NODE_METHOD_NO_RETVAL;
-            }
-        }
-
-        /*
-         * Check predefined method names for correct return behavior
-         * and correct number of arguments. Also, some special checks
-         * For GPE and _REG methods.
-         */
-        if (ApCheckForPredefinedMethod (Op, MethodInfo))
-        {
-            /* Special check for two names like _L01 and _E01 in same scope */
-
-            ApCheckForGpeNameConflict (Op);
-
-            /*
-             * Special check for _REG: Must have an operation region definition
-             * within the same scope!
-             */
-            ApCheckRegMethod (Op);
-        }
-
-        ACPI_FREE (MethodInfo);
-        break;
-
-
-    case PARSEOP_NAME:
-
-         /* Special check for two names like _L01 and _E01 in same scope */
-
-        ApCheckForGpeNameConflict (Op);
-        break;
-
-
-    case PARSEOP_RETURN:
-
-        /*
-         * If the parent is a predefined method name, attempt to typecheck
-         * the return value. Only static types can be validated.
-         */
-        ApCheckPredefinedReturnValue (Op, MethodInfo);
-
-        /*
-         * The parent block does not "exit" and continue execution -- the
-         * method is terminated here with the Return() statement.
-         */
-        Op->Asl.Parent->Asl.CompileFlags |= NODE_HAS_NO_EXIT;
-
-        /* Used in the "typing" pass later */
-
-        Op->Asl.ParentMethod = MethodInfo->Op;
-
-        /*
-         * If there is a peer node after the return statement, then this
-         * node is unreachable code -- i.e., it won't be executed because of
-         * the preceding Return() statement.
-         */
-        if (Op->Asl.Next)
-        {
-            AslError (ASL_WARNING, ASL_MSG_UNREACHABLE_CODE, Op->Asl.Next, NULL);
-        }
-        break;
-
-
-    case PARSEOP_IF:
-
-        if ((Op->Asl.CompileFlags & NODE_HAS_NO_EXIT) &&
-            (Op->Asl.Next) &&
-            (Op->Asl.Next->Asl.ParseOpcode == PARSEOP_ELSE))
-        {
-            /*
-             * This IF has a corresponding ELSE. The IF block has no exit,
-             * (it contains an unconditional Return)
-             * mark the ELSE block to remember this fact.
-             */
-            Op->Asl.Next->Asl.CompileFlags |= NODE_IF_HAS_NO_EXIT;
-        }
-        break;
-
-
-    case PARSEOP_ELSE:
-
-        if ((Op->Asl.CompileFlags & NODE_HAS_NO_EXIT) &&
-            (Op->Asl.CompileFlags & NODE_IF_HAS_NO_EXIT))
-        {
-            /*
-             * This ELSE block has no exit and the corresponding IF block
-             * has no exit either. Therefore, the parent node has no exit.
-             */
-            Op->Asl.Parent->Asl.CompileFlags |= NODE_HAS_NO_EXIT;
-        }
-        break;
-
-
-    default:
-
-        if ((Op->Asl.CompileFlags & NODE_HAS_NO_EXIT) &&
-            (Op->Asl.Parent))
-        {
-            /* If this node has no exit, then the parent has no exit either */
-
-            Op->Asl.Parent->Asl.CompileFlags |= NODE_HAS_NO_EXIT;
-        }
-        break;
-    }
-
-    return (AE_OK);
-}
-
-
-/*******************************************************************************
- *
  * FUNCTION:    AnMethodTypingWalkEnd
  *
  * PARAMETERS:  ASL_WALK_CALLBACK
diff --git a/sys/contrib/dev/acpica/compiler/aslxref.c b/sys/contrib/dev/acpica/compiler/aslxref.c
new file mode 100644
index 0000000..188c214
--- /dev/null
+++ b/sys/contrib/dev/acpica/compiler/aslxref.c
@@ -0,0 +1,874 @@
+/******************************************************************************
+ *
+ * Module Name: aslxref - Namespace cross-reference
+ *
+ *****************************************************************************/
+
+/*
+ * Copyright (C) 2000 - 2013, Intel Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ * 3. Neither the names of the above-listed copyright holders nor the names
+ *    of any contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ */
+
+
+#include <contrib/dev/acpica/compiler/aslcompiler.h>
+#include "aslcompiler.y.h"
+#include <contrib/dev/acpica/include/acparser.h>
+#include <contrib/dev/acpica/include/amlcode.h>
+#include <contrib/dev/acpica/include/acnamesp.h>
+#include <contrib/dev/acpica/include/acdispat.h>
+
+
+#define _COMPONENT          ACPI_COMPILER
+        ACPI_MODULE_NAME    ("aslxref")
+
+/* Local prototypes */
+
+static ACPI_STATUS
+XfNamespaceLocateBegin (
+    ACPI_PARSE_OBJECT       *Op,
+    UINT32                  Level,
+    void                    *Context);
+
+static ACPI_STATUS
+XfNamespaceLocateEnd (
+    ACPI_PARSE_OBJECT       *Op,
+    UINT32                  Level,
+    void                    *Context);
+
+static BOOLEAN
+XfObjectExists (
+    char                    *Name);
+
+static ACPI_STATUS
+XfCompareOneNamespaceObject (
+    ACPI_HANDLE             ObjHandle,
+    UINT32                  Level,
+    void                    *Context,
+    void                    **ReturnValue);
+
+static void
+XfCheckFieldRange (
+    ACPI_PARSE_OBJECT       *Op,
+    UINT32                  RegionBitLength,
+    UINT32                  FieldBitOffset,
+    UINT32                  FieldBitLength,
+    UINT32                  AccessBitWidth);
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    XfCrossReferenceNamespace
+ *
+ * PARAMETERS:  None
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Perform a cross reference check of the parse tree against the
+ *              namespace. Every named referenced within the parse tree
+ *              should be get resolved with a namespace lookup. If not, the
+ *              original reference in the ASL code is invalid -- i.e., refers
+ *              to a non-existent object.
+ *
+ * NOTE:  The ASL "External" operator causes the name to be inserted into the
+ *        namespace so that references to the external name will be resolved
+ *        correctly here.
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+XfCrossReferenceNamespace (
+    void)
+{
+    ACPI_WALK_STATE         *WalkState;
+
+
+    DbgPrint (ASL_DEBUG_OUTPUT, "\nCross referencing namespace\n\n");
+
+    /*
+     * Create a new walk state for use when looking up names
+     * within the namespace (Passed as context to the callbacks)
+     */
+    WalkState = AcpiDsCreateWalkState (0, NULL, NULL, NULL);
+    if (!WalkState)
+    {
+        return (AE_NO_MEMORY);
+    }
+
+    /* Walk the entire parse tree */
+
+    TrWalkParseTree (RootNode, ASL_WALK_VISIT_TWICE, XfNamespaceLocateBegin,
+                        XfNamespaceLocateEnd, WalkState);
+    return (AE_OK);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    XfObjectExists
+ *
+ * PARAMETERS:  Name            - 4 char ACPI name
+ *
+ * RETURN:      TRUE if name exists in namespace
+ *
+ * DESCRIPTION: Walk the namespace to find an object
+ *
+ ******************************************************************************/
+
+static BOOLEAN
+XfObjectExists (
+    char                    *Name)
+{
+    ACPI_STATUS             Status;
+
+
+    /* Walk entire namespace from the supplied root */
+
+    Status = AcpiNsWalkNamespace (ACPI_TYPE_ANY, ACPI_ROOT_OBJECT,
+                ACPI_UINT32_MAX, FALSE, XfCompareOneNamespaceObject, NULL,
+                Name, NULL);
+    if (Status == AE_CTRL_TRUE)
+    {
+        /* At least one instance of the name was found */
+
+        return (TRUE);
+    }
+
+    return (FALSE);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    XfCompareOneNamespaceObject
+ *
+ * PARAMETERS:  ACPI_WALK_CALLBACK
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Compare name of one object.
+ *
+ ******************************************************************************/
+
+static ACPI_STATUS
+XfCompareOneNamespaceObject (
+    ACPI_HANDLE             ObjHandle,
+    UINT32                  Level,
+    void                    *Context,
+    void                    **ReturnValue)
+{
+    ACPI_NAMESPACE_NODE     *Node = (ACPI_NAMESPACE_NODE *) ObjHandle;
+
+
+    /* Simply check the name */
+
+    if (*((UINT32 *) (Context)) == Node->Name.Integer)
+    {
+        /* Abort walk if we found one instance */
+
+        return (AE_CTRL_TRUE);
+    }
+
+    return (AE_OK);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    XfCheckFieldRange
+ *
+ * PARAMETERS:  RegionBitLength     - Length of entire parent region
+ *              FieldBitOffset      - Start of the field unit (within region)
+ *              FieldBitLength      - Entire length of field unit
+ *              AccessBitWidth      - Access width of the field unit
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Check one field unit to make sure it fits in the parent
+ *              op region.
+ *
+ * Note: AccessBitWidth must be either 8,16,32, or 64
+ *
+ ******************************************************************************/
+
+static void
+XfCheckFieldRange (
+    ACPI_PARSE_OBJECT       *Op,
+    UINT32                  RegionBitLength,
+    UINT32                  FieldBitOffset,
+    UINT32                  FieldBitLength,
+    UINT32                  AccessBitWidth)
+{
+    UINT32                  FieldEndBitOffset;
+
+
+    /*
+     * Check each field unit against the region size. The entire
+     * field unit (start offset plus length) must fit within the
+     * region.
+     */
+    FieldEndBitOffset = FieldBitOffset + FieldBitLength;
+
+    if (FieldEndBitOffset > RegionBitLength)
+    {
+        /* Field definition itself is beyond the end-of-region */
+
+        AslError (ASL_ERROR, ASL_MSG_FIELD_UNIT_OFFSET, Op, NULL);
+        return;
+    }
+
+    /*
+     * Now check that the field plus AccessWidth doesn't go beyond
+     * the end-of-region. Assumes AccessBitWidth is a power of 2
+     */
+    FieldEndBitOffset = ACPI_ROUND_UP (FieldEndBitOffset, AccessBitWidth);
+
+    if (FieldEndBitOffset > RegionBitLength)
+    {
+        /* Field definition combined with the access is beyond EOR */
+
+        AslError (ASL_ERROR, ASL_MSG_FIELD_UNIT_ACCESS_WIDTH, Op, NULL);
+    }
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    XfNamespaceLocateBegin
+ *
+ * PARAMETERS:  ASL_WALK_CALLBACK
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Descending callback used during cross-reference. For named
+ *              object references, attempt to locate the name in the
+ *              namespace.
+ *
+ * NOTE: ASL references to named fields within resource descriptors are
+ *       resolved to integer values here. Therefore, this step is an
+ *       important part of the code generation. We don't know that the
+ *       name refers to a resource descriptor until now.
+ *
+ ******************************************************************************/
+
+static ACPI_STATUS
+XfNamespaceLocateBegin (
+    ACPI_PARSE_OBJECT       *Op,
+    UINT32                  Level,
+    void                    *Context)
+{
+    ACPI_WALK_STATE         *WalkState = (ACPI_WALK_STATE *) Context;
+    ACPI_NAMESPACE_NODE     *Node;
+    ACPI_STATUS             Status;
+    ACPI_OBJECT_TYPE        ObjectType;
+    char                    *Path;
+    UINT8                   PassedArgs;
+    ACPI_PARSE_OBJECT       *NextOp;
+    ACPI_PARSE_OBJECT       *OwningOp;
+    ACPI_PARSE_OBJECT       *SpaceIdOp;
+    UINT32                  MinimumLength;
+    UINT32                  Offset;
+    UINT32                  FieldBitLength;
+    UINT32                  TagBitLength;
+    UINT8                   Message = 0;
+    const ACPI_OPCODE_INFO  *OpInfo;
+    UINT32                  Flags;
+
+
+    ACPI_FUNCTION_TRACE_PTR (XfNamespaceLocateBegin, Op);
+
+    /*
+     * If this node is the actual declaration of a name
+     * [such as the XXXX name in "Method (XXXX)"],
+     * we are not interested in it here. We only care about names that are
+     * references to other objects within the namespace and the parent objects
+     * of name declarations
+     */
+    if (Op->Asl.CompileFlags & NODE_IS_NAME_DECLARATION)
+    {
+        return (AE_OK);
+    }
+
+    /* We are only interested in opcodes that have an associated name */
+
+    OpInfo = AcpiPsGetOpcodeInfo (Op->Asl.AmlOpcode);
+
+    if ((!(OpInfo->Flags & AML_NAMED)) &&
+        (!(OpInfo->Flags & AML_CREATE)) &&
+        (Op->Asl.ParseOpcode != PARSEOP_NAMESTRING) &&
+        (Op->Asl.ParseOpcode != PARSEOP_NAMESEG)    &&
+        (Op->Asl.ParseOpcode != PARSEOP_METHODCALL))
+    {
+        return (AE_OK);
+    }
+
+    /*
+     * One special case: CondRefOf operator - we don't care if the name exists
+     * or not at this point, just ignore it, the point of the operator is to
+     * determine if the name exists at runtime.
+     */
+    if ((Op->Asl.Parent) &&
+        (Op->Asl.Parent->Asl.ParseOpcode == PARSEOP_CONDREFOF))
+    {
+        return (AE_OK);
+    }
+
+    /*
+     * We must enable the "search-to-root" for single NameSegs, but
+     * we have to be very careful about opening up scopes
+     */
+    Flags = ACPI_NS_SEARCH_PARENT;
+    if ((Op->Asl.ParseOpcode == PARSEOP_NAMESTRING) ||
+        (Op->Asl.ParseOpcode == PARSEOP_NAMESEG)    ||
+        (Op->Asl.ParseOpcode == PARSEOP_METHODCALL))
+    {
+        /*
+         * These are name references, do not push the scope stack
+         * for them.
+         */
+        Flags |= ACPI_NS_DONT_OPEN_SCOPE;
+    }
+
+    /* Get the NamePath from the appropriate place */
+
+    if (OpInfo->Flags & AML_NAMED)
+    {
+        /* For nearly all NAMED operators, the name reference is the first child */
+
+        Path = Op->Asl.Child->Asl.Value.String;
+        if (Op->Asl.AmlOpcode == AML_ALIAS_OP)
+        {
+            /*
+             * ALIAS is the only oddball opcode, the name declaration
+             * (alias name) is the second operand
+             */
+            Path = Op->Asl.Child->Asl.Next->Asl.Value.String;
+        }
+    }
+    else if (OpInfo->Flags & AML_CREATE)
+    {
+        /* Name must appear as the last parameter */
+
+        NextOp = Op->Asl.Child;
+        while (!(NextOp->Asl.CompileFlags & NODE_IS_NAME_DECLARATION))
+        {
+            NextOp = NextOp->Asl.Next;
+        }
+        Path = NextOp->Asl.Value.String;
+    }
+    else
+    {
+        Path = Op->Asl.Value.String;
+    }
+
+    ObjectType = AslMapNamedOpcodeToDataType (Op->Asl.AmlOpcode);
+    ACPI_DEBUG_PRINT ((ACPI_DB_DISPATCH,
+        "Type=%s\n", AcpiUtGetTypeName (ObjectType)));
+
+    /*
+     * Lookup the name in the namespace. Name must exist at this point, or it
+     * is an invalid reference.
+     *
+     * The namespace is also used as a lookup table for references to resource
+     * descriptors and the fields within them.
+     */
+    Gbl_NsLookupCount++;
+
+    Status = AcpiNsLookup (WalkState->ScopeInfo, Path, ObjectType,
+                ACPI_IMODE_EXECUTE, Flags, WalkState, &(Node));
+    if (ACPI_FAILURE (Status))
+    {
+        if (Status == AE_NOT_FOUND)
+        {
+            /*
+             * We didn't find the name reference by path -- we can qualify this
+             * a little better before we print an error message
+             */
+            if (strlen (Path) == ACPI_NAME_SIZE)
+            {
+                /* A simple, one-segment ACPI name */
+
+                if (XfObjectExists (Path))
+                {
+                    /*
+                     * There exists such a name, but we couldn't get to it
+                     * from this scope
+                     */
+                    AslError (ASL_ERROR, ASL_MSG_NOT_REACHABLE, Op,
+                        Op->Asl.ExternalName);
+                }
+                else
+                {
+                    /* The name doesn't exist, period */
+
+                    AslError (ASL_ERROR, ASL_MSG_NOT_EXIST,
+                        Op, Op->Asl.ExternalName);
+                }
+            }
+            else
+            {
+                /* Check for a fully qualified path */
+
+                if (Path[0] == AML_ROOT_PREFIX)
+                {
+                    /* Gave full path, the object does not exist */
+
+                    AslError (ASL_ERROR, ASL_MSG_NOT_EXIST, Op,
+                        Op->Asl.ExternalName);
+                }
+                else
+                {
+                    /*
+                     * We can't tell whether it doesn't exist or just
+                     * can't be reached.
+                     */
+                    AslError (ASL_ERROR, ASL_MSG_NOT_FOUND, Op,
+                        Op->Asl.ExternalName);
+                }
+            }
+
+            Status = AE_OK;
+        }
+        return (Status);
+    }
+
+    /* Check for a reference vs. name declaration */
+
+    if (!(OpInfo->Flags & AML_NAMED) &&
+        !(OpInfo->Flags & AML_CREATE))
+    {
+        /* This node has been referenced, mark it for reference check */
+
+        Node->Flags |= ANOBJ_IS_REFERENCED;
+    }
+
+    /* Attempt to optimize the NamePath */
+
+    OptOptimizeNamePath (Op, OpInfo->Flags, WalkState, Path, Node);
+
+    /*
+     * 1) Dereference an alias (A name reference that is an alias)
+     *    Aliases are not nested, the alias always points to the final object
+     */
+    if ((Op->Asl.ParseOpcode != PARSEOP_ALIAS) &&
+        (Node->Type == ACPI_TYPE_LOCAL_ALIAS))
+    {
+        /* This node points back to the original PARSEOP_ALIAS */
+
+        NextOp = Node->Op;
+
+        /* The first child is the alias target op */
+
+        NextOp = NextOp->Asl.Child;
+
+        /* That in turn points back to original target alias node */
+
+        if (NextOp->Asl.Node)
+        {
+            Node = NextOp->Asl.Node;
+        }
+
+        /* Else - forward reference to alias, will be resolved later */
+    }
+
+    /* 2) Check for a reference to a resource descriptor */
+
+    if ((Node->Type == ACPI_TYPE_LOCAL_RESOURCE_FIELD) ||
+        (Node->Type == ACPI_TYPE_LOCAL_RESOURCE))
+    {
+        /*
+         * This was a reference to a field within a resource descriptor.
+         * Extract the associated field offset (either a bit or byte
+         * offset depending on the field type) and change the named
+         * reference into an integer for AML code generation
+         */
+        Offset = Node->Value;
+        TagBitLength = Node->Length;
+
+        /*
+         * If a field is being created, generate the length (in bits) of
+         * the field. Note: Opcodes other than CreateXxxField and Index
+         * can come through here. For other opcodes, we just need to
+         * convert the resource tag reference to an integer offset.
+         */
+        switch (Op->Asl.Parent->Asl.AmlOpcode)
+        {
+        case AML_CREATE_FIELD_OP: /* Variable "Length" field, in bits */
+            /*
+             * We know the length operand is an integer constant because
+             * we know that it contains a reference to a resource
+             * descriptor tag.
+             */
+            FieldBitLength = (UINT32) Op->Asl.Next->Asl.Value.Integer;
+            break;
+
+        case AML_CREATE_BIT_FIELD_OP:
+            FieldBitLength = 1;
+            break;
+
+        case AML_CREATE_BYTE_FIELD_OP:
+        case AML_INDEX_OP:
+            FieldBitLength = 8;
+            break;
+
+        case AML_CREATE_WORD_FIELD_OP:
+            FieldBitLength = 16;
+            break;
+
+        case AML_CREATE_DWORD_FIELD_OP:
+            FieldBitLength = 32;
+            break;
+
+        case AML_CREATE_QWORD_FIELD_OP:
+            FieldBitLength = 64;
+            break;
+
+        default:
+            FieldBitLength = 0;
+            break;
+        }
+
+        /* Check the field length against the length of the resource tag */
+
+        if (FieldBitLength)
+        {
+            if (TagBitLength < FieldBitLength)
+            {
+                Message = ASL_MSG_TAG_SMALLER;
+            }
+            else if (TagBitLength > FieldBitLength)
+            {
+                Message = ASL_MSG_TAG_LARGER;
+            }
+
+            if (Message)
+            {
+                sprintf (MsgBuffer, "Size mismatch, Tag: %u bit%s, Field: %u bit%s",
+                    TagBitLength, (TagBitLength > 1) ? "s" : "",
+                    FieldBitLength, (FieldBitLength > 1) ? "s" : "");
+
+                AslError (ASL_WARNING, Message, Op, MsgBuffer);
+            }
+        }
+
+        /* Convert the BitOffset to a ByteOffset for certain opcodes */
+
+        switch (Op->Asl.Parent->Asl.AmlOpcode)
+        {
+        case AML_CREATE_BYTE_FIELD_OP:
+        case AML_CREATE_WORD_FIELD_OP:
+        case AML_CREATE_DWORD_FIELD_OP:
+        case AML_CREATE_QWORD_FIELD_OP:
+        case AML_INDEX_OP:
+
+            Offset = ACPI_DIV_8 (Offset);
+            break;
+
+        default:
+            break;
+        }
+
+        /* Now convert this node to an integer whose value is the field offset */
+
+        Op->Asl.AmlLength = 0;
+        Op->Asl.ParseOpcode = PARSEOP_INTEGER;
+        Op->Asl.Value.Integer = (UINT64) Offset;
+        Op->Asl.CompileFlags |= NODE_IS_RESOURCE_FIELD;
+
+        OpcGenerateAmlOpcode (Op);
+    }
+
+    /* 3) Check for a method invocation */
+
+    else if ((((Op->Asl.ParseOpcode == PARSEOP_NAMESTRING) || (Op->Asl.ParseOpcode == PARSEOP_NAMESEG)) &&
+                (Node->Type == ACPI_TYPE_METHOD) &&
+                (Op->Asl.Parent) &&
+                (Op->Asl.Parent->Asl.ParseOpcode != PARSEOP_METHOD))   ||
+
+                (Op->Asl.ParseOpcode == PARSEOP_METHODCALL))
+    {
+
+        /*
+         * A reference to a method within one of these opcodes is not an
+         * invocation of the method, it is simply a reference to the method.
+         */
+        if ((Op->Asl.Parent) &&
+           ((Op->Asl.Parent->Asl.ParseOpcode == PARSEOP_REFOF)      ||
+            (Op->Asl.Parent->Asl.ParseOpcode == PARSEOP_DEREFOF)    ||
+            (Op->Asl.Parent->Asl.ParseOpcode == PARSEOP_OBJECTTYPE)))
+        {
+            return (AE_OK);
+        }
+        /*
+         * There are two types of method invocation:
+         * 1) Invocation with arguments -- the parser recognizes this
+         *    as a METHODCALL.
+         * 2) Invocation with no arguments --the parser cannot determine that
+         *    this is a method invocation, therefore we have to figure it out
+         *    here.
+         */
+        if (Node->Type != ACPI_TYPE_METHOD)
+        {
+            sprintf (MsgBuffer, "%s is a %s",
+                    Op->Asl.ExternalName, AcpiUtGetTypeName (Node->Type));
+
+            AslError (ASL_ERROR, ASL_MSG_NOT_METHOD, Op, MsgBuffer);
+            return (AE_OK);
+        }
+
+        /* Save the method node in the caller's op */
+
+        Op->Asl.Node = Node;
+        if (Op->Asl.Parent->Asl.ParseOpcode == PARSEOP_CONDREFOF)
+        {
+            return (AE_OK);
+        }
+
+        /*
+         * This is a method invocation, with or without arguments.
+         * Count the number of arguments, each appears as a child
+         * under the parent node
+         */
+        Op->Asl.ParseOpcode = PARSEOP_METHODCALL;
+        UtSetParseOpName (Op);
+
+        PassedArgs = 0;
+        NextOp     = Op->Asl.Child;
+
+        while (NextOp)
+        {
+            PassedArgs++;
+            NextOp = NextOp->Asl.Next;
+        }
+
+        if (Node->Value != ASL_EXTERNAL_METHOD)
+        {
+            /*
+             * Check the parsed arguments with the number expected by the
+             * method declaration itself
+             */
+            if (PassedArgs != Node->Value)
+            {
+                sprintf (MsgBuffer, "%s requires %u", Op->Asl.ExternalName,
+                            Node->Value);
+
+                if (PassedArgs < Node->Value)
+                {
+                    AslError (ASL_ERROR, ASL_MSG_ARG_COUNT_LO, Op, MsgBuffer);
+                }
+                else
+                {
+                    AslError (ASL_ERROR, ASL_MSG_ARG_COUNT_HI, Op, MsgBuffer);
+                }
+            }
+        }
+    }
+
+    /* 4) Check for an ASL Field definition */
+
+    else if ((Op->Asl.Parent) &&
+            ((Op->Asl.Parent->Asl.ParseOpcode == PARSEOP_FIELD)     ||
+             (Op->Asl.Parent->Asl.ParseOpcode == PARSEOP_BANKFIELD)))
+    {
+        /*
+         * Offset checking for fields. If the parent operation region has a
+         * constant length (known at compile time), we can check fields
+         * defined in that region against the region length. This will catch
+         * fields and field units that cannot possibly fit within the region.
+         *
+         * Note: Index fields do not directly reference an operation region,
+         * thus they are not included in this check.
+         */
+        if (Op == Op->Asl.Parent->Asl.Child)
+        {
+            /*
+             * This is the first child of the field node, which is
+             * the name of the region. Get the parse node for the
+             * region -- which contains the length of the region.
+             */
+            OwningOp = Node->Op;
+            Op->Asl.Parent->Asl.ExtraValue =
+                ACPI_MUL_8 ((UINT32) OwningOp->Asl.Value.Integer);
+
+            /* Examine the field access width */
+
+            switch ((UINT8) Op->Asl.Parent->Asl.Value.Integer)
+            {
+            case AML_FIELD_ACCESS_ANY:
+            case AML_FIELD_ACCESS_BYTE:
+            case AML_FIELD_ACCESS_BUFFER:
+            default:
+                MinimumLength = 1;
+                break;
+
+            case AML_FIELD_ACCESS_WORD:
+                MinimumLength = 2;
+                break;
+
+            case AML_FIELD_ACCESS_DWORD:
+                MinimumLength = 4;
+                break;
+
+            case AML_FIELD_ACCESS_QWORD:
+                MinimumLength = 8;
+                break;
+            }
+
+            /*
+             * Is the region at least as big as the access width?
+             * Note: DataTableRegions have 0 length
+             */
+            if (((UINT32) OwningOp->Asl.Value.Integer) &&
+                ((UINT32) OwningOp->Asl.Value.Integer < MinimumLength))
+            {
+                AslError (ASL_ERROR, ASL_MSG_FIELD_ACCESS_WIDTH, Op, NULL);
+            }
+
+            /*
+             * Check EC/CMOS/SMBUS fields to make sure that the correct
+             * access type is used (BYTE for EC/CMOS, BUFFER for SMBUS)
+             */
+            SpaceIdOp = OwningOp->Asl.Child->Asl.Next;
+            switch ((UINT32) SpaceIdOp->Asl.Value.Integer)
+            {
+            case ACPI_ADR_SPACE_EC:
+            case ACPI_ADR_SPACE_CMOS:
+            case ACPI_ADR_SPACE_GPIO:
+
+                if ((UINT8) Op->Asl.Parent->Asl.Value.Integer != AML_FIELD_ACCESS_BYTE)
+                {
+                    AslError (ASL_ERROR, ASL_MSG_REGION_BYTE_ACCESS, Op, NULL);
+                }
+                break;
+
+            case ACPI_ADR_SPACE_SMBUS:
+            case ACPI_ADR_SPACE_IPMI:
+            case ACPI_ADR_SPACE_GSBUS:
+
+                if ((UINT8) Op->Asl.Parent->Asl.Value.Integer != AML_FIELD_ACCESS_BUFFER)
+                {
+                    AslError (ASL_ERROR, ASL_MSG_REGION_BUFFER_ACCESS, Op, NULL);
+                }
+                break;
+
+            default:
+
+                /* Nothing to do for other address spaces */
+                break;
+            }
+        }
+        else
+        {
+            /*
+             * This is one element of the field list. Check to make sure
+             * that it does not go beyond the end of the parent operation region.
+             *
+             * In the code below:
+             *    Op->Asl.Parent->Asl.ExtraValue      - Region Length (bits)
+             *    Op->Asl.ExtraValue                  - Field start offset (bits)
+             *    Op->Asl.Child->Asl.Value.Integer32  - Field length (bits)
+             *    Op->Asl.Child->Asl.ExtraValue       - Field access width (bits)
+             */
+            if (Op->Asl.Parent->Asl.ExtraValue && Op->Asl.Child)
+            {
+                XfCheckFieldRange (Op,
+                            Op->Asl.Parent->Asl.ExtraValue,
+                            Op->Asl.ExtraValue,
+                            (UINT32) Op->Asl.Child->Asl.Value.Integer,
+                            Op->Asl.Child->Asl.ExtraValue);
+            }
+        }
+    }
+
+    Op->Asl.Node = Node;
+    return (Status);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    XfNamespaceLocateEnd
+ *
+ * PARAMETERS:  ASL_WALK_CALLBACK
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Ascending callback used during cross reference. We only
+ *              need to worry about scope management here.
+ *
+ ******************************************************************************/
+
+static ACPI_STATUS
+XfNamespaceLocateEnd (
+    ACPI_PARSE_OBJECT       *Op,
+    UINT32                  Level,
+    void                    *Context)
+{
+    ACPI_WALK_STATE         *WalkState = (ACPI_WALK_STATE *) Context;
+    const ACPI_OPCODE_INFO  *OpInfo;
+
+
+    ACPI_FUNCTION_TRACE (XfNamespaceLocateEnd);
+
+
+    /* We are only interested in opcodes that have an associated name */
+
+    OpInfo = AcpiPsGetOpcodeInfo (Op->Asl.AmlOpcode);
+    if (!(OpInfo->Flags & AML_NAMED))
+    {
+        return (AE_OK);
+    }
+
+    /* Not interested in name references, we did not open a scope for them */
+
+    if ((Op->Asl.ParseOpcode == PARSEOP_NAMESTRING) ||
+        (Op->Asl.ParseOpcode == PARSEOP_NAMESEG)    ||
+        (Op->Asl.ParseOpcode == PARSEOP_METHODCALL))
+    {
+        return (AE_OK);
+    }
+
+    /* Pop the scope stack if necessary */
+
+    if (AcpiNsOpensScope (AslMapNamedOpcodeToDataType (Op->Asl.AmlOpcode)))
+    {
+
+        ACPI_DEBUG_PRINT ((ACPI_DB_DISPATCH,
+            "%s: Popping scope for Op %p\n",
+            AcpiUtGetTypeName (OpInfo->ObjectType), Op));
+
+        (void) AcpiDsScopeStackPop (WalkState);
+    }
+
+    return (AE_OK);
+}
diff --git a/sys/contrib/dev/acpica/compiler/dtcompile.c b/sys/contrib/dev/acpica/compiler/dtcompile.c
index 210b70c..1b24af26 100644
--- a/sys/contrib/dev/acpica/compiler/dtcompile.c
+++ b/sys/contrib/dev/acpica/compiler/dtcompile.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -165,7 +165,7 @@ DtDoCompile (
     /* Write the binary, then the optional hex file */
 
     DtOutputBinary (Gbl_RootTable);
-    LsDoHexOutput ();
+    HxDoHexOutput ();
     DtWriteTableToListing ();
 
 CleanupAndExit:
@@ -284,6 +284,7 @@ DtCompileDataTable (
     char                    *Signature;
     ACPI_TABLE_HEADER       *AcpiTableHeader;
     ACPI_STATUS             Status;
+    DT_FIELD                *RootField = *FieldList;
 
 
     /* Verify that we at least have a table signature and save it */
@@ -354,7 +355,7 @@ DtCompileDataTable (
     if (!TableData || Gbl_CompileGeneric)
     {
         DtCompileGeneric ((void **) FieldList);
-        goto Out;
+        goto FinishHeader;
     }
 
     /* Dispatch to per-table compile */
@@ -391,7 +392,8 @@ DtCompileDataTable (
         return (AE_ERROR);
     }
 
-Out:
+FinishHeader:
+
     /* Set the final table length and then the checksum */
 
     DtSetTableLength ();
@@ -399,6 +401,8 @@ Out:
         ACPI_TABLE_HEADER, Gbl_RootTable->Buffer);
     DtSetTableChecksum (&AcpiTableHeader->Checksum);
 
+    DtDumpFieldList (RootField);
+    DtDumpSubtableList ();
     return (AE_OK);
 }
 
diff --git a/sys/contrib/dev/acpica/compiler/dtcompiler.h b/sys/contrib/dev/acpica/compiler/dtcompiler.h
index 22d1805..e9bdf05 100644
--- a/sys/contrib/dev/acpica/compiler/dtcompiler.h
+++ b/sys/contrib/dev/acpica/compiler/dtcompiler.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -113,6 +113,7 @@ typedef struct dt_subtable
     UINT32                  Length;
     UINT32                  TotalLength;
     UINT32                  SizeOfLengthField;
+    UINT16                  Depth;
     UINT8                   Flags;
 
 } DT_SUBTABLE;
@@ -168,6 +169,14 @@ DtOutputBinary (
     DT_SUBTABLE             *RootTable);
 
 void
+DtDumpSubtableList (
+    void);
+
+void
+DtDumpFieldList (
+    DT_FIELD                *Field);
+
+void
 DtWriteFieldToListing (
     UINT8                   *Buffer,
     DT_FIELD                *Field,
@@ -387,6 +396,10 @@ DtCompileCpep (
     void                    **PFieldList);
 
 ACPI_STATUS
+DtCompileCsrt (
+    void                    **PFieldList);
+
+ACPI_STATUS
 DtCompileDmar (
     void                    **PFieldList);
 
@@ -481,6 +494,7 @@ extern const unsigned char  TemplateBoot[];
 extern const unsigned char  TemplateBert[];
 extern const unsigned char  TemplateBgrt[];
 extern const unsigned char  TemplateCpep[];
+extern const unsigned char  TemplateCsrt[];
 extern const unsigned char  TemplateDbgp[];
 extern const unsigned char  TemplateDmar[];
 extern const unsigned char  TemplateEcdt[];
@@ -507,6 +521,7 @@ extern const unsigned char  TemplateSpcr[];
 extern const unsigned char  TemplateSpmi[];
 extern const unsigned char  TemplateSrat[];
 extern const unsigned char  TemplateTcpa[];
+extern const unsigned char  TemplateTpm2[];
 extern const unsigned char  TemplateUefi[];
 extern const unsigned char  TemplateWaet[];
 extern const unsigned char  TemplateWdat[];
diff --git a/sys/contrib/dev/acpica/compiler/dtexpress.c b/sys/contrib/dev/acpica/compiler/dtexpress.c
index f0a1c8e..1c6ed3f 100644
--- a/sys/contrib/dev/acpica/compiler/dtexpress.c
+++ b/sys/contrib/dev/acpica/compiler/dtexpress.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/dtfield.c b/sys/contrib/dev/acpica/compiler/dtfield.c
index 6bfa649..11e443e 100644
--- a/sys/contrib/dev/acpica/compiler/dtfield.c
+++ b/sys/contrib/dev/acpica/compiler/dtfield.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/dtio.c b/sys/contrib/dev/acpica/compiler/dtio.c
index 5f70a19..f3d5359 100644
--- a/sys/contrib/dev/acpica/compiler/dtio.c
+++ b/sys/contrib/dev/acpica/compiler/dtio.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -79,6 +79,18 @@ DtDumpBuffer (
     UINT32                  Offset,
     UINT32                  Length);
 
+static void
+DtDumpSubtableInfo (
+    DT_SUBTABLE             *Subtable,
+    void                    *Context,
+    void                    *ReturnValue);
+
+static void
+DtDumpSubtableTree (
+    DT_SUBTABLE             *Subtable,
+    void                    *Context,
+    void                    *ReturnValue);
+
 
 /* States for DtGetNextLine */
 
@@ -708,7 +720,6 @@ DtScanFile (
 {
     ACPI_STATUS             Status;
     UINT32                  Offset;
-    DT_FIELD                *Next;
 
 
     ACPI_FUNCTION_NAME (DtScanFile);
@@ -738,28 +749,7 @@ DtScanFile (
 
     /* Dump the parse tree if debug enabled */
 
-    if (Gbl_DebugFlag)
-    {
-        Next = Gbl_FieldList;
-        DbgPrint (ASL_DEBUG_OUTPUT, "Tree:  %32s %32s %8s %8s %8s %8s %8s %8s\n\n",
-            "Name", "Value", "Line", "ByteOff", "NameCol", "Column", "TableOff", "Flags");
-
-        while (Next)
-        {
-            DbgPrint (ASL_DEBUG_OUTPUT, "Field: %32.32s %32.32s %.8X %.8X %.8X %.8X %.8X %.8X\n",
-                Next->Name,
-                Next->Value,
-                Next->Line,
-                Next->ByteOffset,
-                Next->NameColumn,
-                Next->Column,
-                Next->TableOffset,
-                Next->Flags);
-
-            Next = Next->Next;
-        }
-    }
-
+    DtDumpFieldList (Gbl_FieldList);
     return (Gbl_FieldList);
 }
 
@@ -913,6 +903,123 @@ DtDumpBuffer (
 
 /******************************************************************************
  *
+ * FUNCTION:    DtDumpFieldList
+ *
+ * PARAMETERS:  Field               - Root field
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Dump the entire field list
+ *
+ *****************************************************************************/
+
+void
+DtDumpFieldList (
+    DT_FIELD                *Field)
+{
+
+    if (!Gbl_DebugFlag || !Field)
+    {
+        return;
+    }
+
+    DbgPrint (ASL_DEBUG_OUTPUT,  "\nField List:\n"
+        "LineNo   ByteOff  NameCol  Column   TableOff "
+        "Flags    %32s : %s\n\n", "Name", "Value");
+    while (Field)
+    {
+        DbgPrint (ASL_DEBUG_OUTPUT,
+            "%.08X %.08X %.08X %.08X %.08X %.08X %32s : %s\n",
+            Field->Line, Field->ByteOffset, Field->NameColumn,
+            Field->Column, Field->TableOffset, Field->Flags,
+            Field->Name, Field->Value);
+
+        Field = Field->Next;
+    }
+
+    DbgPrint (ASL_DEBUG_OUTPUT,  "\n\n");
+}
+
+
+/******************************************************************************
+ *
+ * FUNCTION:    DtDumpSubtableInfo, DtDumpSubtableTree
+ *
+ * PARAMETERS:  DT_WALK_CALLBACK
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Info - dump a subtable tree entry with extra information.
+ *              Tree - dump a subtable tree formatted by depth indentation.
+ *
+ *****************************************************************************/
+
+static void
+DtDumpSubtableInfo (
+    DT_SUBTABLE             *Subtable,
+    void                    *Context,
+    void                    *ReturnValue)
+{
+
+    DbgPrint (ASL_DEBUG_OUTPUT,
+        "[%.04X] %.08X %.08X %.08X %.08X %.08X %p %p %p\n",
+        Subtable->Depth, Subtable->Length, Subtable->TotalLength,
+        Subtable->SizeOfLengthField, Subtable->Flags, Subtable,
+        Subtable->Parent, Subtable->Child, Subtable->Peer);
+}
+
+static void
+DtDumpSubtableTree (
+    DT_SUBTABLE             *Subtable,
+    void                    *Context,
+    void                    *ReturnValue)
+{
+
+    DbgPrint (ASL_DEBUG_OUTPUT,
+        "[%.04X] %*s%08X (%.02X) - (%.02X)\n",
+        Subtable->Depth, (4 * Subtable->Depth), " ",
+        Subtable, Subtable->Length, Subtable->TotalLength);
+}
+
+
+/******************************************************************************
+ *
+ * FUNCTION:    DtDumpSubtableList
+ *
+ * PARAMETERS:  None
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Dump the raw list of subtables with information, and also
+ *              dump the subtable list in formatted tree format. Assists with
+ *              the development of new table code.
+ *
+ *****************************************************************************/
+
+void
+DtDumpSubtableList (
+    void)
+{
+
+    if (!Gbl_DebugFlag || !Gbl_RootTable)
+    {
+        return;
+    }
+
+    DbgPrint (ASL_DEBUG_OUTPUT,
+        "Subtable Info:\n"
+        "Depth  Length   TotalLen LenSize  Flags    "
+        "This     Parent   Child    Peer\n\n");
+    DtWalkTableTree (Gbl_RootTable, DtDumpSubtableInfo, NULL, NULL);
+
+    DbgPrint (ASL_DEBUG_OUTPUT,
+        "\nSubtable Tree: (Depth, Subtable, Length, TotalLength)\n\n");
+    DtWalkTableTree (Gbl_RootTable, DtDumpSubtableTree, NULL, NULL);
+}
+
+
+/******************************************************************************
+ *
  * FUNCTION:    DtWriteFieldToListing
  *
  * PARAMETERS:  Buffer              - Contains the compiled data
diff --git a/sys/contrib/dev/acpica/compiler/dtparser.l b/sys/contrib/dev/acpica/compiler/dtparser.l
index b6fd6ff..b0f6d6c 100644
--- a/sys/contrib/dev/acpica/compiler/dtparser.l
+++ b/sys/contrib/dev/acpica/compiler/dtparser.l
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/dtparser.y b/sys/contrib/dev/acpica/compiler/dtparser.y
index cf9fb98..47e74bd 100644
--- a/sys/contrib/dev/acpica/compiler/dtparser.y
+++ b/sys/contrib/dev/acpica/compiler/dtparser.y
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/dtsubtable.c b/sys/contrib/dev/acpica/compiler/dtsubtable.c
index 9eebe83..df22710 100644
--- a/sys/contrib/dev/acpica/compiler/dtsubtable.c
+++ b/sys/contrib/dev/acpica/compiler/dtsubtable.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -112,6 +112,7 @@ DtInsertSubtable (
 
     Subtable->Peer = NULL;
     Subtable->Parent = ParentTable;
+    Subtable->Depth = ParentTable->Depth + 1;
 
     /* Link the new entry into the child list */
 
diff --git a/sys/contrib/dev/acpica/compiler/dttable.c b/sys/contrib/dev/acpica/compiler/dttable.c
index 8b58b76..1a8fd81 100644
--- a/sys/contrib/dev/acpica/compiler/dttable.c
+++ b/sys/contrib/dev/acpica/compiler/dttable.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -420,6 +420,94 @@ DtCompileCpep (
 
 /******************************************************************************
  *
+ * FUNCTION:    DtCompileCsrt
+ *
+ * PARAMETERS:  List                - Current field list pointer
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Compile CSRT.
+ *
+ *****************************************************************************/
+
+ACPI_STATUS
+DtCompileCsrt (
+    void                    **List)
+{
+    ACPI_STATUS             Status = AE_OK;
+    DT_SUBTABLE             *Subtable;
+    DT_SUBTABLE             *ParentTable;
+    DT_FIELD                **PFieldList = (DT_FIELD **) List;
+    UINT32                  DescriptorCount;
+    UINT32                  GroupLength;
+
+
+    /* Sub-tables (Resource Groups) */
+
+    while (*PFieldList)
+    {
+        /* Resource group subtable */
+
+        Status = DtCompileTable (PFieldList, AcpiDmTableInfoCsrt0,
+                    &Subtable, TRUE);
+        if (ACPI_FAILURE (Status))
+        {
+            return (Status);
+        }
+
+        /* Compute the number of resource descriptors */
+
+        GroupLength =
+            (ACPI_CAST_PTR (ACPI_CSRT_GROUP,
+                Subtable->Buffer))->Length -
+            (ACPI_CAST_PTR (ACPI_CSRT_GROUP,
+                Subtable->Buffer))->SharedInfoLength -
+            sizeof (ACPI_CSRT_GROUP);
+
+        DescriptorCount = (GroupLength  /
+            sizeof (ACPI_CSRT_DESCRIPTOR));
+
+        ParentTable = DtPeekSubtable ();
+        DtInsertSubtable (ParentTable, Subtable);
+        DtPushSubtable (Subtable);
+
+        /* Shared info subtable (One per resource group) */
+
+        Status = DtCompileTable (PFieldList, AcpiDmTableInfoCsrt1,
+                    &Subtable, TRUE);
+        if (ACPI_FAILURE (Status))
+        {
+            return (Status);
+        }
+
+        ParentTable = DtPeekSubtable ();
+        DtInsertSubtable (ParentTable, Subtable);
+
+        /* Sub-Subtables (Resource Descriptors) */
+
+        while (*PFieldList && DescriptorCount)
+        {
+            Status = DtCompileTable (PFieldList, AcpiDmTableInfoCsrt2,
+                        &Subtable, TRUE);
+            if (ACPI_FAILURE (Status))
+            {
+                return (Status);
+            }
+
+            ParentTable = DtPeekSubtable ();
+            DtInsertSubtable (ParentTable, Subtable);
+            DescriptorCount--;
+        }
+
+        DtPopSubtable ();
+    }
+
+    return (Status);
+}
+
+
+/******************************************************************************
+ *
  * FUNCTION:    DtCompileDmar
  *
  * PARAMETERS:  List                - Current field list pointer
diff --git a/sys/contrib/dev/acpica/compiler/dttemplate.c b/sys/contrib/dev/acpica/compiler/dttemplate.c
index fda712b..c0ff98b 100644
--- a/sys/contrib/dev/acpica/compiler/dttemplate.c
+++ b/sys/contrib/dev/acpica/compiler/dttemplate.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/dttemplate.h b/sys/contrib/dev/acpica/compiler/dttemplate.h
index 1a817cd..07d1f91 100644
--- a/sys/contrib/dev/acpica/compiler/dttemplate.h
+++ b/sys/contrib/dev/acpica/compiler/dttemplate.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -128,6 +128,52 @@ const unsigned char TemplateCpep[] =
     0x00,0x00,0x00,0x00                       /* 00000030    "...."     */
 };
 
+const unsigned char TemplateCsrt[] =
+{
+    0x43,0x53,0x52,0x54,0x4C,0x01,0x00,0x00,  /* 00000000    "CSRTL..." */
+    0x01,0x0D,0x49,0x4E,0x54,0x45,0x4C,0x20,  /* 00000008    "..INTEL " */
+    0x54,0x45,0x4D,0x50,0x4C,0x41,0x54,0x45,  /* 00000010    "TEMPLATE" */
+    0x01,0x00,0x00,0x00,0x49,0x4E,0x54,0x4C,  /* 00000018    "....INTL" */
+    0x14,0x11,0x12,0x20,0x88,0x00,0x00,0x00,  /* 00000020    "... ...." */
+    0x49,0x4E,0x54,0x4C,0x00,0x00,0x00,0x00,  /* 00000028    "INTL...." */
+    0x60,0x9C,0x00,0x00,0x02,0x00,0x00,0x00,  /* 00000030    "`......." */
+    0x1C,0x00,0x00,0x00,0x01,0x00,0x00,0x00,  /* 00000038    "........" */
+    0x00,0x00,0xA0,0xB3,0x00,0x00,0x00,0x00,  /* 00000040    "........" */
+    0x2A,0x00,0x00,0x00,0x02,0x00,0x06,0x20,  /* 00000048    "*...... " */
+    0x00,0x00,0x10,0x00,0xFF,0x0F,0x00,0x00,  /* 00000050    "........" */
+    0x0C,0x00,0x00,0x00,0x03,0x00,0x01,0x00,  /* 00000058    "........" */
+    0x53,0x50,0x49,0x20,0x0C,0x00,0x00,0x00,  /* 00000060    "SPI ...." */
+    0x03,0x00,0x00,0x00,0x43,0x48,0x41,0x30,  /* 00000068    "....CHA0" */
+    0x0C,0x00,0x00,0x00,0x03,0x00,0x00,0x00,  /* 00000070    "........" */
+    0x43,0x48,0x41,0x31,0x0C,0x00,0x00,0x00,  /* 00000078    "CHA1...." */
+    0x03,0x00,0x00,0x00,0x43,0x48,0x41,0x32,  /* 00000080    "....CHA2" */
+    0x0C,0x00,0x00,0x00,0x03,0x00,0x00,0x00,  /* 00000088    "........" */
+    0x43,0x48,0x41,0x33,0x0C,0x00,0x00,0x00,  /* 00000090    "CHA3...." */
+    0x03,0x00,0x00,0x00,0x43,0x48,0x41,0x34,  /* 00000098    "....CHA4" */
+    0x0C,0x00,0x00,0x00,0x03,0x00,0x00,0x00,  /* 000000A0    "........" */
+    0x43,0x48,0x41,0x35,0xA0,0x00,0x00,0x00,  /* 000000A8    "CHA5...." */
+    0x49,0x4E,0x54,0x4C,0x00,0x00,0x00,0x00,  /* 000000B0    "INTL...." */
+    0x60,0x9C,0x00,0x00,0x03,0x00,0x00,0x00,  /* 000000B8    "`......." */
+    0x1C,0x00,0x00,0x00,0x01,0x00,0x00,0x00,  /* 000000C0    "........" */
+    0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,  /* 000000C8    "........" */
+    0x2B,0x00,0x00,0x00,0x02,0x00,0x08,0x20,  /* 000000D0    "+...... " */
+    0x10,0x00,0x10,0x00,0xFF,0x0F,0x00,0x00,  /* 000000D8    "........" */
+    0x0C,0x00,0x00,0x00,0x03,0x00,0x01,0x00,  /* 000000E0    "........" */
+    0x49,0x32,0x43,0x20,0x0C,0x00,0x00,0x00,  /* 000000E8    "I2C ...." */
+    0x03,0x00,0x00,0x00,0x43,0x48,0x41,0x30,  /* 000000F0    "....CHA0" */
+    0x0C,0x00,0x00,0x00,0x03,0x00,0x00,0x00,  /* 000000F8    "........" */
+    0x43,0x48,0x41,0x31,0x0C,0x00,0x00,0x00,  /* 00000100    "CHA1...." */
+    0x03,0x00,0x00,0x00,0x43,0x48,0x41,0x32,  /* 00000108    "....CHA2" */
+    0x0C,0x00,0x00,0x00,0x03,0x00,0x00,0x00,  /* 00000110    "........" */
+    0x43,0x48,0x41,0x33,0x0C,0x00,0x00,0x00,  /* 00000118    "CHA3...." */
+    0x03,0x00,0x00,0x00,0x43,0x48,0x41,0x34,  /* 00000120    "....CHA4" */
+    0x0C,0x00,0x00,0x00,0x03,0x00,0x00,0x00,  /* 00000128    "........" */
+    0x43,0x48,0x41,0x35,0x0C,0x00,0x00,0x00,  /* 00000130    "CHA5...." */
+    0x03,0x00,0x00,0x00,0x43,0x48,0x41,0x36,  /* 00000138    "....CHA6" */
+    0x0C,0x00,0x00,0x00,0x03,0x00,0x00,0x00,  /* 00000140    "........" */
+    0x43,0x48,0x41,0x37                       /* 00000148    "CHA7"     */
+};
+
 const unsigned char TemplateDbgp[] =
 {
     0x44,0x42,0x47,0x50,0x34,0x00,0x00,0x00,  /* 00000000    "DBGP4..." */
@@ -825,6 +871,17 @@ const unsigned char TemplateTcpa[] =
     0x00,0x00                                 /* 00000030    ".."       */
 };
 
+const unsigned char TemplateTpm2[] =
+{
+    0x54,0x50,0x4D,0x32,0x34,0x00,0x00,0x00,  /* 00000000    "TPM24..." */
+    0x03,0x42,0x49,0x4E,0x54,0x45,0x4C,0x20,  /* 00000008    ".BINTEL " */
+    0x54,0x45,0x4D,0x50,0x4C,0x41,0x54,0x45,  /* 00000010    "TEMPLATE" */
+    0x01,0x00,0x00,0x00,0x49,0x4E,0x54,0x4C,  /* 00000018    "....INTL" */
+    0x14,0x11,0x12,0x20,0x00,0x00,0x00,0x00,  /* 00000020    "... ...." */
+    0x77,0x66,0x55,0x44,0x33,0x22,0x11,0x00,  /* 00000028    "wfUD3".." */
+    0x01,0x00,0x00,0x00                       /* 00000030    "...."     */
+};
+
 const unsigned char TemplateUefi[] =
 {
     0x55,0x45,0x46,0x49,0x36,0x00,0x00,0x00,  /* 00000000    "UEFI6..." */
diff --git a/sys/contrib/dev/acpica/compiler/dtutils.c b/sys/contrib/dev/acpica/compiler/dtutils.c
index f87fa20..80b0e44 100644
--- a/sys/contrib/dev/acpica/compiler/dtutils.c
+++ b/sys/contrib/dev/acpica/compiler/dtutils.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/preprocess.h b/sys/contrib/dev/acpica/compiler/preprocess.h
index f97434f..6decc00 100644
--- a/sys/contrib/dev/acpica/compiler/preprocess.h
+++ b/sys/contrib/dev/acpica/compiler/preprocess.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/prexpress.c b/sys/contrib/dev/acpica/compiler/prexpress.c
index c0dccd9..c0aa565 100644
--- a/sys/contrib/dev/acpica/compiler/prexpress.c
+++ b/sys/contrib/dev/acpica/compiler/prexpress.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/prmacros.c b/sys/contrib/dev/acpica/compiler/prmacros.c
index 82663a5..d76de8b 100644
--- a/sys/contrib/dev/acpica/compiler/prmacros.c
+++ b/sys/contrib/dev/acpica/compiler/prmacros.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/prparser.l b/sys/contrib/dev/acpica/compiler/prparser.l
index 34daa48..b580d57 100644
--- a/sys/contrib/dev/acpica/compiler/prparser.l
+++ b/sys/contrib/dev/acpica/compiler/prparser.l
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/prparser.y b/sys/contrib/dev/acpica/compiler/prparser.y
index 81cede3..391ad3f 100644
--- a/sys/contrib/dev/acpica/compiler/prparser.y
+++ b/sys/contrib/dev/acpica/compiler/prparser.y
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/prscan.c b/sys/contrib/dev/acpica/compiler/prscan.c
index 233e45a..0d44d6a 100644
--- a/sys/contrib/dev/acpica/compiler/prscan.c
+++ b/sys/contrib/dev/acpica/compiler/prscan.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/compiler/prutils.c b/sys/contrib/dev/acpica/compiler/prutils.c
index c8faeb6..0dbf307 100644
--- a/sys/contrib/dev/acpica/compiler/prutils.c
+++ b/sys/contrib/dev/acpica/compiler/prutils.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/debugger/dbcmds.c b/sys/contrib/dev/acpica/components/debugger/dbcmds.c
index 5120fcb..64c299c 100644
--- a/sys/contrib/dev/acpica/components/debugger/dbcmds.c
+++ b/sys/contrib/dev/acpica/components/debugger/dbcmds.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -82,17 +82,21 @@ AcpiDbDeviceResources (
     void                    *Context,
     void                    **ReturnValue);
 
+static void
+AcpiDbDoOneSleepState (
+    UINT8                   SleepState);
+
 
 /*******************************************************************************
  *
  * FUNCTION:    AcpiDbConvertToNode
  *
- * PARAMETERS:  InString        - String to convert
+ * PARAMETERS:  InString            - String to convert
  *
  * RETURN:      Pointer to a NS node
  *
  * DESCRIPTION: Convert a string to a valid NS pointer. Handles numeric or
- *              alpha strings.
+ *              alphanumeric strings.
  *
  ******************************************************************************/
 
@@ -126,9 +130,9 @@ AcpiDbConvertToNode (
     }
     else
     {
-        /* Alpha argument */
-        /* The parameter is a name string that must be resolved to a
-         * Named obj
+        /*
+         * Alpha argument: The parameter is a name string that must be
+         * resolved to a Namespace object.
          */
         Node = AcpiDbLocalNsLookup (InString);
         if (!Node)
@@ -145,11 +149,12 @@ AcpiDbConvertToNode (
  *
  * FUNCTION:    AcpiDbSleep
  *
- * PARAMETERS:  ObjectArg       - Desired sleep state (0-5)
+ * PARAMETERS:  ObjectArg           - Desired sleep state (0-5). NULL means
+ *                                    invoke all possible sleep states.
  *
  * RETURN:      Status
  *
- * DESCRIPTION: Simulate a sleep/wake sequence
+ * DESCRIPTION: Simulate sleep/wake sequences
  *
  ******************************************************************************/
 
@@ -157,50 +162,124 @@ ACPI_STATUS
 AcpiDbSleep (
     char                    *ObjectArg)
 {
-    ACPI_STATUS             Status;
     UINT8                   SleepState;
+    UINT32                  i;
 
 
     ACPI_FUNCTION_TRACE (AcpiDbSleep);
 
 
+    /* Null input (no arguments) means to invoke all sleep states */
+
+    if (!ObjectArg)
+    {
+        AcpiOsPrintf ("Invoking all possible sleep states, 0-%d\n",
+            ACPI_S_STATES_MAX);
+
+        for (i = 0; i <= ACPI_S_STATES_MAX; i++)
+        {
+            AcpiDbDoOneSleepState ((UINT8) i);
+        }
+
+        return_ACPI_STATUS (AE_OK);
+    }
+
+    /* Convert argument to binary and invoke the sleep state */
+
     SleepState = (UINT8) ACPI_STRTOUL (ObjectArg, NULL, 0);
+    AcpiDbDoOneSleepState (SleepState);
+    return_ACPI_STATUS (AE_OK);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiDbDoOneSleepState
+ *
+ * PARAMETERS:  SleepState          - Desired sleep state (0-5)
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Simulate a sleep/wake sequence
+ *
+ ******************************************************************************/
+
+static void
+AcpiDbDoOneSleepState (
+    UINT8                   SleepState)
+{
+    ACPI_STATUS             Status;
+    UINT8                   SleepTypeA;
+    UINT8                   SleepTypeB;
+
+
+    /* Validate parameter */
+
+    if (SleepState > ACPI_S_STATES_MAX)
+    {
+        AcpiOsPrintf ("Sleep state %d out of range (%d max)\n",
+            SleepState, ACPI_S_STATES_MAX);
+        return;
+    }
+
+    AcpiOsPrintf ("\n---- Invoking sleep state S%d (%s):\n",
+        SleepState, AcpiGbl_SleepStateNames[SleepState]);
+
+    /* Get the values for the sleep type registers (for display only) */
+
+    Status = AcpiGetSleepTypeData (SleepState, &SleepTypeA, &SleepTypeB);
+    if (ACPI_FAILURE (Status))
+    {
+        AcpiOsPrintf ("Could not evaluate [%s] method, %s\n",
+            AcpiGbl_SleepStateNames[SleepState],
+            AcpiFormatException (Status));
+        return;
+    }
 
-    AcpiOsPrintf ("**** Prepare to sleep ****\n");
+    AcpiOsPrintf (
+        "Register values for sleep state S%d: Sleep-A: %.2X, Sleep-B: %.2X\n",
+        SleepState, SleepTypeA, SleepTypeB);
+
+    /* Invoke the various sleep/wake interfaces */
+
+    AcpiOsPrintf ("**** Sleep: Prepare to sleep (S%d) ****\n",
+        SleepState);
     Status = AcpiEnterSleepStatePrep (SleepState);
     if (ACPI_FAILURE (Status))
     {
         goto ErrorExit;
     }
 
-    AcpiOsPrintf ("**** Going to sleep ****\n");
+    AcpiOsPrintf ("**** Sleep: Going to sleep (S%d) ****\n",
+        SleepState);
     Status = AcpiEnterSleepState (SleepState);
     if (ACPI_FAILURE (Status))
     {
         goto ErrorExit;
     }
 
-    AcpiOsPrintf ("**** Prepare to return from sleep ****\n");
+    AcpiOsPrintf ("**** Wake: Prepare to return from sleep (S%d) ****\n",
+        SleepState);
     Status = AcpiLeaveSleepStatePrep (SleepState);
     if (ACPI_FAILURE (Status))
     {
         goto ErrorExit;
     }
 
-    AcpiOsPrintf ("**** Returning from sleep ****\n");
+    AcpiOsPrintf ("**** Wake: Return from sleep (S%d) ****\n",
+        SleepState);
     Status = AcpiLeaveSleepState (SleepState);
     if (ACPI_FAILURE (Status))
     {
         goto ErrorExit;
     }
 
-    return_ACPI_STATUS (Status);
+    return;
 
 
 ErrorExit:
-
-    ACPI_EXCEPTION ((AE_INFO, Status, "During sleep test"));
-    return_ACPI_STATUS (Status);
+    ACPI_EXCEPTION ((AE_INFO, Status, "During invocation of sleep state S%d",
+        SleepState));
 }
 
 
@@ -236,7 +315,7 @@ AcpiDbDisplayLocks (
  *
  * FUNCTION:    AcpiDbDisplayTableInfo
  *
- * PARAMETERS:  TableArg        - String with name of table to be displayed
+ * PARAMETERS:  TableArg            - Name of table to be displayed
  *
  * RETURN:      None
  *
@@ -377,8 +456,8 @@ AcpiDbUnloadAcpiTable (
  *
  * FUNCTION:    AcpiDbSendNotify
  *
- * PARAMETERS:  Name            - Name of ACPI object to send the notify to
- *              Value           - Value of the notify to send.
+ * PARAMETERS:  Name                - Name of ACPI object where to send notify
+ *              Value               - Value of the notify to send.
  *
  * RETURN:      None
  *
@@ -416,7 +495,8 @@ AcpiDbSendNotify (
     }
     else
     {
-        AcpiOsPrintf ("Named object [%4.4s] Type %s, must be Device/Thermal/Processor type\n",
+        AcpiOsPrintf (
+            "Named object [%4.4s] Type %s, must be Device/Thermal/Processor type\n",
             AcpiUtGetNodeName (Node), AcpiUtGetTypeName (Node->Type));
     }
 }
@@ -453,7 +533,6 @@ AcpiDbDisplayInterfaces (
                     ACPI_WAIT_FOREVER);
 
         NextInterface = AcpiGbl_SupportedInterfaces;
-
         while (NextInterface)
         {
             if (!(NextInterface->Flags & ACPI_OSI_INVALID))
@@ -518,7 +597,7 @@ AcpiDbDisplayInterfaces (
  *
  * FUNCTION:    AcpiDbDisplayTemplate
  *
- * PARAMETERS:  BufferArg           - Buffer name or addrss
+ * PARAMETERS:  BufferArg           - Buffer name or address
  *
  * RETURN:      None
  *
@@ -532,7 +611,7 @@ AcpiDbDisplayTemplate (
 {
     ACPI_NAMESPACE_NODE     *Node;
     ACPI_STATUS             Status;
-    ACPI_BUFFER             ReturnObj;
+    ACPI_BUFFER             ReturnBuffer;
 
 
     /* Translate BufferArg to an Named object */
@@ -553,12 +632,12 @@ AcpiDbDisplayTemplate (
         return;
     }
 
-    ReturnObj.Length = ACPI_DEBUG_BUFFER_SIZE;
-    ReturnObj.Pointer = AcpiGbl_DbBuffer;
+    ReturnBuffer.Length = ACPI_DEBUG_BUFFER_SIZE;
+    ReturnBuffer.Pointer = AcpiGbl_DbBuffer;
 
     /* Attempt to convert the raw buffer to a resource list */
 
-    Status = AcpiRsCreateResourceList (Node->Object, &ReturnObj);
+    Status = AcpiRsCreateResourceList (Node->Object, &ReturnBuffer);
 
     AcpiDbSetOutputDestination (ACPI_DB_REDIRECTABLE_OUTPUT);
     AcpiDbgLevel |= ACPI_LV_RESOURCES;
@@ -573,7 +652,7 @@ AcpiDbDisplayTemplate (
     /* Now we can dump the resource list */
 
     AcpiRsDumpResourceList (ACPI_CAST_PTR (ACPI_RESOURCE,
-        ReturnObj.Pointer));
+        ReturnBuffer.Pointer));
 
 DumpBuffer:
     AcpiOsPrintf ("\nRaw data buffer:\n");
@@ -666,7 +745,8 @@ AcpiDmCompareAmlResources (
             {
                 if (Aml1[i] != Aml2[i])
                 {
-                    AcpiOsPrintf ("Mismatch at byte offset %.2X: is %2.2X, should be %2.2X\n",
+                    AcpiOsPrintf (
+                        "Mismatch at byte offset %.2X: is %2.2X, should be %2.2X\n",
                         i, Aml2[i], Aml1[i]);
                 }
             }
@@ -693,8 +773,8 @@ AcpiDmCompareAmlResources (
  *
  * FUNCTION:    AcpiDmTestResourceConversion
  *
- * PARAMETERS:  Node            - Parent device node
- *              Name            - resource method name (_CRS)
+ * PARAMETERS:  Node                - Parent device node
+ *              Name                - resource method name (_CRS)
  *
  * RETURN:      Status
  *
@@ -709,8 +789,8 @@ AcpiDmTestResourceConversion (
     char                    *Name)
 {
     ACPI_STATUS             Status;
-    ACPI_BUFFER             ReturnObj;
-    ACPI_BUFFER             ResourceObj;
+    ACPI_BUFFER             ReturnBuffer;
+    ACPI_BUFFER             ResourceBuffer;
     ACPI_BUFFER             NewAml;
     ACPI_OBJECT             *OriginalAml;
 
@@ -718,12 +798,12 @@ AcpiDmTestResourceConversion (
     AcpiOsPrintf ("Resource Conversion Comparison:\n");
 
     NewAml.Length = ACPI_ALLOCATE_LOCAL_BUFFER;
-    ReturnObj.Length = ACPI_ALLOCATE_LOCAL_BUFFER;
-    ResourceObj.Length = ACPI_ALLOCATE_LOCAL_BUFFER;
+    ReturnBuffer.Length = ACPI_ALLOCATE_LOCAL_BUFFER;
+    ResourceBuffer.Length = ACPI_ALLOCATE_LOCAL_BUFFER;
 
     /* Get the original _CRS AML resource template */
 
-    Status = AcpiEvaluateObject (Node, Name, NULL, &ReturnObj);
+    Status = AcpiEvaluateObject (Node, Name, NULL, &ReturnBuffer);
     if (ACPI_FAILURE (Status))
     {
         AcpiOsPrintf ("Could not obtain %s: %s\n",
@@ -733,7 +813,7 @@ AcpiDmTestResourceConversion (
 
     /* Get the AML resource template, converted to internal resource structs */
 
-    Status = AcpiGetCurrentResources (Node, &ResourceObj);
+    Status = AcpiGetCurrentResources (Node, &ResourceBuffer);
     if (ACPI_FAILURE (Status))
     {
         AcpiOsPrintf ("AcpiGetCurrentResources failed: %s\n",
@@ -743,7 +823,7 @@ AcpiDmTestResourceConversion (
 
     /* Convert internal resource list to external AML resource template */
 
-    Status = AcpiRsCreateAmlResources (ResourceObj.Pointer, &NewAml);
+    Status = AcpiRsCreateAmlResources (ResourceBuffer.Pointer, &NewAml);
     if (ACPI_FAILURE (Status))
     {
         AcpiOsPrintf ("AcpiRsCreateAmlResources failed: %s\n",
@@ -753,7 +833,7 @@ AcpiDmTestResourceConversion (
 
     /* Compare original AML to the newly created AML resource list */
 
-    OriginalAml = ReturnObj.Pointer;
+    OriginalAml = ReturnBuffer.Pointer;
 
     AcpiDmCompareAmlResources (
         OriginalAml->Buffer.Pointer, (ACPI_RSDESC_SIZE) OriginalAml->Buffer.Length,
@@ -763,9 +843,9 @@ AcpiDmTestResourceConversion (
 
     ACPI_FREE (NewAml.Pointer);
 Exit2:
-    ACPI_FREE (ResourceObj.Pointer);
+    ACPI_FREE (ResourceBuffer.Pointer);
 Exit1:
-    ACPI_FREE (ReturnObj.Pointer);
+    ACPI_FREE (ReturnBuffer.Pointer);
     return (Status);
 }
 
@@ -778,7 +858,8 @@ Exit1:
  *
  * RETURN:      Status
  *
- * DESCRIPTION: Simple callback to exercise AcpiWalkResources
+ * DESCRIPTION: Simple callback to exercise AcpiWalkResources and
+ *              AcpiWalkResourceBuffer.
  *
  ******************************************************************************/
 
@@ -817,7 +898,7 @@ AcpiDbDeviceResources (
     ACPI_NAMESPACE_NODE     *PrsNode = NULL;
     ACPI_NAMESPACE_NODE     *AeiNode = NULL;
     char                    *ParentPath;
-    ACPI_BUFFER             ReturnObj;
+    ACPI_BUFFER             ReturnBuffer;
     ACPI_STATUS             Status;
 
 
@@ -843,8 +924,8 @@ AcpiDbDeviceResources (
 
     /* Prepare for a return object of arbitrary size */
 
-    ReturnObj.Pointer = AcpiGbl_DbBuffer;
-    ReturnObj.Length  = ACPI_DEBUG_BUFFER_SIZE;
+    ReturnBuffer.Pointer = AcpiGbl_DbBuffer;
+    ReturnBuffer.Length  = ACPI_DEBUG_BUFFER_SIZE;
 
 
     /* _PRT */
@@ -853,7 +934,7 @@ AcpiDbDeviceResources (
     {
         AcpiOsPrintf ("Evaluating _PRT\n");
 
-        Status = AcpiEvaluateObject (PrtNode, NULL, NULL, &ReturnObj);
+        Status = AcpiEvaluateObject (PrtNode, NULL, NULL, &ReturnBuffer);
         if (ACPI_FAILURE (Status))
         {
             AcpiOsPrintf ("Could not evaluate _PRT: %s\n",
@@ -861,10 +942,10 @@ AcpiDbDeviceResources (
             goto GetCrs;
         }
 
-        ReturnObj.Pointer = AcpiGbl_DbBuffer;
-        ReturnObj.Length  = ACPI_DEBUG_BUFFER_SIZE;
+        ReturnBuffer.Pointer = AcpiGbl_DbBuffer;
+        ReturnBuffer.Length  = ACPI_DEBUG_BUFFER_SIZE;
 
-        Status = AcpiGetIrqRoutingTable (Node, &ReturnObj);
+        Status = AcpiGetIrqRoutingTable (Node, &ReturnBuffer);
         if (ACPI_FAILURE (Status))
         {
             AcpiOsPrintf ("GetIrqRoutingTable failed: %s\n",
@@ -883,10 +964,10 @@ GetCrs:
     {
         AcpiOsPrintf ("Evaluating _CRS\n");
 
-        ReturnObj.Pointer = AcpiGbl_DbBuffer;
-        ReturnObj.Length  = ACPI_DEBUG_BUFFER_SIZE;
+        ReturnBuffer.Pointer = AcpiGbl_DbBuffer;
+        ReturnBuffer.Length  = ACPI_DEBUG_BUFFER_SIZE;
 
-        Status = AcpiEvaluateObject (CrsNode, NULL, NULL, &ReturnObj);
+        Status = AcpiEvaluateObject (CrsNode, NULL, NULL, &ReturnBuffer);
         if (ACPI_FAILURE (Status))
         {
             AcpiOsPrintf ("Could not evaluate _CRS: %s\n",
@@ -894,7 +975,7 @@ GetCrs:
             goto GetPrs;
         }
 
-        /* This code is here to exercise the AcpiWalkResources interface */
+        /* This code exercises the AcpiWalkResources interface */
 
         Status = AcpiWalkResources (Node, METHOD_NAME__CRS,
             AcpiDbResourceCallback, NULL);
@@ -905,12 +986,12 @@ GetCrs:
             goto GetPrs;
         }
 
-        /* Get the _CRS resource list */
+        /* Get the _CRS resource list (test ALLOCATE buffer) */
 
-        ReturnObj.Pointer = AcpiGbl_DbBuffer;
-        ReturnObj.Length  = ACPI_DEBUG_BUFFER_SIZE;
+        ReturnBuffer.Pointer = NULL;
+        ReturnBuffer.Length  = ACPI_ALLOCATE_LOCAL_BUFFER;
 
-        Status = AcpiGetCurrentResources (Node, &ReturnObj);
+        Status = AcpiGetCurrentResources (Node, &ReturnBuffer);
         if (ACPI_FAILURE (Status))
         {
             AcpiOsPrintf ("AcpiGetCurrentResources failed: %s\n",
@@ -918,26 +999,41 @@ GetCrs:
             goto GetPrs;
         }
 
+        /* This code exercises the AcpiWalkResourceBuffer interface */
+
+        Status = AcpiWalkResourceBuffer (&ReturnBuffer,
+            AcpiDbResourceCallback, NULL);
+        if (ACPI_FAILURE (Status))
+        {
+            AcpiOsPrintf ("AcpiWalkResourceBuffer failed: %s\n",
+                AcpiFormatException (Status));
+            goto EndCrs;
+        }
+
         /* Dump the _CRS resource list */
 
         AcpiRsDumpResourceList (ACPI_CAST_PTR (ACPI_RESOURCE,
-            ReturnObj.Pointer));
+            ReturnBuffer.Pointer));
 
         /*
-         * Perform comparison of original AML to newly created AML. This tests both
-         * the AML->Resource conversion and the Resource->Aml conversion.
+         * Perform comparison of original AML to newly created AML. This
+         * tests both the AML->Resource conversion and the Resource->AML
+         * conversion.
          */
-        Status = AcpiDmTestResourceConversion (Node, METHOD_NAME__CRS);
+        (void) AcpiDmTestResourceConversion (Node, METHOD_NAME__CRS);
 
         /* Execute _SRS with the resource list */
 
-        Status = AcpiSetCurrentResources (Node, &ReturnObj);
+        Status = AcpiSetCurrentResources (Node, &ReturnBuffer);
         if (ACPI_FAILURE (Status))
         {
             AcpiOsPrintf ("AcpiSetCurrentResources failed: %s\n",
                 AcpiFormatException (Status));
-            goto GetPrs;
+            goto EndCrs;
         }
+
+EndCrs:
+        ACPI_FREE_BUFFER (ReturnBuffer);
     }
 
 
@@ -948,10 +1044,10 @@ GetPrs:
     {
         AcpiOsPrintf ("Evaluating _PRS\n");
 
-        ReturnObj.Pointer = AcpiGbl_DbBuffer;
-        ReturnObj.Length  = ACPI_DEBUG_BUFFER_SIZE;
+        ReturnBuffer.Pointer = AcpiGbl_DbBuffer;
+        ReturnBuffer.Length  = ACPI_DEBUG_BUFFER_SIZE;
 
-        Status = AcpiEvaluateObject (PrsNode, NULL, NULL, &ReturnObj);
+        Status = AcpiEvaluateObject (PrsNode, NULL, NULL, &ReturnBuffer);
         if (ACPI_FAILURE (Status))
         {
             AcpiOsPrintf ("Could not evaluate _PRS: %s\n",
@@ -959,10 +1055,10 @@ GetPrs:
             goto GetAei;
         }
 
-        ReturnObj.Pointer = AcpiGbl_DbBuffer;
-        ReturnObj.Length  = ACPI_DEBUG_BUFFER_SIZE;
+        ReturnBuffer.Pointer = AcpiGbl_DbBuffer;
+        ReturnBuffer.Length  = ACPI_DEBUG_BUFFER_SIZE;
 
-        Status = AcpiGetPossibleResources (Node, &ReturnObj);
+        Status = AcpiGetPossibleResources (Node, &ReturnBuffer);
         if (ACPI_FAILURE (Status))
         {
             AcpiOsPrintf ("AcpiGetPossibleResources failed: %s\n",
@@ -981,10 +1077,10 @@ GetAei:
     {
         AcpiOsPrintf ("Evaluating _AEI\n");
 
-        ReturnObj.Pointer = AcpiGbl_DbBuffer;
-        ReturnObj.Length  = ACPI_DEBUG_BUFFER_SIZE;
+        ReturnBuffer.Pointer = AcpiGbl_DbBuffer;
+        ReturnBuffer.Length  = ACPI_DEBUG_BUFFER_SIZE;
 
-        Status = AcpiEvaluateObject (AeiNode, NULL, NULL, &ReturnObj);
+        Status = AcpiEvaluateObject (AeiNode, NULL, NULL, &ReturnBuffer);
         if (ACPI_FAILURE (Status))
         {
             AcpiOsPrintf ("Could not evaluate _AEI: %s\n",
@@ -992,10 +1088,10 @@ GetAei:
             goto Cleanup;
         }
 
-        ReturnObj.Pointer = AcpiGbl_DbBuffer;
-        ReturnObj.Length  = ACPI_DEBUG_BUFFER_SIZE;
+        ReturnBuffer.Pointer = AcpiGbl_DbBuffer;
+        ReturnBuffer.Length  = ACPI_DEBUG_BUFFER_SIZE;
 
-        Status = AcpiGetEventResources (Node, &ReturnObj);
+        Status = AcpiGetEventResources (Node, &ReturnBuffer);
         if (ACPI_FAILURE (Status))
         {
             AcpiOsPrintf ("AcpiGetEventResources failed: %s\n",
@@ -1017,8 +1113,9 @@ Cleanup:
  *
  * FUNCTION:    AcpiDbDisplayResources
  *
- * PARAMETERS:  ObjectArg       - String object name or object pointer.
- *                                "*" means "display resources for all devices"
+ * PARAMETERS:  ObjectArg           - String object name or object pointer.
+ *                                    "*" means "display resources for all
+ *                                    devices"
  *
  * RETURN:      None
  *
@@ -1071,13 +1168,13 @@ AcpiDbDisplayResources (
  *
  * FUNCTION:    AcpiDbGenerateGpe
  *
- * PARAMETERS:  GpeArg          - Raw GPE number, ascii string
- *              BlockArg        - GPE block number, ascii string
- *                                0 or 1 for FADT GPE blocks
+ * PARAMETERS:  GpeArg              - Raw GPE number, ascii string
+ *              BlockArg            - GPE block number, ascii string
+ *                                    0 or 1 for FADT GPE blocks
  *
  * RETURN:      None
  *
- * DESCRIPTION: Generate a GPE
+ * DESCRIPTION: Simulate firing of a GPE
  *
  ******************************************************************************/
 
diff --git a/sys/contrib/dev/acpica/components/debugger/dbconvert.c b/sys/contrib/dev/acpica/components/debugger/dbconvert.c
new file mode 100644
index 0000000..f781f29
--- /dev/null
+++ b/sys/contrib/dev/acpica/components/debugger/dbconvert.c
@@ -0,0 +1,525 @@
+/*******************************************************************************
+ *
+ * Module Name: dbconvert - debugger miscellaneous conversion routines
+ *
+ ******************************************************************************/
+
+/*
+ * Copyright (C) 2000 - 2013, Intel Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ * 3. Neither the names of the above-listed copyright holders nor the names
+ *    of any contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ */
+
+
+#include <contrib/dev/acpica/include/acpi.h>
+#include <contrib/dev/acpica/include/accommon.h>
+#include <contrib/dev/acpica/include/acdebug.h>
+
+#ifdef ACPI_DEBUGGER
+
+#define _COMPONENT          ACPI_CA_DEBUGGER
+        ACPI_MODULE_NAME    ("dbconvert")
+
+
+#define DB_DEFAULT_PKG_ELEMENTS     33
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiDbHexCharToValue
+ *
+ * PARAMETERS:  HexChar             - Ascii Hex digit, 0-9|a-f|A-F
+ *              ReturnValue         - Where the converted value is returned
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Convert a single hex character to a 4-bit number (0-16).
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+AcpiDbHexCharToValue (
+    int                     HexChar,
+    UINT8                   *ReturnValue)
+{
+    UINT8                   Value;
+
+
+    /* Digit must be ascii [0-9a-fA-F] */
+
+    if (!ACPI_IS_XDIGIT (HexChar))
+    {
+        return (AE_BAD_HEX_CONSTANT);
+    }
+
+    if (HexChar <= 0x39)
+    {
+        Value = (UINT8) (HexChar - 0x30);
+    }
+    else
+    {
+        Value = (UINT8) (ACPI_TOUPPER (HexChar) - 0x37);
+    }
+
+    *ReturnValue = Value;
+    return (AE_OK);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiDbHexByteToBinary
+ *
+ * PARAMETERS:  HexByte             - Double hex digit (0x00 - 0xFF) in format:
+ *                                    HiByte then LoByte.
+ *              ReturnValue         - Where the converted value is returned
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Convert two hex characters to an 8 bit number (0 - 255).
+ *
+ ******************************************************************************/
+
+static ACPI_STATUS
+AcpiDbHexByteToBinary (
+    char                    *HexByte,
+    UINT8                   *ReturnValue)
+{
+    UINT8                   Local0;
+    UINT8                   Local1;
+    ACPI_STATUS             Status;
+
+
+    /* High byte */
+
+    Status = AcpiDbHexCharToValue (HexByte[0], &Local0);
+    if (ACPI_FAILURE (Status))
+    {
+        return (Status);
+    }
+
+    /* Low byte */
+
+    Status = AcpiDbHexCharToValue (HexByte[1], &Local1);
+    if (ACPI_FAILURE (Status))
+    {
+        return (Status);
+    }
+
+    *ReturnValue = (UINT8) ((Local0 << 4) | Local1);
+    return (AE_OK);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiDbConvertToBuffer
+ *
+ * PARAMETERS:  String              - Input string to be converted
+ *              Object              - Where the buffer object is returned
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Convert a string to a buffer object. String is treated a list
+ *              of buffer elements, each separated by a space or comma.
+ *
+ ******************************************************************************/
+
+static ACPI_STATUS
+AcpiDbConvertToBuffer (
+    char                    *String,
+    ACPI_OBJECT             *Object)
+{
+    UINT32                  i;
+    UINT32                  j;
+    UINT32                  Length;
+    UINT8                   *Buffer;
+    ACPI_STATUS             Status;
+
+
+    /* Generate the final buffer length */
+
+    for (i = 0, Length = 0; String[i];)
+    {
+        i+=2;
+        Length++;
+
+        while (String[i] &&
+              ((String[i] == ',') || (String[i] == ' ')))
+        {
+            i++;
+        }
+    }
+
+    Buffer = ACPI_ALLOCATE (Length);
+    if (!Buffer)
+    {
+        return (AE_NO_MEMORY);
+    }
+
+    /* Convert the command line bytes to the buffer */
+
+    for (i = 0, j = 0; String[i];)
+    {
+        Status = AcpiDbHexByteToBinary (&String[i], &Buffer[j]);
+        if (ACPI_FAILURE (Status))
+        {
+            ACPI_FREE (Buffer);
+            return (Status);
+        }
+
+        j++;
+        i+=2;
+        while (String[i] &&
+              ((String[i] == ',') || (String[i] == ' ')))
+        {
+            i++;
+        }
+    }
+
+    Object->Type = ACPI_TYPE_BUFFER;
+    Object->Buffer.Pointer = Buffer;
+    Object->Buffer.Length = Length;
+    return (AE_OK);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiDbConvertToPackage
+ *
+ * PARAMETERS:  String              - Input string to be converted
+ *              Object              - Where the package object is returned
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Convert a string to a package object. Handles nested packages
+ *              via recursion with AcpiDbConvertToObject.
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+AcpiDbConvertToPackage (
+    char                    *String,
+    ACPI_OBJECT             *Object)
+{
+    char                    *This;
+    char                    *Next;
+    UINT32                  i;
+    ACPI_OBJECT_TYPE        Type;
+    ACPI_OBJECT             *Elements;
+    ACPI_STATUS             Status;
+
+
+    Elements = ACPI_ALLOCATE_ZEROED (
+        DB_DEFAULT_PKG_ELEMENTS * sizeof (ACPI_OBJECT));
+
+    This = String;
+    for (i = 0; i < (DB_DEFAULT_PKG_ELEMENTS - 1); i++)
+    {
+        This = AcpiDbGetNextToken (This, &Next, &Type);
+        if (!This)
+        {
+            break;
+        }
+
+        /* Recursive call to convert each package element */
+
+        Status = AcpiDbConvertToObject (Type, This, &Elements[i]);
+        if (ACPI_FAILURE (Status))
+        {
+            AcpiDbDeleteObjects (i + 1, Elements);
+            ACPI_FREE (Elements);
+            return (Status);
+        }
+
+        This = Next;
+    }
+
+    Object->Type = ACPI_TYPE_PACKAGE;
+    Object->Package.Count = i;
+    Object->Package.Elements = Elements;
+    return (AE_OK);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiDbConvertToObject
+ *
+ * PARAMETERS:  Type                - Object type as determined by parser
+ *              String              - Input string to be converted
+ *              Object              - Where the new object is returned
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Convert a typed and tokenized string to an ACPI_OBJECT. Typing:
+ *              1) String objects were surrounded by quotes.
+ *              2) Buffer objects were surrounded by parentheses.
+ *              3) Package objects were surrounded by brackets "[]".
+ *              4) All standalone tokens are treated as integers.
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+AcpiDbConvertToObject (
+    ACPI_OBJECT_TYPE        Type,
+    char                    *String,
+    ACPI_OBJECT             *Object)
+{
+    ACPI_STATUS             Status = AE_OK;
+
+
+    switch (Type)
+    {
+    case ACPI_TYPE_STRING:
+        Object->Type = ACPI_TYPE_STRING;
+        Object->String.Pointer = String;
+        Object->String.Length = (UINT32) ACPI_STRLEN (String);
+        break;
+
+    case ACPI_TYPE_BUFFER:
+        Status = AcpiDbConvertToBuffer (String, Object);
+        break;
+
+    case ACPI_TYPE_PACKAGE:
+        Status = AcpiDbConvertToPackage (String, Object);
+        break;
+
+    default:
+        Object->Type = ACPI_TYPE_INTEGER;
+        Status = AcpiUtStrtoul64 (String, 16, &Object->Integer.Value);
+        break;
+    }
+
+    return (Status);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiDbEncodePldBuffer
+ *
+ * PARAMETERS:  PldInfo             - _PLD buffer struct (Using local struct)
+ *
+ * RETURN:      Encode _PLD buffer suitable for return value from _PLD
+ *
+ * DESCRIPTION: Bit-packs a _PLD buffer struct. Used to test the _PLD macros
+ *
+ ******************************************************************************/
+
+UINT8 *
+AcpiDbEncodePldBuffer (
+    ACPI_PLD_INFO           *PldInfo)
+{
+    UINT32                  *Buffer;
+    UINT32                  Dword;
+
+
+    Buffer = ACPI_ALLOCATE_ZEROED (ACPI_PLD_BUFFER_SIZE);
+    if (!Buffer)
+    {
+        return (NULL);
+    }
+
+    /* First 32 bits */
+
+    Dword = 0;
+    ACPI_PLD_SET_REVISION       (&Dword, PldInfo->Revision);
+    ACPI_PLD_SET_IGNORE_COLOR   (&Dword, PldInfo->IgnoreColor);
+    ACPI_PLD_SET_COLOR          (&Dword, PldInfo->Color);
+    ACPI_MOVE_32_TO_32 (&Buffer[0], &Dword);
+
+    /* Second 32 bits */
+
+    Dword = 0;
+    ACPI_PLD_SET_WIDTH          (&Dword, PldInfo->Width);
+    ACPI_PLD_SET_HEIGHT         (&Dword, PldInfo->Height);
+    ACPI_MOVE_32_TO_32 (&Buffer[1], &Dword);
+
+    /* Third 32 bits */
+
+    Dword = 0;
+    ACPI_PLD_SET_USER_VISIBLE   (&Dword, PldInfo->UserVisible);
+    ACPI_PLD_SET_DOCK           (&Dword, PldInfo->Dock);
+    ACPI_PLD_SET_LID            (&Dword, PldInfo->Lid);
+    ACPI_PLD_SET_PANEL          (&Dword, PldInfo->Panel);
+    ACPI_PLD_SET_VERTICAL       (&Dword, PldInfo->VerticalPosition);
+    ACPI_PLD_SET_HORIZONTAL     (&Dword, PldInfo->HorizontalPosition);
+    ACPI_PLD_SET_SHAPE          (&Dword, PldInfo->Shape);
+    ACPI_PLD_SET_ORIENTATION    (&Dword, PldInfo->GroupOrientation);
+    ACPI_PLD_SET_TOKEN          (&Dword, PldInfo->GroupToken);
+    ACPI_PLD_SET_POSITION       (&Dword, PldInfo->GroupPosition);
+    ACPI_PLD_SET_BAY            (&Dword, PldInfo->Bay);
+    ACPI_MOVE_32_TO_32 (&Buffer[2], &Dword);
+
+    /* Fourth 32 bits */
+
+    Dword = 0;
+    ACPI_PLD_SET_EJECTABLE      (&Dword, PldInfo->Ejectable);
+    ACPI_PLD_SET_OSPM_EJECT     (&Dword, PldInfo->OspmEjectRequired);
+    ACPI_PLD_SET_CABINET        (&Dword, PldInfo->CabinetNumber);
+    ACPI_PLD_SET_CARD_CAGE      (&Dword, PldInfo->CardCageNumber);
+    ACPI_PLD_SET_REFERENCE      (&Dword, PldInfo->Reference);
+    ACPI_PLD_SET_ROTATION       (&Dword, PldInfo->Rotation);
+    ACPI_PLD_SET_ORDER          (&Dword, PldInfo->Order);
+    ACPI_MOVE_32_TO_32 (&Buffer[3], &Dword);
+
+    if (PldInfo->Revision >= 2)
+    {
+        /* Fifth 32 bits */
+
+        Dword = 0;
+        ACPI_PLD_SET_VERT_OFFSET    (&Dword, PldInfo->VerticalOffset);
+        ACPI_PLD_SET_HORIZ_OFFSET   (&Dword, PldInfo->HorizontalOffset);
+        ACPI_MOVE_32_TO_32 (&Buffer[4], &Dword);
+    }
+
+    return (ACPI_CAST_PTR (UINT8, Buffer));
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiDbDumpPldBuffer
+ *
+ * PARAMETERS:  ObjDesc             - Object returned from _PLD method
+ *
+ * RETURN:      None.
+ *
+ * DESCRIPTION: Dumps formatted contents of a _PLD return buffer.
+ *
+ ******************************************************************************/
+
+#define ACPI_PLD_OUTPUT     "%20s : %-6X\n"
+
+void
+AcpiDbDumpPldBuffer (
+    ACPI_OBJECT             *ObjDesc)
+{
+    ACPI_OBJECT             *BufferDesc;
+    ACPI_PLD_INFO           *PldInfo;
+    UINT8                   *NewBuffer;
+    ACPI_STATUS             Status;
+
+
+    /* Object must be of type Package with at least one Buffer element */
+
+    if (ObjDesc->Type != ACPI_TYPE_PACKAGE)
+    {
+        return;
+    }
+
+    BufferDesc = &ObjDesc->Package.Elements[0];
+    if (BufferDesc->Type != ACPI_TYPE_BUFFER)
+    {
+        return;
+    }
+
+    /* Convert _PLD buffer to local _PLD struct */
+
+    Status = AcpiDecodePldBuffer (BufferDesc->Buffer.Pointer,
+        BufferDesc->Buffer.Length, &PldInfo);
+    if (ACPI_FAILURE (Status))
+    {
+        return;
+    }
+
+    /* Encode local _PLD struct back to a _PLD buffer */
+
+    NewBuffer = AcpiDbEncodePldBuffer (PldInfo);
+    if (!NewBuffer)
+    {
+        return;
+    }
+
+    /* The two bit-packed buffers should match */
+
+    if (ACPI_MEMCMP (NewBuffer, BufferDesc->Buffer.Pointer,
+        BufferDesc->Buffer.Length))
+    {
+        AcpiOsPrintf ("Converted _PLD buffer does not compare. New:\n");
+
+        AcpiUtDumpBuffer (NewBuffer,
+            BufferDesc->Buffer.Length, DB_BYTE_DISPLAY, 0);
+    }
+
+    /* First 32-bit dword */
+
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Revision", PldInfo->Revision);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "IgnoreColor", PldInfo->IgnoreColor);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Color", PldInfo->Color);
+
+    /* Second 32-bit dword */
+
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Width", PldInfo->Width);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Height", PldInfo->Height);
+
+    /* Third 32-bit dword */
+
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "UserVisible", PldInfo->UserVisible);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Dock", PldInfo->Dock);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Lid", PldInfo->Lid);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Panel", PldInfo->Panel);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "VerticalPosition", PldInfo->VerticalPosition);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "HorizontalPosition", PldInfo->HorizontalPosition);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Shape", PldInfo->Shape);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "GroupOrientation", PldInfo->GroupOrientation);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "GroupToken", PldInfo->GroupToken);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "GroupPosition", PldInfo->GroupPosition);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Bay", PldInfo->Bay);
+
+    /* Fourth 32-bit dword */
+
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Ejectable", PldInfo->Ejectable);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "OspmEjectRequired", PldInfo->OspmEjectRequired);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "CabinetNumber", PldInfo->CabinetNumber);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "CardCageNumber", PldInfo->CardCageNumber);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Reference", PldInfo->Reference);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Rotation", PldInfo->Rotation);
+    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Order", PldInfo->Order);
+
+    /* Fifth 32-bit dword */
+
+    if (BufferDesc->Buffer.Length > 16)
+    {
+        AcpiOsPrintf (ACPI_PLD_OUTPUT, "VerticalOffset", PldInfo->VerticalOffset);
+        AcpiOsPrintf (ACPI_PLD_OUTPUT, "HorizontalOffset", PldInfo->HorizontalOffset);
+    }
+
+    ACPI_FREE (PldInfo);
+    ACPI_FREE (NewBuffer);
+}
+
+#endif /* ACPI_DEBUGGER */
diff --git a/sys/contrib/dev/acpica/components/debugger/dbdisply.c b/sys/contrib/dev/acpica/components/debugger/dbdisply.c
index 7776d55..6e82668 100644
--- a/sys/contrib/dev/acpica/components/debugger/dbdisply.c
+++ b/sys/contrib/dev/acpica/components/debugger/dbdisply.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/debugger/dbexec.c b/sys/contrib/dev/acpica/components/debugger/dbexec.c
index f1dce09..26f62b6 100644
--- a/sys/contrib/dev/acpica/components/debugger/dbexec.c
+++ b/sys/contrib/dev/acpica/components/debugger/dbexec.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -54,7 +54,6 @@
 
 
 static ACPI_DB_METHOD_INFO          AcpiGbl_DbMethodInfo;
-#define DB_DEFAULT_PKG_ELEMENTS     33
 
 /* Local prototypes */
 
@@ -82,308 +81,6 @@ AcpiDbExecutionWalk (
     void                    *Context,
     void                    **ReturnValue);
 
-static ACPI_STATUS
-AcpiDbHexCharToValue (
-    int                     HexChar,
-    UINT8                   *ReturnValue);
-
-static ACPI_STATUS
-AcpiDbConvertToPackage (
-    char                    *String,
-    ACPI_OBJECT             *Object);
-
-static ACPI_STATUS
-AcpiDbConvertToObject (
-    ACPI_OBJECT_TYPE        Type,
-    char                    *String,
-    ACPI_OBJECT             *Object);
-
-static void
-AcpiDbDeleteObjects (
-    UINT32                  Count,
-    ACPI_OBJECT             *Objects);
-
-
-static UINT8 *
-AcpiDbEncodePldBuffer (
-    ACPI_PLD_INFO           *PldInfo);
-
-static void
-AcpiDbDumpPldBuffer (
-    ACPI_OBJECT             *ObjDesc);
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiDbHexCharToValue
- *
- * PARAMETERS:  HexChar             - Ascii Hex digit, 0-9|a-f|A-F
- *              ReturnValue         - Where the converted value is returned
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Convert a single hex character to a 4-bit number (0-16).
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-AcpiDbHexCharToValue (
-    int                     HexChar,
-    UINT8                   *ReturnValue)
-{
-    UINT8                   Value;
-
-
-    /* Digit must be ascii [0-9a-fA-F] */
-
-    if (!ACPI_IS_XDIGIT (HexChar))
-    {
-        return (AE_BAD_HEX_CONSTANT);
-    }
-
-    if (HexChar <= 0x39)
-    {
-        Value = (UINT8) (HexChar - 0x30);
-    }
-    else
-    {
-        Value = (UINT8) (ACPI_TOUPPER (HexChar) - 0x37);
-    }
-
-    *ReturnValue = Value;
-    return (AE_OK);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiDbHexByteToBinary
- *
- * PARAMETERS:  HexByte             - Double hex digit (0x00 - 0xFF) in format:
- *                                    HiByte then LoByte.
- *              ReturnValue         - Where the converted value is returned
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Convert two hex characters to an 8 bit number (0 - 255).
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-AcpiDbHexByteToBinary (
-    char                    *HexByte,
-    UINT8                   *ReturnValue)
-{
-    UINT8                   Local0;
-    UINT8                   Local1;
-    ACPI_STATUS             Status;
-
-
-    /* High byte */
-
-    Status = AcpiDbHexCharToValue (HexByte[0], &Local0);
-    if (ACPI_FAILURE (Status))
-    {
-        return (Status);
-    }
-
-    /* Low byte */
-
-    Status = AcpiDbHexCharToValue (HexByte[1], &Local1);
-    if (ACPI_FAILURE (Status))
-    {
-        return (Status);
-    }
-
-    *ReturnValue = (UINT8) ((Local0 << 4) | Local1);
-    return (AE_OK);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiDbConvertToBuffer
- *
- * PARAMETERS:  String              - Input string to be converted
- *              Object              - Where the buffer object is returned
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Convert a string to a buffer object. String is treated a list
- *              of buffer elements, each separated by a space or comma.
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-AcpiDbConvertToBuffer (
-    char                    *String,
-    ACPI_OBJECT             *Object)
-{
-    UINT32                  i;
-    UINT32                  j;
-    UINT32                  Length;
-    UINT8                   *Buffer;
-    ACPI_STATUS             Status;
-
-
-    /* Generate the final buffer length */
-
-    for (i = 0, Length = 0; String[i];)
-    {
-        i+=2;
-        Length++;
-
-        while (String[i] &&
-              ((String[i] == ',') || (String[i] == ' ')))
-        {
-            i++;
-        }
-    }
-
-    Buffer = ACPI_ALLOCATE (Length);
-    if (!Buffer)
-    {
-        return (AE_NO_MEMORY);
-    }
-
-    /* Convert the command line bytes to the buffer */
-
-    for (i = 0, j = 0; String[i];)
-    {
-        Status = AcpiDbHexByteToBinary (&String[i], &Buffer[j]);
-        if (ACPI_FAILURE (Status))
-        {
-            ACPI_FREE (Buffer);
-            return (Status);
-        }
-
-        j++;
-        i+=2;
-        while (String[i] &&
-              ((String[i] == ',') || (String[i] == ' ')))
-        {
-            i++;
-        }
-    }
-
-    Object->Type = ACPI_TYPE_BUFFER;
-    Object->Buffer.Pointer = Buffer;
-    Object->Buffer.Length = Length;
-    return (AE_OK);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiDbConvertToPackage
- *
- * PARAMETERS:  String              - Input string to be converted
- *              Object              - Where the package object is returned
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Convert a string to a package object. Handles nested packages
- *              via recursion with AcpiDbConvertToObject.
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-AcpiDbConvertToPackage (
-    char                    *String,
-    ACPI_OBJECT             *Object)
-{
-    char                    *This;
-    char                    *Next;
-    UINT32                  i;
-    ACPI_OBJECT_TYPE        Type;
-    ACPI_OBJECT             *Elements;
-    ACPI_STATUS             Status;
-
-
-    Elements = ACPI_ALLOCATE_ZEROED (
-        DB_DEFAULT_PKG_ELEMENTS * sizeof (ACPI_OBJECT));
-
-    This = String;
-    for (i = 0; i < (DB_DEFAULT_PKG_ELEMENTS - 1); i++)
-    {
-        This = AcpiDbGetNextToken (This, &Next, &Type);
-        if (!This)
-        {
-            break;
-        }
-
-        /* Recursive call to convert each package element */
-
-        Status = AcpiDbConvertToObject (Type, This, &Elements[i]);
-        if (ACPI_FAILURE (Status))
-        {
-            AcpiDbDeleteObjects (i + 1, Elements);
-            ACPI_FREE (Elements);
-            return (Status);
-        }
-
-        This = Next;
-    }
-
-    Object->Type = ACPI_TYPE_PACKAGE;
-    Object->Package.Count = i;
-    Object->Package.Elements = Elements;
-    return (AE_OK);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiDbConvertToObject
- *
- * PARAMETERS:  Type                - Object type as determined by parser
- *              String              - Input string to be converted
- *              Object              - Where the new object is returned
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Convert a typed and tokenized string to an ACPI_OBJECT. Typing:
- *              1) String objects were surrounded by quotes.
- *              2) Buffer objects were surrounded by parentheses.
- *              3) Package objects were surrounded by brackets "[]".
- *              4) All standalone tokens are treated as integers.
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-AcpiDbConvertToObject (
-    ACPI_OBJECT_TYPE        Type,
-    char                    *String,
-    ACPI_OBJECT             *Object)
-{
-    ACPI_STATUS             Status = AE_OK;
-
-
-    switch (Type)
-    {
-    case ACPI_TYPE_STRING:
-        Object->Type = ACPI_TYPE_STRING;
-        Object->String.Pointer = String;
-        Object->String.Length = (UINT32) ACPI_STRLEN (String);
-        break;
-
-    case ACPI_TYPE_BUFFER:
-        Status = AcpiDbConvertToBuffer (String, Object);
-        break;
-
-    case ACPI_TYPE_PACKAGE:
-        Status = AcpiDbConvertToPackage (String, Object);
-        break;
-
-    default:
-        Object->Type = ACPI_TYPE_INTEGER;
-        Status = AcpiUtStrtoul64 (String, 16, &Object->Integer.Value);
-        break;
-    }
-
-    return (Status);
-}
-
 
 /*******************************************************************************
  *
@@ -399,7 +96,7 @@ AcpiDbConvertToObject (
  *
  ******************************************************************************/
 
-static void
+void
 AcpiDbDeleteObjects (
     UINT32                  Count,
     ACPI_OBJECT             *Objects)
@@ -720,202 +417,6 @@ AcpiDbExecutionWalk (
 
 /*******************************************************************************
  *
- * FUNCTION:    AcpiDbEncodePldBuffer
- *
- * PARAMETERS:  PldInfo             - _PLD buffer struct (Using local struct)
- *
- * RETURN:      Encode _PLD buffer suitable for return value from _PLD
- *
- * DESCRIPTION: Bit-packs a _PLD buffer struct. Used to test the _PLD macros
- *
- ******************************************************************************/
-
-static UINT8 *
-AcpiDbEncodePldBuffer (
-    ACPI_PLD_INFO           *PldInfo)
-{
-    UINT32                  *Buffer;
-    UINT32                  Dword;
-
-
-    Buffer = ACPI_ALLOCATE_ZEROED (ACPI_PLD_BUFFER_SIZE);
-    if (!Buffer)
-    {
-        return (NULL);
-    }
-
-    /* First 32 bits */
-
-    Dword = 0;
-    ACPI_PLD_SET_REVISION       (&Dword, PldInfo->Revision);
-    ACPI_PLD_SET_IGNORE_COLOR   (&Dword, PldInfo->IgnoreColor);
-    ACPI_PLD_SET_COLOR          (&Dword, PldInfo->Color);
-    ACPI_MOVE_32_TO_32 (&Buffer[0], &Dword);
-
-    /* Second 32 bits */
-
-    Dword = 0;
-    ACPI_PLD_SET_WIDTH          (&Dword, PldInfo->Width);
-    ACPI_PLD_SET_HEIGHT         (&Dword, PldInfo->Height);
-    ACPI_MOVE_32_TO_32 (&Buffer[1], &Dword);
-
-    /* Third 32 bits */
-
-    Dword = 0;
-    ACPI_PLD_SET_USER_VISIBLE   (&Dword, PldInfo->UserVisible);
-    ACPI_PLD_SET_DOCK           (&Dword, PldInfo->Dock);
-    ACPI_PLD_SET_LID            (&Dword, PldInfo->Lid);
-    ACPI_PLD_SET_PANEL          (&Dword, PldInfo->Panel);
-    ACPI_PLD_SET_VERTICAL       (&Dword, PldInfo->VerticalPosition);
-    ACPI_PLD_SET_HORIZONTAL     (&Dword, PldInfo->HorizontalPosition);
-    ACPI_PLD_SET_SHAPE          (&Dword, PldInfo->Shape);
-    ACPI_PLD_SET_ORIENTATION    (&Dword, PldInfo->GroupOrientation);
-    ACPI_PLD_SET_TOKEN          (&Dword, PldInfo->GroupToken);
-    ACPI_PLD_SET_POSITION       (&Dword, PldInfo->GroupPosition);
-    ACPI_PLD_SET_BAY            (&Dword, PldInfo->Bay);
-    ACPI_MOVE_32_TO_32 (&Buffer[2], &Dword);
-
-    /* Fourth 32 bits */
-
-    Dword = 0;
-    ACPI_PLD_SET_EJECTABLE      (&Dword, PldInfo->Ejectable);
-    ACPI_PLD_SET_OSPM_EJECT     (&Dword, PldInfo->OspmEjectRequired);
-    ACPI_PLD_SET_CABINET        (&Dword, PldInfo->CabinetNumber);
-    ACPI_PLD_SET_CARD_CAGE      (&Dword, PldInfo->CardCageNumber);
-    ACPI_PLD_SET_REFERENCE      (&Dword, PldInfo->Reference);
-    ACPI_PLD_SET_ROTATION       (&Dword, PldInfo->Rotation);
-    ACPI_PLD_SET_ORDER          (&Dword, PldInfo->Order);
-    ACPI_MOVE_32_TO_32 (&Buffer[3], &Dword);
-
-    if (PldInfo->Revision >= 2)
-    {
-        /* Fifth 32 bits */
-
-        Dword = 0;
-        ACPI_PLD_SET_VERT_OFFSET    (&Dword, PldInfo->VerticalOffset);
-        ACPI_PLD_SET_HORIZ_OFFSET   (&Dword, PldInfo->HorizontalOffset);
-        ACPI_MOVE_32_TO_32 (&Buffer[4], &Dword);
-    }
-
-    return (ACPI_CAST_PTR (UINT8, Buffer));
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiDbDumpPldBuffer
- *
- * PARAMETERS:  ObjDesc             - Object returned from _PLD method
- *
- * RETURN:      None.
- *
- * DESCRIPTION: Dumps formatted contents of a _PLD return buffer.
- *
- ******************************************************************************/
-
-#define ACPI_PLD_OUTPUT     "%20s : %-6X\n"
-
-static void
-AcpiDbDumpPldBuffer (
-    ACPI_OBJECT             *ObjDesc)
-{
-    ACPI_OBJECT             *BufferDesc;
-    ACPI_PLD_INFO           *PldInfo;
-    UINT8                   *NewBuffer;
-    ACPI_STATUS             Status;
-
-
-    /* Object must be of type Package with at least one Buffer element */
-
-    if (ObjDesc->Type != ACPI_TYPE_PACKAGE)
-    {
-        return;
-    }
-
-    BufferDesc = &ObjDesc->Package.Elements[0];
-    if (BufferDesc->Type != ACPI_TYPE_BUFFER)
-    {
-        return;
-    }
-
-    /* Convert _PLD buffer to local _PLD struct */
-
-    Status = AcpiDecodePldBuffer (BufferDesc->Buffer.Pointer,
-        BufferDesc->Buffer.Length, &PldInfo);
-    if (ACPI_FAILURE (Status))
-    {
-        return;
-    }
-
-    /* Encode local _PLD struct back to a _PLD buffer */
-
-    NewBuffer = AcpiDbEncodePldBuffer (PldInfo);
-    if (!NewBuffer)
-    {
-        return;
-    }
-
-    /* The two bit-packed buffers should match */
-
-    if (ACPI_MEMCMP (NewBuffer, BufferDesc->Buffer.Pointer,
-        BufferDesc->Buffer.Length))
-    {
-        AcpiOsPrintf ("Converted _PLD buffer does not compare. New:\n");
-
-        AcpiUtDumpBuffer (NewBuffer,
-            BufferDesc->Buffer.Length, DB_BYTE_DISPLAY, 0);
-    }
-
-    /* First 32-bit dword */
-
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Revision", PldInfo->Revision);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "IgnoreColor", PldInfo->IgnoreColor);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Color", PldInfo->Color);
-
-    /* Second 32-bit dword */
-
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Width", PldInfo->Width);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Height", PldInfo->Height);
-
-    /* Third 32-bit dword */
-
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "UserVisible", PldInfo->UserVisible);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Dock", PldInfo->Dock);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Lid", PldInfo->Lid);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Panel", PldInfo->Panel);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "VerticalPosition", PldInfo->VerticalPosition);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "HorizontalPosition", PldInfo->HorizontalPosition);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Shape", PldInfo->Shape);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "GroupOrientation", PldInfo->GroupOrientation);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "GroupToken", PldInfo->GroupToken);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "GroupPosition", PldInfo->GroupPosition);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Bay", PldInfo->Bay);
-
-    /* Fourth 32-bit dword */
-
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Ejectable", PldInfo->Ejectable);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "OspmEjectRequired", PldInfo->OspmEjectRequired);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "CabinetNumber", PldInfo->CabinetNumber);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "CardCageNumber", PldInfo->CardCageNumber);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Reference", PldInfo->Reference);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Rotation", PldInfo->Rotation);
-    AcpiOsPrintf (ACPI_PLD_OUTPUT, "Order", PldInfo->Order);
-
-    /* Fifth 32-bit dword */
-
-    if (BufferDesc->Buffer.Length > 16)
-    {
-        AcpiOsPrintf (ACPI_PLD_OUTPUT, "VerticalOffset", PldInfo->VerticalOffset);
-        AcpiOsPrintf (ACPI_PLD_OUTPUT, "HorizontalOffset", PldInfo->HorizontalOffset);
-    }
-
-    ACPI_FREE (PldInfo);
-    ACPI_FREE (NewBuffer);
-}
-
-
-/*******************************************************************************
- *
  * FUNCTION:    AcpiDbExecute
  *
  * PARAMETERS:  Name                - Name of method to execute
@@ -983,12 +484,10 @@ AcpiDbExecute (
 
         Status = AcpiGetHandle (NULL, AcpiGbl_DbMethodInfo.Pathname,
             &AcpiGbl_DbMethodInfo.Method);
-        if (ACPI_FAILURE (Status))
+        if (ACPI_SUCCESS (Status))
         {
-            return;
+            Status = AcpiDbExecuteMethod (&AcpiGbl_DbMethodInfo, &ReturnObj);
         }
-
-        Status = AcpiDbExecuteMethod (&AcpiGbl_DbMethodInfo, &ReturnObj);
         ACPI_FREE (NameString);
     }
 
@@ -998,7 +497,6 @@ AcpiDbExecute (
      */
     AcpiOsSleep ((UINT64) 10);
 
-
 #ifdef ACPI_DEBUG_OUTPUT
 
     /* Memory allocation tracking */
diff --git a/sys/contrib/dev/acpica/components/debugger/dbfileio.c b/sys/contrib/dev/acpica/components/debugger/dbfileio.c
index 5c4b100..e03bb40 100644
--- a/sys/contrib/dev/acpica/components/debugger/dbfileio.c
+++ b/sys/contrib/dev/acpica/components/debugger/dbfileio.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/debugger/dbhistry.c b/sys/contrib/dev/acpica/components/debugger/dbhistry.c
index 1305451..0e05e14 100644
--- a/sys/contrib/dev/acpica/components/debugger/dbhistry.c
+++ b/sys/contrib/dev/acpica/components/debugger/dbhistry.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/debugger/dbinput.c b/sys/contrib/dev/acpica/components/debugger/dbinput.c
index f5db216..b5c46df 100644
--- a/sys/contrib/dev/acpica/components/debugger/dbinput.c
+++ b/sys/contrib/dev/acpica/components/debugger/dbinput.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -210,7 +210,7 @@ static const ACPI_DB_COMMAND_INFO   AcpiGbl_DbCommands[] =
     {"RESOURCES",    1},
     {"RESULTS",      0},
     {"SET",          3},
-    {"SLEEP",        1},
+    {"SLEEP",        0},
     {"STATS",        1},
     {"STOP",         0},
     {"TABLES",       0},
@@ -274,7 +274,7 @@ static const ACPI_DB_COMMAND_HELP   AcpiGbl_DbCommandHelp[] =
     {1, "  References <Addr>",                 "Find all references to object at addr\n"},
     {1, "  Resources <DeviceName | *>",        "Display Device resources (* = all devices)\n"},
     {1, "  Set N <NamedObject> <Value>",       "Set value for named integer\n"},
-    {1, "  Sleep <SleepState>",                "Simulate sleep/wake sequence\n"},
+    {1, "  Sleep [SleepState]",                "Simulate sleep/wake sequence(s) (0-5)\n"},
     {1, "  Template <Object>",                 "Format/dump a Buffer/ResourceTemplate\n"},
     {1, "  Terminate",                         "Delete namespace and all internal objects\n"},
     {1, "  Type <Object>",                     "Display object type\n"},
@@ -1168,6 +1168,8 @@ AcpiDbUserCommands (
     ACPI_STATUS             Status = AE_OK;
 
 
+    AcpiOsPrintf ("\n");
+
     /* TBD: [Restructure] Need a separate command line buffer for step mode */
 
     while (!AcpiGbl_DbTerminateThreads)
diff --git a/sys/contrib/dev/acpica/components/debugger/dbmethod.c b/sys/contrib/dev/acpica/components/debugger/dbmethod.c
index ccfb7d5..d3d86dd 100644
--- a/sys/contrib/dev/acpica/components/debugger/dbmethod.c
+++ b/sys/contrib/dev/acpica/components/debugger/dbmethod.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -396,7 +396,7 @@ AcpiDbDisassembleMethod (
     WalkState->ParseFlags |= ACPI_PARSE_DISASSEMBLE;
 
     Status = AcpiPsParseAml (WalkState);
-    AcpiDmParseDeferredOps (Op);
+    (void) AcpiDmParseDeferredOps (Op);
 
     /* Now we can disassemble the method */
 
diff --git a/sys/contrib/dev/acpica/components/debugger/dbnames.c b/sys/contrib/dev/acpica/components/debugger/dbnames.c
index aed8dfb..8efb74d 100644
--- a/sys/contrib/dev/acpica/components/debugger/dbnames.c
+++ b/sys/contrib/dev/acpica/components/debugger/dbnames.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -159,7 +159,7 @@ AcpiDbSetScope (
 
     AcpiDbPrepNamestring (Name);
 
-    if (Name[0] == '\\')
+    if (ACPI_IS_ROOT_PREFIX (Name[0]))
     {
         /* Validate new scope from the root */
 
diff --git a/sys/contrib/dev/acpica/components/debugger/dbstats.c b/sys/contrib/dev/acpica/components/debugger/dbstats.c
index 7037c2b..6a1c0bb 100644
--- a/sys/contrib/dev/acpica/components/debugger/dbstats.c
+++ b/sys/contrib/dev/acpica/components/debugger/dbstats.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/debugger/dbutils.c b/sys/contrib/dev/acpica/components/debugger/dbutils.c
index 7d1c427..33b9c19 100644
--- a/sys/contrib/dev/acpica/components/debugger/dbutils.c
+++ b/sys/contrib/dev/acpica/components/debugger/dbutils.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -286,7 +286,7 @@ AcpiDbPrepNamestring (
 
     /* Ignore a leading backslash, this is the root prefix */
 
-    if (*Name == '\\')
+    if (ACPI_IS_ROOT_PREFIX (*Name))
     {
         Name++;
     }
diff --git a/sys/contrib/dev/acpica/components/debugger/dbxface.c b/sys/contrib/dev/acpica/components/debugger/dbxface.c
index f265dba..6f97582 100644
--- a/sys/contrib/dev/acpica/components/debugger/dbxface.c
+++ b/sys/contrib/dev/acpica/components/debugger/dbxface.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -421,9 +421,9 @@ AcpiDbInitialize (
     AcpiGbl_DbOutputFlags       = ACPI_DB_CONSOLE_OUTPUT;
 
     AcpiGbl_DbOpt_tables        = FALSE;
+    AcpiGbl_DbOpt_disasm        = FALSE;
     AcpiGbl_DbOpt_stats         = FALSE;
 #ifdef ACPI_DISASSEMBLER
-    AcpiGbl_DbOpt_disasm        = FALSE;
     AcpiGbl_DbOpt_verbose       = TRUE;
 #endif
     AcpiGbl_DbOpt_ini_methods   = TRUE;
@@ -437,7 +437,7 @@ AcpiDbInitialize (
 
     /* Initial scope is the root */
 
-    AcpiGbl_DbScopeBuf [0] = '\\';
+    AcpiGbl_DbScopeBuf [0] = AML_ROOT_PREFIX;
     AcpiGbl_DbScopeBuf [1] =  0;
     AcpiGbl_DbScopeNode = AcpiGbl_RootNode;
 
diff --git a/sys/contrib/dev/acpica/components/disassembler/dmbuffer.c b/sys/contrib/dev/acpica/components/disassembler/dmbuffer.c
index 9be6a50..2f6ff72 100644
--- a/sys/contrib/dev/acpica/components/disassembler/dmbuffer.c
+++ b/sys/contrib/dev/acpica/components/disassembler/dmbuffer.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/disassembler/dmdeferred.c b/sys/contrib/dev/acpica/components/disassembler/dmdeferred.c
index 1e8c224..9a68079 100644
--- a/sys/contrib/dev/acpica/components/disassembler/dmdeferred.c
+++ b/sys/contrib/dev/acpica/components/disassembler/dmdeferred.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -83,7 +83,7 @@ AcpiDmParseDeferredOps (
     ACPI_STATUS             Status;
 
 
-    ACPI_FUNCTION_NAME (DmParseDeferredOps);
+    ACPI_FUNCTION_ENTRY ();
 
 
     /* Traverse the entire parse tree */
diff --git a/sys/contrib/dev/acpica/components/disassembler/dmnames.c b/sys/contrib/dev/acpica/components/disassembler/dmnames.c
index 0149bff..b2ef29d 100644
--- a/sys/contrib/dev/acpica/components/disassembler/dmnames.c
+++ b/sys/contrib/dev/acpica/components/disassembler/dmnames.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -44,7 +44,6 @@
 
 #include <contrib/dev/acpica/include/acpi.h>
 #include <contrib/dev/acpica/include/accommon.h>
-#include <contrib/dev/acpica/include/acparser.h>
 #include <contrib/dev/acpica/include/amlcode.h>
 #include <contrib/dev/acpica/include/acnamesp.h>
 #include <contrib/dev/acpica/include/acdisasm.h>
@@ -226,7 +225,8 @@ AcpiDmNamestring (
 
     /* Handle all Scope Prefix operators */
 
-    while (AcpiPsIsPrefixChar (ACPI_GET8 (Name)))
+    while (ACPI_IS_ROOT_PREFIX (ACPI_GET8 (Name)) ||
+           ACPI_IS_PARENT_PREFIX (ACPI_GET8 (Name)))
     {
         /* Append prefix character */
 
@@ -323,7 +323,7 @@ AcpiDmDisplayPath (
 
         if ((NamePath) &&
             (NamePath->Common.Value.String) &&
-            (NamePath->Common.Value.String[0] == '\\'))
+            (ACPI_IS_ROOT_PREFIX (NamePath->Common.Value.String[0])))
         {
             AcpiDmNamestring (NamePath->Common.Value.String);
             return;
diff --git a/sys/contrib/dev/acpica/components/disassembler/dmobject.c b/sys/contrib/dev/acpica/components/disassembler/dmobject.c
index 6801af5..5dde74a 100644
--- a/sys/contrib/dev/acpica/components/disassembler/dmobject.c
+++ b/sys/contrib/dev/acpica/components/disassembler/dmobject.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/disassembler/dmopcode.c b/sys/contrib/dev/acpica/components/disassembler/dmopcode.c
index 822a1a5..01672fa 100644
--- a/sys/contrib/dev/acpica/components/disassembler/dmopcode.c
+++ b/sys/contrib/dev/acpica/components/disassembler/dmopcode.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/disassembler/dmresrc.c b/sys/contrib/dev/acpica/components/disassembler/dmresrc.c
index 5217084..fd391b3 100644
--- a/sys/contrib/dev/acpica/components/disassembler/dmresrc.c
+++ b/sys/contrib/dev/acpica/components/disassembler/dmresrc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -415,7 +415,8 @@ AcpiDmIsResourceTemplate (
 
     /* Walk the byte list, abort on any invalid descriptor type or length */
 
-    Status = AcpiUtWalkAmlResources (WalkState, Aml, Length, NULL, &EndAml);
+    Status = AcpiUtWalkAmlResources (WalkState, Aml, Length,
+        NULL, ACPI_CAST_INDIRECT_PTR (void, &EndAml));
     if (ACPI_FAILURE (Status))
     {
         return (AE_TYPE);
diff --git a/sys/contrib/dev/acpica/components/disassembler/dmresrcl.c b/sys/contrib/dev/acpica/components/disassembler/dmresrcl.c
index 700370f..c468bbd 100644
--- a/sys/contrib/dev/acpica/components/disassembler/dmresrcl.c
+++ b/sys/contrib/dev/acpica/components/disassembler/dmresrcl.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/disassembler/dmresrcl2.c b/sys/contrib/dev/acpica/components/disassembler/dmresrcl2.c
index a3d046d..5da7df4 100644
--- a/sys/contrib/dev/acpica/components/disassembler/dmresrcl2.c
+++ b/sys/contrib/dev/acpica/components/disassembler/dmresrcl2.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/disassembler/dmresrcs.c b/sys/contrib/dev/acpica/components/disassembler/dmresrcs.c
index 7957920..778d663 100644
--- a/sys/contrib/dev/acpica/components/disassembler/dmresrcs.c
+++ b/sys/contrib/dev/acpica/components/disassembler/dmresrcs.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/disassembler/dmutils.c b/sys/contrib/dev/acpica/components/disassembler/dmutils.c
index 7ac56b7..5d35d83 100644
--- a/sys/contrib/dev/acpica/components/disassembler/dmutils.c
+++ b/sys/contrib/dev/acpica/components/disassembler/dmutils.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/disassembler/dmwalk.c b/sys/contrib/dev/acpica/components/disassembler/dmwalk.c
index f0d65b6..67a7965 100644
--- a/sys/contrib/dev/acpica/components/disassembler/dmwalk.c
+++ b/sys/contrib/dev/acpica/components/disassembler/dmwalk.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -441,7 +441,30 @@ AcpiDmDescendingOp (
              * This is a first-level element of a term list,
              * indent a new line
              */
-            AcpiDmIndent (Level);
+            switch (Op->Common.AmlOpcode)
+            {
+            case AML_NOOP_OP:
+                /*
+                 * Optionally just ignore this opcode. Some tables use
+                 * NoOp opcodes for "padding" out packages that the BIOS
+                 * changes dynamically. This can leave hundreds or
+                 * thousands of NoOp opcodes that if disassembled,
+                 * cannot be compiled because they are syntactically
+                 * incorrect.
+                 */
+                if (AcpiGbl_IgnoreNoopOperator)
+                {
+                    Op->Common.DisasmFlags |= ACPI_PARSEOP_IGNORE;
+                    return (AE_OK);
+                }
+
+                /* Fallthrough */
+
+            default:
+                AcpiDmIndent (Level);
+                break;
+            }
+
             Info->LastLevel = Level;
             Info->Count = 0;
     }
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dsargs.c b/sys/contrib/dev/acpica/components/dispatcher/dsargs.c
index 0602905..2c574ca1 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dsargs.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dsargs.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dscontrol.c b/sys/contrib/dev/acpica/components/dispatcher/dscontrol.c
index 198ba0e..bcc58ca 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dscontrol.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dscontrol.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dsfield.c b/sys/contrib/dev/acpica/components/dispatcher/dsfield.c
index 3574aa7..1ab8c3b 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dsfield.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dsfield.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dsinit.c b/sys/contrib/dev/acpica/components/dispatcher/dsinit.c
index c3766fd..7aca0e8 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dsinit.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dsinit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dsmethod.c b/sys/contrib/dev/acpica/components/dispatcher/dsmethod.c
index b3548e9..d2dccaf 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dsmethod.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dsmethod.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -163,6 +163,7 @@ AcpiDsCreateMethodMutex (
     Status = AcpiOsCreateMutex (&MutexDesc->Mutex.OsMutex);
     if (ACPI_FAILURE (Status))
     {
+        AcpiUtDeleteObjectDesc (MutexDesc);
         return_ACPI_STATUS (Status);
     }
 
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dsmthdat.c b/sys/contrib/dev/acpica/components/dispatcher/dsmthdat.c
index 76c82af..164e0ef 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dsmthdat.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dsmthdat.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dsobject.c b/sys/contrib/dev/acpica/components/dispatcher/dsobject.c
index 69985a9..d718929 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dsobject.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dsobject.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -541,7 +541,7 @@ AcpiDsBuildInternalPackageObj (
         }
 
         ACPI_INFO ((AE_INFO,
-            "Actual Package length (%u) is larger than NumElements field (%u), truncated\n",
+            "Actual Package length (%u) is larger than NumElements field (%u), truncated",
             i, ElementCount));
     }
     else if (i < ElementCount)
@@ -734,7 +734,7 @@ AcpiDsInitObjectFromOp (
                 /* Truncate value if we are executing from a 32-bit ACPI table */
 
 #ifndef ACPI_NO_METHOD_EXECUTION
-                AcpiExTruncateFor32bitTable (ObjDesc);
+                (void) AcpiExTruncateFor32bitTable (ObjDesc);
 #endif
                 break;
 
@@ -756,8 +756,17 @@ AcpiDsInitObjectFromOp (
         case AML_TYPE_LITERAL:
 
             ObjDesc->Integer.Value = Op->Common.Value.Integer;
+
 #ifndef ACPI_NO_METHOD_EXECUTION
-            AcpiExTruncateFor32bitTable (ObjDesc);
+            if (AcpiExTruncateFor32bitTable (ObjDesc))
+            {
+                /* Warn if we found a 64-bit constant in a 32-bit table */
+
+                ACPI_WARNING ((AE_INFO,
+                    "Truncated 64-bit constant found in 32-bit table: %8.8X%8.8X => %8.8X",
+                    ACPI_FORMAT_UINT64 (Op->Common.Value.Integer),
+                    (UINT32) ObjDesc->Integer.Value));
+            }
 #endif
             break;
 
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dsopcode.c b/sys/contrib/dev/acpica/components/dispatcher/dsopcode.c
index 8c7d314..7fed5fd 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dsopcode.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dsopcode.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dsutils.c b/sys/contrib/dev/acpica/components/dispatcher/dsutils.c
index 13ef960..3b44c67 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dsutils.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dsutils.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -201,7 +201,7 @@ AcpiDsIsResultUsed (
     if (!Op)
     {
         ACPI_ERROR ((AE_INFO, "Null Op"));
-        return_UINT8 (TRUE);
+        return_VALUE (TRUE);
     }
 
     /*
@@ -231,7 +231,7 @@ AcpiDsIsResultUsed (
         ACPI_DEBUG_PRINT ((ACPI_DB_DISPATCH,
             "At Method level, result of [%s] not used\n",
             AcpiPsGetOpcodeName (Op->Common.AmlOpcode)));
-        return_UINT8 (FALSE);
+        return_VALUE (FALSE);
     }
 
     /* Get info on the parent. The RootOp is AML_SCOPE */
@@ -241,7 +241,7 @@ AcpiDsIsResultUsed (
     {
         ACPI_ERROR ((AE_INFO,
             "Unknown parent opcode Op=%p", Op));
-        return_UINT8 (FALSE);
+        return_VALUE (FALSE);
     }
 
     /*
@@ -331,7 +331,7 @@ ResultUsed:
         AcpiPsGetOpcodeName (Op->Common.AmlOpcode),
         AcpiPsGetOpcodeName (Op->Common.Parent->Common.AmlOpcode), Op));
 
-    return_UINT8 (TRUE);
+    return_VALUE (TRUE);
 
 
 ResultNotUsed:
@@ -340,7 +340,7 @@ ResultNotUsed:
         AcpiPsGetOpcodeName (Op->Common.AmlOpcode),
         AcpiPsGetOpcodeName (Op->Common.Parent->Common.AmlOpcode), Op));
 
-    return_UINT8 (FALSE);
+    return_VALUE (FALSE);
 }
 
 
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dswexec.c b/sys/contrib/dev/acpica/components/dispatcher/dswexec.c
index eed103d..f92f49b 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dswexec.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dswexec.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -164,7 +164,7 @@ AcpiDsGetPredicateValue (
 
     /* Truncate the predicate to 32-bits if necessary */
 
-    AcpiExTruncateFor32bitTable (LocalObjDesc);
+    (void) AcpiExTruncateFor32bitTable (LocalObjDesc);
 
     /*
      * Save the result of the predicate evaluation on
@@ -739,7 +739,7 @@ AcpiDsExecEndOp (
      * ACPI 2.0 support for 64-bit integers: Truncate numeric
      * result value if we are executing from a 32-bit ACPI table
      */
-    AcpiExTruncateFor32bitTable (WalkState->ResultObj);
+    (void) AcpiExTruncateFor32bitTable (WalkState->ResultObj);
 
     /*
      * Check if we just completed the evaluation of a
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dswload.c b/sys/contrib/dev/acpica/components/dispatcher/dswload.c
index 53257b5..f86dea2 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dswload.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dswload.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dswload2.c b/sys/contrib/dev/acpica/components/dispatcher/dswload2.c
index c3f7bef..9114895 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dswload2.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dswload2.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -236,7 +236,7 @@ AcpiDsLoad2BeginOp (
              */
             ACPI_WARNING ((AE_INFO,
                 "Type override - [%4.4s] had invalid type (%s) "
-                "for Scope operator, changed to type ANY\n",
+                "for Scope operator, changed to type ANY",
                 AcpiUtGetNodeName (Node), AcpiUtGetTypeName (Node->Type)));
 
             Node->Type = ACPI_TYPE_ANY;
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dswscope.c b/sys/contrib/dev/acpica/components/dispatcher/dswscope.c
index bdbd374..8530b5d 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dswscope.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dswscope.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/dispatcher/dswstate.c b/sys/contrib/dev/acpica/components/dispatcher/dswstate.c
index bdea4d7..8fd36bf 100644
--- a/sys/contrib/dev/acpica/components/dispatcher/dswstate.c
+++ b/sys/contrib/dev/acpica/components/dispatcher/dswstate.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/events/evevent.c b/sys/contrib/dev/acpica/components/events/evevent.c
index 5706f8f..adad8c8 100644
--- a/sys/contrib/dev/acpica/components/events/evevent.c
+++ b/sys/contrib/dev/acpica/components/events/evevent.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/events/evglock.c b/sys/contrib/dev/acpica/components/events/evglock.c
index 894e132..4cec447 100644
--- a/sys/contrib/dev/acpica/components/events/evglock.c
+++ b/sys/contrib/dev/acpica/components/events/evglock.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/events/evgpe.c b/sys/contrib/dev/acpica/components/events/evgpe.c
index eb96d72..75ad956 100644
--- a/sys/contrib/dev/acpica/components/events/evgpe.c
+++ b/sys/contrib/dev/acpica/components/events/evgpe.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -770,7 +770,7 @@ AcpiEvGpeDispatch (
         {
             ACPI_EXCEPTION ((AE_INFO, Status,
                 "Unable to clear GPE%02X", GpeNumber));
-            return_UINT32 (ACPI_INTERRUPT_NOT_HANDLED);
+            return_VALUE (ACPI_INTERRUPT_NOT_HANDLED);
         }
     }
 
@@ -788,7 +788,7 @@ AcpiEvGpeDispatch (
     {
         ACPI_EXCEPTION ((AE_INFO, Status,
             "Unable to disable GPE%02X", GpeNumber));
-        return_UINT32 (ACPI_INTERRUPT_NOT_HANDLED);
+        return_VALUE (ACPI_INTERRUPT_NOT_HANDLED);
     }
 
     /*
@@ -846,7 +846,7 @@ AcpiEvGpeDispatch (
         break;
     }
 
-    return_UINT32 (ACPI_INTERRUPT_HANDLED);
+    return_VALUE (ACPI_INTERRUPT_HANDLED);
 }
 
 #endif /* !ACPI_REDUCED_HARDWARE */
diff --git a/sys/contrib/dev/acpica/components/events/evgpeblk.c b/sys/contrib/dev/acpica/components/events/evgpeblk.c
index 6a1c04a..f223b1a 100644
--- a/sys/contrib/dev/acpica/components/events/evgpeblk.c
+++ b/sys/contrib/dev/acpica/components/events/evgpeblk.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -443,8 +443,8 @@ AcpiEvCreateGpeBlock (
         (*ReturnGpeBlock) = GpeBlock;
     }
 
-    ACPI_DEBUG_PRINT ((ACPI_DB_INIT,
-        "GPE %02X to %02X [%4.4s] %u regs on int 0x%X\n",
+    ACPI_DEBUG_PRINT_RAW ((ACPI_DB_INIT,
+        "    Initialized GPE %02X to %02X [%4.4s] %u regs on interrupt 0x%X\n",
         (UINT32) GpeBlock->BlockBaseNumber,
         (UINT32) (GpeBlock->BlockBaseNumber + (GpeBlock->GpeCount - 1)),
         GpeDevice->Name.Ascii, GpeBlock->RegisterCount,
@@ -539,8 +539,10 @@ AcpiEvInitializeGpeBlock (
 
     if (GpeEnabledCount)
     {
-        ACPI_DEBUG_PRINT ((ACPI_DB_INIT,
-            "Enabled %u GPEs in this block\n", GpeEnabledCount));
+        ACPI_INFO ((AE_INFO,
+            "Enabled %u GPEs in block %02X to %02X", GpeEnabledCount,
+            (UINT32) GpeBlock->BlockBaseNumber,
+            (UINT32) (GpeBlock->BlockBaseNumber + (GpeBlock->GpeCount - 1))));
     }
 
     GpeBlock->Initialized = TRUE;
diff --git a/sys/contrib/dev/acpica/components/events/evgpeinit.c b/sys/contrib/dev/acpica/components/events/evgpeinit.c
index 9266d60..9e8630d 100644
--- a/sys/contrib/dev/acpica/components/events/evgpeinit.c
+++ b/sys/contrib/dev/acpica/components/events/evgpeinit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -94,6 +94,9 @@ AcpiEvGpeInitialize (
     ACPI_FUNCTION_TRACE (EvGpeInitialize);
 
 
+    ACPI_DEBUG_PRINT_RAW ((ACPI_DB_INIT,
+        "Initializing General Purpose Events (GPEs):\n"));
+
     Status = AcpiUtAcquireMutex (ACPI_MTX_NAMESPACE);
     if (ACPI_FAILURE (Status))
     {
diff --git a/sys/contrib/dev/acpica/components/events/evgpeutil.c b/sys/contrib/dev/acpica/components/events/evgpeutil.c
index 7f7bb3c..1841f9d 100644
--- a/sys/contrib/dev/acpica/components/events/evgpeutil.c
+++ b/sys/contrib/dev/acpica/components/events/evgpeutil.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/events/evhandler.c b/sys/contrib/dev/acpica/components/events/evhandler.c
new file mode 100644
index 0000000..f32ba65
--- /dev/null
+++ b/sys/contrib/dev/acpica/components/events/evhandler.c
@@ -0,0 +1,578 @@
+/******************************************************************************
+ *
+ * Module Name: evhandler - Support for Address Space handlers
+ *
+ *****************************************************************************/
+
+/*
+ * Copyright (C) 2000 - 2013, Intel Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ * 3. Neither the names of the above-listed copyright holders nor the names
+ *    of any contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ */
+
+
+#define __EVHANDLER_C__
+
+#include <contrib/dev/acpica/include/acpi.h>
+#include <contrib/dev/acpica/include/accommon.h>
+#include <contrib/dev/acpica/include/acevents.h>
+#include <contrib/dev/acpica/include/acnamesp.h>
+#include <contrib/dev/acpica/include/acinterp.h>
+
+#define _COMPONENT          ACPI_EVENTS
+        ACPI_MODULE_NAME    ("evhandler")
+
+
+/* Local prototypes */
+
+static ACPI_STATUS
+AcpiEvInstallHandler (
+    ACPI_HANDLE             ObjHandle,
+    UINT32                  Level,
+    void                    *Context,
+    void                    **ReturnValue);
+
+/* These are the address spaces that will get default handlers */
+
+UINT8        AcpiGbl_DefaultAddressSpaces[ACPI_NUM_DEFAULT_SPACES] =
+{
+    ACPI_ADR_SPACE_SYSTEM_MEMORY,
+    ACPI_ADR_SPACE_SYSTEM_IO,
+    ACPI_ADR_SPACE_PCI_CONFIG,
+    ACPI_ADR_SPACE_DATA_TABLE
+};
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiEvInstallRegionHandlers
+ *
+ * PARAMETERS:  None
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Installs the core subsystem default address space handlers.
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+AcpiEvInstallRegionHandlers (
+    void)
+{
+    ACPI_STATUS             Status;
+    UINT32                  i;
+
+
+    ACPI_FUNCTION_TRACE (EvInstallRegionHandlers);
+
+
+    Status = AcpiUtAcquireMutex (ACPI_MTX_NAMESPACE);
+    if (ACPI_FAILURE (Status))
+    {
+        return_ACPI_STATUS (Status);
+    }
+
+    /*
+     * All address spaces (PCI Config, EC, SMBus) are scope dependent and
+     * registration must occur for a specific device.
+     *
+     * In the case of the system memory and IO address spaces there is
+     * currently no device associated with the address space. For these we
+     * use the root.
+     *
+     * We install the default PCI config space handler at the root so that
+     * this space is immediately available even though the we have not
+     * enumerated all the PCI Root Buses yet. This is to conform to the ACPI
+     * specification which states that the PCI config space must be always
+     * available -- even though we are nowhere near ready to find the PCI root
+     * buses at this point.
+     *
+     * NOTE: We ignore AE_ALREADY_EXISTS because this means that a handler
+     * has already been installed (via AcpiInstallAddressSpaceHandler).
+     * Similar for AE_SAME_HANDLER.
+     */
+    for (i = 0; i < ACPI_NUM_DEFAULT_SPACES; i++)
+    {
+        Status = AcpiEvInstallSpaceHandler (AcpiGbl_RootNode,
+                    AcpiGbl_DefaultAddressSpaces[i],
+                    ACPI_DEFAULT_HANDLER, NULL, NULL);
+        switch (Status)
+        {
+        case AE_OK:
+        case AE_SAME_HANDLER:
+        case AE_ALREADY_EXISTS:
+
+            /* These exceptions are all OK */
+
+            Status = AE_OK;
+            break;
+
+        default:
+
+            goto UnlockAndExit;
+        }
+    }
+
+UnlockAndExit:
+    (void) AcpiUtReleaseMutex (ACPI_MTX_NAMESPACE);
+    return_ACPI_STATUS (Status);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiEvHasDefaultHandler
+ *
+ * PARAMETERS:  Node                - Namespace node for the device
+ *              SpaceId             - The address space ID
+ *
+ * RETURN:      TRUE if default handler is installed, FALSE otherwise
+ *
+ * DESCRIPTION: Check if the default handler is installed for the requested
+ *              space ID.
+ *
+ ******************************************************************************/
+
+BOOLEAN
+AcpiEvHasDefaultHandler (
+    ACPI_NAMESPACE_NODE     *Node,
+    ACPI_ADR_SPACE_TYPE     SpaceId)
+{
+    ACPI_OPERAND_OBJECT     *ObjDesc;
+    ACPI_OPERAND_OBJECT     *HandlerObj;
+
+
+    /* Must have an existing internal object */
+
+    ObjDesc = AcpiNsGetAttachedObject (Node);
+    if (ObjDesc)
+    {
+        HandlerObj = ObjDesc->Device.Handler;
+
+        /* Walk the linked list of handlers for this object */
+
+        while (HandlerObj)
+        {
+            if (HandlerObj->AddressSpace.SpaceId == SpaceId)
+            {
+                if (HandlerObj->AddressSpace.HandlerFlags &
+                        ACPI_ADDR_HANDLER_DEFAULT_INSTALLED)
+                {
+                    return (TRUE);
+                }
+            }
+
+            HandlerObj = HandlerObj->AddressSpace.Next;
+        }
+    }
+
+    return (FALSE);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiEvInstallHandler
+ *
+ * PARAMETERS:  WalkNamespace callback
+ *
+ * DESCRIPTION: This routine installs an address handler into objects that are
+ *              of type Region or Device.
+ *
+ *              If the Object is a Device, and the device has a handler of
+ *              the same type then the search is terminated in that branch.
+ *
+ *              This is because the existing handler is closer in proximity
+ *              to any more regions than the one we are trying to install.
+ *
+ ******************************************************************************/
+
+static ACPI_STATUS
+AcpiEvInstallHandler (
+    ACPI_HANDLE             ObjHandle,
+    UINT32                  Level,
+    void                    *Context,
+    void                    **ReturnValue)
+{
+    ACPI_OPERAND_OBJECT     *HandlerObj;
+    ACPI_OPERAND_OBJECT     *NextHandlerObj;
+    ACPI_OPERAND_OBJECT     *ObjDesc;
+    ACPI_NAMESPACE_NODE     *Node;
+    ACPI_STATUS             Status;
+
+
+    ACPI_FUNCTION_NAME (EvInstallHandler);
+
+
+    HandlerObj = (ACPI_OPERAND_OBJECT  *) Context;
+
+    /* Parameter validation */
+
+    if (!HandlerObj)
+    {
+        return (AE_OK);
+    }
+
+    /* Convert and validate the device handle */
+
+    Node = AcpiNsValidateHandle (ObjHandle);
+    if (!Node)
+    {
+        return (AE_BAD_PARAMETER);
+    }
+
+    /*
+     * We only care about regions and objects that are allowed to have
+     * address space handlers
+     */
+    if ((Node->Type != ACPI_TYPE_DEVICE) &&
+        (Node->Type != ACPI_TYPE_REGION) &&
+        (Node != AcpiGbl_RootNode))
+    {
+        return (AE_OK);
+    }
+
+    /* Check for an existing internal object */
+
+    ObjDesc = AcpiNsGetAttachedObject (Node);
+    if (!ObjDesc)
+    {
+        /* No object, just exit */
+
+        return (AE_OK);
+    }
+
+    /* Devices are handled different than regions */
+
+    if (ObjDesc->Common.Type == ACPI_TYPE_DEVICE)
+    {
+        /* Check if this Device already has a handler for this address space */
+
+        NextHandlerObj = ObjDesc->Device.Handler;
+        while (NextHandlerObj)
+        {
+            /* Found a handler, is it for the same address space? */
+
+            if (NextHandlerObj->AddressSpace.SpaceId ==
+                    HandlerObj->AddressSpace.SpaceId)
+            {
+                ACPI_DEBUG_PRINT ((ACPI_DB_OPREGION,
+                    "Found handler for region [%s] in device %p(%p) "
+                    "handler %p\n",
+                    AcpiUtGetRegionName (HandlerObj->AddressSpace.SpaceId),
+                    ObjDesc, NextHandlerObj, HandlerObj));
+
+                /*
+                 * Since the object we found it on was a device, then it
+                 * means that someone has already installed a handler for
+                 * the branch of the namespace from this device on. Just
+                 * bail out telling the walk routine to not traverse this
+                 * branch. This preserves the scoping rule for handlers.
+                 */
+                return (AE_CTRL_DEPTH);
+            }
+
+            /* Walk the linked list of handlers attached to this device */
+
+            NextHandlerObj = NextHandlerObj->AddressSpace.Next;
+        }
+
+        /*
+         * As long as the device didn't have a handler for this space we
+         * don't care about it. We just ignore it and proceed.
+         */
+        return (AE_OK);
+    }
+
+    /* Object is a Region */
+
+    if (ObjDesc->Region.SpaceId != HandlerObj->AddressSpace.SpaceId)
+    {
+        /* This region is for a different address space, just ignore it */
+
+        return (AE_OK);
+    }
+
+    /*
+     * Now we have a region and it is for the handler's address space type.
+     *
+     * First disconnect region for any previous handler (if any)
+     */
+    AcpiEvDetachRegion (ObjDesc, FALSE);
+
+    /* Connect the region to the new handler */
+
+    Status = AcpiEvAttachRegion (HandlerObj, ObjDesc, FALSE);
+    return (Status);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiEvInstallSpaceHandler
+ *
+ * PARAMETERS:  Node            - Namespace node for the device
+ *              SpaceId         - The address space ID
+ *              Handler         - Address of the handler
+ *              Setup           - Address of the setup function
+ *              Context         - Value passed to the handler on each access
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Install a handler for all OpRegions of a given SpaceId.
+ *              Assumes namespace is locked
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+AcpiEvInstallSpaceHandler (
+    ACPI_NAMESPACE_NODE     *Node,
+    ACPI_ADR_SPACE_TYPE     SpaceId,
+    ACPI_ADR_SPACE_HANDLER  Handler,
+    ACPI_ADR_SPACE_SETUP    Setup,
+    void                    *Context)
+{
+    ACPI_OPERAND_OBJECT     *ObjDesc;
+    ACPI_OPERAND_OBJECT     *HandlerObj;
+    ACPI_STATUS             Status;
+    ACPI_OBJECT_TYPE        Type;
+    UINT8                  Flags = 0;
+
+
+    ACPI_FUNCTION_TRACE (EvInstallSpaceHandler);
+
+
+    /*
+     * This registration is valid for only the types below and the root. This
+     * is where the default handlers get placed.
+     */
+    if ((Node->Type != ACPI_TYPE_DEVICE)     &&
+        (Node->Type != ACPI_TYPE_PROCESSOR)  &&
+        (Node->Type != ACPI_TYPE_THERMAL)    &&
+        (Node != AcpiGbl_RootNode))
+    {
+        Status = AE_BAD_PARAMETER;
+        goto UnlockAndExit;
+    }
+
+    if (Handler == ACPI_DEFAULT_HANDLER)
+    {
+        Flags = ACPI_ADDR_HANDLER_DEFAULT_INSTALLED;
+
+        switch (SpaceId)
+        {
+        case ACPI_ADR_SPACE_SYSTEM_MEMORY:
+            Handler = AcpiExSystemMemorySpaceHandler;
+            Setup   = AcpiEvSystemMemoryRegionSetup;
+            break;
+
+        case ACPI_ADR_SPACE_SYSTEM_IO:
+            Handler = AcpiExSystemIoSpaceHandler;
+            Setup   = AcpiEvIoSpaceRegionSetup;
+            break;
+
+        case ACPI_ADR_SPACE_PCI_CONFIG:
+            Handler = AcpiExPciConfigSpaceHandler;
+            Setup   = AcpiEvPciConfigRegionSetup;
+            break;
+
+        case ACPI_ADR_SPACE_CMOS:
+            Handler = AcpiExCmosSpaceHandler;
+            Setup   = AcpiEvCmosRegionSetup;
+            break;
+
+        case ACPI_ADR_SPACE_PCI_BAR_TARGET:
+            Handler = AcpiExPciBarSpaceHandler;
+            Setup   = AcpiEvPciBarRegionSetup;
+            break;
+
+        case ACPI_ADR_SPACE_DATA_TABLE:
+            Handler = AcpiExDataTableSpaceHandler;
+            Setup   = NULL;
+            break;
+
+        default:
+            Status = AE_BAD_PARAMETER;
+            goto UnlockAndExit;
+        }
+    }
+
+    /* If the caller hasn't specified a setup routine, use the default */
+
+    if (!Setup)
+    {
+        Setup = AcpiEvDefaultRegionSetup;
+    }
+
+    /* Check for an existing internal object */
+
+    ObjDesc = AcpiNsGetAttachedObject (Node);
+    if (ObjDesc)
+    {
+        /*
+         * The attached device object already exists. Make sure the handler
+         * is not already installed.
+         */
+        HandlerObj = ObjDesc->Device.Handler;
+
+        /* Walk the handler list for this device */
+
+        while (HandlerObj)
+        {
+            /* Same SpaceId indicates a handler already installed */
+
+            if (HandlerObj->AddressSpace.SpaceId == SpaceId)
+            {
+                if (HandlerObj->AddressSpace.Handler == Handler)
+                {
+                    /*
+                     * It is (relatively) OK to attempt to install the SAME
+                     * handler twice. This can easily happen with the
+                     * PCI_Config space.
+                     */
+                    Status = AE_SAME_HANDLER;
+                    goto UnlockAndExit;
+                }
+                else
+                {
+                    /* A handler is already installed */
+
+                    Status = AE_ALREADY_EXISTS;
+                }
+                goto UnlockAndExit;
+            }
+
+            /* Walk the linked list of handlers */
+
+            HandlerObj = HandlerObj->AddressSpace.Next;
+        }
+    }
+    else
+    {
+        ACPI_DEBUG_PRINT ((ACPI_DB_OPREGION,
+            "Creating object on Device %p while installing handler\n", Node));
+
+        /* ObjDesc does not exist, create one */
+
+        if (Node->Type == ACPI_TYPE_ANY)
+        {
+            Type = ACPI_TYPE_DEVICE;
+        }
+        else
+        {
+            Type = Node->Type;
+        }
+
+        ObjDesc = AcpiUtCreateInternalObject (Type);
+        if (!ObjDesc)
+        {
+            Status = AE_NO_MEMORY;
+            goto UnlockAndExit;
+        }
+
+        /* Init new descriptor */
+
+        ObjDesc->Common.Type = (UINT8) Type;
+
+        /* Attach the new object to the Node */
+
+        Status = AcpiNsAttachObject (Node, ObjDesc, Type);
+
+        /* Remove local reference to the object */
+
+        AcpiUtRemoveReference (ObjDesc);
+
+        if (ACPI_FAILURE (Status))
+        {
+            goto UnlockAndExit;
+        }
+    }
+
+    ACPI_DEBUG_PRINT ((ACPI_DB_OPREGION,
+        "Installing address handler for region %s(%X) on Device %4.4s %p(%p)\n",
+        AcpiUtGetRegionName (SpaceId), SpaceId,
+        AcpiUtGetNodeName (Node), Node, ObjDesc));
+
+    /*
+     * Install the handler
+     *
+     * At this point there is no existing handler. Just allocate the object
+     * for the handler and link it into the list.
+     */
+    HandlerObj = AcpiUtCreateInternalObject (ACPI_TYPE_LOCAL_ADDRESS_HANDLER);
+    if (!HandlerObj)
+    {
+        Status = AE_NO_MEMORY;
+        goto UnlockAndExit;
+    }
+
+    /* Init handler obj */
+
+    HandlerObj->AddressSpace.SpaceId = (UINT8) SpaceId;
+    HandlerObj->AddressSpace.HandlerFlags = Flags;
+    HandlerObj->AddressSpace.RegionList = NULL;
+    HandlerObj->AddressSpace.Node = Node;
+    HandlerObj->AddressSpace.Handler = Handler;
+    HandlerObj->AddressSpace.Context = Context;
+    HandlerObj->AddressSpace.Setup  = Setup;
+
+    /* Install at head of Device.AddressSpace list */
+
+    HandlerObj->AddressSpace.Next = ObjDesc->Device.Handler;
+
+    /*
+     * The Device object is the first reference on the HandlerObj.
+     * Each region that uses the handler adds a reference.
+     */
+    ObjDesc->Device.Handler = HandlerObj;
+
+    /*
+     * Walk the namespace finding all of the regions this
+     * handler will manage.
+     *
+     * Start at the device and search the branch toward
+     * the leaf nodes until either the leaf is encountered or
+     * a device is detected that has an address handler of the
+     * same type.
+     *
+     * In either case, back up and search down the remainder
+     * of the branch
+     */
+    Status = AcpiNsWalkNamespace (ACPI_TYPE_ANY, Node, ACPI_UINT32_MAX,
+                ACPI_NS_WALK_UNLOCK, AcpiEvInstallHandler, NULL,
+                HandlerObj, NULL);
+
+UnlockAndExit:
+    return_ACPI_STATUS (Status);
+}
diff --git a/sys/contrib/dev/acpica/components/events/evmisc.c b/sys/contrib/dev/acpica/components/events/evmisc.c
index b63aafc..034b8e0 100644
--- a/sys/contrib/dev/acpica/components/events/evmisc.c
+++ b/sys/contrib/dev/acpica/components/events/evmisc.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/events/evregion.c b/sys/contrib/dev/acpica/components/events/evregion.c
index d149181..7bfb1f5 100644
--- a/sys/contrib/dev/acpica/components/events/evregion.c
+++ b/sys/contrib/dev/acpica/components/events/evregion.c
@@ -1,11 +1,11 @@
 /******************************************************************************
  *
- * Module Name: evregion - ACPI AddressSpace (OpRegion) handler dispatch
+ * Module Name: evregion - Operation Region support
  *
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -54,12 +54,9 @@
         ACPI_MODULE_NAME    ("evregion")
 
 
-/* Local prototypes */
+extern UINT8        AcpiGbl_DefaultAddressSpaces[];
 
-static BOOLEAN
-AcpiEvHasDefaultHandler (
-    ACPI_NAMESPACE_NODE     *Node,
-    ACPI_ADR_SPACE_TYPE     SpaceId);
+/* Local prototypes */
 
 static void
 AcpiEvOrphanEcRegMethod (
@@ -72,152 +69,6 @@ AcpiEvRegRun (
     void                    *Context,
     void                    **ReturnValue);
 
-static ACPI_STATUS
-AcpiEvInstallHandler (
-    ACPI_HANDLE             ObjHandle,
-    UINT32                  Level,
-    void                    *Context,
-    void                    **ReturnValue);
-
-/* These are the address spaces that will get default handlers */
-
-#define ACPI_NUM_DEFAULT_SPACES     4
-
-static UINT8        AcpiGbl_DefaultAddressSpaces[ACPI_NUM_DEFAULT_SPACES] =
-{
-    ACPI_ADR_SPACE_SYSTEM_MEMORY,
-    ACPI_ADR_SPACE_SYSTEM_IO,
-    ACPI_ADR_SPACE_PCI_CONFIG,
-    ACPI_ADR_SPACE_DATA_TABLE
-};
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiEvInstallRegionHandlers
- *
- * PARAMETERS:  None
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Installs the core subsystem default address space handlers.
- *
- ******************************************************************************/
-
-ACPI_STATUS
-AcpiEvInstallRegionHandlers (
-    void)
-{
-    ACPI_STATUS             Status;
-    UINT32                  i;
-
-
-    ACPI_FUNCTION_TRACE (EvInstallRegionHandlers);
-
-
-    Status = AcpiUtAcquireMutex (ACPI_MTX_NAMESPACE);
-    if (ACPI_FAILURE (Status))
-    {
-        return_ACPI_STATUS (Status);
-    }
-
-    /*
-     * All address spaces (PCI Config, EC, SMBus) are scope dependent and
-     * registration must occur for a specific device.
-     *
-     * In the case of the system memory and IO address spaces there is
-     * currently no device associated with the address space. For these we
-     * use the root.
-     *
-     * We install the default PCI config space handler at the root so that
-     * this space is immediately available even though the we have not
-     * enumerated all the PCI Root Buses yet. This is to conform to the ACPI
-     * specification which states that the PCI config space must be always
-     * available -- even though we are nowhere near ready to find the PCI root
-     * buses at this point.
-     *
-     * NOTE: We ignore AE_ALREADY_EXISTS because this means that a handler
-     * has already been installed (via AcpiInstallAddressSpaceHandler).
-     * Similar for AE_SAME_HANDLER.
-     */
-    for (i = 0; i < ACPI_NUM_DEFAULT_SPACES; i++)
-    {
-        Status = AcpiEvInstallSpaceHandler (AcpiGbl_RootNode,
-                    AcpiGbl_DefaultAddressSpaces[i],
-                    ACPI_DEFAULT_HANDLER, NULL, NULL);
-        switch (Status)
-        {
-        case AE_OK:
-        case AE_SAME_HANDLER:
-        case AE_ALREADY_EXISTS:
-
-            /* These exceptions are all OK */
-
-            Status = AE_OK;
-            break;
-
-        default:
-
-            goto UnlockAndExit;
-        }
-    }
-
-UnlockAndExit:
-    (void) AcpiUtReleaseMutex (ACPI_MTX_NAMESPACE);
-    return_ACPI_STATUS (Status);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiEvHasDefaultHandler
- *
- * PARAMETERS:  Node                - Namespace node for the device
- *              SpaceId             - The address space ID
- *
- * RETURN:      TRUE if default handler is installed, FALSE otherwise
- *
- * DESCRIPTION: Check if the default handler is installed for the requested
- *              space ID.
- *
- ******************************************************************************/
-
-static BOOLEAN
-AcpiEvHasDefaultHandler (
-    ACPI_NAMESPACE_NODE     *Node,
-    ACPI_ADR_SPACE_TYPE     SpaceId)
-{
-    ACPI_OPERAND_OBJECT     *ObjDesc;
-    ACPI_OPERAND_OBJECT     *HandlerObj;
-
-
-    /* Must have an existing internal object */
-
-    ObjDesc = AcpiNsGetAttachedObject (Node);
-    if (ObjDesc)
-    {
-        HandlerObj = ObjDesc->Device.Handler;
-
-        /* Walk the linked list of handlers for this object */
-
-        while (HandlerObj)
-        {
-            if (HandlerObj->AddressSpace.SpaceId == SpaceId)
-            {
-                if (HandlerObj->AddressSpace.HandlerFlags &
-                        ACPI_ADDR_HANDLER_DEFAULT_INSTALLED)
-                {
-                    return (TRUE);
-                }
-            }
-
-            HandlerObj = HandlerObj->AddressSpace.Next;
-        }
-    }
-
-    return (FALSE);
-}
-
 
 /*******************************************************************************
  *
@@ -275,100 +126,6 @@ AcpiEvInitializeOpRegions (
 
 /*******************************************************************************
  *
- * FUNCTION:    AcpiEvExecuteRegMethod
- *
- * PARAMETERS:  RegionObj           - Region object
- *              Function            - Passed to _REG: On (1) or Off (0)
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Execute _REG method for a region
- *
- ******************************************************************************/
-
-ACPI_STATUS
-AcpiEvExecuteRegMethod (
-    ACPI_OPERAND_OBJECT     *RegionObj,
-    UINT32                  Function)
-{
-    ACPI_EVALUATE_INFO      *Info;
-    ACPI_OPERAND_OBJECT     *Args[3];
-    ACPI_OPERAND_OBJECT     *RegionObj2;
-    ACPI_STATUS             Status;
-
-
-    ACPI_FUNCTION_TRACE (EvExecuteRegMethod);
-
-
-    RegionObj2 = AcpiNsGetSecondaryObject (RegionObj);
-    if (!RegionObj2)
-    {
-        return_ACPI_STATUS (AE_NOT_EXIST);
-    }
-
-    if (RegionObj2->Extra.Method_REG == NULL)
-    {
-        return_ACPI_STATUS (AE_OK);
-    }
-
-    /* Allocate and initialize the evaluation information block */
-
-    Info = ACPI_ALLOCATE_ZEROED (sizeof (ACPI_EVALUATE_INFO));
-    if (!Info)
-    {
-        return_ACPI_STATUS (AE_NO_MEMORY);
-    }
-
-    Info->PrefixNode = RegionObj2->Extra.Method_REG;
-    Info->Pathname = NULL;
-    Info->Parameters = Args;
-    Info->Flags = ACPI_IGNORE_RETURN_VALUE;
-
-    /*
-     * The _REG method has two arguments:
-     *
-     * Arg0 - Integer:
-     *  Operation region space ID Same value as RegionObj->Region.SpaceId
-     *
-     * Arg1 - Integer:
-     *  connection status 1 for connecting the handler, 0 for disconnecting
-     *  the handler (Passed as a parameter)
-     */
-    Args[0] = AcpiUtCreateIntegerObject ((UINT64) RegionObj->Region.SpaceId);
-    if (!Args[0])
-    {
-        Status = AE_NO_MEMORY;
-        goto Cleanup1;
-    }
-
-    Args[1] = AcpiUtCreateIntegerObject ((UINT64) Function);
-    if (!Args[1])
-    {
-        Status = AE_NO_MEMORY;
-        goto Cleanup2;
-    }
-
-    Args[2] = NULL; /* Terminate list */
-
-    /* Execute the method, no return value */
-
-    ACPI_DEBUG_EXEC (
-        AcpiUtDisplayInitPathname (ACPI_TYPE_METHOD, Info->PrefixNode, NULL));
-
-    Status = AcpiNsEvaluate (Info);
-    AcpiUtRemoveReference (Args[1]);
-
-Cleanup2:
-    AcpiUtRemoveReference (Args[0]);
-
-Cleanup1:
-    ACPI_FREE (Info);
-    return_ACPI_STATUS (Status);
-}
-
-
-/*******************************************************************************
- *
  * FUNCTION:    AcpiEvAddressSpaceDispatch
  *
  * PARAMETERS:  RegionObj           - Internal region object
@@ -762,377 +519,94 @@ AcpiEvAttachRegion (
 
 /*******************************************************************************
  *
- * FUNCTION:    AcpiEvInstallHandler
- *
- * PARAMETERS:  WalkNamespace callback
- *
- * DESCRIPTION: This routine installs an address handler into objects that are
- *              of type Region or Device.
- *
- *              If the Object is a Device, and the device has a handler of
- *              the same type then the search is terminated in that branch.
- *
- *              This is because the existing handler is closer in proximity
- *              to any more regions than the one we are trying to install.
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-AcpiEvInstallHandler (
-    ACPI_HANDLE             ObjHandle,
-    UINT32                  Level,
-    void                    *Context,
-    void                    **ReturnValue)
-{
-    ACPI_OPERAND_OBJECT     *HandlerObj;
-    ACPI_OPERAND_OBJECT     *NextHandlerObj;
-    ACPI_OPERAND_OBJECT     *ObjDesc;
-    ACPI_NAMESPACE_NODE     *Node;
-    ACPI_STATUS             Status;
-
-
-    ACPI_FUNCTION_NAME (EvInstallHandler);
-
-
-    HandlerObj = (ACPI_OPERAND_OBJECT  *) Context;
-
-    /* Parameter validation */
-
-    if (!HandlerObj)
-    {
-        return (AE_OK);
-    }
-
-    /* Convert and validate the device handle */
-
-    Node = AcpiNsValidateHandle (ObjHandle);
-    if (!Node)
-    {
-        return (AE_BAD_PARAMETER);
-    }
-
-    /*
-     * We only care about regions and objects that are allowed to have
-     * address space handlers
-     */
-    if ((Node->Type != ACPI_TYPE_DEVICE) &&
-        (Node->Type != ACPI_TYPE_REGION) &&
-        (Node != AcpiGbl_RootNode))
-    {
-        return (AE_OK);
-    }
-
-    /* Check for an existing internal object */
-
-    ObjDesc = AcpiNsGetAttachedObject (Node);
-    if (!ObjDesc)
-    {
-        /* No object, just exit */
-
-        return (AE_OK);
-    }
-
-    /* Devices are handled different than regions */
-
-    if (ObjDesc->Common.Type == ACPI_TYPE_DEVICE)
-    {
-        /* Check if this Device already has a handler for this address space */
-
-        NextHandlerObj = ObjDesc->Device.Handler;
-        while (NextHandlerObj)
-        {
-            /* Found a handler, is it for the same address space? */
-
-            if (NextHandlerObj->AddressSpace.SpaceId ==
-                    HandlerObj->AddressSpace.SpaceId)
-            {
-                ACPI_DEBUG_PRINT ((ACPI_DB_OPREGION,
-                    "Found handler for region [%s] in device %p(%p) "
-                    "handler %p\n",
-                    AcpiUtGetRegionName (HandlerObj->AddressSpace.SpaceId),
-                    ObjDesc, NextHandlerObj, HandlerObj));
-
-                /*
-                 * Since the object we found it on was a device, then it
-                 * means that someone has already installed a handler for
-                 * the branch of the namespace from this device on. Just
-                 * bail out telling the walk routine to not traverse this
-                 * branch. This preserves the scoping rule for handlers.
-                 */
-                return (AE_CTRL_DEPTH);
-            }
-
-            /* Walk the linked list of handlers attached to this device */
-
-            NextHandlerObj = NextHandlerObj->AddressSpace.Next;
-        }
-
-        /*
-         * As long as the device didn't have a handler for this space we
-         * don't care about it. We just ignore it and proceed.
-         */
-        return (AE_OK);
-    }
-
-    /* Object is a Region */
-
-    if (ObjDesc->Region.SpaceId != HandlerObj->AddressSpace.SpaceId)
-    {
-        /* This region is for a different address space, just ignore it */
-
-        return (AE_OK);
-    }
-
-    /*
-     * Now we have a region and it is for the handler's address space type.
-     *
-     * First disconnect region for any previous handler (if any)
-     */
-    AcpiEvDetachRegion (ObjDesc, FALSE);
-
-    /* Connect the region to the new handler */
-
-    Status = AcpiEvAttachRegion (HandlerObj, ObjDesc, FALSE);
-    return (Status);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiEvInstallSpaceHandler
+ * FUNCTION:    AcpiEvExecuteRegMethod
  *
- * PARAMETERS:  Node            - Namespace node for the device
- *              SpaceId         - The address space ID
- *              Handler         - Address of the handler
- *              Setup           - Address of the setup function
- *              Context         - Value passed to the handler on each access
+ * PARAMETERS:  RegionObj           - Region object
+ *              Function            - Passed to _REG: On (1) or Off (0)
  *
  * RETURN:      Status
  *
- * DESCRIPTION: Install a handler for all OpRegions of a given SpaceId.
- *              Assumes namespace is locked
+ * DESCRIPTION: Execute _REG method for a region
  *
  ******************************************************************************/
 
 ACPI_STATUS
-AcpiEvInstallSpaceHandler (
-    ACPI_NAMESPACE_NODE     *Node,
-    ACPI_ADR_SPACE_TYPE     SpaceId,
-    ACPI_ADR_SPACE_HANDLER  Handler,
-    ACPI_ADR_SPACE_SETUP    Setup,
-    void                    *Context)
+AcpiEvExecuteRegMethod (
+    ACPI_OPERAND_OBJECT     *RegionObj,
+    UINT32                  Function)
 {
-    ACPI_OPERAND_OBJECT     *ObjDesc;
-    ACPI_OPERAND_OBJECT     *HandlerObj;
+    ACPI_EVALUATE_INFO      *Info;
+    ACPI_OPERAND_OBJECT     *Args[3];
+    ACPI_OPERAND_OBJECT     *RegionObj2;
     ACPI_STATUS             Status;
-    ACPI_OBJECT_TYPE        Type;
-    UINT8                  Flags = 0;
-
 
-    ACPI_FUNCTION_TRACE (EvInstallSpaceHandler);
 
+    ACPI_FUNCTION_TRACE (EvExecuteRegMethod);
 
-    /*
-     * This registration is valid for only the types below and the root. This
-     * is where the default handlers get placed.
-     */
-    if ((Node->Type != ACPI_TYPE_DEVICE)     &&
-        (Node->Type != ACPI_TYPE_PROCESSOR)  &&
-        (Node->Type != ACPI_TYPE_THERMAL)    &&
-        (Node != AcpiGbl_RootNode))
-    {
-        Status = AE_BAD_PARAMETER;
-        goto UnlockAndExit;
-    }
 
-    if (Handler == ACPI_DEFAULT_HANDLER)
+    RegionObj2 = AcpiNsGetSecondaryObject (RegionObj);
+    if (!RegionObj2)
     {
-        Flags = ACPI_ADDR_HANDLER_DEFAULT_INSTALLED;
-
-        switch (SpaceId)
-        {
-        case ACPI_ADR_SPACE_SYSTEM_MEMORY:
-            Handler = AcpiExSystemMemorySpaceHandler;
-            Setup   = AcpiEvSystemMemoryRegionSetup;
-            break;
-
-        case ACPI_ADR_SPACE_SYSTEM_IO:
-            Handler = AcpiExSystemIoSpaceHandler;
-            Setup   = AcpiEvIoSpaceRegionSetup;
-            break;
-
-        case ACPI_ADR_SPACE_PCI_CONFIG:
-            Handler = AcpiExPciConfigSpaceHandler;
-            Setup   = AcpiEvPciConfigRegionSetup;
-            break;
-
-        case ACPI_ADR_SPACE_CMOS:
-            Handler = AcpiExCmosSpaceHandler;
-            Setup   = AcpiEvCmosRegionSetup;
-            break;
-
-        case ACPI_ADR_SPACE_PCI_BAR_TARGET:
-            Handler = AcpiExPciBarSpaceHandler;
-            Setup   = AcpiEvPciBarRegionSetup;
-            break;
-
-        case ACPI_ADR_SPACE_DATA_TABLE:
-            Handler = AcpiExDataTableSpaceHandler;
-            Setup   = NULL;
-            break;
-
-        default:
-            Status = AE_BAD_PARAMETER;
-            goto UnlockAndExit;
-        }
+        return_ACPI_STATUS (AE_NOT_EXIST);
     }
 
-    /* If the caller hasn't specified a setup routine, use the default */
-
-    if (!Setup)
+    if (RegionObj2->Extra.Method_REG == NULL)
     {
-        Setup = AcpiEvDefaultRegionSetup;
+        return_ACPI_STATUS (AE_OK);
     }
 
-    /* Check for an existing internal object */
-
-    ObjDesc = AcpiNsGetAttachedObject (Node);
-    if (ObjDesc)
-    {
-        /*
-         * The attached device object already exists. Make sure the handler
-         * is not already installed.
-         */
-        HandlerObj = ObjDesc->Device.Handler;
-
-        /* Walk the handler list for this device */
-
-        while (HandlerObj)
-        {
-            /* Same SpaceId indicates a handler already installed */
-
-            if (HandlerObj->AddressSpace.SpaceId == SpaceId)
-            {
-                if (HandlerObj->AddressSpace.Handler == Handler)
-                {
-                    /*
-                     * It is (relatively) OK to attempt to install the SAME
-                     * handler twice. This can easily happen with the
-                     * PCI_Config space.
-                     */
-                    Status = AE_SAME_HANDLER;
-                    goto UnlockAndExit;
-                }
-                else
-                {
-                    /* A handler is already installed */
-
-                    Status = AE_ALREADY_EXISTS;
-                }
-                goto UnlockAndExit;
-            }
-
-            /* Walk the linked list of handlers */
+    /* Allocate and initialize the evaluation information block */
 
-            HandlerObj = HandlerObj->AddressSpace.Next;
-        }
-    }
-    else
+    Info = ACPI_ALLOCATE_ZEROED (sizeof (ACPI_EVALUATE_INFO));
+    if (!Info)
     {
-        ACPI_DEBUG_PRINT ((ACPI_DB_OPREGION,
-            "Creating object on Device %p while installing handler\n", Node));
-
-        /* ObjDesc does not exist, create one */
-
-        if (Node->Type == ACPI_TYPE_ANY)
-        {
-            Type = ACPI_TYPE_DEVICE;
-        }
-        else
-        {
-            Type = Node->Type;
-        }
-
-        ObjDesc = AcpiUtCreateInternalObject (Type);
-        if (!ObjDesc)
-        {
-            Status = AE_NO_MEMORY;
-            goto UnlockAndExit;
-        }
-
-        /* Init new descriptor */
-
-        ObjDesc->Common.Type = (UINT8) Type;
-
-        /* Attach the new object to the Node */
-
-        Status = AcpiNsAttachObject (Node, ObjDesc, Type);
-
-        /* Remove local reference to the object */
-
-        AcpiUtRemoveReference (ObjDesc);
-
-        if (ACPI_FAILURE (Status))
-        {
-            goto UnlockAndExit;
-        }
+        return_ACPI_STATUS (AE_NO_MEMORY);
     }
 
-    ACPI_DEBUG_PRINT ((ACPI_DB_OPREGION,
-        "Installing address handler for region %s(%X) on Device %4.4s %p(%p)\n",
-        AcpiUtGetRegionName (SpaceId), SpaceId,
-        AcpiUtGetNodeName (Node), Node, ObjDesc));
+    Info->PrefixNode = RegionObj2->Extra.Method_REG;
+    Info->Pathname = NULL;
+    Info->Parameters = Args;
+    Info->Flags = ACPI_IGNORE_RETURN_VALUE;
 
     /*
-     * Install the handler
+     * The _REG method has two arguments:
+     *
+     * Arg0 - Integer:
+     *  Operation region space ID Same value as RegionObj->Region.SpaceId
      *
-     * At this point there is no existing handler. Just allocate the object
-     * for the handler and link it into the list.
+     * Arg1 - Integer:
+     *  connection status 1 for connecting the handler, 0 for disconnecting
+     *  the handler (Passed as a parameter)
      */
-    HandlerObj = AcpiUtCreateInternalObject (ACPI_TYPE_LOCAL_ADDRESS_HANDLER);
-    if (!HandlerObj)
+    Args[0] = AcpiUtCreateIntegerObject ((UINT64) RegionObj->Region.SpaceId);
+    if (!Args[0])
     {
         Status = AE_NO_MEMORY;
-        goto UnlockAndExit;
+        goto Cleanup1;
     }
 
-    /* Init handler obj */
+    Args[1] = AcpiUtCreateIntegerObject ((UINT64) Function);
+    if (!Args[1])
+    {
+        Status = AE_NO_MEMORY;
+        goto Cleanup2;
+    }
 
-    HandlerObj->AddressSpace.SpaceId = (UINT8) SpaceId;
-    HandlerObj->AddressSpace.HandlerFlags = Flags;
-    HandlerObj->AddressSpace.RegionList = NULL;
-    HandlerObj->AddressSpace.Node = Node;
-    HandlerObj->AddressSpace.Handler = Handler;
-    HandlerObj->AddressSpace.Context = Context;
-    HandlerObj->AddressSpace.Setup  = Setup;
+    Args[2] = NULL; /* Terminate list */
 
-    /* Install at head of Device.AddressSpace list */
+    /* Execute the method, no return value */
 
-    HandlerObj->AddressSpace.Next = ObjDesc->Device.Handler;
+    ACPI_DEBUG_EXEC (
+        AcpiUtDisplayInitPathname (ACPI_TYPE_METHOD, Info->PrefixNode, NULL));
 
-    /*
-     * The Device object is the first reference on the HandlerObj.
-     * Each region that uses the handler adds a reference.
-     */
-    ObjDesc->Device.Handler = HandlerObj;
+    Status = AcpiNsEvaluate (Info);
+    AcpiUtRemoveReference (Args[1]);
 
-    /*
-     * Walk the namespace finding all of the regions this
-     * handler will manage.
-     *
-     * Start at the device and search the branch toward
-     * the leaf nodes until either the leaf is encountered or
-     * a device is detected that has an address handler of the
-     * same type.
-     *
-     * In either case, back up and search down the remainder
-     * of the branch
-     */
-    Status = AcpiNsWalkNamespace (ACPI_TYPE_ANY, Node, ACPI_UINT32_MAX,
-                ACPI_NS_WALK_UNLOCK, AcpiEvInstallHandler, NULL,
-                HandlerObj, NULL);
+Cleanup2:
+    AcpiUtRemoveReference (Args[0]);
 
-UnlockAndExit:
+Cleanup1:
+    ACPI_FREE (Info);
     return_ACPI_STATUS (Status);
 }
 
diff --git a/sys/contrib/dev/acpica/components/events/evrgnini.c b/sys/contrib/dev/acpica/components/events/evrgnini.c
index 28c367f..90a6b02 100644
--- a/sys/contrib/dev/acpica/components/events/evrgnini.c
+++ b/sys/contrib/dev/acpica/components/events/evrgnini.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/events/evsci.c b/sys/contrib/dev/acpica/components/events/evsci.c
index 0e7a65a..340c400 100644
--- a/sys/contrib/dev/acpica/components/events/evsci.c
+++ b/sys/contrib/dev/acpica/components/events/evsci.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -101,7 +101,7 @@ AcpiEvSciXruptHandler (
     InterruptHandled |= AcpiEvGpeDetect (GpeXruptList);
 
     AcpiSciCount++;
-    return_UINT32 (InterruptHandled);
+    return_VALUE (InterruptHandled);
 }
 
 
@@ -137,7 +137,7 @@ AcpiEvGpeXruptHandler (
 
     InterruptHandled |= AcpiEvGpeDetect (GpeXruptList);
 
-    return_UINT32 (InterruptHandled);
+    return_VALUE (InterruptHandled);
 }
 
 
diff --git a/sys/contrib/dev/acpica/components/events/evxface.c b/sys/contrib/dev/acpica/components/events/evxface.c
index 4027dce..dfc4790 100644
--- a/sys/contrib/dev/acpica/components/events/evxface.c
+++ b/sys/contrib/dev/acpica/components/events/evxface.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/events/evxfevnt.c b/sys/contrib/dev/acpica/components/events/evxfevnt.c
index 4e87cde..e0f36e2 100644
--- a/sys/contrib/dev/acpica/components/events/evxfevnt.c
+++ b/sys/contrib/dev/acpica/components/events/evxfevnt.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/events/evxfgpe.c b/sys/contrib/dev/acpica/components/events/evxfgpe.c
index a07751c..c574b79 100644
--- a/sys/contrib/dev/acpica/components/events/evxfgpe.c
+++ b/sys/contrib/dev/acpica/components/events/evxfgpe.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/events/evxfregn.c b/sys/contrib/dev/acpica/components/events/evxfregn.c
index 59238ef..95e598b 100644
--- a/sys/contrib/dev/acpica/components/events/evxfregn.c
+++ b/sys/contrib/dev/acpica/components/events/evxfregn.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exconfig.c b/sys/contrib/dev/acpica/components/executer/exconfig.c
index 99848a9..fa85344 100644
--- a/sys/contrib/dev/acpica/components/executer/exconfig.c
+++ b/sys/contrib/dev/acpica/components/executer/exconfig.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -50,6 +50,7 @@
 #include <contrib/dev/acpica/include/actables.h>
 #include <contrib/dev/acpica/include/acdispat.h>
 #include <contrib/dev/acpica/include/acevents.h>
+#include <contrib/dev/acpica/include/amlcode.h>
 
 
 #define _COMPONENT          ACPI_EXECUTER
@@ -185,14 +186,15 @@ AcpiExLoadTableOp (
         (Operand[1]->String.Length > ACPI_OEM_ID_SIZE) ||
         (Operand[2]->String.Length > ACPI_OEM_TABLE_ID_SIZE))
     {
-        return_ACPI_STATUS (AE_BAD_PARAMETER);
+        return_ACPI_STATUS (AE_AML_STRING_LIMIT);
     }
 
     /* Find the ACPI table in the RSDT/XSDT */
 
-    Status = AcpiTbFindTable (Operand[0]->String.Pointer,
-                              Operand[1]->String.Pointer,
-                              Operand[2]->String.Pointer, &TableIndex);
+    Status = AcpiTbFindTable (
+                Operand[0]->String.Pointer,
+                Operand[1]->String.Pointer,
+                Operand[2]->String.Pointer, &TableIndex);
     if (ACPI_FAILURE (Status))
     {
         if (Status != AE_NOT_FOUND)
@@ -237,8 +239,8 @@ AcpiExLoadTableOp (
 
     if (Operand[4]->String.Length > 0)
     {
-        if ((Operand[4]->String.Pointer[0] != '\\') &&
-            (Operand[4]->String.Pointer[0] != '^'))
+        if ((Operand[4]->String.Pointer[0] != AML_ROOT_PREFIX) &&
+            (Operand[4]->String.Pointer[0] != AML_PARENT_PREFIX))
         {
             /*
              * Path is not absolute, so it will be relative to the node
@@ -299,7 +301,7 @@ AcpiExLoadTableOp (
     }
 
     *ReturnDesc = DdbHandle;
-    return_ACPI_STATUS  (Status);
+    return_ACPI_STATUS (Status);
 }
 
 
diff --git a/sys/contrib/dev/acpica/components/executer/exconvrt.c b/sys/contrib/dev/acpica/components/executer/exconvrt.c
index c34f326..f185371 100644
--- a/sys/contrib/dev/acpica/components/executer/exconvrt.c
+++ b/sys/contrib/dev/acpica/components/executer/exconvrt.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -199,7 +199,7 @@ AcpiExConvertToInteger (
 
     /* Save the Result */
 
-    AcpiExTruncateFor32bitTable (ReturnDesc);
+    (void) AcpiExTruncateFor32bitTable (ReturnDesc);
     *ResultDesc = ReturnDesc;
     return_ACPI_STATUS (AE_OK);
 }
diff --git a/sys/contrib/dev/acpica/components/executer/excreate.c b/sys/contrib/dev/acpica/components/executer/excreate.c
index b409633..1851957 100644
--- a/sys/contrib/dev/acpica/components/executer/excreate.c
+++ b/sys/contrib/dev/acpica/components/executer/excreate.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exdebug.c b/sys/contrib/dev/acpica/components/executer/exdebug.c
index 600118b..926f8e2 100644
--- a/sys/contrib/dev/acpica/components/executer/exdebug.c
+++ b/sys/contrib/dev/acpica/components/executer/exdebug.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exdump.c b/sys/contrib/dev/acpica/components/executer/exdump.c
index ec648f3..5d1527b 100644
--- a/sys/contrib/dev/acpica/components/executer/exdump.c
+++ b/sys/contrib/dev/acpica/components/executer/exdump.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -483,7 +483,9 @@ AcpiExDumpOperand (
     ACPI_FUNCTION_NAME (ExDumpOperand)
 
 
-    if (!((ACPI_LV_EXEC & AcpiDbgLevel) && (_COMPONENT & AcpiDbgLayer)))
+    /* Check if debug output enabled */
+
+    if (!ACPI_IS_DEBUG_ENABLED (ACPI_LV_EXEC, _COMPONENT))
     {
         return;
     }
@@ -873,7 +875,9 @@ AcpiExDumpNamespaceNode (
 
     if (!Flags)
     {
-        if (!((ACPI_LV_OBJECTS & AcpiDbgLevel) && (_COMPONENT & AcpiDbgLayer)))
+        /* Check if debug output enabled */
+
+        if (!ACPI_IS_DEBUG_ENABLED (ACPI_LV_OBJECTS, _COMPONENT))
         {
             return;
         }
@@ -1080,7 +1084,9 @@ AcpiExDumpObjectDescriptor (
 
     if (!Flags)
     {
-        if (!((ACPI_LV_OBJECTS & AcpiDbgLevel) && (_COMPONENT & AcpiDbgLayer)))
+        /* Check if debug output enabled */
+
+        if (!ACPI_IS_DEBUG_ENABLED (ACPI_LV_OBJECTS, _COMPONENT))
         {
             return_VOID;
         }
diff --git a/sys/contrib/dev/acpica/components/executer/exfield.c b/sys/contrib/dev/acpica/components/executer/exfield.c
index 4a7fa24..bade0f8 100644
--- a/sys/contrib/dev/acpica/components/executer/exfield.c
+++ b/sys/contrib/dev/acpica/components/executer/exfield.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exfldio.c b/sys/contrib/dev/acpica/components/executer/exfldio.c
index c8c3a33..f735aec 100644
--- a/sys/contrib/dev/acpica/components/executer/exfldio.c
+++ b/sys/contrib/dev/acpica/components/executer/exfldio.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exmisc.c b/sys/contrib/dev/acpica/components/executer/exmisc.c
index cf32b76..93f6487 100644
--- a/sys/contrib/dev/acpica/components/executer/exmisc.c
+++ b/sys/contrib/dev/acpica/components/executer/exmisc.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exmutex.c b/sys/contrib/dev/acpica/components/executer/exmutex.c
index c291449..a6e5e14 100644
--- a/sys/contrib/dev/acpica/components/executer/exmutex.c
+++ b/sys/contrib/dev/acpica/components/executer/exmutex.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exnames.c b/sys/contrib/dev/acpica/components/executer/exnames.c
index ce7fd66..16d89e3 100644
--- a/sys/contrib/dev/acpica/components/executer/exnames.c
+++ b/sys/contrib/dev/acpica/components/executer/exnames.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exoparg1.c b/sys/contrib/dev/acpica/components/executer/exoparg1.c
index ac0860b..8ce02de 100644
--- a/sys/contrib/dev/acpica/components/executer/exoparg1.c
+++ b/sys/contrib/dev/acpica/components/executer/exoparg1.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exoparg2.c b/sys/contrib/dev/acpica/components/executer/exoparg2.c
index 6e6e0d9..55defdc 100644
--- a/sys/contrib/dev/acpica/components/executer/exoparg2.c
+++ b/sys/contrib/dev/acpica/components/executer/exoparg2.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exoparg3.c b/sys/contrib/dev/acpica/components/executer/exoparg3.c
index 33aa6fa..5823675 100644
--- a/sys/contrib/dev/acpica/components/executer/exoparg3.c
+++ b/sys/contrib/dev/acpica/components/executer/exoparg3.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exoparg6.c b/sys/contrib/dev/acpica/components/executer/exoparg6.c
index 3677b80..41201f3 100644
--- a/sys/contrib/dev/acpica/components/executer/exoparg6.c
+++ b/sys/contrib/dev/acpica/components/executer/exoparg6.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exprep.c b/sys/contrib/dev/acpica/components/executer/exprep.c
index adf67dc..97fe872 100644
--- a/sys/contrib/dev/acpica/components/executer/exprep.c
+++ b/sys/contrib/dev/acpica/components/executer/exprep.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -292,7 +292,7 @@ AcpiExDecodeFieldAccess (
         ACPI_ERROR ((AE_INFO,
             "Unknown field access type 0x%X",
             Access));
-        return_UINT32 (0);
+        return_VALUE (0);
     }
 
     if (ObjDesc->Common.Type == ACPI_TYPE_BUFFER_FIELD)
@@ -306,7 +306,7 @@ AcpiExDecodeFieldAccess (
     }
 
     *ReturnByteAlignment = ByteAlignment;
-    return_UINT32 (BitLength);
+    return_VALUE (BitLength);
 }
 
 
diff --git a/sys/contrib/dev/acpica/components/executer/exregion.c b/sys/contrib/dev/acpica/components/executer/exregion.c
index abbbd84..bc3417a 100644
--- a/sys/contrib/dev/acpica/components/executer/exregion.c
+++ b/sys/contrib/dev/acpica/components/executer/exregion.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exresnte.c b/sys/contrib/dev/acpica/components/executer/exresnte.c
index 43819a9..c8937ec 100644
--- a/sys/contrib/dev/acpica/components/executer/exresnte.c
+++ b/sys/contrib/dev/acpica/components/executer/exresnte.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exresolv.c b/sys/contrib/dev/acpica/components/executer/exresolv.c
index bebf170..74a1b73 100644
--- a/sys/contrib/dev/acpica/components/executer/exresolv.c
+++ b/sys/contrib/dev/acpica/components/executer/exresolv.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exresop.c b/sys/contrib/dev/acpica/components/executer/exresop.c
index d46c2d4..0b7af01 100644
--- a/sys/contrib/dev/acpica/components/executer/exresop.c
+++ b/sys/contrib/dev/acpica/components/executer/exresop.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exstore.c b/sys/contrib/dev/acpica/components/executer/exstore.c
index ab07400..ae842d2 100644
--- a/sys/contrib/dev/acpica/components/executer/exstore.c
+++ b/sys/contrib/dev/acpica/components/executer/exstore.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -515,13 +515,28 @@ AcpiExStoreObjectToNode (
     default:
 
         ACPI_DEBUG_PRINT ((ACPI_DB_EXEC,
-            "Storing %s (%p) directly into node (%p) with no implicit conversion\n",
-            AcpiUtGetObjectTypeName (SourceDesc), SourceDesc, Node));
+            "Storing [%s] (%p) directly into node [%s] (%p)"
+            " with no implicit conversion\n",
+            AcpiUtGetObjectTypeName (SourceDesc), SourceDesc,
+            AcpiUtGetObjectTypeName (TargetDesc), Node));
 
-        /* No conversions for all other types. Just attach the source object */
+        /*
+         * No conversions for all other types. Directly store a copy of
+         * the source object. NOTE: This is a departure from the ACPI
+         * spec, which states "If conversion is impossible, abort the
+         * running control method".
+         *
+         * This code implements "If conversion is impossible, treat the
+         * Store operation as a CopyObject".
+         */
+        Status = AcpiUtCopyIobjectToIobject (SourceDesc, &NewDesc, WalkState);
+        if (ACPI_FAILURE (Status))
+        {
+            return_ACPI_STATUS (Status);
+        }
 
-        Status = AcpiNsAttachObject (Node, SourceDesc,
-                    SourceDesc->Common.Type);
+        Status = AcpiNsAttachObject (Node, NewDesc, NewDesc->Common.Type);
+        AcpiUtRemoveReference (NewDesc);
         break;
     }
 
diff --git a/sys/contrib/dev/acpica/components/executer/exstoren.c b/sys/contrib/dev/acpica/components/executer/exstoren.c
index 0848e4d..a8ae148 100644
--- a/sys/contrib/dev/acpica/components/executer/exstoren.c
+++ b/sys/contrib/dev/acpica/components/executer/exstoren.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -269,7 +269,7 @@ AcpiExStoreObjectToObject (
 
         /* Truncate value if we are executing from a 32-bit ACPI table */
 
-        AcpiExTruncateFor32bitTable (DestDesc);
+        (void) AcpiExTruncateFor32bitTable (DestDesc);
         break;
 
     case ACPI_TYPE_STRING:
diff --git a/sys/contrib/dev/acpica/components/executer/exstorob.c b/sys/contrib/dev/acpica/components/executer/exstorob.c
index 8cf9de5..c68263c 100644
--- a/sys/contrib/dev/acpica/components/executer/exstorob.c
+++ b/sys/contrib/dev/acpica/components/executer/exstorob.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exsystem.c b/sys/contrib/dev/acpica/components/executer/exsystem.c
index 89fe6be..cc18d80 100644
--- a/sys/contrib/dev/acpica/components/executer/exsystem.c
+++ b/sys/contrib/dev/acpica/components/executer/exsystem.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/executer/exutils.c b/sys/contrib/dev/acpica/components/executer/exutils.c
index f1e9435..fb78123 100644
--- a/sys/contrib/dev/acpica/components/executer/exutils.c
+++ b/sys/contrib/dev/acpica/components/executer/exutils.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -229,14 +229,14 @@ AcpiExRelinquishInterpreter (
  *
  * PARAMETERS:  ObjDesc         - Object to be truncated
  *
- * RETURN:      none
+ * RETURN:      TRUE if a truncation was performed, FALSE otherwise.
  *
  * DESCRIPTION: Truncate an ACPI Integer to 32 bits if the execution mode is
  *              32-bit, as determined by the revision of the DSDT.
  *
  ******************************************************************************/
 
-void
+BOOLEAN
 AcpiExTruncateFor32bitTable (
     ACPI_OPERAND_OBJECT     *ObjDesc)
 {
@@ -246,23 +246,27 @@ AcpiExTruncateFor32bitTable (
 
     /*
      * Object must be a valid number and we must be executing
-     * a control method. NS node could be there for AML_INT_NAMEPATH_OP.
+     * a control method. Object could be NS node for AML_INT_NAMEPATH_OP.
      */
     if ((!ObjDesc) ||
         (ACPI_GET_DESCRIPTOR_TYPE (ObjDesc) != ACPI_DESC_TYPE_OPERAND) ||
         (ObjDesc->Common.Type != ACPI_TYPE_INTEGER))
     {
-        return;
+        return (FALSE);
     }
 
-    if (AcpiGbl_IntegerByteWidth == 4)
+    if ((AcpiGbl_IntegerByteWidth == 4) &&
+        (ObjDesc->Integer.Value > (UINT64) ACPI_UINT32_MAX))
     {
         /*
-         * We are running a method that exists in a 32-bit ACPI table.
+         * We are executing in a 32-bit ACPI table.
          * Truncate the value to 32 bits by zeroing out the upper 32-bit field
          */
         ObjDesc->Integer.Value &= (UINT64) ACPI_UINT32_MAX;
+        return (TRUE);
     }
+
+    return (FALSE);
 }
 
 
@@ -387,7 +391,7 @@ AcpiExDigitsNeeded (
 
     if (Value == 0)
     {
-        return_UINT32 (1);
+        return_VALUE (1);
     }
 
     CurrentValue = Value;
@@ -401,7 +405,7 @@ AcpiExDigitsNeeded (
         NumDigits++;
     }
 
-    return_UINT32 (NumDigits);
+    return_VALUE (NumDigits);
 }
 
 
diff --git a/sys/contrib/dev/acpica/components/hardware/hwacpi.c b/sys/contrib/dev/acpica/components/hardware/hwacpi.c
index e667a47..41830c3 100644
--- a/sys/contrib/dev/acpica/components/hardware/hwacpi.c
+++ b/sys/contrib/dev/acpica/components/hardware/hwacpi.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -141,13 +141,13 @@ AcpiHwSetMode (
     Retry = 3000;
     while (Retry)
     {
-        if (AcpiHwGetMode() == Mode)
+        if (AcpiHwGetMode () == Mode)
         {
             ACPI_DEBUG_PRINT ((ACPI_DB_INFO, "Mode %X successfully enabled\n",
                 Mode));
             return_ACPI_STATUS (AE_OK);
         }
-        AcpiOsStall(1000);
+        AcpiOsStall (ACPI_USEC_PER_MSEC);
         Retry--;
     }
 
@@ -186,22 +186,22 @@ AcpiHwGetMode (
      */
     if (!AcpiGbl_FADT.SmiCommand)
     {
-        return_UINT32 (ACPI_SYS_MODE_ACPI);
+        return_VALUE (ACPI_SYS_MODE_ACPI);
     }
 
     Status = AcpiReadBitRegister (ACPI_BITREG_SCI_ENABLE, &Value);
     if (ACPI_FAILURE (Status))
     {
-        return_UINT32 (ACPI_SYS_MODE_LEGACY);
+        return_VALUE (ACPI_SYS_MODE_LEGACY);
     }
 
     if (Value)
     {
-        return_UINT32 (ACPI_SYS_MODE_ACPI);
+        return_VALUE (ACPI_SYS_MODE_ACPI);
     }
     else
     {
-        return_UINT32 (ACPI_SYS_MODE_LEGACY);
+        return_VALUE (ACPI_SYS_MODE_LEGACY);
     }
 }
 
diff --git a/sys/contrib/dev/acpica/components/hardware/hwesleep.c b/sys/contrib/dev/acpica/components/hardware/hwesleep.c
index 4a9b7ec..6717801 100644
--- a/sys/contrib/dev/acpica/components/hardware/hwesleep.c
+++ b/sys/contrib/dev/acpica/components/hardware/hwesleep.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/hardware/hwgpe.c b/sys/contrib/dev/acpica/components/hardware/hwgpe.c
index afb5f87..c08ce2e 100644
--- a/sys/contrib/dev/acpica/components/hardware/hwgpe.c
+++ b/sys/contrib/dev/acpica/components/hardware/hwgpe.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -150,7 +150,7 @@ AcpiHwLowSetGpe (
         break;
 
     default:
-        ACPI_ERROR ((AE_INFO, "Invalid GPE Action, %u\n", Action));
+        ACPI_ERROR ((AE_INFO, "Invalid GPE Action, %u", Action));
         return (AE_BAD_PARAMETER);
     }
 
diff --git a/sys/contrib/dev/acpica/components/hardware/hwpci.c b/sys/contrib/dev/acpica/components/hardware/hwpci.c
index 2b3ef1c..a89a3dd 100644
--- a/sys/contrib/dev/acpica/components/hardware/hwpci.c
+++ b/sys/contrib/dev/acpica/components/hardware/hwpci.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/hardware/hwregs.c b/sys/contrib/dev/acpica/components/hardware/hwregs.c
index c1440f4..8a7775b 100644
--- a/sys/contrib/dev/acpica/components/hardware/hwregs.c
+++ b/sys/contrib/dev/acpica/components/hardware/hwregs.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/hardware/hwsleep.c b/sys/contrib/dev/acpica/components/hardware/hwsleep.c
index 6da9e5a..0e6691a 100644
--- a/sys/contrib/dev/acpica/components/hardware/hwsleep.c
+++ b/sys/contrib/dev/acpica/components/hardware/hwsleep.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -180,7 +180,7 @@ AcpiHwLegacySleep (
          * to still read the right value. Ideally, this block would go
          * away entirely.
          */
-        AcpiOsStall (10000000);
+        AcpiOsStall (10 * ACPI_USEC_PER_SEC);
 
         Status = AcpiHwRegisterWrite (ACPI_REGISTER_PM1_CONTROL,
                     SleepEnableRegInfo->AccessBitMask);
diff --git a/sys/contrib/dev/acpica/components/hardware/hwtimer.c b/sys/contrib/dev/acpica/components/hardware/hwtimer.c
index ea8f954..1a13f64 100644
--- a/sys/contrib/dev/acpica/components/hardware/hwtimer.c
+++ b/sys/contrib/dev/acpica/components/hardware/hwtimer.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -203,10 +203,11 @@ AcpiGetTimerDuration (
     /*
      * Compute Duration (Requires a 64-bit multiply and divide):
      *
-     * TimeElapsed = (DeltaTicks * 1000000) / PM_TIMER_FREQUENCY;
+     * TimeElapsed (microseconds) =
+     *  (DeltaTicks * ACPI_USEC_PER_SEC) / ACPI_PM_TIMER_FREQUENCY;
      */
-    Status = AcpiUtShortDivide (((UINT64) DeltaTicks) * 1000000,
-                PM_TIMER_FREQUENCY, &Quotient, NULL);
+    Status = AcpiUtShortDivide (((UINT64) DeltaTicks) * ACPI_USEC_PER_SEC,
+                ACPI_PM_TIMER_FREQUENCY, &Quotient, NULL);
 
     *TimeElapsed = (UINT32) Quotient;
     return_ACPI_STATUS (Status);
diff --git a/sys/contrib/dev/acpica/components/hardware/hwvalid.c b/sys/contrib/dev/acpica/components/hardware/hwvalid.c
index c1487d7..85a87d1 100644
--- a/sys/contrib/dev/acpica/components/hardware/hwvalid.c
+++ b/sys/contrib/dev/acpica/components/hardware/hwvalid.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/hardware/hwxface.c b/sys/contrib/dev/acpica/components/hardware/hwxface.c
index d169728..25ed994 100644
--- a/sys/contrib/dev/acpica/components/hardware/hwxface.c
+++ b/sys/contrib/dev/acpica/components/hardware/hwxface.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -377,7 +377,7 @@ ACPI_EXPORT_SYMBOL (AcpiReadBitRegister)
  *
  * PARAMETERS:  RegisterId      - ID of ACPI Bit Register to access
  *              Value           - Value to write to the register, in bit
- *                                position zero. The bit is automaticallly
+ *                                position zero. The bit is automatically
  *                                shifted to the correct position.
  *
  * RETURN:      Status
@@ -492,10 +492,33 @@ ACPI_EXPORT_SYMBOL (AcpiWriteBitRegister)
  *              *SleepTypeA         - Where SLP_TYPa is returned
  *              *SleepTypeB         - Where SLP_TYPb is returned
  *
- * RETURN:      Status - ACPI status
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Obtain the SLP_TYPa and SLP_TYPb values for the requested
+ *              sleep state via the appropriate \_Sx object.
+ *
+ *  The sleep state package returned from the corresponding \_Sx_ object
+ *  must contain at least one integer.
+ *
+ *  March 2005:
+ *  Added support for a package that contains two integers. This
+ *  goes against the ACPI specification which defines this object as a
+ *  package with one encoded DWORD integer. However, existing practice
+ *  by many BIOS vendors is to return a package with 2 or more integer
+ *  elements, at least one per sleep type (A/B).
+ *
+ *  January 2013:
+ *  Therefore, we must be prepared to accept a package with either a
+ *  single integer or multiple integers.
  *
- * DESCRIPTION: Obtain the SLP_TYPa and SLP_TYPb values for the requested sleep
- *              state.
+ *  The single integer DWORD format is as follows:
+ *      BYTE 0 - Value for the PM1A SLP_TYP register
+ *      BYTE 1 - Value for the PM1B SLP_TYP register
+ *      BYTE 2-3 - Reserved
+ *
+ *  The dual integer format is as follows:
+ *      Integer 0 - Value for the PM1A SLP_TYP register
+ *      Integer 1 - Value for the PM1A SLP_TYP register
  *
  ******************************************************************************/
 
@@ -505,8 +528,9 @@ AcpiGetSleepTypeData (
     UINT8                   *SleepTypeA,
     UINT8                   *SleepTypeB)
 {
-    ACPI_STATUS             Status = AE_OK;
+    ACPI_STATUS             Status;
     ACPI_EVALUATE_INFO      *Info;
+    ACPI_OPERAND_OBJECT     **Elements;
 
 
     ACPI_FUNCTION_TRACE (AcpiGetSleepTypeData);
@@ -515,8 +539,7 @@ AcpiGetSleepTypeData (
     /* Validate parameters */
 
     if ((SleepState > ACPI_S_STATES_MAX) ||
-        !SleepTypeA ||
-        !SleepTypeB)
+        !SleepTypeA || !SleepTypeB)
     {
         return_ACPI_STATUS (AE_BAD_PARAMETER);
     }
@@ -529,17 +552,14 @@ AcpiGetSleepTypeData (
         return_ACPI_STATUS (AE_NO_MEMORY);
     }
 
+    /*
+     * Evaluate the \_Sx namespace object containing the register values
+     * for this state
+     */
     Info->Pathname = ACPI_CAST_PTR (char, AcpiGbl_SleepStateNames[SleepState]);
-
-    /* Evaluate the namespace object containing the values for this state */
-
     Status = AcpiNsEvaluate (Info);
     if (ACPI_FAILURE (Status))
     {
-        ACPI_DEBUG_PRINT ((ACPI_DB_EXEC,
-            "%s while evaluating SleepState [%s]\n",
-            AcpiFormatException (Status), Info->Pathname));
-
         goto Cleanup;
     }
 
@@ -549,66 +569,70 @@ AcpiGetSleepTypeData (
     {
         ACPI_ERROR ((AE_INFO, "No Sleep State object returned from [%s]",
             Info->Pathname));
-        Status = AE_NOT_EXIST;
+        Status = AE_AML_NO_RETURN_VALUE;
+        goto Cleanup;
     }
 
-    /* It must be of type Package */
+    /* Return object must be of type Package */
 
-    else if (Info->ReturnObject->Common.Type != ACPI_TYPE_PACKAGE)
+    if (Info->ReturnObject->Common.Type != ACPI_TYPE_PACKAGE)
     {
         ACPI_ERROR ((AE_INFO, "Sleep State return object is not a Package"));
         Status = AE_AML_OPERAND_TYPE;
+        goto Cleanup1;
     }
 
     /*
-     * The package must have at least two elements. NOTE (March 2005): This
-     * goes against the current ACPI spec which defines this object as a
-     * package with one encoded DWORD element. However, existing practice
-     * by BIOS vendors seems to be to have 2 or more elements, at least
-     * one per sleep type (A/B).
+     * Any warnings about the package length or the object types have
+     * already been issued by the predefined name module -- there is no
+     * need to repeat them here.
      */
-    else if (Info->ReturnObject->Package.Count < 2)
+    Elements = Info->ReturnObject->Package.Elements;
+    switch (Info->ReturnObject->Package.Count)
     {
-        ACPI_ERROR ((AE_INFO,
-            "Sleep State return package does not have at least two elements"));
-        Status = AE_AML_NO_OPERAND;
-    }
+    case 0:
+        Status = AE_AML_PACKAGE_LIMIT;
+        break;
 
-    /* The first two elements must both be of type Integer */
+    case 1:
+        if (Elements[0]->Common.Type != ACPI_TYPE_INTEGER)
+        {
+            Status = AE_AML_OPERAND_TYPE;
+            break;
+        }
 
-    else if (((Info->ReturnObject->Package.Elements[0])->Common.Type
-                != ACPI_TYPE_INTEGER) ||
-             ((Info->ReturnObject->Package.Elements[1])->Common.Type
-                != ACPI_TYPE_INTEGER))
-    {
-        ACPI_ERROR ((AE_INFO,
-            "Sleep State return package elements are not both Integers "
-            "(%s, %s)",
-            AcpiUtGetObjectTypeName (Info->ReturnObject->Package.Elements[0]),
-            AcpiUtGetObjectTypeName (Info->ReturnObject->Package.Elements[1])));
-        Status = AE_AML_OPERAND_TYPE;
-    }
-    else
-    {
-        /* Valid _Sx_ package size, type, and value */
+        /* A valid _Sx_ package with one integer */
+
+        *SleepTypeA = (UINT8) Elements[0]->Integer.Value;
+        *SleepTypeB = (UINT8) (Elements[0]->Integer.Value >> 8);
+        break;
+
+    case 2:
+    default:
+        if ((Elements[0]->Common.Type != ACPI_TYPE_INTEGER) ||
+            (Elements[1]->Common.Type != ACPI_TYPE_INTEGER))
+        {
+            Status = AE_AML_OPERAND_TYPE;
+            break;
+        }
+
+        /* A valid _Sx_ package with two integers */
 
-        *SleepTypeA = (UINT8)
-            (Info->ReturnObject->Package.Elements[0])->Integer.Value;
-        *SleepTypeB = (UINT8)
-            (Info->ReturnObject->Package.Elements[1])->Integer.Value;
+        *SleepTypeA = (UINT8) Elements[0]->Integer.Value;
+        *SleepTypeB = (UINT8) Elements[1]->Integer.Value;
+        break;
     }
 
+Cleanup1:
+    AcpiUtRemoveReference (Info->ReturnObject);
+
+Cleanup:
     if (ACPI_FAILURE (Status))
     {
         ACPI_EXCEPTION ((AE_INFO, Status,
-            "While evaluating SleepState [%s], bad Sleep object %p type %s",
-            Info->Pathname, Info->ReturnObject,
-            AcpiUtGetObjectTypeName (Info->ReturnObject)));
+            "While evaluating Sleep State [%s]", Info->Pathname));
     }
 
-    AcpiUtRemoveReference (Info->ReturnObject);
-
-Cleanup:
     ACPI_FREE (Info);
     return_ACPI_STATUS (Status);
 }
diff --git a/sys/contrib/dev/acpica/components/hardware/hwxfsleep.c b/sys/contrib/dev/acpica/components/hardware/hwxfsleep.c
index e467164..3643799 100644
--- a/sys/contrib/dev/acpica/components/hardware/hwxfsleep.c
+++ b/sys/contrib/dev/acpica/components/hardware/hwxfsleep.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -227,7 +227,7 @@ AcpiEnterSleepStateS4bios (
                 (UINT32) AcpiGbl_FADT.S4BiosRequest, 8);
 
     do {
-        AcpiOsStall(1000);
+        AcpiOsStall (ACPI_USEC_PER_MSEC);
         Status = AcpiReadBitRegister (ACPI_BITREG_WAKE_STATUS, &InValue);
         if (ACPI_FAILURE (Status))
         {
diff --git a/sys/contrib/dev/acpica/components/namespace/nsaccess.c b/sys/contrib/dev/acpica/components/namespace/nsaccess.c
index 246628c..8dc8b5f 100644
--- a/sys/contrib/dev/acpica/components/namespace/nsaccess.c
+++ b/sys/contrib/dev/acpica/components/namespace/nsaccess.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/namespace/nsalloc.c b/sys/contrib/dev/acpica/components/namespace/nsalloc.c
index c76f422..ecec172 100644
--- a/sys/contrib/dev/acpica/components/namespace/nsalloc.c
+++ b/sys/contrib/dev/acpica/components/namespace/nsalloc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/namespace/nsdump.c b/sys/contrib/dev/acpica/components/namespace/nsdump.c
index fd82cdb..1411534 100644
--- a/sys/contrib/dev/acpica/components/namespace/nsdump.c
+++ b/sys/contrib/dev/acpica/components/namespace/nsdump.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -46,6 +46,7 @@
 #include <contrib/dev/acpica/include/acpi.h>
 #include <contrib/dev/acpica/include/accommon.h>
 #include <contrib/dev/acpica/include/acnamesp.h>
+#include <contrib/dev/acpica/include/acoutput.h>
 
 
 #define _COMPONENT          ACPI_NAMESPACE
@@ -92,7 +93,9 @@ AcpiNsPrintPathname (
     ACPI_FUNCTION_NAME (NsPrintPathname);
 
 
-    if (!(AcpiDbgLevel & ACPI_LV_NAMES) || !(AcpiDbgLayer & ACPI_NAMESPACE))
+    /* Check if debug output enabled */
+
+    if (!ACPI_IS_DEBUG_ENABLED (ACPI_LV_NAMES, ACPI_NAMESPACE))
     {
         return;
     }
@@ -151,7 +154,7 @@ AcpiNsDumpPathname (
 
     /* Do this only if the requested debug level and component are enabled */
 
-    if (!(AcpiDbgLevel & Level) || !(AcpiDbgLayer & Component))
+    if (!ACPI_IS_DEBUG_ENABLED (Level, Component))
     {
         return_VOID;
     }
diff --git a/sys/contrib/dev/acpica/components/namespace/nsdumpdv.c b/sys/contrib/dev/acpica/components/namespace/nsdumpdv.c
index 4b93fbe..868b104 100644
--- a/sys/contrib/dev/acpica/components/namespace/nsdumpdv.c
+++ b/sys/contrib/dev/acpica/components/namespace/nsdumpdv.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/namespace/nseval.c b/sys/contrib/dev/acpica/components/namespace/nseval.c
index a7a3577..61e783c 100644
--- a/sys/contrib/dev/acpica/components/namespace/nseval.c
+++ b/sys/contrib/dev/acpica/components/namespace/nseval.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/namespace/nsinit.c b/sys/contrib/dev/acpica/components/namespace/nsinit.c
index be2c60d..e9c9781 100644
--- a/sys/contrib/dev/acpica/components/namespace/nsinit.c
+++ b/sys/contrib/dev/acpica/components/namespace/nsinit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -104,7 +104,7 @@ AcpiNsInitializeObjects (
     ACPI_DEBUG_PRINT ((ACPI_DB_DISPATCH,
         "**** Starting initialization of namespace objects ****\n"));
     ACPI_DEBUG_PRINT_RAW ((ACPI_DB_INIT,
-        "Completing Region/Field/Buffer/Package initialization:"));
+        "Completing Region/Field/Buffer/Package initialization:\n"));
 
     /* Set all init info to zero */
 
@@ -121,7 +121,7 @@ AcpiNsInitializeObjects (
     }
 
     ACPI_DEBUG_PRINT_RAW ((ACPI_DB_INIT,
-        "\nInitialized %u/%u Regions %u/%u Fields %u/%u "
+        "    Initialized %u/%u Regions %u/%u Fields %u/%u "
         "Buffers %u/%u Packages (%u nodes)\n",
         Info.OpRegionInit,  Info.OpRegionCount,
         Info.FieldInit,     Info.FieldCount,
@@ -172,7 +172,7 @@ AcpiNsInitializeDevices (
 
     ACPI_DEBUG_PRINT_RAW ((ACPI_DB_INIT,
         "Initializing Device/Processor/Thermal objects "
-        "by executing _INI methods:"));
+        "and executing _INI/_STA methods:\n"));
 
     /* Tree analysis: find all subtrees that contain _INI methods */
 
@@ -230,7 +230,7 @@ AcpiNsInitializeDevices (
     }
 
     ACPI_DEBUG_PRINT_RAW ((ACPI_DB_INIT,
-        "\nExecuted %u _INI methods requiring %u _STA executions "
+        "    Executed %u _INI methods requiring %u _STA executions "
         "(examined %u objects)\n",
         Info.Num_INI, Info.Num_STA, Info.DeviceCount));
 
@@ -381,15 +381,6 @@ AcpiNsInitOneObject (
     }
 
     /*
-     * Print a dot for each object unless we are going to print the entire
-     * pathname
-     */
-    if (!(AcpiDbgLevel & ACPI_LV_INIT_NAMES))
-    {
-        ACPI_DEBUG_PRINT_RAW ((ACPI_DB_INIT, "."));
-    }
-
-    /*
      * We ignore errors from above, and always return OK, since we don't want
      * to abort the walk on any single error.
      */
@@ -627,12 +618,6 @@ AcpiNsInitOneDevice (
     if (ACPI_SUCCESS (Status))
     {
         WalkInfo->Num_INI++;
-
-        if ((AcpiDbgLevel <= ACPI_LV_ALL_EXCEPTIONS) &&
-            (!(AcpiDbgLevel & ACPI_LV_INFO)))
-        {
-            ACPI_DEBUG_PRINT_RAW ((ACPI_DB_INIT, "."));
-        }
     }
 
 #ifdef ACPI_DEBUG_OUTPUT
diff --git a/sys/contrib/dev/acpica/components/namespace/nsload.c b/sys/contrib/dev/acpica/components/namespace/nsload.c
index 9cfa412..d29a891 100644
--- a/sys/contrib/dev/acpica/components/namespace/nsload.c
+++ b/sys/contrib/dev/acpica/components/namespace/nsload.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/namespace/nsnames.c b/sys/contrib/dev/acpica/components/namespace/nsnames.c
index cd2fc2c..cd4d440 100644
--- a/sys/contrib/dev/acpica/components/namespace/nsnames.c
+++ b/sys/contrib/dev/acpica/components/namespace/nsnames.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/namespace/nsobject.c b/sys/contrib/dev/acpica/components/namespace/nsobject.c
index 0920db3..b1d1f01 100644
--- a/sys/contrib/dev/acpica/components/namespace/nsobject.c
+++ b/sys/contrib/dev/acpica/components/namespace/nsobject.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/namespace/nsparse.c b/sys/contrib/dev/acpica/components/namespace/nsparse.c
index 231a440..f327f26 100644
--- a/sys/contrib/dev/acpica/components/namespace/nsparse.c
+++ b/sys/contrib/dev/acpica/components/namespace/nsparse.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/namespace/nspredef.c b/sys/contrib/dev/acpica/components/namespace/nspredef.c
index fd003fb..8c65b84 100644
--- a/sys/contrib/dev/acpica/components/namespace/nspredef.c
+++ b/sys/contrib/dev/acpica/components/namespace/nspredef.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -78,35 +78,6 @@
 /* Local prototypes */
 
 static ACPI_STATUS
-AcpiNsCheckPackage (
-    ACPI_PREDEFINED_DATA        *Data,
-    ACPI_OPERAND_OBJECT         **ReturnObjectPtr);
-
-static ACPI_STATUS
-AcpiNsCheckPackageList (
-    ACPI_PREDEFINED_DATA        *Data,
-    const ACPI_PREDEFINED_INFO  *Package,
-    ACPI_OPERAND_OBJECT         **Elements,
-    UINT32                      Count);
-
-static ACPI_STATUS
-AcpiNsCheckPackageElements (
-    ACPI_PREDEFINED_DATA        *Data,
-    ACPI_OPERAND_OBJECT         **Elements,
-    UINT8                       Type1,
-    UINT32                      Count1,
-    UINT8                       Type2,
-    UINT32                      Count2,
-    UINT32                      StartIndex);
-
-static ACPI_STATUS
-AcpiNsCheckObjectType (
-    ACPI_PREDEFINED_DATA        *Data,
-    ACPI_OPERAND_OBJECT         **ReturnObjectPtr,
-    UINT32                      ExpectedBtypes,
-    UINT32                      PackageIndex);
-
-static ACPI_STATUS
 AcpiNsCheckReference (
     ACPI_PREDEFINED_DATA        *Data,
     ACPI_OPERAND_OBJECT         *ReturnObject);
@@ -450,574 +421,6 @@ AcpiNsCheckForPredefinedName (
 
 /*******************************************************************************
  *
- * FUNCTION:    AcpiNsCheckPackage
- *
- * PARAMETERS:  Data            - Pointer to validation data structure
- *              ReturnObjectPtr - Pointer to the object returned from the
- *                                evaluation of a method or object
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Check a returned package object for the correct count and
- *              correct type of all sub-objects.
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-AcpiNsCheckPackage (
-    ACPI_PREDEFINED_DATA        *Data,
-    ACPI_OPERAND_OBJECT         **ReturnObjectPtr)
-{
-    ACPI_OPERAND_OBJECT         *ReturnObject = *ReturnObjectPtr;
-    const ACPI_PREDEFINED_INFO  *Package;
-    ACPI_OPERAND_OBJECT         **Elements;
-    ACPI_STATUS                 Status = AE_OK;
-    UINT32                      ExpectedCount;
-    UINT32                      Count;
-    UINT32                      i;
-
-
-    ACPI_FUNCTION_NAME (NsCheckPackage);
-
-
-    /* The package info for this name is in the next table entry */
-
-    Package = Data->Predefined + 1;
-
-    ACPI_DEBUG_PRINT ((ACPI_DB_NAMES,
-        "%s Validating return Package of Type %X, Count %X\n",
-        Data->Pathname, Package->RetInfo.Type, ReturnObject->Package.Count));
-
-    /*
-     * For variable-length Packages, we can safely remove all embedded
-     * and trailing NULL package elements
-     */
-    AcpiNsRemoveNullElements (Data, Package->RetInfo.Type, ReturnObject);
-
-    /* Extract package count and elements array */
-
-    Elements = ReturnObject->Package.Elements;
-    Count = ReturnObject->Package.Count;
-
-    /* The package must have at least one element, else invalid */
-
-    if (!Count)
-    {
-        ACPI_WARN_PREDEFINED ((AE_INFO, Data->Pathname, Data->NodeFlags,
-            "Return Package has no elements (empty)"));
-
-        return (AE_AML_OPERAND_VALUE);
-    }
-
-    /*
-     * Decode the type of the expected package contents
-     *
-     * PTYPE1 packages contain no subpackages
-     * PTYPE2 packages contain sub-packages
-     */
-    switch (Package->RetInfo.Type)
-    {
-    case ACPI_PTYPE1_FIXED:
-
-        /*
-         * The package count is fixed and there are no sub-packages
-         *
-         * If package is too small, exit.
-         * If package is larger than expected, issue warning but continue
-         */
-        ExpectedCount = Package->RetInfo.Count1 + Package->RetInfo.Count2;
-        if (Count < ExpectedCount)
-        {
-            goto PackageTooSmall;
-        }
-        else if (Count > ExpectedCount)
-        {
-            ACPI_DEBUG_PRINT ((ACPI_DB_REPAIR,
-                "%s: Return Package is larger than needed - "
-                "found %u, expected %u\n",
-                Data->Pathname, Count, ExpectedCount));
-        }
-
-        /* Validate all elements of the returned package */
-
-        Status = AcpiNsCheckPackageElements (Data, Elements,
-                    Package->RetInfo.ObjectType1, Package->RetInfo.Count1,
-                    Package->RetInfo.ObjectType2, Package->RetInfo.Count2, 0);
-        break;
-
-
-    case ACPI_PTYPE1_VAR:
-
-        /*
-         * The package count is variable, there are no sub-packages, and all
-         * elements must be of the same type
-         */
-        for (i = 0; i < Count; i++)
-        {
-            Status = AcpiNsCheckObjectType (Data, Elements,
-                        Package->RetInfo.ObjectType1, i);
-            if (ACPI_FAILURE (Status))
-            {
-                return (Status);
-            }
-            Elements++;
-        }
-        break;
-
-
-    case ACPI_PTYPE1_OPTION:
-
-        /*
-         * The package count is variable, there are no sub-packages. There are
-         * a fixed number of required elements, and a variable number of
-         * optional elements.
-         *
-         * Check if package is at least as large as the minimum required
-         */
-        ExpectedCount = Package->RetInfo3.Count;
-        if (Count < ExpectedCount)
-        {
-            goto PackageTooSmall;
-        }
-
-        /* Variable number of sub-objects */
-
-        for (i = 0; i < Count; i++)
-        {
-            if (i < Package->RetInfo3.Count)
-            {
-                /* These are the required package elements (0, 1, or 2) */
-
-                Status = AcpiNsCheckObjectType (Data, Elements,
-                            Package->RetInfo3.ObjectType[i], i);
-                if (ACPI_FAILURE (Status))
-                {
-                    return (Status);
-                }
-            }
-            else
-            {
-                /* These are the optional package elements */
-
-                Status = AcpiNsCheckObjectType (Data, Elements,
-                            Package->RetInfo3.TailObjectType, i);
-                if (ACPI_FAILURE (Status))
-                {
-                    return (Status);
-                }
-            }
-            Elements++;
-        }
-        break;
-
-
-    case ACPI_PTYPE2_REV_FIXED:
-
-        /* First element is the (Integer) revision */
-
-        Status = AcpiNsCheckObjectType (Data, Elements,
-                    ACPI_RTYPE_INTEGER, 0);
-        if (ACPI_FAILURE (Status))
-        {
-            return (Status);
-        }
-
-        Elements++;
-        Count--;
-
-        /* Examine the sub-packages */
-
-        Status = AcpiNsCheckPackageList (Data, Package, Elements, Count);
-        break;
-
-
-    case ACPI_PTYPE2_PKG_COUNT:
-
-        /* First element is the (Integer) count of sub-packages to follow */
-
-        Status = AcpiNsCheckObjectType (Data, Elements,
-                    ACPI_RTYPE_INTEGER, 0);
-        if (ACPI_FAILURE (Status))
-        {
-            return (Status);
-        }
-
-        /*
-         * Count cannot be larger than the parent package length, but allow it
-         * to be smaller. The >= accounts for the Integer above.
-         */
-        ExpectedCount = (UINT32) (*Elements)->Integer.Value;
-        if (ExpectedCount >= Count)
-        {
-            goto PackageTooSmall;
-        }
-
-        Count = ExpectedCount;
-        Elements++;
-
-        /* Examine the sub-packages */
-
-        Status = AcpiNsCheckPackageList (Data, Package, Elements, Count);
-        break;
-
-
-    case ACPI_PTYPE2:
-    case ACPI_PTYPE2_FIXED:
-    case ACPI_PTYPE2_MIN:
-    case ACPI_PTYPE2_COUNT:
-    case ACPI_PTYPE2_FIX_VAR:
-
-        /*
-         * These types all return a single Package that consists of a
-         * variable number of sub-Packages.
-         *
-         * First, ensure that the first element is a sub-Package. If not,
-         * the BIOS may have incorrectly returned the object as a single
-         * package instead of a Package of Packages (a common error if
-         * there is only one entry). We may be able to repair this by
-         * wrapping the returned Package with a new outer Package.
-         */
-        if (*Elements && ((*Elements)->Common.Type != ACPI_TYPE_PACKAGE))
-        {
-            /* Create the new outer package and populate it */
-
-            Status = AcpiNsWrapWithPackage (Data, ReturnObject, ReturnObjectPtr);
-            if (ACPI_FAILURE (Status))
-            {
-                return (Status);
-            }
-
-            /* Update locals to point to the new package (of 1 element) */
-
-            ReturnObject = *ReturnObjectPtr;
-            Elements = ReturnObject->Package.Elements;
-            Count = 1;
-        }
-
-        /* Examine the sub-packages */
-
-        Status = AcpiNsCheckPackageList (Data, Package, Elements, Count);
-        break;
-
-
-    default:
-
-        /* Should not get here if predefined info table is correct */
-
-        ACPI_WARN_PREDEFINED ((AE_INFO, Data->Pathname, Data->NodeFlags,
-            "Invalid internal return type in table entry: %X",
-            Package->RetInfo.Type));
-
-        return (AE_AML_INTERNAL);
-    }
-
-    return (Status);
-
-
-PackageTooSmall:
-
-    /* Error exit for the case with an incorrect package count */
-
-    ACPI_WARN_PREDEFINED ((AE_INFO, Data->Pathname, Data->NodeFlags,
-        "Return Package is too small - found %u elements, expected %u",
-        Count, ExpectedCount));
-
-    return (AE_AML_OPERAND_VALUE);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiNsCheckPackageList
- *
- * PARAMETERS:  Data            - Pointer to validation data structure
- *              Package         - Pointer to package-specific info for method
- *              Elements        - Element list of parent package. All elements
- *                                of this list should be of type Package.
- *              Count           - Count of subpackages
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Examine a list of subpackages
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-AcpiNsCheckPackageList (
-    ACPI_PREDEFINED_DATA        *Data,
-    const ACPI_PREDEFINED_INFO  *Package,
-    ACPI_OPERAND_OBJECT         **Elements,
-    UINT32                      Count)
-{
-    ACPI_OPERAND_OBJECT         *SubPackage;
-    ACPI_OPERAND_OBJECT         **SubElements;
-    ACPI_STATUS                 Status;
-    UINT32                      ExpectedCount;
-    UINT32                      i;
-    UINT32                      j;
-
-
-    /*
-     * Validate each sub-Package in the parent Package
-     *
-     * NOTE: assumes list of sub-packages contains no NULL elements.
-     * Any NULL elements should have been removed by earlier call
-     * to AcpiNsRemoveNullElements.
-     */
-    for (i = 0; i < Count; i++)
-    {
-        SubPackage = *Elements;
-        SubElements = SubPackage->Package.Elements;
-        Data->ParentPackage = SubPackage;
-
-        /* Each sub-object must be of type Package */
-
-        Status = AcpiNsCheckObjectType (Data, &SubPackage,
-                    ACPI_RTYPE_PACKAGE, i);
-        if (ACPI_FAILURE (Status))
-        {
-            return (Status);
-        }
-
-        /* Examine the different types of expected sub-packages */
-
-        Data->ParentPackage = SubPackage;
-        switch (Package->RetInfo.Type)
-        {
-        case ACPI_PTYPE2:
-        case ACPI_PTYPE2_PKG_COUNT:
-        case ACPI_PTYPE2_REV_FIXED:
-
-            /* Each subpackage has a fixed number of elements */
-
-            ExpectedCount = Package->RetInfo.Count1 + Package->RetInfo.Count2;
-            if (SubPackage->Package.Count < ExpectedCount)
-            {
-                goto PackageTooSmall;
-            }
-
-            Status = AcpiNsCheckPackageElements (Data, SubElements,
-                        Package->RetInfo.ObjectType1,
-                        Package->RetInfo.Count1,
-                        Package->RetInfo.ObjectType2,
-                        Package->RetInfo.Count2, 0);
-            if (ACPI_FAILURE (Status))
-            {
-                return (Status);
-            }
-            break;
-
-
-        case ACPI_PTYPE2_FIX_VAR:
-            /*
-             * Each subpackage has a fixed number of elements and an
-             * optional element
-             */
-            ExpectedCount = Package->RetInfo.Count1 + Package->RetInfo.Count2;
-            if (SubPackage->Package.Count < ExpectedCount)
-            {
-                goto PackageTooSmall;
-            }
-
-            Status = AcpiNsCheckPackageElements (Data, SubElements,
-                        Package->RetInfo.ObjectType1,
-                        Package->RetInfo.Count1,
-                        Package->RetInfo.ObjectType2,
-                        SubPackage->Package.Count - Package->RetInfo.Count1, 0);
-            if (ACPI_FAILURE (Status))
-            {
-                return (Status);
-            }
-            break;
-
-
-        case ACPI_PTYPE2_FIXED:
-
-            /* Each sub-package has a fixed length */
-
-            ExpectedCount = Package->RetInfo2.Count;
-            if (SubPackage->Package.Count < ExpectedCount)
-            {
-                goto PackageTooSmall;
-            }
-
-            /* Check the type of each sub-package element */
-
-            for (j = 0; j < ExpectedCount; j++)
-            {
-                Status = AcpiNsCheckObjectType (Data, &SubElements[j],
-                            Package->RetInfo2.ObjectType[j], j);
-                if (ACPI_FAILURE (Status))
-                {
-                    return (Status);
-                }
-            }
-            break;
-
-
-        case ACPI_PTYPE2_MIN:
-
-            /* Each sub-package has a variable but minimum length */
-
-            ExpectedCount = Package->RetInfo.Count1;
-            if (SubPackage->Package.Count < ExpectedCount)
-            {
-                goto PackageTooSmall;
-            }
-
-            /* Check the type of each sub-package element */
-
-            Status = AcpiNsCheckPackageElements (Data, SubElements,
-                        Package->RetInfo.ObjectType1,
-                        SubPackage->Package.Count, 0, 0, 0);
-            if (ACPI_FAILURE (Status))
-            {
-                return (Status);
-            }
-            break;
-
-
-        case ACPI_PTYPE2_COUNT:
-
-            /*
-             * First element is the (Integer) count of elements, including
-             * the count field (the ACPI name is NumElements)
-             */
-            Status = AcpiNsCheckObjectType (Data, SubElements,
-                        ACPI_RTYPE_INTEGER, 0);
-            if (ACPI_FAILURE (Status))
-            {
-                return (Status);
-            }
-
-            /*
-             * Make sure package is large enough for the Count and is
-             * is as large as the minimum size
-             */
-            ExpectedCount = (UINT32) (*SubElements)->Integer.Value;
-            if (SubPackage->Package.Count < ExpectedCount)
-            {
-                goto PackageTooSmall;
-            }
-            if (SubPackage->Package.Count < Package->RetInfo.Count1)
-            {
-                ExpectedCount = Package->RetInfo.Count1;
-                goto PackageTooSmall;
-            }
-            if (ExpectedCount == 0)
-            {
-                /*
-                 * Either the NumEntries element was originally zero or it was
-                 * a NULL element and repaired to an Integer of value zero.
-                 * In either case, repair it by setting NumEntries to be the
-                 * actual size of the subpackage.
-                 */
-                ExpectedCount = SubPackage->Package.Count;
-                (*SubElements)->Integer.Value = ExpectedCount;
-            }
-
-            /* Check the type of each sub-package element */
-
-            Status = AcpiNsCheckPackageElements (Data, (SubElements + 1),
-                        Package->RetInfo.ObjectType1,
-                        (ExpectedCount - 1), 0, 0, 1);
-            if (ACPI_FAILURE (Status))
-            {
-                return (Status);
-            }
-            break;
-
-
-        default: /* Should not get here, type was validated by caller */
-
-            return (AE_AML_INTERNAL);
-        }
-
-        Elements++;
-    }
-
-    return (AE_OK);
-
-
-PackageTooSmall:
-
-    /* The sub-package count was smaller than required */
-
-    ACPI_WARN_PREDEFINED ((AE_INFO, Data->Pathname, Data->NodeFlags,
-        "Return Sub-Package[%u] is too small - found %u elements, expected %u",
-        i, SubPackage->Package.Count, ExpectedCount));
-
-    return (AE_AML_OPERAND_VALUE);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiNsCheckPackageElements
- *
- * PARAMETERS:  Data            - Pointer to validation data structure
- *              Elements        - Pointer to the package elements array
- *              Type1           - Object type for first group
- *              Count1          - Count for first group
- *              Type2           - Object type for second group
- *              Count2          - Count for second group
- *              StartIndex      - Start of the first group of elements
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Check that all elements of a package are of the correct object
- *              type. Supports up to two groups of different object types.
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-AcpiNsCheckPackageElements (
-    ACPI_PREDEFINED_DATA        *Data,
-    ACPI_OPERAND_OBJECT         **Elements,
-    UINT8                       Type1,
-    UINT32                      Count1,
-    UINT8                       Type2,
-    UINT32                      Count2,
-    UINT32                      StartIndex)
-{
-    ACPI_OPERAND_OBJECT         **ThisElement = Elements;
-    ACPI_STATUS                 Status;
-    UINT32                      i;
-
-
-    /*
-     * Up to two groups of package elements are supported by the data
-     * structure. All elements in each group must be of the same type.
-     * The second group can have a count of zero.
-     */
-    for (i = 0; i < Count1; i++)
-    {
-        Status = AcpiNsCheckObjectType (Data, ThisElement,
-                    Type1, i + StartIndex);
-        if (ACPI_FAILURE (Status))
-        {
-            return (Status);
-        }
-        ThisElement++;
-    }
-
-    for (i = 0; i < Count2; i++)
-    {
-        Status = AcpiNsCheckObjectType (Data, ThisElement,
-                    Type2, (i + Count1 + StartIndex));
-        if (ACPI_FAILURE (Status))
-        {
-            return (Status);
-        }
-        ThisElement++;
-    }
-
-    return (AE_OK);
-}
-
-
-/*******************************************************************************
- *
  * FUNCTION:    AcpiNsCheckObjectType
  *
  * PARAMETERS:  Data            - Pointer to validation data structure
@@ -1035,7 +438,7 @@ AcpiNsCheckPackageElements (
  *
  ******************************************************************************/
 
-static ACPI_STATUS
+ACPI_STATUS
 AcpiNsCheckObjectType (
     ACPI_PREDEFINED_DATA        *Data,
     ACPI_OPERAND_OBJECT         **ReturnObjectPtr,
diff --git a/sys/contrib/dev/acpica/components/namespace/nsprepkg.c b/sys/contrib/dev/acpica/components/namespace/nsprepkg.c
new file mode 100644
index 0000000..7b65b80
--- /dev/null
+++ b/sys/contrib/dev/acpica/components/namespace/nsprepkg.c
@@ -0,0 +1,639 @@
+/******************************************************************************
+ *
+ * Module Name: nsprepkg - Validation of package objects for predefined names
+ *
+ *****************************************************************************/
+
+/*
+ * Copyright (C) 2000 - 2013, Intel Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ * 3. Neither the names of the above-listed copyright holders nor the names
+ *    of any contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ */
+
+#include <contrib/dev/acpica/include/acpi.h>
+#include <contrib/dev/acpica/include/accommon.h>
+#include <contrib/dev/acpica/include/acnamesp.h>
+#include <contrib/dev/acpica/include/acpredef.h>
+
+
+#define _COMPONENT          ACPI_NAMESPACE
+        ACPI_MODULE_NAME    ("nsprepkg")
+
+
+/* Local prototypes */
+
+static ACPI_STATUS
+AcpiNsCheckPackageList (
+    ACPI_PREDEFINED_DATA        *Data,
+    const ACPI_PREDEFINED_INFO  *Package,
+    ACPI_OPERAND_OBJECT         **Elements,
+    UINT32                      Count);
+
+static ACPI_STATUS
+AcpiNsCheckPackageElements (
+    ACPI_PREDEFINED_DATA        *Data,
+    ACPI_OPERAND_OBJECT         **Elements,
+    UINT8                       Type1,
+    UINT32                      Count1,
+    UINT8                       Type2,
+    UINT32                      Count2,
+    UINT32                      StartIndex);
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiNsCheckPackage
+ *
+ * PARAMETERS:  Data                - Pointer to validation data structure
+ *              ReturnObjectPtr     - Pointer to the object returned from the
+ *                                    evaluation of a method or object
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Check a returned package object for the correct count and
+ *              correct type of all sub-objects.
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+AcpiNsCheckPackage (
+    ACPI_PREDEFINED_DATA        *Data,
+    ACPI_OPERAND_OBJECT         **ReturnObjectPtr)
+{
+    ACPI_OPERAND_OBJECT         *ReturnObject = *ReturnObjectPtr;
+    const ACPI_PREDEFINED_INFO  *Package;
+    ACPI_OPERAND_OBJECT         **Elements;
+    ACPI_STATUS                 Status = AE_OK;
+    UINT32                      ExpectedCount;
+    UINT32                      Count;
+    UINT32                      i;
+
+
+    ACPI_FUNCTION_NAME (NsCheckPackage);
+
+
+    /* The package info for this name is in the next table entry */
+
+    Package = Data->Predefined + 1;
+
+    ACPI_DEBUG_PRINT ((ACPI_DB_NAMES,
+        "%s Validating return Package of Type %X, Count %X\n",
+        Data->Pathname, Package->RetInfo.Type, ReturnObject->Package.Count));
+
+    /*
+     * For variable-length Packages, we can safely remove all embedded
+     * and trailing NULL package elements
+     */
+    AcpiNsRemoveNullElements (Data, Package->RetInfo.Type, ReturnObject);
+
+    /* Extract package count and elements array */
+
+    Elements = ReturnObject->Package.Elements;
+    Count = ReturnObject->Package.Count;
+
+    /* The package must have at least one element, else invalid */
+
+    if (!Count)
+    {
+        ACPI_WARN_PREDEFINED ((AE_INFO, Data->Pathname, Data->NodeFlags,
+            "Return Package has no elements (empty)"));
+
+        return (AE_AML_OPERAND_VALUE);
+    }
+
+    /*
+     * Decode the type of the expected package contents
+     *
+     * PTYPE1 packages contain no subpackages
+     * PTYPE2 packages contain sub-packages
+     */
+    switch (Package->RetInfo.Type)
+    {
+    case ACPI_PTYPE1_FIXED:
+
+        /*
+         * The package count is fixed and there are no sub-packages
+         *
+         * If package is too small, exit.
+         * If package is larger than expected, issue warning but continue
+         */
+        ExpectedCount = Package->RetInfo.Count1 + Package->RetInfo.Count2;
+        if (Count < ExpectedCount)
+        {
+            goto PackageTooSmall;
+        }
+        else if (Count > ExpectedCount)
+        {
+            ACPI_DEBUG_PRINT ((ACPI_DB_REPAIR,
+                "%s: Return Package is larger than needed - "
+                "found %u, expected %u\n",
+                Data->Pathname, Count, ExpectedCount));
+        }
+
+        /* Validate all elements of the returned package */
+
+        Status = AcpiNsCheckPackageElements (Data, Elements,
+                    Package->RetInfo.ObjectType1, Package->RetInfo.Count1,
+                    Package->RetInfo.ObjectType2, Package->RetInfo.Count2, 0);
+        break;
+
+
+    case ACPI_PTYPE1_VAR:
+
+        /*
+         * The package count is variable, there are no sub-packages, and all
+         * elements must be of the same type
+         */
+        for (i = 0; i < Count; i++)
+        {
+            Status = AcpiNsCheckObjectType (Data, Elements,
+                        Package->RetInfo.ObjectType1, i);
+            if (ACPI_FAILURE (Status))
+            {
+                return (Status);
+            }
+            Elements++;
+        }
+        break;
+
+
+    case ACPI_PTYPE1_OPTION:
+
+        /*
+         * The package count is variable, there are no sub-packages. There are
+         * a fixed number of required elements, and a variable number of
+         * optional elements.
+         *
+         * Check if package is at least as large as the minimum required
+         */
+        ExpectedCount = Package->RetInfo3.Count;
+        if (Count < ExpectedCount)
+        {
+            goto PackageTooSmall;
+        }
+
+        /* Variable number of sub-objects */
+
+        for (i = 0; i < Count; i++)
+        {
+            if (i < Package->RetInfo3.Count)
+            {
+                /* These are the required package elements (0, 1, or 2) */
+
+                Status = AcpiNsCheckObjectType (Data, Elements,
+                            Package->RetInfo3.ObjectType[i], i);
+                if (ACPI_FAILURE (Status))
+                {
+                    return (Status);
+                }
+            }
+            else
+            {
+                /* These are the optional package elements */
+
+                Status = AcpiNsCheckObjectType (Data, Elements,
+                            Package->RetInfo3.TailObjectType, i);
+                if (ACPI_FAILURE (Status))
+                {
+                    return (Status);
+                }
+            }
+            Elements++;
+        }
+        break;
+
+
+    case ACPI_PTYPE2_REV_FIXED:
+
+        /* First element is the (Integer) revision */
+
+        Status = AcpiNsCheckObjectType (Data, Elements,
+                    ACPI_RTYPE_INTEGER, 0);
+        if (ACPI_FAILURE (Status))
+        {
+            return (Status);
+        }
+
+        Elements++;
+        Count--;
+
+        /* Examine the sub-packages */
+
+        Status = AcpiNsCheckPackageList (Data, Package, Elements, Count);
+        break;
+
+
+    case ACPI_PTYPE2_PKG_COUNT:
+
+        /* First element is the (Integer) count of sub-packages to follow */
+
+        Status = AcpiNsCheckObjectType (Data, Elements,
+                    ACPI_RTYPE_INTEGER, 0);
+        if (ACPI_FAILURE (Status))
+        {
+            return (Status);
+        }
+
+        /*
+         * Count cannot be larger than the parent package length, but allow it
+         * to be smaller. The >= accounts for the Integer above.
+         */
+        ExpectedCount = (UINT32) (*Elements)->Integer.Value;
+        if (ExpectedCount >= Count)
+        {
+            goto PackageTooSmall;
+        }
+
+        Count = ExpectedCount;
+        Elements++;
+
+        /* Examine the sub-packages */
+
+        Status = AcpiNsCheckPackageList (Data, Package, Elements, Count);
+        break;
+
+
+    case ACPI_PTYPE2:
+    case ACPI_PTYPE2_FIXED:
+    case ACPI_PTYPE2_MIN:
+    case ACPI_PTYPE2_COUNT:
+    case ACPI_PTYPE2_FIX_VAR:
+
+        /*
+         * These types all return a single Package that consists of a
+         * variable number of sub-Packages.
+         *
+         * First, ensure that the first element is a sub-Package. If not,
+         * the BIOS may have incorrectly returned the object as a single
+         * package instead of a Package of Packages (a common error if
+         * there is only one entry). We may be able to repair this by
+         * wrapping the returned Package with a new outer Package.
+         */
+        if (*Elements && ((*Elements)->Common.Type != ACPI_TYPE_PACKAGE))
+        {
+            /* Create the new outer package and populate it */
+
+            Status = AcpiNsWrapWithPackage (Data, ReturnObject, ReturnObjectPtr);
+            if (ACPI_FAILURE (Status))
+            {
+                return (Status);
+            }
+
+            /* Update locals to point to the new package (of 1 element) */
+
+            ReturnObject = *ReturnObjectPtr;
+            Elements = ReturnObject->Package.Elements;
+            Count = 1;
+        }
+
+        /* Examine the sub-packages */
+
+        Status = AcpiNsCheckPackageList (Data, Package, Elements, Count);
+        break;
+
+
+    default:
+
+        /* Should not get here if predefined info table is correct */
+
+        ACPI_WARN_PREDEFINED ((AE_INFO, Data->Pathname, Data->NodeFlags,
+            "Invalid internal return type in table entry: %X",
+            Package->RetInfo.Type));
+
+        return (AE_AML_INTERNAL);
+    }
+
+    return (Status);
+
+
+PackageTooSmall:
+
+    /* Error exit for the case with an incorrect package count */
+
+    ACPI_WARN_PREDEFINED ((AE_INFO, Data->Pathname, Data->NodeFlags,
+        "Return Package is too small - found %u elements, expected %u",
+        Count, ExpectedCount));
+
+    return (AE_AML_OPERAND_VALUE);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiNsCheckPackageList
+ *
+ * PARAMETERS:  Data            - Pointer to validation data structure
+ *              Package         - Pointer to package-specific info for method
+ *              Elements        - Element list of parent package. All elements
+ *                                of this list should be of type Package.
+ *              Count           - Count of subpackages
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Examine a list of subpackages
+ *
+ ******************************************************************************/
+
+static ACPI_STATUS
+AcpiNsCheckPackageList (
+    ACPI_PREDEFINED_DATA        *Data,
+    const ACPI_PREDEFINED_INFO  *Package,
+    ACPI_OPERAND_OBJECT         **Elements,
+    UINT32                      Count)
+{
+    ACPI_OPERAND_OBJECT         *SubPackage;
+    ACPI_OPERAND_OBJECT         **SubElements;
+    ACPI_STATUS                 Status;
+    UINT32                      ExpectedCount;
+    UINT32                      i;
+    UINT32                      j;
+
+
+    /*
+     * Validate each sub-Package in the parent Package
+     *
+     * NOTE: assumes list of sub-packages contains no NULL elements.
+     * Any NULL elements should have been removed by earlier call
+     * to AcpiNsRemoveNullElements.
+     */
+    for (i = 0; i < Count; i++)
+    {
+        SubPackage = *Elements;
+        SubElements = SubPackage->Package.Elements;
+        Data->ParentPackage = SubPackage;
+
+        /* Each sub-object must be of type Package */
+
+        Status = AcpiNsCheckObjectType (Data, &SubPackage,
+                    ACPI_RTYPE_PACKAGE, i);
+        if (ACPI_FAILURE (Status))
+        {
+            return (Status);
+        }
+
+        /* Examine the different types of expected sub-packages */
+
+        Data->ParentPackage = SubPackage;
+        switch (Package->RetInfo.Type)
+        {
+        case ACPI_PTYPE2:
+        case ACPI_PTYPE2_PKG_COUNT:
+        case ACPI_PTYPE2_REV_FIXED:
+
+            /* Each subpackage has a fixed number of elements */
+
+            ExpectedCount = Package->RetInfo.Count1 + Package->RetInfo.Count2;
+            if (SubPackage->Package.Count < ExpectedCount)
+            {
+                goto PackageTooSmall;
+            }
+
+            Status = AcpiNsCheckPackageElements (Data, SubElements,
+                        Package->RetInfo.ObjectType1,
+                        Package->RetInfo.Count1,
+                        Package->RetInfo.ObjectType2,
+                        Package->RetInfo.Count2, 0);
+            if (ACPI_FAILURE (Status))
+            {
+                return (Status);
+            }
+            break;
+
+
+        case ACPI_PTYPE2_FIX_VAR:
+            /*
+             * Each subpackage has a fixed number of elements and an
+             * optional element
+             */
+            ExpectedCount = Package->RetInfo.Count1 + Package->RetInfo.Count2;
+            if (SubPackage->Package.Count < ExpectedCount)
+            {
+                goto PackageTooSmall;
+            }
+
+            Status = AcpiNsCheckPackageElements (Data, SubElements,
+                        Package->RetInfo.ObjectType1,
+                        Package->RetInfo.Count1,
+                        Package->RetInfo.ObjectType2,
+                        SubPackage->Package.Count - Package->RetInfo.Count1, 0);
+            if (ACPI_FAILURE (Status))
+            {
+                return (Status);
+            }
+            break;
+
+
+        case ACPI_PTYPE2_FIXED:
+
+            /* Each sub-package has a fixed length */
+
+            ExpectedCount = Package->RetInfo2.Count;
+            if (SubPackage->Package.Count < ExpectedCount)
+            {
+                goto PackageTooSmall;
+            }
+
+            /* Check the type of each sub-package element */
+
+            for (j = 0; j < ExpectedCount; j++)
+            {
+                Status = AcpiNsCheckObjectType (Data, &SubElements[j],
+                            Package->RetInfo2.ObjectType[j], j);
+                if (ACPI_FAILURE (Status))
+                {
+                    return (Status);
+                }
+            }
+            break;
+
+
+        case ACPI_PTYPE2_MIN:
+
+            /* Each sub-package has a variable but minimum length */
+
+            ExpectedCount = Package->RetInfo.Count1;
+            if (SubPackage->Package.Count < ExpectedCount)
+            {
+                goto PackageTooSmall;
+            }
+
+            /* Check the type of each sub-package element */
+
+            Status = AcpiNsCheckPackageElements (Data, SubElements,
+                        Package->RetInfo.ObjectType1,
+                        SubPackage->Package.Count, 0, 0, 0);
+            if (ACPI_FAILURE (Status))
+            {
+                return (Status);
+            }
+            break;
+
+
+        case ACPI_PTYPE2_COUNT:
+
+            /*
+             * First element is the (Integer) count of elements, including
+             * the count field (the ACPI name is NumElements)
+             */
+            Status = AcpiNsCheckObjectType (Data, SubElements,
+                        ACPI_RTYPE_INTEGER, 0);
+            if (ACPI_FAILURE (Status))
+            {
+                return (Status);
+            }
+
+            /*
+             * Make sure package is large enough for the Count and is
+             * is as large as the minimum size
+             */
+            ExpectedCount = (UINT32) (*SubElements)->Integer.Value;
+            if (SubPackage->Package.Count < ExpectedCount)
+            {
+                goto PackageTooSmall;
+            }
+            if (SubPackage->Package.Count < Package->RetInfo.Count1)
+            {
+                ExpectedCount = Package->RetInfo.Count1;
+                goto PackageTooSmall;
+            }
+            if (ExpectedCount == 0)
+            {
+                /*
+                 * Either the NumEntries element was originally zero or it was
+                 * a NULL element and repaired to an Integer of value zero.
+                 * In either case, repair it by setting NumEntries to be the
+                 * actual size of the subpackage.
+                 */
+                ExpectedCount = SubPackage->Package.Count;
+                (*SubElements)->Integer.Value = ExpectedCount;
+            }
+
+            /* Check the type of each sub-package element */
+
+            Status = AcpiNsCheckPackageElements (Data, (SubElements + 1),
+                        Package->RetInfo.ObjectType1,
+                        (ExpectedCount - 1), 0, 0, 1);
+            if (ACPI_FAILURE (Status))
+            {
+                return (Status);
+            }
+            break;
+
+
+        default: /* Should not get here, type was validated by caller */
+
+            return (AE_AML_INTERNAL);
+        }
+
+        Elements++;
+    }
+
+    return (AE_OK);
+
+
+PackageTooSmall:
+
+    /* The sub-package count was smaller than required */
+
+    ACPI_WARN_PREDEFINED ((AE_INFO, Data->Pathname, Data->NodeFlags,
+        "Return Sub-Package[%u] is too small - found %u elements, expected %u",
+        i, SubPackage->Package.Count, ExpectedCount));
+
+    return (AE_AML_OPERAND_VALUE);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiNsCheckPackageElements
+ *
+ * PARAMETERS:  Data            - Pointer to validation data structure
+ *              Elements        - Pointer to the package elements array
+ *              Type1           - Object type for first group
+ *              Count1          - Count for first group
+ *              Type2           - Object type for second group
+ *              Count2          - Count for second group
+ *              StartIndex      - Start of the first group of elements
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Check that all elements of a package are of the correct object
+ *              type. Supports up to two groups of different object types.
+ *
+ ******************************************************************************/
+
+static ACPI_STATUS
+AcpiNsCheckPackageElements (
+    ACPI_PREDEFINED_DATA        *Data,
+    ACPI_OPERAND_OBJECT         **Elements,
+    UINT8                       Type1,
+    UINT32                      Count1,
+    UINT8                       Type2,
+    UINT32                      Count2,
+    UINT32                      StartIndex)
+{
+    ACPI_OPERAND_OBJECT         **ThisElement = Elements;
+    ACPI_STATUS                 Status;
+    UINT32                      i;
+
+
+    /*
+     * Up to two groups of package elements are supported by the data
+     * structure. All elements in each group must be of the same type.
+     * The second group can have a count of zero.
+     */
+    for (i = 0; i < Count1; i++)
+    {
+        Status = AcpiNsCheckObjectType (Data, ThisElement,
+                    Type1, i + StartIndex);
+        if (ACPI_FAILURE (Status))
+        {
+            return (Status);
+        }
+        ThisElement++;
+    }
+
+    for (i = 0; i < Count2; i++)
+    {
+        Status = AcpiNsCheckObjectType (Data, ThisElement,
+                    Type2, (i + Count1 + StartIndex));
+        if (ACPI_FAILURE (Status))
+        {
+            return (Status);
+        }
+        ThisElement++;
+    }
+
+    return (AE_OK);
+}
diff --git a/sys/contrib/dev/acpica/components/namespace/nsrepair.c b/sys/contrib/dev/acpica/components/namespace/nsrepair.c
index 7a1ee9f..c5c9309 100644
--- a/sys/contrib/dev/acpica/components/namespace/nsrepair.c
+++ b/sys/contrib/dev/acpica/components/namespace/nsrepair.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/namespace/nsrepair2.c b/sys/contrib/dev/acpica/components/namespace/nsrepair2.c
index 06286f1..a6559f4 100644
--- a/sys/contrib/dev/acpica/components/namespace/nsrepair2.c
+++ b/sys/contrib/dev/acpica/components/namespace/nsrepair2.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/namespace/nssearch.c b/sys/contrib/dev/acpica/components/namespace/nssearch.c
index 93e060c..255956f 100644
--- a/sys/contrib/dev/acpica/components/namespace/nssearch.c
+++ b/sys/contrib/dev/acpica/components/namespace/nssearch.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/namespace/nsutils.c b/sys/contrib/dev/acpica/components/namespace/nsutils.c
index 1003098..070ed26 100644
--- a/sys/contrib/dev/acpica/components/namespace/nsutils.c
+++ b/sys/contrib/dev/acpica/components/namespace/nsutils.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -54,10 +54,6 @@
 
 /* Local prototypes */
 
-static BOOLEAN
-AcpiNsValidPathSeparator (
-    char                    Sep);
-
 #ifdef ACPI_OBSOLETE_FUNCTIONS
 ACPI_NAME
 AcpiNsFindParentName (
@@ -112,48 +108,6 @@ AcpiNsPrintNodePathname (
 
 /*******************************************************************************
  *
- * FUNCTION:    AcpiNsValidRootPrefix
- *
- * PARAMETERS:  Prefix          - Character to be checked
- *
- * RETURN:      TRUE if a valid prefix
- *
- * DESCRIPTION: Check if a character is a valid ACPI Root prefix
- *
- ******************************************************************************/
-
-BOOLEAN
-AcpiNsValidRootPrefix (
-    char                    Prefix)
-{
-
-    return ((BOOLEAN) (Prefix == '\\'));
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiNsValidPathSeparator
- *
- * PARAMETERS:  Sep         - Character to be checked
- *
- * RETURN:      TRUE if a valid path separator
- *
- * DESCRIPTION: Check if a character is a valid ACPI path separator
- *
- ******************************************************************************/
-
-static BOOLEAN
-AcpiNsValidPathSeparator (
-    char                    Sep)
-{
-
-    return ((BOOLEAN) (Sep == '.'));
-}
-
-
-/*******************************************************************************
- *
  * FUNCTION:    AcpiNsGetType
  *
  * PARAMETERS:  Node        - Parent Node to be examined
@@ -174,10 +128,10 @@ AcpiNsGetType (
     if (!Node)
     {
         ACPI_WARNING ((AE_INFO, "Null Node parameter"));
-        return_UINT32 (ACPI_TYPE_ANY);
+        return_VALUE (ACPI_TYPE_ANY);
     }
 
-    return_UINT32 ((ACPI_OBJECT_TYPE) Node->Type);
+    return_VALUE (Node->Type);
 }
 
 
@@ -206,10 +160,10 @@ AcpiNsLocal (
         /* Type code out of range  */
 
         ACPI_WARNING ((AE_INFO, "Invalid Object Type 0x%X", Type));
-        return_UINT32 (ACPI_NS_NORMAL);
+        return_VALUE (ACPI_NS_NORMAL);
     }
 
-    return_UINT32 ((UINT32) AcpiGbl_NsProperties[Type] & ACPI_NS_LOCAL);
+    return_VALUE (AcpiGbl_NsProperties[Type] & ACPI_NS_LOCAL);
 }
 
 
@@ -250,14 +204,14 @@ AcpiNsGetInternalNameLength (
      *
      * strlen() + 1 covers the first NameSeg, which has no path separator
      */
-    if (AcpiNsValidRootPrefix (*NextExternalChar))
+    if (ACPI_IS_ROOT_PREFIX (*NextExternalChar))
     {
         Info->FullyQualified = TRUE;
         NextExternalChar++;
 
         /* Skip redundant RootPrefix, like \\_SB.PCI0.SBRG.EC0 */
 
-        while (AcpiNsValidRootPrefix (*NextExternalChar))
+        while (ACPI_IS_ROOT_PREFIX (*NextExternalChar))
         {
             NextExternalChar++;
         }
@@ -266,7 +220,7 @@ AcpiNsGetInternalNameLength (
     {
         /* Handle Carat prefixes */
 
-        while (*NextExternalChar == '^')
+        while (ACPI_IS_PARENT_PREFIX (*NextExternalChar))
         {
             Info->NumCarats++;
             NextExternalChar++;
@@ -283,7 +237,7 @@ AcpiNsGetInternalNameLength (
         Info->NumSegments = 1;
         for (i = 0; NextExternalChar[i]; i++)
         {
-            if (AcpiNsValidPathSeparator (NextExternalChar[i]))
+            if (ACPI_IS_PATH_SEPARATOR (NextExternalChar[i]))
             {
                 Info->NumSegments++;
             }
@@ -328,7 +282,7 @@ AcpiNsBuildInternalName (
 
     if (Info->FullyQualified)
     {
-        InternalName[0] = '\\';
+        InternalName[0] = AML_ROOT_PREFIX;
 
         if (NumSegments <= 1)
         {
@@ -357,7 +311,7 @@ AcpiNsBuildInternalName (
         {
             for (i = 0; i < Info->NumCarats; i++)
             {
-                InternalName[i] = '^';
+                InternalName[i] = AML_PARENT_PREFIX;
             }
         }
 
@@ -384,7 +338,7 @@ AcpiNsBuildInternalName (
     {
         for (i = 0; i < ACPI_NAME_SIZE; i++)
         {
-            if (AcpiNsValidPathSeparator (*ExternalName) ||
+            if (ACPI_IS_PATH_SEPARATOR (*ExternalName) ||
                (*ExternalName == 0))
             {
                 /* Pad the segment with underscore(s) if segment is short */
@@ -402,7 +356,7 @@ AcpiNsBuildInternalName (
 
         /* Now we must have a path separator, or the pathname is bad */
 
-        if (!AcpiNsValidPathSeparator (*ExternalName) &&
+        if (!ACPI_IS_PATH_SEPARATOR (*ExternalName) &&
             (*ExternalName != 0))
         {
             return_ACPI_STATUS (AE_BAD_PATHNAME);
@@ -542,14 +496,14 @@ AcpiNsExternalizeName (
 
     switch (InternalName[0])
     {
-    case '\\':
+    case AML_ROOT_PREFIX:
         PrefixLength = 1;
         break;
 
-    case '^':
+    case AML_PARENT_PREFIX:
         for (i = 0; i < InternalNameLength; i++)
         {
-            if (InternalName[i] == '^')
+            if (ACPI_IS_PARENT_PREFIX (InternalName[i]))
             {
                 PrefixLength = i + 1;
             }
@@ -829,6 +783,8 @@ AcpiNsGetNode (
     ACPI_FUNCTION_TRACE_PTR (NsGetNode, ACPI_CAST_PTR (char, Pathname));
 
 
+    /* Simplest case is a null pathname */
+
     if (!Pathname)
     {
         *ReturnNode = PrefixNode;
@@ -839,6 +795,14 @@ AcpiNsGetNode (
         return_ACPI_STATUS (AE_OK);
     }
 
+    /* Quick check for a reference to the root */
+
+    if (ACPI_IS_ROOT_PREFIX (Pathname[0]) && (!Pathname[1]))
+    {
+        *ReturnNode = AcpiGbl_RootNode;
+        return_ACPI_STATUS (AE_OK);
+    }
+
     /* Convert path to internal representation */
 
     Status = AcpiNsInternalizeName (Pathname, &InternalPath);
diff --git a/sys/contrib/dev/acpica/components/namespace/nswalk.c b/sys/contrib/dev/acpica/components/namespace/nswalk.c
index 008a957..3b42d91 100644
--- a/sys/contrib/dev/acpica/components/namespace/nswalk.c
+++ b/sys/contrib/dev/acpica/components/namespace/nswalk.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/namespace/nsxfeval.c b/sys/contrib/dev/acpica/components/namespace/nsxfeval.c
index 5d3204d..ff580be 100644
--- a/sys/contrib/dev/acpica/components/namespace/nsxfeval.c
+++ b/sys/contrib/dev/acpica/components/namespace/nsxfeval.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -255,7 +255,7 @@ AcpiEvaluateObject (
      * 3) Valid handle
      */
     if ((Pathname) &&
-        (AcpiNsValidRootPrefix (Pathname[0])))
+        (ACPI_IS_ROOT_PREFIX (Pathname[0])))
     {
         /* The path is fully qualified, just evaluate by name */
 
diff --git a/sys/contrib/dev/acpica/components/namespace/nsxfname.c b/sys/contrib/dev/acpica/components/namespace/nsxfname.c
index a6e502b1..582ecf3 100644
--- a/sys/contrib/dev/acpica/components/namespace/nsxfname.c
+++ b/sys/contrib/dev/acpica/components/namespace/nsxfname.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -120,7 +120,7 @@ AcpiGetHandle (
      *
      * Error for <null Parent + relative path>
      */
-    if (AcpiNsValidRootPrefix (Pathname[0]))
+    if (ACPI_IS_ROOT_PREFIX (Pathname[0]))
     {
         /* Pathname is fully qualified (starts with '\') */
 
diff --git a/sys/contrib/dev/acpica/components/namespace/nsxfobj.c b/sys/contrib/dev/acpica/components/namespace/nsxfobj.c
index fb0df50..153dfaa 100644
--- a/sys/contrib/dev/acpica/components/namespace/nsxfobj.c
+++ b/sys/contrib/dev/acpica/components/namespace/nsxfobj.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/parser/psargs.c b/sys/contrib/dev/acpica/components/parser/psargs.c
index 9bddc21..c57f485 100644
--- a/sys/contrib/dev/acpica/components/parser/psargs.c
+++ b/sys/contrib/dev/acpica/components/parser/psargs.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -118,7 +118,7 @@ AcpiPsGetNextPackageLength (
     /* Byte 0 is a special case, either bits [0:3] or [0:5] are used */
 
     PackageLength |= (Aml[0] & ByteZeroMask);
-    return_UINT32 (PackageLength);
+    return_VALUE (PackageLength);
 }
 
 
@@ -182,7 +182,8 @@ AcpiPsGetNextNamestring (
 
     /* Point past any namestring prefix characters (backslash or carat) */
 
-    while (AcpiPsIsPrefixChar (*End))
+    while (ACPI_IS_ROOT_PREFIX (*End) ||
+           ACPI_IS_PARENT_PREFIX (*End))
     {
         End++;
     }
@@ -868,7 +869,8 @@ AcpiPsGetNextArg (
         Subop = AcpiPsPeekOpcode (ParserState);
         if (Subop == 0                  ||
             AcpiPsIsLeadingChar (Subop) ||
-            AcpiPsIsPrefixChar (Subop))
+            ACPI_IS_ROOT_PREFIX (Subop) ||
+            ACPI_IS_PARENT_PREFIX (Subop))
         {
             /* NullName or NameString */
 
diff --git a/sys/contrib/dev/acpica/components/parser/psloop.c b/sys/contrib/dev/acpica/components/parser/psloop.c
index 814bd93..4472bf1 100644
--- a/sys/contrib/dev/acpica/components/parser/psloop.c
+++ b/sys/contrib/dev/acpica/components/parser/psloop.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -59,46 +59,15 @@
 #define _COMPONENT          ACPI_PARSER
         ACPI_MODULE_NAME    ("psloop")
 
-static UINT32               AcpiGbl_Depth = 0;
-
 
 /* Local prototypes */
 
 static ACPI_STATUS
-AcpiPsGetAmlOpcode (
-    ACPI_WALK_STATE         *WalkState);
-
-static ACPI_STATUS
-AcpiPsBuildNamedOp (
-    ACPI_WALK_STATE         *WalkState,
-    UINT8                   *AmlOpStart,
-    ACPI_PARSE_OBJECT       *UnnamedOp,
-    ACPI_PARSE_OBJECT       **Op);
-
-static ACPI_STATUS
-AcpiPsCreateOp (
-    ACPI_WALK_STATE         *WalkState,
-    UINT8                   *AmlOpStart,
-    ACPI_PARSE_OBJECT       **NewOp);
-
-static ACPI_STATUS
 AcpiPsGetArguments (
     ACPI_WALK_STATE         *WalkState,
     UINT8                   *AmlOpStart,
     ACPI_PARSE_OBJECT       *Op);
 
-static ACPI_STATUS
-AcpiPsCompleteOp (
-    ACPI_WALK_STATE         *WalkState,
-    ACPI_PARSE_OBJECT       **Op,
-    ACPI_STATUS             Status);
-
-static ACPI_STATUS
-AcpiPsCompleteFinalOp (
-    ACPI_WALK_STATE         *WalkState,
-    ACPI_PARSE_OBJECT       *Op,
-    ACPI_STATUS             Status);
-
 static void
 AcpiPsLinkModuleCode (
     ACPI_PARSE_OBJECT       *ParentOp,
@@ -109,341 +78,6 @@ AcpiPsLinkModuleCode (
 
 /*******************************************************************************
  *
- * FUNCTION:    AcpiPsGetAmlOpcode
- *
- * PARAMETERS:  WalkState           - Current state
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Extract the next AML opcode from the input stream.
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-AcpiPsGetAmlOpcode (
-    ACPI_WALK_STATE         *WalkState)
-{
-
-    ACPI_FUNCTION_TRACE_PTR (PsGetAmlOpcode, WalkState);
-
-
-    WalkState->AmlOffset = (UINT32) ACPI_PTR_DIFF (WalkState->ParserState.Aml,
-                                WalkState->ParserState.AmlStart);
-    WalkState->Opcode = AcpiPsPeekOpcode (&(WalkState->ParserState));
-
-    /*
-     * First cut to determine what we have found:
-     * 1) A valid AML opcode
-     * 2) A name string
-     * 3) An unknown/invalid opcode
-     */
-    WalkState->OpInfo = AcpiPsGetOpcodeInfo (WalkState->Opcode);
-
-    switch (WalkState->OpInfo->Class)
-    {
-    case AML_CLASS_ASCII:
-    case AML_CLASS_PREFIX:
-        /*
-         * Starts with a valid prefix or ASCII char, this is a name
-         * string. Convert the bare name string to a namepath.
-         */
-        WalkState->Opcode = AML_INT_NAMEPATH_OP;
-        WalkState->ArgTypes = ARGP_NAMESTRING;
-        break;
-
-    case AML_CLASS_UNKNOWN:
-
-        /* The opcode is unrecognized. Complain and skip unknown opcodes */
-
-        if (WalkState->PassNumber == 2)
-        {
-            ACPI_ERROR ((AE_INFO,
-                "Unknown opcode 0x%.2X at table offset 0x%.4X, ignoring",
-                WalkState->Opcode,
-                (UINT32) (WalkState->AmlOffset + sizeof (ACPI_TABLE_HEADER))));
-
-            ACPI_DUMP_BUFFER (WalkState->ParserState.Aml - 16, 48);
-
-#ifdef ACPI_ASL_COMPILER
-            /*
-             * This is executed for the disassembler only. Output goes
-             * to the disassembled ASL output file.
-             */
-            AcpiOsPrintf (
-                "/*\nError: Unknown opcode 0x%.2X at table offset 0x%.4X, context:\n",
-                WalkState->Opcode,
-                (UINT32) (WalkState->AmlOffset + sizeof (ACPI_TABLE_HEADER)));
-
-            /* Dump the context surrounding the invalid opcode */
-
-            AcpiUtDumpBuffer (((UINT8 *) WalkState->ParserState.Aml - 16),
-                48, DB_BYTE_DISPLAY,
-                WalkState->AmlOffset + sizeof (ACPI_TABLE_HEADER) - 16);
-            AcpiOsPrintf (" */\n");
-#endif
-        }
-
-        /* Increment past one-byte or two-byte opcode */
-
-        WalkState->ParserState.Aml++;
-        if (WalkState->Opcode > 0xFF) /* Can only happen if first byte is 0x5B */
-        {
-            WalkState->ParserState.Aml++;
-        }
-
-        return_ACPI_STATUS (AE_CTRL_PARSE_CONTINUE);
-
-    default:
-
-        /* Found opcode info, this is a normal opcode */
-
-        WalkState->ParserState.Aml += AcpiPsGetOpcodeSize (WalkState->Opcode);
-        WalkState->ArgTypes = WalkState->OpInfo->ParseArgs;
-        break;
-    }
-
-    return_ACPI_STATUS (AE_OK);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiPsBuildNamedOp
- *
- * PARAMETERS:  WalkState           - Current state
- *              AmlOpStart          - Begin of named Op in AML
- *              UnnamedOp           - Early Op (not a named Op)
- *              Op                  - Returned Op
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Parse a named Op
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-AcpiPsBuildNamedOp (
-    ACPI_WALK_STATE         *WalkState,
-    UINT8                   *AmlOpStart,
-    ACPI_PARSE_OBJECT       *UnnamedOp,
-    ACPI_PARSE_OBJECT       **Op)
-{
-    ACPI_STATUS             Status = AE_OK;
-    ACPI_PARSE_OBJECT       *Arg = NULL;
-
-
-    ACPI_FUNCTION_TRACE_PTR (PsBuildNamedOp, WalkState);
-
-
-    UnnamedOp->Common.Value.Arg = NULL;
-    UnnamedOp->Common.ArgListLength = 0;
-    UnnamedOp->Common.AmlOpcode = WalkState->Opcode;
-
-    /*
-     * Get and append arguments until we find the node that contains
-     * the name (the type ARGP_NAME).
-     */
-    while (GET_CURRENT_ARG_TYPE (WalkState->ArgTypes) &&
-          (GET_CURRENT_ARG_TYPE (WalkState->ArgTypes) != ARGP_NAME))
-    {
-        Status = AcpiPsGetNextArg (WalkState, &(WalkState->ParserState),
-                    GET_CURRENT_ARG_TYPE (WalkState->ArgTypes), &Arg);
-        if (ACPI_FAILURE (Status))
-        {
-            return_ACPI_STATUS (Status);
-        }
-
-        AcpiPsAppendArg (UnnamedOp, Arg);
-        INCREMENT_ARG_LIST (WalkState->ArgTypes);
-    }
-
-    /*
-     * Make sure that we found a NAME and didn't run out of arguments
-     */
-    if (!GET_CURRENT_ARG_TYPE (WalkState->ArgTypes))
-    {
-        return_ACPI_STATUS (AE_AML_NO_OPERAND);
-    }
-
-    /* We know that this arg is a name, move to next arg */
-
-    INCREMENT_ARG_LIST (WalkState->ArgTypes);
-
-    /*
-     * Find the object. This will either insert the object into
-     * the namespace or simply look it up
-     */
-    WalkState->Op = NULL;
-
-    Status = WalkState->DescendingCallback (WalkState, Op);
-    if (ACPI_FAILURE (Status))
-    {
-        ACPI_EXCEPTION ((AE_INFO, Status, "During name lookup/catalog"));
-        return_ACPI_STATUS (Status);
-    }
-
-    if (!*Op)
-    {
-        return_ACPI_STATUS (AE_CTRL_PARSE_CONTINUE);
-    }
-
-    Status = AcpiPsNextParseState (WalkState, *Op, Status);
-    if (ACPI_FAILURE (Status))
-    {
-        if (Status == AE_CTRL_PENDING)
-        {
-            return_ACPI_STATUS (AE_CTRL_PARSE_PENDING);
-        }
-        return_ACPI_STATUS (Status);
-    }
-
-    AcpiPsAppendArg (*Op, UnnamedOp->Common.Value.Arg);
-    AcpiGbl_Depth++;
-
-    if ((*Op)->Common.AmlOpcode == AML_REGION_OP ||
-        (*Op)->Common.AmlOpcode == AML_DATA_REGION_OP)
-    {
-        /*
-         * Defer final parsing of an OperationRegion body, because we don't
-         * have enough info in the first pass to parse it correctly (i.e.,
-         * there may be method calls within the TermArg elements of the body.)
-         *
-         * However, we must continue parsing because the opregion is not a
-         * standalone package -- we don't know where the end is at this point.
-         *
-         * (Length is unknown until parse of the body complete)
-         */
-        (*Op)->Named.Data = AmlOpStart;
-        (*Op)->Named.Length = 0;
-    }
-
-    return_ACPI_STATUS (AE_OK);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiPsCreateOp
- *
- * PARAMETERS:  WalkState           - Current state
- *              AmlOpStart          - Op start in AML
- *              NewOp               - Returned Op
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Get Op from AML
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-AcpiPsCreateOp (
-    ACPI_WALK_STATE         *WalkState,
-    UINT8                   *AmlOpStart,
-    ACPI_PARSE_OBJECT       **NewOp)
-{
-    ACPI_STATUS             Status = AE_OK;
-    ACPI_PARSE_OBJECT       *Op;
-    ACPI_PARSE_OBJECT       *NamedOp = NULL;
-    ACPI_PARSE_OBJECT       *ParentScope;
-    UINT8                   ArgumentCount;
-    const ACPI_OPCODE_INFO  *OpInfo;
-
-
-    ACPI_FUNCTION_TRACE_PTR (PsCreateOp, WalkState);
-
-
-    Status = AcpiPsGetAmlOpcode (WalkState);
-    if (Status == AE_CTRL_PARSE_CONTINUE)
-    {
-        return_ACPI_STATUS (AE_CTRL_PARSE_CONTINUE);
-    }
-
-    /* Create Op structure and append to parent's argument list */
-
-    WalkState->OpInfo = AcpiPsGetOpcodeInfo (WalkState->Opcode);
-    Op = AcpiPsAllocOp (WalkState->Opcode);
-    if (!Op)
-    {
-        return_ACPI_STATUS (AE_NO_MEMORY);
-    }
-
-    if (WalkState->OpInfo->Flags & AML_NAMED)
-    {
-        Status = AcpiPsBuildNamedOp (WalkState, AmlOpStart, Op, &NamedOp);
-        AcpiPsFreeOp (Op);
-        if (ACPI_FAILURE (Status))
-        {
-            return_ACPI_STATUS (Status);
-        }
-
-        *NewOp = NamedOp;
-        return_ACPI_STATUS (AE_OK);
-    }
-
-    /* Not a named opcode, just allocate Op and append to parent */
-
-    if (WalkState->OpInfo->Flags & AML_CREATE)
-    {
-        /*
-         * Backup to beginning of CreateXXXfield declaration
-         * BodyLength is unknown until we parse the body
-         */
-        Op->Named.Data = AmlOpStart;
-        Op->Named.Length = 0;
-    }
-
-    if (WalkState->Opcode == AML_BANK_FIELD_OP)
-    {
-        /*
-         * Backup to beginning of BankField declaration
-         * BodyLength is unknown until we parse the body
-         */
-        Op->Named.Data = AmlOpStart;
-        Op->Named.Length = 0;
-    }
-
-    ParentScope = AcpiPsGetParentScope (&(WalkState->ParserState));
-    AcpiPsAppendArg (ParentScope, Op);
-
-    if (ParentScope)
-    {
-        OpInfo = AcpiPsGetOpcodeInfo (ParentScope->Common.AmlOpcode);
-        if (OpInfo->Flags & AML_HAS_TARGET)
-        {
-            ArgumentCount = AcpiPsGetArgumentCount (OpInfo->Type);
-            if (ParentScope->Common.ArgListLength > ArgumentCount)
-            {
-                Op->Common.Flags |= ACPI_PARSEOP_TARGET;
-            }
-        }
-        else if (ParentScope->Common.AmlOpcode == AML_INCREMENT_OP)
-        {
-            Op->Common.Flags |= ACPI_PARSEOP_TARGET;
-        }
-    }
-
-    if (WalkState->DescendingCallback != NULL)
-    {
-        /*
-         * Find the object. This will either insert the object into
-         * the namespace or simply look it up
-         */
-        WalkState->Op = *NewOp = Op;
-
-        Status = WalkState->DescendingCallback (WalkState, &Op);
-        Status = AcpiPsNextParseState (WalkState, Op, Status);
-        if (Status == AE_CTRL_PENDING)
-        {
-            Status = AE_CTRL_PARSE_PENDING;
-        }
-    }
-
-    return_ACPI_STATUS (Status);
-}
-
-
-/*******************************************************************************
- *
  * FUNCTION:    AcpiPsGetArguments
  *
  * PARAMETERS:  WalkState           - Current state
@@ -746,298 +380,6 @@ AcpiPsLinkModuleCode (
     }
 }
 
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiPsCompleteOp
- *
- * PARAMETERS:  WalkState           - Current state
- *              Op                  - Returned Op
- *              Status              - Parse status before complete Op
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Complete Op
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-AcpiPsCompleteOp (
-    ACPI_WALK_STATE         *WalkState,
-    ACPI_PARSE_OBJECT       **Op,
-    ACPI_STATUS             Status)
-{
-    ACPI_STATUS             Status2;
-
-
-    ACPI_FUNCTION_TRACE_PTR (PsCompleteOp, WalkState);
-
-
-    /*
-     * Finished one argument of the containing scope
-     */
-    WalkState->ParserState.Scope->ParseScope.ArgCount--;
-
-    /* Close this Op (will result in parse subtree deletion) */
-
-    Status2 = AcpiPsCompleteThisOp (WalkState, *Op);
-    if (ACPI_FAILURE (Status2))
-    {
-        return_ACPI_STATUS (Status2);
-    }
-
-    *Op = NULL;
-
-    switch (Status)
-    {
-    case AE_OK:
-        break;
-
-
-    case AE_CTRL_TRANSFER:
-
-        /* We are about to transfer to a called method */
-
-        WalkState->PrevOp = NULL;
-        WalkState->PrevArgTypes = WalkState->ArgTypes;
-        return_ACPI_STATUS (Status);
-
-
-    case AE_CTRL_END:
-
-        AcpiPsPopScope (&(WalkState->ParserState), Op,
-            &WalkState->ArgTypes, &WalkState->ArgCount);
-
-        if (*Op)
-        {
-            WalkState->Op = *Op;
-            WalkState->OpInfo = AcpiPsGetOpcodeInfo ((*Op)->Common.AmlOpcode);
-            WalkState->Opcode = (*Op)->Common.AmlOpcode;
-
-            Status = WalkState->AscendingCallback (WalkState);
-            Status = AcpiPsNextParseState (WalkState, *Op, Status);
-
-            Status2 = AcpiPsCompleteThisOp (WalkState, *Op);
-            if (ACPI_FAILURE (Status2))
-            {
-                return_ACPI_STATUS (Status2);
-            }
-        }
-
-        Status = AE_OK;
-        break;
-
-
-    case AE_CTRL_BREAK:
-    case AE_CTRL_CONTINUE:
-
-        /* Pop off scopes until we find the While */
-
-        while (!(*Op) || ((*Op)->Common.AmlOpcode != AML_WHILE_OP))
-        {
-            AcpiPsPopScope (&(WalkState->ParserState), Op,
-                &WalkState->ArgTypes, &WalkState->ArgCount);
-        }
-
-        /* Close this iteration of the While loop */
-
-        WalkState->Op = *Op;
-        WalkState->OpInfo = AcpiPsGetOpcodeInfo ((*Op)->Common.AmlOpcode);
-        WalkState->Opcode = (*Op)->Common.AmlOpcode;
-
-        Status = WalkState->AscendingCallback (WalkState);
-        Status = AcpiPsNextParseState (WalkState, *Op, Status);
-
-        Status2 = AcpiPsCompleteThisOp (WalkState, *Op);
-        if (ACPI_FAILURE (Status2))
-        {
-            return_ACPI_STATUS (Status2);
-        }
-
-        Status = AE_OK;
-        break;
-
-
-    case AE_CTRL_TERMINATE:
-
-        /* Clean up */
-        do
-        {
-            if (*Op)
-            {
-                Status2 = AcpiPsCompleteThisOp (WalkState, *Op);
-                if (ACPI_FAILURE (Status2))
-                {
-                    return_ACPI_STATUS (Status2);
-                }
-
-                AcpiUtDeleteGenericState (
-                    AcpiUtPopGenericState (&WalkState->ControlState));
-            }
-
-            AcpiPsPopScope (&(WalkState->ParserState), Op,
-                &WalkState->ArgTypes, &WalkState->ArgCount);
-
-        } while (*Op);
-
-        return_ACPI_STATUS (AE_OK);
-
-
-    default:  /* All other non-AE_OK status */
-
-        do
-        {
-            if (*Op)
-            {
-                Status2 = AcpiPsCompleteThisOp (WalkState, *Op);
-                if (ACPI_FAILURE (Status2))
-                {
-                    return_ACPI_STATUS (Status2);
-                }
-            }
-
-            AcpiPsPopScope (&(WalkState->ParserState), Op,
-                &WalkState->ArgTypes, &WalkState->ArgCount);
-
-        } while (*Op);
-
-
-#if 0
-        /*
-         * TBD: Cleanup parse ops on error
-         */
-        if (*Op == NULL)
-        {
-            AcpiPsPopScope (ParserState, Op,
-                &WalkState->ArgTypes, &WalkState->ArgCount);
-        }
-#endif
-        WalkState->PrevOp = NULL;
-        WalkState->PrevArgTypes = WalkState->ArgTypes;
-        return_ACPI_STATUS (Status);
-    }
-
-    /* This scope complete? */
-
-    if (AcpiPsHasCompletedScope (&(WalkState->ParserState)))
-    {
-        AcpiPsPopScope (&(WalkState->ParserState), Op,
-            &WalkState->ArgTypes, &WalkState->ArgCount);
-        ACPI_DEBUG_PRINT ((ACPI_DB_PARSE, "Popped scope, Op=%p\n", *Op));
-    }
-    else
-    {
-        *Op = NULL;
-    }
-
-    return_ACPI_STATUS (AE_OK);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiPsCompleteFinalOp
- *
- * PARAMETERS:  WalkState           - Current state
- *              Op                  - Current Op
- *              Status              - Current parse status before complete last
- *                                    Op
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Complete last Op.
- *
- ******************************************************************************/
-
-static ACPI_STATUS
-AcpiPsCompleteFinalOp (
-    ACPI_WALK_STATE         *WalkState,
-    ACPI_PARSE_OBJECT       *Op,
-    ACPI_STATUS             Status)
-{
-    ACPI_STATUS             Status2;
-
-
-    ACPI_FUNCTION_TRACE_PTR (PsCompleteFinalOp, WalkState);
-
-
-    /*
-     * Complete the last Op (if not completed), and clear the scope stack.
-     * It is easily possible to end an AML "package" with an unbounded number
-     * of open scopes (such as when several ASL blocks are closed with
-     * sequential closing braces). We want to terminate each one cleanly.
-     */
-    ACPI_DEBUG_PRINT ((ACPI_DB_PARSE, "AML package complete at Op %p\n", Op));
-    do
-    {
-        if (Op)
-        {
-            if (WalkState->AscendingCallback != NULL)
-            {
-                WalkState->Op = Op;
-                WalkState->OpInfo = AcpiPsGetOpcodeInfo (Op->Common.AmlOpcode);
-                WalkState->Opcode = Op->Common.AmlOpcode;
-
-                Status = WalkState->AscendingCallback (WalkState);
-                Status = AcpiPsNextParseState (WalkState, Op, Status);
-                if (Status == AE_CTRL_PENDING)
-                {
-                    Status = AcpiPsCompleteOp (WalkState, &Op, AE_OK);
-                    if (ACPI_FAILURE (Status))
-                    {
-                        return_ACPI_STATUS (Status);
-                    }
-                }
-
-                if (Status == AE_CTRL_TERMINATE)
-                {
-                    Status = AE_OK;
-
-                    /* Clean up */
-                    do
-                    {
-                        if (Op)
-                        {
-                            Status2 = AcpiPsCompleteThisOp (WalkState, Op);
-                            if (ACPI_FAILURE (Status2))
-                            {
-                                return_ACPI_STATUS (Status2);
-                            }
-                        }
-
-                        AcpiPsPopScope (&(WalkState->ParserState), &Op,
-                            &WalkState->ArgTypes, &WalkState->ArgCount);
-
-                    } while (Op);
-
-                    return_ACPI_STATUS (Status);
-                }
-
-                else if (ACPI_FAILURE (Status))
-                {
-                    /* First error is most important */
-
-                    (void) AcpiPsCompleteThisOp (WalkState, Op);
-                    return_ACPI_STATUS (Status);
-                }
-            }
-
-            Status2 = AcpiPsCompleteThisOp (WalkState, Op);
-            if (ACPI_FAILURE (Status2))
-            {
-                return_ACPI_STATUS (Status2);
-            }
-        }
-
-        AcpiPsPopScope (&(WalkState->ParserState), &Op, &WalkState->ArgTypes,
-            &WalkState->ArgCount);
-
-    } while (Op);
-
-    return_ACPI_STATUS (Status);
-}
-
-
 /*******************************************************************************
  *
  * FUNCTION:    AcpiPsParseLoop
@@ -1225,11 +567,6 @@ AcpiPsParseLoop (
         WalkState->OpInfo = AcpiPsGetOpcodeInfo (Op->Common.AmlOpcode);
         if (WalkState->OpInfo->Flags & AML_NAMED)
         {
-            if (AcpiGbl_Depth)
-            {
-                AcpiGbl_Depth--;
-            }
-
             if (Op->Common.AmlOpcode == AML_REGION_OP ||
                 Op->Common.AmlOpcode == AML_DATA_REGION_OP)
             {
diff --git a/sys/contrib/dev/acpica/components/parser/psobject.c b/sys/contrib/dev/acpica/components/parser/psobject.c
new file mode 100644
index 0000000..397c6f9
--- /dev/null
+++ b/sys/contrib/dev/acpica/components/parser/psobject.c
@@ -0,0 +1,683 @@
+/******************************************************************************
+ *
+ * Module Name: psobject - Support for parse objects
+ *
+ *****************************************************************************/
+
+/*
+ * Copyright (C) 2000 - 2013, Intel Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ * 3. Neither the names of the above-listed copyright holders nor the names
+ *    of any contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ */
+
+
+#include <contrib/dev/acpica/include/acpi.h>
+#include <contrib/dev/acpica/include/accommon.h>
+#include <contrib/dev/acpica/include/acparser.h>
+#include <contrib/dev/acpica/include/amlcode.h>
+
+#define _COMPONENT          ACPI_PARSER
+        ACPI_MODULE_NAME    ("psobject")
+
+
+/* Local prototypes */
+
+static ACPI_STATUS
+AcpiPsGetAmlOpcode (
+    ACPI_WALK_STATE         *WalkState);
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiPsGetAmlOpcode
+ *
+ * PARAMETERS:  WalkState           - Current state
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Extract the next AML opcode from the input stream.
+ *
+ ******************************************************************************/
+
+static ACPI_STATUS
+AcpiPsGetAmlOpcode (
+    ACPI_WALK_STATE         *WalkState)
+{
+
+    ACPI_FUNCTION_TRACE_PTR (PsGetAmlOpcode, WalkState);
+
+
+    WalkState->AmlOffset = (UINT32) ACPI_PTR_DIFF (WalkState->ParserState.Aml,
+                                WalkState->ParserState.AmlStart);
+    WalkState->Opcode = AcpiPsPeekOpcode (&(WalkState->ParserState));
+
+    /*
+     * First cut to determine what we have found:
+     * 1) A valid AML opcode
+     * 2) A name string
+     * 3) An unknown/invalid opcode
+     */
+    WalkState->OpInfo = AcpiPsGetOpcodeInfo (WalkState->Opcode);
+
+    switch (WalkState->OpInfo->Class)
+    {
+    case AML_CLASS_ASCII:
+    case AML_CLASS_PREFIX:
+        /*
+         * Starts with a valid prefix or ASCII char, this is a name
+         * string. Convert the bare name string to a namepath.
+         */
+        WalkState->Opcode = AML_INT_NAMEPATH_OP;
+        WalkState->ArgTypes = ARGP_NAMESTRING;
+        break;
+
+    case AML_CLASS_UNKNOWN:
+
+        /* The opcode is unrecognized. Complain and skip unknown opcodes */
+
+        if (WalkState->PassNumber == 2)
+        {
+            ACPI_ERROR ((AE_INFO,
+                "Unknown opcode 0x%.2X at table offset 0x%.4X, ignoring",
+                WalkState->Opcode,
+                (UINT32) (WalkState->AmlOffset + sizeof (ACPI_TABLE_HEADER))));
+
+            ACPI_DUMP_BUFFER ((WalkState->ParserState.Aml - 16), 48);
+
+#ifdef ACPI_ASL_COMPILER
+            /*
+             * This is executed for the disassembler only. Output goes
+             * to the disassembled ASL output file.
+             */
+            AcpiOsPrintf (
+                "/*\nError: Unknown opcode 0x%.2X at table offset 0x%.4X, context:\n",
+                WalkState->Opcode,
+                (UINT32) (WalkState->AmlOffset + sizeof (ACPI_TABLE_HEADER)));
+
+            /* Dump the context surrounding the invalid opcode */
+
+            AcpiUtDumpBuffer (((UINT8 *) WalkState->ParserState.Aml - 16),
+                48, DB_BYTE_DISPLAY,
+                (WalkState->AmlOffset + sizeof (ACPI_TABLE_HEADER) - 16));
+            AcpiOsPrintf (" */\n");
+#endif
+        }
+
+        /* Increment past one-byte or two-byte opcode */
+
+        WalkState->ParserState.Aml++;
+        if (WalkState->Opcode > 0xFF) /* Can only happen if first byte is 0x5B */
+        {
+            WalkState->ParserState.Aml++;
+        }
+
+        return_ACPI_STATUS (AE_CTRL_PARSE_CONTINUE);
+
+    default:
+
+        /* Found opcode info, this is a normal opcode */
+
+        WalkState->ParserState.Aml += AcpiPsGetOpcodeSize (WalkState->Opcode);
+        WalkState->ArgTypes = WalkState->OpInfo->ParseArgs;
+        break;
+    }
+
+    return_ACPI_STATUS (AE_OK);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiPsBuildNamedOp
+ *
+ * PARAMETERS:  WalkState           - Current state
+ *              AmlOpStart          - Begin of named Op in AML
+ *              UnnamedOp           - Early Op (not a named Op)
+ *              Op                  - Returned Op
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Parse a named Op
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+AcpiPsBuildNamedOp (
+    ACPI_WALK_STATE         *WalkState,
+    UINT8                   *AmlOpStart,
+    ACPI_PARSE_OBJECT       *UnnamedOp,
+    ACPI_PARSE_OBJECT       **Op)
+{
+    ACPI_STATUS             Status = AE_OK;
+    ACPI_PARSE_OBJECT       *Arg = NULL;
+
+
+    ACPI_FUNCTION_TRACE_PTR (PsBuildNamedOp, WalkState);
+
+
+    UnnamedOp->Common.Value.Arg = NULL;
+    UnnamedOp->Common.ArgListLength = 0;
+    UnnamedOp->Common.AmlOpcode = WalkState->Opcode;
+
+    /*
+     * Get and append arguments until we find the node that contains
+     * the name (the type ARGP_NAME).
+     */
+    while (GET_CURRENT_ARG_TYPE (WalkState->ArgTypes) &&
+          (GET_CURRENT_ARG_TYPE (WalkState->ArgTypes) != ARGP_NAME))
+    {
+        Status = AcpiPsGetNextArg (WalkState, &(WalkState->ParserState),
+                    GET_CURRENT_ARG_TYPE (WalkState->ArgTypes), &Arg);
+        if (ACPI_FAILURE (Status))
+        {
+            return_ACPI_STATUS (Status);
+        }
+
+        AcpiPsAppendArg (UnnamedOp, Arg);
+        INCREMENT_ARG_LIST (WalkState->ArgTypes);
+    }
+
+    /*
+     * Make sure that we found a NAME and didn't run out of arguments
+     */
+    if (!GET_CURRENT_ARG_TYPE (WalkState->ArgTypes))
+    {
+        return_ACPI_STATUS (AE_AML_NO_OPERAND);
+    }
+
+    /* We know that this arg is a name, move to next arg */
+
+    INCREMENT_ARG_LIST (WalkState->ArgTypes);
+
+    /*
+     * Find the object. This will either insert the object into
+     * the namespace or simply look it up
+     */
+    WalkState->Op = NULL;
+
+    Status = WalkState->DescendingCallback (WalkState, Op);
+    if (ACPI_FAILURE (Status))
+    {
+        ACPI_EXCEPTION ((AE_INFO, Status, "During name lookup/catalog"));
+        return_ACPI_STATUS (Status);
+    }
+
+    if (!*Op)
+    {
+        return_ACPI_STATUS (AE_CTRL_PARSE_CONTINUE);
+    }
+
+    Status = AcpiPsNextParseState (WalkState, *Op, Status);
+    if (ACPI_FAILURE (Status))
+    {
+        if (Status == AE_CTRL_PENDING)
+        {
+            return_ACPI_STATUS (AE_CTRL_PARSE_PENDING);
+        }
+        return_ACPI_STATUS (Status);
+    }
+
+    AcpiPsAppendArg (*Op, UnnamedOp->Common.Value.Arg);
+
+    if ((*Op)->Common.AmlOpcode == AML_REGION_OP ||
+        (*Op)->Common.AmlOpcode == AML_DATA_REGION_OP)
+    {
+        /*
+         * Defer final parsing of an OperationRegion body, because we don't
+         * have enough info in the first pass to parse it correctly (i.e.,
+         * there may be method calls within the TermArg elements of the body.)
+         *
+         * However, we must continue parsing because the opregion is not a
+         * standalone package -- we don't know where the end is at this point.
+         *
+         * (Length is unknown until parse of the body complete)
+         */
+        (*Op)->Named.Data = AmlOpStart;
+        (*Op)->Named.Length = 0;
+    }
+
+    return_ACPI_STATUS (AE_OK);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiPsCreateOp
+ *
+ * PARAMETERS:  WalkState           - Current state
+ *              AmlOpStart          - Op start in AML
+ *              NewOp               - Returned Op
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Get Op from AML
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+AcpiPsCreateOp (
+    ACPI_WALK_STATE         *WalkState,
+    UINT8                   *AmlOpStart,
+    ACPI_PARSE_OBJECT       **NewOp)
+{
+    ACPI_STATUS             Status = AE_OK;
+    ACPI_PARSE_OBJECT       *Op;
+    ACPI_PARSE_OBJECT       *NamedOp = NULL;
+    ACPI_PARSE_OBJECT       *ParentScope;
+    UINT8                   ArgumentCount;
+    const ACPI_OPCODE_INFO  *OpInfo;
+
+
+    ACPI_FUNCTION_TRACE_PTR (PsCreateOp, WalkState);
+
+
+    Status = AcpiPsGetAmlOpcode (WalkState);
+    if (Status == AE_CTRL_PARSE_CONTINUE)
+    {
+        return_ACPI_STATUS (AE_CTRL_PARSE_CONTINUE);
+    }
+
+    /* Create Op structure and append to parent's argument list */
+
+    WalkState->OpInfo = AcpiPsGetOpcodeInfo (WalkState->Opcode);
+    Op = AcpiPsAllocOp (WalkState->Opcode);
+    if (!Op)
+    {
+        return_ACPI_STATUS (AE_NO_MEMORY);
+    }
+
+    if (WalkState->OpInfo->Flags & AML_NAMED)
+    {
+        Status = AcpiPsBuildNamedOp (WalkState, AmlOpStart, Op, &NamedOp);
+        AcpiPsFreeOp (Op);
+        if (ACPI_FAILURE (Status))
+        {
+            return_ACPI_STATUS (Status);
+        }
+
+        *NewOp = NamedOp;
+        return_ACPI_STATUS (AE_OK);
+    }
+
+    /* Not a named opcode, just allocate Op and append to parent */
+
+    if (WalkState->OpInfo->Flags & AML_CREATE)
+    {
+        /*
+         * Backup to beginning of CreateXXXfield declaration
+         * BodyLength is unknown until we parse the body
+         */
+        Op->Named.Data = AmlOpStart;
+        Op->Named.Length = 0;
+    }
+
+    if (WalkState->Opcode == AML_BANK_FIELD_OP)
+    {
+        /*
+         * Backup to beginning of BankField declaration
+         * BodyLength is unknown until we parse the body
+         */
+        Op->Named.Data = AmlOpStart;
+        Op->Named.Length = 0;
+    }
+
+    ParentScope = AcpiPsGetParentScope (&(WalkState->ParserState));
+    AcpiPsAppendArg (ParentScope, Op);
+
+    if (ParentScope)
+    {
+        OpInfo = AcpiPsGetOpcodeInfo (ParentScope->Common.AmlOpcode);
+        if (OpInfo->Flags & AML_HAS_TARGET)
+        {
+            ArgumentCount = AcpiPsGetArgumentCount (OpInfo->Type);
+            if (ParentScope->Common.ArgListLength > ArgumentCount)
+            {
+                Op->Common.Flags |= ACPI_PARSEOP_TARGET;
+            }
+        }
+        else if (ParentScope->Common.AmlOpcode == AML_INCREMENT_OP)
+        {
+            Op->Common.Flags |= ACPI_PARSEOP_TARGET;
+        }
+    }
+
+    if (WalkState->DescendingCallback != NULL)
+    {
+        /*
+         * Find the object. This will either insert the object into
+         * the namespace or simply look it up
+         */
+        WalkState->Op = *NewOp = Op;
+
+        Status = WalkState->DescendingCallback (WalkState, &Op);
+        Status = AcpiPsNextParseState (WalkState, Op, Status);
+        if (Status == AE_CTRL_PENDING)
+        {
+            Status = AE_CTRL_PARSE_PENDING;
+        }
+    }
+
+    return_ACPI_STATUS (Status);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiPsCompleteOp
+ *
+ * PARAMETERS:  WalkState           - Current state
+ *              Op                  - Returned Op
+ *              Status              - Parse status before complete Op
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Complete Op
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+AcpiPsCompleteOp (
+    ACPI_WALK_STATE         *WalkState,
+    ACPI_PARSE_OBJECT       **Op,
+    ACPI_STATUS             Status)
+{
+    ACPI_STATUS             Status2;
+
+
+    ACPI_FUNCTION_TRACE_PTR (PsCompleteOp, WalkState);
+
+
+    /*
+     * Finished one argument of the containing scope
+     */
+    WalkState->ParserState.Scope->ParseScope.ArgCount--;
+
+    /* Close this Op (will result in parse subtree deletion) */
+
+    Status2 = AcpiPsCompleteThisOp (WalkState, *Op);
+    if (ACPI_FAILURE (Status2))
+    {
+        return_ACPI_STATUS (Status2);
+    }
+
+    *Op = NULL;
+
+    switch (Status)
+    {
+    case AE_OK:
+        break;
+
+
+    case AE_CTRL_TRANSFER:
+
+        /* We are about to transfer to a called method */
+
+        WalkState->PrevOp = NULL;
+        WalkState->PrevArgTypes = WalkState->ArgTypes;
+        return_ACPI_STATUS (Status);
+
+
+    case AE_CTRL_END:
+
+        AcpiPsPopScope (&(WalkState->ParserState), Op,
+            &WalkState->ArgTypes, &WalkState->ArgCount);
+
+        if (*Op)
+        {
+            WalkState->Op = *Op;
+            WalkState->OpInfo = AcpiPsGetOpcodeInfo ((*Op)->Common.AmlOpcode);
+            WalkState->Opcode = (*Op)->Common.AmlOpcode;
+
+            Status = WalkState->AscendingCallback (WalkState);
+            Status = AcpiPsNextParseState (WalkState, *Op, Status);
+
+            Status2 = AcpiPsCompleteThisOp (WalkState, *Op);
+            if (ACPI_FAILURE (Status2))
+            {
+                return_ACPI_STATUS (Status2);
+            }
+        }
+
+        Status = AE_OK;
+        break;
+
+
+    case AE_CTRL_BREAK:
+    case AE_CTRL_CONTINUE:
+
+        /* Pop off scopes until we find the While */
+
+        while (!(*Op) || ((*Op)->Common.AmlOpcode != AML_WHILE_OP))
+        {
+            AcpiPsPopScope (&(WalkState->ParserState), Op,
+                &WalkState->ArgTypes, &WalkState->ArgCount);
+        }
+
+        /* Close this iteration of the While loop */
+
+        WalkState->Op = *Op;
+        WalkState->OpInfo = AcpiPsGetOpcodeInfo ((*Op)->Common.AmlOpcode);
+        WalkState->Opcode = (*Op)->Common.AmlOpcode;
+
+        Status = WalkState->AscendingCallback (WalkState);
+        Status = AcpiPsNextParseState (WalkState, *Op, Status);
+
+        Status2 = AcpiPsCompleteThisOp (WalkState, *Op);
+        if (ACPI_FAILURE (Status2))
+        {
+            return_ACPI_STATUS (Status2);
+        }
+
+        Status = AE_OK;
+        break;
+
+
+    case AE_CTRL_TERMINATE:
+
+        /* Clean up */
+        do
+        {
+            if (*Op)
+            {
+                Status2 = AcpiPsCompleteThisOp (WalkState, *Op);
+                if (ACPI_FAILURE (Status2))
+                {
+                    return_ACPI_STATUS (Status2);
+                }
+
+                AcpiUtDeleteGenericState (
+                    AcpiUtPopGenericState (&WalkState->ControlState));
+            }
+
+            AcpiPsPopScope (&(WalkState->ParserState), Op,
+                &WalkState->ArgTypes, &WalkState->ArgCount);
+
+        } while (*Op);
+
+        return_ACPI_STATUS (AE_OK);
+
+
+    default:  /* All other non-AE_OK status */
+
+        do
+        {
+            if (*Op)
+            {
+                Status2 = AcpiPsCompleteThisOp (WalkState, *Op);
+                if (ACPI_FAILURE (Status2))
+                {
+                    return_ACPI_STATUS (Status2);
+                }
+            }
+
+            AcpiPsPopScope (&(WalkState->ParserState), Op,
+                &WalkState->ArgTypes, &WalkState->ArgCount);
+
+        } while (*Op);
+
+
+#if 0
+        /*
+         * TBD: Cleanup parse ops on error
+         */
+        if (*Op == NULL)
+        {
+            AcpiPsPopScope (ParserState, Op,
+                &WalkState->ArgTypes, &WalkState->ArgCount);
+        }
+#endif
+        WalkState->PrevOp = NULL;
+        WalkState->PrevArgTypes = WalkState->ArgTypes;
+        return_ACPI_STATUS (Status);
+    }
+
+    /* This scope complete? */
+
+    if (AcpiPsHasCompletedScope (&(WalkState->ParserState)))
+    {
+        AcpiPsPopScope (&(WalkState->ParserState), Op,
+            &WalkState->ArgTypes, &WalkState->ArgCount);
+        ACPI_DEBUG_PRINT ((ACPI_DB_PARSE, "Popped scope, Op=%p\n", *Op));
+    }
+    else
+    {
+        *Op = NULL;
+    }
+
+    return_ACPI_STATUS (AE_OK);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiPsCompleteFinalOp
+ *
+ * PARAMETERS:  WalkState           - Current state
+ *              Op                  - Current Op
+ *              Status              - Current parse status before complete last
+ *                                    Op
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Complete last Op.
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+AcpiPsCompleteFinalOp (
+    ACPI_WALK_STATE         *WalkState,
+    ACPI_PARSE_OBJECT       *Op,
+    ACPI_STATUS             Status)
+{
+    ACPI_STATUS             Status2;
+
+
+    ACPI_FUNCTION_TRACE_PTR (PsCompleteFinalOp, WalkState);
+
+
+    /*
+     * Complete the last Op (if not completed), and clear the scope stack.
+     * It is easily possible to end an AML "package" with an unbounded number
+     * of open scopes (such as when several ASL blocks are closed with
+     * sequential closing braces). We want to terminate each one cleanly.
+     */
+    ACPI_DEBUG_PRINT ((ACPI_DB_PARSE, "AML package complete at Op %p\n", Op));
+    do
+    {
+        if (Op)
+        {
+            if (WalkState->AscendingCallback != NULL)
+            {
+                WalkState->Op = Op;
+                WalkState->OpInfo = AcpiPsGetOpcodeInfo (Op->Common.AmlOpcode);
+                WalkState->Opcode = Op->Common.AmlOpcode;
+
+                Status = WalkState->AscendingCallback (WalkState);
+                Status = AcpiPsNextParseState (WalkState, Op, Status);
+                if (Status == AE_CTRL_PENDING)
+                {
+                    Status = AcpiPsCompleteOp (WalkState, &Op, AE_OK);
+                    if (ACPI_FAILURE (Status))
+                    {
+                        return_ACPI_STATUS (Status);
+                    }
+                }
+
+                if (Status == AE_CTRL_TERMINATE)
+                {
+                    Status = AE_OK;
+
+                    /* Clean up */
+                    do
+                    {
+                        if (Op)
+                        {
+                            Status2 = AcpiPsCompleteThisOp (WalkState, Op);
+                            if (ACPI_FAILURE (Status2))
+                            {
+                                return_ACPI_STATUS (Status2);
+                            }
+                        }
+
+                        AcpiPsPopScope (&(WalkState->ParserState), &Op,
+                            &WalkState->ArgTypes, &WalkState->ArgCount);
+
+                    } while (Op);
+
+                    return_ACPI_STATUS (Status);
+                }
+
+                else if (ACPI_FAILURE (Status))
+                {
+                    /* First error is most important */
+
+                    (void) AcpiPsCompleteThisOp (WalkState, Op);
+                    return_ACPI_STATUS (Status);
+                }
+            }
+
+            Status2 = AcpiPsCompleteThisOp (WalkState, Op);
+            if (ACPI_FAILURE (Status2))
+            {
+                return_ACPI_STATUS (Status2);
+            }
+        }
+
+        AcpiPsPopScope (&(WalkState->ParserState), &Op, &WalkState->ArgTypes,
+            &WalkState->ArgCount);
+
+    } while (Op);
+
+    return_ACPI_STATUS (Status);
+}
diff --git a/sys/contrib/dev/acpica/components/parser/psopcode.c b/sys/contrib/dev/acpica/components/parser/psopcode.c
index 85e170d..8ae0c80 100644
--- a/sys/contrib/dev/acpica/components/parser/psopcode.c
+++ b/sys/contrib/dev/acpica/components/parser/psopcode.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -44,7 +44,6 @@
 
 #include <contrib/dev/acpica/include/acpi.h>
 #include <contrib/dev/acpica/include/accommon.h>
-#include <contrib/dev/acpica/include/acparser.h>
 #include <contrib/dev/acpica/include/acopcode.h>
 #include <contrib/dev/acpica/include/amlcode.h>
 
@@ -53,9 +52,6 @@
         ACPI_MODULE_NAME    ("psopcode")
 
 
-static const UINT8          AcpiGbl_ArgumentCount[] = {0,1,1,1,1,2,2,2,2,3,3,6};
-
-
 /*******************************************************************************
  *
  * NAME:        AcpiGbl_AmlOpInfo
@@ -342,181 +338,3 @@ const ACPI_OPCODE_INFO    AcpiGbl_AmlOpInfo[AML_NUM_OPCODES] =
 
 /*! [End] no source code translation !*/
 };
-
-/*
- * This table is directly indexed by the opcodes, and returns an
- * index into the table above
- */
-static const UINT8 AcpiGbl_ShortOpIndex[256] =
-{
-/*              0     1     2     3     4     5     6     7  */
-/*              8     9     A     B     C     D     E     F  */
-/* 0x00 */    0x00, 0x01, _UNK, _UNK, _UNK, _UNK, 0x02, _UNK,
-/* 0x08 */    0x03, _UNK, 0x04, 0x05, 0x06, 0x07, 0x6E, _UNK,
-/* 0x10 */    0x08, 0x09, 0x0a, 0x6F, 0x0b, _UNK, _UNK, _UNK,
-/* 0x18 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0x20 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0x28 */    _UNK, _UNK, _UNK, _UNK, _UNK, 0x63, _PFX, _PFX,
-/* 0x30 */    0x67, 0x66, 0x68, 0x65, 0x69, 0x64, 0x6A, 0x7D,
-/* 0x38 */    0x7F, 0x80, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0x40 */    _UNK, _ASC, _ASC, _ASC, _ASC, _ASC, _ASC, _ASC,
-/* 0x48 */    _ASC, _ASC, _ASC, _ASC, _ASC, _ASC, _ASC, _ASC,
-/* 0x50 */    _ASC, _ASC, _ASC, _ASC, _ASC, _ASC, _ASC, _ASC,
-/* 0x58 */    _ASC, _ASC, _ASC, _UNK, _PFX, _UNK, _PFX, _ASC,
-/* 0x60 */    0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13,
-/* 0x68 */    0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, _UNK,
-/* 0x70 */    0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22,
-/* 0x78 */    0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a,
-/* 0x80 */    0x2b, 0x2c, 0x2d, 0x2e, 0x70, 0x71, 0x2f, 0x30,
-/* 0x88 */    0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x72,
-/* 0x90 */    0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x73, 0x74,
-/* 0x98 */    0x75, 0x76, _UNK, _UNK, 0x77, 0x78, 0x79, 0x7A,
-/* 0xA0 */    0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x60, 0x61,
-/* 0xA8 */    0x62, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0xB0 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0xB8 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0xC0 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0xC8 */    _UNK, _UNK, _UNK, _UNK, 0x44, _UNK, _UNK, _UNK,
-/* 0xD0 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0xD8 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0xE0 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0xE8 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0xF0 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0xF8 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, 0x45,
-};
-
-/*
- * This table is indexed by the second opcode of the extended opcode
- * pair. It returns an index into the opcode table (AcpiGbl_AmlOpInfo)
- */
-static const UINT8 AcpiGbl_LongOpIndex[NUM_EXTENDED_OPCODE] =
-{
-/*              0     1     2     3     4     5     6     7  */
-/*              8     9     A     B     C     D     E     F  */
-/* 0x00 */    _UNK, 0x46, 0x47, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0x08 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0x10 */    _UNK, _UNK, 0x48, 0x49, _UNK, _UNK, _UNK, _UNK,
-/* 0x18 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, 0x7B,
-/* 0x20 */    0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51,
-/* 0x28 */    0x52, 0x53, 0x54, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0x30 */    0x55, 0x56, 0x57, 0x7e, _UNK, _UNK, _UNK, _UNK,
-/* 0x38 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0x40 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0x48 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0x50 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0x58 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0x60 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0x68 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0x70 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0x78 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
-/* 0x80 */    0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,
-/* 0x88 */    0x7C,
-};
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiPsGetOpcodeInfo
- *
- * PARAMETERS:  Opcode              - The AML opcode
- *
- * RETURN:      A pointer to the info about the opcode.
- *
- * DESCRIPTION: Find AML opcode description based on the opcode.
- *              NOTE: This procedure must ALWAYS return a valid pointer!
- *
- ******************************************************************************/
-
-const ACPI_OPCODE_INFO *
-AcpiPsGetOpcodeInfo (
-    UINT16                  Opcode)
-{
-    ACPI_FUNCTION_NAME (PsGetOpcodeInfo);
-
-
-    /*
-     * Detect normal 8-bit opcode or extended 16-bit opcode
-     */
-    if (!(Opcode & 0xFF00))
-    {
-        /* Simple (8-bit) opcode: 0-255, can't index beyond table  */
-
-        return (&AcpiGbl_AmlOpInfo [AcpiGbl_ShortOpIndex [(UINT8) Opcode]]);
-    }
-
-    if (((Opcode & 0xFF00) == AML_EXTENDED_OPCODE) &&
-        (((UINT8) Opcode) <= MAX_EXTENDED_OPCODE))
-    {
-        /* Valid extended (16-bit) opcode */
-
-        return (&AcpiGbl_AmlOpInfo [AcpiGbl_LongOpIndex [(UINT8) Opcode]]);
-    }
-
-    /* Unknown AML opcode */
-
-    ACPI_DEBUG_PRINT ((ACPI_DB_EXEC,
-        "Unknown AML opcode [%4.4X]\n", Opcode));
-
-    return (&AcpiGbl_AmlOpInfo [_UNK]);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiPsGetOpcodeName
- *
- * PARAMETERS:  Opcode              - The AML opcode
- *
- * RETURN:      A pointer to the name of the opcode (ASCII String)
- *              Note: Never returns NULL.
- *
- * DESCRIPTION: Translate an opcode into a human-readable string
- *
- ******************************************************************************/
-
-char *
-AcpiPsGetOpcodeName (
-    UINT16                  Opcode)
-{
-#if defined(ACPI_DISASSEMBLER) || defined (ACPI_DEBUG_OUTPUT)
-
-    const ACPI_OPCODE_INFO  *Op;
-
-
-    Op = AcpiPsGetOpcodeInfo (Opcode);
-
-    /* Always guaranteed to return a valid pointer */
-
-    return (Op->Name);
-
-#else
-    return ("OpcodeName unavailable");
-
-#endif
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiPsGetArgumentCount
- *
- * PARAMETERS:  OpType              - Type associated with the AML opcode
- *
- * RETURN:      Argument count
- *
- * DESCRIPTION: Obtain the number of expected arguments for an AML opcode
- *
- ******************************************************************************/
-
-UINT8
-AcpiPsGetArgumentCount (
-    UINT32                  OpType)
-{
-
-    if (OpType <= AML_TYPE_EXEC_6A_0T_1R)
-    {
-        return (AcpiGbl_ArgumentCount[OpType]);
-    }
-
-    return (0);
-}
diff --git a/sys/contrib/dev/acpica/components/parser/psopinfo.c b/sys/contrib/dev/acpica/components/parser/psopinfo.c
new file mode 100644
index 0000000..3c6e4a5
--- /dev/null
+++ b/sys/contrib/dev/acpica/components/parser/psopinfo.c
@@ -0,0 +1,238 @@
+/******************************************************************************
+ *
+ * Module Name: psopinfo - AML opcode information functions and dispatch tables
+ *
+ *****************************************************************************/
+
+/*
+ * Copyright (C) 2000 - 2013, Intel Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ * 3. Neither the names of the above-listed copyright holders nor the names
+ *    of any contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ */
+
+
+#include <contrib/dev/acpica/include/acpi.h>
+#include <contrib/dev/acpica/include/accommon.h>
+#include <contrib/dev/acpica/include/acparser.h>
+#include <contrib/dev/acpica/include/acopcode.h>
+#include <contrib/dev/acpica/include/amlcode.h>
+
+
+#define _COMPONENT          ACPI_PARSER
+        ACPI_MODULE_NAME    ("psopinfo")
+
+
+extern const UINT8      AcpiGbl_ShortOpIndex[];
+extern const UINT8      AcpiGbl_LongOpIndex[];
+
+static const UINT8      AcpiGbl_ArgumentCount[] = {0,1,1,1,1,2,2,2,2,3,3,6};
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiPsGetOpcodeInfo
+ *
+ * PARAMETERS:  Opcode              - The AML opcode
+ *
+ * RETURN:      A pointer to the info about the opcode.
+ *
+ * DESCRIPTION: Find AML opcode description based on the opcode.
+ *              NOTE: This procedure must ALWAYS return a valid pointer!
+ *
+ ******************************************************************************/
+
+const ACPI_OPCODE_INFO *
+AcpiPsGetOpcodeInfo (
+    UINT16                  Opcode)
+{
+    ACPI_FUNCTION_NAME (PsGetOpcodeInfo);
+
+
+    /*
+     * Detect normal 8-bit opcode or extended 16-bit opcode
+     */
+    if (!(Opcode & 0xFF00))
+    {
+        /* Simple (8-bit) opcode: 0-255, can't index beyond table  */
+
+        return (&AcpiGbl_AmlOpInfo [AcpiGbl_ShortOpIndex [(UINT8) Opcode]]);
+    }
+
+    if (((Opcode & 0xFF00) == AML_EXTENDED_OPCODE) &&
+        (((UINT8) Opcode) <= MAX_EXTENDED_OPCODE))
+    {
+        /* Valid extended (16-bit) opcode */
+
+        return (&AcpiGbl_AmlOpInfo [AcpiGbl_LongOpIndex [(UINT8) Opcode]]);
+    }
+
+    /* Unknown AML opcode */
+
+    ACPI_DEBUG_PRINT ((ACPI_DB_EXEC,
+        "Unknown AML opcode [%4.4X]\n", Opcode));
+
+    return (&AcpiGbl_AmlOpInfo [_UNK]);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiPsGetOpcodeName
+ *
+ * PARAMETERS:  Opcode              - The AML opcode
+ *
+ * RETURN:      A pointer to the name of the opcode (ASCII String)
+ *              Note: Never returns NULL.
+ *
+ * DESCRIPTION: Translate an opcode into a human-readable string
+ *
+ ******************************************************************************/
+
+char *
+AcpiPsGetOpcodeName (
+    UINT16                  Opcode)
+{
+#if defined(ACPI_DISASSEMBLER) || defined (ACPI_DEBUG_OUTPUT)
+
+    const ACPI_OPCODE_INFO  *Op;
+
+
+    Op = AcpiPsGetOpcodeInfo (Opcode);
+
+    /* Always guaranteed to return a valid pointer */
+
+    return (Op->Name);
+
+#else
+    return ("OpcodeName unavailable");
+
+#endif
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiPsGetArgumentCount
+ *
+ * PARAMETERS:  OpType              - Type associated with the AML opcode
+ *
+ * RETURN:      Argument count
+ *
+ * DESCRIPTION: Obtain the number of expected arguments for an AML opcode
+ *
+ ******************************************************************************/
+
+UINT8
+AcpiPsGetArgumentCount (
+    UINT32                  OpType)
+{
+
+    if (OpType <= AML_TYPE_EXEC_6A_0T_1R)
+    {
+        return (AcpiGbl_ArgumentCount[OpType]);
+    }
+
+    return (0);
+}
+
+
+/*
+ * This table is directly indexed by the opcodes It returns
+ * an index into the opcode table (AcpiGbl_AmlOpInfo)
+ */
+const UINT8 AcpiGbl_ShortOpIndex[256] =
+{
+/*              0     1     2     3     4     5     6     7  */
+/*              8     9     A     B     C     D     E     F  */
+/* 0x00 */    0x00, 0x01, _UNK, _UNK, _UNK, _UNK, 0x02, _UNK,
+/* 0x08 */    0x03, _UNK, 0x04, 0x05, 0x06, 0x07, 0x6E, _UNK,
+/* 0x10 */    0x08, 0x09, 0x0a, 0x6F, 0x0b, _UNK, _UNK, _UNK,
+/* 0x18 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0x20 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0x28 */    _UNK, _UNK, _UNK, _UNK, _UNK, 0x63, _PFX, _PFX,
+/* 0x30 */    0x67, 0x66, 0x68, 0x65, 0x69, 0x64, 0x6A, 0x7D,
+/* 0x38 */    0x7F, 0x80, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0x40 */    _UNK, _ASC, _ASC, _ASC, _ASC, _ASC, _ASC, _ASC,
+/* 0x48 */    _ASC, _ASC, _ASC, _ASC, _ASC, _ASC, _ASC, _ASC,
+/* 0x50 */    _ASC, _ASC, _ASC, _ASC, _ASC, _ASC, _ASC, _ASC,
+/* 0x58 */    _ASC, _ASC, _ASC, _UNK, _PFX, _UNK, _PFX, _ASC,
+/* 0x60 */    0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13,
+/* 0x68 */    0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, _UNK,
+/* 0x70 */    0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22,
+/* 0x78 */    0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a,
+/* 0x80 */    0x2b, 0x2c, 0x2d, 0x2e, 0x70, 0x71, 0x2f, 0x30,
+/* 0x88 */    0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x72,
+/* 0x90 */    0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x73, 0x74,
+/* 0x98 */    0x75, 0x76, _UNK, _UNK, 0x77, 0x78, 0x79, 0x7A,
+/* 0xA0 */    0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x60, 0x61,
+/* 0xA8 */    0x62, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0xB0 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0xB8 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0xC0 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0xC8 */    _UNK, _UNK, _UNK, _UNK, 0x44, _UNK, _UNK, _UNK,
+/* 0xD0 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0xD8 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0xE0 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0xE8 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0xF0 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0xF8 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, 0x45,
+};
+
+/*
+ * This table is indexed by the second opcode of the extended opcode
+ * pair. It returns an index into the opcode table (AcpiGbl_AmlOpInfo)
+ */
+const UINT8 AcpiGbl_LongOpIndex[NUM_EXTENDED_OPCODE] =
+{
+/*              0     1     2     3     4     5     6     7  */
+/*              8     9     A     B     C     D     E     F  */
+/* 0x00 */    _UNK, 0x46, 0x47, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0x08 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0x10 */    _UNK, _UNK, 0x48, 0x49, _UNK, _UNK, _UNK, _UNK,
+/* 0x18 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, 0x7B,
+/* 0x20 */    0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51,
+/* 0x28 */    0x52, 0x53, 0x54, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0x30 */    0x55, 0x56, 0x57, 0x7e, _UNK, _UNK, _UNK, _UNK,
+/* 0x38 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0x40 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0x48 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0x50 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0x58 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0x60 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0x68 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0x70 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0x78 */    _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK, _UNK,
+/* 0x80 */    0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,
+/* 0x88 */    0x7C,
+};
diff --git a/sys/contrib/dev/acpica/components/parser/psparse.c b/sys/contrib/dev/acpica/components/parser/psparse.c
index 1193262..1dafdc1 100644
--- a/sys/contrib/dev/acpica/components/parser/psparse.c
+++ b/sys/contrib/dev/acpica/components/parser/psparse.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/parser/psscope.c b/sys/contrib/dev/acpica/components/parser/psscope.c
index 3d88272..3aa102bd 100644
--- a/sys/contrib/dev/acpica/components/parser/psscope.c
+++ b/sys/contrib/dev/acpica/components/parser/psscope.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/parser/pstree.c b/sys/contrib/dev/acpica/components/parser/pstree.c
index 9ad3412..277c702 100644
--- a/sys/contrib/dev/acpica/components/parser/pstree.c
+++ b/sys/contrib/dev/acpica/components/parser/pstree.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/parser/psutils.c b/sys/contrib/dev/acpica/components/parser/psutils.c
index ca0e1d3..4b555cc 100644
--- a/sys/contrib/dev/acpica/components/parser/psutils.c
+++ b/sys/contrib/dev/acpica/components/parser/psutils.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -238,17 +238,6 @@ AcpiPsIsLeadingChar (
 
 
 /*
- * Is "c" a namestring prefix character?
- */
-BOOLEAN
-AcpiPsIsPrefixChar (
-    UINT32                  c)
-{
-    return ((BOOLEAN) (c == '\\' || c == '^'));
-}
-
-
-/*
  * Get op's name (4-byte name segment) or 0 if unnamed
  */
 UINT32
diff --git a/sys/contrib/dev/acpica/components/parser/pswalk.c b/sys/contrib/dev/acpica/components/parser/pswalk.c
index 6dd2b0d..6b6e289 100644
--- a/sys/contrib/dev/acpica/components/parser/pswalk.c
+++ b/sys/contrib/dev/acpica/components/parser/pswalk.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/parser/psxface.c b/sys/contrib/dev/acpica/components/parser/psxface.c
index 1131e8c..6177baf 100644
--- a/sys/contrib/dev/acpica/components/parser/psxface.c
+++ b/sys/contrib/dev/acpica/components/parser/psxface.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/resources/rsaddr.c b/sys/contrib/dev/acpica/components/resources/rsaddr.c
index 80eb57a..a8fa79d 100644
--- a/sys/contrib/dev/acpica/components/resources/rsaddr.c
+++ b/sys/contrib/dev/acpica/components/resources/rsaddr.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/resources/rscalc.c b/sys/contrib/dev/acpica/components/resources/rscalc.c
index 3820aa0..3b43775 100644
--- a/sys/contrib/dev/acpica/components/resources/rscalc.c
+++ b/sys/contrib/dev/acpica/components/resources/rscalc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/resources/rscreate.c b/sys/contrib/dev/acpica/components/resources/rscreate.c
index 87a9ecf..86c10aa 100644
--- a/sys/contrib/dev/acpica/components/resources/rscreate.c
+++ b/sys/contrib/dev/acpica/components/resources/rscreate.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/resources/rsdump.c b/sys/contrib/dev/acpica/components/resources/rsdump.c
index acccad3..20c1893 100644
--- a/sys/contrib/dev/acpica/components/resources/rsdump.c
+++ b/sys/contrib/dev/acpica/components/resources/rsdump.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -119,315 +119,16 @@ AcpiRsDumpDescriptor (
     ACPI_RSDUMP_INFO *Table);
 
 
-#define ACPI_RSD_OFFSET(f)          (UINT8) ACPI_OFFSET (ACPI_RESOURCE_DATA,f)
-#define ACPI_PRT_OFFSET(f)          (UINT8) ACPI_OFFSET (ACPI_PCI_ROUTING_TABLE,f)
-#define ACPI_RSD_TABLE_SIZE(name)   (sizeof(name) / sizeof (ACPI_RSDUMP_INFO))
-
-
-/*******************************************************************************
- *
- * Resource Descriptor info tables
- *
- * Note: The first table entry must be a Title or Literal and must contain
- * the table length (number of table entries)
- *
- ******************************************************************************/
-
-ACPI_RSDUMP_INFO        AcpiRsDumpIrq[7] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpIrq),                "IRQ",                      NULL},
-    {ACPI_RSD_UINT8 ,   ACPI_RSD_OFFSET (Irq.DescriptorLength),             "Descriptor Length",        NULL},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Irq.Triggering),                   "Triggering",               AcpiGbl_HeDecode},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Irq.Polarity),                     "Polarity",                 AcpiGbl_LlDecode},
-    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Irq.Sharable),                     "Sharing",                  AcpiGbl_ShrDecode},
-    {ACPI_RSD_UINT8 ,   ACPI_RSD_OFFSET (Irq.InterruptCount),               "Interrupt Count",          NULL},
-    {ACPI_RSD_SHORTLIST,ACPI_RSD_OFFSET (Irq.Interrupts[0]),                "Interrupt List",           NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpDma[6] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpDma),                "DMA",                      NULL},
-    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Dma.Type),                         "Speed",                    AcpiGbl_TypDecode},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Dma.BusMaster),                    "Mastering",                AcpiGbl_BmDecode},
-    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Dma.Transfer),                     "Transfer Type",            AcpiGbl_SizDecode},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (Dma.ChannelCount),                 "Channel Count",            NULL},
-    {ACPI_RSD_SHORTLIST,ACPI_RSD_OFFSET (Dma.Channels[0]),                  "Channel List",             NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpStartDpf[4] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpStartDpf),           "Start-Dependent-Functions",NULL},
-    {ACPI_RSD_UINT8 ,   ACPI_RSD_OFFSET (StartDpf.DescriptorLength),        "Descriptor Length",        NULL},
-    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (StartDpf.CompatibilityPriority),   "Compatibility Priority",   AcpiGbl_ConfigDecode},
-    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (StartDpf.PerformanceRobustness),   "Performance/Robustness",   AcpiGbl_ConfigDecode}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpEndDpf[1] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpEndDpf),             "End-Dependent-Functions",  NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpIo[6] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpIo),                 "I/O",                      NULL},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Io.IoDecode),                      "Address Decoding",         AcpiGbl_IoDecode},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Io.Minimum),                       "Address Minimum",          NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Io.Maximum),                       "Address Maximum",          NULL},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (Io.Alignment),                     "Alignment",                NULL},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (Io.AddressLength),                 "Address Length",           NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpFixedIo[3] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpFixedIo),            "Fixed I/O",                NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (FixedIo.Address),                  "Address",                  NULL},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (FixedIo.AddressLength),            "Address Length",           NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpVendor[3] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpVendor),             "Vendor Specific",          NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Vendor.ByteLength),                "Length",                   NULL},
-    {ACPI_RSD_LONGLIST, ACPI_RSD_OFFSET (Vendor.ByteData[0]),               "Vendor Data",              NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpEndTag[1] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpEndTag),             "EndTag",                   NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpMemory24[6] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpMemory24),           "24-Bit Memory Range",      NULL},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Memory24.WriteProtect),            "Write Protect",            AcpiGbl_RwDecode},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Memory24.Minimum),                 "Address Minimum",          NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Memory24.Maximum),                 "Address Maximum",          NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Memory24.Alignment),               "Alignment",                NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Memory24.AddressLength),           "Address Length",           NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpMemory32[6] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpMemory32),           "32-Bit Memory Range",      NULL},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Memory32.WriteProtect),            "Write Protect",            AcpiGbl_RwDecode},
-    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Memory32.Minimum),                 "Address Minimum",          NULL},
-    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Memory32.Maximum),                 "Address Maximum",          NULL},
-    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Memory32.Alignment),               "Alignment",                NULL},
-    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Memory32.AddressLength),           "Address Length",           NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpFixedMemory32[4] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpFixedMemory32),      "32-Bit Fixed Memory Range",NULL},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (FixedMemory32.WriteProtect),       "Write Protect",            AcpiGbl_RwDecode},
-    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (FixedMemory32.Address),            "Address",                  NULL},
-    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (FixedMemory32.AddressLength),      "Address Length",           NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpAddress16[8] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpAddress16),          "16-Bit WORD Address Space",NULL},
-    {ACPI_RSD_ADDRESS,  0,                                                  NULL,                       NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Address16.Granularity),            "Granularity",              NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Address16.Minimum),                "Address Minimum",          NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Address16.Maximum),                "Address Maximum",          NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Address16.TranslationOffset),      "Translation Offset",       NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Address16.AddressLength),          "Address Length",           NULL},
-    {ACPI_RSD_SOURCE,   ACPI_RSD_OFFSET (Address16.ResourceSource),         NULL,                       NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpAddress32[8] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpAddress32),         "32-Bit DWORD Address Space", NULL},
-    {ACPI_RSD_ADDRESS,  0,                                                  NULL,                       NULL},
-    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Address32.Granularity),            "Granularity",              NULL},
-    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Address32.Minimum),                "Address Minimum",          NULL},
-    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Address32.Maximum),                "Address Maximum",          NULL},
-    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Address32.TranslationOffset),      "Translation Offset",       NULL},
-    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Address32.AddressLength),          "Address Length",           NULL},
-    {ACPI_RSD_SOURCE,   ACPI_RSD_OFFSET (Address32.ResourceSource),         NULL,                       NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpAddress64[8] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpAddress64),          "64-Bit QWORD Address Space", NULL},
-    {ACPI_RSD_ADDRESS,  0,                                                  NULL,                       NULL},
-    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (Address64.Granularity),            "Granularity",              NULL},
-    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (Address64.Minimum),                "Address Minimum",          NULL},
-    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (Address64.Maximum),                "Address Maximum",          NULL},
-    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (Address64.TranslationOffset),      "Translation Offset",       NULL},
-    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (Address64.AddressLength),          "Address Length",           NULL},
-    {ACPI_RSD_SOURCE,   ACPI_RSD_OFFSET (Address64.ResourceSource),         NULL,                       NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpExtAddress64[8] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpExtAddress64),       "64-Bit Extended Address Space", NULL},
-    {ACPI_RSD_ADDRESS,  0,                                                  NULL,                       NULL},
-    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (ExtAddress64.Granularity),         "Granularity",              NULL},
-    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (ExtAddress64.Minimum),             "Address Minimum",          NULL},
-    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (ExtAddress64.Maximum),             "Address Maximum",          NULL},
-    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (ExtAddress64.TranslationOffset),   "Translation Offset",       NULL},
-    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (ExtAddress64.AddressLength),       "Address Length",           NULL},
-    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (ExtAddress64.TypeSpecific),        "Type-Specific Attribute",  NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpExtIrq[8] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpExtIrq),             "Extended IRQ",             NULL},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (ExtendedIrq.ProducerConsumer),     "Type",                     AcpiGbl_ConsumeDecode},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (ExtendedIrq.Triggering),           "Triggering",               AcpiGbl_HeDecode},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (ExtendedIrq.Polarity),             "Polarity",                 AcpiGbl_LlDecode},
-    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (ExtendedIrq.Sharable),             "Sharing",                  AcpiGbl_ShrDecode},
-    {ACPI_RSD_SOURCE,   ACPI_RSD_OFFSET (ExtendedIrq.ResourceSource),       NULL,                       NULL},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (ExtendedIrq.InterruptCount),       "Interrupt Count",          NULL},
-    {ACPI_RSD_DWORDLIST,ACPI_RSD_OFFSET (ExtendedIrq.Interrupts[0]),        "Interrupt List",           NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpGenericReg[6] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpGenericReg),         "Generic Register",         NULL},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (GenericReg.SpaceId),               "Space ID",                 NULL},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (GenericReg.BitWidth),              "Bit Width",                NULL},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (GenericReg.BitOffset),             "Bit Offset",               NULL},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (GenericReg.AccessSize),            "Access Size",              NULL},
-    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (GenericReg.Address),               "Address",                  NULL}
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpGpio[16] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpGpio),               "GPIO",                     NULL},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (Gpio.RevisionId),                  "RevisionId",               NULL},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (Gpio.ConnectionType),              "ConnectionType",           AcpiGbl_CtDecode},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Gpio.ProducerConsumer),            "ProducerConsumer",         AcpiGbl_ConsumeDecode},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (Gpio.PinConfig),                   "PinConfig",                AcpiGbl_PpcDecode},
-    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Gpio.Sharable),                    "Sharing",                  AcpiGbl_ShrDecode},
-    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Gpio.IoRestriction),               "IoRestriction",            AcpiGbl_IorDecode},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Gpio.Triggering),                  "Triggering",               AcpiGbl_HeDecode},
-    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Gpio.Polarity),                    "Polarity",                 AcpiGbl_LlDecode},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Gpio.DriveStrength),               "DriveStrength",            NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Gpio.DebounceTimeout),             "DebounceTimeout",          NULL},
-    {ACPI_RSD_SOURCE,   ACPI_RSD_OFFSET (Gpio.ResourceSource),              "ResourceSource",           NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Gpio.PinTableLength),              "PinTableLength",           NULL},
-    {ACPI_RSD_WORDLIST, ACPI_RSD_OFFSET (Gpio.PinTable),                    "PinTable",                 NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Gpio.VendorLength),                "VendorLength",             NULL},
-    {ACPI_RSD_SHORTLISTX,ACPI_RSD_OFFSET (Gpio.VendorData),                 "VendorData",               NULL},
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpFixedDma[4] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpFixedDma),           "FixedDma",                 NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (FixedDma.RequestLines),            "RequestLines",             NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (FixedDma.Channels),                "Channels",                 NULL},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (FixedDma.Width),                   "TransferWidth",            AcpiGbl_DtsDecode},
-};
-
-#define ACPI_RS_DUMP_COMMON_SERIAL_BUS \
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (CommonSerialBus.RevisionId),       "RevisionId",               NULL}, \
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (CommonSerialBus.Type),             "Type",                     AcpiGbl_SbtDecode}, \
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (CommonSerialBus.ProducerConsumer), "ProducerConsumer",         AcpiGbl_ConsumeDecode}, \
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (CommonSerialBus.SlaveMode),        "SlaveMode",                AcpiGbl_SmDecode}, \
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (CommonSerialBus.TypeRevisionId),   "TypeRevisionId",           NULL}, \
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (CommonSerialBus.TypeDataLength),   "TypeDataLength",           NULL}, \
-    {ACPI_RSD_SOURCE,   ACPI_RSD_OFFSET (CommonSerialBus.ResourceSource),   "ResourceSource",           NULL}, \
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (CommonSerialBus.VendorLength),     "VendorLength",             NULL}, \
-    {ACPI_RSD_SHORTLISTX,ACPI_RSD_OFFSET (CommonSerialBus.VendorData),      "VendorData",               NULL},
-
-ACPI_RSDUMP_INFO        AcpiRsDumpCommonSerialBus[10] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpCommonSerialBus),    "Common Serial Bus",        NULL},
-    ACPI_RS_DUMP_COMMON_SERIAL_BUS
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpI2cSerialBus[13] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpI2cSerialBus),       "I2C Serial Bus",           NULL},
-    ACPI_RS_DUMP_COMMON_SERIAL_BUS
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (I2cSerialBus.AccessMode),          "AccessMode",               AcpiGbl_AmDecode},
-    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (I2cSerialBus.ConnectionSpeed),     "ConnectionSpeed",          NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (I2cSerialBus.SlaveAddress),        "SlaveAddress",             NULL},
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpSpiSerialBus[17] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpSpiSerialBus),       "Spi Serial Bus",           NULL},
-    ACPI_RS_DUMP_COMMON_SERIAL_BUS
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (SpiSerialBus.WireMode),            "WireMode",                 AcpiGbl_WmDecode},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (SpiSerialBus.DevicePolarity),      "DevicePolarity",           AcpiGbl_DpDecode},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (SpiSerialBus.DataBitLength),       "DataBitLength",            NULL},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (SpiSerialBus.ClockPhase),          "ClockPhase",               AcpiGbl_CphDecode},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (SpiSerialBus.ClockPolarity),       "ClockPolarity",            AcpiGbl_CpoDecode},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (SpiSerialBus.DeviceSelection),     "DeviceSelection",          NULL},
-    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (SpiSerialBus.ConnectionSpeed),     "ConnectionSpeed",          NULL},
-};
-
-ACPI_RSDUMP_INFO        AcpiRsDumpUartSerialBus[19] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpUartSerialBus),       "Uart Serial Bus",         NULL},
-    ACPI_RS_DUMP_COMMON_SERIAL_BUS
-    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (UartSerialBus.FlowControl),         "FlowControl",             AcpiGbl_FcDecode},
-    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (UartSerialBus.StopBits),            "StopBits",                AcpiGbl_SbDecode},
-    {ACPI_RSD_3BITFLAG, ACPI_RSD_OFFSET (UartSerialBus.DataBits),            "DataBits",                AcpiGbl_BpbDecode},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (UartSerialBus.Endian),              "Endian",                  AcpiGbl_EdDecode},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (UartSerialBus.Parity),              "Parity",                  AcpiGbl_PtDecode},
-    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (UartSerialBus.LinesEnabled),        "LinesEnabled",            NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (UartSerialBus.RxFifoSize),          "RxFifoSize",              NULL},
-    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (UartSerialBus.TxFifoSize),          "TxFifoSize",              NULL},
-    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (UartSerialBus.DefaultBaudRate),     "ConnectionSpeed",         NULL},
-};
-
-/*
- * Tables used for common address descriptor flag fields
- */
-static ACPI_RSDUMP_INFO AcpiRsDumpGeneralFlags[5] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpGeneralFlags),       NULL,                       NULL},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.ProducerConsumer),         "Consumer/Producer",        AcpiGbl_ConsumeDecode},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.Decode),                   "Address Decode",           AcpiGbl_DecDecode},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.MinAddressFixed),          "Min Relocatability",       AcpiGbl_MinDecode},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.MaxAddressFixed),          "Max Relocatability",       AcpiGbl_MaxDecode}
-};
-
-static ACPI_RSDUMP_INFO AcpiRsDumpMemoryFlags[5] =
-{
-    {ACPI_RSD_LITERAL,  ACPI_RSD_TABLE_SIZE (AcpiRsDumpMemoryFlags),        "Resource Type",            (void *) "Memory Range"},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.Info.Mem.WriteProtect),    "Write Protect",            AcpiGbl_RwDecode},
-    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Address.Info.Mem.Caching),         "Caching",                  AcpiGbl_MemDecode},
-    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Address.Info.Mem.RangeType),       "Range Type",               AcpiGbl_MtpDecode},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.Info.Mem.Translation),     "Translation",              AcpiGbl_TtpDecode}
-};
-
-static ACPI_RSDUMP_INFO AcpiRsDumpIoFlags[4] =
-{
-    {ACPI_RSD_LITERAL,  ACPI_RSD_TABLE_SIZE (AcpiRsDumpIoFlags),            "Resource Type",            (void *) "I/O Range"},
-    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Address.Info.Io.RangeType),        "Range Type",               AcpiGbl_RngDecode},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.Info.Io.Translation),      "Translation",              AcpiGbl_TtpDecode},
-    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.Info.Io.TranslationType),  "Translation Type",         AcpiGbl_TrsDecode}
-};
-
-
-/*
- * Table used to dump _PRT contents
- */
-static ACPI_RSDUMP_INFO   AcpiRsDumpPrt[5] =
-{
-    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpPrt),                NULL,                       NULL},
-    {ACPI_RSD_UINT64,   ACPI_PRT_OFFSET (Address),                          "Address",                  NULL},
-    {ACPI_RSD_UINT32,   ACPI_PRT_OFFSET (Pin),                              "Pin",                      NULL},
-    {ACPI_RSD_STRING,   ACPI_PRT_OFFSET (Source[0]),                        "Source",                   NULL},
-    {ACPI_RSD_UINT32,   ACPI_PRT_OFFSET (SourceIndex),                      "Source Index",             NULL}
-};
-
-
 /*******************************************************************************
  *
  * FUNCTION:    AcpiRsDumpDescriptor
  *
- * PARAMETERS:  Resource
+ * PARAMETERS:  Resource            - Buffer containing the resource
+ *              Table               - Table entry to decode the resource
  *
  * RETURN:      None
  *
- * DESCRIPTION:
+ * DESCRIPTION: Dump a resource descriptor based on a dump table entry.
  *
  ******************************************************************************/
 
@@ -712,7 +413,9 @@ AcpiRsDumpResourceList (
     ACPI_FUNCTION_ENTRY ();
 
 
-    if (!(AcpiDbgLevel & ACPI_LV_RESOURCES) || !( _COMPONENT & AcpiDbgLayer))
+    /* Check if debug output enabled */
+
+    if (!ACPI_IS_DEBUG_ENABLED (ACPI_LV_RESOURCES, _COMPONENT))
     {
         return;
     }
@@ -781,7 +484,9 @@ AcpiRsDumpIrqList (
     ACPI_FUNCTION_ENTRY ();
 
 
-    if (!(AcpiDbgLevel & ACPI_LV_RESOURCES) || !( _COMPONENT & AcpiDbgLayer))
+    /* Check if debug output enabled */
+
+    if (!ACPI_IS_DEBUG_ENABLED (ACPI_LV_RESOURCES, _COMPONENT))
     {
         return;
     }
diff --git a/sys/contrib/dev/acpica/components/resources/rsdumpinfo.c b/sys/contrib/dev/acpica/components/resources/rsdumpinfo.c
new file mode 100644
index 0000000..865209b
--- /dev/null
+++ b/sys/contrib/dev/acpica/components/resources/rsdumpinfo.c
@@ -0,0 +1,357 @@
+/*******************************************************************************
+ *
+ * Module Name: rsdumpinfo - Tables used to display resource descriptors.
+ *
+ ******************************************************************************/
+
+/*
+ * Copyright (C) 2000 - 2013, Intel Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ * 3. Neither the names of the above-listed copyright holders nor the names
+ *    of any contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ */
+
+
+#define __RSDUMPINFO_C__
+
+#include <contrib/dev/acpica/include/acpi.h>
+#include <contrib/dev/acpica/include/accommon.h>
+#include <contrib/dev/acpica/include/acresrc.h>
+
+#define _COMPONENT          ACPI_RESOURCES
+        ACPI_MODULE_NAME    ("rsdumpinfo")
+
+
+#if defined(ACPI_DEBUG_OUTPUT) || defined(ACPI_DEBUGGER)
+
+
+#define ACPI_RSD_OFFSET(f)          (UINT8) ACPI_OFFSET (ACPI_RESOURCE_DATA,f)
+#define ACPI_PRT_OFFSET(f)          (UINT8) ACPI_OFFSET (ACPI_PCI_ROUTING_TABLE,f)
+#define ACPI_RSD_TABLE_SIZE(name)   (sizeof(name) / sizeof (ACPI_RSDUMP_INFO))
+
+
+/*******************************************************************************
+ *
+ * Resource Descriptor info tables
+ *
+ * Note: The first table entry must be a Title or Literal and must contain
+ * the table length (number of table entries)
+ *
+ ******************************************************************************/
+
+ACPI_RSDUMP_INFO        AcpiRsDumpIrq[7] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpIrq),                "IRQ",                      NULL},
+    {ACPI_RSD_UINT8 ,   ACPI_RSD_OFFSET (Irq.DescriptorLength),             "Descriptor Length",        NULL},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Irq.Triggering),                   "Triggering",               AcpiGbl_HeDecode},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Irq.Polarity),                     "Polarity",                 AcpiGbl_LlDecode},
+    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Irq.Sharable),                     "Sharing",                  AcpiGbl_ShrDecode},
+    {ACPI_RSD_UINT8 ,   ACPI_RSD_OFFSET (Irq.InterruptCount),               "Interrupt Count",          NULL},
+    {ACPI_RSD_SHORTLIST,ACPI_RSD_OFFSET (Irq.Interrupts[0]),                "Interrupt List",           NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpDma[6] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpDma),                "DMA",                      NULL},
+    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Dma.Type),                         "Speed",                    AcpiGbl_TypDecode},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Dma.BusMaster),                    "Mastering",                AcpiGbl_BmDecode},
+    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Dma.Transfer),                     "Transfer Type",            AcpiGbl_SizDecode},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (Dma.ChannelCount),                 "Channel Count",            NULL},
+    {ACPI_RSD_SHORTLIST,ACPI_RSD_OFFSET (Dma.Channels[0]),                  "Channel List",             NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpStartDpf[4] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpStartDpf),           "Start-Dependent-Functions",NULL},
+    {ACPI_RSD_UINT8 ,   ACPI_RSD_OFFSET (StartDpf.DescriptorLength),        "Descriptor Length",        NULL},
+    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (StartDpf.CompatibilityPriority),   "Compatibility Priority",   AcpiGbl_ConfigDecode},
+    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (StartDpf.PerformanceRobustness),   "Performance/Robustness",   AcpiGbl_ConfigDecode}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpEndDpf[1] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpEndDpf),             "End-Dependent-Functions",  NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpIo[6] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpIo),                 "I/O",                      NULL},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Io.IoDecode),                      "Address Decoding",         AcpiGbl_IoDecode},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Io.Minimum),                       "Address Minimum",          NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Io.Maximum),                       "Address Maximum",          NULL},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (Io.Alignment),                     "Alignment",                NULL},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (Io.AddressLength),                 "Address Length",           NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpFixedIo[3] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpFixedIo),            "Fixed I/O",                NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (FixedIo.Address),                  "Address",                  NULL},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (FixedIo.AddressLength),            "Address Length",           NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpVendor[3] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpVendor),             "Vendor Specific",          NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Vendor.ByteLength),                "Length",                   NULL},
+    {ACPI_RSD_LONGLIST, ACPI_RSD_OFFSET (Vendor.ByteData[0]),               "Vendor Data",              NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpEndTag[1] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpEndTag),             "EndTag",                   NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpMemory24[6] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpMemory24),           "24-Bit Memory Range",      NULL},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Memory24.WriteProtect),            "Write Protect",            AcpiGbl_RwDecode},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Memory24.Minimum),                 "Address Minimum",          NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Memory24.Maximum),                 "Address Maximum",          NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Memory24.Alignment),               "Alignment",                NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Memory24.AddressLength),           "Address Length",           NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpMemory32[6] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpMemory32),           "32-Bit Memory Range",      NULL},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Memory32.WriteProtect),            "Write Protect",            AcpiGbl_RwDecode},
+    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Memory32.Minimum),                 "Address Minimum",          NULL},
+    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Memory32.Maximum),                 "Address Maximum",          NULL},
+    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Memory32.Alignment),               "Alignment",                NULL},
+    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Memory32.AddressLength),           "Address Length",           NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpFixedMemory32[4] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpFixedMemory32),      "32-Bit Fixed Memory Range",NULL},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (FixedMemory32.WriteProtect),       "Write Protect",            AcpiGbl_RwDecode},
+    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (FixedMemory32.Address),            "Address",                  NULL},
+    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (FixedMemory32.AddressLength),      "Address Length",           NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpAddress16[8] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpAddress16),          "16-Bit WORD Address Space",NULL},
+    {ACPI_RSD_ADDRESS,  0,                                                  NULL,                       NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Address16.Granularity),            "Granularity",              NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Address16.Minimum),                "Address Minimum",          NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Address16.Maximum),                "Address Maximum",          NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Address16.TranslationOffset),      "Translation Offset",       NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Address16.AddressLength),          "Address Length",           NULL},
+    {ACPI_RSD_SOURCE,   ACPI_RSD_OFFSET (Address16.ResourceSource),         NULL,                       NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpAddress32[8] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpAddress32),         "32-Bit DWORD Address Space", NULL},
+    {ACPI_RSD_ADDRESS,  0,                                                  NULL,                       NULL},
+    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Address32.Granularity),            "Granularity",              NULL},
+    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Address32.Minimum),                "Address Minimum",          NULL},
+    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Address32.Maximum),                "Address Maximum",          NULL},
+    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Address32.TranslationOffset),      "Translation Offset",       NULL},
+    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (Address32.AddressLength),          "Address Length",           NULL},
+    {ACPI_RSD_SOURCE,   ACPI_RSD_OFFSET (Address32.ResourceSource),         NULL,                       NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpAddress64[8] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpAddress64),          "64-Bit QWORD Address Space", NULL},
+    {ACPI_RSD_ADDRESS,  0,                                                  NULL,                       NULL},
+    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (Address64.Granularity),            "Granularity",              NULL},
+    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (Address64.Minimum),                "Address Minimum",          NULL},
+    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (Address64.Maximum),                "Address Maximum",          NULL},
+    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (Address64.TranslationOffset),      "Translation Offset",       NULL},
+    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (Address64.AddressLength),          "Address Length",           NULL},
+    {ACPI_RSD_SOURCE,   ACPI_RSD_OFFSET (Address64.ResourceSource),         NULL,                       NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpExtAddress64[8] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpExtAddress64),       "64-Bit Extended Address Space", NULL},
+    {ACPI_RSD_ADDRESS,  0,                                                  NULL,                       NULL},
+    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (ExtAddress64.Granularity),         "Granularity",              NULL},
+    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (ExtAddress64.Minimum),             "Address Minimum",          NULL},
+    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (ExtAddress64.Maximum),             "Address Maximum",          NULL},
+    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (ExtAddress64.TranslationOffset),   "Translation Offset",       NULL},
+    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (ExtAddress64.AddressLength),       "Address Length",           NULL},
+    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (ExtAddress64.TypeSpecific),        "Type-Specific Attribute",  NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpExtIrq[8] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpExtIrq),             "Extended IRQ",             NULL},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (ExtendedIrq.ProducerConsumer),     "Type",                     AcpiGbl_ConsumeDecode},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (ExtendedIrq.Triggering),           "Triggering",               AcpiGbl_HeDecode},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (ExtendedIrq.Polarity),             "Polarity",                 AcpiGbl_LlDecode},
+    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (ExtendedIrq.Sharable),             "Sharing",                  AcpiGbl_ShrDecode},
+    {ACPI_RSD_SOURCE,   ACPI_RSD_OFFSET (ExtendedIrq.ResourceSource),       NULL,                       NULL},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (ExtendedIrq.InterruptCount),       "Interrupt Count",          NULL},
+    {ACPI_RSD_DWORDLIST,ACPI_RSD_OFFSET (ExtendedIrq.Interrupts[0]),        "Interrupt List",           NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpGenericReg[6] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpGenericReg),         "Generic Register",         NULL},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (GenericReg.SpaceId),               "Space ID",                 NULL},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (GenericReg.BitWidth),              "Bit Width",                NULL},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (GenericReg.BitOffset),             "Bit Offset",               NULL},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (GenericReg.AccessSize),            "Access Size",              NULL},
+    {ACPI_RSD_UINT64,   ACPI_RSD_OFFSET (GenericReg.Address),               "Address",                  NULL}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpGpio[16] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpGpio),               "GPIO",                     NULL},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (Gpio.RevisionId),                  "RevisionId",               NULL},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (Gpio.ConnectionType),              "ConnectionType",           AcpiGbl_CtDecode},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Gpio.ProducerConsumer),            "ProducerConsumer",         AcpiGbl_ConsumeDecode},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (Gpio.PinConfig),                   "PinConfig",                AcpiGbl_PpcDecode},
+    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Gpio.Sharable),                    "Sharing",                  AcpiGbl_ShrDecode},
+    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Gpio.IoRestriction),               "IoRestriction",            AcpiGbl_IorDecode},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Gpio.Triggering),                  "Triggering",               AcpiGbl_HeDecode},
+    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Gpio.Polarity),                    "Polarity",                 AcpiGbl_LlDecode},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Gpio.DriveStrength),               "DriveStrength",            NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Gpio.DebounceTimeout),             "DebounceTimeout",          NULL},
+    {ACPI_RSD_SOURCE,   ACPI_RSD_OFFSET (Gpio.ResourceSource),              "ResourceSource",           NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Gpio.PinTableLength),              "PinTableLength",           NULL},
+    {ACPI_RSD_WORDLIST, ACPI_RSD_OFFSET (Gpio.PinTable),                    "PinTable",                 NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (Gpio.VendorLength),                "VendorLength",             NULL},
+    {ACPI_RSD_SHORTLISTX,ACPI_RSD_OFFSET (Gpio.VendorData),                 "VendorData",               NULL},
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpFixedDma[4] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpFixedDma),           "FixedDma",                 NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (FixedDma.RequestLines),            "RequestLines",             NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (FixedDma.Channels),                "Channels",                 NULL},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (FixedDma.Width),                   "TransferWidth",            AcpiGbl_DtsDecode},
+};
+
+#define ACPI_RS_DUMP_COMMON_SERIAL_BUS \
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (CommonSerialBus.RevisionId),       "RevisionId",               NULL}, \
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (CommonSerialBus.Type),             "Type",                     AcpiGbl_SbtDecode}, \
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (CommonSerialBus.ProducerConsumer), "ProducerConsumer",         AcpiGbl_ConsumeDecode}, \
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (CommonSerialBus.SlaveMode),        "SlaveMode",                AcpiGbl_SmDecode}, \
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (CommonSerialBus.TypeRevisionId),   "TypeRevisionId",           NULL}, \
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (CommonSerialBus.TypeDataLength),   "TypeDataLength",           NULL}, \
+    {ACPI_RSD_SOURCE,   ACPI_RSD_OFFSET (CommonSerialBus.ResourceSource),   "ResourceSource",           NULL}, \
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (CommonSerialBus.VendorLength),     "VendorLength",             NULL}, \
+    {ACPI_RSD_SHORTLISTX,ACPI_RSD_OFFSET (CommonSerialBus.VendorData),      "VendorData",               NULL},
+
+ACPI_RSDUMP_INFO        AcpiRsDumpCommonSerialBus[10] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpCommonSerialBus),    "Common Serial Bus",        NULL},
+    ACPI_RS_DUMP_COMMON_SERIAL_BUS
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpI2cSerialBus[13] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpI2cSerialBus),       "I2C Serial Bus",           NULL},
+    ACPI_RS_DUMP_COMMON_SERIAL_BUS
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (I2cSerialBus.AccessMode),          "AccessMode",               AcpiGbl_AmDecode},
+    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (I2cSerialBus.ConnectionSpeed),     "ConnectionSpeed",          NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (I2cSerialBus.SlaveAddress),        "SlaveAddress",             NULL},
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpSpiSerialBus[17] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpSpiSerialBus),       "Spi Serial Bus",           NULL},
+    ACPI_RS_DUMP_COMMON_SERIAL_BUS
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (SpiSerialBus.WireMode),            "WireMode",                 AcpiGbl_WmDecode},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (SpiSerialBus.DevicePolarity),      "DevicePolarity",           AcpiGbl_DpDecode},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (SpiSerialBus.DataBitLength),       "DataBitLength",            NULL},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (SpiSerialBus.ClockPhase),          "ClockPhase",               AcpiGbl_CphDecode},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (SpiSerialBus.ClockPolarity),       "ClockPolarity",            AcpiGbl_CpoDecode},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (SpiSerialBus.DeviceSelection),     "DeviceSelection",          NULL},
+    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (SpiSerialBus.ConnectionSpeed),     "ConnectionSpeed",          NULL},
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpUartSerialBus[19] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpUartSerialBus),       "Uart Serial Bus",         NULL},
+    ACPI_RS_DUMP_COMMON_SERIAL_BUS
+    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (UartSerialBus.FlowControl),         "FlowControl",             AcpiGbl_FcDecode},
+    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (UartSerialBus.StopBits),            "StopBits",                AcpiGbl_SbDecode},
+    {ACPI_RSD_3BITFLAG, ACPI_RSD_OFFSET (UartSerialBus.DataBits),            "DataBits",                AcpiGbl_BpbDecode},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (UartSerialBus.Endian),              "Endian",                  AcpiGbl_EdDecode},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (UartSerialBus.Parity),              "Parity",                  AcpiGbl_PtDecode},
+    {ACPI_RSD_UINT8,    ACPI_RSD_OFFSET (UartSerialBus.LinesEnabled),        "LinesEnabled",            NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (UartSerialBus.RxFifoSize),          "RxFifoSize",              NULL},
+    {ACPI_RSD_UINT16,   ACPI_RSD_OFFSET (UartSerialBus.TxFifoSize),          "TxFifoSize",              NULL},
+    {ACPI_RSD_UINT32,   ACPI_RSD_OFFSET (UartSerialBus.DefaultBaudRate),     "ConnectionSpeed",         NULL},
+};
+
+/*
+ * Tables used for common address descriptor flag fields
+ */
+ACPI_RSDUMP_INFO        AcpiRsDumpGeneralFlags[5] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpGeneralFlags),       NULL,                       NULL},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.ProducerConsumer),         "Consumer/Producer",        AcpiGbl_ConsumeDecode},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.Decode),                   "Address Decode",           AcpiGbl_DecDecode},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.MinAddressFixed),          "Min Relocatability",       AcpiGbl_MinDecode},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.MaxAddressFixed),          "Max Relocatability",       AcpiGbl_MaxDecode}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpMemoryFlags[5] =
+{
+    {ACPI_RSD_LITERAL,  ACPI_RSD_TABLE_SIZE (AcpiRsDumpMemoryFlags),        "Resource Type",            (void *) "Memory Range"},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.Info.Mem.WriteProtect),    "Write Protect",            AcpiGbl_RwDecode},
+    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Address.Info.Mem.Caching),         "Caching",                  AcpiGbl_MemDecode},
+    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Address.Info.Mem.RangeType),       "Range Type",               AcpiGbl_MtpDecode},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.Info.Mem.Translation),     "Translation",              AcpiGbl_TtpDecode}
+};
+
+ACPI_RSDUMP_INFO        AcpiRsDumpIoFlags[4] =
+{
+    {ACPI_RSD_LITERAL,  ACPI_RSD_TABLE_SIZE (AcpiRsDumpIoFlags),            "Resource Type",            (void *) "I/O Range"},
+    {ACPI_RSD_2BITFLAG, ACPI_RSD_OFFSET (Address.Info.Io.RangeType),        "Range Type",               AcpiGbl_RngDecode},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.Info.Io.Translation),      "Translation",              AcpiGbl_TtpDecode},
+    {ACPI_RSD_1BITFLAG, ACPI_RSD_OFFSET (Address.Info.Io.TranslationType),  "Translation Type",         AcpiGbl_TrsDecode}
+};
+
+
+/*
+ * Table used to dump _PRT contents
+ */
+ACPI_RSDUMP_INFO        AcpiRsDumpPrt[5] =
+{
+    {ACPI_RSD_TITLE,    ACPI_RSD_TABLE_SIZE (AcpiRsDumpPrt),                NULL,                       NULL},
+    {ACPI_RSD_UINT64,   ACPI_PRT_OFFSET (Address),                          "Address",                  NULL},
+    {ACPI_RSD_UINT32,   ACPI_PRT_OFFSET (Pin),                              "Pin",                      NULL},
+    {ACPI_RSD_STRING,   ACPI_PRT_OFFSET (Source[0]),                        "Source",                   NULL},
+    {ACPI_RSD_UINT32,   ACPI_PRT_OFFSET (SourceIndex),                      "Source Index",             NULL}
+};
+
+#endif
diff --git a/sys/contrib/dev/acpica/components/resources/rsinfo.c b/sys/contrib/dev/acpica/components/resources/rsinfo.c
index ab2254e..cb72332 100644
--- a/sys/contrib/dev/acpica/components/resources/rsinfo.c
+++ b/sys/contrib/dev/acpica/components/resources/rsinfo.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/resources/rsio.c b/sys/contrib/dev/acpica/components/resources/rsio.c
index 08119bf..1a41fb9 100644
--- a/sys/contrib/dev/acpica/components/resources/rsio.c
+++ b/sys/contrib/dev/acpica/components/resources/rsio.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/resources/rsirq.c b/sys/contrib/dev/acpica/components/resources/rsirq.c
index 0179749..b3b5ee57 100644
--- a/sys/contrib/dev/acpica/components/resources/rsirq.c
+++ b/sys/contrib/dev/acpica/components/resources/rsirq.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -57,7 +57,7 @@
  *
  ******************************************************************************/
 
-ACPI_RSCONVERT_INFO     AcpiRsGetIrq[8] =
+ACPI_RSCONVERT_INFO     AcpiRsGetIrq[9] =
 {
     {ACPI_RSC_INITGET,  ACPI_RESOURCE_TYPE_IRQ,
                         ACPI_RS_SIZE (ACPI_RESOURCE_IRQ),
@@ -85,7 +85,7 @@ ACPI_RSCONVERT_INFO     AcpiRsGetIrq[8] =
 
     {ACPI_RSC_EXIT_NE,  ACPI_RSC_COMPARE_AML_LENGTH, 0, 3},
 
-    /* Get flags: Triggering[0], Polarity[3], Sharing[4] */
+    /* Get flags: Triggering[0], Polarity[3], Sharing[4], Wake[5] */
 
     {ACPI_RSC_1BITFLAG, ACPI_RS_OFFSET (Data.Irq.Triggering),
                         AML_OFFSET (Irq.Flags),
@@ -97,7 +97,11 @@ ACPI_RSCONVERT_INFO     AcpiRsGetIrq[8] =
 
     {ACPI_RSC_1BITFLAG, ACPI_RS_OFFSET (Data.Irq.Sharable),
                         AML_OFFSET (Irq.Flags),
-                        4}
+                        4},
+
+    {ACPI_RSC_1BITFLAG, ACPI_RS_OFFSET (Data.Irq.WakeCapable),
+                        AML_OFFSET (Irq.Flags),
+                        5}
 };
 
 
@@ -107,7 +111,7 @@ ACPI_RSCONVERT_INFO     AcpiRsGetIrq[8] =
  *
  ******************************************************************************/
 
-ACPI_RSCONVERT_INFO     AcpiRsSetIrq[13] =
+ACPI_RSCONVERT_INFO     AcpiRsSetIrq[14] =
 {
     /* Start with a default descriptor of length 3 */
 
@@ -121,7 +125,7 @@ ACPI_RSCONVERT_INFO     AcpiRsSetIrq[13] =
                         AML_OFFSET (Irq.IrqMask),
                         ACPI_RS_OFFSET (Data.Irq.InterruptCount)},
 
-    /* Set the flags byte */
+    /* Set flags: Triggering[0], Polarity[3], Sharing[4], Wake[5] */
 
     {ACPI_RSC_1BITFLAG, ACPI_RS_OFFSET (Data.Irq.Triggering),
                         AML_OFFSET (Irq.Flags),
@@ -135,6 +139,10 @@ ACPI_RSCONVERT_INFO     AcpiRsSetIrq[13] =
                         AML_OFFSET (Irq.Flags),
                         4},
 
+    {ACPI_RSC_1BITFLAG, ACPI_RS_OFFSET (Data.Irq.WakeCapable),
+                        AML_OFFSET (Irq.Flags),
+                        5},
+
     /*
      * All done if the output descriptor length is required to be 3
      * (i.e., optimization to 2 bytes cannot be attempted)
@@ -189,7 +197,7 @@ ACPI_RSCONVERT_INFO     AcpiRsSetIrq[13] =
  *
  ******************************************************************************/
 
-ACPI_RSCONVERT_INFO     AcpiRsConvertExtIrq[9] =
+ACPI_RSCONVERT_INFO     AcpiRsConvertExtIrq[10] =
 {
     {ACPI_RSC_INITGET,  ACPI_RESOURCE_TYPE_EXTENDED_IRQ,
                         ACPI_RS_SIZE (ACPI_RESOURCE_EXTENDED_IRQ),
@@ -199,8 +207,10 @@ ACPI_RSCONVERT_INFO     AcpiRsConvertExtIrq[9] =
                         sizeof (AML_RESOURCE_EXTENDED_IRQ),
                         0},
 
-    /* Flag bits */
-
+    /*
+     * Flags: Producer/Consumer[0], Triggering[1], Polarity[2],
+     *        Sharing[3], Wake[4]
+     */
     {ACPI_RSC_1BITFLAG, ACPI_RS_OFFSET (Data.ExtendedIrq.ProducerConsumer),
                         AML_OFFSET (ExtendedIrq.Flags),
                         0},
@@ -217,6 +227,10 @@ ACPI_RSCONVERT_INFO     AcpiRsConvertExtIrq[9] =
                         AML_OFFSET (ExtendedIrq.Flags),
                         3},
 
+    {ACPI_RSC_1BITFLAG, ACPI_RS_OFFSET (Data.ExtendedIrq.WakeCapable),
+                        AML_OFFSET (ExtendedIrq.Flags),
+                        4},
+
     /* IRQ Table length (Byte4) */
 
     {ACPI_RSC_COUNT,    ACPI_RS_OFFSET (Data.ExtendedIrq.InterruptCount),
@@ -296,7 +310,6 @@ ACPI_RSCONVERT_INFO     AcpiRsConvertFixedDma[4] =
      * RequestLines
      * Channels
      */
-
     {ACPI_RSC_MOVE16,   ACPI_RS_OFFSET (Data.FixedDma.RequestLines),
                         AML_OFFSET (FixedDma.RequestLines),
                         2},
@@ -304,5 +317,4 @@ ACPI_RSCONVERT_INFO     AcpiRsConvertFixedDma[4] =
     {ACPI_RSC_MOVE8,    ACPI_RS_OFFSET (Data.FixedDma.Width),
                         AML_OFFSET (FixedDma.Width),
                         1},
-
 };
diff --git a/sys/contrib/dev/acpica/components/resources/rslist.c b/sys/contrib/dev/acpica/components/resources/rslist.c
index 96b8c67..2c69c349 100644
--- a/sys/contrib/dev/acpica/components/resources/rslist.c
+++ b/sys/contrib/dev/acpica/components/resources/rslist.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -72,7 +72,7 @@ AcpiRsConvertAmlToResources (
     UINT32                  Length,
     UINT32                  Offset,
     UINT8                   ResourceIndex,
-    void                    *Context)
+    void                    **Context)
 {
     ACPI_RESOURCE           **ResourcePtr = ACPI_CAST_INDIRECT_PTR (
                                 ACPI_RESOURCE, Context);
diff --git a/sys/contrib/dev/acpica/components/resources/rsmemory.c b/sys/contrib/dev/acpica/components/resources/rsmemory.c
index e4e4ae0..049a7b1 100644
--- a/sys/contrib/dev/acpica/components/resources/rsmemory.c
+++ b/sys/contrib/dev/acpica/components/resources/rsmemory.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/resources/rsmisc.c b/sys/contrib/dev/acpica/components/resources/rsmisc.c
index 733ac8f..c0687ca 100644
--- a/sys/contrib/dev/acpica/components/resources/rsmisc.c
+++ b/sys/contrib/dev/acpica/components/resources/rsmisc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/resources/rsserial.c b/sys/contrib/dev/acpica/components/resources/rsserial.c
index 09cc7de..69e2259 100644
--- a/sys/contrib/dev/acpica/components/resources/rsserial.c
+++ b/sys/contrib/dev/acpica/components/resources/rsserial.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -57,7 +57,7 @@
  *
  ******************************************************************************/
 
-ACPI_RSCONVERT_INFO     AcpiRsConvertGpio[17] =
+ACPI_RSCONVERT_INFO     AcpiRsConvertGpio[18] =
 {
     {ACPI_RSC_INITGET,  ACPI_RESOURCE_TYPE_GPIO,
                         ACPI_RS_SIZE (ACPI_RESOURCE_GPIO),
@@ -80,10 +80,14 @@ ACPI_RSCONVERT_INFO     AcpiRsConvertGpio[17] =
                         AML_OFFSET (Gpio.Flags),
                         0},
 
-    {ACPI_RSC_2BITFLAG, ACPI_RS_OFFSET (Data.Gpio.Sharable),
+    {ACPI_RSC_1BITFLAG, ACPI_RS_OFFSET (Data.Gpio.Sharable),
                         AML_OFFSET (Gpio.IntFlags),
                         3},
 
+    {ACPI_RSC_1BITFLAG, ACPI_RS_OFFSET (Data.Gpio.WakeCapable),
+                        AML_OFFSET (Gpio.IntFlags),
+                        4},
+
     {ACPI_RSC_2BITFLAG, ACPI_RS_OFFSET (Data.Gpio.IoRestriction),
                         AML_OFFSET (Gpio.IntFlags),
                         0},
diff --git a/sys/contrib/dev/acpica/components/resources/rsutils.c b/sys/contrib/dev/acpica/components/resources/rsutils.c
index cdbb389..78f4549 100644
--- a/sys/contrib/dev/acpica/components/resources/rsutils.c
+++ b/sys/contrib/dev/acpica/components/resources/rsutils.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -746,7 +746,8 @@ AcpiRsGetMethodData (
 
     /* Execute the method, no parameters */
 
-    Status = AcpiUtEvaluateObject (Handle, Path, ACPI_BTYPE_BUFFER, &ObjDesc);
+    Status = AcpiUtEvaluateObject (ACPI_CAST_PTR (ACPI_NAMESPACE_NODE, Handle),
+        Path, ACPI_BTYPE_BUFFER, &ObjDesc);
     if (ACPI_FAILURE (Status))
     {
         return_ACPI_STATUS (Status);
diff --git a/sys/contrib/dev/acpica/components/resources/rsxface.c b/sys/contrib/dev/acpica/components/resources/rsxface.c
index 9251aec..f70d8f6 100644
--- a/sys/contrib/dev/acpica/components/resources/rsxface.c
+++ b/sys/contrib/dev/acpica/components/resources/rsxface.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -474,7 +474,7 @@ ACPI_EXPORT_SYMBOL (AcpiResourceToAddress64)
  *
  * RETURN:      Status
  *
- * DESCRIPTION: Walk a resource template for the specified evice to find a
+ * DESCRIPTION: Walk a resource template for the specified device to find a
  *              vendor-defined resource that matches the supplied UUID and
  *              UUID subtype. Returns a ACPI_RESOURCE of type Vendor.
  *
@@ -586,63 +586,45 @@ AcpiRsMatchVendorResource (
 
 /*******************************************************************************
  *
- * FUNCTION:    AcpiWalkResources
+ * FUNCTION:    AcpiWalkResourceBuffer
  *
- * PARAMETERS:  DeviceHandle    - Handle to the device object for the
- *                                device we are querying
- *              Name            - Method name of the resources we want.
- *                                (METHOD_NAME__CRS, METHOD_NAME__PRS, or
- *                                METHOD_NAME__AEI)
+ * PARAMETERS:  Buffer          - Formatted buffer returned by one of the
+ *                                various Get*Resource functions
  *              UserFunction    - Called for each resource
  *              Context         - Passed to UserFunction
  *
  * RETURN:      Status
  *
- * DESCRIPTION: Retrieves the current or possible resource list for the
- *              specified device. The UserFunction is called once for
- *              each resource in the list.
+ * DESCRIPTION: Walks the input resource template. The UserFunction is called
+ *              once for each resource in the list.
  *
  ******************************************************************************/
 
 ACPI_STATUS
-AcpiWalkResources (
-    ACPI_HANDLE                 DeviceHandle,
-    char                        *Name,
+AcpiWalkResourceBuffer (
+    ACPI_BUFFER                 *Buffer,
     ACPI_WALK_RESOURCE_CALLBACK UserFunction,
     void                        *Context)
 {
-    ACPI_STATUS                 Status;
-    ACPI_BUFFER                 Buffer;
+    ACPI_STATUS                 Status = AE_OK;
     ACPI_RESOURCE               *Resource;
     ACPI_RESOURCE               *ResourceEnd;
 
 
-    ACPI_FUNCTION_TRACE (AcpiWalkResources);
+    ACPI_FUNCTION_TRACE (AcpiWalkResourceBuffer);
 
 
     /* Parameter validation */
 
-    if (!DeviceHandle || !UserFunction || !Name ||
-        (!ACPI_COMPARE_NAME (Name, METHOD_NAME__CRS) &&
-         !ACPI_COMPARE_NAME (Name, METHOD_NAME__PRS) &&
-         !ACPI_COMPARE_NAME (Name, METHOD_NAME__AEI)))
+    if (!Buffer || !Buffer->Pointer || !UserFunction)
     {
         return_ACPI_STATUS (AE_BAD_PARAMETER);
     }
 
-    /* Get the _CRS/_PRS/_AEI resource list */
-
-    Buffer.Length = ACPI_ALLOCATE_LOCAL_BUFFER;
-    Status = AcpiRsGetMethodData (DeviceHandle, Name, &Buffer);
-    if (ACPI_FAILURE (Status))
-    {
-        return_ACPI_STATUS (Status);
-    }
-
-    /* Buffer now contains the resource list */
+    /* Buffer contains the resource list and length */
 
-    Resource = ACPI_CAST_PTR (ACPI_RESOURCE, Buffer.Pointer);
-    ResourceEnd = ACPI_ADD_PTR (ACPI_RESOURCE, Buffer.Pointer, Buffer.Length);
+    Resource = ACPI_CAST_PTR (ACPI_RESOURCE, Buffer->Pointer);
+    ResourceEnd = ACPI_ADD_PTR (ACPI_RESOURCE, Buffer->Pointer, Buffer->Length);
 
     /* Walk the resource list until the EndTag is found (or buffer end) */
 
@@ -682,6 +664,68 @@ AcpiWalkResources (
         Resource = ACPI_NEXT_RESOURCE (Resource);
     }
 
+    return_ACPI_STATUS (Status);
+}
+
+ACPI_EXPORT_SYMBOL (AcpiWalkResourceBuffer)
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiWalkResources
+ *
+ * PARAMETERS:  DeviceHandle    - Handle to the device object for the
+ *                                device we are querying
+ *              Name            - Method name of the resources we want.
+ *                                (METHOD_NAME__CRS, METHOD_NAME__PRS, or
+ *                                METHOD_NAME__AEI)
+ *              UserFunction    - Called for each resource
+ *              Context         - Passed to UserFunction
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Retrieves the current or possible resource list for the
+ *              specified device. The UserFunction is called once for
+ *              each resource in the list.
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+AcpiWalkResources (
+    ACPI_HANDLE                 DeviceHandle,
+    char                        *Name,
+    ACPI_WALK_RESOURCE_CALLBACK UserFunction,
+    void                        *Context)
+{
+    ACPI_STATUS                 Status;
+    ACPI_BUFFER                 Buffer;
+
+
+    ACPI_FUNCTION_TRACE (AcpiWalkResources);
+
+
+    /* Parameter validation */
+
+    if (!DeviceHandle || !UserFunction || !Name ||
+        (!ACPI_COMPARE_NAME (Name, METHOD_NAME__CRS) &&
+         !ACPI_COMPARE_NAME (Name, METHOD_NAME__PRS) &&
+         !ACPI_COMPARE_NAME (Name, METHOD_NAME__AEI)))
+    {
+        return_ACPI_STATUS (AE_BAD_PARAMETER);
+    }
+
+    /* Get the _CRS/_PRS/_AEI resource list */
+
+    Buffer.Length = ACPI_ALLOCATE_LOCAL_BUFFER;
+    Status = AcpiRsGetMethodData (DeviceHandle, Name, &Buffer);
+    if (ACPI_FAILURE (Status))
+    {
+        return_ACPI_STATUS (Status);
+    }
+
+    /* Walk the resource list and cleanup */
+
+    Status = AcpiWalkResourceBuffer (&Buffer, UserFunction, Context);
     ACPI_FREE (Buffer.Pointer);
     return_ACPI_STATUS (Status);
 }
diff --git a/sys/contrib/dev/acpica/components/tables/tbfadt.c b/sys/contrib/dev/acpica/components/tables/tbfadt.c
index 3a53ae0..327f3a6 100644
--- a/sys/contrib/dev/acpica/components/tables/tbfadt.c
+++ b/sys/contrib/dev/acpica/components/tables/tbfadt.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/tables/tbfind.c b/sys/contrib/dev/acpica/components/tables/tbfind.c
index ccb4029..6cd30e8 100644
--- a/sys/contrib/dev/acpica/components/tables/tbfind.c
+++ b/sys/contrib/dev/acpica/components/tables/tbfind.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/tables/tbinstal.c b/sys/contrib/dev/acpica/components/tables/tbinstal.c
index c2035aa..e6558ff 100644
--- a/sys/contrib/dev/acpica/components/tables/tbinstal.c
+++ b/sys/contrib/dev/acpica/components/tables/tbinstal.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/tables/tbutils.c b/sys/contrib/dev/acpica/components/tables/tbutils.c
index a97a1e9..500c2f1 100644
--- a/sys/contrib/dev/acpica/components/tables/tbutils.c
+++ b/sys/contrib/dev/acpica/components/tables/tbutils.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/tables/tbxface.c b/sys/contrib/dev/acpica/components/tables/tbxface.c
index 33d3591..e040944 100644
--- a/sys/contrib/dev/acpica/components/tables/tbxface.c
+++ b/sys/contrib/dev/acpica/components/tables/tbxface.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/tables/tbxfload.c b/sys/contrib/dev/acpica/components/tables/tbxfload.c
index d06683d..5a4906f 100644
--- a/sys/contrib/dev/acpica/components/tables/tbxfload.c
+++ b/sys/contrib/dev/acpica/components/tables/tbxfload.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -208,7 +208,7 @@ AcpiTbLoadNamespace (
         (void) AcpiUtAcquireMutex (ACPI_MTX_TABLES);
     }
 
-    ACPI_DEBUG_PRINT ((ACPI_DB_INIT, "ACPI Tables successfully acquired\n"));
+    ACPI_INFO ((AE_INFO, "All ACPI Tables successfully acquired"));
 
 UnlockAndExit:
     (void) AcpiUtReleaseMutex (ACPI_MTX_TABLES);
diff --git a/sys/contrib/dev/acpica/components/tables/tbxfroot.c b/sys/contrib/dev/acpica/components/tables/tbxfroot.c
index db06a8a..1d8909b 100644
--- a/sys/contrib/dev/acpica/components/tables/tbxfroot.c
+++ b/sys/contrib/dev/acpica/components/tables/tbxfroot.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utaddress.c b/sys/contrib/dev/acpica/components/utilities/utaddress.c
index ca73b69..715c77b 100644
--- a/sys/contrib/dev/acpica/components/utilities/utaddress.c
+++ b/sys/contrib/dev/acpica/components/utilities/utaddress.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -237,7 +237,7 @@ AcpiUtCheckAddressRange (
     if ((SpaceId != ACPI_ADR_SPACE_SYSTEM_MEMORY) &&
         (SpaceId != ACPI_ADR_SPACE_SYSTEM_IO))
     {
-        return_UINT32 (0);
+        return_VALUE (0);
     }
 
     RangeInfo = AcpiGbl_AddressRangeList[SpaceId];
@@ -278,7 +278,7 @@ AcpiUtCheckAddressRange (
         RangeInfo = RangeInfo->Next;
     }
 
-    return_UINT32 (OverlapCount);
+    return_VALUE (OverlapCount);
 }
 
 
diff --git a/sys/contrib/dev/acpica/components/utilities/utalloc.c b/sys/contrib/dev/acpica/components/utilities/utalloc.c
index 0ea15af..0bdb2ff 100644
--- a/sys/contrib/dev/acpica/components/utilities/utalloc.c
+++ b/sys/contrib/dev/acpica/components/utilities/utalloc.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utcache.c b/sys/contrib/dev/acpica/components/utilities/utcache.c
index e21cb48..2aa3e5b 100644
--- a/sys/contrib/dev/acpica/components/utilities/utcache.c
+++ b/sys/contrib/dev/acpica/components/utilities/utcache.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -95,7 +95,6 @@ AcpiOsCreateCache (
     /* Populate the cache object and return it */
 
     ACPI_MEMSET (Cache, 0, sizeof (ACPI_MEMORY_LIST));
-    Cache->LinkOffset = 8;
     Cache->ListName   = CacheName;
     Cache->ObjectSize = ObjectSize;
     Cache->MaxDepth   = MaxDepth;
@@ -121,7 +120,7 @@ ACPI_STATUS
 AcpiOsPurgeCache (
     ACPI_MEMORY_LIST        *Cache)
 {
-    char                    *Next;
+    void                    *Next;
     ACPI_STATUS             Status;
 
 
@@ -145,8 +144,7 @@ AcpiOsPurgeCache (
     {
         /* Delete and unlink one cached state object */
 
-        Next = *(ACPI_CAST_INDIRECT_PTR (char,
-                    &(((char *) Cache->ListHead)[Cache->LinkOffset])));
+        Next = ACPI_GET_DESCRIPTOR_PTR (Cache->ListHead);
         ACPI_FREE (Cache->ListHead);
 
         Cache->ListHead = Next;
@@ -251,8 +249,7 @@ AcpiOsReleaseObject (
 
         /* Put the object at the head of the cache list */
 
-        * (ACPI_CAST_INDIRECT_PTR (char,
-            &(((char *) Object)[Cache->LinkOffset]))) = Cache->ListHead;
+        ACPI_SET_DESCRIPTOR_PTR (Object, Cache->ListHead);
         Cache->ListHead = Object;
         Cache->CurrentDepth++;
 
@@ -307,8 +304,7 @@ AcpiOsAcquireObject (
         /* There is an object available, use it */
 
         Object = Cache->ListHead;
-        Cache->ListHead = *(ACPI_CAST_INDIRECT_PTR (char,
-                                &(((char *) Object)[Cache->LinkOffset])));
+        Cache->ListHead = ACPI_GET_DESCRIPTOR_PTR (Object);
 
         Cache->CurrentDepth--;
 
diff --git a/sys/contrib/dev/acpica/components/utilities/utcopy.c b/sys/contrib/dev/acpica/components/utilities/utcopy.c
index dd8de9e..696acf0 100644
--- a/sys/contrib/dev/acpica/components/utilities/utcopy.c
+++ b/sys/contrib/dev/acpica/components/utilities/utcopy.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utdebug.c b/sys/contrib/dev/acpica/components/utilities/utdebug.c
index 5834c0f..a299680 100644
--- a/sys/contrib/dev/acpica/components/utilities/utdebug.c
+++ b/sys/contrib/dev/acpica/components/utilities/utdebug.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -189,11 +189,9 @@ AcpiDebugPrint (
     va_list                 args;
 
 
-    /*
-     * Stay silent if the debug level or component ID is disabled
-     */
-    if (!(RequestedDebugLevel & AcpiDbgLevel) ||
-        !(ComponentId & AcpiDbgLayer))
+    /* Check if debug output enabled */
+
+    if (!ACPI_IS_DEBUG_ENABLED (RequestedDebugLevel, ComponentId))
     {
         return;
     }
@@ -268,8 +266,9 @@ AcpiDebugPrintRaw (
     va_list                 args;
 
 
-    if (!(RequestedDebugLevel & AcpiDbgLevel) ||
-        !(ComponentId & AcpiDbgLayer))
+    /* Check if debug output enabled */
+
+    if (!ACPI_IS_DEBUG_ENABLED (RequestedDebugLevel, ComponentId))
     {
         return;
     }
@@ -309,9 +308,14 @@ AcpiUtTrace (
     AcpiGbl_NestingLevel++;
     AcpiUtTrackStackPtr ();
 
-    AcpiDebugPrint (ACPI_LV_FUNCTIONS,
-        LineNumber, FunctionName, ModuleName, ComponentId,
-        "%s\n", AcpiGbl_FnEntryStr);
+    /* Check if enabled up-front for performance */
+
+    if (ACPI_IS_DEBUG_ENABLED (ACPI_LV_FUNCTIONS, ComponentId))
+    {
+        AcpiDebugPrint (ACPI_LV_FUNCTIONS,
+            LineNumber, FunctionName, ModuleName, ComponentId,
+            "%s\n", AcpiGbl_FnEntryStr);
+    }
 }
 
 ACPI_EXPORT_SYMBOL (AcpiUtTrace)
@@ -346,9 +350,14 @@ AcpiUtTracePtr (
     AcpiGbl_NestingLevel++;
     AcpiUtTrackStackPtr ();
 
-    AcpiDebugPrint (ACPI_LV_FUNCTIONS,
-        LineNumber, FunctionName, ModuleName, ComponentId,
-        "%s %p\n", AcpiGbl_FnEntryStr, Pointer);
+    /* Check if enabled up-front for performance */
+
+    if (ACPI_IS_DEBUG_ENABLED (ACPI_LV_FUNCTIONS, ComponentId))
+    {
+        AcpiDebugPrint (ACPI_LV_FUNCTIONS,
+            LineNumber, FunctionName, ModuleName, ComponentId,
+            "%s %p\n", AcpiGbl_FnEntryStr, Pointer);
+    }
 }
 
 
@@ -381,9 +390,14 @@ AcpiUtTraceStr (
     AcpiGbl_NestingLevel++;
     AcpiUtTrackStackPtr ();
 
-    AcpiDebugPrint (ACPI_LV_FUNCTIONS,
-        LineNumber, FunctionName, ModuleName, ComponentId,
-        "%s %s\n", AcpiGbl_FnEntryStr, String);
+    /* Check if enabled up-front for performance */
+
+    if (ACPI_IS_DEBUG_ENABLED (ACPI_LV_FUNCTIONS, ComponentId))
+    {
+        AcpiDebugPrint (ACPI_LV_FUNCTIONS,
+            LineNumber, FunctionName, ModuleName, ComponentId,
+            "%s %s\n", AcpiGbl_FnEntryStr, String);
+    }
 }
 
 
@@ -416,9 +430,14 @@ AcpiUtTraceU32 (
     AcpiGbl_NestingLevel++;
     AcpiUtTrackStackPtr ();
 
-    AcpiDebugPrint (ACPI_LV_FUNCTIONS,
-        LineNumber, FunctionName, ModuleName, ComponentId,
-        "%s %08X\n", AcpiGbl_FnEntryStr, Integer);
+    /* Check if enabled up-front for performance */
+
+    if (ACPI_IS_DEBUG_ENABLED (ACPI_LV_FUNCTIONS, ComponentId))
+    {
+        AcpiDebugPrint (ACPI_LV_FUNCTIONS,
+            LineNumber, FunctionName, ModuleName, ComponentId,
+            "%s %08X\n", AcpiGbl_FnEntryStr, Integer);
+    }
 }
 
 
@@ -446,9 +465,14 @@ AcpiUtExit (
     UINT32                  ComponentId)
 {
 
-    AcpiDebugPrint (ACPI_LV_FUNCTIONS,
-        LineNumber, FunctionName, ModuleName, ComponentId,
-        "%s\n", AcpiGbl_FnExitStr);
+    /* Check if enabled up-front for performance */
+
+    if (ACPI_IS_DEBUG_ENABLED (ACPI_LV_FUNCTIONS, ComponentId))
+    {
+        AcpiDebugPrint (ACPI_LV_FUNCTIONS,
+            LineNumber, FunctionName, ModuleName, ComponentId,
+            "%s\n", AcpiGbl_FnExitStr);
+    }
 
     AcpiGbl_NestingLevel--;
 }
@@ -482,19 +506,24 @@ AcpiUtStatusExit (
     ACPI_STATUS             Status)
 {
 
-    if (ACPI_SUCCESS (Status))
-    {
-        AcpiDebugPrint (ACPI_LV_FUNCTIONS,
-            LineNumber, FunctionName, ModuleName, ComponentId,
-            "%s %s\n", AcpiGbl_FnExitStr,
-            AcpiFormatException (Status));
-    }
-    else
+    /* Check if enabled up-front for performance */
+
+    if (ACPI_IS_DEBUG_ENABLED (ACPI_LV_FUNCTIONS, ComponentId))
     {
-        AcpiDebugPrint (ACPI_LV_FUNCTIONS,
-            LineNumber, FunctionName, ModuleName, ComponentId,
-            "%s ****Exception****: %s\n", AcpiGbl_FnExitStr,
-            AcpiFormatException (Status));
+        if (ACPI_SUCCESS (Status))
+        {
+            AcpiDebugPrint (ACPI_LV_FUNCTIONS,
+                LineNumber, FunctionName, ModuleName, ComponentId,
+                "%s %s\n", AcpiGbl_FnExitStr,
+                AcpiFormatException (Status));
+        }
+        else
+        {
+            AcpiDebugPrint (ACPI_LV_FUNCTIONS,
+                LineNumber, FunctionName, ModuleName, ComponentId,
+                "%s ****Exception****: %s\n", AcpiGbl_FnExitStr,
+                AcpiFormatException (Status));
+        }
     }
 
     AcpiGbl_NestingLevel--;
@@ -529,10 +558,15 @@ AcpiUtValueExit (
     UINT64                  Value)
 {
 
-    AcpiDebugPrint (ACPI_LV_FUNCTIONS,
-        LineNumber, FunctionName, ModuleName, ComponentId,
-        "%s %8.8X%8.8X\n", AcpiGbl_FnExitStr,
-        ACPI_FORMAT_UINT64 (Value));
+    /* Check if enabled up-front for performance */
+
+    if (ACPI_IS_DEBUG_ENABLED (ACPI_LV_FUNCTIONS, ComponentId))
+    {
+        AcpiDebugPrint (ACPI_LV_FUNCTIONS,
+            LineNumber, FunctionName, ModuleName, ComponentId,
+            "%s %8.8X%8.8X\n", AcpiGbl_FnExitStr,
+            ACPI_FORMAT_UINT64 (Value));
+    }
 
     AcpiGbl_NestingLevel--;
 }
@@ -566,9 +600,14 @@ AcpiUtPtrExit (
     UINT8                   *Ptr)
 {
 
-    AcpiDebugPrint (ACPI_LV_FUNCTIONS,
-        LineNumber, FunctionName, ModuleName, ComponentId,
-        "%s %p\n", AcpiGbl_FnExitStr, Ptr);
+    /* Check if enabled up-front for performance */
+
+    if (ACPI_IS_DEBUG_ENABLED (ACPI_LV_FUNCTIONS, ComponentId))
+    {
+        AcpiDebugPrint (ACPI_LV_FUNCTIONS,
+            LineNumber, FunctionName, ModuleName, ComponentId,
+            "%s %p\n", AcpiGbl_FnExitStr, Ptr);
+    }
 
     AcpiGbl_NestingLevel--;
 }
diff --git a/sys/contrib/dev/acpica/components/utilities/utdecode.c b/sys/contrib/dev/acpica/components/utilities/utdecode.c
index ef423e9..3075479 100644
--- a/sys/contrib/dev/acpica/components/utilities/utdecode.c
+++ b/sys/contrib/dev/acpica/components/utilities/utdecode.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utdelete.c b/sys/contrib/dev/acpica/components/utilities/utdelete.c
index 77dbe1b..b73e2d8 100644
--- a/sys/contrib/dev/acpica/components/utilities/utdelete.c
+++ b/sys/contrib/dev/acpica/components/utilities/utdelete.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -368,7 +368,7 @@ AcpiUtDeleteInternalObjectList (
     ACPI_OPERAND_OBJECT     **InternalObj;
 
 
-    ACPI_FUNCTION_NAME (UtDeleteInternalObjectList);
+    ACPI_FUNCTION_ENTRY ();
 
 
     /* Walk the null-terminated internal list */
diff --git a/sys/contrib/dev/acpica/components/utilities/uteval.c b/sys/contrib/dev/acpica/components/utilities/uteval.c
index 57dbec0..95233f4 100644
--- a/sys/contrib/dev/acpica/components/utilities/uteval.c
+++ b/sys/contrib/dev/acpica/components/utilities/uteval.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utexcep.c b/sys/contrib/dev/acpica/components/utilities/utexcep.c
index 0f38227..2d3ae17 100644
--- a/sys/contrib/dev/acpica/components/utilities/utexcep.c
+++ b/sys/contrib/dev/acpica/components/utilities/utexcep.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utglobal.c b/sys/contrib/dev/acpica/components/utilities/utglobal.c
index 5bb5991..6d2f906 100644
--- a/sys/contrib/dev/acpica/components/utilities/utglobal.c
+++ b/sys/contrib/dev/acpica/components/utilities/utglobal.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utids.c b/sys/contrib/dev/acpica/components/utilities/utids.c
index 17ad5ed..c73c8ed 100644
--- a/sys/contrib/dev/acpica/components/utilities/utids.c
+++ b/sys/contrib/dev/acpica/components/utilities/utids.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utinit.c b/sys/contrib/dev/acpica/components/utilities/utinit.c
index 7b53c9a..760d789 100644
--- a/sys/contrib/dev/acpica/components/utilities/utinit.c
+++ b/sys/contrib/dev/acpica/components/utilities/utinit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utlock.c b/sys/contrib/dev/acpica/components/utilities/utlock.c
index 40fc8f8..da933c2 100644
--- a/sys/contrib/dev/acpica/components/utilities/utlock.c
+++ b/sys/contrib/dev/acpica/components/utilities/utlock.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utmath.c b/sys/contrib/dev/acpica/components/utilities/utmath.c
index ec51855..b535ffc 100644
--- a/sys/contrib/dev/acpica/components/utilities/utmath.c
+++ b/sys/contrib/dev/acpica/components/utilities/utmath.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utmisc.c b/sys/contrib/dev/acpica/components/utilities/utmisc.c
index 70fdb50..7d013fc 100644
--- a/sys/contrib/dev/acpica/components/utilities/utmisc.c
+++ b/sys/contrib/dev/acpica/components/utilities/utmisc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -53,43 +53,6 @@
         ACPI_MODULE_NAME    ("utmisc")
 
 
-#if defined ACPI_ASL_COMPILER || defined ACPI_EXEC_APP
-/*******************************************************************************
- *
- * FUNCTION:    UtConvertBackslashes
- *
- * PARAMETERS:  Pathname        - File pathname string to be converted
- *
- * RETURN:      Modifies the input Pathname
- *
- * DESCRIPTION: Convert all backslashes (0x5C) to forward slashes (0x2F) within
- *              the entire input file pathname string.
- *
- ******************************************************************************/
-
-void
-UtConvertBackslashes (
-    char                    *Pathname)
-{
-
-    if (!Pathname)
-    {
-        return;
-    }
-
-    while (*Pathname)
-    {
-        if (*Pathname == '\\')
-        {
-            *Pathname = '/';
-        }
-
-        Pathname++;
-    }
-}
-#endif
-
-
 /*******************************************************************************
  *
  * FUNCTION:    AcpiUtIsPciRootBridge
@@ -158,411 +121,6 @@ AcpiUtIsAmlTable (
 
 /*******************************************************************************
  *
- * FUNCTION:    AcpiUtAllocateOwnerId
- *
- * PARAMETERS:  OwnerId         - Where the new owner ID is returned
- *
- * RETURN:      Status
- *
- * DESCRIPTION: Allocate a table or method owner ID. The owner ID is used to
- *              track objects created by the table or method, to be deleted
- *              when the method exits or the table is unloaded.
- *
- ******************************************************************************/
-
-ACPI_STATUS
-AcpiUtAllocateOwnerId (
-    ACPI_OWNER_ID           *OwnerId)
-{
-    UINT32                  i;
-    UINT32                  j;
-    UINT32                  k;
-    ACPI_STATUS             Status;
-
-
-    ACPI_FUNCTION_TRACE (UtAllocateOwnerId);
-
-
-    /* Guard against multiple allocations of ID to the same location */
-
-    if (*OwnerId)
-    {
-        ACPI_ERROR ((AE_INFO, "Owner ID [0x%2.2X] already exists", *OwnerId));
-        return_ACPI_STATUS (AE_ALREADY_EXISTS);
-    }
-
-    /* Mutex for the global ID mask */
-
-    Status = AcpiUtAcquireMutex (ACPI_MTX_CACHES);
-    if (ACPI_FAILURE (Status))
-    {
-        return_ACPI_STATUS (Status);
-    }
-
-    /*
-     * Find a free owner ID, cycle through all possible IDs on repeated
-     * allocations. (ACPI_NUM_OWNERID_MASKS + 1) because first index may have
-     * to be scanned twice.
-     */
-    for (i = 0, j = AcpiGbl_LastOwnerIdIndex;
-         i < (ACPI_NUM_OWNERID_MASKS + 1);
-         i++, j++)
-    {
-        if (j >= ACPI_NUM_OWNERID_MASKS)
-        {
-            j = 0;  /* Wraparound to start of mask array */
-        }
-
-        for (k = AcpiGbl_NextOwnerIdOffset; k < 32; k++)
-        {
-            if (AcpiGbl_OwnerIdMask[j] == ACPI_UINT32_MAX)
-            {
-                /* There are no free IDs in this mask */
-
-                break;
-            }
-
-            if (!(AcpiGbl_OwnerIdMask[j] & (1 << k)))
-            {
-                /*
-                 * Found a free ID. The actual ID is the bit index plus one,
-                 * making zero an invalid Owner ID. Save this as the last ID
-                 * allocated and update the global ID mask.
-                 */
-                AcpiGbl_OwnerIdMask[j] |= (1 << k);
-
-                AcpiGbl_LastOwnerIdIndex = (UINT8) j;
-                AcpiGbl_NextOwnerIdOffset = (UINT8) (k + 1);
-
-                /*
-                 * Construct encoded ID from the index and bit position
-                 *
-                 * Note: Last [j].k (bit 255) is never used and is marked
-                 * permanently allocated (prevents +1 overflow)
-                 */
-                *OwnerId = (ACPI_OWNER_ID) ((k + 1) + ACPI_MUL_32 (j));
-
-                ACPI_DEBUG_PRINT ((ACPI_DB_VALUES,
-                    "Allocated OwnerId: %2.2X\n", (unsigned int) *OwnerId));
-                goto Exit;
-            }
-        }
-
-        AcpiGbl_NextOwnerIdOffset = 0;
-    }
-
-    /*
-     * All OwnerIds have been allocated. This typically should
-     * not happen since the IDs are reused after deallocation. The IDs are
-     * allocated upon table load (one per table) and method execution, and
-     * they are released when a table is unloaded or a method completes
-     * execution.
-     *
-     * If this error happens, there may be very deep nesting of invoked control
-     * methods, or there may be a bug where the IDs are not released.
-     */
-    Status = AE_OWNER_ID_LIMIT;
-    ACPI_ERROR ((AE_INFO,
-        "Could not allocate new OwnerId (255 max), AE_OWNER_ID_LIMIT"));
-
-Exit:
-    (void) AcpiUtReleaseMutex (ACPI_MTX_CACHES);
-    return_ACPI_STATUS (Status);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiUtReleaseOwnerId
- *
- * PARAMETERS:  OwnerIdPtr          - Pointer to a previously allocated OwnerID
- *
- * RETURN:      None. No error is returned because we are either exiting a
- *              control method or unloading a table. Either way, we would
- *              ignore any error anyway.
- *
- * DESCRIPTION: Release a table or method owner ID. Valid IDs are 1 - 255
- *
- ******************************************************************************/
-
-void
-AcpiUtReleaseOwnerId (
-    ACPI_OWNER_ID           *OwnerIdPtr)
-{
-    ACPI_OWNER_ID           OwnerId = *OwnerIdPtr;
-    ACPI_STATUS             Status;
-    UINT32                  Index;
-    UINT32                  Bit;
-
-
-    ACPI_FUNCTION_TRACE_U32 (UtReleaseOwnerId, OwnerId);
-
-
-    /* Always clear the input OwnerId (zero is an invalid ID) */
-
-    *OwnerIdPtr = 0;
-
-    /* Zero is not a valid OwnerID */
-
-    if (OwnerId == 0)
-    {
-        ACPI_ERROR ((AE_INFO, "Invalid OwnerId: 0x%2.2X", OwnerId));
-        return_VOID;
-    }
-
-    /* Mutex for the global ID mask */
-
-    Status = AcpiUtAcquireMutex (ACPI_MTX_CACHES);
-    if (ACPI_FAILURE (Status))
-    {
-        return_VOID;
-    }
-
-    /* Normalize the ID to zero */
-
-    OwnerId--;
-
-    /* Decode ID to index/offset pair */
-
-    Index = ACPI_DIV_32 (OwnerId);
-    Bit = 1 << ACPI_MOD_32 (OwnerId);
-
-    /* Free the owner ID only if it is valid */
-
-    if (AcpiGbl_OwnerIdMask[Index] & Bit)
-    {
-        AcpiGbl_OwnerIdMask[Index] ^= Bit;
-    }
-    else
-    {
-        ACPI_ERROR ((AE_INFO,
-            "Release of non-allocated OwnerId: 0x%2.2X", OwnerId + 1));
-    }
-
-    (void) AcpiUtReleaseMutex (ACPI_MTX_CACHES);
-    return_VOID;
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiUtStrupr (strupr)
- *
- * PARAMETERS:  SrcString       - The source string to convert
- *
- * RETURN:      None
- *
- * DESCRIPTION: Convert string to uppercase
- *
- * NOTE: This is not a POSIX function, so it appears here, not in utclib.c
- *
- ******************************************************************************/
-
-void
-AcpiUtStrupr (
-    char                    *SrcString)
-{
-    char                    *String;
-
-
-    ACPI_FUNCTION_ENTRY ();
-
-
-    if (!SrcString)
-    {
-        return;
-    }
-
-    /* Walk entire string, uppercasing the letters */
-
-    for (String = SrcString; *String; String++)
-    {
-        *String = (char) ACPI_TOUPPER (*String);
-    }
-
-    return;
-}
-
-
-#ifdef ACPI_ASL_COMPILER
-/*******************************************************************************
- *
- * FUNCTION:    AcpiUtStrlwr (strlwr)
- *
- * PARAMETERS:  SrcString       - The source string to convert
- *
- * RETURN:      None
- *
- * DESCRIPTION: Convert string to lowercase
- *
- * NOTE: This is not a POSIX function, so it appears here, not in utclib.c
- *
- ******************************************************************************/
-
-void
-AcpiUtStrlwr (
-    char                    *SrcString)
-{
-    char                    *String;
-
-
-    ACPI_FUNCTION_ENTRY ();
-
-
-    if (!SrcString)
-    {
-        return;
-    }
-
-    /* Walk entire string, lowercasing the letters */
-
-    for (String = SrcString; *String; String++)
-    {
-        *String = (char) ACPI_TOLOWER (*String);
-    }
-
-    return;
-}
-
-
-/******************************************************************************
- *
- * FUNCTION:    AcpiUtStricmp
- *
- * PARAMETERS:  String1             - first string to compare
- *              String2             - second string to compare
- *
- * RETURN:      int that signifies string relationship. Zero means strings
- *              are equal.
- *
- * DESCRIPTION: Implementation of the non-ANSI stricmp function (compare
- *              strings with no case sensitivity)
- *
- ******************************************************************************/
-
-int
-AcpiUtStricmp (
-    char                    *String1,
-    char                    *String2)
-{
-    int                     c1;
-    int                     c2;
-
-
-    do
-    {
-        c1 = tolower ((int) *String1);
-        c2 = tolower ((int) *String2);
-
-        String1++;
-        String2++;
-    }
-    while ((c1 == c2) && (c1));
-
-    return (c1 - c2);
-}
-#endif
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiUtPrintString
- *
- * PARAMETERS:  String          - Null terminated ASCII string
- *              MaxLength       - Maximum output length
- *
- * RETURN:      None
- *
- * DESCRIPTION: Dump an ASCII string with support for ACPI-defined escape
- *              sequences.
- *
- ******************************************************************************/
-
-void
-AcpiUtPrintString (
-    char                    *String,
-    UINT8                   MaxLength)
-{
-    UINT32                  i;
-
-
-    if (!String)
-    {
-        AcpiOsPrintf ("<\"NULL STRING PTR\">");
-        return;
-    }
-
-    AcpiOsPrintf ("\"");
-    for (i = 0; String[i] && (i < MaxLength); i++)
-    {
-        /* Escape sequences */
-
-        switch (String[i])
-        {
-        case 0x07:
-            AcpiOsPrintf ("\\a");       /* BELL */
-            break;
-
-        case 0x08:
-            AcpiOsPrintf ("\\b");       /* BACKSPACE */
-            break;
-
-        case 0x0C:
-            AcpiOsPrintf ("\\f");       /* FORMFEED */
-            break;
-
-        case 0x0A:
-            AcpiOsPrintf ("\\n");       /* LINEFEED */
-            break;
-
-        case 0x0D:
-            AcpiOsPrintf ("\\r");       /* CARRIAGE RETURN*/
-            break;
-
-        case 0x09:
-            AcpiOsPrintf ("\\t");       /* HORIZONTAL TAB */
-            break;
-
-        case 0x0B:
-            AcpiOsPrintf ("\\v");       /* VERTICAL TAB */
-            break;
-
-        case '\'':                      /* Single Quote */
-        case '\"':                      /* Double Quote */
-        case '\\':                      /* Backslash */
-            AcpiOsPrintf ("\\%c", (int) String[i]);
-            break;
-
-        default:
-
-            /* Check for printable character or hex escape */
-
-            if (ACPI_IS_PRINT (String[i]))
-            {
-                /* This is a normal character */
-
-                AcpiOsPrintf ("%c", (int) String[i]);
-            }
-            else
-            {
-                /* All others will be Hex escapes */
-
-                AcpiOsPrintf ("\\x%2.2X", (INT32) String[i]);
-            }
-            break;
-        }
-    }
-    AcpiOsPrintf ("\"");
-
-    if (i == MaxLength && String[i])
-    {
-        AcpiOsPrintf ("...");
-    }
-}
-
-
-/*******************************************************************************
- *
  * FUNCTION:    AcpiUtDwordByteSwap
  *
  * PARAMETERS:  Value           - Value to be converted
@@ -642,444 +200,6 @@ AcpiUtSetIntegerWidth (
 }
 
 
-#ifdef ACPI_DEBUG_OUTPUT
-/*******************************************************************************
- *
- * FUNCTION:    AcpiUtDisplayInitPathname
- *
- * PARAMETERS:  Type                - Object type of the node
- *              ObjHandle           - Handle whose pathname will be displayed
- *              Path                - Additional path string to be appended.
- *                                      (NULL if no extra path)
- *
- * RETURN:      ACPI_STATUS
- *
- * DESCRIPTION: Display full pathname of an object, DEBUG ONLY
- *
- ******************************************************************************/
-
-void
-AcpiUtDisplayInitPathname (
-    UINT8                   Type,
-    ACPI_NAMESPACE_NODE     *ObjHandle,
-    char                    *Path)
-{
-    ACPI_STATUS             Status;
-    ACPI_BUFFER             Buffer;
-
-
-    ACPI_FUNCTION_ENTRY ();
-
-
-    /* Only print the path if the appropriate debug level is enabled */
-
-    if (!(AcpiDbgLevel & ACPI_LV_INIT_NAMES))
-    {
-        return;
-    }
-
-    /* Get the full pathname to the node */
-
-    Buffer.Length = ACPI_ALLOCATE_LOCAL_BUFFER;
-    Status = AcpiNsHandleToPathname (ObjHandle, &Buffer);
-    if (ACPI_FAILURE (Status))
-    {
-        return;
-    }
-
-    /* Print what we're doing */
-
-    switch (Type)
-    {
-    case ACPI_TYPE_METHOD:
-        AcpiOsPrintf ("Executing    ");
-        break;
-
-    default:
-        AcpiOsPrintf ("Initializing ");
-        break;
-    }
-
-    /* Print the object type and pathname */
-
-    AcpiOsPrintf ("%-12s  %s",
-        AcpiUtGetTypeName (Type), (char *) Buffer.Pointer);
-
-    /* Extra path is used to append names like _STA, _INI, etc. */
-
-    if (Path)
-    {
-        AcpiOsPrintf (".%s", Path);
-    }
-    AcpiOsPrintf ("\n");
-
-    ACPI_FREE (Buffer.Pointer);
-}
-#endif
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiUtValidAcpiChar
- *
- * PARAMETERS:  Char            - The character to be examined
- *              Position        - Byte position (0-3)
- *
- * RETURN:      TRUE if the character is valid, FALSE otherwise
- *
- * DESCRIPTION: Check for a valid ACPI character. Must be one of:
- *              1) Upper case alpha
- *              2) numeric
- *              3) underscore
- *
- *              We allow a '!' as the last character because of the ASF! table
- *
- ******************************************************************************/
-
-BOOLEAN
-AcpiUtValidAcpiChar (
-    char                    Character,
-    UINT32                  Position)
-{
-
-    if (!((Character >= 'A' && Character <= 'Z') ||
-          (Character >= '0' && Character <= '9') ||
-          (Character == '_')))
-    {
-        /* Allow a '!' in the last position */
-
-        if (Character == '!' && Position == 3)
-        {
-            return (TRUE);
-        }
-
-        return (FALSE);
-    }
-
-    return (TRUE);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiUtValidAcpiName
- *
- * PARAMETERS:  Name            - The name to be examined
- *
- * RETURN:      TRUE if the name is valid, FALSE otherwise
- *
- * DESCRIPTION: Check for a valid ACPI name. Each character must be one of:
- *              1) Upper case alpha
- *              2) numeric
- *              3) underscore
- *
- ******************************************************************************/
-
-BOOLEAN
-AcpiUtValidAcpiName (
-    UINT32                  Name)
-{
-    UINT32                  i;
-
-
-    ACPI_FUNCTION_ENTRY ();
-
-
-    for (i = 0; i < ACPI_NAME_SIZE; i++)
-    {
-        if (!AcpiUtValidAcpiChar ((ACPI_CAST_PTR (char, &Name))[i], i))
-        {
-            return (FALSE);
-        }
-    }
-
-    return (TRUE);
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiUtRepairName
- *
- * PARAMETERS:  Name            - The ACPI name to be repaired
- *
- * RETURN:      Repaired version of the name
- *
- * DESCRIPTION: Repair an ACPI name: Change invalid characters to '*' and
- *              return the new name. NOTE: the Name parameter must reside in
- *              read/write memory, cannot be a const.
- *
- * An ACPI Name must consist of valid ACPI characters. We will repair the name
- * if necessary because we don't want to abort because of this, but we want
- * all namespace names to be printable. A warning message is appropriate.
- *
- * This issue came up because there are in fact machines that exhibit
- * this problem, and we want to be able to enable ACPI support for them,
- * even though there are a few bad names.
- *
- ******************************************************************************/
-
-void
-AcpiUtRepairName (
-    char                    *Name)
-{
-    UINT32                  i;
-    BOOLEAN                 FoundBadChar = FALSE;
-    UINT32                  OriginalName;
-
-
-    ACPI_FUNCTION_NAME (UtRepairName);
-
-
-    ACPI_MOVE_NAME (&OriginalName, Name);
-
-    /* Check each character in the name */
-
-    for (i = 0; i < ACPI_NAME_SIZE; i++)
-    {
-        if (AcpiUtValidAcpiChar (Name[i], i))
-        {
-            continue;
-        }
-
-        /*
-         * Replace a bad character with something printable, yet technically
-         * still invalid. This prevents any collisions with existing "good"
-         * names in the namespace.
-         */
-        Name[i] = '*';
-        FoundBadChar = TRUE;
-    }
-
-    if (FoundBadChar)
-    {
-        /* Report warning only if in strict mode or debug mode */
-
-        if (!AcpiGbl_EnableInterpreterSlack)
-        {
-            ACPI_WARNING ((AE_INFO,
-                "Invalid character(s) in name (0x%.8X), repaired: [%4.4s]",
-                OriginalName, Name));
-        }
-        else
-        {
-            ACPI_DEBUG_PRINT ((ACPI_DB_INFO,
-                "Invalid character(s) in name (0x%.8X), repaired: [%4.4s]",
-                OriginalName, Name));
-        }
-    }
-}
-
-
-/*******************************************************************************
- *
- * FUNCTION:    AcpiUtStrtoul64
- *
- * PARAMETERS:  String          - Null terminated string
- *              Base            - Radix of the string: 16 or ACPI_ANY_BASE;
- *                                ACPI_ANY_BASE means 'in behalf of ToInteger'
- *              RetInteger      - Where the converted integer is returned
- *
- * RETURN:      Status and Converted value
- *
- * DESCRIPTION: Convert a string into an unsigned value. Performs either a
- *              32-bit or 64-bit conversion, depending on the current mode
- *              of the interpreter.
- *              NOTE: Does not support Octal strings, not needed.
- *
- ******************************************************************************/
-
-ACPI_STATUS
-AcpiUtStrtoul64 (
-    char                    *String,
-    UINT32                  Base,
-    UINT64                  *RetInteger)
-{
-    UINT32                  ThisDigit = 0;
-    UINT64                  ReturnValue = 0;
-    UINT64                  Quotient;
-    UINT64                  Dividend;
-    UINT32                  ToIntegerOp = (Base == ACPI_ANY_BASE);
-    UINT32                  Mode32 = (AcpiGbl_IntegerByteWidth == 4);
-    UINT8                   ValidDigits = 0;
-    UINT8                   SignOf0x = 0;
-    UINT8                   Term = 0;
-
-
-    ACPI_FUNCTION_TRACE_STR (UtStroul64, String);
-
-
-    switch (Base)
-    {
-    case ACPI_ANY_BASE:
-    case 16:
-        break;
-
-    default:
-        /* Invalid Base */
-        return_ACPI_STATUS (AE_BAD_PARAMETER);
-    }
-
-    if (!String)
-    {
-        goto ErrorExit;
-    }
-
-    /* Skip over any white space in the buffer */
-
-    while ((*String) && (ACPI_IS_SPACE (*String) || *String == '\t'))
-    {
-        String++;
-    }
-
-    if (ToIntegerOp)
-    {
-        /*
-         * Base equal to ACPI_ANY_BASE means 'ToInteger operation case'.
-         * We need to determine if it is decimal or hexadecimal.
-         */
-        if ((*String == '0') && (ACPI_TOLOWER (*(String + 1)) == 'x'))
-        {
-            SignOf0x = 1;
-            Base = 16;
-
-            /* Skip over the leading '0x' */
-            String += 2;
-        }
-        else
-        {
-            Base = 10;
-        }
-    }
-
-    /* Any string left? Check that '0x' is not followed by white space. */
-
-    if (!(*String) || ACPI_IS_SPACE (*String) || *String == '\t')
-    {
-        if (ToIntegerOp)
-        {
-            goto ErrorExit;
-        }
-        else
-        {
-            goto AllDone;
-        }
-    }
-
-    /*
-     * Perform a 32-bit or 64-bit conversion, depending upon the current
-     * execution mode of the interpreter
-     */
-    Dividend = (Mode32) ? ACPI_UINT32_MAX : ACPI_UINT64_MAX;
-
-    /* Main loop: convert the string to a 32- or 64-bit integer */
-
-    while (*String)
-    {
-        if (ACPI_IS_DIGIT (*String))
-        {
-            /* Convert ASCII 0-9 to Decimal value */
-
-            ThisDigit = ((UINT8) *String) - '0';
-        }
-        else if (Base == 10)
-        {
-            /* Digit is out of range; possible in ToInteger case only */
-
-            Term = 1;
-        }
-        else
-        {
-            ThisDigit = (UINT8) ACPI_TOUPPER (*String);
-            if (ACPI_IS_XDIGIT ((char) ThisDigit))
-            {
-                /* Convert ASCII Hex char to value */
-
-                ThisDigit = ThisDigit - 'A' + 10;
-            }
-            else
-            {
-                Term = 1;
-            }
-        }
-
-        if (Term)
-        {
-            if (ToIntegerOp)
-            {
-                goto ErrorExit;
-            }
-            else
-            {
-                break;
-            }
-        }
-        else if ((ValidDigits == 0) && (ThisDigit == 0) && !SignOf0x)
-        {
-            /* Skip zeros */
-            String++;
-            continue;
-        }
-
-        ValidDigits++;
-
-        if (SignOf0x && ((ValidDigits > 16) || ((ValidDigits > 8) && Mode32)))
-        {
-            /*
-             * This is ToInteger operation case.
-             * No any restrictions for string-to-integer conversion,
-             * see ACPI spec.
-             */
-            goto ErrorExit;
-        }
-
-        /* Divide the digit into the correct position */
-
-        (void) AcpiUtShortDivide ((Dividend - (UINT64) ThisDigit),
-                    Base, &Quotient, NULL);
-
-        if (ReturnValue > Quotient)
-        {
-            if (ToIntegerOp)
-            {
-                goto ErrorExit;
-            }
-            else
-            {
-                break;
-            }
-        }
-
-        ReturnValue *= Base;
-        ReturnValue += ThisDigit;
-        String++;
-    }
-
-    /* All done, normal exit */
-
-AllDone:
-
-    ACPI_DEBUG_PRINT ((ACPI_DB_EXEC, "Converted value: %8.8X%8.8X\n",
-        ACPI_FORMAT_UINT64 (ReturnValue)));
-
-    *RetInteger = ReturnValue;
-    return_ACPI_STATUS (AE_OK);
-
-
-ErrorExit:
-    /* Base was set/validated above */
-
-    if (Base == 10)
-    {
-        return_ACPI_STATUS (AE_BAD_DECIMAL_CONSTANT);
-    }
-    else
-    {
-        return_ACPI_STATUS (AE_BAD_HEX_CONSTANT);
-    }
-}
-
-
 /*******************************************************************************
  *
  * FUNCTION:    AcpiUtCreateUpdateStateAndPush
@@ -1257,3 +377,79 @@ AcpiUtWalkPackageTree (
 
     return_ACPI_STATUS (AE_AML_INTERNAL);
 }
+
+
+#ifdef ACPI_DEBUG_OUTPUT
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiUtDisplayInitPathname
+ *
+ * PARAMETERS:  Type                - Object type of the node
+ *              ObjHandle           - Handle whose pathname will be displayed
+ *              Path                - Additional path string to be appended.
+ *                                      (NULL if no extra path)
+ *
+ * RETURN:      ACPI_STATUS
+ *
+ * DESCRIPTION: Display full pathname of an object, DEBUG ONLY
+ *
+ ******************************************************************************/
+
+void
+AcpiUtDisplayInitPathname (
+    UINT8                   Type,
+    ACPI_NAMESPACE_NODE     *ObjHandle,
+    char                    *Path)
+{
+    ACPI_STATUS             Status;
+    ACPI_BUFFER             Buffer;
+
+
+    ACPI_FUNCTION_ENTRY ();
+
+
+    /* Only print the path if the appropriate debug level is enabled */
+
+    if (!(AcpiDbgLevel & ACPI_LV_INIT_NAMES))
+    {
+        return;
+    }
+
+    /* Get the full pathname to the node */
+
+    Buffer.Length = ACPI_ALLOCATE_LOCAL_BUFFER;
+    Status = AcpiNsHandleToPathname (ObjHandle, &Buffer);
+    if (ACPI_FAILURE (Status))
+    {
+        return;
+    }
+
+    /* Print what we're doing */
+
+    switch (Type)
+    {
+    case ACPI_TYPE_METHOD:
+        AcpiOsPrintf ("Executing    ");
+        break;
+
+    default:
+        AcpiOsPrintf ("Initializing ");
+        break;
+    }
+
+    /* Print the object type and pathname */
+
+    AcpiOsPrintf ("%-12s  %s",
+        AcpiUtGetTypeName (Type), (char *) Buffer.Pointer);
+
+    /* Extra path is used to append names like _STA, _INI, etc. */
+
+    if (Path)
+    {
+        AcpiOsPrintf (".%s", Path);
+    }
+    AcpiOsPrintf ("\n");
+
+    ACPI_FREE (Buffer.Pointer);
+}
+#endif
diff --git a/sys/contrib/dev/acpica/components/utilities/utmutex.c b/sys/contrib/dev/acpica/components/utilities/utmutex.c
index 14b9aa7..90717f1 100644
--- a/sys/contrib/dev/acpica/components/utilities/utmutex.c
+++ b/sys/contrib/dev/acpica/components/utilities/utmutex.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utobject.c b/sys/contrib/dev/acpica/components/utilities/utobject.c
index d4d211a..daa8e40 100644
--- a/sys/contrib/dev/acpica/components/utilities/utobject.c
+++ b/sys/contrib/dev/acpica/components/utilities/utobject.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -725,7 +725,7 @@ AcpiUtGetPackageObjectSize (
     Info.NumPackages = 1;
 
     Status = AcpiUtWalkPackageTree (InternalObject, NULL,
-                            AcpiUtGetElementLength, &Info);
+        AcpiUtGetElementLength, &Info);
     if (ACPI_FAILURE (Status))
     {
         return_ACPI_STATUS (Status);
diff --git a/sys/contrib/dev/acpica/components/utilities/utosi.c b/sys/contrib/dev/acpica/components/utilities/utosi.c
index bd173b7..424cfb6 100644
--- a/sys/contrib/dev/acpica/components/utilities/utosi.c
+++ b/sys/contrib/dev/acpica/components/utilities/utosi.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utownerid.c b/sys/contrib/dev/acpica/components/utilities/utownerid.c
new file mode 100644
index 0000000..74f4d83
--- /dev/null
+++ b/sys/contrib/dev/acpica/components/utilities/utownerid.c
@@ -0,0 +1,241 @@
+/*******************************************************************************
+ *
+ * Module Name: utownerid - Support for Table/Method Owner IDs
+ *
+ ******************************************************************************/
+
+/*
+ * Copyright (C) 2000 - 2013, Intel Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ * 3. Neither the names of the above-listed copyright holders nor the names
+ *    of any contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ */
+
+
+#define __UTOWNERID_C__
+
+#include <contrib/dev/acpica/include/acpi.h>
+#include <contrib/dev/acpica/include/accommon.h>
+#include <contrib/dev/acpica/include/acnamesp.h>
+
+
+#define _COMPONENT          ACPI_UTILITIES
+        ACPI_MODULE_NAME    ("utownerid")
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiUtAllocateOwnerId
+ *
+ * PARAMETERS:  OwnerId         - Where the new owner ID is returned
+ *
+ * RETURN:      Status
+ *
+ * DESCRIPTION: Allocate a table or method owner ID. The owner ID is used to
+ *              track objects created by the table or method, to be deleted
+ *              when the method exits or the table is unloaded.
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+AcpiUtAllocateOwnerId (
+    ACPI_OWNER_ID           *OwnerId)
+{
+    UINT32                  i;
+    UINT32                  j;
+    UINT32                  k;
+    ACPI_STATUS             Status;
+
+
+    ACPI_FUNCTION_TRACE (UtAllocateOwnerId);
+
+
+    /* Guard against multiple allocations of ID to the same location */
+
+    if (*OwnerId)
+    {
+        ACPI_ERROR ((AE_INFO, "Owner ID [0x%2.2X] already exists", *OwnerId));
+        return_ACPI_STATUS (AE_ALREADY_EXISTS);
+    }
+
+    /* Mutex for the global ID mask */
+
+    Status = AcpiUtAcquireMutex (ACPI_MTX_CACHES);
+    if (ACPI_FAILURE (Status))
+    {
+        return_ACPI_STATUS (Status);
+    }
+
+    /*
+     * Find a free owner ID, cycle through all possible IDs on repeated
+     * allocations. (ACPI_NUM_OWNERID_MASKS + 1) because first index may have
+     * to be scanned twice.
+     */
+    for (i = 0, j = AcpiGbl_LastOwnerIdIndex;
+         i < (ACPI_NUM_OWNERID_MASKS + 1);
+         i++, j++)
+    {
+        if (j >= ACPI_NUM_OWNERID_MASKS)
+        {
+            j = 0;  /* Wraparound to start of mask array */
+        }
+
+        for (k = AcpiGbl_NextOwnerIdOffset; k < 32; k++)
+        {
+            if (AcpiGbl_OwnerIdMask[j] == ACPI_UINT32_MAX)
+            {
+                /* There are no free IDs in this mask */
+
+                break;
+            }
+
+            if (!(AcpiGbl_OwnerIdMask[j] & (1 << k)))
+            {
+                /*
+                 * Found a free ID. The actual ID is the bit index plus one,
+                 * making zero an invalid Owner ID. Save this as the last ID
+                 * allocated and update the global ID mask.
+                 */
+                AcpiGbl_OwnerIdMask[j] |= (1 << k);
+
+                AcpiGbl_LastOwnerIdIndex = (UINT8) j;
+                AcpiGbl_NextOwnerIdOffset = (UINT8) (k + 1);
+
+                /*
+                 * Construct encoded ID from the index and bit position
+                 *
+                 * Note: Last [j].k (bit 255) is never used and is marked
+                 * permanently allocated (prevents +1 overflow)
+                 */
+                *OwnerId = (ACPI_OWNER_ID) ((k + 1) + ACPI_MUL_32 (j));
+
+                ACPI_DEBUG_PRINT ((ACPI_DB_VALUES,
+                    "Allocated OwnerId: %2.2X\n", (unsigned int) *OwnerId));
+                goto Exit;
+            }
+        }
+
+        AcpiGbl_NextOwnerIdOffset = 0;
+    }
+
+    /*
+     * All OwnerIds have been allocated. This typically should
+     * not happen since the IDs are reused after deallocation. The IDs are
+     * allocated upon table load (one per table) and method execution, and
+     * they are released when a table is unloaded or a method completes
+     * execution.
+     *
+     * If this error happens, there may be very deep nesting of invoked control
+     * methods, or there may be a bug where the IDs are not released.
+     */
+    Status = AE_OWNER_ID_LIMIT;
+    ACPI_ERROR ((AE_INFO,
+        "Could not allocate new OwnerId (255 max), AE_OWNER_ID_LIMIT"));
+
+Exit:
+    (void) AcpiUtReleaseMutex (ACPI_MTX_CACHES);
+    return_ACPI_STATUS (Status);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiUtReleaseOwnerId
+ *
+ * PARAMETERS:  OwnerIdPtr          - Pointer to a previously allocated OwnerID
+ *
+ * RETURN:      None. No error is returned because we are either exiting a
+ *              control method or unloading a table. Either way, we would
+ *              ignore any error anyway.
+ *
+ * DESCRIPTION: Release a table or method owner ID. Valid IDs are 1 - 255
+ *
+ ******************************************************************************/
+
+void
+AcpiUtReleaseOwnerId (
+    ACPI_OWNER_ID           *OwnerIdPtr)
+{
+    ACPI_OWNER_ID           OwnerId = *OwnerIdPtr;
+    ACPI_STATUS             Status;
+    UINT32                  Index;
+    UINT32                  Bit;
+
+
+    ACPI_FUNCTION_TRACE_U32 (UtReleaseOwnerId, OwnerId);
+
+
+    /* Always clear the input OwnerId (zero is an invalid ID) */
+
+    *OwnerIdPtr = 0;
+
+    /* Zero is not a valid OwnerID */
+
+    if (OwnerId == 0)
+    {
+        ACPI_ERROR ((AE_INFO, "Invalid OwnerId: 0x%2.2X", OwnerId));
+        return_VOID;
+    }
+
+    /* Mutex for the global ID mask */
+
+    Status = AcpiUtAcquireMutex (ACPI_MTX_CACHES);
+    if (ACPI_FAILURE (Status))
+    {
+        return_VOID;
+    }
+
+    /* Normalize the ID to zero */
+
+    OwnerId--;
+
+    /* Decode ID to index/offset pair */
+
+    Index = ACPI_DIV_32 (OwnerId);
+    Bit = 1 << ACPI_MOD_32 (OwnerId);
+
+    /* Free the owner ID only if it is valid */
+
+    if (AcpiGbl_OwnerIdMask[Index] & Bit)
+    {
+        AcpiGbl_OwnerIdMask[Index] ^= Bit;
+    }
+    else
+    {
+        ACPI_ERROR ((AE_INFO,
+            "Release of non-allocated OwnerId: 0x%2.2X", OwnerId + 1));
+    }
+
+    (void) AcpiUtReleaseMutex (ACPI_MTX_CACHES);
+    return_VOID;
+}
diff --git a/sys/contrib/dev/acpica/components/utilities/utresrc.c b/sys/contrib/dev/acpica/components/utilities/utresrc.c
index 08b8550..3a47a0c 100644
--- a/sys/contrib/dev/acpica/components/utilities/utresrc.c
+++ b/sys/contrib/dev/acpica/components/utilities/utresrc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -456,7 +456,7 @@ AcpiUtWalkAmlResources (
     UINT8                   *Aml,
     ACPI_SIZE               AmlLength,
     ACPI_WALK_AML_CALLBACK  UserFunction,
-    void                    *Context)
+    void                    **Context)
 {
     ACPI_STATUS             Status;
     UINT8                   *EndAml;
@@ -528,7 +528,7 @@ AcpiUtWalkAmlResources (
 
             if (!UserFunction)
             {
-                *(void **) Context = Aml;
+                *Context = Aml;
             }
 
             /* Normal exit */
@@ -919,7 +919,7 @@ AcpiUtGetResourceEndTag (
     /* Validate the template and get a pointer to the EndTag */
 
     Status = AcpiUtWalkAmlResources (NULL, ObjDesc->Buffer.Pointer,
-                ObjDesc->Buffer.Length, NULL, EndTag);
+                ObjDesc->Buffer.Length, NULL, (void **) EndTag);
 
     return_ACPI_STATUS (Status);
 }
diff --git a/sys/contrib/dev/acpica/components/utilities/utstate.c b/sys/contrib/dev/acpica/components/utilities/utstate.c
index 8edb9cd..e0e5cce 100644
--- a/sys/contrib/dev/acpica/components/utilities/utstate.c
+++ b/sys/contrib/dev/acpica/components/utilities/utstate.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utstring.c b/sys/contrib/dev/acpica/components/utilities/utstring.c
new file mode 100644
index 0000000..49a47fe
--- /dev/null
+++ b/sys/contrib/dev/acpica/components/utilities/utstring.c
@@ -0,0 +1,674 @@
+/*******************************************************************************
+ *
+ * Module Name: utstring - Common functions for strings and characters
+ *
+ ******************************************************************************/
+
+/*
+ * Copyright (C) 2000 - 2013, Intel Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon
+ *    including a substantially similar Disclaimer requirement for further
+ *    binary redistribution.
+ * 3. Neither the names of the above-listed copyright holders nor the names
+ *    of any contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * Alternatively, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") version 2 as published by the Free
+ * Software Foundation.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGES.
+ */
+
+
+#define __UTSTRING_C__
+
+#include <contrib/dev/acpica/include/acpi.h>
+#include <contrib/dev/acpica/include/accommon.h>
+#include <contrib/dev/acpica/include/acnamesp.h>
+
+
+#define _COMPONENT          ACPI_UTILITIES
+        ACPI_MODULE_NAME    ("utstring")
+
+
+/*
+ * Non-ANSI C library functions - strlwr, strupr, stricmp, and a 64-bit
+ * version of strtoul.
+ */
+
+#ifdef ACPI_ASL_COMPILER
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiUtStrlwr (strlwr)
+ *
+ * PARAMETERS:  SrcString       - The source string to convert
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Convert string to lowercase
+ *
+ * NOTE: This is not a POSIX function, so it appears here, not in utclib.c
+ *
+ ******************************************************************************/
+
+void
+AcpiUtStrlwr (
+    char                    *SrcString)
+{
+    char                    *String;
+
+
+    ACPI_FUNCTION_ENTRY ();
+
+
+    if (!SrcString)
+    {
+        return;
+    }
+
+    /* Walk entire string, lowercasing the letters */
+
+    for (String = SrcString; *String; String++)
+    {
+        *String = (char) ACPI_TOLOWER (*String);
+    }
+
+    return;
+}
+
+
+/******************************************************************************
+ *
+ * FUNCTION:    AcpiUtStricmp (stricmp)
+ *
+ * PARAMETERS:  String1             - first string to compare
+ *              String2             - second string to compare
+ *
+ * RETURN:      int that signifies string relationship. Zero means strings
+ *              are equal.
+ *
+ * DESCRIPTION: Implementation of the non-ANSI stricmp function (compare
+ *              strings with no case sensitivity)
+ *
+ ******************************************************************************/
+
+int
+AcpiUtStricmp (
+    char                    *String1,
+    char                    *String2)
+{
+    int                     c1;
+    int                     c2;
+
+
+    do
+    {
+        c1 = tolower ((int) *String1);
+        c2 = tolower ((int) *String2);
+
+        String1++;
+        String2++;
+    }
+    while ((c1 == c2) && (c1));
+
+    return (c1 - c2);
+}
+#endif
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiUtStrupr (strupr)
+ *
+ * PARAMETERS:  SrcString       - The source string to convert
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Convert string to uppercase
+ *
+ * NOTE: This is not a POSIX function, so it appears here, not in utclib.c
+ *
+ ******************************************************************************/
+
+void
+AcpiUtStrupr (
+    char                    *SrcString)
+{
+    char                    *String;
+
+
+    ACPI_FUNCTION_ENTRY ();
+
+
+    if (!SrcString)
+    {
+        return;
+    }
+
+    /* Walk entire string, uppercasing the letters */
+
+    for (String = SrcString; *String; String++)
+    {
+        *String = (char) ACPI_TOUPPER (*String);
+    }
+
+    return;
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiUtStrtoul64
+ *
+ * PARAMETERS:  String          - Null terminated string
+ *              Base            - Radix of the string: 16 or ACPI_ANY_BASE;
+ *                                ACPI_ANY_BASE means 'in behalf of ToInteger'
+ *              RetInteger      - Where the converted integer is returned
+ *
+ * RETURN:      Status and Converted value
+ *
+ * DESCRIPTION: Convert a string into an unsigned value. Performs either a
+ *              32-bit or 64-bit conversion, depending on the current mode
+ *              of the interpreter.
+ *              NOTE: Does not support Octal strings, not needed.
+ *
+ ******************************************************************************/
+
+ACPI_STATUS
+AcpiUtStrtoul64 (
+    char                    *String,
+    UINT32                  Base,
+    UINT64                  *RetInteger)
+{
+    UINT32                  ThisDigit = 0;
+    UINT64                  ReturnValue = 0;
+    UINT64                  Quotient;
+    UINT64                  Dividend;
+    UINT32                  ToIntegerOp = (Base == ACPI_ANY_BASE);
+    UINT32                  Mode32 = (AcpiGbl_IntegerByteWidth == 4);
+    UINT8                   ValidDigits = 0;
+    UINT8                   SignOf0x = 0;
+    UINT8                   Term = 0;
+
+
+    ACPI_FUNCTION_TRACE_STR (UtStroul64, String);
+
+
+    switch (Base)
+    {
+    case ACPI_ANY_BASE:
+    case 16:
+        break;
+
+    default:
+        /* Invalid Base */
+        return_ACPI_STATUS (AE_BAD_PARAMETER);
+    }
+
+    if (!String)
+    {
+        goto ErrorExit;
+    }
+
+    /* Skip over any white space in the buffer */
+
+    while ((*String) && (ACPI_IS_SPACE (*String) || *String == '\t'))
+    {
+        String++;
+    }
+
+    if (ToIntegerOp)
+    {
+        /*
+         * Base equal to ACPI_ANY_BASE means 'ToInteger operation case'.
+         * We need to determine if it is decimal or hexadecimal.
+         */
+        if ((*String == '0') && (ACPI_TOLOWER (*(String + 1)) == 'x'))
+        {
+            SignOf0x = 1;
+            Base = 16;
+
+            /* Skip over the leading '0x' */
+            String += 2;
+        }
+        else
+        {
+            Base = 10;
+        }
+    }
+
+    /* Any string left? Check that '0x' is not followed by white space. */
+
+    if (!(*String) || ACPI_IS_SPACE (*String) || *String == '\t')
+    {
+        if (ToIntegerOp)
+        {
+            goto ErrorExit;
+        }
+        else
+        {
+            goto AllDone;
+        }
+    }
+
+    /*
+     * Perform a 32-bit or 64-bit conversion, depending upon the current
+     * execution mode of the interpreter
+     */
+    Dividend = (Mode32) ? ACPI_UINT32_MAX : ACPI_UINT64_MAX;
+
+    /* Main loop: convert the string to a 32- or 64-bit integer */
+
+    while (*String)
+    {
+        if (ACPI_IS_DIGIT (*String))
+        {
+            /* Convert ASCII 0-9 to Decimal value */
+
+            ThisDigit = ((UINT8) *String) - '0';
+        }
+        else if (Base == 10)
+        {
+            /* Digit is out of range; possible in ToInteger case only */
+
+            Term = 1;
+        }
+        else
+        {
+            ThisDigit = (UINT8) ACPI_TOUPPER (*String);
+            if (ACPI_IS_XDIGIT ((char) ThisDigit))
+            {
+                /* Convert ASCII Hex char to value */
+
+                ThisDigit = ThisDigit - 'A' + 10;
+            }
+            else
+            {
+                Term = 1;
+            }
+        }
+
+        if (Term)
+        {
+            if (ToIntegerOp)
+            {
+                goto ErrorExit;
+            }
+            else
+            {
+                break;
+            }
+        }
+        else if ((ValidDigits == 0) && (ThisDigit == 0) && !SignOf0x)
+        {
+            /* Skip zeros */
+            String++;
+            continue;
+        }
+
+        ValidDigits++;
+
+        if (SignOf0x && ((ValidDigits > 16) || ((ValidDigits > 8) && Mode32)))
+        {
+            /*
+             * This is ToInteger operation case.
+             * No any restrictions for string-to-integer conversion,
+             * see ACPI spec.
+             */
+            goto ErrorExit;
+        }
+
+        /* Divide the digit into the correct position */
+
+        (void) AcpiUtShortDivide ((Dividend - (UINT64) ThisDigit),
+                    Base, &Quotient, NULL);
+
+        if (ReturnValue > Quotient)
+        {
+            if (ToIntegerOp)
+            {
+                goto ErrorExit;
+            }
+            else
+            {
+                break;
+            }
+        }
+
+        ReturnValue *= Base;
+        ReturnValue += ThisDigit;
+        String++;
+    }
+
+    /* All done, normal exit */
+
+AllDone:
+
+    ACPI_DEBUG_PRINT ((ACPI_DB_EXEC, "Converted value: %8.8X%8.8X\n",
+        ACPI_FORMAT_UINT64 (ReturnValue)));
+
+    *RetInteger = ReturnValue;
+    return_ACPI_STATUS (AE_OK);
+
+
+ErrorExit:
+    /* Base was set/validated above */
+
+    if (Base == 10)
+    {
+        return_ACPI_STATUS (AE_BAD_DECIMAL_CONSTANT);
+    }
+    else
+    {
+        return_ACPI_STATUS (AE_BAD_HEX_CONSTANT);
+    }
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiUtPrintString
+ *
+ * PARAMETERS:  String          - Null terminated ASCII string
+ *              MaxLength       - Maximum output length
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Dump an ASCII string with support for ACPI-defined escape
+ *              sequences.
+ *
+ ******************************************************************************/
+
+void
+AcpiUtPrintString (
+    char                    *String,
+    UINT8                   MaxLength)
+{
+    UINT32                  i;
+
+
+    if (!String)
+    {
+        AcpiOsPrintf ("<\"NULL STRING PTR\">");
+        return;
+    }
+
+    AcpiOsPrintf ("\"");
+    for (i = 0; String[i] && (i < MaxLength); i++)
+    {
+        /* Escape sequences */
+
+        switch (String[i])
+        {
+        case 0x07:
+            AcpiOsPrintf ("\\a");       /* BELL */
+            break;
+
+        case 0x08:
+            AcpiOsPrintf ("\\b");       /* BACKSPACE */
+            break;
+
+        case 0x0C:
+            AcpiOsPrintf ("\\f");       /* FORMFEED */
+            break;
+
+        case 0x0A:
+            AcpiOsPrintf ("\\n");       /* LINEFEED */
+            break;
+
+        case 0x0D:
+            AcpiOsPrintf ("\\r");       /* CARRIAGE RETURN*/
+            break;
+
+        case 0x09:
+            AcpiOsPrintf ("\\t");       /* HORIZONTAL TAB */
+            break;
+
+        case 0x0B:
+            AcpiOsPrintf ("\\v");       /* VERTICAL TAB */
+            break;
+
+        case '\'':                      /* Single Quote */
+        case '\"':                      /* Double Quote */
+        case '\\':                      /* Backslash */
+            AcpiOsPrintf ("\\%c", (int) String[i]);
+            break;
+
+        default:
+
+            /* Check for printable character or hex escape */
+
+            if (ACPI_IS_PRINT (String[i]))
+            {
+                /* This is a normal character */
+
+                AcpiOsPrintf ("%c", (int) String[i]);
+            }
+            else
+            {
+                /* All others will be Hex escapes */
+
+                AcpiOsPrintf ("\\x%2.2X", (INT32) String[i]);
+            }
+            break;
+        }
+    }
+    AcpiOsPrintf ("\"");
+
+    if (i == MaxLength && String[i])
+    {
+        AcpiOsPrintf ("...");
+    }
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiUtValidAcpiChar
+ *
+ * PARAMETERS:  Char            - The character to be examined
+ *              Position        - Byte position (0-3)
+ *
+ * RETURN:      TRUE if the character is valid, FALSE otherwise
+ *
+ * DESCRIPTION: Check for a valid ACPI character. Must be one of:
+ *              1) Upper case alpha
+ *              2) numeric
+ *              3) underscore
+ *
+ *              We allow a '!' as the last character because of the ASF! table
+ *
+ ******************************************************************************/
+
+BOOLEAN
+AcpiUtValidAcpiChar (
+    char                    Character,
+    UINT32                  Position)
+{
+
+    if (!((Character >= 'A' && Character <= 'Z') ||
+          (Character >= '0' && Character <= '9') ||
+          (Character == '_')))
+    {
+        /* Allow a '!' in the last position */
+
+        if (Character == '!' && Position == 3)
+        {
+            return (TRUE);
+        }
+
+        return (FALSE);
+    }
+
+    return (TRUE);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiUtValidAcpiName
+ *
+ * PARAMETERS:  Name            - The name to be examined
+ *
+ * RETURN:      TRUE if the name is valid, FALSE otherwise
+ *
+ * DESCRIPTION: Check for a valid ACPI name. Each character must be one of:
+ *              1) Upper case alpha
+ *              2) numeric
+ *              3) underscore
+ *
+ ******************************************************************************/
+
+BOOLEAN
+AcpiUtValidAcpiName (
+    UINT32                  Name)
+{
+    UINT32                  i;
+
+
+    ACPI_FUNCTION_ENTRY ();
+
+
+    for (i = 0; i < ACPI_NAME_SIZE; i++)
+    {
+        if (!AcpiUtValidAcpiChar ((ACPI_CAST_PTR (char, &Name))[i], i))
+        {
+            return (FALSE);
+        }
+    }
+
+    return (TRUE);
+}
+
+
+/*******************************************************************************
+ *
+ * FUNCTION:    AcpiUtRepairName
+ *
+ * PARAMETERS:  Name            - The ACPI name to be repaired
+ *
+ * RETURN:      Repaired version of the name
+ *
+ * DESCRIPTION: Repair an ACPI name: Change invalid characters to '*' and
+ *              return the new name. NOTE: the Name parameter must reside in
+ *              read/write memory, cannot be a const.
+ *
+ * An ACPI Name must consist of valid ACPI characters. We will repair the name
+ * if necessary because we don't want to abort because of this, but we want
+ * all namespace names to be printable. A warning message is appropriate.
+ *
+ * This issue came up because there are in fact machines that exhibit
+ * this problem, and we want to be able to enable ACPI support for them,
+ * even though there are a few bad names.
+ *
+ ******************************************************************************/
+
+void
+AcpiUtRepairName (
+    char                    *Name)
+{
+    UINT32                  i;
+    BOOLEAN                 FoundBadChar = FALSE;
+    UINT32                  OriginalName;
+
+
+    ACPI_FUNCTION_NAME (UtRepairName);
+
+
+    ACPI_MOVE_NAME (&OriginalName, Name);
+
+    /* Check each character in the name */
+
+    for (i = 0; i < ACPI_NAME_SIZE; i++)
+    {
+        if (AcpiUtValidAcpiChar (Name[i], i))
+        {
+            continue;
+        }
+
+        /*
+         * Replace a bad character with something printable, yet technically
+         * still invalid. This prevents any collisions with existing "good"
+         * names in the namespace.
+         */
+        Name[i] = '*';
+        FoundBadChar = TRUE;
+    }
+
+    if (FoundBadChar)
+    {
+        /* Report warning only if in strict mode or debug mode */
+
+        if (!AcpiGbl_EnableInterpreterSlack)
+        {
+            ACPI_WARNING ((AE_INFO,
+                "Invalid character(s) in name (0x%.8X), repaired: [%4.4s]",
+                OriginalName, Name));
+        }
+        else
+        {
+            ACPI_DEBUG_PRINT ((ACPI_DB_INFO,
+                "Invalid character(s) in name (0x%.8X), repaired: [%4.4s]",
+                OriginalName, Name));
+        }
+    }
+}
+
+
+#if defined ACPI_ASL_COMPILER || defined ACPI_EXEC_APP
+/*******************************************************************************
+ *
+ * FUNCTION:    UtConvertBackslashes
+ *
+ * PARAMETERS:  Pathname        - File pathname string to be converted
+ *
+ * RETURN:      Modifies the input Pathname
+ *
+ * DESCRIPTION: Convert all backslashes (0x5C) to forward slashes (0x2F) within
+ *              the entire input file pathname string.
+ *
+ ******************************************************************************/
+
+void
+UtConvertBackslashes (
+    char                    *Pathname)
+{
+
+    if (!Pathname)
+    {
+        return;
+    }
+
+    while (*Pathname)
+    {
+        if (*Pathname == '\\')
+        {
+            *Pathname = '/';
+        }
+
+        Pathname++;
+    }
+}
+#endif
diff --git a/sys/contrib/dev/acpica/components/utilities/uttrack.c b/sys/contrib/dev/acpica/components/utilities/uttrack.c
index 76b5bb3..c829fb0 100644
--- a/sys/contrib/dev/acpica/components/utilities/uttrack.c
+++ b/sys/contrib/dev/acpica/components/utilities/uttrack.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utxface.c b/sys/contrib/dev/acpica/components/utilities/utxface.c
index 4abe72e..cd3e7b2 100644
--- a/sys/contrib/dev/acpica/components/utilities/utxface.c
+++ b/sys/contrib/dev/acpica/components/utilities/utxface.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utxferror.c b/sys/contrib/dev/acpica/components/utilities/utxferror.c
index e106c9b..3067294 100644
--- a/sys/contrib/dev/acpica/components/utilities/utxferror.c
+++ b/sys/contrib/dev/acpica/components/utilities/utxferror.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utxfinit.c b/sys/contrib/dev/acpica/components/utilities/utxfinit.c
index e6e0f5d..c828a71 100644
--- a/sys/contrib/dev/acpica/components/utilities/utxfinit.c
+++ b/sys/contrib/dev/acpica/components/utilities/utxfinit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/components/utilities/utxfmutex.c b/sys/contrib/dev/acpica/components/utilities/utxfmutex.c
index ca07871..ae5a345 100644
--- a/sys/contrib/dev/acpica/components/utilities/utxfmutex.c
+++ b/sys/contrib/dev/acpica/components/utilities/utxfmutex.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/acapps.h b/sys/contrib/dev/acpica/include/acapps.h
index 0b3d72a..acc6e4a 100644
--- a/sys/contrib/dev/acpica/include/acapps.h
+++ b/sys/contrib/dev/acpica/include/acapps.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -52,7 +52,7 @@
 /* Common info for tool signons */
 
 #define ACPICA_NAME                 "Intel ACPI Component Architecture"
-#define ACPICA_COPYRIGHT            "Copyright (c) 2000 - 2012 Intel Corporation"
+#define ACPICA_COPYRIGHT            "Copyright (c) 2000 - 2013 Intel Corporation"
 
 #if ACPI_MACHINE_WIDTH == 64
 #define ACPI_WIDTH          "-64"
diff --git a/sys/contrib/dev/acpica/include/acbuffer.h b/sys/contrib/dev/acpica/include/acbuffer.h
index 4b34ccd..9651024 100644
--- a/sys/contrib/dev/acpica/include/acbuffer.h
+++ b/sys/contrib/dev/acpica/include/acbuffer.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/accommon.h b/sys/contrib/dev/acpica/include/accommon.h
index 93c9031..074b6c1 100644
--- a/sys/contrib/dev/acpica/include/accommon.h
+++ b/sys/contrib/dev/acpica/include/accommon.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/acconfig.h b/sys/contrib/dev/acpica/include/acconfig.h
index b414383..452ab28 100644
--- a/sys/contrib/dev/acpica/include/acconfig.h
+++ b/sys/contrib/dev/acpica/include/acconfig.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -194,6 +194,7 @@
 /* Maximum SpaceIds for Operation Regions */
 
 #define ACPI_MAX_ADDRESS_SPACE          255
+#define ACPI_NUM_DEFAULT_SPACES         4
 
 /* Array sizes. Used for range checking also */
 
diff --git a/sys/contrib/dev/acpica/include/acdebug.h b/sys/contrib/dev/acpica/include/acdebug.h
index de915b4..5fef182 100644
--- a/sys/contrib/dev/acpica/include/acdebug.h
+++ b/sys/contrib/dev/acpica/include/acdebug.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -161,6 +161,34 @@ AcpiDbGenerateGpe (
 
 
 /*
+ * dbconvert - miscellaneous conversion routines
+ */
+ACPI_STATUS
+AcpiDbHexCharToValue (
+    int                     HexChar,
+    UINT8                   *ReturnValue);
+
+ACPI_STATUS
+AcpiDbConvertToPackage (
+    char                    *String,
+    ACPI_OBJECT             *Object);
+
+ACPI_STATUS
+AcpiDbConvertToObject (
+    ACPI_OBJECT_TYPE        Type,
+    char                    *String,
+    ACPI_OBJECT             *Object);
+
+UINT8 *
+AcpiDbEncodePldBuffer (
+    ACPI_PLD_INFO           *PldInfo);
+
+void
+AcpiDbDumpPldBuffer (
+    ACPI_OBJECT             *ObjDesc);
+
+
+/*
  * dbmethod - control method commands
  */
 void
@@ -299,6 +327,11 @@ AcpiDbCreateExecutionThreads (
     char                    *NumLoopsArg,
     char                    *MethodNameArg);
 
+void
+AcpiDbDeleteObjects (
+    UINT32                  Count,
+    ACPI_OBJECT             *Objects);
+
 #ifdef ACPI_DBG_TRACK_ALLOCATIONS
 UINT32
 AcpiDbGetCacheInfo (
diff --git a/sys/contrib/dev/acpica/include/acdisasm.h b/sys/contrib/dev/acpica/include/acdisasm.h
index 20e056a..eb6553b 100644
--- a/sys/contrib/dev/acpica/include/acdisasm.h
+++ b/sys/contrib/dev/acpica/include/acdisasm.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -237,6 +237,7 @@ extern ACPI_DMTABLE_INFO        AcpiDmTableInfoCpep[];
 extern ACPI_DMTABLE_INFO        AcpiDmTableInfoCpep0[];
 extern ACPI_DMTABLE_INFO        AcpiDmTableInfoCsrt0[];
 extern ACPI_DMTABLE_INFO        AcpiDmTableInfoCsrt1[];
+extern ACPI_DMTABLE_INFO        AcpiDmTableInfoCsrt2[];
 extern ACPI_DMTABLE_INFO        AcpiDmTableInfoDbg2[];
 extern ACPI_DMTABLE_INFO        AcpiDmTableInfoDbg2Device[];
 extern ACPI_DMTABLE_INFO        AcpiDmTableInfoDbg2Addr[];
@@ -340,6 +341,7 @@ extern ACPI_DMTABLE_INFO        AcpiDmTableInfoSrat0[];
 extern ACPI_DMTABLE_INFO        AcpiDmTableInfoSrat1[];
 extern ACPI_DMTABLE_INFO        AcpiDmTableInfoSrat2[];
 extern ACPI_DMTABLE_INFO        AcpiDmTableInfoTcpa[];
+extern ACPI_DMTABLE_INFO        AcpiDmTableInfoTpm2[];
 extern ACPI_DMTABLE_INFO        AcpiDmTableInfoUefi[];
 extern ACPI_DMTABLE_INFO        AcpiDmTableInfoWaet[];
 extern ACPI_DMTABLE_INFO        AcpiDmTableInfoWdat[];
diff --git a/sys/contrib/dev/acpica/include/acdispat.h b/sys/contrib/dev/acpica/include/acdispat.h
index 0699ee7..fc747f1 100644
--- a/sys/contrib/dev/acpica/include/acdispat.h
+++ b/sys/contrib/dev/acpica/include/acdispat.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/acevents.h b/sys/contrib/dev/acpica/include/acevents.h
index 3b874f1..fc652b4 100644
--- a/sys/contrib/dev/acpica/include/acevents.h
+++ b/sys/contrib/dev/acpica/include/acevents.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -167,6 +167,7 @@ AcpiEvGpeDispatch (
     ACPI_GPE_EVENT_INFO     *GpeEventInfo,
     UINT32                  GpeNumber);
 
+
 /*
  * evgpeinit - GPE initialization and update
  */
@@ -186,6 +187,7 @@ AcpiEvMatchGpeMethod (
     void                    *Context,
     void                    **ReturnValue);
 
+
 /*
  * evgpeutil - GPE utilities
  */
@@ -220,13 +222,30 @@ AcpiEvDeleteGpeHandlers (
 
 
 /*
- * evregion - Address Space handling
+ * evhandler - Address space handling
  */
+BOOLEAN
+AcpiEvHasDefaultHandler (
+    ACPI_NAMESPACE_NODE     *Node,
+    ACPI_ADR_SPACE_TYPE     SpaceId);
+
 ACPI_STATUS
 AcpiEvInstallRegionHandlers (
     void);
 
 ACPI_STATUS
+AcpiEvInstallSpaceHandler (
+    ACPI_NAMESPACE_NODE     *Node,
+    ACPI_ADR_SPACE_TYPE     SpaceId,
+    ACPI_ADR_SPACE_HANDLER  Handler,
+    ACPI_ADR_SPACE_SETUP    Setup,
+    void                    *Context);
+
+
+/*
+ * evregion - Operation region support
+ */
+ACPI_STATUS
 AcpiEvInitializeOpRegions (
     void);
 
@@ -251,14 +270,6 @@ AcpiEvDetachRegion (
     BOOLEAN                 AcpiNsIsLocked);
 
 ACPI_STATUS
-AcpiEvInstallSpaceHandler (
-    ACPI_NAMESPACE_NODE     *Node,
-    ACPI_ADR_SPACE_TYPE     SpaceId,
-    ACPI_ADR_SPACE_HANDLER  Handler,
-    ACPI_ADR_SPACE_SETUP    Setup,
-    void                    *Context);
-
-ACPI_STATUS
 AcpiEvExecuteRegMethods (
     ACPI_NAMESPACE_NODE     *Node,
     ACPI_ADR_SPACE_TYPE     SpaceId);
diff --git a/sys/contrib/dev/acpica/include/acexcep.h b/sys/contrib/dev/acpica/include/acexcep.h
index 072ddf9..110ef18 100644
--- a/sys/contrib/dev/acpica/include/acexcep.h
+++ b/sys/contrib/dev/acpica/include/acexcep.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/acglobal.h b/sys/contrib/dev/acpica/include/acglobal.h
index 42c6234..83547e6 100644
--- a/sys/contrib/dev/acpica/include/acglobal.h
+++ b/sys/contrib/dev/acpica/include/acglobal.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -349,7 +349,6 @@ ACPI_EXTERN UINT32                      AcpiGbl_DeepestNesting;
  *
  ****************************************************************************/
 
-
 ACPI_EXTERN ACPI_THREAD_STATE          *AcpiGbl_CurrentWalkList;
 
 /* Control method single step flag */
@@ -413,7 +412,7 @@ ACPI_EXTERN UINT32                      AcpiGbl_TraceDbgLayer;
 
 /*****************************************************************************
  *
- * Debugger globals
+ * Debugger and Disassembler globals
  *
  ****************************************************************************/
 
@@ -421,6 +420,8 @@ ACPI_EXTERN UINT8                       AcpiGbl_DbOutputFlags;
 
 #ifdef ACPI_DISASSEMBLER
 
+ACPI_EXTERN BOOLEAN     ACPI_INIT_GLOBAL (AcpiGbl_IgnoreNoopOperator, FALSE);
+
 ACPI_EXTERN BOOLEAN                     AcpiGbl_DbOpt_disasm;
 ACPI_EXTERN BOOLEAN                     AcpiGbl_DbOpt_verbose;
 ACPI_EXTERN ACPI_EXTERNAL_LIST         *AcpiGbl_ExternalList;
diff --git a/sys/contrib/dev/acpica/include/achware.h b/sys/contrib/dev/acpica/include/achware.h
index 6dc1191..9577fd1 100644
--- a/sys/contrib/dev/acpica/include/achware.h
+++ b/sys/contrib/dev/acpica/include/achware.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/acinterp.h b/sys/contrib/dev/acpica/include/acinterp.h
index 45e9814..ebf4c98 100644
--- a/sys/contrib/dev/acpica/include/acinterp.h
+++ b/sys/contrib/dev/acpica/include/acinterp.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -620,7 +620,7 @@ void
 AcpiExRelinquishInterpreter (
     void);
 
-void
+BOOLEAN
 AcpiExTruncateFor32bitTable (
     ACPI_OPERAND_OBJECT     *ObjDesc);
 
diff --git a/sys/contrib/dev/acpica/include/aclocal.h b/sys/contrib/dev/acpica/include/aclocal.h
index d54455f..798c4cb 100644
--- a/sys/contrib/dev/acpica/include/aclocal.h
+++ b/sys/contrib/dev/acpica/include/aclocal.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/acmacros.h b/sys/contrib/dev/acpica/include/acmacros.h
index 9b44671..a56dcd7 100644
--- a/sys/contrib/dev/acpica/include/acmacros.h
+++ b/sys/contrib/dev/acpica/include/acmacros.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -314,15 +314,23 @@
 #define ACPI_EXTRACT_3BIT_FLAG(Field, Position)     (ACPI_GET_3BIT_FLAG ((Field) >> Position))
 #define ACPI_EXTRACT_4BIT_FLAG(Field, Position)     (ACPI_GET_4BIT_FLAG ((Field) >> Position))
 
+/* ACPI Pathname helpers */
+
+#define ACPI_IS_ROOT_PREFIX(c)      ((c) == (UINT8) 0x5C) /* Backslash */
+#define ACPI_IS_PARENT_PREFIX(c)    ((c) == (UINT8) 0x5E) /* Carat */
+#define ACPI_IS_PATH_SEPARATOR(c)   ((c) == (UINT8) 0x2E) /* Period (dot) */
+
 /*
  * An object of type ACPI_NAMESPACE_NODE can appear in some contexts
  * where a pointer to an object of type ACPI_OPERAND_OBJECT can also
  * appear. This macro is used to distinguish them.
  *
- * The "Descriptor" field is the first field in both structures.
+ * The "DescriptorType" field is the second field in both structures.
  */
+#define ACPI_GET_DESCRIPTOR_PTR(d)      (((ACPI_DESCRIPTOR *)(void *)(d))->Common.CommonPointer)
+#define ACPI_SET_DESCRIPTOR_PTR(d, p)   (((ACPI_DESCRIPTOR *)(void *)(d))->Common.CommonPointer = (p))
 #define ACPI_GET_DESCRIPTOR_TYPE(d)     (((ACPI_DESCRIPTOR *)(void *)(d))->Common.DescriptorType)
-#define ACPI_SET_DESCRIPTOR_TYPE(d, t)  (((ACPI_DESCRIPTOR *)(void *)(d))->Common.DescriptorType = t)
+#define ACPI_SET_DESCRIPTOR_TYPE(d, t)  (((ACPI_DESCRIPTOR *)(void *)(d))->Common.DescriptorType = (t))
 
 /*
  * Macros for the master AML opcode table
@@ -385,139 +393,6 @@
 
 #endif /* ACPI_NO_ERROR_MESSAGES */
 
-/*
- * Debug macros that are conditionally compiled
- */
-#ifdef ACPI_DEBUG_OUTPUT
-/*
- * Function entry tracing
- */
-#define ACPI_FUNCTION_TRACE(a)          ACPI_FUNCTION_NAME(a) \
-                                            AcpiUtTrace(ACPI_DEBUG_PARAMETERS)
-#define ACPI_FUNCTION_TRACE_PTR(a, b)   ACPI_FUNCTION_NAME(a) \
-                                            AcpiUtTracePtr(ACPI_DEBUG_PARAMETERS, (void *)b)
-#define ACPI_FUNCTION_TRACE_U32(a, b)   ACPI_FUNCTION_NAME(a) \
-                                            AcpiUtTraceU32(ACPI_DEBUG_PARAMETERS, (UINT32)b)
-#define ACPI_FUNCTION_TRACE_STR(a, b)   ACPI_FUNCTION_NAME(a) \
-                                            AcpiUtTraceStr(ACPI_DEBUG_PARAMETERS, (char *)b)
-
-#define ACPI_FUNCTION_ENTRY()           AcpiUtTrackStackPtr()
-
-/*
- * Function exit tracing.
- * WARNING: These macros include a return statement. This is usually considered
- * bad form, but having a separate exit macro is very ugly and difficult to maintain.
- * One of the FUNCTION_TRACE macros above must be used in conjunction with these macros
- * so that "_AcpiFunctionName" is defined.
- *
- * Note: the DO_WHILE0 macro is used to prevent some compilers from complaining
- * about these constructs.
- */
-#ifdef ACPI_USE_DO_WHILE_0
-#define ACPI_DO_WHILE0(a)               do a while(0)
-#else
-#define ACPI_DO_WHILE0(a)               a
-#endif
-
-#define return_VOID                     ACPI_DO_WHILE0 ({ \
-                                            AcpiUtExit (ACPI_DEBUG_PARAMETERS); \
-                                            return;})
-/*
- * There are two versions of most of the return macros. The default version is
- * safer, since it avoids side-effects by guaranteeing that the argument will
- * not be evaluated twice.
- *
- * A less-safe version of the macros is provided for optional use if the
- * compiler uses excessive CPU stack (for example, this may happen in the
- * debug case if code optimzation is disabled.)
- */
-#ifndef ACPI_SIMPLE_RETURN_MACROS
-
-#define return_ACPI_STATUS(s)           ACPI_DO_WHILE0 ({ \
-                                            register ACPI_STATUS _s = (s); \
-                                            AcpiUtStatusExit (ACPI_DEBUG_PARAMETERS, _s); \
-                                            return (_s); })
-#define return_PTR(s)                   ACPI_DO_WHILE0 ({ \
-                                            register void *_s = (void *) (s); \
-                                            AcpiUtPtrExit (ACPI_DEBUG_PARAMETERS, (UINT8 *) _s); \
-                                            return (_s); })
-#define return_VALUE(s)                 ACPI_DO_WHILE0 ({ \
-                                            register UINT64 _s = (s); \
-                                            AcpiUtValueExit (ACPI_DEBUG_PARAMETERS, _s); \
-                                            return (_s); })
-#define return_UINT8(s)                 ACPI_DO_WHILE0 ({ \
-                                            register UINT8 _s = (UINT8) (s); \
-                                            AcpiUtValueExit (ACPI_DEBUG_PARAMETERS, (UINT64) _s); \
-                                            return (_s); })
-#define return_UINT32(s)                ACPI_DO_WHILE0 ({ \
-                                            register UINT32 _s = (UINT32) (s); \
-                                            AcpiUtValueExit (ACPI_DEBUG_PARAMETERS, (UINT64) _s); \
-                                            return (_s); })
-#else /* Use original less-safe macros */
-
-#define return_ACPI_STATUS(s)           ACPI_DO_WHILE0 ({ \
-                                            AcpiUtStatusExit (ACPI_DEBUG_PARAMETERS, (s)); \
-                                            return((s)); })
-#define return_PTR(s)                   ACPI_DO_WHILE0 ({ \
-                                            AcpiUtPtrExit (ACPI_DEBUG_PARAMETERS, (UINT8 *) (s)); \
-                                            return((s)); })
-#define return_VALUE(s)                 ACPI_DO_WHILE0 ({ \
-                                            AcpiUtValueExit (ACPI_DEBUG_PARAMETERS, (UINT64) (s)); \
-                                            return((s)); })
-#define return_UINT8(s)                 return_VALUE(s)
-#define return_UINT32(s)                return_VALUE(s)
-
-#endif /* ACPI_SIMPLE_RETURN_MACROS */
-
-/* Conditional execution */
-
-#define ACPI_DEBUG_EXEC(a)              a
-#define ACPI_DEBUG_ONLY_MEMBERS(a)      a;
-#define _VERBOSE_STRUCTURES
-
-
-/* Various object display routines for debug */
-
-#define ACPI_DUMP_STACK_ENTRY(a)        AcpiExDumpOperand((a), 0)
-#define ACPI_DUMP_OPERANDS(a, b ,c)     AcpiExDumpOperands(a, b, c)
-#define ACPI_DUMP_ENTRY(a, b)           AcpiNsDumpEntry (a, b)
-#define ACPI_DUMP_PATHNAME(a, b, c, d)  AcpiNsDumpPathname(a, b, c, d)
-#define ACPI_DUMP_BUFFER(a, b)          AcpiUtDebugDumpBuffer((UINT8 *) a, b, DB_BYTE_DISPLAY, _COMPONENT)
-
-#else
-/*
- * This is the non-debug case -- make everything go away,
- * leaving no executable debug code!
- */
-#define ACPI_DEBUG_EXEC(a)
-#define ACPI_DEBUG_ONLY_MEMBERS(a)
-#define ACPI_FUNCTION_TRACE(a)
-#define ACPI_FUNCTION_TRACE_PTR(a, b)
-#define ACPI_FUNCTION_TRACE_U32(a, b)
-#define ACPI_FUNCTION_TRACE_STR(a, b)
-#define ACPI_FUNCTION_EXIT
-#define ACPI_FUNCTION_STATUS_EXIT(s)
-#define ACPI_FUNCTION_VALUE_EXIT(s)
-#define ACPI_FUNCTION_ENTRY()
-#define ACPI_DUMP_STACK_ENTRY(a)
-#define ACPI_DUMP_OPERANDS(a, b, c)
-#define ACPI_DUMP_ENTRY(a, b)
-#define ACPI_DUMP_TABLES(a, b)
-#define ACPI_DUMP_PATHNAME(a, b, c, d)
-#define ACPI_DUMP_BUFFER(a, b)
-#define ACPI_DEBUG_PRINT(pl)
-#define ACPI_DEBUG_PRINT_RAW(pl)
-
-#define return_VOID                     return
-#define return_ACPI_STATUS(s)           return(s)
-#define return_VALUE(s)                 return(s)
-#define return_UINT8(s)                 return(s)
-#define return_UINT32(s)                return(s)
-#define return_PTR(s)                   return(s)
-
-#endif /* ACPI_DEBUG_OUTPUT */
-
-
 #if (!ACPI_REDUCED_HARDWARE)
 #define ACPI_HW_OPTIONAL_FUNCTION(addr)     addr
 #else
diff --git a/sys/contrib/dev/acpica/include/acnames.h b/sys/contrib/dev/acpica/include/acnames.h
index 1987b16..d6fdaa6 100644
--- a/sys/contrib/dev/acpica/include/acnames.h
+++ b/sys/contrib/dev/acpica/include/acnames.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/acnamesp.h b/sys/contrib/dev/acpica/include/acnamesp.h
index 4a91108..7a92143 100644
--- a/sys/contrib/dev/acpica/include/acnamesp.h
+++ b/sys/contrib/dev/acpica/include/acnamesp.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -277,6 +277,22 @@ AcpiNsCheckParameterCount (
     UINT32                      UserParamCount,
     const ACPI_PREDEFINED_INFO  *Info);
 
+ACPI_STATUS
+AcpiNsCheckObjectType (
+    ACPI_PREDEFINED_DATA        *Data,
+    ACPI_OPERAND_OBJECT         **ReturnObjectPtr,
+    UINT32                      ExpectedBtypes,
+    UINT32                      PackageIndex);
+
+
+/*
+ * nsprepkg - Validation of predefined name packages
+ */
+ACPI_STATUS
+AcpiNsCheckPackage (
+    ACPI_PREDEFINED_DATA        *Data,
+    ACPI_OPERAND_OBJECT         **ReturnObjectPtr);
+
 
 /*
  * nsnames - Name and Scope manipulation
@@ -430,10 +446,6 @@ AcpiNsInstallNode (
 /*
  * nsutils - Utility functions
  */
-BOOLEAN
-AcpiNsValidRootPrefix (
-    char                    Prefix);
-
 ACPI_OBJECT_TYPE
 AcpiNsGetType (
     ACPI_NAMESPACE_NODE     *Node);
diff --git a/sys/contrib/dev/acpica/include/acobject.h b/sys/contrib/dev/acpica/include/acobject.h
index 2f17375..e755daa 100644
--- a/sys/contrib/dev/acpica/include/acobject.h
+++ b/sys/contrib/dev/acpica/include/acobject.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/acopcode.h b/sys/contrib/dev/acpica/include/acopcode.h
index edd1349..ba37842 100644
--- a/sys/contrib/dev/acpica/include/acopcode.h
+++ b/sys/contrib/dev/acpica/include/acopcode.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/acoutput.h b/sys/contrib/dev/acpica/include/acoutput.h
index 193d0ac..4b5ca9c 100644
--- a/sys/contrib/dev/acpica/include/acoutput.h
+++ b/sys/contrib/dev/acpica/include/acoutput.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -267,18 +267,165 @@
  * Common parameters used for debug output functions:
  * line number, function name, module(file) name, component ID
  */
-#define ACPI_DEBUG_PARAMETERS           __LINE__, ACPI_GET_FUNCTION_NAME, _AcpiModuleName, _COMPONENT
+#define ACPI_DEBUG_PARAMETERS \
+    __LINE__, ACPI_GET_FUNCTION_NAME, _AcpiModuleName, _COMPONENT
+
+/* Check if debug output is currently dynamically enabled */
+
+#define ACPI_IS_DEBUG_ENABLED(Level, Component) \
+    ((Level & AcpiDbgLevel) && (Component & AcpiDbgLayer))
 
 /*
  * Master debug print macros
  * Print message if and only if:
  *    1) Debug print for the current component is enabled
  *    2) Debug error level or trace level for the print statement is enabled
+ *
+ * November 2012: Moved the runtime check for whether to actually emit the
+ * debug message outside of the print function itself. This improves overall
+ * performance at a relatively small code cost. Implementation involves the
+ * use of variadic macros supported by C99.
+ *
+ * Note: the ACPI_DO_WHILE0 macro is used to prevent some compilers from
+ * complaining about these constructs. On other compilers the do...while
+ * adds some extra code, so this feature is optional.
  */
-#define ACPI_DEBUG_PRINT(plist)         AcpiDebugPrint plist
-#define ACPI_DEBUG_PRINT_RAW(plist)     AcpiDebugPrintRaw plist
-
+#ifdef ACPI_USE_DO_WHILE_0
+#define ACPI_DO_WHILE0(a)               do a while(0)
 #else
+#define ACPI_DO_WHILE0(a)               a
+#endif
+
+/* DEBUG_PRINT functions */
+
+#define ACPI_DEBUG_PRINT(plist)         ACPI_ACTUAL_DEBUG plist
+#define ACPI_DEBUG_PRINT_RAW(plist)     ACPI_ACTUAL_DEBUG_RAW plist
+
+/* Helper macros for DEBUG_PRINT */
+
+#define ACPI_DO_DEBUG_PRINT(Function, Level, Line, Filename, Modulename, Component, ...) \
+    ACPI_DO_WHILE0 ({ \
+        if (ACPI_IS_DEBUG_ENABLED (Level, Component)) \
+        { \
+            Function (Level, Line, Filename, Modulename, Component, __VA_ARGS__); \
+        } \
+    })
+
+#define ACPI_ACTUAL_DEBUG(Level, Line, Filename, Modulename, Component, ...) \
+    ACPI_DO_DEBUG_PRINT (AcpiDebugPrint, Level, Line, \
+        Filename, Modulename, Component, __VA_ARGS__)
+
+#define ACPI_ACTUAL_DEBUG_RAW(Level, Line, Filename, Modulename, Component, ...) \
+    ACPI_DO_DEBUG_PRINT (AcpiDebugPrintRaw, Level, Line, \
+        Filename, Modulename, Component, __VA_ARGS__)
+
+
+/*
+ * Function entry tracing
+ *
+ * The name of the function is emitted as a local variable that is
+ * intended to be used by both the entry trace and the exit trace.
+ */
+
+/* Helper macro */
+
+#define ACPI_TRACE_ENTRY(Name, Function, Type, Param) \
+    ACPI_FUNCTION_NAME (Name) \
+    Function (ACPI_DEBUG_PARAMETERS, (Type) (Param))
+
+/* The actual entry trace macros */
+
+#define ACPI_FUNCTION_TRACE(Name) \
+    ACPI_FUNCTION_NAME(Name) \
+    AcpiUtTrace (ACPI_DEBUG_PARAMETERS)
+
+#define ACPI_FUNCTION_TRACE_PTR(Name, Pointer) \
+    ACPI_TRACE_ENTRY (Name, AcpiUtTracePtr, void *, Pointer)
+
+#define ACPI_FUNCTION_TRACE_U32(Name, Value) \
+    ACPI_TRACE_ENTRY (Name, AcpiUtTraceU32, UINT32, Value)
+
+#define ACPI_FUNCTION_TRACE_STR(Name, String) \
+    ACPI_TRACE_ENTRY (Name, AcpiUtTraceStr, char *, String)
+
+#define ACPI_FUNCTION_ENTRY() \
+    AcpiUtTrackStackPtr()
+
+
+/*
+ * Function exit tracing
+ *
+ * These macros include a return statement. This is usually considered
+ * bad form, but having a separate exit macro before the actual return
+ * is very ugly and difficult to maintain.
+ *
+ * One of the FUNCTION_TRACE macros above must be used in conjunction
+ * with these macros so that "_AcpiFunctionName" is defined.
+ *
+ * There are two versions of most of the return macros. The default version is
+ * safer, since it avoids side-effects by guaranteeing that the argument will
+ * not be evaluated twice.
+ *
+ * A less-safe version of the macros is provided for optional use if the
+ * compiler uses excessive CPU stack (for example, this may happen in the
+ * debug case if code optimzation is disabled.)
+ */
+
+/* Exit trace helper macro */
+
+#ifndef ACPI_SIMPLE_RETURN_MACROS
+
+#define ACPI_TRACE_EXIT(Function, Type, Param) \
+    ACPI_DO_WHILE0 ({ \
+        register Type _Param = (Type) (Param); \
+        Function (ACPI_DEBUG_PARAMETERS, _Param); \
+        return (_Param); \
+    })
+
+#else /* Use original less-safe macros */
+
+#define ACPI_TRACE_EXIT(Function, Type, Param) \
+    ACPI_DO_WHILE0 ({ \
+        Function (ACPI_DEBUG_PARAMETERS, (Type) (Param)); \
+        return (Param); \
+    })
+
+#endif /* ACPI_SIMPLE_RETURN_MACROS */
+
+/* The actual exit macros */
+
+#define return_VOID \
+    ACPI_DO_WHILE0 ({ \
+        AcpiUtExit (ACPI_DEBUG_PARAMETERS); \
+        return; \
+    })
+
+#define return_ACPI_STATUS(Status) \
+    ACPI_TRACE_EXIT (AcpiUtStatusExit, ACPI_STATUS, Status)
+
+#define return_PTR(Pointer) \
+    ACPI_TRACE_EXIT (AcpiUtPtrExit, void *, Pointer)
+
+#define return_VALUE(Value) \
+    ACPI_TRACE_EXIT (AcpiUtValueExit, UINT64, Value)
+
+
+/* Conditional execution */
+
+#define ACPI_DEBUG_EXEC(a)              a
+#define ACPI_DEBUG_ONLY_MEMBERS(a)      a;
+#define _VERBOSE_STRUCTURES
+
+
+/* Various object display routines for debug */
+
+#define ACPI_DUMP_STACK_ENTRY(a)        AcpiExDumpOperand((a), 0)
+#define ACPI_DUMP_OPERANDS(a, b ,c)     AcpiExDumpOperands(a, b, c)
+#define ACPI_DUMP_ENTRY(a, b)           AcpiNsDumpEntry (a, b)
+#define ACPI_DUMP_PATHNAME(a, b, c, d)  AcpiNsDumpPathname(a, b, c, d)
+#define ACPI_DUMP_BUFFER(a, b)          AcpiUtDebugDumpBuffer((UINT8 *) a, b, DB_BYTE_DISPLAY, _COMPONENT)
+
+#else /* ACPI_DEBUG_OUTPUT */
 /*
  * This is the non-debug case -- make everything go away,
  * leaving no executable debug code!
@@ -286,6 +433,32 @@
 #define ACPI_FUNCTION_NAME(a)
 #define ACPI_DEBUG_PRINT(pl)
 #define ACPI_DEBUG_PRINT_RAW(pl)
+#define ACPI_DEBUG_EXEC(a)
+#define ACPI_DEBUG_ONLY_MEMBERS(a)
+#define ACPI_FUNCTION_TRACE(a)
+#define ACPI_FUNCTION_TRACE_PTR(a, b)
+#define ACPI_FUNCTION_TRACE_U32(a, b)
+#define ACPI_FUNCTION_TRACE_STR(a, b)
+#define ACPI_FUNCTION_EXIT
+#define ACPI_FUNCTION_STATUS_EXIT(s)
+#define ACPI_FUNCTION_VALUE_EXIT(s)
+#define ACPI_FUNCTION_ENTRY()
+#define ACPI_DUMP_STACK_ENTRY(a)
+#define ACPI_DUMP_OPERANDS(a, b, c)
+#define ACPI_DUMP_ENTRY(a, b)
+#define ACPI_DUMP_TABLES(a, b)
+#define ACPI_DUMP_PATHNAME(a, b, c, d)
+#define ACPI_DUMP_BUFFER(a, b)
+#define ACPI_DEBUG_PRINT(pl)
+#define ACPI_DEBUG_PRINT_RAW(pl)
+#define ACPI_IS_DEBUG_ENABLED(Level, Component) 0
+
+/* Return macros must have a return statement at the minimum */
+
+#define return_VOID                     return
+#define return_ACPI_STATUS(s)           return(s)
+#define return_VALUE(s)                 return(s)
+#define return_PTR(s)                   return(s)
 
 #endif /* ACPI_DEBUG_OUTPUT */
 
diff --git a/sys/contrib/dev/acpica/include/acparser.h b/sys/contrib/dev/acpica/include/acparser.h
index f46d3cb..7f236d8 100644
--- a/sys/contrib/dev/acpica/include/acparser.h
+++ b/sys/contrib/dev/acpica/include/acparser.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -129,7 +129,36 @@ AcpiPsGetParent (
 
 
 /*
- * psopcode - AML Opcode information
+ * psobject - support for parse object processing
+ */
+ACPI_STATUS
+AcpiPsBuildNamedOp (
+    ACPI_WALK_STATE         *WalkState,
+    UINT8                   *AmlOpStart,
+    ACPI_PARSE_OBJECT       *UnnamedOp,
+    ACPI_PARSE_OBJECT       **Op);
+
+ACPI_STATUS
+AcpiPsCreateOp (
+    ACPI_WALK_STATE         *WalkState,
+    UINT8                   *AmlOpStart,
+    ACPI_PARSE_OBJECT       **NewOp);
+
+ACPI_STATUS
+AcpiPsCompleteOp (
+    ACPI_WALK_STATE         *WalkState,
+    ACPI_PARSE_OBJECT       **Op,
+    ACPI_STATUS             Status);
+
+ACPI_STATUS
+AcpiPsCompleteFinalOp (
+    ACPI_WALK_STATE         *WalkState,
+    ACPI_PARSE_OBJECT       *Op,
+    ACPI_STATUS             Status);
+
+
+/*
+ * psopinfo - AML Opcode information
  */
 const ACPI_OPCODE_INFO *
 AcpiPsGetOpcodeInfo (
@@ -294,10 +323,6 @@ BOOLEAN
 AcpiPsIsLeadingChar (
     UINT32                  c);
 
-BOOLEAN
-AcpiPsIsPrefixChar (
-    UINT32                  c);
-
 UINT32
 AcpiPsGetName(
     ACPI_PARSE_OBJECT       *op);
diff --git a/sys/contrib/dev/acpica/include/acpi.h b/sys/contrib/dev/acpica/include/acpi.h
index ad0f17b..5450ad4 100644
--- a/sys/contrib/dev/acpica/include/acpi.h
+++ b/sys/contrib/dev/acpica/include/acpi.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/acpiosxf.h b/sys/contrib/dev/acpica/include/acpiosxf.h
index 462d030..ae9cf3c 100644
--- a/sys/contrib/dev/acpica/include/acpiosxf.h
+++ b/sys/contrib/dev/acpica/include/acpiosxf.h
@@ -8,7 +8,7 @@
 
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/acpixf.h b/sys/contrib/dev/acpica/include/acpixf.h
index 086e2d3..1cd6cb9 100644
--- a/sys/contrib/dev/acpica/include/acpixf.h
+++ b/sys/contrib/dev/acpica/include/acpixf.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -47,7 +47,7 @@
 
 /* Current ACPICA subsystem version in YYYYMMDD format */
 
-#define ACPI_CA_VERSION                 0x20121114
+#define ACPI_CA_VERSION                 0x20130117
 
 #include <contrib/dev/acpica/include/acconfig.h>
 #include <contrib/dev/acpica/include/actypes.h>
@@ -618,6 +618,12 @@ AcpiGetEventResources (
     ACPI_BUFFER             *RetBuffer);
 
 ACPI_STATUS
+AcpiWalkResourceBuffer (
+    ACPI_BUFFER                 *Buffer,
+    ACPI_WALK_RESOURCE_CALLBACK UserFunction,
+    void                        *Context);
+
+ACPI_STATUS
 AcpiWalkResources (
     ACPI_HANDLE                 Device,
     char                        *Name,
diff --git a/sys/contrib/dev/acpica/include/acpredef.h b/sys/contrib/dev/acpica/include/acpredef.h
index c5b115e..771e344 100644
--- a/sys/contrib/dev/acpica/include/acpredef.h
+++ b/sys/contrib/dev/acpica/include/acpredef.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -345,8 +345,8 @@ static const ACPI_PREDEFINED_INFO     PredefinedNames[] =
     {{"_MBM", 0, ACPI_RTYPE_PACKAGE}}, /* Fixed-length (8 Int) */
                     {{{ACPI_PTYPE1_FIXED, ACPI_RTYPE_INTEGER, 8,0}, 0,0}},
 
-    {{"_MLS", 0, ACPI_RTYPE_PACKAGE}}, /* Variable-length (Pkgs) each (2 Str) */
-                    {{{ACPI_PTYPE2, ACPI_RTYPE_STRING, 2,0}, 0,0}},
+    {{"_MLS", 0, ACPI_RTYPE_PACKAGE}}, /* Variable-length (Pkgs) each (1 Str/1 Buf) */
+                    {{{ACPI_PTYPE2, ACPI_RTYPE_STRING, 1, ACPI_RTYPE_BUFFER}, 1,0}},
 
     {{"_MSG", 1, 0}},
     {{"_MSM", 4, ACPI_RTYPE_INTEGER}},
diff --git a/sys/contrib/dev/acpica/include/acresrc.h b/sys/contrib/dev/acpica/include/acresrc.h
index e9ae052..b894e2f 100644
--- a/sys/contrib/dev/acpica/include/acresrc.h
+++ b/sys/contrib/dev/acpica/include/acresrc.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -265,7 +265,7 @@ AcpiRsConvertAmlToResources (
     UINT32                  Length,
     UINT32                  Offset,
     UINT8                   ResourceIndex,
-    void                    *Context);
+    void                    **Context);
 
 ACPI_STATUS
 AcpiRsConvertResourcesToAml (
@@ -405,18 +405,21 @@ extern ACPI_RSDUMP_INFO         *AcpiGbl_DumpResourceDispatch[];
 extern ACPI_RSDUMP_INFO         *AcpiGbl_DumpSerialBusDispatch[];
 
 /*
- * rsdump
+ * rsdumpinfo
  */
 extern ACPI_RSDUMP_INFO         AcpiRsDumpIrq[];
+extern ACPI_RSDUMP_INFO         AcpiRsDumpPrt[];
 extern ACPI_RSDUMP_INFO         AcpiRsDumpDma[];
 extern ACPI_RSDUMP_INFO         AcpiRsDumpStartDpf[];
 extern ACPI_RSDUMP_INFO         AcpiRsDumpEndDpf[];
 extern ACPI_RSDUMP_INFO         AcpiRsDumpIo[];
+extern ACPI_RSDUMP_INFO         AcpiRsDumpIoFlags[];
 extern ACPI_RSDUMP_INFO         AcpiRsDumpFixedIo[];
 extern ACPI_RSDUMP_INFO         AcpiRsDumpVendor[];
 extern ACPI_RSDUMP_INFO         AcpiRsDumpEndTag[];
 extern ACPI_RSDUMP_INFO         AcpiRsDumpMemory24[];
 extern ACPI_RSDUMP_INFO         AcpiRsDumpMemory32[];
+extern ACPI_RSDUMP_INFO         AcpiRsDumpMemoryFlags[];
 extern ACPI_RSDUMP_INFO         AcpiRsDumpFixedMemory32[];
 extern ACPI_RSDUMP_INFO         AcpiRsDumpAddress16[];
 extern ACPI_RSDUMP_INFO         AcpiRsDumpAddress32[];
@@ -430,6 +433,7 @@ extern ACPI_RSDUMP_INFO         AcpiRsDumpCommonSerialBus[];
 extern ACPI_RSDUMP_INFO         AcpiRsDumpI2cSerialBus[];
 extern ACPI_RSDUMP_INFO         AcpiRsDumpSpiSerialBus[];
 extern ACPI_RSDUMP_INFO         AcpiRsDumpUartSerialBus[];
+extern ACPI_RSDUMP_INFO         AcpiRsDumpGeneralFlags[];
 #endif
 
 #endif  /* __ACRESRC_H__ */
diff --git a/sys/contrib/dev/acpica/include/acrestyp.h b/sys/contrib/dev/acpica/include/acrestyp.h
index 07a6076..9d69825 100644
--- a/sys/contrib/dev/acpica/include/acrestyp.h
+++ b/sys/contrib/dev/acpica/include/acrestyp.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -103,8 +103,11 @@ typedef UINT32                          ACPI_RSDESC_SIZE;  /* Max Resource Descr
 
 #define ACPI_EXCLUSIVE                  (UINT8) 0x00
 #define ACPI_SHARED                     (UINT8) 0x01
-#define ACPI_EXCLUSIVE_AND_WAKE         (UINT8) 0x02
-#define ACPI_SHARED_AND_WAKE            (UINT8) 0x03
+
+/* Wake */
+
+#define ACPI_NOT_WAKE_CAPABLE           (UINT8) 0x00
+#define ACPI_WAKE_CAPABLE               (UINT8) 0x01
 
 /*
  * DMA Attributes
@@ -177,6 +180,7 @@ typedef struct acpi_resource_irq
     UINT8                           Triggering;
     UINT8                           Polarity;
     UINT8                           Sharable;
+    UINT8                           WakeCapable;
     UINT8                           InterruptCount;
     UINT8                           Interrupts[1];
 
@@ -402,6 +406,7 @@ typedef struct acpi_resource_extended_irq
     UINT8                           Triggering;
     UINT8                           Polarity;
     UINT8                           Sharable;
+    UINT8                           WakeCapable;
     UINT8                           InterruptCount;
     ACPI_RESOURCE_SOURCE            ResourceSource;
     UINT32                          Interrupts[1];
@@ -425,6 +430,7 @@ typedef struct acpi_resource_gpio
     UINT8                           ProducerConsumer;   /* For values, see Producer/Consumer above */
     UINT8                           PinConfig;
     UINT8                           Sharable;           /* For values, see Interrupt Attributes above */
+    UINT8                           WakeCapable;        /* For values, see Interrupt Attributes above */
     UINT8                           IoRestriction;
     UINT8                           Triggering;         /* For values, see Interrupt Attributes above */
     UINT8                           Polarity;           /* For values, see Interrupt Attributes above */
diff --git a/sys/contrib/dev/acpica/include/acstruct.h b/sys/contrib/dev/acpica/include/acstruct.h
index ff01e86..8923d81 100644
--- a/sys/contrib/dev/acpica/include/acstruct.h
+++ b/sys/contrib/dev/acpica/include/acstruct.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/actables.h b/sys/contrib/dev/acpica/include/actables.h
index 26a7947d..b16c6e3 100644
--- a/sys/contrib/dev/acpica/include/actables.h
+++ b/sys/contrib/dev/acpica/include/actables.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/actbl.h b/sys/contrib/dev/acpica/include/actbl.h
index 90264dd..d77d6c4 100644
--- a/sys/contrib/dev/acpica/include/actbl.h
+++ b/sys/contrib/dev/acpica/include/actbl.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/actbl1.h b/sys/contrib/dev/acpica/include/actbl1.h
index d7cc59b..67579da 100644
--- a/sys/contrib/dev/acpica/include/actbl1.h
+++ b/sys/contrib/dev/acpica/include/actbl1.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/actbl2.h b/sys/contrib/dev/acpica/include/actbl2.h
index 2d2d5ca..49b9578 100644
--- a/sys/contrib/dev/acpica/include/actbl2.h
+++ b/sys/contrib/dev/acpica/include/actbl2.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -285,6 +285,7 @@ typedef struct acpi_table_csrt
 
 } ACPI_TABLE_CSRT;
 
+
 /* Resource Group subtable */
 
 typedef struct acpi_csrt_group
@@ -296,12 +297,33 @@ typedef struct acpi_csrt_group
     UINT16                  SubdeviceId;
     UINT16                  Revision;
     UINT16                  Reserved;
-    UINT32                  InfoLength;
+    UINT32                  SharedInfoLength;
 
-    /* Shared data (length = InfoLength) immediately follows */
+    /* Shared data immediately follows (Length = SharedInfoLength) */
 
 } ACPI_CSRT_GROUP;
 
+/* Shared Info subtable */
+
+typedef struct acpi_csrt_shared_info
+{
+    UINT16                  MajorVersion;
+    UINT16                  MinorVersion;
+    UINT32                  MmioBaseLow;
+    UINT32                  MmioBaseHigh;
+    UINT32                  GsiInterrupt;
+    UINT8                   InterruptPolarity;
+    UINT8                   InterruptMode;
+    UINT8                   NumChannels;
+    UINT8                   DmaAddressWidth;
+    UINT16                  BaseRequestLine;
+    UINT16                  NumHandshakeSignals;
+    UINT32                  MaxBlockSize;
+
+    /* Resource descriptors immediately follow (Length = Group Length - SharedInfoLength) */
+
+} ACPI_CSRT_SHARED_INFO;
+
 /* Resource Descriptor subtable */
 
 typedef struct acpi_csrt_descriptor
diff --git a/sys/contrib/dev/acpica/include/actbl3.h b/sys/contrib/dev/acpica/include/actbl3.h
index 874e348..46c3c6d 100644
--- a/sys/contrib/dev/acpica/include/actbl3.h
+++ b/sys/contrib/dev/acpica/include/actbl3.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -70,13 +70,13 @@
 #define ACPI_SIG_PCCT           "PCCT"      /* Platform Communications Channel Table */
 #define ACPI_SIG_PMTT           "PMTT"      /* Platform Memory Topology Table */
 #define ACPI_SIG_RASF           "RASF"      /* RAS Feature table */
+#define ACPI_SIG_TPM2           "TPM2"      /* Trusted Platform Module 2.0 H/W interface table */
 
 #define ACPI_SIG_S3PT           "S3PT"      /* S3 Performance (sub)Table */
 #define ACPI_SIG_PCCS           "PCC"       /* PCC Shared Memory Region */
 
 /* Reserved table signatures */
 
-#define ACPI_SIG_CSRT           "CSRT"      /* Core System Resources Table */
 #define ACPI_SIG_MATR           "MATR"      /* Memory Address Translation Table */
 #define ACPI_SIG_MSDM           "MSDM"      /* Microsoft Data Management Table */
 #define ACPI_SIG_WPBT           "WPBT"      /* Windows Platform Binary Table */
@@ -648,6 +648,41 @@ enum AcpiRasfStatus
 #define ACPI_RASF_STATUS                (0x1F<<3)
 
 
+/*******************************************************************************
+ *
+ * TPM2 - Trusted Platform Module (TPM) 2.0 Hardware Interface Table
+ *        Version 3
+ *
+ * Conforms to "TPM 2.0 Hardware Interface Table (TPM2)" 29 November 2011
+ *
+ ******************************************************************************/
+
+typedef struct acpi_table_tpm2
+{
+    ACPI_TABLE_HEADER       Header;             /* Common ACPI table header */
+    UINT32                  Flags;
+    UINT64                  ControlAddress;
+    UINT32                  StartMethod;
+
+} ACPI_TABLE_TPM2;
+
+/* Control area structure (not part of table, pointed to by ControlAddress) */
+
+typedef struct acpi_tpm2_control
+{
+    UINT32                  Reserved;
+    UINT32                  Error;
+    UINT32                  Cancel;
+    UINT32                  Start;
+    UINT64                  InterruptControl;
+    UINT32                  CommandSize;
+    UINT64                  CommandAddress;
+    UINT32                  ResponseSize;
+    UINT64                  ResponseAddress;
+
+} ACPI_TPM2_CONTROL;
+
+
 /* Reset to default packing */
 
 #pragma pack()
diff --git a/sys/contrib/dev/acpica/include/actypes.h b/sys/contrib/dev/acpica/include/actypes.h
index 611c418..9c8ec7d 100644
--- a/sys/contrib/dev/acpica/include/actypes.h
+++ b/sys/contrib/dev/acpica/include/actypes.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -295,7 +295,7 @@ typedef UINT32                          ACPI_PHYSICAL_ADDRESS;
 /*
  * Some compilers complain about unused variables. Sometimes we don't want to
  * use all the variables (for example, _AcpiModuleName). This allows us
- * to to tell the compiler in a per-variable manner that a variable
+ * to tell the compiler in a per-variable manner that a variable
  * is unused
  */
 #ifndef ACPI_UNUSED_VAR
@@ -346,7 +346,7 @@ typedef UINT32                          ACPI_PHYSICAL_ADDRESS;
 
 /* PM Timer ticks per second (HZ) */
 
-#define PM_TIMER_FREQUENCY              3579545
+#define ACPI_PM_TIMER_FREQUENCY         3579545
 
 
 /*******************************************************************************
@@ -381,6 +381,22 @@ typedef char *                          ACPI_STRING;    /* Null terminated ASCII
 typedef void *                          ACPI_HANDLE;    /* Actually a ptr to a NS Node */
 
 
+/* Time constants for timer calculations */
+
+#define ACPI_MSEC_PER_SEC               1000L
+
+#define ACPI_USEC_PER_MSEC              1000L
+#define ACPI_USEC_PER_SEC               1000000L
+
+#define ACPI_100NSEC_PER_USEC           10L
+#define ACPI_100NSEC_PER_MSEC           10000L
+#define ACPI_100NSEC_PER_SEC            10000000L
+
+#define ACPI_NSEC_PER_USEC              1000L
+#define ACPI_NSEC_PER_MSEC              1000000L
+#define ACPI_NSEC_PER_SEC               1000000000L
+
+
 /* Owner IDs are used to track namespace nodes for selective deletion */
 
 typedef UINT8                           ACPI_OWNER_ID;
@@ -596,7 +612,7 @@ typedef UINT32                          ACPI_OBJECT_TYPE;
 
 /*
  * These are special object types that never appear in
- * a Namespace node, only in an ACPI_OPERAND_OBJECT
+ * a Namespace node, only in an object of ACPI_OPERAND_OBJECT
  */
 #define ACPI_TYPE_LOCAL_EXTRA           0x1C
 #define ACPI_TYPE_LOCAL_DATA            0x1D
@@ -904,6 +920,10 @@ typedef struct acpi_buffer
 
 } ACPI_BUFFER;
 
+/* Free a buffer created in an ACPI_BUFFER via ACPI_ALLOCATE_LOCAL_BUFFER */
+
+#define ACPI_FREE_BUFFER(b)         ACPI_FREE(b.Pointer)
+
 
 /*
  * NameType for AcpiGetName
@@ -1206,7 +1226,6 @@ typedef struct acpi_memory_list
     UINT16                          ObjectSize;
     UINT16                          MaxDepth;
     UINT16                          CurrentDepth;
-    UINT16                          LinkOffset;
 
 #ifdef ACPI_DBG_TRACK_ALLOCATIONS
 
diff --git a/sys/contrib/dev/acpica/include/acutils.h b/sys/contrib/dev/acpica/include/acutils.h
index f663a1e..e75e248 100644
--- a/sys/contrib/dev/acpica/include/acutils.h
+++ b/sys/contrib/dev/acpica/include/acutils.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -101,7 +101,7 @@ ACPI_STATUS (*ACPI_WALK_AML_CALLBACK) (
     UINT32                  Length,
     UINT32                  Offset,
     UINT8                   ResourceIndex,
-    void                    *Context);
+    void                    **Context);
 
 typedef
 ACPI_STATUS (*ACPI_PKG_CALLBACK) (
@@ -709,10 +709,6 @@ AcpiUtShortDivide (
 /*
  * utmisc
  */
-void
-UtConvertBackslashes (
-    char                    *Pathname);
-
 const char *
 AcpiUtValidateException (
     ACPI_STATUS             Status);
@@ -726,56 +722,12 @@ AcpiUtIsAmlTable (
     ACPI_TABLE_HEADER       *Table);
 
 ACPI_STATUS
-AcpiUtAllocateOwnerId (
-    ACPI_OWNER_ID           *OwnerId);
-
-void
-AcpiUtReleaseOwnerId (
-    ACPI_OWNER_ID           *OwnerId);
-
-ACPI_STATUS
 AcpiUtWalkPackageTree (
     ACPI_OPERAND_OBJECT     *SourceObject,
     void                    *TargetObject,
     ACPI_PKG_CALLBACK       WalkCallback,
     void                    *Context);
 
-void
-AcpiUtStrupr (
-    char                    *SrcString);
-
-void
-AcpiUtStrlwr (
-    char                    *SrcString);
-
-int
-AcpiUtStricmp (
-    char                    *String1,
-    char                    *String2);
-
-void
-AcpiUtPrintString (
-    char                    *String,
-    UINT8                   MaxLength);
-
-BOOLEAN
-AcpiUtValidAcpiName (
-    UINT32                  Name);
-
-void
-AcpiUtRepairName (
-    char                    *Name);
-
-BOOLEAN
-AcpiUtValidAcpiChar (
-    char                    Character,
-    UINT32                  Position);
-
-ACPI_STATUS
-AcpiUtStrtoul64 (
-    char                    *String,
-    UINT32                  Base,
-    UINT64                  *RetInteger);
 
 /* Values for Base above (16=Hex, 10=Decimal) */
 
@@ -799,6 +751,18 @@ AcpiUtDisplayInitPathname (
 
 
 /*
+ * utownerid - Support for Table/Method Owner IDs
+ */
+ACPI_STATUS
+AcpiUtAllocateOwnerId (
+    ACPI_OWNER_ID           *OwnerId);
+
+void
+AcpiUtReleaseOwnerId (
+    ACPI_OWNER_ID           *OwnerId);
+
+
+/*
  * utresrc
  */
 ACPI_STATUS
@@ -807,7 +771,7 @@ AcpiUtWalkAmlResources (
     UINT8                   *Aml,
     ACPI_SIZE               AmlLength,
     ACPI_WALK_AML_CALLBACK  UserFunction,
-    void                    *Context);
+    void                    **Context);
 
 ACPI_STATUS
 AcpiUtValidateResource (
@@ -838,6 +802,51 @@ AcpiUtGetResourceEndTag (
 
 
 /*
+ * utstring - String and character utilities
+ */
+void
+AcpiUtStrupr (
+    char                    *SrcString);
+
+void
+AcpiUtStrlwr (
+    char                    *SrcString);
+
+int
+AcpiUtStricmp (
+    char                    *String1,
+    char                    *String2);
+
+ACPI_STATUS
+AcpiUtStrtoul64 (
+    char                    *String,
+    UINT32                  Base,
+    UINT64                  *RetInteger);
+
+void
+AcpiUtPrintString (
+    char                    *String,
+    UINT8                   MaxLength);
+
+void
+UtConvertBackslashes (
+    char                    *Pathname);
+
+BOOLEAN
+AcpiUtValidAcpiName (
+    UINT32                  Name);
+
+BOOLEAN
+AcpiUtValidAcpiChar (
+    char                    Character,
+    UINT32                  Position);
+
+void
+AcpiUtRepairName (
+    char                    *Name);
+
+
+/*
  * utmutex - mutex support
  */
 ACPI_STATUS
diff --git a/sys/contrib/dev/acpica/include/amlcode.h b/sys/contrib/dev/acpica/include/amlcode.h
index f778bb0..66af4c2 100644
--- a/sys/contrib/dev/acpica/include/amlcode.h
+++ b/sys/contrib/dev/acpica/include/amlcode.h
@@ -7,7 +7,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/amlresrc.h b/sys/contrib/dev/acpica/include/amlresrc.h
index 2202bb9..8e68559 100644
--- a/sys/contrib/dev/acpica/include/amlresrc.h
+++ b/sys/contrib/dev/acpica/include/amlresrc.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/platform/acenv.h b/sys/contrib/dev/acpica/include/platform/acenv.h
index 4c23f30..dea4db2 100644
--- a/sys/contrib/dev/acpica/include/platform/acenv.h
+++ b/sys/contrib/dev/acpica/include/platform/acenv.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -300,7 +300,7 @@
  */
 #ifdef ACPI_USE_SYSTEM_CLIBRARY
 
-/* Use the standard C library headers. We want to keep these to a minimum */
+/* Use the standard C library headers. We want to keep these to a minimum. */
 
 #ifdef ACPI_USE_STANDARD_HEADERS
 
diff --git a/sys/contrib/dev/acpica/include/platform/acfreebsd.h b/sys/contrib/dev/acpica/include/platform/acfreebsd.h
index d7aa7ed..6b32ce7 100644
--- a/sys/contrib/dev/acpica/include/platform/acfreebsd.h
+++ b/sys/contrib/dev/acpica/include/platform/acfreebsd.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/sys/contrib/dev/acpica/include/platform/acgcc.h b/sys/contrib/dev/acpica/include/platform/acgcc.h
index d2eafff..1662a9b 100644
--- a/sys/contrib/dev/acpica/include/platform/acgcc.h
+++ b/sys/contrib/dev/acpica/include/platform/acgcc.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -59,7 +59,7 @@
 /*
  * Some compilers complain about unused variables. Sometimes we don't want to
  * use all the variables (for example, _AcpiModuleName). This allows us
- * to to tell the compiler warning in a per-variable manner that a variable
+ * to tell the compiler warning in a per-variable manner that a variable
  * is unused.
  */
 #define ACPI_UNUSED_VAR __attribute__ ((unused))
diff --git a/sys/contrib/dev/acpica/os_specific/service_layers/osunixxf.c b/sys/contrib/dev/acpica/os_specific/service_layers/osunixxf.c
index cf6382b..c18d65c 100644
--- a/sys/contrib/dev/acpica/os_specific/service_layers/osunixxf.c
+++ b/sys/contrib/dev/acpica/os_specific/service_layers/osunixxf.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2012, Intel Corp.
+ * Copyright (C) 2000 - 2013, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -84,6 +84,10 @@ AeTableOverride (
 
 typedef void* (*PTHREAD_CALLBACK) (void *);
 
+/* Buffer used by AcpiOsVprintf */
+
+#define ACPI_VPRINTF_BUFFER_SIZE        512
+
 /* Apple-specific */
 
 #ifdef __APPLE__
@@ -268,7 +272,8 @@ AcpiOsRedirectOutput (
  *
  * RETURN:      None
  *
- * DESCRIPTION: Formatted output
+ * DESCRIPTION: Formatted output. Note: very similar to AcpiOsVprintf
+ *              (performance), changes should be tracked in both functions.
  *
  *****************************************************************************/
 
@@ -278,11 +283,36 @@ AcpiOsPrintf (
     ...)
 {
     va_list                 Args;
+    UINT8                   Flags;
+
+
+    Flags = AcpiGbl_DbOutputFlags;
+    if (Flags & ACPI_DB_REDIRECTABLE_OUTPUT)
+    {
+        /* Output is directable to either a file (if open) or the console */
+
+        if (AcpiGbl_DebugFile)
+        {
+            /* Output file is open, send the output there */
+
+            va_start (Args, Fmt);
+            vfprintf (AcpiGbl_DebugFile, Fmt, Args);
+            va_end (Args);
+        }
+        else
+        {
+            /* No redirection, send output to console (once only!) */
 
+            Flags |= ACPI_DB_CONSOLE_OUTPUT;
+        }
+    }
 
-    va_start (Args, Fmt);
-    AcpiOsVprintf (Fmt, Args);
-    va_end (Args);
+    if (Flags & ACPI_DB_CONSOLE_OUTPUT)
+    {
+        va_start (Args, Fmt);
+        vfprintf (AcpiGbl_OutputFile, Fmt, Args);
+        va_end (Args);
+    }
 }
 
 
@@ -295,7 +325,9 @@ AcpiOsPrintf (
  *
  * RETURN:      None
  *
- * DESCRIPTION: Formatted output with argument list pointer
+ * DESCRIPTION: Formatted output with argument list pointer. Note: very
+ *              similar to AcpiOsPrintf, changes should be tracked in both
+ *              functions.
  *
  *****************************************************************************/
 
@@ -305,7 +337,20 @@ AcpiOsVprintf (
     va_list                 Args)
 {
     UINT8                   Flags;
+    char                    Buffer[ACPI_VPRINTF_BUFFER_SIZE];
+
 
+    /*
+     * We build the output string in a local buffer because we may be
+     * outputting the buffer twice. Using vfprintf is problematic because
+     * some implementations modify the args pointer/structure during
+     * execution. Thus, we use the local buffer for portability.
+     *
+     * Note: Since this module is intended for use by the various ACPICA
+     * utilities/applications, we can safely declare the buffer on the stack.
+     * Also, This function is used for relatively small error messages only.
+     */
+    vsnprintf (Buffer, ACPI_VPRINTF_BUFFER_SIZE, Fmt, Args);
 
     Flags = AcpiGbl_DbOutputFlags;
     if (Flags & ACPI_DB_REDIRECTABLE_OUTPUT)
@@ -316,7 +361,7 @@ AcpiOsVprintf (
         {
             /* Output file is open, send the output there */
 
-            vfprintf (AcpiGbl_DebugFile, Fmt, Args);
+            fputs (Buffer, AcpiGbl_DebugFile);
         }
         else
         {
@@ -328,7 +373,7 @@ AcpiOsVprintf (
 
     if (Flags & ACPI_DB_CONSOLE_OUTPUT)
     {
-        vfprintf (AcpiGbl_OutputFile, Fmt, Args);
+        fputs (Buffer, AcpiGbl_OutputFile);
     }
 }
 
@@ -626,7 +671,7 @@ AcpiOsDeleteSemaphore (
  *
  * PARAMETERS:  Handle              - Handle returned by AcpiOsCreateSemaphore
  *              Units               - How many units to wait for
- *              Timeout             - How long to wait
+ *              MsecTimeout         - How long to wait (milliseconds)
  *
  * RETURN:      Status
  *
@@ -638,11 +683,14 @@ ACPI_STATUS
 AcpiOsWaitSemaphore (
     ACPI_HANDLE         Handle,
     UINT32              Units,
-    UINT16              Timeout)
+    UINT16              MsecTimeout)
 {
     ACPI_STATUS         Status = AE_OK;
     sem_t               *Sem = (sem_t *) Handle;
-    struct timespec     T;
+#ifndef ACPI_USE_ALTERNATE_TIMEOUT
+    struct timespec     Time;
+    int                 RetVal;
+#endif
 
 
     if (!Sem)
@@ -650,7 +698,7 @@ AcpiOsWaitSemaphore (
         return (AE_BAD_PARAMETER);
     }
 
-    switch (Timeout)
+    switch (MsecTimeout)
     {
     /*
      * No Wait:
@@ -677,37 +725,71 @@ AcpiOsWaitSemaphore (
         }
         break;
 
-    /* Wait with Timeout */
+    /* Wait with MsecTimeout */
 
     default:
 
-        T.tv_sec = Timeout / 1000;
-        T.tv_nsec = (Timeout - (T.tv_sec * 1000)) * 1000000;
-
 #ifdef ACPI_USE_ALTERNATE_TIMEOUT
         /*
          * Alternate timeout mechanism for environments where
          * sem_timedwait is not available or does not work properly.
          */
-        while (Timeout)
+        while (MsecTimeout)
         {
             if (sem_trywait (Sem) == 0)
             {
                 /* Got the semaphore */
                 return (AE_OK);
             }
-            usleep (1000);  /* one millisecond */
-            Timeout--;
+
+            if (MsecTimeout >= 10)
+            {
+                MsecTimeout -= 10;
+                usleep (10 * ACPI_USEC_PER_MSEC); /* ten milliseconds */
+            }
+            else
+            {
+                MsecTimeout--;
+                usleep (ACPI_USEC_PER_MSEC); /* one millisecond */
+            }
         }
         Status = (AE_TIME);
 #else
+        /*
+         * The interface to sem_timedwait is an absolute time, so we need to
+         * get the current time, then add in the millisecond Timeout value.
+         */
+        if (clock_gettime (CLOCK_REALTIME, &Time) == -1)
+        {
+            perror ("clock_gettime");
+            return (AE_TIME);
+        }
+
+        Time.tv_sec += (MsecTimeout / ACPI_MSEC_PER_SEC);
+        Time.tv_nsec += ((MsecTimeout % ACPI_MSEC_PER_SEC) * ACPI_NSEC_PER_MSEC);
+
+        /* Handle nanosecond overflow (field must be less than one second) */
+
+        if (Time.tv_nsec >= ACPI_NSEC_PER_SEC)
+        {
+            Time.tv_sec += (Time.tv_nsec / ACPI_NSEC_PER_SEC);
+            Time.tv_nsec = (Time.tv_nsec % ACPI_NSEC_PER_SEC);
+        }
+
+        while (((RetVal = sem_timedwait (Sem, &Time)) == -1) && (errno == EINTR))
+        {
+            continue;
+        }
 
-        if (sem_timedwait (Sem, &T))
+        if (RetVal != 0)
         {
+            if (errno != ETIMEDOUT)
+            {
+                perror ("sem_timedwait");
+            }
             Status = (AE_TIME);
         }
 #endif
-
         break;
     }
 
@@ -884,12 +966,15 @@ AcpiOsSleep (
     UINT64                  milliseconds)
 {
 
-    sleep (milliseconds / 1000);    /* Sleep for whole seconds */
+    /* Sleep for whole seconds */
+
+    sleep (milliseconds / ACPI_MSEC_PER_SEC);
 
     /*
-     * Arg to usleep() must be less than 1,000,000 (1 second)
+     * Sleep for remaining microseconds.
+     * Arg to usleep() is in usecs and must be less than 1,000,000 (1 second).
      */
-    usleep ((milliseconds % 1000) * 1000);      /* Sleep for remaining usecs */
+    usleep ((milliseconds % ACPI_MSEC_PER_SEC) * ACPI_USEC_PER_MSEC);
 }
 
 
@@ -912,11 +997,14 @@ AcpiOsGetTimer (
     struct timeval          time;
 
 
+    /* This timer has sufficient resolution for user-space application code */
+
     gettimeofday (&time, NULL);
 
-    /* Seconds * 10^7 = 100ns(10^-7), Microseconds(10^-6) * 10^1 = 100ns */
+    /* (Seconds * 10^7 = 100ns(10^-7)) + (Microseconds(10^-6) * 10^1 = 100ns) */
 
-    return (((UINT64) time.tv_sec * 10000000) + ((UINT64) time.tv_usec * 10));
+    return (((UINT64) time.tv_sec * ACPI_100NSEC_PER_SEC) +
+            ((UINT64) time.tv_usec * ACPI_100NSEC_PER_USEC));
 }
 
 
diff --git a/sys/contrib/octeon-sdk/cvmx-helper-board.c b/sys/contrib/octeon-sdk/cvmx-helper-board.c
index 8d2b0bd..d97813e 100644
--- a/sys/contrib/octeon-sdk/cvmx-helper-board.c
+++ b/sys/contrib/octeon-sdk/cvmx-helper-board.c
@@ -1394,6 +1394,21 @@ int __cvmx_helper_board_hardware_enable(int interface)
             }
         }
     }
+#if defined(OCTEON_VENDOR_UBIQUITI)
+    else if (cvmx_sysinfo_get()->board_type == CVMX_BOARD_TYPE_CUST_UBIQUITI_E100)
+    {
+	/* Configure ASX cloks for all ports on interface 0.  */
+	if (interface == 0)
+	{
+	    int port;
+
+	    for (port = 0; port < 3; port++) {
+                cvmx_write_csr(CVMX_ASXX_TX_CLK_SETX(port, interface), 16);
+                cvmx_write_csr(CVMX_ASXX_RX_CLK_SETX(port, interface), 0);
+	    }
+	}
+    }
+#endif
     return 0;
 }
 
diff --git a/sys/dev/acpi_support/acpi_asus_wmi.c b/sys/dev/acpi_support/acpi_asus_wmi.c
index 9e2a498..54a2893 100644
--- a/sys/dev/acpi_support/acpi_asus_wmi.c
+++ b/sys/dev/acpi_support/acpi_asus_wmi.c
@@ -304,7 +304,8 @@ static device_method_t acpi_asus_wmi_methods[] = {
 	DEVMETHOD(device_probe, acpi_asus_wmi_probe),
 	DEVMETHOD(device_attach, acpi_asus_wmi_attach),
 	DEVMETHOD(device_detach, acpi_asus_wmi_detach),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t	acpi_asus_wmi_driver = {
diff --git a/sys/dev/acpi_support/acpi_fujitsu.c b/sys/dev/acpi_support/acpi_fujitsu.c
index 2455b50..6f96bc5 100644
--- a/sys/dev/acpi_support/acpi_fujitsu.c
+++ b/sys/dev/acpi_support/acpi_fujitsu.c
@@ -154,7 +154,8 @@ static device_method_t acpi_fujitsu_methods[] = {
 	DEVMETHOD(device_detach,	acpi_fujitsu_detach),
 	DEVMETHOD(device_suspend,	acpi_fujitsu_suspend),
 	DEVMETHOD(device_resume,	acpi_fujitsu_resume),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t acpi_fujitsu_driver = {
diff --git a/sys/dev/acpi_support/acpi_hp.c b/sys/dev/acpi_support/acpi_hp.c
index edc04b7..c190166 100644
--- a/sys/dev/acpi_support/acpi_hp.c
+++ b/sys/dev/acpi_support/acpi_hp.c
@@ -324,7 +324,8 @@ static device_method_t acpi_hp_methods[] = {
 	DEVMETHOD(device_probe, acpi_hp_probe),
 	DEVMETHOD(device_attach, acpi_hp_attach),
 	DEVMETHOD(device_detach, acpi_hp_detach),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t	acpi_hp_driver = {
diff --git a/sys/dev/acpi_support/acpi_ibm.c b/sys/dev/acpi_support/acpi_ibm.c
index 46a985e..4160112 100644
--- a/sys/dev/acpi_support/acpi_ibm.c
+++ b/sys/dev/acpi_support/acpi_ibm.c
@@ -303,7 +303,7 @@ static device_method_t acpi_ibm_methods[] = {
 	DEVMETHOD(device_detach, acpi_ibm_detach),
 	DEVMETHOD(device_resume, acpi_ibm_resume),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t	acpi_ibm_driver = {
diff --git a/sys/dev/acpi_support/acpi_panasonic.c b/sys/dev/acpi_support/acpi_panasonic.c
index ad8b239..2542b0c 100644
--- a/sys/dev/acpi_support/acpi_panasonic.c
+++ b/sys/dev/acpi_support/acpi_panasonic.c
@@ -118,7 +118,7 @@ static device_method_t acpi_panasonic_methods[] = {
 	DEVMETHOD(device_detach,	acpi_panasonic_detach),
 	DEVMETHOD(device_shutdown,	acpi_panasonic_shutdown),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t acpi_panasonic_driver = {
diff --git a/sys/dev/acpi_support/acpi_sony.c b/sys/dev/acpi_support/acpi_sony.c
index 2a0d40e..95e7b4f 100644
--- a/sys/dev/acpi_support/acpi_sony.c
+++ b/sys/dev/acpi_support/acpi_sony.c
@@ -95,7 +95,7 @@ static device_method_t acpi_sony_methods[] = {
 	DEVMETHOD(device_attach, acpi_sony_attach),
 	DEVMETHOD(device_detach, acpi_sony_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t	acpi_sony_driver = {
diff --git a/sys/dev/acpi_support/acpi_toshiba.c b/sys/dev/acpi_support/acpi_toshiba.c
index 343d3ae..9e3dc5b 100644
--- a/sys/dev/acpi_support/acpi_toshiba.c
+++ b/sys/dev/acpi_support/acpi_toshiba.c
@@ -172,7 +172,7 @@ static device_method_t acpi_toshiba_methods[] = {
 	DEVMETHOD(device_attach,	acpi_toshiba_attach),
 	DEVMETHOD(device_detach,	acpi_toshiba_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t acpi_toshiba_driver = {
@@ -190,7 +190,7 @@ static device_method_t acpi_toshiba_video_methods[] = {
 	DEVMETHOD(device_probe,		acpi_toshiba_video_probe),
 	DEVMETHOD(device_attach,	acpi_toshiba_video_attach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t acpi_toshiba_video_driver = {
diff --git a/sys/dev/acpica/Osd/OsdSchedule.c b/sys/dev/acpica/Osd/OsdSchedule.c
index 6d517cb..4afcec5 100644
--- a/sys/dev/acpica/Osd/OsdSchedule.c
+++ b/sys/dev/acpica/Osd/OsdSchedule.c
@@ -181,12 +181,13 @@ ACPI_STATUS
 AcpiOsExecute(ACPI_EXECUTE_TYPE Type, ACPI_OSD_EXEC_CALLBACK Function,
     void *Context)
 {
+    ACPI_STATUS status;
     int pri;
 
     ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
 
     if (Function == NULL)
-	return_ACPI_STATUS (AE_BAD_PARAMETER);
+	return_ACPI_STATUS(AE_BAD_PARAMETER);
 
     switch (Type) {
     case OSL_GPE_HANDLER:
@@ -208,10 +209,11 @@ AcpiOsExecute(ACPI_EXECUTE_TYPE Type, ACPI_OSD_EXEC_CALLBACK Function,
 	pri = 0;
 	break;
     default:
-	return_ACPI_STATUS (AE_BAD_PARAMETER);
+	return_ACPI_STATUS(AE_BAD_PARAMETER);
     }
 
-    return_ACPI_STATUS (acpi_task_enqueue(pri, Function, Context));
+    status = acpi_task_enqueue(pri, Function, Context);
+    return_ACPI_STATUS(status);
 }
 
 void
diff --git a/sys/dev/acpica/acpi.c b/sys/dev/acpica/acpi.c
index e2f8c1c..f0c750f 100644
--- a/sys/dev/acpica/acpi.c
+++ b/sys/dev/acpica/acpi.c
@@ -217,7 +217,7 @@ static device_method_t acpi_methods[] = {
     /* ISA emulation */
     DEVMETHOD(isa_pnp_probe,		acpi_isa_pnp_probe),
 
-    {0, 0}
+    DEVMETHOD_END
 };
 
 static driver_t acpi_driver = {
@@ -613,7 +613,9 @@ acpi_attach(device_t dev)
     /* Probe all supported sleep states. */
     acpi_sleep_states[ACPI_STATE_S0] = TRUE;
     for (state = ACPI_STATE_S1; state < ACPI_S_STATE_COUNT; state++)
-	if (ACPI_SUCCESS(AcpiGetSleepTypeData(state, &TypeA, &TypeB)))
+	if (ACPI_SUCCESS(AcpiEvaluateObject(ACPI_ROOT_OBJECT,
+	    __DECONST(char *, AcpiGbl_SleepStateNames[state]), NULL, NULL)) &&
+	    ACPI_SUCCESS(AcpiGetSleepTypeData(state, &TypeA, &TypeB)))
 	    acpi_sleep_states[state] = TRUE;
 
     /*
@@ -2657,6 +2659,7 @@ acpi_EnterSleepState(struct acpi_softc *sc, int state)
 {
     register_t intr;
     ACPI_STATUS status;
+    ACPI_EVENT_STATUS power_button_status;
     enum acpi_sleep_state slp_state;
     int sleep_result;
 
@@ -2742,7 +2745,44 @@ acpi_EnterSleepState(struct acpi_softc *sc, int state)
     if (state != ACPI_STATE_S1) {
 	sleep_result = acpi_sleep_machdep(sc, state);
 	acpi_wakeup_machdep(sc, state, sleep_result, 0);
+
+	/*
+	 * XXX According to ACPI specification SCI_EN bit should be restored
+	 * by ACPI platform (BIOS, firmware) to its pre-sleep state.
+	 * Unfortunately some BIOSes fail to do that and that leads to
+	 * unexpected and serious consequences during wake up like a system
+	 * getting stuck in SMI handlers.
+	 * This hack is picked up from Linux, which claims that it follows
+	 * Windows behavior.
+	 */
+	if (sleep_result == 1 && state != ACPI_STATE_S4)
+	    AcpiWriteBitRegister(ACPI_BITREG_SCI_ENABLE, ACPI_ENABLE_EVENT);
+
 	AcpiLeaveSleepStatePrep(state);
+
+	if (sleep_result == 1 && state == ACPI_STATE_S3) {
+	    /*
+	     * Prevent mis-interpretation of the wakeup by power button
+	     * as a request for power off.
+	     * Ideally we should post an appropriate wakeup event,
+	     * perhaps using acpi_event_power_button_wake or alike.
+	     *
+	     * Clearing of power button status after wakeup is mandated
+	     * by ACPI specification in section "Fixed Power Button".
+	     *
+	     * XXX As of ACPICA 20121114 AcpiGetEventStatus provides
+	     * status as 0/1 corressponding to inactive/active despite
+	     * its type being ACPI_EVENT_STATUS.  In other words,
+	     * we should not test for ACPI_EVENT_FLAG_SET for time being.
+	     */
+	    if (ACPI_SUCCESS(AcpiGetEventStatus(ACPI_EVENT_POWER_BUTTON,
+		&power_button_status)) && power_button_status != 0) {
+		AcpiClearEvent(ACPI_EVENT_POWER_BUTTON);
+		device_printf(sc->acpi_dev,
+		    "cleared fixed power button status\n");
+	    }
+	}
+
 	intr_restore(intr);
 
 	/* call acpi_wakeup_machdep() again with interrupt enabled */
diff --git a/sys/dev/acpica/acpi_acad.c b/sys/dev/acpica/acpi_acad.c
index 8153abc..4c9dc19 100644
--- a/sys/dev/acpica/acpi_acad.c
+++ b/sys/dev/acpica/acpi_acad.c
@@ -74,7 +74,7 @@ static device_method_t acpi_acad_methods[] = {
     DEVMETHOD(device_probe,	acpi_acad_probe),
     DEVMETHOD(device_attach,	acpi_acad_attach),
 
-    {0, 0}
+    DEVMETHOD_END
 };
 
 static driver_t acpi_acad_driver = {
diff --git a/sys/dev/acpica/acpi_button.c b/sys/dev/acpica/acpi_button.c
index d1f774a..72bf5b1 100644
--- a/sys/dev/acpica/acpi_button.c
+++ b/sys/dev/acpica/acpi_button.c
@@ -79,8 +79,7 @@ static device_method_t acpi_button_methods[] = {
     DEVMETHOD(device_suspend,	acpi_button_suspend),
     DEVMETHOD(device_shutdown,	acpi_button_suspend),
     DEVMETHOD(device_resume,	acpi_button_resume),
-
-    {0, 0}
+    DEVMETHOD_END
 };
 
 static driver_t acpi_button_driver = {
diff --git a/sys/dev/acpica/acpi_cmbat.c b/sys/dev/acpica/acpi_cmbat.c
index ba44da8..be57a32 100644
--- a/sys/dev/acpica/acpi_cmbat.c
+++ b/sys/dev/acpica/acpi_cmbat.c
@@ -99,7 +99,7 @@ static device_method_t acpi_cmbat_methods[] = {
     DEVMETHOD(acpi_batt_get_info, acpi_cmbat_bif),
     DEVMETHOD(acpi_batt_get_status, acpi_cmbat_bst),
 
-    {0, 0}
+    DEVMETHOD_END
 };
 
 static driver_t acpi_cmbat_driver = {
diff --git a/sys/dev/acpica/acpi_dock.c b/sys/dev/acpica/acpi_dock.c
index b184053..d27b4bd 100644
--- a/sys/dev/acpica/acpi_dock.c
+++ b/sys/dev/acpica/acpi_dock.c
@@ -521,7 +521,7 @@ static device_method_t acpi_dock_methods[] = {
 	DEVMETHOD(device_probe, acpi_dock_probe),
 	DEVMETHOD(device_attach, acpi_dock_attach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t	acpi_dock_driver = {
diff --git a/sys/dev/acpica/acpi_ec.c b/sys/dev/acpica/acpi_ec.c
index 00a3073..6e2f0dd 100644
--- a/sys/dev/acpica/acpi_ec.c
+++ b/sys/dev/acpica/acpi_ec.c
@@ -253,7 +253,7 @@ static device_method_t acpi_ec_methods[] = {
     DEVMETHOD(acpi_ec_read,	acpi_ec_read_method),
     DEVMETHOD(acpi_ec_write,	acpi_ec_write_method),
 
-    {0, 0}
+    DEVMETHOD_END
 };
 
 static driver_t acpi_ec_driver = {
diff --git a/sys/dev/acpica/acpi_hpet.c b/sys/dev/acpica/acpi_hpet.c
index 4b0c356..c00bda4 100644
--- a/sys/dev/acpica/acpi_hpet.c
+++ b/sys/dev/acpica/acpi_hpet.c
@@ -849,7 +849,7 @@ static device_method_t hpet_methods[] = {
 	DEVMETHOD(bus_remap_intr, hpet_remap_intr),
 #endif
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t	hpet_driver = {
diff --git a/sys/dev/acpica/acpi_lid.c b/sys/dev/acpica/acpi_lid.c
index d84e3c9..dcdd2e6 100644
--- a/sys/dev/acpica/acpi_lid.c
+++ b/sys/dev/acpica/acpi_lid.c
@@ -69,7 +69,7 @@ static device_method_t acpi_lid_methods[] = {
     DEVMETHOD(device_suspend,	acpi_lid_suspend),
     DEVMETHOD(device_resume,	acpi_lid_resume),
 
-    {0, 0}
+    DEVMETHOD_END
 };
 
 static driver_t acpi_lid_driver = {
diff --git a/sys/dev/acpica/acpi_pci.c b/sys/dev/acpica/acpi_pci.c
index 185731a..39fba88 100644
--- a/sys/dev/acpica/acpi_pci.c
+++ b/sys/dev/acpica/acpi_pci.c
@@ -94,7 +94,7 @@ static device_method_t acpi_pci_methods[] = {
 	/* PCI interface */
 	DEVMETHOD(pci_set_powerstate,	acpi_pci_set_powerstate_method),
 
-	{ 0, 0 }
+	DEVMETHOD_END
 };
 
 static devclass_t pci_devclass;
diff --git a/sys/dev/acpica/acpi_pci_link.c b/sys/dev/acpica/acpi_pci_link.c
index ba03d72..1a1ae4d 100644
--- a/sys/dev/acpica/acpi_pci_link.c
+++ b/sys/dev/acpica/acpi_pci_link.c
@@ -1097,7 +1097,7 @@ static device_method_t acpi_pci_link_methods[] = {
 	DEVMETHOD(device_attach,	acpi_pci_link_attach),
 	DEVMETHOD(device_resume,	acpi_pci_link_resume),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t acpi_pci_link_driver = {
diff --git a/sys/dev/acpica/acpi_pcib.c b/sys/dev/acpica/acpi_pcib.c
index 1b26b4f..1a42d74 100644
--- a/sys/dev/acpica/acpi_pcib.c
+++ b/sys/dev/acpica/acpi_pcib.c
@@ -129,7 +129,8 @@ prt_attach_devices(ACPI_PCI_ROUTING_TABLE *entry, void *arg)
 int
 acpi_pcib_attach(device_t dev, ACPI_BUFFER *prt, int busno)
 {
-    ACPI_STATUS			status;
+    ACPI_STATUS status;
+    int error;
 
     ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
 
@@ -168,7 +169,8 @@ acpi_pcib_attach(device_t dev, ACPI_BUFFER *prt, int busno)
      */
     prt_walk_table(prt, prt_attach_devices, dev);
 
-    return_VALUE (bus_generic_attach(dev));
+    error = bus_generic_attach(dev);
+    return_VALUE(error);
 }
 
 static void
@@ -273,7 +275,7 @@ acpi_pcib_route_interrupt(device_t pcib, device_t dev, int pin,
 out:
     ACPI_SERIAL_END(pcib);
 
-    return_VALUE (interrupt);
+    return_VALUE(interrupt);
 }
 
 int
@@ -285,4 +287,3 @@ acpi_pcib_power_for_sleep(device_t pcib, device_t dev, int *pstate)
     acpi_device_pwr_for_sleep(acpi_dev, dev, pstate);
     return (0);
 }
-
diff --git a/sys/dev/acpica/acpi_pcib_pci.c b/sys/dev/acpica/acpi_pcib_pci.c
index 7dbccd6..de26a4a 100644
--- a/sys/dev/acpica/acpi_pcib_pci.c
+++ b/sys/dev/acpica/acpi_pcib_pci.c
@@ -83,7 +83,7 @@ static device_method_t acpi_pcib_pci_methods[] = {
     DEVMETHOD(pcib_route_interrupt,	acpi_pcib_pci_route_interrupt),
     DEVMETHOD(pcib_power_for_sleep,	acpi_pcib_power_for_sleep),
 
-    {0, 0}
+    DEVMETHOD_END
 };
 
 static devclass_t pcib_devclass;
diff --git a/sys/dev/acpica/acpi_perf.c b/sys/dev/acpica/acpi_perf.c
index 3f047cc..d95cc79 100644
--- a/sys/dev/acpica/acpi_perf.c
+++ b/sys/dev/acpica/acpi_perf.c
@@ -122,7 +122,8 @@ static device_method_t acpi_perf_methods[] = {
 	DEVMETHOD(cpufreq_drv_get,	acpi_px_get),
 	DEVMETHOD(cpufreq_drv_type,	acpi_px_type),
 	DEVMETHOD(cpufreq_drv_settings,	acpi_px_settings),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t acpi_perf_driver = {
diff --git a/sys/dev/acpica/acpi_resource.c b/sys/dev/acpica/acpi_resource.c
index ce6732f..0878d96 100644
--- a/sys/dev/acpica/acpi_resource.c
+++ b/sys/dev/acpica/acpi_resource.c
@@ -615,7 +615,7 @@ static device_method_t acpi_sysres_methods[] = {
     DEVMETHOD(device_probe,	acpi_sysres_probe),
     DEVMETHOD(device_attach,	acpi_sysres_attach),
 
-    {0, 0}
+    DEVMETHOD_END
 };
 
 static driver_t acpi_sysres_driver = {
diff --git a/sys/dev/acpica/acpi_smbat.c b/sys/dev/acpica/acpi_smbat.c
index f83392f..dda5e75 100644
--- a/sys/dev/acpica/acpi_smbat.c
+++ b/sys/dev/acpica/acpi_smbat.c
@@ -89,7 +89,7 @@ static device_method_t acpi_smbat_methods[] = {
 	DEVMETHOD(acpi_batt_get_status, acpi_smbat_get_bst),
 	DEVMETHOD(acpi_batt_get_info, acpi_smbat_get_bif),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t	acpi_smbat_driver = {
diff --git a/sys/dev/acpica/acpi_thermal.c b/sys/dev/acpica/acpi_thermal.c
index baa8205..485da9e 100644
--- a/sys/dev/acpica/acpi_thermal.c
+++ b/sys/dev/acpica/acpi_thermal.c
@@ -152,7 +152,7 @@ static device_method_t acpi_tz_methods[] = {
     DEVMETHOD(device_probe,	acpi_tz_probe),
     DEVMETHOD(device_attach,	acpi_tz_attach),
 
-    {0, 0}
+    DEVMETHOD_END
 };
 
 static driver_t acpi_tz_driver = {
@@ -509,15 +509,8 @@ acpi_tz_monitor(void *Context)
      */
     newactive = TZ_ACTIVE_NONE;
     for (i = TZ_NUMLEVELS - 1; i >= 0; i--) {
-	if (sc->tz_zone.ac[i] != -1 && temp >= sc->tz_zone.ac[i]) {
+	if (sc->tz_zone.ac[i] != -1 && temp >= sc->tz_zone.ac[i])
 	    newactive = i;
-	    if (sc->tz_active != newactive) {
-		ACPI_VPRINT(sc->tz_dev,
-			    acpi_device_get_parent_softc(sc->tz_dev),
-			    "_AC%d: temperature %d.%d >= setpoint %d.%d\n", i,
-			    TZ_KELVTOC(temp), TZ_KELVTOC(sc->tz_zone.ac[i]));
-	    }
-	}
     }
 
     /*
diff --git a/sys/dev/acpica/acpi_throttle.c b/sys/dev/acpica/acpi_throttle.c
index 40476e0..ca14215 100644
--- a/sys/dev/acpica/acpi_throttle.c
+++ b/sys/dev/acpica/acpi_throttle.c
@@ -114,7 +114,7 @@ static device_method_t acpi_throttle_methods[] = {
 	DEVMETHOD(cpufreq_drv_get,	acpi_thr_get),
 	DEVMETHOD(cpufreq_drv_type,	acpi_thr_type),
 	DEVMETHOD(cpufreq_drv_settings,	acpi_thr_settings),
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t acpi_throttle_driver = {
diff --git a/sys/dev/acpica/acpi_timer.c b/sys/dev/acpica/acpi_timer.c
index 9f61cd8..80e6d18 100644
--- a/sys/dev/acpica/acpi_timer.c
+++ b/sys/dev/acpica/acpi_timer.c
@@ -82,7 +82,7 @@ static device_method_t acpi_timer_methods[] = {
     DEVMETHOD(device_probe,	acpi_timer_probe),
     DEVMETHOD(device_attach,	acpi_timer_attach),
 
-    {0, 0}
+    DEVMETHOD_END
 };
 
 static driver_t acpi_timer_driver = {
diff --git a/sys/dev/adlink/adlink.c b/sys/dev/adlink/adlink.c
index b625d26..4180f57 100644
--- a/sys/dev/adlink/adlink.c
+++ b/sys/dev/adlink/adlink.c
@@ -407,7 +407,8 @@ static device_method_t adlink_methods[] = {
 	DEVMETHOD(device_suspend,	bus_generic_suspend),
 	DEVMETHOD(device_resume,	bus_generic_resume),
 	DEVMETHOD(device_shutdown,	bus_generic_shutdown),
-	{0, 0}
+
+	DEVMETHOD_END
 };
  
 static driver_t adlink_driver = {
diff --git a/sys/dev/agp/agp_i810.c b/sys/dev/agp/agp_i810.c
index 21c7ce6..9ec3992 100644
--- a/sys/dev/agp/agp_i810.c
+++ b/sys/dev/agp/agp_i810.c
@@ -2228,7 +2228,7 @@ agp_i830_chipset_flush(device_t dev)
 	bus_write_4(sc->sc_res[0], AGP_I830_HIC, hic | (1 << 31));
 	for (i = 0; i < 20000 /* 1 sec */; i++) {
 		hic = bus_read_4(sc->sc_res[0], AGP_I830_HIC);
-		if ((hic & (1 << 31)) != 0)
+		if ((hic & (1 << 31)) == 0)
 			break;
 		DELAY(50);
 	}
diff --git a/sys/dev/ahci/ahci.c b/sys/dev/ahci/ahci.c
index 5882c86..b17b9f3 100644
--- a/sys/dev/ahci/ahci.c
+++ b/sys/dev/ahci/ahci.c
@@ -114,8 +114,9 @@ static struct {
 #define AHCI_Q_NOAA	512
 #define AHCI_Q_NOCOUNT	1024
 #define AHCI_Q_ALTSIG	2048
+#define AHCI_Q_NOMSI	4096
 } ahci_ids[] = {
-	{0x43801002, 0x00, "ATI IXP600",	0},
+	{0x43801002, 0x00, "ATI IXP600",	AHCI_Q_NOMSI},
 	{0x43901002, 0x00, "ATI IXP700",	0},
 	{0x43911002, 0x00, "ATI IXP700",	0},
 	{0x43921002, 0x00, "ATI IXP700",	0},
@@ -634,6 +635,8 @@ ahci_setup_interrupt(device_t dev)
 	int i, msi = 1;
 
 	/* Process hints. */
+	if (ctlr->quirks & AHCI_Q_NOMSI)
+		msi = 0;
 	resource_int_value(device_get_name(dev),
 	    device_get_unit(dev), "msi", &msi);
 	if (msi < 0)
diff --git a/sys/dev/altera/avgen/altera_avgen.c b/sys/dev/altera/avgen/altera_avgen.c
index a90b546..5429c9d 100644
--- a/sys/dev/altera/avgen/altera_avgen.c
+++ b/sys/dev/altera/avgen/altera_avgen.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2012 Robert N. M. Watson
+ * Copyright (c) 2012-2013 Robert N. M. Watson
  * All rights reserved.
  *
  * This software was developed by SRI International and the University of
@@ -60,6 +60,8 @@ __FBSDID("$FreeBSD$");
  * system-on-chip bus environments would work fine with the same code.
  */
 
+devclass_t altera_avgen_devclass;
+
 static d_mmap_t altera_avgen_mmap;
 static d_read_t altera_avgen_read;
 static d_write_t altera_avgen_write;
@@ -226,13 +228,6 @@ altera_avgen_mmap(struct cdev *dev, vm_ooffset_t offset, vm_paddr_t *paddr,
 	return (0);
 }
 
-static int
-altera_avgen_nexus_probe(device_t dev)
-{
-
-	device_set_desc(dev, "Generic Altera Avalon device attachment");
-	return (BUS_PROBE_DEFAULT);
-}
 
 static int
 altera_avgen_process_options(struct altera_avgen_softc *sc,
@@ -321,42 +316,14 @@ altera_avgen_process_options(struct altera_avgen_softc *sc,
 	return (0);
 }
 
-static int
-altera_avgen_nexus_attach(device_t dev)
+int
+altera_avgen_attach(struct altera_avgen_softc *sc, const char *str_fileio,
+    const char *str_mmapio, const char *str_devname, int devunit)
 {
-	struct altera_avgen_softc *sc;
-	const char *str_fileio, *str_mmapio;
-	const char *str_devname;
+	device_t dev = sc->avg_dev;
 	char devname[SPECNAMELEN + 1];
-	int devunit, error;
-
-	sc = device_get_softc(dev);
-	sc->avg_dev = dev;
-	sc->avg_unit = device_get_unit(dev);
+	int error;
 
-	/*
-	 * Query non-standard hints to find out what operations are permitted
-	 * on the device, and whether it is cached.
-	 */
-	str_fileio = NULL;
-	str_mmapio = NULL;
-	str_devname = NULL;
-	devunit = -1;
-	sc->avg_width = 1;
-	error = resource_int_value(device_get_name(dev), device_get_unit(dev),
-	    ALTERA_AVALON_STR_WIDTH, &sc->avg_width);
-	if (error != 0 && error != ENOENT) {
-		device_printf(dev, "invalid %s\n", ALTERA_AVALON_STR_WIDTH);
-		return (error);
-	}
-	(void)resource_string_value(device_get_name(dev),
-	    device_get_unit(dev), ALTERA_AVALON_STR_FILEIO, &str_fileio);
-	(void)resource_string_value(device_get_name(dev),
-	    device_get_unit(dev), ALTERA_AVALON_STR_MMAPIO, &str_mmapio);
-	(void)resource_string_value(device_get_name(dev),
-	    device_get_unit(dev), ALTERA_AVALON_STR_DEVNAME, &str_devname);
-	(void)resource_int_value(device_get_name(dev), device_get_unit(dev),
-	    ALTERA_AVALON_STR_DEVUNIT, &devunit);
 	error = altera_avgen_process_options(sc, str_fileio, str_mmapio,
 	    str_devname, devunit);
 	if (error)
@@ -374,25 +341,15 @@ altera_avgen_nexus_attach(device_t dev)
 		snprintf(devname, sizeof(devname), "%s%d", "avgen",
 		    sc->avg_unit);
 
-	/* Memory allocation and checking. */
-	sc->avg_rid = 0;
-	sc->avg_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
-	    &sc->avg_rid, RF_ACTIVE);
-	if (sc->avg_res == NULL) {
-		device_printf(dev, "couldn't map memory\n");
-		return (ENXIO);
-	}
 	if (rman_get_size(sc->avg_res) >= PAGE_SIZE || str_mmapio != NULL) {
 		if (rman_get_size(sc->avg_res) % PAGE_SIZE != 0) {
 			device_printf(dev,
 			    "memory region not even multiple of page size\n");
-			error = ENXIO;
-			goto error;
+			return (ENXIO);
 		}
 		if (rman_get_start(sc->avg_res) % PAGE_SIZE != 0) {
 			device_printf(dev, "memory region not page-aligned\n");
-			error = ENXIO;
-			goto error;
+			return (ENXIO);
 		}
 	}
 
@@ -409,43 +366,16 @@ altera_avgen_nexus_attach(device_t dev)
 		    GID_WHEEL, S_IRUSR | S_IWUSR, str_devname);
 	if (sc->avg_cdev == NULL) {
 		device_printf(sc->avg_dev, "%s: make_dev failed\n", __func__);
-		error = ENXIO;
-		goto error;
+		return (ENXIO);
 	}
 	/* XXXRW: Slight race between make_dev(9) and here. */
 	sc->avg_cdev->si_drv1 = sc;
 	return (0);
-
-error:
-	bus_release_resource(dev, SYS_RES_MEMORY, sc->avg_rid, sc->avg_res);
-	return (error);
 }
 
-static int
-altera_avgen_nexus_detach(device_t dev)
+void
+altera_avgen_detach(struct altera_avgen_softc *sc)
 {
-	struct altera_avgen_softc *sc;
 
-	sc = device_get_softc(dev);
 	destroy_dev(sc->avg_cdev);
-	bus_release_resource(dev, SYS_RES_MEMORY, sc->avg_rid, sc->avg_res);
-	return (0);
 }
-
-static device_method_t altera_avgen_nexus_methods[] = {
-	DEVMETHOD(device_probe,		altera_avgen_nexus_probe),
-	DEVMETHOD(device_attach,	altera_avgen_nexus_attach),
-	DEVMETHOD(device_detach,	altera_avgen_nexus_detach),
-	{ 0, 0 }
-};
-
-static driver_t altera_avgen_nexus_driver = {
-	"altera_avgen",
-	altera_avgen_nexus_methods,
-	sizeof(struct altera_avgen_softc),
-};
-
-static devclass_t altera_avgen_devclass;
-
-DRIVER_MODULE(avgen, nexus, altera_avgen_nexus_driver, altera_avgen_devclass,
-    0, 0);
diff --git a/sys/dev/altera/avgen/altera_avgen.h b/sys/dev/altera/avgen/altera_avgen.h
index c8c7947..fb25a3c 100644
--- a/sys/dev/altera/avgen/altera_avgen.h
+++ b/sys/dev/altera/avgen/altera_avgen.h
@@ -77,7 +77,11 @@ struct altera_avgen_softc {
 /*
  * Driver setup routines from the bus attachment/teardown.
  */
-int	altera_avgen_attach(struct altera_avgen_softc *sc);
+int	altera_avgen_attach(struct altera_avgen_softc *sc,
+	    const char *str_fileio, const char *str_mmapio,
+	    const char *str_devname, int devunit);
 void	altera_avgen_detach(struct altera_avgen_softc *sc);
 
+extern devclass_t	altera_avgen_devclass;
+
 #endif /* _DEV_ALTERA_AVALON_H_ */
diff --git a/sys/dev/altera/avgen/altera_avgen_fdt.c b/sys/dev/altera/avgen/altera_avgen_fdt.c
new file mode 100644
index 0000000..2762ff4
--- /dev/null
+++ b/sys/dev/altera/avgen/altera_avgen_fdt.c
@@ -0,0 +1,154 @@
+/*-
+ * Copyright (c) 2012-2013 Robert N. M. Watson
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
+ * ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/condvar.h>
+#include <sys/conf.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/rman.h>
+#include <sys/stat.h>
+#include <sys/systm.h>
+#include <sys/uio.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <machine/vm.h>
+
+#include <vm/vm.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <dev/altera/avgen/altera_avgen.h>
+
+static int
+altera_avgen_fdt_probe(device_t dev)
+{
+
+	if (ofw_bus_is_compatible(dev, "sri-cambridge,avgen")) {
+		device_set_desc(dev, "Generic Altera Avalon device attachment");
+		return (BUS_PROBE_DEFAULT);
+	}
+	return (ENXIO);
+}
+
+static int
+altera_avgen_fdt_attach(device_t dev)
+{
+	struct altera_avgen_softc *sc;
+	char *str_fileio, *str_mmapio;
+	char *str_devname;
+	phandle_t node;
+	pcell_t cell;
+	int devunit, error;
+
+	sc = device_get_softc(dev);
+	sc->avg_dev = dev;
+	sc->avg_unit = device_get_unit(dev);
+
+	/*
+	 * Query driver-specific OpenFirmware properties to determine how to
+	 * expose the device via /dev.
+	 */
+	str_fileio = NULL;
+	str_mmapio = NULL;
+	str_devname = NULL;
+	devunit = -1;
+	sc->avg_width = 1;
+	node = ofw_bus_get_node(dev);
+	if (OF_getprop(node, "sri-cambridge,width", &cell, sizeof(cell)) > 0)
+		sc->avg_width = cell;
+	(void)OF_getprop_alloc(node, "sri-cambridge,fileio", sizeof(char),
+	    (void **)&str_fileio);
+	(void)OF_getprop_alloc(node, "sri-cambridge,mmapio", sizeof(char),
+	    (void **)&str_mmapio);
+	(void)OF_getprop_alloc(node,  "sri-cambridge,devname", sizeof(char),
+	    (void **)&str_devname);
+	if (OF_getprop(node, "sri-cambridge,devunit", &cell, sizeof(cell)) > 0)
+		devunit = cell;
+
+	/* Memory allocation and checking. */
+	sc->avg_rid = 0;
+	sc->avg_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+	    &sc->avg_rid, RF_ACTIVE);
+	if (sc->avg_res == NULL) {
+		device_printf(dev, "couldn't map memory\n");
+		return (ENXIO);
+	}
+	error = altera_avgen_attach(sc, str_fileio, str_mmapio, str_devname,
+	    devunit);
+	if (error != 0)
+		bus_release_resource(dev, SYS_RES_MEMORY, sc->avg_rid,
+		    sc->avg_res);
+	if (str_fileio != NULL)
+		free(str_fileio, M_OFWPROP);
+	if (str_mmapio != NULL)
+		free(str_mmapio, M_OFWPROP);
+	if (str_devname != NULL)
+		free(str_devname, M_OFWPROP);
+	return (error);
+}
+
+static int
+altera_avgen_fdt_detach(device_t dev)
+{
+	struct altera_avgen_softc *sc;
+
+	sc = device_get_softc(dev);
+	altera_avgen_detach(sc);
+	bus_release_resource(dev, SYS_RES_MEMORY, sc->avg_rid, sc->avg_res);
+	return (0);
+}
+
+static device_method_t altera_avgen_fdt_methods[] = {
+	DEVMETHOD(device_probe,		altera_avgen_fdt_probe),
+	DEVMETHOD(device_attach,	altera_avgen_fdt_attach),
+	DEVMETHOD(device_detach,	altera_avgen_fdt_detach),
+	{ 0, 0 }
+};
+
+static driver_t altera_avgen_fdt_driver = {
+	"altera_avgen",
+	altera_avgen_fdt_methods,
+	sizeof(struct altera_avgen_softc),
+};
+
+DRIVER_MODULE(avgen, simplebus, altera_avgen_fdt_driver,
+    altera_avgen_devclass, 0, 0);
diff --git a/sys/dev/altera/avgen/altera_avgen_nexus.c b/sys/dev/altera/avgen/altera_avgen_nexus.c
new file mode 100644
index 0000000..d0e4f9b
--- /dev/null
+++ b/sys/dev/altera/avgen/altera_avgen_nexus.c
@@ -0,0 +1,141 @@
+/*-
+ * Copyright (c) 2012-2013 Robert N. M. Watson
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
+ * ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/condvar.h>
+#include <sys/conf.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/rman.h>
+#include <sys/stat.h>
+#include <sys/systm.h>
+#include <sys/uio.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <machine/vm.h>
+
+#include <vm/vm.h>
+
+#include <dev/altera/avgen/altera_avgen.h>
+
+static int
+altera_avgen_nexus_probe(device_t dev)
+{
+
+	device_set_desc(dev, "Generic Altera Avalon device attachment");
+	return (BUS_PROBE_DEFAULT);
+}
+
+static int
+altera_avgen_nexus_attach(device_t dev)
+{
+	struct altera_avgen_softc *sc;
+	const char *str_fileio, *str_mmapio;
+	const char *str_devname;
+	int devunit, error;
+
+	sc = device_get_softc(dev);
+	sc->avg_dev = dev;
+	sc->avg_unit = device_get_unit(dev);
+
+	/*
+	 * Query non-standard hints to find out what operations are permitted
+	 * on the device, and whether it is cached.
+	 */
+	str_fileio = NULL;
+	str_mmapio = NULL;
+	str_devname = NULL;
+	devunit = -1;
+	sc->avg_width = 1;
+	error = resource_int_value(device_get_name(dev), device_get_unit(dev),
+	    ALTERA_AVALON_STR_WIDTH, &sc->avg_width);
+	if (error != 0 && error != ENOENT) {
+		device_printf(dev, "invalid %s\n", ALTERA_AVALON_STR_WIDTH);
+		return (error);
+	}
+	(void)resource_string_value(device_get_name(dev),
+	    device_get_unit(dev), ALTERA_AVALON_STR_FILEIO, &str_fileio);
+	(void)resource_string_value(device_get_name(dev),
+	    device_get_unit(dev), ALTERA_AVALON_STR_MMAPIO, &str_mmapio);
+	(void)resource_string_value(device_get_name(dev),
+	    device_get_unit(dev), ALTERA_AVALON_STR_DEVNAME, &str_devname);
+	(void)resource_int_value(device_get_name(dev), device_get_unit(dev),
+	    ALTERA_AVALON_STR_DEVUNIT, &devunit);
+
+	/* Memory allocation and checking. */
+	sc->avg_rid = 0;
+	sc->avg_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+	    &sc->avg_rid, RF_ACTIVE);
+	if (sc->avg_res == NULL) {
+		device_printf(dev, "couldn't map memory\n");
+		return (ENXIO);
+	}
+	error = altera_avgen_attach(sc, str_fileio, str_mmapio, str_devname,
+	    devunit);
+	if (error != 0)
+		bus_release_resource(dev, SYS_RES_MEMORY, sc->avg_rid,
+		    sc->avg_res);
+	return (error);
+}
+
+static int
+altera_avgen_nexus_detach(device_t dev)
+{
+	struct altera_avgen_softc *sc;
+
+	sc = device_get_softc(dev);
+	altera_avgen_detach(sc);
+	bus_release_resource(dev, SYS_RES_MEMORY, sc->avg_rid, sc->avg_res);
+	return (0);
+}
+
+static device_method_t altera_avgen_nexus_methods[] = {
+	DEVMETHOD(device_probe,		altera_avgen_nexus_probe),
+	DEVMETHOD(device_attach,	altera_avgen_nexus_attach),
+	DEVMETHOD(device_detach,	altera_avgen_nexus_detach),
+	{ 0, 0 }
+};
+
+static driver_t altera_avgen_nexus_driver = {
+	"altera_avgen",
+	altera_avgen_nexus_methods,
+	sizeof(struct altera_avgen_softc),
+};
+
+DRIVER_MODULE(avgen, nexus, altera_avgen_nexus_driver, altera_avgen_devclass,
+    0, 0);
diff --git a/sys/dev/altera/jtag_uart/altera_jtag_uart.h b/sys/dev/altera/jtag_uart/altera_jtag_uart.h
index 1ff3bcb..5b5b29f 100644
--- a/sys/dev/altera/jtag_uart/altera_jtag_uart.h
+++ b/sys/dev/altera/jtag_uart/altera_jtag_uart.h
@@ -194,4 +194,6 @@ extern u_int		aju_cons_jtag_missed;
 int	altera_jtag_uart_attach(struct altera_jtag_uart_softc *sc);
 void	altera_jtag_uart_detach(struct altera_jtag_uart_softc *sc);
 
+extern devclass_t	altera_jtag_uart_devclass;
+
 #endif /* _DEV_ALTERA_JTAG_UART_H_ */
diff --git a/sys/dev/altera/jtag_uart/altera_jtag_uart_cons.c b/sys/dev/altera/jtag_uart/altera_jtag_uart_cons.c
index 679ebe2..ae84597 100644
--- a/sys/dev/altera/jtag_uart/altera_jtag_uart_cons.c
+++ b/sys/dev/altera/jtag_uart/altera_jtag_uart_cons.c
@@ -32,6 +32,7 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
+#include <sys/bus.h>
 #include <sys/cons.h>
 #include <sys/endian.h>
 #include <sys/kdb.h>
@@ -46,6 +47,8 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/altera/jtag_uart/altera_jtag_uart.h>
 
+devclass_t	altera_jtag_uart_devclass;
+
 /*
  * One-byte buffer as we can't check whether the UART is readable without
  * actually reading from it, synchronised by a spinlock; this lock also
diff --git a/sys/dev/altera/jtag_uart/altera_jtag_uart_fdt.c b/sys/dev/altera/jtag_uart/altera_jtag_uart_fdt.c
new file mode 100644
index 0000000..47f39a4
--- /dev/null
+++ b/sys/dev/altera/jtag_uart/altera_jtag_uart_fdt.c
@@ -0,0 +1,147 @@
+/*-
+ * Copyright (c) 2012 Robert N. M. Watson
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
+ * ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/condvar.h>
+#include <sys/conf.h>
+#include <sys/bio.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/rman.h>
+#include <sys/systm.h>
+#include <sys/taskqueue.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+
+#include <geom/geom_disk.h>
+
+#include <dev/altera/jtag_uart/altera_jtag_uart.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+/*
+ * FDT bus attachment for Altera JTAG UARTs.
+ */
+static int
+altera_jtag_uart_fdt_probe(device_t dev)
+{
+
+	if (ofw_bus_is_compatible(dev, "altera,jtag_uart-11_0")) {
+		device_set_desc(dev, "Altera JTAG UART");
+		return (BUS_PROBE_DEFAULT);
+	}
+	return (ENXIO);
+}
+
+static int
+altera_jtag_uart_fdt_attach(device_t dev)
+{
+	struct altera_jtag_uart_softc *sc;
+	int error;
+
+	error = 0;
+	sc = device_get_softc(dev);
+	sc->ajus_dev = dev;
+	sc->ajus_unit = device_get_unit(dev);
+	sc->ajus_mem_rid = 0;
+	sc->ajus_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+	    &sc->ajus_mem_rid, RF_ACTIVE);
+	if (sc->ajus_mem_res == NULL) {
+		device_printf(dev, "couldn't map memory\n");
+		error = ENXIO;
+		goto out;
+	}
+
+	/*
+	 * Interrupt support is optional -- if we can't allocate an IRQ, then
+	 * we fall back on polling.
+	 */
+	sc->ajus_irq_rid = 0;
+	sc->ajus_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
+	    &sc->ajus_irq_rid, RF_ACTIVE | RF_SHAREABLE);
+	if (sc->ajus_irq_res == NULL)
+		device_printf(dev,
+		    "IRQ unavailable; selecting polled operation\n");
+	error = altera_jtag_uart_attach(sc);
+out:
+	if (error) {
+		if (sc->ajus_irq_res != NULL)
+			bus_release_resource(dev, SYS_RES_IRQ,
+			    sc->ajus_irq_rid, sc->ajus_irq_res);
+		if (sc->ajus_mem_res != NULL)
+			bus_release_resource(dev, SYS_RES_MEMORY,
+			    sc->ajus_mem_rid, sc->ajus_mem_res);
+	}
+	return (error);
+}
+
+static int
+altera_jtag_uart_fdt_detach(device_t dev)
+{
+	struct altera_jtag_uart_softc *sc;
+
+	sc = device_get_softc(dev);
+	KASSERT(sc->ajus_mem_res != NULL, ("%s: resources not allocated",
+	    __func__));
+
+	altera_jtag_uart_detach(sc);
+	bus_release_resource(dev, SYS_RES_IRQ, sc->ajus_irq_rid,
+	    sc->ajus_irq_res);
+	bus_release_resource(dev, SYS_RES_MEMORY, sc->ajus_mem_rid,
+	    sc->ajus_mem_res);
+	return (0);
+}
+
+static device_method_t altera_jtag_uart_fdt_methods[] = {
+	DEVMETHOD(device_probe,		altera_jtag_uart_fdt_probe),
+	DEVMETHOD(device_attach,	altera_jtag_uart_fdt_attach),
+	DEVMETHOD(device_detach,	altera_jtag_uart_fdt_detach),
+	{ 0, 0 }
+};
+
+static driver_t altera_jtag_uart_fdt_driver = {
+	"altera_jtag_uart",
+	altera_jtag_uart_fdt_methods,
+	sizeof(struct altera_jtag_uart_softc),
+};
+
+DRIVER_MODULE(altera_jtag_uart, simplebus, altera_jtag_uart_fdt_driver,
+    altera_jtag_uart_devclass, 0, 0);
diff --git a/sys/dev/altera/jtag_uart/altera_jtag_uart_nexus.c b/sys/dev/altera/jtag_uart/altera_jtag_uart_nexus.c
index 2592249..2fe772a 100644
--- a/sys/dev/altera/jtag_uart/altera_jtag_uart_nexus.c
+++ b/sys/dev/altera/jtag_uart/altera_jtag_uart_nexus.c
@@ -137,7 +137,5 @@ static driver_t altera_jtag_uart_nexus_driver = {
 	sizeof(struct altera_jtag_uart_softc),
 };
 
-static devclass_t altera_jtag_uart_devclass;
-
 DRIVER_MODULE(altera_jtag_uart, nexus, altera_jtag_uart_nexus_driver,
     altera_jtag_uart_devclass, 0, 0);
diff --git a/sys/dev/altera/sdcard/altera_sdcard.c b/sys/dev/altera/sdcard/altera_sdcard.c
index 4572206..f64b9c5 100644
--- a/sys/dev/altera/sdcard/altera_sdcard.c
+++ b/sys/dev/altera/sdcard/altera_sdcard.c
@@ -70,6 +70,7 @@ __FBSDID("$FreeBSD$");
  * 3. Handle read-only SD Cards.
  * 4. Tune timeouts based on real-world SD Card speeds.
  */
+devclass_t	altera_sdcard_devclass;
 
 void
 altera_sdcard_attach(struct altera_sdcard_softc *sc)
diff --git a/sys/dev/altera/sdcard/altera_sdcard.h b/sys/dev/altera/sdcard/altera_sdcard.h
index 4a491c4..96019b2 100644
--- a/sys/dev/altera/sdcard/altera_sdcard.h
+++ b/sys/dev/altera/sdcard/altera_sdcard.h
@@ -138,7 +138,7 @@ void	altera_sdcard_io_start(struct altera_sdcard_softc *sc,
  * data.  Software ease of use was not a design consideration.
  */
 #define	ALTERA_SDCARD_CSD_C_SIZE_BYTE0		7
-#define	ALTERA_SDCARD_CSD_C_SIZE_MASK0		0xc	/* top 2 bits */
+#define	ALTERA_SDCARD_CSD_C_SIZE_MASK0		0xc0	/* top 2 bits */
 #define	ALTERA_SDCARD_CSD_C_SIZE_RSHIFT0	6
 
 #define	ALTERA_SDCARD_CSD_C_SIZE_BYTE1		8
@@ -244,4 +244,6 @@ void	altera_sdcard_start(struct altera_sdcard_softc *sc);
 void	altera_sdcard_disk_insert(struct altera_sdcard_softc *sc);
 void	altera_sdcard_disk_remove(struct altera_sdcard_softc *sc);
 
+extern devclass_t	altera_sdcard_devclass;
+
 #endif	/* _DEV_ALTERA_SDCARD_H_ */
diff --git a/sys/dev/altera/sdcard/altera_sdcard_fdt.c b/sys/dev/altera/sdcard/altera_sdcard_fdt.c
new file mode 100644
index 0000000..027ca2d
--- /dev/null
+++ b/sys/dev/altera/sdcard/altera_sdcard_fdt.c
@@ -0,0 +1,120 @@
+/*-
+ * Copyright (c) 2012 Robert N. M. Watson
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
+ * ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/condvar.h>
+#include <sys/conf.h>
+#include <sys/bio.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/rman.h>
+#include <sys/systm.h>
+#include <sys/taskqueue.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+
+#include <geom/geom_disk.h>
+
+#include <dev/altera/sdcard/altera_sdcard.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+/*
+ * FDT bus attachment for the Altera SD Card IP core.
+ */
+static int
+altera_sdcard_fdt_probe(device_t dev)
+{
+
+	if (ofw_bus_is_compatible(dev, "altera,sdcard_11_2011")) {
+		device_set_desc(dev, "Altera Secure Data Card IP Core");
+		return (BUS_PROBE_DEFAULT);
+	}
+	return (ENXIO);
+}
+
+static int
+altera_sdcard_fdt_attach(device_t dev)
+{
+	struct altera_sdcard_softc *sc;
+
+	sc = device_get_softc(dev);
+	sc->as_dev = dev;
+	sc->as_unit = device_get_unit(dev);
+	sc->as_rid = 0;
+	sc->as_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+	    &sc->as_rid, RF_ACTIVE);
+	if (sc->as_res == NULL) {
+		device_printf(dev, "couldn't map memory\n");
+		return (ENXIO);
+	}
+	altera_sdcard_attach(sc);
+	return (0);
+}
+
+static int
+altera_sdcard_fdt_detach(device_t dev)
+{
+	struct altera_sdcard_softc *sc;
+
+	sc = device_get_softc(dev);
+	KASSERT(sc->as_res != NULL, ("%s: resources not allocated",
+	    __func__));
+	altera_sdcard_detach(sc);
+	bus_release_resource(dev, SYS_RES_MEMORY, sc->as_rid, sc->as_res);
+	return (0);
+}
+
+static device_method_t altera_sdcard_fdt_methods[] = {
+	DEVMETHOD(device_probe,		altera_sdcard_fdt_probe),
+	DEVMETHOD(device_attach,	altera_sdcard_fdt_attach),
+	DEVMETHOD(device_detach,	altera_sdcard_fdt_detach),
+	{ 0, 0 }
+};
+
+static driver_t altera_sdcard_fdt_driver = {
+	"altera_sdcardc",
+	altera_sdcard_fdt_methods,
+	sizeof(struct altera_sdcard_softc),
+};
+
+DRIVER_MODULE(altera_sdcard, simplebus, altera_sdcard_fdt_driver,
+    altera_sdcard_devclass, 0, 0);
diff --git a/sys/dev/altera/sdcard/altera_sdcard_io.c b/sys/dev/altera/sdcard/altera_sdcard_io.c
index adbd5d1..b256c0b 100644
--- a/sys/dev/altera/sdcard/altera_sdcard_io.c
+++ b/sys/dev/altera/sdcard/altera_sdcard_io.c
@@ -93,13 +93,6 @@ altera_sdcard_process_csd0(struct altera_sdcard_softc *sc)
 	read_bl_len = sc->as_csd.csd_data[ALTERA_SDCARD_CSD_READ_BL_LEN_BYTE];
 	read_bl_len &= ALTERA_SDCARD_CSD_READ_BL_LEN_MASK;
 
-	byte0 = sc->as_csd.csd_data[ALTERA_SDCARD_CSD_C_SIZE_MULT_BYTE0];
-	byte0 &= ALTERA_SDCARD_CSD_C_SIZE_MULT_MASK0;
-	byte1 = sc->as_csd.csd_data[ALTERA_SDCARD_CSD_C_SIZE_MULT_BYTE1];
-	byte1 &= ALTERA_SDCARD_CSD_C_SIZE_MULT_MASK1;
-	c_size_mult = (byte0 >> ALTERA_SDCARD_CSD_C_SIZE_MULT_RSHIFT0) |
-	    (byte0 << ALTERA_SDCARD_CSD_C_SIZE_MULT_LSHIFT1);
-
 	byte0 = sc->as_csd.csd_data[ALTERA_SDCARD_CSD_C_SIZE_BYTE0];
 	byte0 &= ALTERA_SDCARD_CSD_C_SIZE_MASK0;
 	byte1 = sc->as_csd.csd_data[ALTERA_SDCARD_CSD_C_SIZE_BYTE1];
diff --git a/sys/dev/altera/sdcard/altera_sdcard_nexus.c b/sys/dev/altera/sdcard/altera_sdcard_nexus.c
index e0fddcb..88abe1f 100644
--- a/sys/dev/altera/sdcard/altera_sdcard_nexus.c
+++ b/sys/dev/altera/sdcard/altera_sdcard_nexus.c
@@ -110,7 +110,5 @@ static driver_t altera_sdcard_nexus_driver = {
 	sizeof(struct altera_sdcard_softc),
 };
 
-static devclass_t altera_sdcard_devclass;
-
 DRIVER_MODULE(altera_sdcard, nexus, altera_sdcard_nexus_driver,
     altera_sdcard_devclass, 0, 0);
diff --git a/sys/dev/amdsbwd/amdsbwd.c b/sys/dev/amdsbwd/amdsbwd.c
index 4256381..4726416 100644
--- a/sys/dev/amdsbwd/amdsbwd.c
+++ b/sys/dev/amdsbwd/amdsbwd.c
@@ -139,7 +139,7 @@ static device_method_t amdsbwd_methods[] = {
 #if 0
 	DEVMETHOD(device_shutdown,	amdsbwd_detach),
 #endif
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static devclass_t	amdsbwd_devclass;
diff --git a/sys/dev/amdtemp/amdtemp.c b/sys/dev/amdtemp/amdtemp.c
index 6d106ae..c55e680 100644
--- a/sys/dev/amdtemp/amdtemp.c
+++ b/sys/dev/amdtemp/amdtemp.c
@@ -134,7 +134,7 @@ static device_method_t amdtemp_methods[] = {
 	DEVMETHOD(device_attach,	amdtemp_attach),
 	DEVMETHOD(device_detach,	amdtemp_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t amdtemp_driver = {
diff --git a/sys/dev/ata/ata-all.h b/sys/dev/ata/ata-all.h
index 3a6710c..28c93f9 100644
--- a/sys/dev/ata/ata-all.h
+++ b/sys/dev/ata/ata-all.h
@@ -698,6 +698,8 @@ MALLOC_DECLARE(M_ATA);
 
 #define ATA_INW(res, offset) \
 	bus_read_2((res), (offset))
+#define ATA_INW_STRM(res, offset) \
+	bus_read_stream_2((res), (offset))
 #define ATA_INL(res, offset) \
 	bus_read_4((res), (offset))
 #define ATA_INSW(res, offset, addr, count) \
@@ -712,6 +714,8 @@ MALLOC_DECLARE(M_ATA);
 	bus_write_1((res), (offset), (value))
 #define ATA_OUTW(res, offset, value) \
 	bus_write_2((res), (offset), (value))
+#define ATA_OUTW_STRM(res, offset, value) \
+	bus_write_stream_2((res), (offset), (value))
 #define ATA_OUTL(res, offset, value) \
 	bus_write_4((res), (offset), (value))
 #define ATA_OUTSW(res, offset, addr, count) \
@@ -729,6 +733,9 @@ MALLOC_DECLARE(M_ATA);
 #define ATA_IDX_INW(ch, idx) \
 	ATA_INW(ch->r_io[idx].res, ch->r_io[idx].offset)
 
+#define ATA_IDX_INW_STRM(ch, idx) \
+	ATA_INW_STRM(ch->r_io[idx].res, ch->r_io[idx].offset)
+
 #define ATA_IDX_INL(ch, idx) \
 	ATA_INL(ch->r_io[idx].res, ch->r_io[idx].offset)
 
@@ -750,6 +757,9 @@ MALLOC_DECLARE(M_ATA);
 #define ATA_IDX_OUTW(ch, idx, value) \
 	ATA_OUTW(ch->r_io[idx].res, ch->r_io[idx].offset, value)
 
+#define ATA_IDX_OUTW_STRM(ch, idx, value) \
+	ATA_OUTW_STRM(ch->r_io[idx].res, ch->r_io[idx].offset, value)
+
 #define ATA_IDX_OUTL(ch, idx, value) \
 	ATA_OUTL(ch->r_io[idx].res, ch->r_io[idx].offset, value)
 
diff --git a/sys/dev/ata/ata-lowlevel.c b/sys/dev/ata/ata-lowlevel.c
index de30603..6f83db7 100644
--- a/sys/dev/ata/ata-lowlevel.c
+++ b/sys/dev/ata/ata-lowlevel.c
@@ -846,14 +846,28 @@ ata_pio_read(struct ata_request *request, int length)
     uint8_t *addr;
     int size = min(request->transfersize, length);
     int resid;
-    uint8_t buf[2];
+    uint8_t buf[2] __aligned(sizeof(int16_t));
+#ifndef __NO_STRICT_ALIGNMENT
+    int i;
+#endif
 
     addr = (uint8_t *)request->data + request->donecount;
-    if (ch->flags & ATA_USE_16BIT || (size % sizeof(int32_t)) ||
-	((uintptr_t)addr % sizeof(int32_t))) {
-	ATA_IDX_INSW_STRM(ch, ATA_DATA, (void*)addr, size / sizeof(int16_t));
+    if (__predict_false(ch->flags & ATA_USE_16BIT ||
+      (size % sizeof(int32_t)) || ((uintptr_t)addr % sizeof(int32_t)))) {
+#ifndef __NO_STRICT_ALIGNMENT
+	if (__predict_false((uintptr_t)addr % sizeof(int16_t))) {
+	    for (i = 0, resid = size & ~1; resid > 0; resid -=
+	      sizeof(int16_t)) {
+		*(uint16_t *)&buf = ATA_IDX_INW_STRM(ch, ATA_DATA);
+	        addr[i++] = buf[0];
+	        addr[i++] = buf[1];
+	    }
+	} else
+#endif
+	    ATA_IDX_INSW_STRM(ch, ATA_DATA, (void*)addr, size /
+	      sizeof(int16_t));
 	if (size & 1) {
-	    ATA_IDX_INSW_STRM(ch, ATA_DATA, (void*)buf, 1);
+	    *(uint16_t *)&buf = ATA_IDX_INW_STRM(ch, ATA_DATA);
 	    (addr + (size & ~1))[0] = buf[0];
 	}
     } else
@@ -875,15 +889,30 @@ ata_pio_write(struct ata_request *request, int length)
     uint8_t *addr;
     int size = min(request->transfersize, length);
     int resid;
-    uint8_t buf[2];
+    uint8_t buf[2] __aligned(sizeof(int16_t));
+#ifndef __NO_STRICT_ALIGNMENT
+    int i;
+#endif
 
+    size = min(request->transfersize, length);
     addr = (uint8_t *)request->data + request->donecount;
-    if (ch->flags & ATA_USE_16BIT || (size % sizeof(int32_t)) ||
-	((uintptr_t)addr % sizeof(int32_t))) {
-	ATA_IDX_OUTSW_STRM(ch, ATA_DATA, (void*)addr, size / sizeof(int16_t));
+    if (__predict_false(ch->flags & ATA_USE_16BIT ||
+      (size % sizeof(int32_t)) || ((uintptr_t)addr % sizeof(int32_t)))) {
+#ifndef __NO_STRICT_ALIGNMENT
+	if (__predict_false((uintptr_t)addr % sizeof(int16_t))) {
+	    for (i = 0, resid = size & ~1; resid > 0; resid -=
+	      sizeof(int16_t)) {
+	        buf[0] = addr[i++];
+	        buf[1] = addr[i++];
+		ATA_IDX_OUTW_STRM(ch, ATA_DATA, *(uint16_t *)&buf);
+	    }
+	} else
+#endif
+	    ATA_IDX_OUTSW_STRM(ch, ATA_DATA, (void*)addr, size /
+	      sizeof(int16_t));
 	if (size & 1) {
 	    buf[0] = (addr + (size & ~1))[0];
-	    ATA_IDX_OUTSW_STRM(ch, ATA_DATA, (void*)buf, 1);
+	    ATA_IDX_OUTW_STRM(ch, ATA_DATA, *(uint16_t *)&buf);
 	}
     } else
 	ATA_IDX_OUTSL_STRM(ch, ATA_DATA, (void*)addr, size / sizeof(int32_t));
diff --git a/sys/dev/ata/ata-raid.c b/sys/dev/ata/ata-raid.c
index c96e252..5b0595c 100644
--- a/sys/dev/ata/ata-raid.c
+++ b/sys/dev/ata/ata-raid.c
@@ -1351,10 +1351,11 @@ static int
 ata_raid_read_metadata(device_t subdisk)
 {
     devclass_t pci_devclass = devclass_find("pci");
+    devclass_t atapci_devclass = devclass_find("atapci");
     devclass_t devclass=device_get_devclass(GRANDPARENT(GRANDPARENT(subdisk)));
 
     /* prioritize vendor native metadata layout if possible */
-    if (devclass == pci_devclass) {
+    if (devclass == pci_devclass || devclass == atapci_devclass) {
 	switch (pci_get_vendor(GRANDPARENT(device_get_parent(subdisk)))) {
 	case ATA_HIGHPOINT_ID: 
 	    if (ata_raid_hptv3_read_meta(subdisk, ata_raid_arrays))
diff --git a/sys/dev/ath/ath_hal/ah.h b/sys/dev/ath/ath_hal/ah.h
index 19beffe..ddc4b6a 100644
--- a/sys/dev/ath/ath_hal/ah.h
+++ b/sys/dev/ath/ath_hal/ah.h
@@ -941,6 +941,8 @@ typedef struct {
 	int8_t		ss_nf_cal[AH_MAX_CHAINS*2];     /* nf calibrated values for ctl+ext from eeprom */
 	int8_t		ss_nf_pwr[AH_MAX_CHAINS*2];     /* nf pwr values for ctl+ext from eeprom */
 	int32_t		ss_nf_temp_data;	/* temperature data taken during nf scan */
+	int		ss_enabled;
+	int		ss_active;
 } HAL_SPECTRAL_PARAM;
 #define	HAL_SPECTRAL_PARAM_NOVAL	0xFFFF
 #define	HAL_SPECTRAL_PARAM_ENABLE	0x8000	/* Enable/Disable if applicable */
diff --git a/sys/dev/ath/ath_hal/ar5211/boss.ini b/sys/dev/ath/ath_hal/ar5211/boss.ini
index e054436..e054436 100755..100644
--- a/sys/dev/ath/ath_hal/ar5211/boss.ini
+++ b/sys/dev/ath/ath_hal/ar5211/boss.ini
diff --git a/sys/dev/ath/if_ath.c b/sys/dev/ath/if_ath.c
index bd2580f..41019fb 100644
--- a/sys/dev/ath/if_ath.c
+++ b/sys/dev/ath/if_ath.c
@@ -152,7 +152,6 @@ static void	ath_init(void *);
 static void	ath_stop_locked(struct ifnet *);
 static void	ath_stop(struct ifnet *);
 static int	ath_reset_vap(struct ieee80211vap *, u_long);
-static void	ath_start_queue(struct ifnet *ifp);
 static int	ath_media_change(struct ifnet *);
 static void	ath_watchdog(void *);
 static int	ath_ioctl(struct ifnet *, u_long, caddr_t);
@@ -213,6 +212,14 @@ static void	ath_dfs_tasklet(void *, int);
 static void	ath_node_powersave(struct ieee80211_node *, int);
 static int	ath_node_set_tim(struct ieee80211_node *, int);
 
+static int	ath_transmit(struct ifnet *ifp, struct mbuf *m);
+static void	ath_qflush(struct ifnet *ifp);
+
+static void	ath_txq_qinit(struct ifnet *ifp);
+static void	ath_txq_qflush(struct ifnet *ifp);
+static int	ath_txq_qadd(struct ifnet *ifp, struct mbuf *m0);
+static void	ath_txq_qrun(struct ifnet *ifp);
+
 #ifdef IEEE80211_SUPPORT_TDMA
 #include <dev/ath/if_ath_tdma.h>
 #endif
@@ -429,12 +436,30 @@ ath_attach(u_int16_t devid, struct ath_softc *sc)
 	taskqueue_start_threads(&sc->sc_tq, 1, PI_NET,
 		"%s taskq", ifp->if_xname);
 
+	/*
+	 * This taskqueue doesn't get any higher priority
+	 * than the ath(4) taskqueue (PI_NET) so TX won't
+	 * pre-empt RX and other task priorities.
+	 *
+	 * This may not be optimal - the previous behaviour
+	 * was to direct-dispatch frames via the sending
+	 * task context, rather than (always) software
+	 * queuing.
+	 */
+	sc->sc_tx_tq = taskqueue_create("ath_tx_taskq", M_NOWAIT,
+		taskqueue_thread_enqueue, &sc->sc_tx_tq);
+	taskqueue_start_threads(&sc->sc_tx_tq, 1, PI_NET,
+		"%s TX taskq", ifp->if_xname);
+
 	TASK_INIT(&sc->sc_rxtask, 0, sc->sc_rx.recv_tasklet, sc);
 	TASK_INIT(&sc->sc_bmisstask, 0, ath_bmiss_proc, sc);
 	TASK_INIT(&sc->sc_bstucktask,0, ath_bstuck_proc, sc);
 	TASK_INIT(&sc->sc_resettask,0, ath_reset_proc, sc);
-	TASK_INIT(&sc->sc_txqtask,0, ath_txq_sched_tasklet, sc);
-	TASK_INIT(&sc->sc_fataltask,0, ath_fatal_proc, sc);
+	TASK_INIT(&sc->sc_txqtask, 0, ath_txq_sched_tasklet, sc);
+	TASK_INIT(&sc->sc_fataltask, 0, ath_fatal_proc, sc);
+
+	/* XXX make this a higher priority taskqueue? */
+	TASK_INIT(&sc->sc_txpkttask, 0, ath_start_task, sc);
 
 	/*
 	 * Allocate hardware transmit queues: one queue for
@@ -554,13 +579,18 @@ ath_attach(u_int16_t devid, struct ath_softc *sc)
 
 	ifp->if_softc = sc;
 	ifp->if_flags = IFF_SIMPLEX | IFF_BROADCAST | IFF_MULTICAST;
-	ifp->if_start = ath_start_queue;
+	/* XXX net80211 uses if_start to re-start ifnet processing */
+	ifp->if_start = ath_start;
+	ifp->if_transmit = ath_transmit;
+	ifp->if_qflush = ath_qflush;
 	ifp->if_ioctl = ath_ioctl;
 	ifp->if_init = ath_init;
 	IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen);
 	ifp->if_snd.ifq_drv_maxlen = ifqmaxlen;
 	IFQ_SET_READY(&ifp->if_snd);
 
+	ath_txq_qinit(ifp);
+
 	ic->ic_ifp = ifp;
 	/* XXX not right but it's not used anywhere important */
 	ic->ic_phytype = IEEE80211_T_OFDM;
@@ -966,16 +996,16 @@ ath_detach(struct ath_softc *sc)
 	ath_stop(ifp);
 	ieee80211_ifdetach(ifp->if_l2com);
 	taskqueue_free(sc->sc_tq);
+	taskqueue_free(sc->sc_tx_tq);
 #ifdef ATH_TX99_DIAG
 	if (sc->sc_tx99 != NULL)
 		sc->sc_tx99->detach(sc->sc_tx99);
 #endif
 	ath_rate_detach(sc->sc_rc);
-
 #ifdef	ATH_DEBUG_ALQ
 	if_ath_alq_tidyup(&sc->sc_alq);
 #endif
-
+	ath_txq_qflush(ifp);
 	ath_spectral_detach(sc);
 	ath_dfs_detach(sc);
 	ath_desc_free(sc);
@@ -1863,11 +1893,19 @@ ath_bmiss_proc(void *arg, int pending)
 
 	DPRINTF(sc, ATH_DEBUG_ANY, "%s: pending %u\n", __func__, pending);
 
+	/*
+	 * Do a reset upon any becaon miss event.
+	 *
+	 * It may be a non-recognised RX clear hang which needs a reset
+	 * to clear.
+	 */
 	if (ath_hal_gethangstate(sc->sc_ah, 0xff, &hangs) && hangs != 0) {
+		ath_reset(ifp, ATH_RESET_NOLOSS);
 		if_printf(ifp, "bb hang detected (0x%x), resetting\n", hangs);
+	} else {
 		ath_reset(ifp, ATH_RESET_NOLOSS);
-	} else
 		ieee80211_beacon_miss(ifp->if_l2com);
+	}
 }
 
 /*
@@ -2107,6 +2145,7 @@ ath_txrx_start(struct ath_softc *sc)
 {
 
 	taskqueue_unblock(sc->sc_tq);
+	taskqueue_unblock(sc->sc_tx_tq);
 }
 
 /*
@@ -2207,6 +2246,7 @@ ath_reset(struct ifnet *ifp, ATH_RESET_TYPE reset_type)
 
 	/* Try to (stop any further TX/RX from occuring */
 	taskqueue_block(sc->sc_tq);
+	taskqueue_block(sc->sc_tx_tq);
 
 	ATH_PCU_LOCK(sc);
 	ath_hal_intrset(ah, 0);		/* disable interrupts */
@@ -2454,6 +2494,14 @@ ath_buf_clone(struct ath_softc *sc, const struct ath_buf *bf)
 	tbf->bf_flags = bf->bf_flags & ~ATH_BUF_BUSY;
 	tbf->bf_status = bf->bf_status;
 	tbf->bf_m = bf->bf_m;
+	/*
+	 * XXX Copy the node reference, the caller is responsible
+	 * for deleting the node reference before it frees its
+	 * buffer.
+	 *
+	 * XXX It's done like this so we don't call the net80211
+	 * code whilst having active TX queue locks held.
+	 */
 	tbf->bf_node = bf->bf_node;
 	/* will be setup by the chain/setup function */
 	tbf->bf_lastds = NULL;
@@ -2498,13 +2546,70 @@ ath_getbuf(struct ath_softc *sc, ath_buf_type_t btype)
 }
 
 static void
-ath_start_queue(struct ifnet *ifp)
+ath_qflush(struct ifnet *ifp)
 {
-	struct ath_softc *sc = ifp->if_softc;
 
-	ATH_KTR(sc, ATH_KTR_TX, 0, "ath_start_queue: start");
+	/* XXX complete/suspend TX */
+	ath_txq_qflush(ifp);
+
+	/* Unsuspend TX? */
+}
+
+/*
+ * Transmit a frame from net80211.
+ */
+static int
+ath_transmit(struct ifnet *ifp, struct mbuf *m)
+{
+	struct ieee80211_node *ni;
+	struct ath_softc *sc = (struct ath_softc *) ifp->if_softc;
+
+	ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
+
+	if (ath_txq_qadd(ifp, m) < 0) {
+		/*
+		 * If queuing fails, the if_transmit() API makes the
+		 * callee responsible for freeing the mbuf (rather than
+		 * the caller, who just assumes the mbuf has been dealt
+		 * with somehow).
+		 *
+		 * BUT, net80211 will free node references if if_transmit()
+		 * fails _on encapsulated buffers_.  Since drivers
+		 * only get fully encapsulated frames from net80211 (via
+		 * raw or otherwise APIs), we must be absolutely careful
+		 * to not free the node ref or things will get loopy
+		 * down the track.
+		 *
+		 * For tx fragments, the TX code must free whatever
+		 * new references it created, but NOT the original
+		 * TX node ref that was passed in.
+		 */
+		ath_freetx(m);
+		return (ENOBUFS);
+	}
+
+	/*
+	 * Unconditionally kick the taskqueue.
+	 *
+	 * Now, there's a subtle race condition possible here if we
+	 * went down the path of only kicking the taskqueue if it
+	 * wasn't running.  If we're not absolutely, positively
+	 * careful, we could have a small race window between
+	 * finishing the taskqueue and clearing the TX flag, which
+	 * would be interpreted in _this_ context as "we don't need
+	 * to kick the TX taskqueue, as said taskqueue is already
+	 * running."
+	 *
+	 * It's a problem in some of the 1GE/10GE NIC drivers.
+	 * So until a _correct_ method for implementing this is
+	 * drafted up and written, which avoids (potentially)
+	 * large amounts of locking contention per-frame, let's
+	 * just do the inefficient "kick taskqueue each time"
+	 * method.
+	 */
 	ath_tx_kick(sc);
-	ATH_KTR(sc, ATH_KTR_TX, 0, "ath_start_queue: finished");
+
+	return (0);
 }
 
 void
@@ -2531,9 +2636,7 @@ ath_start_task(void *arg, int npending)
 	sc->sc_txstart_cnt++;
 	ATH_PCU_UNLOCK(sc);
 
-	ATH_TX_LOCK(sc);
-	ath_start(sc->sc_ifp);
-	ATH_TX_UNLOCK(sc);
+	ath_txq_qrun(ifp);
 
 	ATH_PCU_LOCK(sc);
 	sc->sc_txstart_cnt--;
@@ -2541,91 +2644,298 @@ ath_start_task(void *arg, int npending)
 	ATH_KTR(sc, ATH_KTR_TX, 0, "ath_start_task: finished");
 }
 
-void
-ath_start(struct ifnet *ifp)
+/*
+ * Pending TX buffer chain management routines.
+ */
+
+
+/*
+ * Initialise the TX queue!
+ */
+static void
+ath_txq_qinit(struct ifnet *ifp)
+{
+	struct ath_softc *sc = ifp->if_softc;
+
+	TAILQ_INIT(&sc->sc_txbuf_list);
+}
+
+/*
+ * Add this mbuf to the TX buffer chain.
+ *
+ * This allocates an ath_buf, links the mbuf into it, and
+ * appends it to the end of the TX buffer chain.
+ * It doesn't fill out the ath_buf in any way besides
+ * that.
+ *
+ * Since the mbuf may be a list of mbufs representing
+ * 802.11 fragments, handle allocating ath_bufs for each
+ * of the mbuf fragments.
+ *
+ * If we queued it, 0 is returned. Else, < 0 is returned.
+ *
+ * If <0 is returned, the sender is responsible for
+ * freeing the mbuf if appropriate.
+ */
+static int
+ath_txq_qadd(struct ifnet *ifp, struct mbuf *m0)
 {
 	struct ath_softc *sc = ifp->if_softc;
-	struct ieee80211_node *ni;
 	struct ath_buf *bf;
-	struct mbuf *m, *next;
 	ath_bufhead frags;
-	int npkts = 0;
+	struct ieee80211_node *ni;
+	struct mbuf *m;
 
-	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || sc->sc_invalid)
-		return;
+	/* XXX recursive TX completion -> TX? */
+	ATH_TX_UNLOCK_ASSERT(sc);
+
+	/*
+	 * We grab the node pointer, but we don't deref
+	 * the node.  The caller must be responsible for
+	 * freeing the node reference if it decides to
+	 * free the mbuf.
+	 */
+	ni = (struct ieee80211_node *) m0->m_pkthdr.rcvif;
+
+	ATH_TXBUF_LOCK(sc);
+	if (sc->sc_txbuf_cnt <= sc->sc_txq_data_minfree) {
+		/* XXX increment counter? */
+		ATH_TXBUF_UNLOCK(sc);
+		IF_LOCK(&ifp->if_snd);
+		ifp->if_drv_flags |= IFF_DRV_OACTIVE;
+		IF_UNLOCK(&ifp->if_snd);
+		return (-1);
+	}
+	ATH_TXBUF_UNLOCK(sc);
+
+	/*
+	 * Grab a TX buffer and associated resources.
+	 */
+	bf = ath_getbuf(sc, ATH_BUFTYPE_NORMAL);
+	if (bf == NULL) {
+		device_printf(sc->sc_dev,
+		    "%s: couldn't allocate a buffer\n",
+		    __func__);
+		return (-1);
+	}
 
-	ATH_TX_LOCK_ASSERT(sc);
+	/* Setup the initial buffer node contents */
+	bf->bf_m = m0;
+	bf->bf_node = ni;
 
-	ATH_KTR(sc, ATH_KTR_TX, 0, "ath_start: called");
+	/*
+	 * Check for fragmentation.  If this frame
+	 * has been broken up verify we have enough
+	 * buffers to send all the fragments so all
+	 * go out or none...
+	 */
+	TAILQ_INIT(&frags);
+	if (m0->m_flags & M_FRAG)
+		DPRINTF(sc, ATH_DEBUG_XMIT, "%s: txfrag\n", __func__);
+	if ((m0->m_flags & M_FRAG) &&
+	    !ath_txfrag_setup(sc, &frags, m0, ni)) {
+		DPRINTF(sc, ATH_DEBUG_XMIT,
+		    "%s: out of txfrag buffers\n", __func__);
+		sc->sc_stats.ast_tx_nofrag++;
+		ifp->if_oerrors++;
+		goto bad;
+	}
 
-	for (;;) {
+	/*
+	 * Don't stuff the non-fragment frame onto the fragment
+	 * queue. ath_txfrag_cleanup() should only be called on fragments -
+	 * ie, the _extra_ ieee80211_node references - and not the single
+	 * node reference already done as part of the net08211 TX call
+	 * into the driver.
+	 */
+
+	ATH_TX_LOCK(sc);
+
+	/*
+	 * Throw the single frame onto the queue.
+	 */
+	TAILQ_INSERT_TAIL(&sc->sc_txbuf_list, bf, bf_list);
+
+	/*
+	 * Update next packet duration length if it's a fragment.
+	 * It's needed for accurate NAV calculations (which for
+	 * fragments include the length of the NEXT fragment.)
+	 */
+	if (m0->m_nextpkt != NULL)
+		bf->bf_state.bfs_nextpktlen =
+		    m0->m_nextpkt->m_pkthdr.len;
+
+	/*
+	 * Append the fragments.  We have to populate bf and node
+	 * references here as although the txfrag setup code does
+	 * create buffers and increment the node ref, it doesn't
+	 * populate the fields for us.
+	 */
+	m = m0->m_nextpkt;
+	while ( (bf = TAILQ_FIRST(&frags)) != NULL) {
+		bf->bf_m = m;
+		bf->bf_node = ni;
+		device_printf(sc->sc_dev, "%s: adding bf=%p, m=%p, ni=%p\n",
+		    __func__,
+		    bf,
+		    bf->bf_m,
+		    bf->bf_node);
+		TAILQ_REMOVE(&frags, bf, bf_list);
+		TAILQ_INSERT_TAIL(&sc->sc_txbuf_list, bf, bf_list);
+
+		/*
+		 * For duration (NAV) calculations, we need
+		 * to know the next fragment size.
+		 *
+		 * XXX This isn't entirely accurate as it doesn't
+		 * take pad bytes and such into account, but it'll do
+		 * for fragment length / NAV calculations.
+		 */
+		if (m->m_nextpkt != NULL)
+			bf->bf_state.bfs_nextpktlen =
+			    m->m_nextpkt->m_pkthdr.len;
+
+		m = m->m_nextpkt;
+	}
+	ATH_TX_UNLOCK(sc);
+
+	return (0);
+bad:
+	device_printf(sc->sc_dev, "%s: bad?!\n", __func__);
+	bf->bf_m = NULL;
+	bf->bf_node = NULL;
+	ATH_TXBUF_LOCK(sc);
+	ath_returnbuf_head(sc, bf);
+	ath_txfrag_cleanup(sc, &frags, ni);
+	ATH_TXBUF_UNLOCK(sc);
+	return (-1);
+}
+
+/*
+ * Flush the pending TX buffer chain.
+ */
+static void
+ath_txq_qflush(struct ifnet *ifp)
+{
+	struct ath_softc *sc = ifp->if_softc;
+	ath_bufhead txlist;
+	struct ath_buf *bf;
+
+	device_printf(sc->sc_dev, "%s: called\n", __func__);
+	TAILQ_INIT(&txlist);
+
+	/* Grab lock */
+	ATH_TX_LOCK(sc);
+
+	/* Copy everything out of sc_txbuf_list into txlist */
+	TAILQ_CONCAT(&txlist, &sc->sc_txbuf_list, bf_list);
+
+	/* Unlock */
+	ATH_TX_UNLOCK(sc);
+
+	/* Now, walk the list, freeing things */
+	while ((bf = TAILQ_FIRST(&txlist)) != NULL) {
+		TAILQ_REMOVE(&txlist, bf, bf_list);
+	
+		if (bf->bf_node)
+			ieee80211_free_node(bf->bf_node);
+
+		m_free(bf->bf_m);
+
+		/* XXX paranoia! */
+		bf->bf_m = NULL;
+		bf->bf_node = NULL;
+
+		/*
+		 * XXX Perhaps do a second pass with the TXBUF lock
+		 * held and free them all at once?
+		 */
 		ATH_TXBUF_LOCK(sc);
-		if (sc->sc_txbuf_cnt <= sc->sc_txq_data_minfree) {
-			/* XXX increment counter? */
-			ATH_TXBUF_UNLOCK(sc);
-			IF_LOCK(&ifp->if_snd);
-			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
-			IF_UNLOCK(&ifp->if_snd);
-			break;
-		}
+		ath_returnbuf_head(sc, bf);
 		ATH_TXBUF_UNLOCK(sc);
-		
+	}
+}
+
+/*
+ * Walk the TX buffer queue and call ath_tx_start() on each
+ * of them.
+ */
+static void
+ath_txq_qrun(struct ifnet *ifp)
+{
+	struct ath_softc *sc = ifp->if_softc;
+	ath_bufhead txlist;
+	struct ath_buf *bf, *bf_next;
+	struct ieee80211_node *ni;
+	struct mbuf *m;
+
+	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || sc->sc_invalid)
+		return;
+
+	TAILQ_INIT(&txlist);
+
+	/*
+	 * Grab the frames to transmit from the tx queue
+	 */
+
+	/* Copy everything out of sc_txbuf_list into txlist */
+	ATH_TX_LOCK(sc);
+	TAILQ_CONCAT(&txlist, &sc->sc_txbuf_list, bf_list);
+	ATH_TX_UNLOCK(sc);
+
+	/*
+	 * For now, the ath_tx_start() code sits behind the same lock;
+	 * worry about serialising this in a taskqueue later.
+	 */
+
+	ATH_TX_LOCK(sc);
+
+	/*
+	 * Attempt to transmit each frame.
+	 *
+	 * In the old code path - if a TX fragment fails, subsequent
+	 * fragments in that group would be aborted.
+	 *
+	 * It would be nice to chain together TX fragments in this
+	 * way so they can be aborted together.
+	 */
+	TAILQ_FOREACH_SAFE(bf, &txlist, bf_list, bf_next) {
 		/*
-		 * Grab a TX buffer and associated resources.
+		 * Clear, because we're going to reuse this
+		 * as a real ath_buf now
 		 */
-		bf = ath_getbuf(sc, ATH_BUFTYPE_NORMAL);
-		if (bf == NULL)
-			break;
+		ni = bf->bf_node;
+		m = bf->bf_m;
+
+		bf->bf_node = NULL;
+		bf->bf_m = NULL;
 
-		IFQ_DEQUEUE(&ifp->if_snd, m);
-		if (m == NULL) {
-			ATH_TXBUF_LOCK(sc);
-			ath_returnbuf_head(sc, bf);
-			ATH_TXBUF_UNLOCK(sc);
-			break;
-		}
-		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
-		npkts ++;
 		/*
-		 * Check for fragmentation.  If this frame
-		 * has been broken up verify we have enough
-		 * buffers to send all the fragments so all
-		 * go out or none...
+		 * Remove it from the list.
 		 */
-		TAILQ_INIT(&frags);
-		if ((m->m_flags & M_FRAG) &&
-		    !ath_txfrag_setup(sc, &frags, m, ni)) {
-			DPRINTF(sc, ATH_DEBUG_XMIT,
-			    "%s: out of txfrag buffers\n", __func__);
-			sc->sc_stats.ast_tx_nofrag++;
-			ifp->if_oerrors++;
-			ath_freetx(m);
-			goto bad;
-		}
-		ifp->if_opackets++;
-	nextfrag:
+		TAILQ_REMOVE(&txlist, bf, bf_list);
+
 		/*
-		 * Pass the frame to the h/w for transmission.
-		 * Fragmented frames have each frag chained together
-		 * with m_nextpkt.  We know there are sufficient ath_buf's
-		 * to send all the frags because of work done by
-		 * ath_txfrag_setup.  We leave m_nextpkt set while
-		 * calling ath_tx_start so it can use it to extend the
-		 * the tx duration to cover the subsequent frag and
-		 * so it can reclaim all the mbufs in case of an error;
-		 * ath_tx_start clears m_nextpkt once it commits to
-		 * handing the frame to the hardware.
+		 * If we fail, free this buffer and go to the next one;
+		 * ath_tx_start() frees the mbuf but not the node
+		 * reference.
 		 */
-		next = m->m_nextpkt;
 		if (ath_tx_start(sc, ni, bf, m)) {
-	bad:
+			/*
+			 * XXX m is freed by ath_tx_start(); node reference
+			 * is not!
+			 */
+			DPRINTF(sc, ATH_DEBUG_XMIT,
+			    "%s: failed; bf=%p, ni=%p, m=%p\n",
+			    __func__,
+			    bf,
+			    ni,
+			    m);
 			ifp->if_oerrors++;
-	reclaim:
 			bf->bf_m = NULL;
 			bf->bf_node = NULL;
 			ATH_TXBUF_LOCK(sc);
 			ath_returnbuf_head(sc, bf);
-			ath_txfrag_cleanup(sc, &frags, ni);
 			ATH_TXBUF_UNLOCK(sc);
 			/*
 			 * XXX todo, free the node outside of
@@ -2633,37 +2943,84 @@ ath_start(struct ifnet *ifp)
 			 */
 			if (ni != NULL)
 				ieee80211_free_node(ni);
-			continue;
+		} else {
+			/*
+			 * Check here if the node is in power save state.
+			 * XXX we should hold a node ref here, and release
+			 * it after the TX has completed.
+			 */
+			ath_tx_update_tim(sc, ni, 1);
+			ifp->if_opackets++;
 		}
 
 		/*
-		 * Check here if the node is in power save state.
+		 * XXX should check for state change and flip out
+		 * if needed.
 		 */
-		ath_tx_update_tim(sc, ni, 1);
+	}
+	ATH_TX_UNLOCK(sc);
 
-		if (next != NULL) {
-			/*
-			 * Beware of state changing between frags.
-			 * XXX check sta power-save state?
-			 */
-			if (ni->ni_vap->iv_state != IEEE80211_S_RUN) {
-				DPRINTF(sc, ATH_DEBUG_XMIT,
-				    "%s: flush fragmented packet, state %s\n",
-				    __func__,
-				    ieee80211_state_name[ni->ni_vap->iv_state]);
-				ath_freetx(next);
-				goto reclaim;
-			}
-			m = next;
-			bf = TAILQ_FIRST(&frags);
-			KASSERT(bf != NULL, ("no buf for txfrag"));
-			TAILQ_REMOVE(&frags, bf, bf_list);
-			goto nextfrag;
+	/*
+	 * If we break out early (eg a state change) we should prepend these
+	 * frames onto the TX queue.
+	 */
+}
+
+/*
+ * This is now primarily used by the net80211 layer to kick-start
+ * queue processing.
+ */
+void
+ath_start(struct ifnet *ifp)
+{
+	struct mbuf *m;
+	struct ath_softc *sc = ifp->if_softc;
+	struct ieee80211_node *ni;
+	int npkts = 0;
+
+	ATH_TX_UNLOCK_ASSERT(sc);
+
+	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || sc->sc_invalid)
+		return;
+
+	/*
+	 * If we're below the free buffer limit, don't dequeue anything.
+	 * The original code would not dequeue anything from the queue
+	 * if allocating an ath_buf failed.
+	 *
+	 * For if_transmit, we have to either queue or drop the frame.
+	 * So we have to try and queue it _somewhere_.
+	 */
+	for (;;) {
+		IFQ_DEQUEUE(&ifp->if_snd, m);
+		if (m == NULL) {
+			break;
 		}
 
-		sc->sc_wd_timer = 5;
+		/*
+		 * If we do fail here, just break out for now
+		 * and wait until we've transmitted something
+		 * before we attempt again?
+		 */
+		if (ath_txq_qadd(ifp, m) < 0) {
+			DPRINTF(sc, ATH_DEBUG_XMIT,
+			    "%s: ath_txq_qadd failed\n",
+			    __func__);
+			ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
+			if (ni != NULL)
+				ieee80211_free_node(ni);
+			ath_freetx(m);
+			break;
+		}
+		npkts++;
 	}
-	ATH_KTR(sc, ATH_KTR_TX, 1, "ath_start: finished; npkts=%d", npkts);
+
+	/*
+	 * Kick the taskqueue into activity, but only if we
+	 * queued something.
+	 */
+	if (npkts > 0)
+		ath_tx_kick(sc);
 }
 
 static int
@@ -2688,6 +3045,7 @@ ath_key_update_begin(struct ieee80211vap *vap)
 
 	DPRINTF(sc, ATH_DEBUG_KEYCACHE, "%s:\n", __func__);
 	taskqueue_block(sc->sc_tq);
+	taskqueue_block(sc->sc_tx_tq);
 	IF_LOCK(&ifp->if_snd);		/* NB: doesn't block mgmt frames */
 }
 
@@ -2700,6 +3058,7 @@ ath_key_update_end(struct ieee80211vap *vap)
 	DPRINTF(sc, ATH_DEBUG_KEYCACHE, "%s:\n", __func__);
 	IF_UNLOCK(&ifp->if_snd);
 	taskqueue_unblock(sc->sc_tq);
+	taskqueue_unblock(sc->sc_tx_tq);
 }
 
 static void
@@ -3873,9 +4232,7 @@ ath_tx_processq(struct ath_softc *sc, struct ath_txq *txq, int dosched)
 
 	/* Kick the TXQ scheduler */
 	if (dosched) {
-		ATH_TX_LOCK(sc);
-		ath_txq_sched(sc, txq);
-		ATH_TX_UNLOCK(sc);
+		taskqueue_enqueue(sc->sc_tx_tq, &sc->sc_txqtask);
 	}
 
 	ATH_KTR(sc, ATH_KTR_TXCOMP, 1,
@@ -4373,6 +4730,7 @@ ath_chan_set(struct ath_softc *sc, struct ieee80211_channel *chan)
 
 	/* (Try to) stop TX/RX from occuring */
 	taskqueue_block(sc->sc_tq);
+	taskqueue_block(sc->sc_tx_tq);
 
 	ATH_PCU_LOCK(sc);
 	ath_hal_intrset(ah, 0);		/* Stop new RX/TX completion */
@@ -4719,6 +5077,7 @@ ath_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 	int i, error, stamode;
 	u_int32_t rfilt;
 	int csa_run_transition = 0;
+
 	static const HAL_LED_STATE leds[] = {
 	    HAL_LED_INIT,	/* IEEE80211_S_INIT */
 	    HAL_LED_SCAN,	/* IEEE80211_S_SCAN */
@@ -4761,6 +5120,7 @@ ath_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 		sc->sc_imask &= ~(HAL_INT_SWBA | HAL_INT_BMISS);
 		sc->sc_beacons = 0;
 		taskqueue_unblock(sc->sc_tq);
+		taskqueue_unblock(sc->sc_tx_tq);
 	}
 
 	ni = ieee80211_ref_node(vap->iv_bss);
@@ -4863,10 +5223,32 @@ ath_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 			 * force a beacon update so we pick up a lack of
 			 * beacons from an AP in CAC and thus force a
 			 * scan.
+			 *
+			 * And, there's also corner cases here where
+			 * after a scan, the AP may have disappeared.
+			 * In that case, we may not receive an actual
+			 * beacon to update the beacon timer and thus we
+			 * won't get notified of the missing beacons.
 			 */
 			sc->sc_syncbeacon = 1;
+#if 0
 			if (csa_run_transition)
+#endif
 				ath_beacon_config(sc, vap);
+
+			/*
+			 * PR: kern/175227
+			 *
+			 * Reconfigure beacons during reset; as otherwise
+			 * we won't get the beacon timers reprogrammed
+			 * after a reset and thus we won't pick up a
+			 * beacon miss interrupt.
+			 *
+			 * Hopefully we'll see a beacon before the BMISS
+			 * timer fires (too often), leading to a STA
+			 * disassociation.
+			 */
+			sc->sc_beacons = 1;
 			break;
 		case IEEE80211_M_MONITOR:
 			/*
@@ -4904,6 +5286,7 @@ ath_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 			    "%s: calibration disabled\n", __func__);
 		}
 		taskqueue_unblock(sc->sc_tq);
+		taskqueue_unblock(sc->sc_tx_tq);
 	} else if (nstate == IEEE80211_S_INIT) {
 		/*
 		 * If there are no vaps left in RUN state then
@@ -4917,6 +5300,7 @@ ath_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 			/* disable interrupts  */
 			ath_hal_intrset(ah, sc->sc_imask &~ HAL_INT_GLOBAL);
 			taskqueue_block(sc->sc_tq);
+			taskqueue_block(sc->sc_tx_tq);
 			sc->sc_beacons = 0;
 		}
 #ifdef IEEE80211_SUPPORT_TDMA
diff --git a/sys/dev/ath/if_ath_beacon.c b/sys/dev/ath/if_ath_beacon.c
index bb1e393..51467cd 100644
--- a/sys/dev/ath/if_ath_beacon.c
+++ b/sys/dev/ath/if_ath_beacon.c
@@ -705,6 +705,16 @@ ath_beacon_config(struct ath_softc *sc, struct ieee80211vap *vap)
 
 	if (vap == NULL)
 		vap = TAILQ_FIRST(&ic->ic_vaps);	/* XXX */
+	/*
+	 * Just ensure that we aren't being called when the last
+	 * VAP is destroyed.
+	 */
+	if (vap == NULL) {
+		device_printf(sc->sc_dev, "%s: called with no VAPs\n",
+		    __func__);
+		return;
+	}
+
 	ni = ieee80211_ref_node(vap->iv_bss);
 
 	/* extract tstamp from last beacon and convert to TU */
diff --git a/sys/dev/ath/if_ath_misc.h b/sys/dev/ath/if_ath_misc.h
index c3b101c..a97f897 100644
--- a/sys/dev/ath/if_ath_misc.h
+++ b/sys/dev/ath/if_ath_misc.h
@@ -124,9 +124,8 @@ static inline void
 ath_tx_kick(struct ath_softc *sc)
 {
 
-	ATH_TX_LOCK(sc);
-	ath_start(sc->sc_ifp);
-	ATH_TX_UNLOCK(sc);
+	/* XXX eventually try sc_tx_tq? */
+	taskqueue_enqueue(sc->sc_tx_tq, &sc->sc_txpkttask);
 }
 
 #endif
diff --git a/sys/dev/ath/if_ath_rx.c b/sys/dev/ath/if_ath_rx.c
index 5a39d16..c8d9536 100644
--- a/sys/dev/ath/if_ath_rx.c
+++ b/sys/dev/ath/if_ath_rx.c
@@ -431,18 +431,16 @@ ath_rx_tap(struct ifnet *ifp, struct mbuf *m,
 #ifdef AH_SUPPORT_AR5416
 	sc->sc_rx_th.wr_chan_flags &= ~CHAN_HT;
 	if (rs->rs_status & HAL_RXERR_PHY) {
-		struct ieee80211com *ic = ifp->if_l2com;
-
 		/*
 		 * PHY error - make sure the channel flags
 		 * reflect the actual channel configuration,
 		 * not the received frame.
 		 */
-		if (IEEE80211_IS_CHAN_HT40U(ic->ic_curchan))
+		if (IEEE80211_IS_CHAN_HT40U(sc->sc_curchan))
 			sc->sc_rx_th.wr_chan_flags |= CHAN_HT40U;
-		else if (IEEE80211_IS_CHAN_HT40D(ic->ic_curchan))
+		else if (IEEE80211_IS_CHAN_HT40D(sc->sc_curchan))
 			sc->sc_rx_th.wr_chan_flags |= CHAN_HT40D;
-		else if (IEEE80211_IS_CHAN_HT20(ic->ic_curchan))
+		else if (IEEE80211_IS_CHAN_HT20(sc->sc_curchan))
 			sc->sc_rx_th.wr_chan_flags |= CHAN_HT20;
 	} else if (sc->sc_rx_th.wr_rate & IEEE80211_RATE_MCS) {	/* HT rate */
 		struct ieee80211com *ic = ifp->if_l2com;
@@ -845,6 +843,7 @@ ath_rx_proc(struct ath_softc *sc, int resched)
 	int16_t nf;
 	u_int64_t tsf;
 	int npkts = 0;
+	int kickpcu = 0;
 
 	/* XXX we must not hold the ATH_LOCK here */
 	ATH_UNLOCK_ASSERT(sc);
@@ -852,6 +851,7 @@ ath_rx_proc(struct ath_softc *sc, int resched)
 
 	ATH_PCU_LOCK(sc);
 	sc->sc_rxproc_cnt++;
+	kickpcu = sc->sc_kickpcu;
 	ATH_PCU_UNLOCK(sc);
 
 	DPRINTF(sc, ATH_DEBUG_RX_PROC, "%s: called\n", __func__);
@@ -866,7 +866,7 @@ ath_rx_proc(struct ath_softc *sc, int resched)
 		 * latency can jump by quite a bit, causing throughput
 		 * degredation.
 		 */
-		if (npkts >= ATH_RX_MAX)
+		if (!kickpcu && npkts >= ATH_RX_MAX)
 			break;
 
 		bf = TAILQ_FIRST(&sc->sc_rxbuf);
@@ -961,6 +961,9 @@ rx_proc_next:
 		    __func__, npkts);
 
 		/* XXX rxslink? */
+#if 0
+		ath_startrecv(sc);
+#else
 		/*
 		 * XXX can we hold the PCU lock here?
 		 * Are there any net80211 buffer calls involved?
@@ -970,6 +973,7 @@ rx_proc_next:
 		ath_hal_rxena(ah);		/* enable recv descriptors */
 		ath_mode_init(sc);		/* set filters, etc. */
 		ath_hal_startpcurecv(ah);	/* re-enable PCU/DMA engine */
+#endif
 
 		ath_hal_intrset(ah, sc->sc_imask);
 		sc->sc_kickpcu = 0;
diff --git a/sys/dev/ath/if_ath_sysctl.c b/sys/dev/ath/if_ath_sysctl.c
index 5de29af..abb180b 100644
--- a/sys/dev/ath/if_ath_sysctl.c
+++ b/sys/dev/ath/if_ath_sysctl.c
@@ -505,6 +505,33 @@ ath_sysctl_forcebstuck(SYSCTL_HANDLER_ARGS)
 	return 0;
 }
 
+static int
+ath_sysctl_hangcheck(SYSCTL_HANDLER_ARGS)
+{
+	struct ath_softc *sc = arg1;
+	int val = 0;
+	int error;
+	uint32_t mask = 0xffffffff;
+	uint32_t *sp;
+	uint32_t rsize;
+	struct ath_hal *ah = sc->sc_ah;
+
+	error = sysctl_handle_int(oidp, &val, 0, req);
+	if (error || !req->newptr)
+		return error;
+	if (val == 0)
+		return 0;
+
+	/* Do a hang check */
+	if (!ath_hal_getdiagstate(ah, HAL_DIAG_CHECK_HANGS,
+	    &mask, sizeof(mask),
+	    (void *) &sp, &rsize))
+		return (0);
+	device_printf(sc->sc_dev, "%s: sp=0x%08x\n", __func__, *sp);
+
+	val = 0;
+	return 0;
+}
 
 #ifdef ATH_DEBUG_ALQ
 static int
@@ -661,6 +688,10 @@ ath_sysctlattach(struct ath_softc *sc)
 		"forcebstuck", CTLTYPE_INT | CTLFLAG_RW, sc, 0,
 		ath_sysctl_forcebstuck, "I", "");
 
+	SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+		"hangcheck", CTLTYPE_INT | CTLFLAG_RW, sc, 0,
+		ath_sysctl_hangcheck, "I", "");
+
 	if (ath_hal_hasintmit(ah)) {
 		SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 			"intmit", CTLTYPE_INT | CTLFLAG_RW, sc, 0,
diff --git a/sys/dev/ath/if_ath_tx.c b/sys/dev/ath/if_ath_tx.c
index 263e69c..b3b6079 100644
--- a/sys/dev/ath/if_ath_tx.c
+++ b/sys/dev/ath/if_ath_tx.c
@@ -1124,7 +1124,11 @@ ath_tx_calc_duration(struct ath_softc *sc, struct ath_buf *bf)
 			dur = rt->info[rix].lpAckDuration;
 		if (wh->i_fc[1] & IEEE80211_FC1_MORE_FRAG) {
 			dur += dur;		/* additional SIFS+ACK */
-			KASSERT(bf->bf_m->m_nextpkt != NULL, ("no fragment"));
+			if (bf->bf_state.bfs_nextpktlen == 0) {
+				device_printf(sc->sc_dev,
+				    "%s: next txfrag len=0?\n",
+				    __func__);
+			}
 			/*
 			 * Include the size of next fragment so NAV is
 			 * updated properly.  The last fragment uses only
@@ -1135,7 +1139,7 @@ ath_tx_calc_duration(struct ath_softc *sc, struct ath_buf *bf)
 			 * first fragment!
 			 */
 			dur += ath_hal_computetxtime(ah, rt,
-					bf->bf_m->m_nextpkt->m_pkthdr.len,
+					bf->bf_state.bfs_nextpktlen,
 					rix, shortPreamble);
 		}
 		if (isfrag) {
@@ -1393,12 +1397,13 @@ static void
 ath_tx_update_clrdmask(struct ath_softc *sc, struct ath_tid *tid,
     struct ath_buf *bf)
 {
+	struct ath_node *an = ATH_NODE(bf->bf_node);
 
 	ATH_TX_LOCK_ASSERT(sc);
 
-	if (tid->clrdmask == 1) {
+	if (an->clrdmask == 1) {
 		bf->bf_state.bfs_txflags |= HAL_TXDESC_CLRDMASK;
-		tid->clrdmask = 0;
+		an->clrdmask = 0;
 	}
 }
 
@@ -2884,6 +2889,29 @@ ath_tx_swq(struct ath_softc *sc, struct ieee80211_node *ni, struct ath_txq *txq,
 }
 
 /*
+ * Only set the clrdmask bit if none of the nodes are currently
+ * filtered.
+ *
+ * XXX TODO: go through all the callers and check to see
+ * which are being called in the context of looping over all
+ * TIDs (eg, if all tids are being paused, resumed, etc.)
+ * That'll avoid O(n^2) complexity here.
+ */
+static void
+ath_tx_set_clrdmask(struct ath_softc *sc, struct ath_node *an)
+{
+	int i;
+
+	ATH_TX_LOCK_ASSERT(sc);
+
+	for (i = 0; i < IEEE80211_TID_SIZE; i++) {
+		if (an->an_tid[i].isfiltered == 1)
+			return;
+	}
+	an->clrdmask = 1;
+}
+
+/*
  * Configure the per-TID node state.
  *
  * This likely belongs in if_ath_node.c but I can't think of anywhere
@@ -2914,12 +2942,12 @@ ath_tx_tid_init(struct ath_softc *sc, struct ath_node *an)
 		atid->sched = 0;
 		atid->hwq_depth = 0;
 		atid->cleanup_inprogress = 0;
-		atid->clrdmask = 1;	/* Always start by setting this bit */
 		if (i == IEEE80211_NONQOS_TID)
 			atid->ac = ATH_NONQOS_TID_AC;
 		else
 			atid->ac = TID_TO_WME_AC(i);
 	}
+	an->clrdmask = 1;	/* Always start by setting this bit */
 }
 
 /*
@@ -2945,7 +2973,6 @@ ath_tx_tid_pause(struct ath_softc *sc, struct ath_tid *tid)
 static void
 ath_tx_tid_resume(struct ath_softc *sc, struct ath_tid *tid)
 {
-
 	ATH_TX_LOCK_ASSERT(sc);
 
 	tid->paused--;
@@ -2960,7 +2987,7 @@ ath_tx_tid_resume(struct ath_softc *sc, struct ath_tid *tid)
 	 * Override the clrdmask configuration for the next frame
 	 * from this TID, just to get the ball rolling.
 	 */
-	tid->clrdmask = 1;
+	ath_tx_set_clrdmask(sc, tid->an);
 
 	if (tid->axq_depth == 0)
 		return;
@@ -2974,7 +3001,7 @@ ath_tx_tid_resume(struct ath_softc *sc, struct ath_tid *tid)
 	ath_tx_tid_sched(sc, tid);
 	/* Punt some frames to the hardware if needed */
 	//ath_txq_sched(sc, sc->sc_ac2q[tid->ac]);
-	taskqueue_enqueue(sc->sc_tq, &sc->sc_txqtask);
+	taskqueue_enqueue(sc->sc_tx_tq, &sc->sc_txqtask);
 }
 
 /*
@@ -3043,7 +3070,8 @@ ath_tx_tid_filt_comp_complete(struct ath_softc *sc, struct ath_tid *tid)
 	DPRINTF(sc, ATH_DEBUG_SW_TX_FILT, "%s: hwq=0, transition back\n",
 	    __func__);
 	tid->isfiltered = 0;
-	tid->clrdmask = 1;
+	/* XXX ath_tx_tid_resume() also calls ath_tx_set_clrdmask()! */
+	ath_tx_set_clrdmask(sc, tid->an);
 
 	/* XXX this is really quite inefficient */
 	while ((bf = ATH_TID_FILT_LAST(tid, ath_bufhead_s)) != NULL) {
@@ -3299,7 +3327,7 @@ ath_tx_tid_bar_tx(struct ath_softc *sc, struct ath_tid *tid)
 	 * Override the clrdmask configuration for the next frame,
 	 * just to get the ball rolling.
 	 */
-	tid->clrdmask = 1;
+	ath_tx_set_clrdmask(sc, tid->an);
 
 	/*
 	 * Calculate new BAW left edge, now that all frames have either
@@ -3480,7 +3508,7 @@ ath_tx_tid_drain(struct ath_softc *sc, struct ath_node *an,
 	 *
 	 * This won't hurt things if the TID is about to be freed.
 	 */
-	tid->clrdmask = 1;
+	ath_tx_set_clrdmask(sc, tid->an);
 
 	/*
 	 * Now that it's completed, grab the TID lock and update
diff --git a/sys/dev/ath/if_ath_tx_edma.c b/sys/dev/ath/if_ath_tx_edma.c
index 40a0378..a2eef65 100644
--- a/sys/dev/ath/if_ath_tx_edma.c
+++ b/sys/dev/ath/if_ath_tx_edma.c
@@ -655,7 +655,7 @@ ath_edma_tx_processq(struct ath_softc *sc, int dosched)
 	 * the txq task for _one_ TXQ.  This should be fixed.
 	 */
 	if (dosched)
-		taskqueue_enqueue(sc->sc_tq, &sc->sc_txqtask);
+		taskqueue_enqueue(sc->sc_tx_tq, &sc->sc_txqtask);
 }
 
 static void
diff --git a/sys/dev/ath/if_athvar.h b/sys/dev/ath/if_athvar.h
index 2e170a3..070fe53 100644
--- a/sys/dev/ath/if_athvar.h
+++ b/sys/dev/ath/if_athvar.h
@@ -132,7 +132,6 @@ struct ath_tid {
 	int			bar_wait;	/* waiting for BAR */
 	int			bar_tx;		/* BAR TXed */
 	int			isfiltered;	/* is this node currently filtered */
-	int			clrdmask;	/* has clrdmask been set */
 
 	/*
 	 * Is the TID being cleaned up after a transition
@@ -182,6 +181,7 @@ struct ath_node {
 	struct mtx	an_mtx;		/* protecting the ath_node state */
 	uint32_t	an_swq_depth;	/* how many SWQ packets for this
 					   node */
+	int			clrdmask;	/* has clrdmask been set */
 	/* variable-length rate control state follows */
 };
 #define	ATH_NODE(ni)	((struct ath_node *)(ni))
@@ -279,6 +279,8 @@ struct ath_buf {
 		int32_t bfs_keyix;		/* crypto key index */
 		int32_t bfs_txantenna;	/* TX antenna config */
 
+		uint16_t bfs_nextpktlen;	/* length of next frag pkt */
+
 		/* Make this an 8 bit value? */
 		enum ieee80211_protmode bfs_protmode;
 
@@ -494,6 +496,13 @@ struct ath_softc {
 	struct ath_tx_methods	sc_tx;
 	struct ath_tx_edma_fifo	sc_txedma[HAL_NUM_TX_QUEUES];
 
+	/*
+	 * This is (currently) protected by the TX queue lock;
+	 * it should migrate to a separate lock later
+	 * so as to minimise contention.
+	 */
+	ath_bufhead		sc_txbuf_list;
+
 	int			sc_rx_statuslen;
 	int			sc_tx_desclen;
 	int			sc_tx_statuslen;
@@ -514,6 +523,7 @@ struct ath_softc {
 	struct mtx		sc_tx_mtx;	/* TX access mutex */
 	char			sc_tx_mtx_name[32];
 	struct taskqueue	*sc_tq;		/* private task queue */
+	struct taskqueue	*sc_tx_tq;	/* private TX task queue */
 	struct ath_hal		*sc_ah;		/* Atheros HAL */
 	struct ath_ratectrl	*sc_rc;		/* tx rate control support */
 	struct ath_tx99		*sc_tx99;	/* tx99 adjunct state */
@@ -660,6 +670,7 @@ struct ath_softc {
 	struct ath_txq		*sc_ac2q[5];	/* WME AC -> h/w q map */ 
 	struct task		sc_txtask;	/* tx int processing */
 	struct task		sc_txqtask;	/* tx proc processing */
+	struct task		sc_txpkttask;	/* tx frame processing */
 
 	struct ath_descdma	sc_txcompdma;	/* TX EDMA completion */
 	struct mtx		sc_txcomplock;	/* TX EDMA completion lock */
diff --git a/sys/dev/atkbdc/atkbd.c b/sys/dev/atkbdc/atkbd.c
index e58ffc7..199fcb1 100644
--- a/sys/dev/atkbdc/atkbd.c
+++ b/sys/dev/atkbdc/atkbd.c
@@ -66,7 +66,7 @@ static timeout_t	atkbd_timeout;
 static void		atkbd_shutdown_final(void *v);
 
 int
-atkbd_probe_unit(int unit, int ctlr, int irq, int flags)
+atkbd_probe_unit(device_t dev, int irq, int flags)
 {
 	keyboard_switch_t *sw;
 	int args[2];
@@ -76,27 +76,29 @@ atkbd_probe_unit(int unit, int ctlr, int irq, int flags)
 	if (sw == NULL)
 		return ENXIO;
 
-	args[0] = ctlr;
+	args[0] = device_get_unit(device_get_parent(dev));
 	args[1] = irq;
-	error = (*sw->probe)(unit, args, flags);
+	error = (*sw->probe)(device_get_unit(dev), args, flags);
 	if (error)
 		return error;
 	return 0;
 }
 
 int
-atkbd_attach_unit(int unit, keyboard_t **kbd, int ctlr, int irq, int flags)
+atkbd_attach_unit(device_t dev, keyboard_t **kbd, int irq, int flags)
 {
 	keyboard_switch_t *sw;
 	int args[2];
 	int error;
+	int unit;
 
 	sw = kbd_get_switch(ATKBD_DRIVER_NAME);
 	if (sw == NULL)
 		return ENXIO;
 
 	/* reset, initialize and enable the device */
-	args[0] = ctlr;
+	unit = device_get_unit(dev);
+	args[0] = device_get_unit(device_get_parent(dev));
 	args[1] = irq;
 	*kbd = NULL;
 	error = (*sw->probe)(unit, args, flags);
@@ -401,7 +403,7 @@ atkbd_init(int unit, keyboard_t **kbdp, void *arg, int flags)
 		bcopy(&key_map, keymap, sizeof(key_map));
 		bcopy(&accent_map, accmap, sizeof(accent_map));
 		bcopy(fkey_tab, fkeymap,
-		      imin(fkeymap_size*sizeof(fkeymap[0]), sizeof(fkey_tab)));
+		    imin(fkeymap_size * sizeof(fkeymap[0]), sizeof(fkey_tab)));
 		kbd_set_maps(kbd, keymap, accmap, fkeymap, fkeymap_size);
 		kbd->kb_data = (void *)state;
 	
@@ -424,8 +426,8 @@ atkbd_init(int unit, keyboard_t **kbdp, void *arg, int flags)
 	if (!KBD_IS_INITIALIZED(kbd) && !(flags & KB_CONF_PROBE_ONLY)) {
 		kbd->kb_config = flags & ~KB_CONF_PROBE_ONLY;
 		if (KBD_HAS_DEVICE(kbd)
-	    	    && init_keyboard(state->kbdc, &kbd->kb_type, kbd->kb_config)
-	    	    && (kbd->kb_config & KB_CONF_FAIL_IF_NO_KBD)) {
+		    && init_keyboard(state->kbdc, &kbd->kb_type, kbd->kb_config)
+		    && (kbd->kb_config & KB_CONF_FAIL_IF_NO_KBD)) {
 			kbd_unregister(kbd);
 			error = ENXIO;
 			goto bad;
@@ -485,8 +487,7 @@ atkbd_intr(keyboard_t *kbd, void *arg)
 		 * The keyboard was not detected before;
 		 * it must have been reconnected!
 		 */
-		init_keyboard(state->kbdc, &kbd->kb_type,
-			      kbd->kb_config);
+		init_keyboard(state->kbdc, &kbd->kb_type, kbd->kb_config);
 		KBD_FOUND_DEVICE(kbd);
 		atkbd_ioctl(kbd, KDSETLED, (caddr_t)&state->ks_state);
 		get_typematic(kbd);
@@ -645,7 +646,7 @@ next_code:
 			goto next_code;
 		}
 		break;
-	case 0xE0:      /* 0xE0 prefix */
+	case 0xE0:		/* 0xE0 prefix */
 		state->ks_prefix = 0;
 		switch (keycode) {
 		case 0x1C:	/* right enter key */
@@ -655,57 +656,57 @@ next_code:
 			keycode = 0x5A;
 			break;
 		case 0x35:	/* keypad divide key */
-	    		keycode = 0x5B;
-	    		break;
+			keycode = 0x5B;
+			break;
 		case 0x37:	/* print scrn key */
-	    		keycode = 0x5C;
-	    		break;
+			keycode = 0x5C;
+			break;
 		case 0x38:	/* right alt key (alt gr) */
-	    		keycode = 0x5D;
-	    		break;
+			keycode = 0x5D;
+			break;
 		case 0x46:	/* ctrl-pause/break on AT 101 (see below) */
 			keycode = 0x68;
-	    		break;
+			break;
 		case 0x47:	/* grey home key */
-	    		keycode = 0x5E;
-	    		break;
+			keycode = 0x5E;
+			break;
 		case 0x48:	/* grey up arrow key */
-	    		keycode = 0x5F;
-	    		break;
+			keycode = 0x5F;
+			break;
 		case 0x49:	/* grey page up key */
-	    		keycode = 0x60;
-	    		break;
+			keycode = 0x60;
+			break;
 		case 0x4B:	/* grey left arrow key */
-	    		keycode = 0x61;
-	    		break;
+			keycode = 0x61;
+			break;
 		case 0x4D:	/* grey right arrow key */
-	    		keycode = 0x62;
-	    		break;
+			keycode = 0x62;
+			break;
 		case 0x4F:	/* grey end key */
-	    		keycode = 0x63;
-	    		break;
+			keycode = 0x63;
+			break;
 		case 0x50:	/* grey down arrow key */
-	    		keycode = 0x64;
-	    		break;
+			keycode = 0x64;
+			break;
 		case 0x51:	/* grey page down key */
-	    		keycode = 0x65;
-	    		break;
+			keycode = 0x65;
+			break;
 		case 0x52:	/* grey insert key */
-	    		keycode = 0x66;
-	    		break;
+			keycode = 0x66;
+			break;
 		case 0x53:	/* grey delete key */
-	    		keycode = 0x67;
-	    		break;
-		/* the following 3 are only used on the MS "Natural" keyboard */
+			keycode = 0x67;
+			break;
+			/* the following 3 are only used on the MS "Natural" keyboard */
 		case 0x5b:	/* left Window key */
-	    		keycode = 0x69;
-	    		break;
+			keycode = 0x69;
+			break;
 		case 0x5c:	/* right Window key */
-	    		keycode = 0x6a;
-	    		break;
+			keycode = 0x6a;
+			break;
 		case 0x5d:	/* menu key */
-	    		keycode = 0x6b;
-	    		break;
+			keycode = 0x6b;
+			break;
 		case 0x5e:	/* power key */
 			keycode = 0x6d;
 			break;
@@ -716,10 +717,10 @@ next_code:
 			keycode = 0x6f;
 			break;
 		default:	/* ignore everything else */
-	    		goto next_code;
+			goto next_code;
 		}
 		break;
-    	case 0xE1:	/* 0xE1 prefix */
+   	case 0xE1:	/* 0xE1 prefix */
 		/* 
 		 * The pause/break key on the 101 keyboard produces:
 		 * E1-1D-45 E1-9D-C5
@@ -728,10 +729,10 @@ next_code:
 		 */
 		state->ks_prefix = 0;
 		if (keycode == 0x1D)
-	    		state->ks_prefix = 0x1D;
+			state->ks_prefix = 0x1D;
 		goto next_code;
 		/* NOT REACHED */
-    	case 0x1D:	/* pause / break */
+   	case 0x1D:	/* pause / break */
 		state->ks_prefix = 0;
 		if (keycode != 0x45)
 			goto next_code;
@@ -743,7 +744,7 @@ next_code:
 		switch (keycode) {
 		case 0x37:	/* *(numpad)/print screen */
 			if (state->ks_flags & SHIFTS)
-	    			keycode = 0x5c;	/* print screen */
+				keycode = 0x5c;	/* print screen */
 			break;
 		case 0x45:	/* num lock/pause */
 			if (state->ks_flags & CTLS)
@@ -1177,7 +1178,7 @@ get_kbd_echo(KBDC kbdc)
 		 */
 		return ENXIO;
 	}
-    
+
 	return 0;
 }
 
@@ -1275,7 +1276,7 @@ init_keyboard(KBDC kbdc, int *type, int flags)
 	}
 	if (bootverbose)
 		printf("atkbd: the current kbd controller command byte %04x\n",
-		       c);
+		   c);
 #if 0
 	/* override the keyboard lock switch */
 	c |= KBD_OVERRIDE_KBD_LOCK;
@@ -1415,52 +1416,49 @@ init_keyboard(KBDC kbdc, int *type, int flags)
 static int
 write_kbd(KBDC kbdc, int command, int data)
 {
-    int s;
+	int s;
 
-    /* prevent the timeout routine from polling the keyboard */
-    if (!kbdc_lock(kbdc, TRUE)) 
-	return EBUSY;
+	/* prevent the timeout routine from polling the keyboard */
+	if (!kbdc_lock(kbdc, TRUE)) 
+		return EBUSY;
 
-    /* disable the keyboard and mouse interrupt */
-    s = spltty();
+	/* disable the keyboard and mouse interrupt */
+	s = spltty();
 #if 0
-    c = get_controller_command_byte(kbdc);
-    if ((c == -1) 
-	|| !set_controller_command_byte(kbdc, 
-            kbdc_get_device_mask(kbdc),
-            KBD_DISABLE_KBD_PORT | KBD_DISABLE_KBD_INT
-                | KBD_DISABLE_AUX_PORT | KBD_DISABLE_AUX_INT)) {
-	/* CONTROLLER ERROR */
-        kbdc_lock(kbdc, FALSE);
+	c = get_controller_command_byte(kbdc);
+	if ((c == -1) 
+	    || !set_controller_command_byte(kbdc, 
+		kbdc_get_device_mask(kbdc),
+		KBD_DISABLE_KBD_PORT | KBD_DISABLE_KBD_INT
+		| KBD_DISABLE_AUX_PORT | KBD_DISABLE_AUX_INT)) {
+		/* CONTROLLER ERROR */
+		kbdc_lock(kbdc, FALSE);
+		splx(s);
+		return EIO;
+	}
+	/* 
+	 * Now that the keyboard controller is told not to generate 
+	 * the keyboard and mouse interrupts, call `splx()' to allow 
+	 * the other tty interrupts. The clock interrupt may also occur, 
+	 * but the timeout routine (`scrn_timer()') will be blocked 
+	 * by the lock flag set via `kbdc_lock()'
+	 */
 	splx(s);
-	return EIO;
-    }
-    /* 
-     * Now that the keyboard controller is told not to generate 
-     * the keyboard and mouse interrupts, call `splx()' to allow 
-     * the other tty interrupts. The clock interrupt may also occur, 
-     * but the timeout routine (`scrn_timer()') will be blocked 
-     * by the lock flag set via `kbdc_lock()'
-     */
-    splx(s);
 #endif
-
-    if (send_kbd_command_and_data(kbdc, command, data) != KBD_ACK)
-        send_kbd_command(kbdc, KBDC_ENABLE_KBD);
-
+	if (send_kbd_command_and_data(kbdc, command, data) != KBD_ACK)
+		send_kbd_command(kbdc, KBDC_ENABLE_KBD);
 #if 0
-    /* restore the interrupts */
-    if (!set_controller_command_byte(kbdc,
-            kbdc_get_device_mask(kbdc),
+	/* restore the interrupts */
+	if (!set_controller_command_byte(kbdc, kbdc_get_device_mask(kbdc),
 	    c & (KBD_KBD_CONTROL_BITS | KBD_AUX_CONTROL_BITS))) { 
-	/* CONTROLLER ERROR */
-    }
+		/* CONTROLLER ERROR */
+	}
 #else
-    splx(s);
+	splx(s);
 #endif
-    kbdc_lock(kbdc, FALSE);
+	kbdc_lock(kbdc, FALSE);
 
-    return 0;
+	return 0;
 }
 
 static int
diff --git a/sys/dev/atkbdc/atkbd_atkbdc.c b/sys/dev/atkbdc/atkbd_atkbdc.c
index 8181820..dd0a9d6 100644
--- a/sys/dev/atkbdc/atkbd_atkbdc.c
+++ b/sys/dev/atkbdc/atkbd_atkbdc.c
@@ -104,9 +104,7 @@ atkbdprobe(device_t dev)
 	bus_release_resource(dev, SYS_RES_IRQ, rid, res);
 
 	/* probe the device */
-	return atkbd_probe_unit(device_get_unit(dev),
-				device_get_unit(device_get_parent(dev)),
-				irq, flags);
+	return atkbd_probe_unit(dev, irq, flags);
 }
 
 static int
@@ -124,9 +122,7 @@ atkbdattach(device_t dev)
 	rid = KBDC_RID_KBD;
 	irq = bus_get_resource_start(dev, SYS_RES_IRQ, rid);
 	flags = device_get_flags(dev);
-	error = atkbd_attach_unit(device_get_unit(dev), &kbd,
-				  device_get_unit(device_get_parent(dev)),
-				  irq, flags);
+	error = atkbd_attach_unit(dev, &kbd, irq, flags);
 	if (error)
 		return error;
 
diff --git a/sys/dev/atkbdc/atkbdreg.h b/sys/dev/atkbdc/atkbdreg.h
index cf7ee6b..3d09e44 100644
--- a/sys/dev/atkbdc/atkbdreg.h
+++ b/sys/dev/atkbdc/atkbdreg.h
@@ -39,9 +39,8 @@
 
 #ifdef _KERNEL
 
-int		atkbd_probe_unit(int unit, int ctlr, int irq, int flags);
-int		atkbd_attach_unit(int unit, keyboard_t **kbd,
-				 int ctlr, int irq, int flags);
+int		atkbd_probe_unit(device_t dev, int irq, int flags);
+int		atkbd_attach_unit(device_t dev, keyboard_t **kbd, int irq, int flags);
 
 #endif
 
diff --git a/sys/dev/auxio/auxio.c b/sys/dev/auxio/auxio.c
index 35fd307..b104d5a 100644
--- a/sys/dev/auxio/auxio.c
+++ b/sys/dev/auxio/auxio.c
@@ -131,7 +131,8 @@ static device_method_t auxio_sbus_methods[] = {
 	DEVMETHOD(device_probe,		auxio_bus_probe),
 	DEVMETHOD(device_attach,	auxio_sbus_attach),
 	DEVMETHOD(device_detach,	auxio_bus_detach),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t auxio_sbus_driver = {
@@ -151,7 +152,8 @@ static device_method_t auxio_ebus_methods[] = {
 	DEVMETHOD(device_probe,		auxio_bus_probe),
 	DEVMETHOD(device_attach,	auxio_ebus_attach),
 	DEVMETHOD(device_detach,	auxio_bus_detach),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t auxio_ebus_driver = {
diff --git a/sys/dev/bge/if_bge.c b/sys/dev/bge/if_bge.c
index e7cb2f7..db86ec2 100644
--- a/sys/dev/bge/if_bge.c
+++ b/sys/dev/bge/if_bge.c
@@ -35,10 +35,10 @@
 __FBSDID("$FreeBSD$");
 
 /*
- * Broadcom BCM570x family gigabit ethernet driver for FreeBSD.
+ * Broadcom BCM57xx(x)/BCM590x NetXtreme and NetLink family Ethernet driver
  *
  * The Broadcom BCM5700 is based on technology originally developed by
- * Alteon Networks as part of the Tigon I and Tigon II gigabit ethernet
+ * Alteon Networks as part of the Tigon I and Tigon II Gigabit Ethernet
  * MAC chips. The BCM5700, sometimes referred to as the Tigon III, has
  * two on-board MIPS R4000 CPUs and can have as much as 16MB of external
  * SSRAM. The BCM5700 supports TCP, UDP and IP checksum offload, jumbo
@@ -367,8 +367,9 @@ static const struct bge_revision bge_majorrevs[] = {
 #define	BGE_IS_5717_PLUS(sc)		((sc)->bge_flags & BGE_FLAG_5717_PLUS)
 #define	BGE_IS_57765_PLUS(sc)		((sc)->bge_flags & BGE_FLAG_57765_PLUS)
 
-const struct bge_revision * bge_lookup_rev(uint32_t);
-const struct bge_vendor * bge_lookup_vendor(uint16_t);
+static uint32_t bge_chipid(device_t);
+static const struct bge_vendor * bge_lookup_vendor(uint16_t);
+static const struct bge_revision * bge_lookup_rev(uint32_t);
 
 typedef int	(*bge_eaddr_fcn_t)(struct bge_softc *, uint8_t[]);
 
@@ -1916,7 +1917,7 @@ bge_chipinit(struct bge_softc *sc)
 	PCI_CLRBIT(sc->bge_dev, BGE_PCI_CMD,
 	    PCIM_CMD_INTxDIS | PCIM_CMD_MWIEN, 4);
 
-	/* Set the timer prescaler (always 66Mhz) */
+	/* Set the timer prescaler (always 66 MHz). */
 	CSR_WRITE_4(sc, BGE_MISC_CFG, BGE_32BITTIME_66MHZ);
 
 	/* XXX: The Linux tg3 driver does this at the start of brgphy_reset. */
@@ -2586,7 +2587,7 @@ bge_blockinit(struct bge_softc *sc)
 	return (0);
 }
 
-const struct bge_revision *
+static const struct bge_revision *
 bge_lookup_rev(uint32_t chipid)
 {
 	const struct bge_revision *br;
@@ -2604,7 +2605,7 @@ bge_lookup_rev(uint32_t chipid)
 	return (NULL);
 }
 
-const struct bge_vendor *
+static const struct bge_vendor *
 bge_lookup_vendor(uint16_t vid)
 {
 	const struct bge_vendor *v;
@@ -2613,10 +2614,47 @@ bge_lookup_vendor(uint16_t vid)
 		if (v->v_id == vid)
 			return (v);
 
-	panic("%s: unknown vendor %d", __func__, vid);
 	return (NULL);
 }
 
+static uint32_t
+bge_chipid(device_t dev)
+{
+	uint32_t id;
+
+	id = pci_read_config(dev, BGE_PCI_MISC_CTL, 4) >>
+	    BGE_PCIMISCCTL_ASICREV_SHIFT;
+	if (BGE_ASICREV(id) == BGE_ASICREV_USE_PRODID_REG) {
+		/*
+		 * Find the ASCI revision.  Different chips use different
+		 * registers.
+		 */
+		switch (pci_get_device(dev)) {
+		case BCOM_DEVICEID_BCM5717:
+		case BCOM_DEVICEID_BCM5718:
+		case BCOM_DEVICEID_BCM5719:
+		case BCOM_DEVICEID_BCM5720:
+			id = pci_read_config(dev,
+			    BGE_PCI_GEN2_PRODID_ASICREV, 4);
+			break;
+		case BCOM_DEVICEID_BCM57761:
+		case BCOM_DEVICEID_BCM57762:
+		case BCOM_DEVICEID_BCM57765:
+		case BCOM_DEVICEID_BCM57766:
+		case BCOM_DEVICEID_BCM57781:
+		case BCOM_DEVICEID_BCM57785:
+		case BCOM_DEVICEID_BCM57791:
+		case BCOM_DEVICEID_BCM57795:
+			id = pci_read_config(dev,
+			    BGE_PCI_GEN15_PRODID_ASICREV, 4);
+			break;
+		default:
+			id = pci_read_config(dev, BGE_PCI_PRODID_ASICREV, 4);
+		}
+	}
+	return (id);
+}
+
 /*
  * Probe for a Broadcom chip. Check the PCI vendor and device IDs
  * against our list and return its name if we find a match.
@@ -2634,61 +2672,34 @@ bge_probe(device_t dev)
 	char model[64];
 	const struct bge_revision *br;
 	const char *pname;
-	struct bge_softc *sc = device_get_softc(dev);
+	struct bge_softc *sc;
 	const struct bge_type *t = bge_devs;
 	const struct bge_vendor *v;
 	uint32_t id;
 	uint16_t did, vid;
 
+	sc = device_get_softc(dev);
 	sc->bge_dev = dev;
 	vid = pci_get_vendor(dev);
 	did = pci_get_device(dev);
 	while(t->bge_vid != 0) {
 		if ((vid == t->bge_vid) && (did == t->bge_did)) {
-			id = pci_read_config(dev, BGE_PCI_MISC_CTL, 4) >>
-			    BGE_PCIMISCCTL_ASICREV_SHIFT;
-			if (BGE_ASICREV(id) == BGE_ASICREV_USE_PRODID_REG) {
-				/*
-				 * Find the ASCI revision.  Different chips
-				 * use different registers.
-				 */
-				switch (pci_get_device(dev)) {
-				case BCOM_DEVICEID_BCM5717:
-				case BCOM_DEVICEID_BCM5718:
-				case BCOM_DEVICEID_BCM5719:
-				case BCOM_DEVICEID_BCM5720:
-					id = pci_read_config(dev,
-					    BGE_PCI_GEN2_PRODID_ASICREV, 4);
-					break;
-				case BCOM_DEVICEID_BCM57761:
-				case BCOM_DEVICEID_BCM57762:
-				case BCOM_DEVICEID_BCM57765:
-				case BCOM_DEVICEID_BCM57766:
-				case BCOM_DEVICEID_BCM57781:
-				case BCOM_DEVICEID_BCM57785:
-				case BCOM_DEVICEID_BCM57791:
-				case BCOM_DEVICEID_BCM57795:
-					id = pci_read_config(dev,
-					    BGE_PCI_GEN15_PRODID_ASICREV, 4);
-					break;
-				default:
-					id = pci_read_config(dev,
-					    BGE_PCI_PRODID_ASICREV, 4);
-				}
-			}
+			id = bge_chipid(dev);
 			br = bge_lookup_rev(id);
-			v = bge_lookup_vendor(vid);
 			if (bge_has_eaddr(sc) &&
 			    pci_get_vpd_ident(dev, &pname) == 0)
-				snprintf(model, 64, "%s", pname);
-			else
-				snprintf(model, 64, "%s %s", v->v_name,
+				snprintf(model, sizeof(model), "%s", pname);
+			else {
+				v = bge_lookup_vendor(vid);
+				snprintf(model, sizeof(model), "%s %s",
+				    v != NULL ? v->v_name : "Unknown",
 				    br != NULL ? br->br_name :
-				    "NetXtreme Ethernet Controller");
-			snprintf(buf, 96, "%s, %sASIC rev. %#08x", model,
-			    br != NULL ? "" : "unknown ", id);
+				    "NetXtreme/NetLink Ethernet Controller");
+			}
+			snprintf(buf, sizeof(buf), "%s, %sASIC rev. %#08x",
+			    model, br != NULL ? "" : "unknown ", id);
 			device_set_desc_copy(dev, buf);
-			return (0);
+			return (BUS_PROBE_DEFAULT);
 		}
 		t++;
 	}
@@ -3247,38 +3258,7 @@ bge_attach(device_t dev)
 
 	/* Save various chip information. */
 	sc->bge_func_addr = pci_get_function(dev);
-	sc->bge_chipid =
-	    pci_read_config(dev, BGE_PCI_MISC_CTL, 4) >>
-	    BGE_PCIMISCCTL_ASICREV_SHIFT;
-	if (BGE_ASICREV(sc->bge_chipid) == BGE_ASICREV_USE_PRODID_REG) {
-		/*
-		 * Find the ASCI revision.  Different chips use different
-		 * registers.
-		 */
-		switch (pci_get_device(dev)) {
-		case BCOM_DEVICEID_BCM5717:
-		case BCOM_DEVICEID_BCM5718:
-		case BCOM_DEVICEID_BCM5719:
-		case BCOM_DEVICEID_BCM5720:
-			sc->bge_chipid = pci_read_config(dev,
-			    BGE_PCI_GEN2_PRODID_ASICREV, 4);
-			break;
-		case BCOM_DEVICEID_BCM57761:
-		case BCOM_DEVICEID_BCM57762:
-		case BCOM_DEVICEID_BCM57765:
-		case BCOM_DEVICEID_BCM57766:
-		case BCOM_DEVICEID_BCM57781:
-		case BCOM_DEVICEID_BCM57785:
-		case BCOM_DEVICEID_BCM57791:
-		case BCOM_DEVICEID_BCM57795:
-			sc->bge_chipid = pci_read_config(dev,
-			    BGE_PCI_GEN15_PRODID_ASICREV, 4);
-			break;
-		default:
-			sc->bge_chipid = pci_read_config(dev,
-			    BGE_PCI_PRODID_ASICREV, 4);
-		}
-	}
+	sc->bge_chipid = bge_chipid(dev);
 	sc->bge_asicrev = BGE_ASICREV(sc->bge_chipid);
 	sc->bge_chiprev = BGE_CHIPREV(sc->bge_chipid);
 
@@ -3468,6 +3448,8 @@ bge_attach(device_t dev)
 	    pci_get_device(dev) == BCOM_DEVICEID_BCM5753F ||
 	    pci_get_device(dev) == BCOM_DEVICEID_BCM5787F)) ||
 	    pci_get_device(dev) == BCOM_DEVICEID_BCM57790 ||
+	    pci_get_device(dev) == BCOM_DEVICEID_BCM57791 ||
+	    pci_get_device(dev) == BCOM_DEVICEID_BCM57795 ||
 	    sc->bge_asicrev == BGE_ASICREV_BCM5906) {
 		/* These chips are 10/100 only. */
 		capmask &= ~BMSR_EXTSTAT;
@@ -3479,8 +3461,8 @@ bge_attach(device_t dev)
 	 * TSO. But the firmware is not available to FreeBSD and Linux
 	 * claims that the TSO performed by the firmware is slower than
 	 * hardware based TSO. Moreover the firmware based TSO has one
-	 * known bug which can't handle TSO if ethernet header + IP/TCP
-	 * header is greater than 80 bytes. The workaround for the TSO
+	 * known bug which can't handle TSO if Ethernet header + IP/TCP
+	 * header is greater than 80 bytes. A workaround for the TSO
 	 * bug exist but it seems it's too expensive than not using
 	 * TSO at all. Some hardwares also have the TSO bug so limit
 	 * the TSO to the controllers that are not affected TSO issues
@@ -3853,8 +3835,13 @@ again:
 			error = ENOMEM;
 			goto fail;
 		}
-		taskqueue_start_threads(&sc->bge_tq, 1, PI_NET, "%s taskq",
-		    device_get_nameunit(sc->bge_dev));
+		error = taskqueue_start_threads(&sc->bge_tq, 1, PI_NET,
+		    "%s taskq", device_get_nameunit(sc->bge_dev));
+		if (error != 0) {
+			device_printf(dev, "could not start threads.\n");
+			ether_ifdetach(ifp);
+			goto fail;
+		}
 		error = bus_setup_intr(dev, sc->bge_irq,
 		    INTR_TYPE_NET | INTR_MPSAFE, bge_msi_intr, NULL, sc,
 		    &sc->bge_intrhand);
@@ -3899,9 +3886,9 @@ bge_detach(device_t dev)
 	if (sc->bge_tq)
 		taskqueue_drain(sc->bge_tq, &sc->bge_intr_task);
 
-	if (sc->bge_flags & BGE_FLAG_TBI) {
+	if (sc->bge_flags & BGE_FLAG_TBI)
 		ifmedia_removeall(&sc->bge_ifmedia);
-	} else {
+	else if (sc->bge_miibus != NULL) {
 		bus_generic_detach(dev);
 		device_delete_child(dev, sc->bge_miibus);
 	}
@@ -5427,7 +5414,7 @@ bge_init_locked(struct bge_softc *sc)
 	 * this number of frames, it will drop subsequent incoming
 	 * frames until the MBUF High Watermark is reached.
 	 */
-	if (sc->bge_asicrev == BGE_ASICREV_BCM57765)
+	if (BGE_IS_57765_PLUS(sc))
 		CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 1);
 	else
 		CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 2);
diff --git a/sys/dev/bktr/CHANGELOG.TXT b/sys/dev/bktr/CHANGELOG.TXT
index b885e15..b885e15 100755..100644
--- a/sys/dev/bktr/CHANGELOG.TXT
+++ b/sys/dev/bktr/CHANGELOG.TXT
diff --git a/sys/dev/bvm/bvm_console.c b/sys/dev/bvm/bvm_console.c
new file mode 100644
index 0000000..a0e70e5
--- /dev/null
+++ b/sys/dev/bvm/bvm_console.c
@@ -0,0 +1,240 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/types.h>
+#include <sys/cons.h>
+#include <sys/tty.h>
+#include <sys/reboot.h>
+#include <sys/bus.h>
+
+#include <sys/kdb.h>
+#include <ddb/ddb.h>
+
+#ifndef	BVMCONS_POLL_HZ
+#define	BVMCONS_POLL_HZ	4
+#endif
+#define BVMBURSTLEN	16	/* max number of bytes to write in one chunk */
+
+static tsw_open_t bvm_tty_open;
+static tsw_close_t bvm_tty_close;
+static tsw_outwakeup_t bvm_tty_outwakeup;
+
+static struct ttydevsw bvm_ttydevsw = {
+	.tsw_flags	= TF_NOPREFIX,
+	.tsw_open	= bvm_tty_open,
+	.tsw_close	= bvm_tty_close,
+	.tsw_outwakeup	= bvm_tty_outwakeup,
+};
+
+static int			polltime;
+static struct callout_handle	bvm_timeouthandle
+    = CALLOUT_HANDLE_INITIALIZER(&bvm_timeouthandle);
+
+#if defined(KDB)
+static int			alt_break_state;
+#endif
+
+#define	BVM_CONS_PORT	0x220
+static int bvm_cons_port = BVM_CONS_PORT;
+
+#define BVM_CONS_SIG	('b' << 8 | 'v')
+
+static void	bvm_timeout(void *);
+
+static cn_probe_t	bvm_cnprobe;
+static cn_init_t	bvm_cninit;
+static cn_term_t	bvm_cnterm;
+static cn_getc_t	bvm_cngetc;
+static cn_putc_t	bvm_cnputc;
+static cn_grab_t 	bvm_cngrab;
+static cn_ungrab_t 	bvm_cnungrab;
+
+CONSOLE_DRIVER(bvm);
+
+static int
+bvm_rcons(u_char *ch)
+{
+	int c;
+
+	c = inl(bvm_cons_port);
+	if (c != -1) {
+		*ch = (u_char)c;
+		return (0);
+	} else
+		return (-1);
+}
+
+static void
+bvm_wcons(u_char ch)
+{
+
+	outl(bvm_cons_port, ch);
+}
+
+static void
+cn_drvinit(void *unused)
+{
+	struct tty *tp;
+
+	if (bvm_consdev.cn_pri != CN_DEAD &&
+	    bvm_consdev.cn_name[0] != '\0') {
+		tp = tty_alloc(&bvm_ttydevsw, NULL);
+		tty_makedev(tp, NULL, "bvmcons");
+	}
+}
+
+static int
+bvm_tty_open(struct tty *tp)
+{
+	polltime = hz / BVMCONS_POLL_HZ;
+	if (polltime < 1)
+		polltime = 1;
+	bvm_timeouthandle = timeout(bvm_timeout, tp, polltime);
+
+	return (0);
+}
+
+static void
+bvm_tty_close(struct tty *tp)
+{
+
+	/* XXX Should be replaced with callout_stop(9) */
+	untimeout(bvm_timeout, tp, bvm_timeouthandle);
+}
+
+static void
+bvm_tty_outwakeup(struct tty *tp)
+{
+	int len, written;
+	u_char buf[BVMBURSTLEN];
+
+	for (;;) {
+		len = ttydisc_getc(tp, buf, sizeof(buf));
+		if (len == 0)
+			break;
+
+		written = 0;
+		while (written < len)
+			bvm_wcons(buf[written++]);
+	}
+}
+
+static void
+bvm_timeout(void *v)
+{
+	struct	tty *tp;
+	int 	c;
+
+	tp = (struct tty *)v;
+
+	tty_lock(tp);
+	while ((c = bvm_cngetc(NULL)) != -1)
+		ttydisc_rint(tp, c, 0);
+	ttydisc_rint_done(tp);
+	tty_unlock(tp);
+
+	bvm_timeouthandle = timeout(bvm_timeout, tp, polltime);
+}
+
+static void
+bvm_cnprobe(struct consdev *cp)
+{
+	int disabled, port;
+
+	disabled = 0;
+	cp->cn_pri = CN_DEAD;
+
+	resource_int_value("bvmconsole", 0, "disabled", &disabled);
+	if (!disabled) {
+		if (resource_int_value("bvmconsole", 0, "port", &port) == 0)
+			bvm_cons_port = port;
+
+		if (inw(bvm_cons_port) == BVM_CONS_SIG)
+			cp->cn_pri = CN_REMOTE;
+	}
+}
+
+static void
+bvm_cninit(struct consdev *cp)
+{
+	int i;
+	const char *bootmsg = "Using bvm console.\n";
+
+	if (boothowto & RB_VERBOSE) {
+		for (i = 0; i < strlen(bootmsg); i++)
+			bvm_cnputc(cp, bootmsg[i]);
+	}
+
+	strcpy(cp->cn_name, "bvmcons");
+}
+
+static void
+bvm_cnterm(struct consdev *cp)
+{
+
+}
+
+static int
+bvm_cngetc(struct consdev *cp)
+{
+	unsigned char ch;
+
+	if (bvm_rcons(&ch) == 0) {
+#if defined(KDB)
+		kdb_alt_break(ch, &alt_break_state);
+#endif
+		return (ch);
+	}
+
+	return (-1);
+}
+
+static void
+bvm_cnputc(struct consdev *cp, int c)
+{
+
+	bvm_wcons(c);
+}
+
+static void
+bvm_cngrab(struct consdev *cp)
+{
+}
+
+static void
+bvm_cnungrab(struct consdev *cp)
+{
+}
+
+SYSINIT(cndev, SI_SUB_CONFIGURE, SI_ORDER_MIDDLE, cn_drvinit, NULL);
diff --git a/sys/dev/bvm/bvm_dbg.c b/sys/dev/bvm/bvm_dbg.c
new file mode 100644
index 0000000..1ba7ce0
--- /dev/null
+++ b/sys/dev/bvm/bvm_dbg.c
@@ -0,0 +1,100 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+
+#include <gdb/gdb.h>
+
+#include <machine/cpufunc.h>
+
+static gdb_probe_f bvm_dbg_probe;
+static gdb_init_f bvm_dbg_init;
+static gdb_term_f bvm_dbg_term;
+static gdb_getc_f bvm_dbg_getc;
+static gdb_putc_f bvm_dbg_putc;
+
+GDB_DBGPORT(bvm, bvm_dbg_probe, bvm_dbg_init, bvm_dbg_term,
+    bvm_dbg_getc, bvm_dbg_putc);
+
+#define	BVM_DBG_PORT	0x224
+static int bvm_dbg_port = BVM_DBG_PORT;
+
+#define BVM_DBG_SIG	('B' << 8 | 'V')
+
+static int
+bvm_dbg_probe(void)
+{
+	int disabled, port;
+
+	disabled = 0;
+	resource_int_value("bvmdbg", 0, "disabled", &disabled);
+
+	if (!disabled) {
+		if (resource_int_value("bvmdbg", 0, "port", &port) == 0)
+			bvm_dbg_port = port;
+
+		if (inw(bvm_dbg_port) == BVM_DBG_SIG) {
+			/*
+			 * Return a higher priority than 0 to override other
+			 * gdb dbgport providers that may be present (e.g. uart)
+			 */
+			return (1);
+		}
+	}
+
+	return (-1);
+}
+
+static void
+bvm_dbg_init(void)
+{
+}
+
+static void
+bvm_dbg_term(void)
+{
+}
+
+static void
+bvm_dbg_putc(int c)
+{
+
+	outl(bvm_dbg_port, c);
+}
+
+static int
+bvm_dbg_getc(void)
+{
+
+	return (inl(bvm_dbg_port));
+}
diff --git a/sys/dev/cas/if_cas.c b/sys/dev/cas/if_cas.c
index f846350..f98d4f0 100644
--- a/sys/dev/cas/if_cas.c
+++ b/sys/dev/cas/if_cas.c
@@ -214,8 +214,12 @@ cas_attach(struct cas_softc *sc)
 		error = ENXIO;
 		goto fail_ifnet;
 	}
-	taskqueue_start_threads(&sc->sc_tq, 1, PI_NET, "%s taskq",
+	error = taskqueue_start_threads(&sc->sc_tq, 1, PI_NET, "%s taskq",
 	    device_get_nameunit(sc->sc_dev));
+	if (error != 0) {
+		device_printf(sc->sc_dev, "could not start threads\n");
+		goto fail_taskq;
+	}
 
 	/* Make sure the chip is stopped. */
 	cas_reset(sc);
@@ -339,10 +343,13 @@ cas_attach(struct cas_softc *sc)
 			    BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
 			/* Enable/unfreeze the GMII pins of Saturn. */
 			if (sc->sc_variant == CAS_SATURN) {
-				CAS_WRITE_4(sc, CAS_SATURN_PCFG, 0);
+				CAS_WRITE_4(sc, CAS_SATURN_PCFG,
+				    CAS_READ_4(sc, CAS_SATURN_PCFG) &
+				    ~CAS_SATURN_PCFG_FSI);
 				CAS_BARRIER(sc, CAS_SATURN_PCFG, 4,
 				    BUS_SPACE_BARRIER_READ |
 				    BUS_SPACE_BARRIER_WRITE);
+				DELAY(10000);
 			}
 			error = mii_attach(sc->sc_dev, &sc->sc_miibus, ifp,
 			    cas_mediachange, cas_mediastatus, BMSR_DEFCAPMASK,
@@ -359,10 +366,12 @@ cas_attach(struct cas_softc *sc)
 			/* Freeze the GMII pins of Saturn for saving power. */
 			if (sc->sc_variant == CAS_SATURN) {
 				CAS_WRITE_4(sc, CAS_SATURN_PCFG,
+				    CAS_READ_4(sc, CAS_SATURN_PCFG) |
 				    CAS_SATURN_PCFG_FSI);
 				CAS_BARRIER(sc, CAS_SATURN_PCFG, 4,
 				    BUS_SPACE_BARRIER_READ |
 				    BUS_SPACE_BARRIER_WRITE);
+				DELAY(10000);
 			}
 			error = mii_attach(sc->sc_dev, &sc->sc_miibus, ifp,
 			    cas_mediachange, cas_mediastatus, BMSR_DEFCAPMASK,
@@ -2865,7 +2874,7 @@ cas_pci_attach(device_t dev)
 		goto fail;
 	}
 	i = 0;
-	if (lma > 1 && pci_get_slot(dev) < sizeof(enaddr) / sizeof(*enaddr))
+	if (lma > 1 && pci_get_slot(dev) < nitems(enaddr))
 		i = pci_get_slot(dev);
 	memcpy(sc->sc_enaddr, enaddr[i], ETHER_ADDR_LEN);
 
@@ -2874,7 +2883,7 @@ cas_pci_attach(device_t dev)
 		goto fail;
 	}
 	i = 0;
-	if (phy > 1 && pci_get_slot(dev) < sizeof(pcs) / sizeof(*pcs))
+	if (phy > 1 && pci_get_slot(dev) < nitems(pcs))
 		i = pci_get_slot(dev);
 	if (pcs[i] != 0)
 		sc->sc_flags |= CAS_SERDES;
diff --git a/sys/dev/ce/if_ce.c b/sys/dev/ce/if_ce.c
index 700ab80a..3f1a550 100644
--- a/sys/dev/ce/if_ce.c
+++ b/sys/dev/ce/if_ce.c
@@ -145,7 +145,7 @@ static	device_method_t ce_methods[] = {
 	DEVMETHOD(device_attach,	ce_attach),
 	DEVMETHOD(device_detach,	ce_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 typedef struct _ce_dma_mem_t {
diff --git a/sys/dev/cfi/cfi_bus_fdt.c b/sys/dev/cfi/cfi_bus_fdt.c
index bfbd65f..f775e05 100644
--- a/sys/dev/cfi/cfi_bus_fdt.c
+++ b/sys/dev/cfi/cfi_bus_fdt.c
@@ -51,7 +51,7 @@ static device_method_t cfi_fdt_methods[] = {
 	DEVMETHOD(device_attach,	cfi_attach),
 	DEVMETHOD(device_detach,	cfi_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t cfi_fdt_driver = {
diff --git a/sys/dev/cfi/cfi_bus_ixp4xx.c b/sys/dev/cfi/cfi_bus_ixp4xx.c
index ef9da9ee..93d03fe 100644
--- a/sys/dev/cfi/cfi_bus_ixp4xx.c
+++ b/sys/dev/cfi/cfi_bus_ixp4xx.c
@@ -69,7 +69,7 @@ static device_method_t cfi_ixp4xx_methods[] = {
 	DEVMETHOD(device_attach,	cfi_attach),
 	DEVMETHOD(device_detach,	cfi_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t cfi_ixp4xx_driver = {
diff --git a/sys/dev/ciss/ciss.c b/sys/dev/ciss/ciss.c
index 3291f03..a3f0a4a 100644
--- a/sys/dev/ciss/ciss.c
+++ b/sys/dev/ciss/ciss.c
@@ -1203,13 +1203,21 @@ ciss_identify_adapter(struct ciss_softc *sc)
     /* XXX only really required for old 5300 adapters? */
     sc->ciss_flags |= CISS_FLAG_BMIC_ABORT;
 
+    /*
+     * Earlier controller specs do not contain these config
+     * entries, so assume that a 0 means its old and assign
+     * these values to the defaults that were established 
+     * when this driver was developed for them
+     */
+    if (sc->ciss_cfg->max_logical_supported == 0) 
+        sc->ciss_cfg->max_logical_supported = CISS_MAX_LOGICAL;
+    if (sc->ciss_cfg->max_physical_supported == 0) 
+	sc->ciss_cfg->max_physical_supported = CISS_MAX_PHYSICAL;
     /* print information */
     if (bootverbose) {
-#if 0	/* XXX proxy volumes??? */
 	ciss_printf(sc, "  %d logical drive%s configured\n",
 		    sc->ciss_id->configured_logical_drives,
 		    (sc->ciss_id->configured_logical_drives == 1) ? "" : "s");
-#endif
 	ciss_printf(sc, "  firmware %4.4s\n", sc->ciss_id->running_firmware_revision);
 	ciss_printf(sc, "  %d SCSI channels\n", sc->ciss_id->scsi_bus_count);
 
@@ -1232,6 +1240,9 @@ ciss_identify_adapter(struct ciss_softc *sc)
 		    "\20\1ultra2\2ultra3\10fibre1\11fibre2\n");
 	ciss_printf(sc, "  server name '%.16s'\n", sc->ciss_cfg->server_name);
 	ciss_printf(sc, "  heartbeat 0x%x\n", sc->ciss_cfg->heartbeat);
+    	ciss_printf(sc, "  max logical logical volumes: %d\n", sc->ciss_cfg->max_logical_supported);
+    	ciss_printf(sc, "  max physical disks supported: %d\n", sc->ciss_cfg->max_physical_supported);
+    	ciss_printf(sc, "  max physical disks per logical volume: %d\n", sc->ciss_cfg->max_physical_per_logical);
     }
 
 out:
@@ -1319,7 +1330,7 @@ ciss_report_luns(struct ciss_softc *sc, int opcode, int nunits)
 	break;
     case CISS_CMD_STATUS_DATA_OVERRUN:
 	ciss_printf(sc, "WARNING: more units than driver limit (%d)\n",
-		    CISS_MAX_LOGICAL);
+		    sc->ciss_cfg->max_logical_supported);
 	break;
     default:
 	ciss_printf(sc, "error detecting logical drive configuration (%s)\n",
@@ -1353,7 +1364,7 @@ ciss_init_logical(struct ciss_softc *sc)
     debug_called(1);
 
     cll = ciss_report_luns(sc, CISS_OPCODE_REPORT_LOGICAL_LUNS,
-			   CISS_MAX_LOGICAL);
+			   sc->ciss_cfg->max_logical_supported);
     if (cll == NULL) {
 	error = ENXIO;
 	goto out;
@@ -1361,9 +1372,9 @@ ciss_init_logical(struct ciss_softc *sc)
 
     /* sanity-check reply */
     ndrives = (ntohl(cll->list_size) / sizeof(union ciss_device_address));
-    if ((ndrives < 0) || (ndrives > CISS_MAX_LOGICAL)) {
+    if ((ndrives < 0) || (ndrives > sc->ciss_cfg->max_logical_supported)) {
 	ciss_printf(sc, "adapter claims to report absurd number of logical drives (%d > %d)\n",
-		    ndrives, CISS_MAX_LOGICAL);
+	    	ndrives, sc->ciss_cfg->max_logical_supported);
 	error = ENXIO;
 	goto out;
     }
@@ -1386,19 +1397,20 @@ ciss_init_logical(struct ciss_softc *sc)
 
     for (i = 0; i <= sc->ciss_max_logical_bus; i++) {
 	sc->ciss_logical[i] =
-	    malloc(CISS_MAX_LOGICAL * sizeof(struct ciss_ldrive),
+	    malloc(sc->ciss_cfg->max_logical_supported *
+		   sizeof(struct ciss_ldrive),
 		   CISS_MALLOC_CLASS, M_NOWAIT | M_ZERO);
 	if (sc->ciss_logical[i] == NULL) {
 	    error = ENXIO;
 	    goto out;
 	}
 
-	for (j = 0; j < CISS_MAX_LOGICAL; j++)
+	for (j = 0; j < sc->ciss_cfg->max_logical_supported; j++)
 	    sc->ciss_logical[i][j].cl_status = CISS_LD_NONEXISTENT;
     }
 
 
-    for (i = 0; i < CISS_MAX_LOGICAL; i++) {
+    for (i = 0; i < sc->ciss_cfg->max_logical_supported; i++) {
 	if (i < ndrives) {
 	    struct ciss_ldrive	*ld;
 	    int			bus, target;
@@ -1440,7 +1452,7 @@ ciss_init_physical(struct ciss_softc *sc)
     target = 0;
 
     cll = ciss_report_luns(sc, CISS_OPCODE_REPORT_PHYSICAL_LUNS,
-			   CISS_MAX_PHYSICAL);
+			   sc->ciss_cfg->max_physical_supported);
     if (cll == NULL) {
 	error = ENXIO;
 	goto out;
@@ -1983,7 +1995,7 @@ ciss_free(struct ciss_softc *sc)
 	bus_dma_tag_destroy(sc->ciss_parent_dmat);
     if (sc->ciss_logical) {
 	for (i = 0; i <= sc->ciss_max_logical_bus; i++) {
-	    for (j = 0; j < CISS_MAX_LOGICAL; j++) {
+	    for (j = 0; j < sc->ciss_cfg->max_logical_supported; j++) {
 		if (sc->ciss_logical[i][j].cl_ldrive)
 		    free(sc->ciss_logical[i][j].cl_ldrive, CISS_MALLOC_CLASS);
 		if (sc->ciss_logical[i][j].cl_lstatus)
@@ -2966,9 +2978,9 @@ ciss_cam_action(struct cam_sim *sim, union ccb *ccb)
 	cpi->hba_inquiry = PI_TAG_ABLE;	/* XXX is this correct? */
 	cpi->target_sprt = 0;
 	cpi->hba_misc = 0;
-	cpi->max_target = CISS_MAX_LOGICAL;
+	cpi->max_target = sc->ciss_cfg->max_logical_supported;
 	cpi->max_lun = 0;		/* 'logical drive' channel only */
-	cpi->initiator_id = CISS_MAX_LOGICAL;
+	cpi->initiator_id = sc->ciss_cfg->max_logical_supported;
 	strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
         strncpy(cpi->hba_vid, "msmith@freebsd.org", HBA_IDLEN);
         strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
@@ -3879,7 +3891,7 @@ ciss_notify_rescan_logical(struct ciss_softc *sc)
      * drive address.
      */
     cll = ciss_report_luns(sc, CISS_OPCODE_REPORT_LOGICAL_LUNS,
-                           CISS_MAX_LOGICAL);
+                           sc->ciss_cfg->max_logical_supported);
     if (cll == NULL)
         return;
 
@@ -3890,7 +3902,7 @@ ciss_notify_rescan_logical(struct ciss_softc *sc)
      * firmware.
      */
     for (i = 0; i < sc->ciss_max_logical_bus; i++) {
-	for (j = 0; j < CISS_MAX_LOGICAL; j++) {
+	for (j = 0; j < sc->ciss_cfg->max_logical_supported; j++) {
 	    ld = &sc->ciss_logical[i][j];
 
 	    if (ld->cl_update == 0)
@@ -4059,7 +4071,7 @@ ciss_notify_hotplug(struct ciss_softc *sc, struct ciss_notify *cn)
 	     * Rescan the physical lun list for new items
 	     */
 	    cll = ciss_report_luns(sc, CISS_OPCODE_REPORT_PHYSICAL_LUNS,
-				   CISS_MAX_PHYSICAL);
+				   sc->ciss_cfg->max_physical_supported);
 	    if (cll == NULL) {
 		ciss_printf(sc, "Warning, cannot get physical lun list\n");
 		break;
@@ -4307,7 +4319,7 @@ ciss_print_adapter(struct ciss_softc *sc)
 	"\20\1notify_ok\2control_open\3aborting\4running\21fake_synch\22bmic_abort\n");
 
     for (i = 0; i < sc->ciss_max_logical_bus; i++) {
-	for (j = 0; j < CISS_MAX_LOGICAL; j++) {
+	for (j = 0; j < sc->ciss_cfg->max_logical_supported; j++) {
 	    ciss_printf(sc, "LOGICAL DRIVE %d:  ", i);
 	    ciss_print_ldrive(sc, &sc->ciss_logical[i][j]);
 	}
diff --git a/sys/dev/ciss/cissreg.h b/sys/dev/ciss/cissreg.h
index feae826..4a3349f 100644
--- a/sys/dev/ciss/cissreg.h
+++ b/sys/dev/ciss/cissreg.h
@@ -425,6 +425,15 @@ struct ciss_config_table
 #define CISS_DRIVER_DAUGHTER_ATTACHED		(1<<8)
 #define CISS_DRIVER_SCSI_PREFETCH		(1<<9)
     u_int32_t	max_sg_length;		/* 31 in older firmware */
+/*
+ * these fields appear in OpenCISS Spec 1.06
+ * http://cciss.sourceforge.net/#docs
+ */
+    u_int32_t	max_logical_supported;
+    u_int32_t	max_physical_supported;
+    u_int32_t	max_physical_per_logical;
+    u_int32_t	max_perfomant_mode_cmds;
+    u_int32_t	max_block_fetch_count;
 } __packed;
 
 /*
diff --git a/sys/dev/ciss/cissvar.h b/sys/dev/ciss/cissvar.h
index d109b7a..e89e4e9 100644
--- a/sys/dev/ciss/cissvar.h
+++ b/sys/dev/ciss/cissvar.h
@@ -45,8 +45,11 @@ typedef STAILQ_HEAD(, ciss_request)	cr_qhead_t;
 
 /*
  * Maximum number of logical drives we support.
+ * If the controller does not indicate a maximum
+ * value.  This is a compatibiliy value to support
+ * older ciss controllers (e.g. model 6i)
  */
-#define CISS_MAX_LOGICAL	63
+#define CISS_MAX_LOGICAL	16
 
 /*
  * Maximum number of physical devices we support.
diff --git a/sys/dev/coretemp/coretemp.c b/sys/dev/coretemp/coretemp.c
index 54e5a43..55efca2 100644
--- a/sys/dev/coretemp/coretemp.c
+++ b/sys/dev/coretemp/coretemp.c
@@ -85,7 +85,7 @@ static device_method_t coretemp_methods[] = {
 	DEVMETHOD(device_attach,	coretemp_attach),
 	DEVMETHOD(device_detach,	coretemp_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t coretemp_driver = {
diff --git a/sys/dev/cp/if_cp.c b/sys/dev/cp/if_cp.c
index 4ff0bcc..299a51c 100644
--- a/sys/dev/cp/if_cp.c
+++ b/sys/dev/cp/if_cp.c
@@ -93,7 +93,7 @@ static	device_method_t cp_methods[] = {
 	DEVMETHOD(device_attach,	cp_attach),
 	DEVMETHOD(device_detach,	cp_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 typedef struct _cp_dma_mem_t {
diff --git a/sys/dev/cpufreq/ichss.c b/sys/dev/cpufreq/ichss.c
index fc7f0ac..0eae0c9 100644
--- a/sys/dev/cpufreq/ichss.c
+++ b/sys/dev/cpufreq/ichss.c
@@ -113,7 +113,7 @@ static device_method_t ichss_methods[] = {
 	DEVMETHOD(cpufreq_drv_get,	ichss_get),
 	DEVMETHOD(cpufreq_drv_type,	ichss_type),
 	DEVMETHOD(cpufreq_drv_settings,	ichss_settings),
-	{0, 0}
+	DEVMETHOD_END
 };
 static driver_t ichss_driver = {
 	"ichss", ichss_methods, sizeof(struct ichss_softc)
diff --git a/sys/dev/ctau/if_ct.c b/sys/dev/ctau/if_ct.c
index 7b4abc3..fc3e916 100644
--- a/sys/dev/ctau/if_ct.c
+++ b/sys/dev/ctau/if_ct.c
@@ -93,7 +93,8 @@ static device_method_t ct_isa_methods [] = {
 	DEVMETHOD(device_probe,		ct_probe),
 	DEVMETHOD(device_attach,	ct_attach),
 	DEVMETHOD(device_detach,	ct_detach),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 typedef struct _ct_dma_mem_t {
diff --git a/sys/dev/cx/if_cx.c b/sys/dev/cx/if_cx.c
index cb15486..ad8312c 100644
--- a/sys/dev/cx/if_cx.c
+++ b/sys/dev/cx/if_cx.c
@@ -113,7 +113,8 @@ static device_method_t cx_isa_methods [] = {
 	DEVMETHOD(device_probe,		cx_probe),
 	DEVMETHOD(device_attach,	cx_attach),
 	DEVMETHOD(device_detach,	cx_detach),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 typedef struct _cx_dma_mem_t {
diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index c1d8a6b..55cfa52 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -158,6 +158,16 @@ enum {
 };
 
 enum {
+	/* flags understood by begin_synchronized_op */
+	HOLD_LOCK	= (1 << 0),
+	SLEEP_OK	= (1 << 1),
+	INTR_OK		= (1 << 2),
+
+	/* flags understood by end_synchronized_op */
+	LOCK_HELD	= HOLD_LOCK,
+};
+
+enum {
 	/* adapter flags */
 	FULL_INIT_DONE	= (1 << 0),
 	FW_OK		= (1 << 1),
@@ -174,11 +184,11 @@ enum {
 	PORT_SYSCTL_CTX	= (1 << 2),
 };
 
-#define IS_DOOMED(pi)	(pi->flags & DOOMED)
-#define SET_DOOMED(pi)	do {pi->flags |= DOOMED;} while (0)
-#define IS_BUSY(sc)	(sc->flags & CXGBE_BUSY)
-#define SET_BUSY(sc)	do {sc->flags |= CXGBE_BUSY;} while (0)
-#define CLR_BUSY(sc)	do {sc->flags &= ~CXGBE_BUSY;} while (0)
+#define IS_DOOMED(pi)	((pi)->flags & DOOMED)
+#define SET_DOOMED(pi)	do {(pi)->flags |= DOOMED;} while (0)
+#define IS_BUSY(sc)	((sc)->flags & CXGBE_BUSY)
+#define SET_BUSY(sc)	do {(sc)->flags |= CXGBE_BUSY;} while (0)
+#define CLR_BUSY(sc)	do {(sc)->flags &= ~CXGBE_BUSY;} while (0)
 
 struct port_info {
 	device_t dev;
@@ -567,7 +577,8 @@ struct adapter {
 	int flags;
 
 	char fw_version[32];
-	unsigned int cfcsum;
+	char cfg_file[32];
+	u_int cfcsum;
 	struct adapter_params params;
 	struct t4_virt_res vres;
 
@@ -591,6 +602,11 @@ struct adapter {
 	an_handler_t an_handler __aligned(CACHE_LINE_SIZE);
 	fw_msg_handler_t fw_msg_handler[4];	/* NUM_FW6_TYPES */
 	cpl_handler_t cpl_handler[0xef];	/* NUM_CPL_CMDS */
+
+#ifdef INVARIANTS
+	const char *last_op;
+	const void *last_op_thr;
+#endif
 };
 
 #define ADAPTER_LOCK(sc)		mtx_lock(&(sc)->sc_lock)
@@ -598,6 +614,12 @@ struct adapter {
 #define ADAPTER_LOCK_ASSERT_OWNED(sc)	mtx_assert(&(sc)->sc_lock, MA_OWNED)
 #define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) mtx_assert(&(sc)->sc_lock, MA_NOTOWNED)
 
+/* XXX: not bulletproof, but much better than nothing */
+#define ASSERT_SYNCHRONIZED_OP(sc)	\
+    KASSERT(IS_BUSY(sc) && \
+	(mtx_owned(&(sc)->sc_lock) || sc->last_op_thr == curthread), \
+	("%s: operation not synchronized.", __func__))
+
 #define PORT_LOCK(pi)			mtx_lock(&(pi)->pi_lock)
 #define PORT_UNLOCK(pi)			mtx_unlock(&(pi)->pi_lock)
 #define PORT_LOCK_ASSERT_OWNED(pi)	mtx_assert(&(pi)->pi_lock, MA_OWNED)
@@ -626,18 +648,18 @@ struct adapter {
 #define TXQ_LOCK_ASSERT_OWNED(txq)	EQ_LOCK_ASSERT_OWNED(&(txq)->eq)
 #define TXQ_LOCK_ASSERT_NOTOWNED(txq)	EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq)
 
-#define for_each_txq(pi, iter, txq) \
-	txq = &pi->adapter->sge.txq[pi->first_txq]; \
-	for (iter = 0; iter < pi->ntxq; ++iter, ++txq)
-#define for_each_rxq(pi, iter, rxq) \
-	rxq = &pi->adapter->sge.rxq[pi->first_rxq]; \
-	for (iter = 0; iter < pi->nrxq; ++iter, ++rxq)
-#define for_each_ofld_txq(pi, iter, ofld_txq) \
-	ofld_txq = &pi->adapter->sge.ofld_txq[pi->first_ofld_txq]; \
-	for (iter = 0; iter < pi->nofldtxq; ++iter, ++ofld_txq)
-#define for_each_ofld_rxq(pi, iter, ofld_rxq) \
-	ofld_rxq = &pi->adapter->sge.ofld_rxq[pi->first_ofld_rxq]; \
-	for (iter = 0; iter < pi->nofldrxq; ++iter, ++ofld_rxq)
+#define for_each_txq(pi, iter, q) \
+	for (q = &pi->adapter->sge.txq[pi->first_txq], iter = 0; \
+	    iter < pi->ntxq; ++iter, ++q)
+#define for_each_rxq(pi, iter, q) \
+	for (q = &pi->adapter->sge.rxq[pi->first_rxq], iter = 0; \
+	    iter < pi->nrxq; ++iter, ++q)
+#define for_each_ofld_txq(pi, iter, q) \
+	for (q = &pi->adapter->sge.ofld_txq[pi->first_ofld_txq], iter = 0; \
+	    iter < pi->nofldtxq; ++iter, ++q)
+#define for_each_ofld_rxq(pi, iter, q) \
+	for (q = &pi->adapter->sge.ofld_rxq[pi->first_ofld_rxq], iter = 0; \
+	    iter < pi->nofldrxq; ++iter, ++q)
 
 /* One for errors, one for firmware events */
 #define T4_EXTRA_INTR 2
@@ -751,6 +773,8 @@ int t4_register_cpl_handler(struct adapter *, int, cpl_handler_t);
 int t4_register_an_handler(struct adapter *, an_handler_t);
 int t4_register_fw_msg_handler(struct adapter *, int, fw_msg_handler_t);
 int t4_filter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *);
+int begin_synchronized_op(struct adapter *, struct port_info *, int, char *);
+void end_synchronized_op(struct adapter *, int);
 
 /* t4_sge.c */
 void t4_sge_modload(void);
diff --git a/sys/dev/cxgbe/common/jhash.h b/sys/dev/cxgbe/common/jhash.h
deleted file mode 100644
index 4546b7b..0000000
--- a/sys/dev/cxgbe/common/jhash.h
+++ /dev/null
@@ -1,140 +0,0 @@
-#ifndef _JHASH_H
-#define _JHASH_H
-
-/* jhash.h: Jenkins hash support.
- *
- * Copyright (C) 1996 Bob Jenkins (bob_jenkins@burtleburtle.net)
- *
- * http://burtleburtle.net/bob/hash/
- *
- * These are the credits from Bob's sources:
- *
- * lookup2.c, by Bob Jenkins, December 1996, Public Domain.
- * hash(), hash2(), hash3, and mix() are externally useful functions.
- * Routines to test the hash are included if SELF_TEST is defined.
- * You can use this free for any purpose.  It has no warranty.
- *
- * $FreeBSD$
- */
-
-/* NOTE: Arguments are modified. */
-#define __jhash_mix(a, b, c) \
-{ \
-  a -= b; a -= c; a ^= (c>>13); \
-  b -= c; b -= a; b ^= (a<<8); \
-  c -= a; c -= b; c ^= (b>>13); \
-  a -= b; a -= c; a ^= (c>>12);  \
-  b -= c; b -= a; b ^= (a<<16); \
-  c -= a; c -= b; c ^= (b>>5); \
-  a -= b; a -= c; a ^= (c>>3);  \
-  b -= c; b -= a; b ^= (a<<10); \
-  c -= a; c -= b; c ^= (b>>15); \
-}
-
-/* The golden ration: an arbitrary value */
-#define JHASH_GOLDEN_RATIO	0x9e3779b9
-
-/* The most generic version, hashes an arbitrary sequence
- * of bytes.  No alignment or length assumptions are made about
- * the input key.
- */
-static inline u32 jhash(const void *key, u32 length, u32 initval)
-{
-	u32 a, b, c, len;
-	const u8 *k = key;
-
-	len = length;
-	a = b = JHASH_GOLDEN_RATIO;
-	c = initval;
-
-	while (len >= 12) {
-		a += (k[0] +((u32)k[1]<<8) +((u32)k[2]<<16) +((u32)k[3]<<24));
-		b += (k[4] +((u32)k[5]<<8) +((u32)k[6]<<16) +((u32)k[7]<<24));
-		c += (k[8] +((u32)k[9]<<8) +((u32)k[10]<<16)+((u32)k[11]<<24));
-
-		__jhash_mix(a,b,c);
-
-		k += 12;
-		len -= 12;
-	}
-
-	c += length;
-	switch (len) {
-	case 11: c += ((u32)k[10]<<24);
-	case 10: c += ((u32)k[9]<<16);
-	case 9 : c += ((u32)k[8]<<8);
-	case 8 : b += ((u32)k[7]<<24);
-	case 7 : b += ((u32)k[6]<<16);
-	case 6 : b += ((u32)k[5]<<8);
-	case 5 : b += k[4];
-	case 4 : a += ((u32)k[3]<<24);
-	case 3 : a += ((u32)k[2]<<16);
-	case 2 : a += ((u32)k[1]<<8);
-	case 1 : a += k[0];
-	};
-
-	__jhash_mix(a,b,c);
-
-	return c;
-}
-
-/* A special optimized version that handles 1 or more of u32s.
- * The length parameter here is the number of u32s in the key.
- */
-static inline u32 jhash2(u32 *k, u32 length, u32 initval)
-{
-	u32 a, b, c, len;
-
-	a = b = JHASH_GOLDEN_RATIO;
-	c = initval;
-	len = length;
-
-	while (len >= 3) {
-		a += k[0];
-		b += k[1];
-		c += k[2];
-		__jhash_mix(a, b, c);
-		k += 3; len -= 3;
-	}
-
-	c += length * 4;
-
-	switch (len) {
-	case 2 : b += k[1];
-	case 1 : a += k[0];
-	};
-
-	__jhash_mix(a,b,c);
-
-	return c;
-}
-
-
-/* A special ultra-optimized versions that knows they are hashing exactly
- * 3, 2 or 1 word(s).
- *
- * NOTE: In partilar the "c += length; __jhash_mix(a,b,c);" normally
- *       done at the end is not done here.
- */
-static inline u32 jhash_3words(u32 a, u32 b, u32 c, u32 initval)
-{
-	a += JHASH_GOLDEN_RATIO;
-	b += JHASH_GOLDEN_RATIO;
-	c += initval;
-
-	__jhash_mix(a, b, c);
-
-	return c;
-}
-
-static inline u32 jhash_2words(u32 a, u32 b, u32 initval)
-{
-	return jhash_3words(a, b, 0, initval);
-}
-
-static inline u32 jhash_1word(u32 a, u32 initval)
-{
-	return jhash_3words(a, 0, 0, initval);
-}
-
-#endif /* _JHASH_H */
diff --git a/sys/dev/cxgbe/common/t4_msg.h b/sys/dev/cxgbe/common/t4_msg.h
index 5bd3cef..92f760b 100644
--- a/sys/dev/cxgbe/common/t4_msg.h
+++ b/sys/dev/cxgbe/common/t4_msg.h
@@ -159,6 +159,8 @@ enum CPL_error {
 	CPL_ERR_KEEPALIVE_TIMEDOUT = 34,
 	CPL_ERR_RTX_NEG_ADVICE     = 35,
 	CPL_ERR_PERSIST_NEG_ADVICE = 36,
+	CPL_ERR_KEEPALV_NEG_ADVICE = 37,
+	CPL_ERR_WAIT_ARP_RPL       = 41,
 	CPL_ERR_ABORT_FAILED       = 42,
 	CPL_ERR_IWARP_FLM          = 50,
 };
diff --git a/sys/dev/cxgbe/firmware/t4fw_cfg.txt b/sys/dev/cxgbe/firmware/t4fw_cfg.txt
index bf6a9a1..2a9db62 100644
--- a/sys/dev/cxgbe/firmware/t4fw_cfg.txt
+++ b/sys/dev/cxgbe/firmware/t4fw_cfg.txt
@@ -56,7 +56,7 @@
 [function "4"]
 	wx_caps = all
 	r_caps = all
-	nvi = 54
+	nvi = 32
 	niqflint = 256
 	nethctrl = 128
 	neq = 256
@@ -74,8 +74,8 @@
 	# Each entry in these categories takes 4 cells each.  nhash will use the
 	# TCAM iff there is room left (that is, the rest don't add up to 2048).
 	nroute = 32
-	nclip = 0	# needed only for IPv6 offload
-	nfilter = 1488
+	nclip = 32
+	nfilter = 1456
 	nserver = 512
 	nhash = 16384
 
@@ -137,7 +137,7 @@
 
 [fini]
 	version = 0x1
-	checksum = 0x162df193
+	checksum = 0xfdebb6ef
 #
 # $FreeBSD$
 #
diff --git a/sys/dev/cxgbe/offload.h b/sys/dev/cxgbe/offload.h
index ced15a6..55ac71b 100644
--- a/sys/dev/cxgbe/offload.h
+++ b/sys/dev/cxgbe/offload.h
@@ -54,16 +54,20 @@
 	OPCODE_TID(w) = htonl(MK_OPCODE_TID(cpl, tid)); \
 } while (0)
 
+TAILQ_HEAD(stid_head, stid_region);
+struct listen_ctx;
+
+struct stid_region {
+	TAILQ_ENTRY(stid_region) link;
+	int used;	/* # of stids used by this region */
+	int free;	/* # of contiguous stids free right after this region */
+};
+
 /*
  * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
  */
 #define MAX_ATIDS 8192U
 
-union serv_entry {
-	void *data;
-	union serv_entry *next;
-};
-
 union aopen_entry {
 	void *data;
 	union aopen_entry *next;
@@ -75,34 +79,33 @@ union aopen_entry {
  */
 struct tid_info {
 	void **tid_tab;
-	unsigned int ntids;
-
-	union serv_entry *stid_tab;
-	unsigned int nstids;
-	unsigned int stid_base;
-
+	u_int ntids;
+	u_int tids_in_use;
+
+	struct mtx stid_lock __aligned(CACHE_LINE_SIZE);
+	struct listen_ctx **stid_tab;
+	u_int nstids;
+	u_int stid_base;
+	u_int stids_in_use;
+	u_int nstids_free_head;	/* # of available stids at the begining */
+	struct stid_head stids;
+
+	struct mtx atid_lock __aligned(CACHE_LINE_SIZE);
 	union aopen_entry *atid_tab;
-	unsigned int natids;
-
-	struct filter_entry *ftid_tab;
-	unsigned int nftids;
-	unsigned int ftid_base;
-	unsigned int ftids_in_use;
-
-	struct mtx atid_lock;
+	u_int natids;
 	union aopen_entry *afree;
-	unsigned int atids_in_use;
-
-	struct mtx stid_lock;
-	union serv_entry *sfree;
-	unsigned int stids_in_use;
+	u_int atids_in_use;
 
-	unsigned int tids_in_use;
+	struct mtx ftid_lock __aligned(CACHE_LINE_SIZE);
+	struct filter_entry *ftid_tab;
+	u_int nftids;
+	u_int ftid_base;
+	u_int ftids_in_use;
 };
 
 struct t4_range {
-	unsigned int start;
-	unsigned int size;
+	u_int start;
+	u_int size;
 };
 
 struct t4_virt_res {                      /* virtualized HW resources */
@@ -114,6 +117,7 @@ struct t4_virt_res {                      /* virtualized HW resources */
 	struct t4_range qp;
 	struct t4_range cq;
 	struct t4_range ocq;
+	struct t4_range l2t;
 };
 
 #ifdef TCP_OFFLOAD
diff --git a/sys/dev/cxgbe/t4_l2t.c b/sys/dev/cxgbe/t4_l2t.c
index dd8748e..dcff5e8 100644
--- a/sys/dev/cxgbe/t4_l2t.c
+++ b/sys/dev/cxgbe/t4_l2t.c
@@ -42,7 +42,6 @@ __FBSDID("$FreeBSD$");
 #include <netinet/in.h>
 
 #include "common/common.h"
-#include "common/jhash.h"
 #include "common/t4_msg.h"
 #include "t4_l2t.h"
 
@@ -78,7 +77,7 @@ t4_alloc_l2e(struct l2t_data *d)
 		return (NULL);
 
 	/* there's definitely a free entry */
-	for (e = d->rover, end = &d->l2tab[L2T_SIZE]; e != end; ++e)
+	for (e = d->rover, end = &d->l2tab[d->l2t_size]; e != end; ++e)
 		if (atomic_load_acq_int(&e->refcnt) == 0)
 			goto found;
 
@@ -115,6 +114,7 @@ t4_write_l2e(struct adapter *sc, struct l2t_entry *e, int sync)
 {
 	struct wrqe *wr;
 	struct cpl_l2t_write_req *req;
+	int idx = e->idx + sc->vres.l2t.start;
 
 	mtx_assert(&e->lock, MA_OWNED);
 
@@ -124,10 +124,10 @@ t4_write_l2e(struct adapter *sc, struct l2t_entry *e, int sync)
 	req = wrtod(wr);
 
 	INIT_TP_WR(req, 0);
-	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, e->idx |
+	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, idx |
 	    V_SYNC_WR(sync) | V_TID_QID(sc->sge.fwq.abs_id)));
 	req->params = htons(V_L2T_W_PORT(e->lport) | V_L2T_W_NOREPLY(!sync));
-	req->l2t_idx = htons(e->idx);
+	req->l2t_idx = htons(idx);
 	req->vlan = htons(e->vlan);
 	memcpy(req->dst_mac, e->dmac, sizeof(req->dst_mac));
 
@@ -183,18 +183,24 @@ t4_l2t_set_switching(struct adapter *sc, struct l2t_entry *e, uint16_t vlan,
 int
 t4_init_l2t(struct adapter *sc, int flags)
 {
-	int i;
+	int i, l2t_size;
 	struct l2t_data *d;
 
-	d = malloc(sizeof(*d), M_CXGBE, M_ZERO | flags);
+	l2t_size = sc->vres.l2t.size;
+	if (l2t_size < 2)	/* At least 1 bucket for IP and 1 for IPv6 */
+		return (EINVAL);
+
+	d = malloc(sizeof(*d) + l2t_size * sizeof (struct l2t_entry), M_CXGBE,
+	    M_ZERO | flags);
 	if (!d)
 		return (ENOMEM);
 
+	d->l2t_size = l2t_size;
 	d->rover = d->l2tab;
-	atomic_store_rel_int(&d->nfree, L2T_SIZE);
+	atomic_store_rel_int(&d->nfree, l2t_size);
 	rw_init(&d->lock, "L2T");
 
-	for (i = 0; i < L2T_SIZE; i++) {
+	for (i = 0; i < l2t_size; i++) {
 		struct l2t_entry *e = &d->l2tab[i];
 
 		e->idx = i;
@@ -215,7 +221,7 @@ t4_free_l2t(struct l2t_data *d)
 {
 	int i;
 
-	for (i = 0; i < L2T_SIZE; i++)
+	for (i = 0; i < d->l2t_size; i++)
 		mtx_destroy(&d->l2tab[i].lock);
 	rw_destroy(&d->lock);
 	free(d, M_CXGBE);
@@ -229,11 +235,11 @@ do_l2t_write_rpl(struct sge_iq *iq, const struct rss_header *rss,
 {
 	const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1);
 	unsigned int tid = GET_TID(rpl);
-	unsigned int idx = tid & (L2T_SIZE - 1);
+	unsigned int idx = tid % L2T_SIZE;
 
 	if (__predict_false(rpl->status != CPL_ERR_NONE)) {
 		log(LOG_ERR,
-		    "Unexpected L2T_WRITE_RPL status %u for entry %u\n",
+		    "Unexpected L2T_WRITE_RPL (%u) for entry at hw_idx %u\n",
 		    rpl->status, idx);
 		return (EINVAL);
 	}
@@ -269,7 +275,7 @@ sysctl_l2t(SYSCTL_HANDLER_ARGS)
 	struct l2t_entry *e;
 	struct sbuf *sb;
 	int rc, i, header = 0;
-	char ip[60];
+	char ip[INET6_ADDRSTRLEN];
 
 	if (l2t == NULL)
 		return (ENXIO);
@@ -283,7 +289,7 @@ sysctl_l2t(SYSCTL_HANDLER_ARGS)
 		return (ENOMEM);
 
 	e = &l2t->l2tab[0];
-	for (i = 0; i < L2T_SIZE; i++, e++) {
+	for (i = 0; i < l2t->l2t_size; i++, e++) {
 		mtx_lock(&e->lock);
 		if (e->state == L2T_STATE_UNUSED)
 			goto skip;
@@ -295,11 +301,15 @@ sysctl_l2t(SYSCTL_HANDLER_ARGS)
 		}
 		if (e->state == L2T_STATE_SWITCHING)
 			ip[0] = 0;
-		else
-			snprintf(ip, sizeof(ip), "%s",
-			    inet_ntoa(*(struct in_addr *)&e->addr));
+		else {
+			inet_ntop(e->ipv6 ? AF_INET6 : AF_INET, &e->addr[0],
+			    &ip[0], sizeof(ip));
+		}
 
-		/* XXX: e->ifp may not be around */
+		/*
+		 * XXX: e->ifp may not be around.
+		 * XXX: IPv6 addresses may not align properly in the output.
+		 */
 		sbuf_printf(sb, "\n%4u %-15s %02x:%02x:%02x:%02x:%02x:%02x %4d"
 			   " %u %2u   %c   %5u %s",
 			   e->idx, ip, e->dmac[0], e->dmac[1], e->dmac[2],
diff --git a/sys/dev/cxgbe/t4_l2t.h b/sys/dev/cxgbe/t4_l2t.h
index 6927b81..c60eef1 100644
--- a/sys/dev/cxgbe/t4_l2t.h
+++ b/sys/dev/cxgbe/t4_l2t.h
@@ -60,7 +60,7 @@ enum {
 struct l2t_entry {
 	uint16_t state;			/* entry state */
 	uint16_t idx;			/* entry index */
-	uint32_t addr;			/* next hop IP address */
+	uint32_t addr[4];		/* next hop IP or IPv6 address */
 	struct ifnet *ifp;		/* outgoing interface */
 	uint16_t smt_idx;		/* SMT index */
 	uint16_t vlan;			/* VLAN TCI (id: 0-11, prio: 13-15) */
@@ -70,15 +70,17 @@ struct l2t_entry {
 	struct mtx lock;
 	volatile int refcnt;		/* entry reference count */
 	uint16_t hash;			/* hash bucket the entry is on */
+	uint8_t ipv6;			/* entry is for an IPv6 address */
 	uint8_t lport;			/* associated offload logical port */
 	uint8_t dmac[ETHER_ADDR_LEN];	/* next hop's MAC address */
 };
 
 struct l2t_data {
 	struct rwlock lock;
+	u_int l2t_size;
 	volatile int nfree;	/* number of free entries */
 	struct l2t_entry *rover;/* starting point for next allocation */
-	struct l2t_entry l2tab[L2T_SIZE];
+	struct l2t_entry l2tab[];
 };
 
 
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index aeaa4d2..c22ec21 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -284,9 +284,7 @@ static int get_params__post_init(struct adapter *);
 static void t4_set_desc(struct adapter *);
 static void build_medialist(struct port_info *);
 static int update_mac_settings(struct port_info *, int);
-static int cxgbe_init_locked(struct port_info *);
 static int cxgbe_init_synchronized(struct port_info *);
-static int cxgbe_uninit_locked(struct port_info *);
 static int cxgbe_uninit_synchronized(struct port_info *);
 static int setup_intr_handlers(struct adapter *);
 static int adapter_full_init(struct adapter *);
@@ -348,6 +346,7 @@ static void clear_filter(struct filter_entry *);
 static int set_filter_wr(struct adapter *, int);
 static int del_filter_wr(struct adapter *, int);
 static int get_sge_context(struct adapter *, struct t4_sge_context *);
+static int load_fw(struct adapter *, struct t4_data *);
 static int read_card_mem(struct adapter *, struct t4_mem_range *);
 static int read_i2c(struct adapter *, struct t4_i2c_data *);
 #ifdef TCP_OFFLOAD
@@ -820,6 +819,8 @@ t4_detach(device_t dev)
 		mtx_destroy(&sc->sc_lock);
 	}
 
+	if (mtx_initialized(&sc->tids.ftid_lock))
+		mtx_destroy(&sc->tids.ftid_lock);
 	if (mtx_initialized(&sc->sfl_lock))
 		mtx_destroy(&sc->sfl_lock);
 
@@ -874,7 +875,7 @@ cxgbe_attach(device_t dev)
 	ifp->if_capabilities = T4_CAP;
 #ifdef TCP_OFFLOAD
 	if (is_offload(pi->adapter))
-		ifp->if_capabilities |= IFCAP_TOE4;
+		ifp->if_capabilities |= IFCAP_TOE;
 #endif
 	ifp->if_capenable = T4_CAP_ENABLE;
 	ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
@@ -918,6 +919,10 @@ cxgbe_detach(device_t dev)
 	while (IS_BUSY(sc))
 		mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
 	SET_BUSY(sc);
+#ifdef INVARIANTS
+	sc->last_op = "t4detach";
+	sc->last_op_thr = curthread;
+#endif
 	ADAPTER_UNLOCK(sc);
 
 	if (pi->vlan_c)
@@ -939,7 +944,7 @@ cxgbe_detach(device_t dev)
 
 	ADAPTER_LOCK(sc);
 	CLR_BUSY(sc);
-	wakeup_one(&sc->flags);
+	wakeup(&sc->flags);
 	ADAPTER_UNLOCK(sc);
 
 	return (0);
@@ -951,9 +956,10 @@ cxgbe_init(void *arg)
 	struct port_info *pi = arg;
 	struct adapter *sc = pi->adapter;
 
-	ADAPTER_LOCK(sc);
-	cxgbe_init_locked(pi); /* releases adapter lock */
-	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
+	if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
+		return;
+	cxgbe_init_synchronized(pi);
+	end_synchronized_op(sc, 0);
 }
 
 static int
@@ -967,81 +973,56 @@ cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
 
 	switch (cmd) {
 	case SIOCSIFMTU:
-		ADAPTER_LOCK(sc);
-		rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
-		if (rc) {
-fail:
-			ADAPTER_UNLOCK(sc);
-			return (rc);
-		}
-
 		mtu = ifr->ifr_mtu;
-		if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
-			rc = EINVAL;
-		} else {
-			ifp->if_mtu = mtu;
-			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
-				t4_update_fl_bufsize(ifp);
-				PORT_LOCK(pi);
-				rc = update_mac_settings(pi, XGMAC_MTU);
-				PORT_UNLOCK(pi);
-			}
+		if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
+			return (EINVAL);
+
+		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
+		if (rc)
+			return (rc);
+		ifp->if_mtu = mtu;
+		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+			t4_update_fl_bufsize(ifp);
+			rc = update_mac_settings(pi, XGMAC_MTU);
 		}
-		ADAPTER_UNLOCK(sc);
+		end_synchronized_op(sc, 0);
 		break;
 
 	case SIOCSIFFLAGS:
-		ADAPTER_LOCK(sc);
-		if (IS_DOOMED(pi)) {
-			rc = ENXIO;
-			goto fail;
-		}
+		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg");
+		if (rc)
+			return (rc);
+
 		if (ifp->if_flags & IFF_UP) {
 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 				flags = pi->if_flags;
 				if ((ifp->if_flags ^ flags) &
 				    (IFF_PROMISC | IFF_ALLMULTI)) {
-					if (IS_BUSY(sc)) {
-						rc = EBUSY;
-						goto fail;
-					}
-					PORT_LOCK(pi);
 					rc = update_mac_settings(pi,
 					    XGMAC_PROMISC | XGMAC_ALLMULTI);
-					PORT_UNLOCK(pi);
 				}
-				ADAPTER_UNLOCK(sc);
 			} else
-				rc = cxgbe_init_locked(pi);
+				rc = cxgbe_init_synchronized(pi);
 			pi->if_flags = ifp->if_flags;
 		} else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
-			rc = cxgbe_uninit_locked(pi);
-		else
-			ADAPTER_UNLOCK(sc);
-
-		ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
+			rc = cxgbe_uninit_synchronized(pi);
+		end_synchronized_op(sc, 0);
 		break;
 
 	case SIOCADDMULTI:	
-	case SIOCDELMULTI: /* these two can be called with a mutex held :-( */
-		ADAPTER_LOCK(sc);
-		rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
+	case SIOCDELMULTI: /* these two are called with a mutex held :-( */
+		rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
 		if (rc)
-			goto fail;
-
-		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
-			PORT_LOCK(pi);
+			return (rc);
+		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 			rc = update_mac_settings(pi, XGMAC_MCADDRS);
-			PORT_UNLOCK(pi);
-		}
-		ADAPTER_UNLOCK(sc);
+		end_synchronized_op(sc, LOCK_HELD);
 		break;
 
 	case SIOCSIFCAP:
-		ADAPTER_LOCK(sc);
-		rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
+		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
 		if (rc)
-			goto fail;
+			return (rc);
 
 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 		if (mask & IFCAP_TXCSUM) {
@@ -1122,11 +1103,8 @@ fail:
 #endif
 		if (mask & IFCAP_VLAN_HWTAGGING) {
 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
-			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
-				PORT_LOCK(pi);
+			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 				rc = update_mac_settings(pi, XGMAC_VLANEX);
-				PORT_UNLOCK(pi);
-			}
 		}
 		if (mask & IFCAP_VLAN_MTU) {
 			ifp->if_capenable ^= IFCAP_VLAN_MTU;
@@ -1141,7 +1119,8 @@ fail:
 #ifdef VLAN_CAPABILITIES
 		VLAN_CAPABILITIES(ifp);
 #endif
-		ADAPTER_UNLOCK(sc);
+fail:
+		end_synchronized_op(sc, 0);
 		break;
 
 	case SIOCSIFMEDIA:
@@ -1625,21 +1604,28 @@ prep_firmware(struct adapter *sc)
 
 	/* Partition adapter resources as specified in the config file. */
 	if (sc->flags & MASTER_PF) {
-		if (strncmp(t4_cfg_file, "default", sizeof(t4_cfg_file))) {
+		snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s",
+		    pci_get_device(sc->dev) == 0x440a ? "uwire" : t4_cfg_file);
+		if (strncmp(sc->cfg_file, "default", sizeof(sc->cfg_file))) {
 			char s[32];
 
-			snprintf(s, sizeof(s), "t4fw_cfg_%s", t4_cfg_file);
+			snprintf(s, sizeof(s), "t4fw_cfg_%s", sc->cfg_file);
 			cfg = firmware_get(s);
 			if (cfg == NULL) {
 				device_printf(sc->dev,
 				    "unable to locate %s module, "
 				    "will use default config file.\n", s);
+				snprintf(sc->cfg_file, sizeof(sc->cfg_file),
+				    "%s", "default");
 			}
 		}
 
 		rc = partition_resources(sc, cfg ? cfg : default_cfg);
 		if (rc != 0)
 			goto done;	/* error message displayed already */
+	} else {
+		snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", "notme");
+		sc->cfcsum = (u_int)-1;
 	}
 
 	sc->flags |= FW_OK;
@@ -1887,7 +1873,9 @@ get_params__post_init(struct adapter *sc)
 	param[1] = FW_PARAM_PFVF(EQ_START);
 	param[2] = FW_PARAM_PFVF(FILTER_START);
 	param[3] = FW_PARAM_PFVF(FILTER_END);
-	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val);
+	param[4] = FW_PARAM_PFVF(L2T_START);
+	param[5] = FW_PARAM_PFVF(L2T_END);
+	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to query parameters (post_init): %d.\n", rc);
@@ -1898,6 +1886,11 @@ get_params__post_init(struct adapter *sc)
 	sc->sge.eq_start = val[1];
 	sc->tids.ftid_base = val[2];
 	sc->tids.nftids = val[3] - val[2] + 1;
+	sc->vres.l2t.start = val[4];
+	sc->vres.l2t.size = val[5] - val[4] + 1;
+	KASSERT(sc->vres.l2t.size <= L2T_SIZE,
+	    ("%s: L2 table size (%u) larger than expected (%u)",
+	    __func__, sc->vres.l2t.size, L2T_SIZE));
 
 	/* get capabilites */
 	bzero(&caps, sizeof(caps));
@@ -2111,7 +2104,7 @@ update_mac_settings(struct port_info *pi, int flags)
 	struct adapter *sc = pi->adapter;
 	int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
 
-	PORT_LOCK_ASSERT_OWNED(pi);
+	ASSERT_SYNCHRONIZED_OP(sc);
 	KASSERT(flags, ("%s: not told what to update.", __func__));
 
 	if (flags & XGMAC_MTU)
@@ -2213,39 +2206,74 @@ mcfail:
 	return (rc);
 }
 
-static int
-cxgbe_init_locked(struct port_info *pi)
+int
+begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
+    char *wmesg)
 {
-	struct adapter *sc = pi->adapter;
-	int rc = 0;
+	int rc, pri;
 
-	ADAPTER_LOCK_ASSERT_OWNED(sc);
+#ifdef WITNESS
+	/* the caller thinks it's ok to sleep, but is it really? */
+	if (flags & SLEEP_OK)
+		pause("t4slptst", 1);
+#endif
 
-	while (!IS_DOOMED(pi) && IS_BUSY(sc)) {
-		if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4init", 0)) {
+	if (INTR_OK)
+		pri = PCATCH;
+	else
+		pri = 0;
+
+	ADAPTER_LOCK(sc);
+	for (;;) {
+
+		if (pi && IS_DOOMED(pi)) {
+			rc = ENXIO;
+			goto done;
+		}
+
+		if (!IS_BUSY(sc)) {
+			rc = 0;
+			break;
+		}
+
+		if (!(flags & SLEEP_OK)) {
+			rc = EBUSY;
+			goto done;
+		}
+
+		if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
 			rc = EINTR;
 			goto done;
 		}
 	}
-	if (IS_DOOMED(pi)) {
-		rc = ENXIO;
-		goto done;
-	}
-	KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
 
-	/* Give up the adapter lock, port init code can sleep. */
+	KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
 	SET_BUSY(sc);
-	ADAPTER_UNLOCK(sc);
-
-	rc = cxgbe_init_synchronized(pi);
+#ifdef INVARIANTS
+	sc->last_op = wmesg;
+	sc->last_op_thr = curthread;
+#endif
 
 done:
-	ADAPTER_LOCK(sc);
+	if (!(flags & HOLD_LOCK) || rc)
+		ADAPTER_UNLOCK(sc);
+
+	return (rc);
+}
+
+void
+end_synchronized_op(struct adapter *sc, int flags)
+{
+
+	if (flags & LOCK_HELD)
+		ADAPTER_LOCK_ASSERT_OWNED(sc);
+	else
+		ADAPTER_LOCK(sc);
+
 	KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
 	CLR_BUSY(sc);
-	wakeup_one(&sc->flags);
+	wakeup(&sc->flags);
 	ADAPTER_UNLOCK(sc);
-	return (rc);
 }
 
 static int
@@ -2255,7 +2283,7 @@ cxgbe_init_synchronized(struct port_info *pi)
 	struct ifnet *ifp = pi->ifp;
 	int rc = 0;
 
-	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
+	ASSERT_SYNCHRONIZED_OP(sc);
 
 	if (isset(&sc->open_device_map, pi->port_id)) {
 		KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
@@ -2271,9 +2299,7 @@ cxgbe_init_synchronized(struct port_info *pi)
 	    ((rc = port_full_init(pi)) != 0))
 		return (rc); /* error message displayed already */
 
-	PORT_LOCK(pi);
 	rc = update_mac_settings(pi, XGMAC_ALL);
-	PORT_UNLOCK(pi);
 	if (rc)
 		goto done;	/* error message displayed already */
 
@@ -2291,7 +2317,9 @@ cxgbe_init_synchronized(struct port_info *pi)
 
 	/* all ok */
 	setbit(&sc->open_device_map, pi->port_id);
+	PORT_LOCK(pi);
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
+	PORT_UNLOCK(pi);
 
 	callout_reset(&pi->tick, hz, cxgbe_tick, pi);
 done:
@@ -2301,39 +2329,6 @@ done:
 	return (rc);
 }
 
-static int
-cxgbe_uninit_locked(struct port_info *pi)
-{
-	struct adapter *sc = pi->adapter;
-	int rc;
-
-	ADAPTER_LOCK_ASSERT_OWNED(sc);
-
-	while (!IS_DOOMED(pi) && IS_BUSY(sc)) {
-		if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4uninit", 0)) {
-			rc = EINTR;
-			goto done;
-		}
-	}
-	if (IS_DOOMED(pi)) {
-		rc = ENXIO;
-		goto done;
-	}
-	KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
-	SET_BUSY(sc);
-	ADAPTER_UNLOCK(sc);
-
-	rc = cxgbe_uninit_synchronized(pi);
-
-	ADAPTER_LOCK(sc);
-	KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
-	CLR_BUSY(sc);
-	wakeup_one(&sc->flags);
-done:
-	ADAPTER_UNLOCK(sc);
-	return (rc);
-}
-
 /*
  * Idempotent.
  */
@@ -2344,7 +2339,7 @@ cxgbe_uninit_synchronized(struct port_info *pi)
 	struct ifnet *ifp = pi->ifp;
 	int rc;
 
-	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
+	ASSERT_SYNCHRONIZED_OP(sc);
 
 	/*
 	 * Disable the VI so that all its data in either direction is discarded
@@ -2360,7 +2355,9 @@ cxgbe_uninit_synchronized(struct port_info *pi)
 	}
 
 	clrbit(&sc->open_device_map, pi->port_id);
+	PORT_LOCK(pi);
 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+	PORT_UNLOCK(pi);
 
 	pi->link_cfg.link_ok = 0;
 	pi->link_cfg.speed = 0;
@@ -2539,7 +2536,7 @@ port_full_init(struct port_info *pi)
 	struct sge_rxq *rxq;
 	int rc, i;
 
-	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
+	ASSERT_SYNCHRONIZED_OP(sc);
 	KASSERT((pi->flags & PORT_INIT_DONE) == 0,
 	    ("%s: PORT_INIT_DONE already", __func__));
 
@@ -3119,7 +3116,7 @@ t4_sysctls(struct adapter *sc)
 	    CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
 
 	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
-	    CTLFLAG_RD, &t4_cfg_file, 0, "configuration file");
+	    CTLFLAG_RD, &sc->cfg_file, 0, "configuration file");
 
 	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD,
 	    &sc->cfcsum, 0, "config file checksum");
@@ -3524,6 +3521,8 @@ sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
 	struct port_info *pi = arg1;
 	struct adapter *sc = pi->adapter;
 	int idx, rc, i;
+	struct sge_rxq *rxq;
+	uint8_t v;
 
 	idx = pi->tmr_idx;
 
@@ -3534,25 +3533,23 @@ sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
 	if (idx < 0 || idx >= SGE_NTIMERS)
 		return (EINVAL);
 
-	ADAPTER_LOCK(sc);
-	rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
-	if (rc == 0) {
-		struct sge_rxq *rxq;
-		uint8_t v;
+	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
+	    "t4tmr");
+	if (rc)
+		return (rc);
 
-		v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
-		for_each_rxq(pi, i, rxq) {
+	v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
+	for_each_rxq(pi, i, rxq) {
 #ifdef atomic_store_rel_8
-			atomic_store_rel_8(&rxq->iq.intr_params, v);
+		atomic_store_rel_8(&rxq->iq.intr_params, v);
 #else
-			rxq->iq.intr_params = v;
+		rxq->iq.intr_params = v;
 #endif
-		}
-		pi->tmr_idx = idx;
 	}
+	pi->tmr_idx = idx;
 
-	ADAPTER_UNLOCK(sc);
-	return (rc);
+	end_synchronized_op(sc, LOCK_HELD);
+	return (0);
 }
 
 static int
@@ -3571,15 +3568,17 @@ sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
 	if (idx < -1 || idx >= SGE_NCOUNTERS)
 		return (EINVAL);
 
-	ADAPTER_LOCK(sc);
-	rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
-	if (rc == 0 && pi->flags & PORT_INIT_DONE)
-		rc = EBUSY; /* cannot be changed once the queues are created */
+	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
+	    "t4pktc");
+	if (rc)
+		return (rc);
 
-	if (rc == 0)
+	if (pi->flags & PORT_INIT_DONE)
+		rc = EBUSY; /* cannot be changed once the queues are created */
+	else
 		pi->pktc_idx = idx;
 
-	ADAPTER_UNLOCK(sc);
+	end_synchronized_op(sc, LOCK_HELD);
 	return (rc);
 }
 
@@ -3599,15 +3598,17 @@ sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
 	if (qsize < 128 || (qsize & 7))
 		return (EINVAL);
 
-	ADAPTER_LOCK(sc);
-	rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
-	if (rc == 0 && pi->flags & PORT_INIT_DONE)
-		rc = EBUSY; /* cannot be changed once the queues are created */
+	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
+	    "t4rxqs");
+	if (rc)
+		return (rc);
 
-	if (rc == 0)
+	if (pi->flags & PORT_INIT_DONE)
+		rc = EBUSY; /* cannot be changed once the queues are created */
+	else
 		pi->qsize_rxq = qsize;
 
-	ADAPTER_UNLOCK(sc);
+	end_synchronized_op(sc, LOCK_HELD);
 	return (rc);
 }
 
@@ -3624,18 +3625,21 @@ sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
 	if (rc != 0 || req->newptr == NULL)
 		return (rc);
 
-	if (qsize < 128)
+	/* bufring size must be powerof2 */
+	if (qsize < 128 || !powerof2(qsize))
 		return (EINVAL);
 
-	ADAPTER_LOCK(sc);
-	rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
-	if (rc == 0 && pi->flags & PORT_INIT_DONE)
-		rc = EBUSY; /* cannot be changed once the queues are created */
+	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
+	    "t4txqs");
+	if (rc)
+		return (rc);
 
-	if (rc == 0)
+	if (pi->flags & PORT_INIT_DONE)
+		rc = EBUSY; /* cannot be changed once the queues are created */
+	else
 		pi->qsize_txq = qsize;
 
-	ADAPTER_UNLOCK(sc);
+	end_synchronized_op(sc, LOCK_HELD);
 	return (rc);
 }
 
@@ -4674,8 +4678,14 @@ fspec_to_fconf(struct t4_filter_specification *fs)
 static int
 get_filter_mode(struct adapter *sc, uint32_t *mode)
 {
+	int rc;
 	uint32_t fconf;
 
+	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
+	    "t4getfm");
+	if (rc)
+		return (rc);
+
 	t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
 	    A_TP_VLAN_PRI_MAP);
 
@@ -4687,6 +4697,7 @@ get_filter_mode(struct adapter *sc, uint32_t *mode)
 
 	*mode = fconf_to_mode(sc->filter_mode);
 
+	end_synchronized_op(sc, LOCK_HELD);
 	return (0);
 }
 
@@ -4698,11 +4709,10 @@ set_filter_mode(struct adapter *sc, uint32_t mode)
 
 	fconf = mode_to_fconf(mode);
 
-	ADAPTER_LOCK(sc);
-	if (IS_BUSY(sc)) {
-		rc = EAGAIN;
-		goto done;
-	}
+	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
+	    "t4setfm");
+	if (rc)
+		return (rc);
 
 	if (sc->tids.ftids_in_use > 0) {
 		rc = EBUSY;
@@ -4725,7 +4735,7 @@ set_filter_mode(struct adapter *sc, uint32_t mode)
 #endif
 
 done:
-	ADAPTER_UNLOCK(sc);
+	end_synchronized_op(sc, LOCK_HELD);
 	return (rc);
 }
 
@@ -4746,18 +4756,18 @@ get_filter_hits(struct adapter *sc, uint32_t fid)
 static int
 get_filter(struct adapter *sc, struct t4_filter *t)
 {
-	int i, nfilters = sc->tids.nftids;
+	int i, rc, nfilters = sc->tids.nftids;
 	struct filter_entry *f;
 
-	ADAPTER_LOCK_ASSERT_OWNED(sc);
-
-	if (IS_BUSY(sc))
-		return (EAGAIN);
+	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
+	    "t4getf");
+	if (rc)
+		return (rc);
 
 	if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
 	    t->idx >= nfilters) {
 		t->idx = 0xffffffff;
-		return (0);
+		goto done;
 	}
 
 	f = &sc->tids.ftid_tab[t->idx];
@@ -4772,11 +4782,13 @@ get_filter(struct adapter *sc, struct t4_filter *t)
 				t->hits = UINT64_MAX;
 			t->fs = f->fs;
 
-			return (0);
+			goto done;
 		}
 	}
 
 	t->idx = 0xffffffff;
+done:
+	end_synchronized_op(sc, LOCK_HELD);
 	return (0);
 }
 
@@ -4785,40 +4797,58 @@ set_filter(struct adapter *sc, struct t4_filter *t)
 {
 	unsigned int nfilters, nports;
 	struct filter_entry *f;
-	int i;
+	int i, rc;
 
-	ADAPTER_LOCK_ASSERT_OWNED(sc);
+	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
+	if (rc)
+		return (rc);
 
 	nfilters = sc->tids.nftids;
 	nports = sc->params.nports;
 
-	if (nfilters == 0)
-		return (ENOTSUP);
+	if (nfilters == 0) {
+		rc = ENOTSUP;
+		goto done;
+	}
 
-	if (!(sc->flags & FULL_INIT_DONE))
-		return (EAGAIN);
+	if (!(sc->flags & FULL_INIT_DONE)) {
+		rc = EAGAIN;
+		goto done;
+	}
 
-	if (t->idx >= nfilters)
-		return (EINVAL);
+	if (t->idx >= nfilters) {
+		rc = EINVAL;
+		goto done;
+	}
 
 	/* Validate against the global filter mode */
-	if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode)
-		return (E2BIG);
+	if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode) {
+		rc = E2BIG;
+		goto done;
+	}
 
-	if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports)
-		return (EINVAL);
+	if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
+		rc = EINVAL;
+		goto done;
+	}
 
-	if (t->fs.val.iport >= nports)
-		return (EINVAL);
+	if (t->fs.val.iport >= nports) {
+		rc = EINVAL;
+		goto done;
+	}
 
 	/* Can't specify an iq if not steering to it */
-	if (!t->fs.dirsteer && t->fs.iq)
-		return (EINVAL);
+	if (!t->fs.dirsteer && t->fs.iq) {
+		rc = EINVAL;
+		goto done;
+	}
 
 	/* IPv6 filter idx must be 4 aligned */
 	if (t->fs.type == 1 &&
-	    ((t->idx & 0x3) || t->idx + 4 >= nfilters))
-		return (EINVAL);
+	    ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
+		rc = EINVAL;
+		goto done;
+	}
 
 	if (sc->tids.ftid_tab == NULL) {
 		KASSERT(sc->tids.ftids_in_use == 0,
@@ -4827,17 +4857,24 @@ set_filter(struct adapter *sc, struct t4_filter *t)
 
 		sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
 		    nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
-		if (sc->tids.ftid_tab == NULL)
-			return (ENOMEM);
+		if (sc->tids.ftid_tab == NULL) {
+			rc = ENOMEM;
+			goto done;
+		}
+		mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
 	}
 
 	for (i = 0; i < 4; i++) {
 		f = &sc->tids.ftid_tab[t->idx + i];
 
-		if (f->pending || f->valid)
-			return (EBUSY);
-		if (f->locked)
-			return (EPERM);
+		if (f->pending || f->valid) {
+			rc = EBUSY;
+			goto done;
+		}
+		if (f->locked) {
+			rc = EPERM;
+			goto done;
+		}
 
 		if (t->fs.type == 0)
 			break;
@@ -4846,7 +4883,27 @@ set_filter(struct adapter *sc, struct t4_filter *t)
 	f = &sc->tids.ftid_tab[t->idx];
 	f->fs = t->fs;
 
-	return set_filter_wr(sc, t->idx);
+	rc = set_filter_wr(sc, t->idx);
+done:
+	end_synchronized_op(sc, 0);
+
+	if (rc == 0) {
+		mtx_lock(&sc->tids.ftid_lock);
+		for (;;) {
+			if (f->pending == 0) {
+				rc = f->valid ? 0 : EIO;
+				break;
+			}
+
+			if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
+			    PCATCH, "t4setfw", 0)) {
+				rc = EINPROGRESS;
+				break;
+			}
+		}
+		mtx_unlock(&sc->tids.ftid_lock);
+	}
+	return (rc);
 }
 
 static int
@@ -4854,37 +4911,67 @@ del_filter(struct adapter *sc, struct t4_filter *t)
 {
 	unsigned int nfilters;
 	struct filter_entry *f;
+	int rc;
 
-	ADAPTER_LOCK_ASSERT_OWNED(sc);
-
-	if (IS_BUSY(sc))
-		return (EAGAIN);
+	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
+	if (rc)
+		return (rc);
 
 	nfilters = sc->tids.nftids;
 
-	if (nfilters == 0)
-		return (ENOTSUP);
+	if (nfilters == 0) {
+		rc = ENOTSUP;
+		goto done;
+	}
 
 	if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
-	    t->idx >= nfilters)
-		return (EINVAL);
+	    t->idx >= nfilters) {
+		rc = EINVAL;
+		goto done;
+	}
 
-	if (!(sc->flags & FULL_INIT_DONE))
-		return (EAGAIN);
+	if (!(sc->flags & FULL_INIT_DONE)) {
+		rc = EAGAIN;
+		goto done;
+	}
 
 	f = &sc->tids.ftid_tab[t->idx];
 
-	if (f->pending)
-		return (EBUSY);
-	if (f->locked)
-		return (EPERM);
+	if (f->pending) {
+		rc = EBUSY;
+		goto done;
+	}
+	if (f->locked) {
+		rc = EPERM;
+		goto done;
+	}
 
 	if (f->valid) {
 		t->fs = f->fs;	/* extra info for the caller */
-		return del_filter_wr(sc, t->idx);
+		rc = del_filter_wr(sc, t->idx);
 	}
 
-	return (0);
+done:
+	end_synchronized_op(sc, 0);
+
+	if (rc == 0) {
+		mtx_lock(&sc->tids.ftid_lock);
+		for (;;) {
+			if (f->pending == 0) {
+				rc = f->valid ? EIO : 0;
+				break;
+			}
+
+			if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
+			    PCATCH, "t4delfw", 0)) {
+				rc = EINPROGRESS;
+				break;
+			}
+		}
+		mtx_unlock(&sc->tids.ftid_lock);
+	}
+
+	return (rc);
 }
 
 static void
@@ -4904,7 +4991,7 @@ set_filter_wr(struct adapter *sc, int fidx)
 	struct fw_filter_wr *fwr;
 	unsigned int ftid;
 
-	ADAPTER_LOCK_ASSERT_OWNED(sc);
+	ASSERT_SYNCHRONIZED_OP(sc);
 
 	if (f->fs.newdmac || f->fs.newvlan) {
 		/* This filter needs an L2T entry; allocate one. */
@@ -5007,8 +5094,6 @@ del_filter_wr(struct adapter *sc, int fidx)
 	struct fw_filter_wr *fwr;
 	unsigned int ftid;
 
-	ADAPTER_LOCK_ASSERT_OWNED(sc);
-
 	ftid = sc->tids.ftid_base + fidx;
 
 	wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
@@ -5039,8 +5124,10 @@ t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 		unsigned int rc = G_COOKIE(rpl->cookie);
 		struct filter_entry *f = &sc->tids.ftid_tab[idx];
 
-		ADAPTER_LOCK(sc);
+		mtx_lock(&sc->tids.ftid_lock);
 		if (rc == FW_FILTER_WR_FLT_ADDED) {
+			KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
+			    __func__, idx));
 			f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
 			f->pending = 0;  /* asynchronous setup completed */
 			f->valid = 1;
@@ -5055,7 +5142,8 @@ t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 			clear_filter(f);
 			sc->tids.ftids_in_use--;
 		}
-		ADAPTER_UNLOCK(sc);
+		wakeup(&sc->tids.ftid_tab);
+		mtx_unlock(&sc->tids.ftid_lock);
 	}
 
 	return (0);
@@ -5064,29 +5152,63 @@ t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 static int
 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
 {
-	int rc = EINVAL;
+	int rc;
 
 	if (cntxt->cid > M_CTXTQID)
-		return (rc);
+		return (EINVAL);
 
 	if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
 	    cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
-		return (rc);
+		return (EINVAL);
 
 	if (sc->flags & FW_OK) {
-		ADAPTER_LOCK(sc);	/* Avoid parallel t4_wr_mbox */
-		rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
-		    &cntxt->data[0]);
-		ADAPTER_UNLOCK(sc);
+		rc = begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4ctxt");
+		if (rc == 0) {
+			rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid,
+			    cntxt->mem_id, &cntxt->data[0]);
+			end_synchronized_op(sc, LOCK_HELD);
+			if (rc == 0)
+				return (0);
+		}
 	}
 
-	if (rc != 0) {
-		/* Read via firmware failed or wasn't even attempted */
+	/*
+	 * Read via firmware failed or wasn't even attempted.  Read directly via
+	 * the backdoor.
+	 */
+	rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id,
+	    &cntxt->data[0]);
+	return (rc);
+}
+
+static int
+load_fw(struct adapter *sc, struct t4_data *fw)
+{
+	int rc;
+	uint8_t *fw_data;
+
+	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
+	if (rc)
+		return (rc);
+
+	if (sc->flags & FULL_INIT_DONE) {
+		rc = EBUSY;
+		goto done;
+	}
 
-		rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id,
-		    &cntxt->data[0]);
+	fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
+	if (fw_data == NULL) {
+		rc = ENOMEM;
+		goto done;
 	}
 
+	rc = copyin(fw->data, fw_data, fw->len);
+	if (rc == 0)
+		rc = -t4_load_fw(sc, fw_data, fw->len);
+
+	free(fw_data, M_CXGBE);
+done:
+	end_synchronized_op(sc, 0);
 	return (rc);
 }
 
@@ -5173,8 +5295,6 @@ read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
 {
 	int rc;
 
-	ADAPTER_LOCK_ASSERT_OWNED(sc);	/* for mbox */
-
 	if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
 		return (EINVAL);
 
@@ -5183,8 +5303,12 @@ read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
 		return (ENOTSUP);
 	}
 
+	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
+	if (rc)
+		return (rc);
 	rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
 	    i2cd->offset, &i2cd->data[0]);
+	end_synchronized_op(sc, 0);
 
 	return (rc);
 }
@@ -5354,56 +5478,78 @@ t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
 		rc = set_filter_mode(sc, *(uint32_t *)data);
 		break;
 	case CHELSIO_T4_GET_FILTER:
-		ADAPTER_LOCK(sc);
 		rc = get_filter(sc, (struct t4_filter *)data);
-		ADAPTER_UNLOCK(sc);
 		break;
 	case CHELSIO_T4_SET_FILTER:
-		ADAPTER_LOCK(sc);
 		rc = set_filter(sc, (struct t4_filter *)data);
-		ADAPTER_UNLOCK(sc);
 		break;
 	case CHELSIO_T4_DEL_FILTER:
-		ADAPTER_LOCK(sc);
 		rc = del_filter(sc, (struct t4_filter *)data);
-		ADAPTER_UNLOCK(sc);
 		break;
 	case CHELSIO_T4_GET_SGE_CONTEXT:
 		rc = get_sge_context(sc, (struct t4_sge_context *)data);
 		break;
-	case CHELSIO_T4_LOAD_FW: {
-		struct t4_data *fw = (struct t4_data *)data;
-		uint8_t *fw_data;
-
-		if (sc->flags & FULL_INIT_DONE)
-			return (EBUSY);
-
-		fw_data = malloc(fw->len, M_CXGBE, M_NOWAIT);
-		if (fw_data == NULL)
-			return (ENOMEM);
-
-		rc = copyin(fw->data, fw_data, fw->len);
-		if (rc == 0)
-			rc = -t4_load_fw(sc, fw_data, fw->len);
-
-		free(fw_data, M_CXGBE);
+	case CHELSIO_T4_LOAD_FW:
+		rc = load_fw(sc, (struct t4_data *)data);
 		break;
-	}
 	case CHELSIO_T4_GET_MEM:
 		rc = read_card_mem(sc, (struct t4_mem_range *)data);
 		break;
 	case CHELSIO_T4_GET_I2C:
-		ADAPTER_LOCK(sc);
 		rc = read_i2c(sc, (struct t4_i2c_data *)data);
-		ADAPTER_UNLOCK(sc);
 		break;
 	case CHELSIO_T4_CLEAR_STATS: {
+		int i;
 		u_int port_id = *(uint32_t *)data;
+		struct port_info *pi;
 
 		if (port_id >= sc->params.nports)
 			return (EINVAL);
 
+		/* MAC stats */
 		t4_clr_port_stats(sc, port_id);
+
+		pi = sc->port[port_id];
+		if (pi->flags & PORT_INIT_DONE) {
+			struct sge_rxq *rxq;
+			struct sge_txq *txq;
+			struct sge_wrq *wrq;
+
+			for_each_rxq(pi, i, rxq) {
+#if defined(INET) || defined(INET6)
+				rxq->lro.lro_queued = 0;
+				rxq->lro.lro_flushed = 0;
+#endif
+				rxq->rxcsum = 0;
+				rxq->vlan_extraction = 0;
+			}
+
+			for_each_txq(pi, i, txq) {
+				txq->txcsum = 0;
+				txq->tso_wrs = 0;
+				txq->vlan_insertion = 0;
+				txq->imm_wrs = 0;
+				txq->sgl_wrs = 0;
+				txq->txpkt_wrs = 0;
+				txq->txpkts_wrs = 0;
+				txq->txpkts_pkts = 0;
+				txq->br->br_drops = 0;
+				txq->no_dmamap = 0;
+				txq->no_desc = 0;
+			}
+
+#ifdef TCP_OFFLOAD
+			/* nothing to clear for each ofld_rxq */
+
+			for_each_ofld_txq(pi, i, wrq) {
+				wrq->tx_wrs = 0;
+				wrq->no_desc = 0;
+			}
+#endif
+			wrq = &sc->sge.ctrlq[pi->port_id];
+			wrq->tx_wrs = 0;
+			wrq->no_desc = 0;
+		}
 		break;
 	}
 	default:
@@ -5420,16 +5566,16 @@ toe_capability(struct port_info *pi, int enable)
 	int rc;
 	struct adapter *sc = pi->adapter;
 
-	ADAPTER_LOCK_ASSERT_OWNED(sc);
+	ASSERT_SYNCHRONIZED_OP(sc);
 
 	if (!is_offload(sc))
 		return (ENODEV);
 
 	if (enable) {
 		if (!(sc->flags & FULL_INIT_DONE)) {
-			log(LOG_WARNING,
-			    "You must enable a cxgbe interface first\n");
-			return (EAGAIN);
+			rc = cxgbe_init_synchronized(pi);
+			if (rc)
+				return (rc);
 		}
 
 		if (isset(&sc->offload_map, pi->port_id))
@@ -5518,6 +5664,8 @@ t4_activate_uld(struct adapter *sc, int id)
 	int rc = EAGAIN;
 	struct uld_info *ui;
 
+	ASSERT_SYNCHRONIZED_OP(sc);
+
 	mtx_lock(&t4_uld_list_lock);
 
 	SLIST_FOREACH(ui, &t4_uld_list, link) {
@@ -5540,6 +5688,8 @@ t4_deactivate_uld(struct adapter *sc, int id)
 	int rc = EINVAL;
 	struct uld_info *ui;
 
+	ASSERT_SYNCHRONIZED_OP(sc);
+
 	mtx_lock(&t4_uld_list_lock);
 
 	SLIST_FOREACH(ui, &t4_uld_list, link) {
diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c
index 62d9eb3..62ceec4 100644
--- a/sys/dev/cxgbe/t4_sge.c
+++ b/sys/dev/cxgbe/t4_sge.c
@@ -2362,6 +2362,8 @@ alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx,
 	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "txpkts_pkts", CTLFLAG_RD,
 	    &txq->txpkts_pkts, "# of frames tx'd using txpkts work requests");
 
+	SYSCTL_ADD_UQUAD(&pi->ctx, children, OID_AUTO, "br_drops", CTLFLAG_RD,
+	    &txq->br->br_drops, "# of drops in the buf_ring for this queue");
 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "no_dmamap", CTLFLAG_RD,
 	    &txq->no_dmamap, 0, "# of times txq ran out of DMA maps");
 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "no_desc", CTLFLAG_RD,
diff --git a/sys/dev/cxgbe/tom/t4_connect.c b/sys/dev/cxgbe/tom/t4_connect.c
index 8d36b1e..17ed1d3 100644
--- a/sys/dev/cxgbe/tom/t4_connect.c
+++ b/sys/dev/cxgbe/tom/t4_connect.c
@@ -29,6 +29,7 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
+#include "opt_inet6.h"
 
 #ifdef TCP_OFFLOAD
 #include <sys/param.h>
@@ -195,7 +196,7 @@ do_act_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
 	CTR3(KTR_CXGBE, "%s: atid %u, status %u ", __func__, atid, status);
 
 	/* Ignore negative advice */
-	if (status == CPL_ERR_RTX_NEG_ADVICE)
+	if (negative_advice(status))
 		return (0);
 
 	free_atid(sc, atid);
@@ -220,10 +221,9 @@ do_act_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
  * Options2 for active open.
  */
 static uint32_t
-calc_opt2a(struct socket *so)
+calc_opt2a(struct socket *so, struct toepcb *toep)
 {
 	struct tcpcb *tp = so_sototcpcb(so);
-	struct toepcb *toep = tp->t_toe;
 	struct port_info *pi = toep->port;
 	struct adapter *sc = pi->adapter;
 	uint32_t opt2 = 0;
@@ -260,6 +260,12 @@ t4_init_connect_cpl_handlers(struct adapter *sc)
 	t4_register_cpl_handler(sc, CPL_ACT_OPEN_RPL, do_act_open_rpl);
 }
 
+#define DONT_OFFLOAD_ACTIVE_OPEN(x)	do { \
+	reason = __LINE__; \
+	rc = (x); \
+	goto failed; \
+} while (0)
+
 /*
  * active open (soconnect).
  *
@@ -275,20 +281,19 @@ t4_connect(struct toedev *tod, struct socket *so, struct rtentry *rt,
     struct sockaddr *nam)
 {
 	struct adapter *sc = tod->tod_softc;
+	struct tom_data *td = tod_td(tod);
 	struct toepcb *toep = NULL;
 	struct wrqe *wr = NULL;
-	struct cpl_act_open_req *cpl;
-	struct l2t_entry *e = NULL;
 	struct ifnet *rt_ifp = rt->rt_ifp;
 	struct port_info *pi;
-	int atid = -1, mtu_idx, rscale, qid_atid, rc = ENOMEM;
+	int mtu_idx, rscale, qid_atid, rc, isipv6;
 	struct inpcb *inp = sotoinpcb(so);
 	struct tcpcb *tp = intotcpcb(inp);
+	int reason;
 
 	INP_WLOCK_ASSERT(inp);
-
-	if (nam->sa_family != AF_INET)
-		CXGBE_UNIMPLEMENTED("IPv6 connect");
+	KASSERT(nam->sa_family == AF_INET || nam->sa_family == AF_INET6,
+	    ("%s: dest addr %p has family %u", __func__, nam, nam->sa_family));
 
 	if (rt_ifp->if_type == IFT_ETHER)
 		pi = rt_ifp->if_softc;
@@ -297,30 +302,29 @@ t4_connect(struct toedev *tod, struct socket *so, struct rtentry *rt,
 
 		pi = ifp->if_softc;
 	} else if (rt_ifp->if_type == IFT_IEEE8023ADLAG)
-		return (ENOSYS);	/* XXX: implement lagg support */
+		DONT_OFFLOAD_ACTIVE_OPEN(ENOSYS); /* XXX: implement lagg+TOE */
 	else
-		return (ENOTSUP);
+		DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
 
 	toep = alloc_toepcb(pi, -1, -1, M_NOWAIT);
 	if (toep == NULL)
-		goto failed;
+		DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
 
-	atid = alloc_atid(sc, toep);
-	if (atid < 0)
-		goto failed;
+	toep->tid = alloc_atid(sc, toep);
+	if (toep->tid < 0)
+		DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
 
-	e = t4_l2t_get(pi, rt_ifp,
+	toep->l2te = t4_l2t_get(pi, rt_ifp,
 	    rt->rt_flags & RTF_GATEWAY ? rt->rt_gateway : nam);
-	if (e == NULL)
-		goto failed;
+	if (toep->l2te == NULL)
+		DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
 
-	wr = alloc_wrqe(sizeof(*cpl), toep->ctrlq);
+	isipv6 = nam->sa_family == AF_INET6;
+	wr = alloc_wrqe(isipv6 ? sizeof(struct cpl_act_open_req6) :
+	    sizeof(struct cpl_act_open_req), toep->ctrlq);
 	if (wr == NULL)
-		goto failed;
-	cpl = wrtod(wr);
+		DONT_OFFLOAD_ACTIVE_OPEN(ENOMEM);
 
-	toep->tid = atid;
-	toep->l2te = e;
 	if (sc->tt.ddp && (so->so_options & SO_NO_DDP) == 0)
 		set_tcpddp_ulp_mode(toep);
 	else
@@ -330,8 +334,6 @@ t4_connect(struct toedev *tod, struct socket *so, struct rtentry *rt,
 	toep->rx_credits = min(select_rcv_wnd(so) >> 10, M_RCV_BUFSIZ);
 	SOCKBUF_UNLOCK(&so->so_rcv);
 
-	offload_socket(so, toep);
-
 	/*
 	 * The kernel sets request_r_scale based on sb_max whereas we need to
 	 * take hardware's MAX_RCV_WND into account too.  This is normally a
@@ -342,39 +344,78 @@ t4_connect(struct toedev *tod, struct socket *so, struct rtentry *rt,
 	else
 		rscale = 0;
 	mtu_idx = find_best_mtu_idx(sc, &inp->inp_inc, 0);
-	qid_atid = (toep->ofld_rxq->iq.abs_id << 14) | atid;
-
-	INIT_TP_WR(cpl, 0);
-	OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));
-	inp_4tuple_get(inp, &cpl->local_ip, &cpl->local_port, &cpl->peer_ip,
-	    &cpl->peer_port);
-	cpl->opt0 = calc_opt0(so, pi, e, mtu_idx, rscale, toep->rx_credits,
-	    toep->ulp_mode);
-	cpl->params = select_ntuple(pi, e, sc->filter_mode);
-	cpl->opt2 = calc_opt2a(so);
+	qid_atid = (toep->ofld_rxq->iq.abs_id << 14) | toep->tid;
+
+	if (isipv6) {
+		struct cpl_act_open_req6 *cpl = wrtod(wr);
+
+		if ((inp->inp_vflag & INP_IPV6) == 0) {
+			/* XXX think about this a bit more */
+			log(LOG_ERR,
+			    "%s: time to think about AF_INET6 + vflag 0x%x.\n",
+			    __func__, inp->inp_vflag);
+			DONT_OFFLOAD_ACTIVE_OPEN(ENOTSUP);
+		}
+
+		toep->ce = hold_lip(td, &inp->in6p_laddr);
+		if (toep->ce == NULL)
+			DONT_OFFLOAD_ACTIVE_OPEN(ENOENT);
+
+		INIT_TP_WR(cpl, 0);
+		OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
+		    qid_atid));
+
+		cpl->local_port = inp->inp_lport;
+		cpl->local_ip_hi = *(uint64_t *)&inp->in6p_laddr.s6_addr[0];
+		cpl->local_ip_lo = *(uint64_t *)&inp->in6p_laddr.s6_addr[8];
+		cpl->peer_port = inp->inp_fport;
+		cpl->peer_ip_hi = *(uint64_t *)&inp->in6p_faddr.s6_addr[0];
+		cpl->peer_ip_lo = *(uint64_t *)&inp->in6p_faddr.s6_addr[8];
+		cpl->opt0 = calc_opt0(so, pi, toep->l2te, mtu_idx, rscale,
+		    toep->rx_credits, toep->ulp_mode);
+		cpl->params = select_ntuple(pi, toep->l2te, sc->filter_mode);
+		cpl->opt2 = calc_opt2a(so, toep);
+	} else {
+		struct cpl_act_open_req *cpl = wrtod(wr);
+
+		INIT_TP_WR(cpl, 0);
+		OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
+		    qid_atid));
+		inp_4tuple_get(inp, &cpl->local_ip, &cpl->local_port,
+		    &cpl->peer_ip, &cpl->peer_port);
+		cpl->opt0 = calc_opt0(so, pi, toep->l2te, mtu_idx, rscale,
+		    toep->rx_credits, toep->ulp_mode);
+		cpl->params = select_ntuple(pi, toep->l2te, sc->filter_mode);
+		cpl->opt2 = calc_opt2a(so, toep);
+	}
 
 	CTR5(KTR_CXGBE, "%s: atid %u (%s), toep %p, inp %p", __func__,
 	    toep->tid, tcpstates[tp->t_state], toep, inp);
 
-	rc = t4_l2t_send(sc, wr, e);
+	offload_socket(so, toep);
+	rc = t4_l2t_send(sc, wr, toep->l2te);
 	if (rc == 0) {
 		toep->flags |= TPF_CPL_PENDING;
 		return (0);
 	}
 
 	undo_offload_socket(so);
+	reason = __LINE__;
 failed:
-	CTR5(KTR_CXGBE, "%s: FAILED, atid %d, toep %p, l2te %p, wr %p",
-	    __func__, atid, toep, e, wr);
+	CTR3(KTR_CXGBE, "%s: not offloading (%d), rc %d", __func__, reason, rc);
 
-	if (e)
-		t4_l2t_release(e);
 	if (wr)
 		free_wrqe(wr);
-	if (atid >= 0)
-		free_atid(sc, atid);
-	if (toep)
+
+	if (toep) {
+		if (toep->tid >= 0)
+			free_atid(sc, toep->tid);
+		if (toep->l2te)
+			t4_l2t_release(toep->l2te);
+		if (toep->ce)
+			release_lip(td, toep->ce);
 		free_toepcb(toep);
+	}
 
 	return (rc);
 }
diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c
index 6ae1ec4..9aead9f 100644
--- a/sys/dev/cxgbe/tom/t4_cpl_io.c
+++ b/sys/dev/cxgbe/tom/t4_cpl_io.c
@@ -1018,8 +1018,7 @@ do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 
 	KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
 
-	if (cpl->status == CPL_ERR_RTX_NEG_ADVICE ||
-	    cpl->status == CPL_ERR_PERSIST_NEG_ADVICE) {
+	if (negative_advice(cpl->status)) {
 		CTR4(KTR_CXGBE, "%s: negative advice %d for tid %d (0x%x)",
 		    __func__, cpl->status, tid, toep->flags);
 		return (0);	/* Ignore negative advice */
diff --git a/sys/dev/cxgbe/tom/t4_listen.c b/sys/dev/cxgbe/tom/t4_listen.c
index 523f7f3..b80702d 100644
--- a/sys/dev/cxgbe/tom/t4_listen.c
+++ b/sys/dev/cxgbe/tom/t4_listen.c
@@ -29,6 +29,7 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
+#include "opt_inet6.h"
 
 #ifdef TCP_OFFLOAD
 #include <sys/param.h>
@@ -50,6 +51,8 @@ __FBSDID("$FreeBSD$");
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
 #include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <netinet6/scope6_var.h>
 #include <netinet/tcp_timer.h>
 #include <netinet/tcp_var.h>
 #define TCPSTATES
@@ -63,9 +66,9 @@ __FBSDID("$FreeBSD$");
 #include "tom/t4_tom.h"
 
 /* stid services */
-static int alloc_stid(struct adapter *, void *);
-static void *lookup_stid(struct adapter *, int);
-static void free_stid(struct adapter *, int);
+static int alloc_stid(struct adapter *, struct listen_ctx *, int);
+static struct listen_ctx *lookup_stid(struct adapter *, int);
+static void free_stid(struct adapter *, struct listen_ctx *);
 
 /* lctx services */
 static struct listen_ctx *alloc_lctx(struct adapter *, struct inpcb *,
@@ -81,45 +84,105 @@ static inline void save_qids_in_mbuf(struct mbuf *, struct port_info *);
 static inline void get_qids_from_mbuf(struct mbuf *m, int *, int *);
 static void send_reset_synqe(struct toedev *, struct synq_entry *);
 
-/* XXX: won't work for IPv6 */
 static int
-alloc_stid(struct adapter *sc, void *ctx)
+alloc_stid(struct adapter *sc, struct listen_ctx *lctx, int isipv6)
 {
 	struct tid_info *t = &sc->tids;
-	int stid = -1;
+	u_int stid, n, f, mask;
+	struct stid_region *sr = &lctx->stid_region;
+
+	/*
+	 * An IPv6 server needs 2 naturally aligned stids (1 stid = 4 cells) in
+	 * the TCAM.  The start of the stid region is properly aligned (the chip
+	 * requires each region to be 128-cell aligned).
+	 */
+	n = isipv6 ? 2 : 1;
+	mask = n - 1;
+	KASSERT((t->stid_base & mask) == 0 && (t->nstids & mask) == 0,
+	    ("%s: stid region (%u, %u) not properly aligned.  n = %u",
+	    __func__, t->stid_base, t->nstids, n));
 
 	mtx_lock(&t->stid_lock);
-	if (t->sfree) {
-		union serv_entry *p = t->sfree;
-
-		stid = p - t->stid_tab;
-		stid += t->stid_base;
-		t->sfree = p->next;
-		p->data = ctx;
-		t->stids_in_use++;
+	if (n > t->nstids - t->stids_in_use) {
+		mtx_unlock(&t->stid_lock);
+		return (-1);
 	}
+
+	if (t->nstids_free_head >= n) {
+		/*
+		 * This allocation will definitely succeed because the region
+		 * starts at a good alignment and we just checked we have enough
+		 * stids free.
+		 */
+		f = t->nstids_free_head & mask;
+		t->nstids_free_head -= n + f;
+		stid = t->nstids_free_head;
+		TAILQ_INSERT_HEAD(&t->stids, sr, link);
+	} else {
+		struct stid_region *s;
+
+		stid = t->nstids_free_head;
+		TAILQ_FOREACH(s, &t->stids, link) {
+			stid += s->used + s->free;
+			f = stid & mask;
+			if (n <= s->free - f) {
+				stid -= n + f;
+				s->free -= n + f;
+				TAILQ_INSERT_AFTER(&t->stids, s, sr, link);
+				goto allocated;
+			}
+		}
+
+		if (__predict_false(stid != t->nstids)) {
+			panic("%s: stids TAILQ (%p) corrupt."
+			    "  At %d instead of %d at the end of the queue.",
+			    __func__, &t->stids, stid, t->nstids);
+		}
+
+		mtx_unlock(&t->stid_lock);
+		return (-1);
+	}
+
+allocated:
+	sr->used = n;
+	sr->free = f;
+	t->stids_in_use += n;
+	t->stid_tab[stid] = lctx;
 	mtx_unlock(&t->stid_lock);
-	return (stid);
+
+	KASSERT(((stid + t->stid_base) & mask) == 0,
+	    ("%s: EDOOFUS.", __func__));
+	return (stid + t->stid_base);
 }
 
-static void *
+static struct listen_ctx *
 lookup_stid(struct adapter *sc, int stid)
 {
 	struct tid_info *t = &sc->tids;
 
-	return (t->stid_tab[stid - t->stid_base].data);
+	return (t->stid_tab[stid - t->stid_base]);
 }
 
 static void
-free_stid(struct adapter *sc, int stid)
+free_stid(struct adapter *sc, struct listen_ctx *lctx)
 {
 	struct tid_info *t = &sc->tids;
-	union serv_entry *p = &t->stid_tab[stid - t->stid_base];
+	struct stid_region *sr = &lctx->stid_region;
+	struct stid_region *s;
+
+	KASSERT(sr->used > 0, ("%s: nonsense free (%d)", __func__, sr->used));
 
 	mtx_lock(&t->stid_lock);
-	p->next = t->sfree;
-	t->sfree = p;
-	t->stids_in_use--;
+	s = TAILQ_PREV(sr, stid_head, link);
+	if (s != NULL)
+		s->free += sr->used + sr->free;
+	else
+		t->nstids_free_head += sr->used + sr->free;
+	KASSERT(t->stids_in_use >= sr->used,
+	    ("%s: stids_in_use (%u) < stids being freed (%u)", __func__,
+	    t->stids_in_use, sr->used));
+	t->stids_in_use -= sr->used;
+	TAILQ_REMOVE(&t->stids, sr, link);
 	mtx_unlock(&t->stid_lock);
 }
 
@@ -134,7 +197,7 @@ alloc_lctx(struct adapter *sc, struct inpcb *inp, struct port_info *pi)
 	if (lctx == NULL)
 		return (NULL);
 
-	lctx->stid = alloc_stid(sc, lctx);
+	lctx->stid = alloc_stid(sc, lctx, inp->inp_vflag & INP_IPV6);
 	if (lctx->stid < 0) {
 		free(lctx, M_CXGBE);
 		return (NULL);
@@ -167,7 +230,7 @@ free_lctx(struct adapter *sc, struct listen_ctx *lctx)
 	CTR4(KTR_CXGBE, "%s: stid %u, lctx %p, inp %p",
 	    __func__, lctx->stid, lctx, lctx->inp);
 
-	free_stid(sc, lctx->stid);
+	free_stid(sc, lctx);
 	free(lctx, M_CXGBE);
 
 	return (in_pcbrele_wlocked(inp));
@@ -339,7 +402,7 @@ create_server(struct adapter *sc, struct listen_ctx *lctx)
 {
 	struct wrqe *wr;
 	struct cpl_pass_open_req *req;
-	struct in_conninfo *inc = &lctx->inp->inp_inc;
+	struct inpcb *inp = lctx->inp;
 
 	wr = alloc_wrqe(sizeof(*req), lctx->ctrlq);
 	if (wr == NULL) {
@@ -350,9 +413,9 @@ create_server(struct adapter *sc, struct listen_ctx *lctx)
 
 	INIT_TP_WR(req, 0);
 	OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, lctx->stid));
-	req->local_port = inc->inc_lport;
+	req->local_port = inp->inp_lport;
 	req->peer_port = 0;
-	req->local_ip = inc->inc_laddr.s_addr;
+	req->local_ip = inp->inp_laddr.s_addr;
 	req->peer_ip = 0;
 	req->opt0 = htobe64(V_TX_CHAN(lctx->ctrlq->eq.tx_chan));
 	req->opt1 = htobe64(V_CONN_POLICY(CPL_CONN_POLICY_ASK) |
@@ -363,6 +426,36 @@ create_server(struct adapter *sc, struct listen_ctx *lctx)
 }
 
 static int
+create_server6(struct adapter *sc, struct listen_ctx *lctx)
+{
+	struct wrqe *wr;
+	struct cpl_pass_open_req6 *req;
+	struct inpcb *inp = lctx->inp;
+
+	wr = alloc_wrqe(sizeof(*req), lctx->ctrlq);
+	if (wr == NULL) {
+		log(LOG_ERR, "%s: allocation failure", __func__);
+		return (ENOMEM);
+	}
+	req = wrtod(wr);
+
+	INIT_TP_WR(req, 0);
+	OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, lctx->stid));
+	req->local_port = inp->inp_lport;
+	req->peer_port = 0;
+	req->local_ip_hi = *(uint64_t *)&inp->in6p_laddr.s6_addr[0];
+	req->local_ip_lo = *(uint64_t *)&inp->in6p_laddr.s6_addr[8];
+	req->peer_ip_hi = 0;
+	req->peer_ip_lo = 0;
+	req->opt0 = htobe64(V_TX_CHAN(lctx->ctrlq->eq.tx_chan));
+	req->opt1 = htobe64(V_CONN_POLICY(CPL_CONN_POLICY_ASK) |
+	    F_SYN_RSS_ENABLE | V_SYN_RSS_QUEUE(lctx->ofld_rxq->iq.abs_id));
+
+	t4_wrq_tx(sc, wr);
+	return (0);
+}
+
+static int
 destroy_server(struct adapter *sc, struct listen_ctx *lctx)
 {
 	struct wrqe *wr;
@@ -398,13 +491,10 @@ t4_listen_start(struct toedev *tod, struct tcpcb *tp)
 	struct port_info *pi;
 	struct inpcb *inp = tp->t_inpcb;
 	struct listen_ctx *lctx;
-	int i;
+	int i, rc;
 
 	INP_WLOCK_ASSERT(inp);
 
-	if ((inp->inp_vflag & INP_IPV4) == 0)
-		return (0);
-
 #if 0
 	ADAPTER_LOCK(sc);
 	if (IS_BUSY(sc)) {
@@ -421,8 +511,9 @@ t4_listen_start(struct toedev *tod, struct tcpcb *tp)
 		goto done;	/* no port that's UP with IFCAP_TOE enabled */
 
 	/*
-	 * Find a running port with IFCAP_TOE4.  We'll use the first such port's
-	 * queues to send the passive open and receive the reply to it.
+	 * Find a running port with IFCAP_TOE (4 or 6).  We'll use the first
+	 * such port's queues to send the passive open and receive the reply to
+	 * it.
 	 *
 	 * XXX: need a way to mark a port in use by offload.  if_cxgbe should
 	 * then reject any attempt to bring down such a port (and maybe reject
@@ -430,7 +521,7 @@ t4_listen_start(struct toedev *tod, struct tcpcb *tp)
 	 */
 	for_each_port(sc, i) {
 		if (isset(&sc->open_device_map, i) &&
-		    sc->port[i]->ifp->if_capenable & IFCAP_TOE4)
+		    sc->port[i]->ifp->if_capenable & IFCAP_TOE)
 				break;
 	}
 	KASSERT(i < sc->params.nports,
@@ -449,12 +540,17 @@ t4_listen_start(struct toedev *tod, struct tcpcb *tp)
 	}
 	listen_hash_add(sc, lctx);
 
-	CTR5(KTR_CXGBE, "%s: stid %u (%s), lctx %p, inp %p", __func__,
-	    lctx->stid, tcpstates[tp->t_state], lctx, inp);
+	CTR6(KTR_CXGBE, "%s: stid %u (%s), lctx %p, inp %p vflag 0x%x",
+	    __func__, lctx->stid, tcpstates[tp->t_state], lctx, inp,
+	    inp->inp_vflag);
 
-	if (create_server(sc, lctx) != 0) {
-		log(LOG_ERR, "%s: %s failed to create hw listener.\n", __func__,
-		    device_get_nameunit(sc->dev));
+	if (inp->inp_vflag & INP_IPV6)
+		rc = create_server6(sc, lctx);
+	else
+		rc = create_server(sc, lctx);
+	if (rc != 0) {
+		log(LOG_ERR, "%s: %s failed to create hw listener: %d.\n",
+		    __func__, device_get_nameunit(sc->dev), rc);
 		(void) listen_hash_del(sc, inp);
 		inp = release_lctx(sc, lctx);
 		/* can't be freed, host stack has a reference */
@@ -558,7 +654,7 @@ t4_syncache_respond(struct toedev *tod, void *arg, struct mbuf *m)
 	struct l2t_entry *e;
 	struct tcpopt to;
 	struct ip *ip = mtod(m, struct ip *);
-	struct tcphdr *th = (void *)(ip + 1);
+	struct tcphdr *th;
 
 	wr = (struct wrqe *)atomic_readandclear_ptr(&synqe->wr);
 	if (wr == NULL) {
@@ -566,6 +662,10 @@ t4_syncache_respond(struct toedev *tod, void *arg, struct mbuf *m)
 		return (EALREADY);
 	}
 
+	if (ip->ip_v == IPVERSION)
+		th = (void *)(ip + 1);
+	else
+		th = (void *)((struct ip6_hdr *)ip + 1);
 	bzero(&to, sizeof(to));
 	tcp_dooptions(&to, (void *)(th + 1), (th->th_off << 2) - sizeof(*th),
 	    TO_SYN);
@@ -608,7 +708,7 @@ do_pass_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
 	lctx->flags &= ~LCTX_RPL_PENDING;
 
 	if (status != CPL_ERR_NONE)
-		log(LOG_ERR, "listener with stid %u failed: %d", stid, status);
+		log(LOG_ERR, "listener (stid %u) failed: %d\n", stid, status);
 
 #ifdef INVARIANTS
 	/*
@@ -678,7 +778,7 @@ do_close_server_rpl(struct sge_iq *iq, const struct rss_header *rss,
 	CTR3(KTR_CXGBE, "%s: stid %u, status %u", __func__, stid, status);
 
 	if (status != CPL_ERR_NONE) {
-		log(LOG_ERR, "%s: failed (%u) to close listener for stid %u",
+		log(LOG_ERR, "%s: failed (%u) to close listener for stid %u\n",
 		    __func__, status, stid);
 		return (status);
 	}
@@ -735,8 +835,7 @@ do_abort_req_synqe(struct sge_iq *iq, const struct rss_header *rss,
 	CTR6(KTR_CXGBE, "%s: tid %u, synqe %p (0x%x), lctx %p, status %d",
 	    __func__, tid, synqe, synqe->flags, synqe->lctx, cpl->status);
 
-	if (cpl->status == CPL_ERR_RTX_NEG_ADVICE ||
-	    cpl->status == CPL_ERR_PERSIST_NEG_ADVICE)
+	if (negative_advice(cpl->status))
 		return (0);	/* Ignore negative advice */
 
 	INP_WLOCK(inp);
@@ -855,7 +954,7 @@ mbuf_to_synqe(struct mbuf *m)
 			return (NULL);
 		synqe->flags = TPF_SYNQE | TPF_SYNQE_NEEDFREE;
 	} else {
-		synqe = (void *)(m->m_data + m->m_len + tspace - sizeof(*synqe));
+		synqe = (void *)(m->m_data + m->m_len + tspace - len);
 		synqe->flags = TPF_SYNQE;
 	}
 
@@ -936,21 +1035,29 @@ pass_accept_req_to_protohdrs(const struct mbuf *m, struct in_conninfo *inc,
 	const struct cpl_pass_accept_req *cpl = mtod(m, const void *);
 	const struct ether_header *eh;
 	unsigned int hlen = be32toh(cpl->hdr_len);
-	const struct ip *ip;
+	uintptr_t l3hdr;
 	const struct tcphdr *tcp;
 
 	eh = (const void *)(cpl + 1);
-	ip = (const void *)((uintptr_t)eh + G_ETH_HDR_LEN(hlen));
-	tcp = (const void *)((uintptr_t)ip + G_IP_HDR_LEN(hlen));
+	l3hdr = ((uintptr_t)eh + G_ETH_HDR_LEN(hlen));
+	tcp = (const void *)(l3hdr + G_IP_HDR_LEN(hlen));
 
 	if (inc) {
 		bzero(inc, sizeof(*inc));
-		inc->inc_faddr = ip->ip_src;
-		inc->inc_laddr = ip->ip_dst;
 		inc->inc_fport = tcp->th_sport;
 		inc->inc_lport = tcp->th_dport;
-		if (ip->ip_v == 6)
+		if (((struct ip *)l3hdr)->ip_v == IPVERSION) {
+			const struct ip *ip = (const void *)l3hdr;
+
+			inc->inc_faddr = ip->ip_src;
+			inc->inc_laddr = ip->ip_dst;
+		} else {
+			const struct ip6_hdr *ip6 = (const void *)l3hdr;
+
 			inc->inc_flags |= INC_ISIPV6;
+			inc->inc6_faddr = ip6->ip6_src;
+			inc->inc6_laddr = ip6->ip6_dst;
+		}
 	}
 
 	if (th) {
@@ -959,6 +1066,105 @@ pass_accept_req_to_protohdrs(const struct mbuf *m, struct in_conninfo *inc,
 	}
 }
 
+static int
+ifnet_has_ip6(struct ifnet *ifp, struct in6_addr *ip6)
+{
+	struct ifaddr *ifa;
+	struct sockaddr_in6 *sin6;
+	int found = 0;
+	struct in6_addr in6 = *ip6;
+
+	/* Just as in ip6_input */
+	if (in6_clearscope(&in6) || in6_clearscope(&in6))
+		return (0);
+	in6_setscope(&in6, ifp, NULL);
+
+	if_addr_rlock(ifp);
+	TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
+		sin6 = (void *)ifa->ifa_addr;
+		if (sin6->sin6_family != AF_INET6)
+			continue;
+
+		if (IN6_ARE_ADDR_EQUAL(&sin6->sin6_addr, &in6)) {
+			found = 1;
+			break;
+		}
+	}
+	if_addr_runlock(ifp);
+
+	return (found);
+}
+
+static struct l2t_entry *
+get_l2te_for_nexthop(struct port_info *pi, struct ifnet *ifp,
+    struct in_conninfo *inc)
+{
+	struct rtentry *rt;
+	struct l2t_entry *e;
+	struct sockaddr_in6 sin6;
+	struct sockaddr *dst = (void *)&sin6;
+ 
+	if (inc->inc_flags & INC_ISIPV6) {
+		dst->sa_len = sizeof(struct sockaddr_in6);
+		dst->sa_family = AF_INET6;
+		((struct sockaddr_in6 *)dst)->sin6_addr = inc->inc6_faddr;
+
+		if (IN6_IS_ADDR_LINKLOCAL(&inc->inc6_laddr)) {
+			/* no need for route lookup */
+			e = t4_l2t_get(pi, ifp, dst);
+			return (e);
+		}
+	} else {
+		dst->sa_len = sizeof(struct sockaddr_in);
+		dst->sa_family = AF_INET;
+		((struct sockaddr_in *)dst)->sin_addr = inc->inc_faddr;
+	}
+
+	rt = rtalloc1(dst, 0, 0);
+	if (rt == NULL)
+		return (NULL);
+	else {
+		struct sockaddr *nexthop;
+
+		RT_UNLOCK(rt);
+		if (rt->rt_ifp != ifp)
+			e = NULL;
+		else {
+			if (rt->rt_flags & RTF_GATEWAY)
+				nexthop = rt->rt_gateway;
+			else
+				nexthop = dst;
+			e = t4_l2t_get(pi, ifp, nexthop);
+		}
+		RTFREE(rt);
+	}
+
+	return (e);
+}
+
+static int
+ifnet_has_ip(struct ifnet *ifp, struct in_addr in)
+{
+	struct ifaddr *ifa;
+	struct sockaddr_in *sin;
+	int found = 0;
+
+	if_addr_rlock(ifp);
+	TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
+		sin = (void *)ifa->ifa_addr;
+		if (sin->sin_family != AF_INET)
+			continue;
+
+		if (sin->sin_addr.s_addr == in.s_addr) {
+			found = 1;
+			break;
+		}
+	}
+	if_addr_runlock(ifp);
+
+	return (found);
+}
+
 #define REJECT_PASS_ACCEPT()	do { \
 	reject_reason = __LINE__; \
 	goto reject; \
@@ -994,10 +1200,8 @@ do_pass_accept_req(struct sge_iq *iq, const struct rss_header *rss,
 	struct tcphdr th;
 	struct tcpopt to;
 	struct port_info *pi;
-	struct ifnet *ifp, *ifp_vlan = NULL;
+	struct ifnet *hw_ifp, *ifp;
 	struct l2t_entry *e = NULL;
-	struct rtentry *rt;
-	struct sockaddr_in nam;
 	int rscale, mtu_idx, rx_credits, rxqid, ulp_mode;
 	struct synq_entry *synqe = NULL;
 	int reject_reason;
@@ -1017,31 +1221,24 @@ do_pass_accept_req(struct sge_iq *iq, const struct rss_header *rss,
 	t4opt_to_tcpopt(&cpl->tcpopt, &to);
 
 	pi = sc->port[G_SYN_INTF(be16toh(cpl->l2info))];
-	ifp = pi->ifp;
-	m->m_pkthdr.rcvif = ifp;
-	tod = TOEDEV(ifp);
+	hw_ifp = pi->ifp;	/* the cxgbeX ifnet */
+	m->m_pkthdr.rcvif = hw_ifp;
+	tod = TOEDEV(hw_ifp);
 
 	/*
-	 * Don't offload if the interface that received the SYN doesn't have
-	 * IFCAP_TOE enabled.
-	 */
-	if ((ifp->if_capenable & IFCAP_TOE4) == 0)
-		REJECT_PASS_ACCEPT();
-
-	/* Don't offload IPv6 connections. XXX: add IPv6 support */
-	if (inc.inc_flags & INC_ISIPV6)
-		REJECT_PASS_ACCEPT();
-
-	/*
-	 * Don't offload if the SYN had a VLAN tag and the vid doesn't match
-	 * anything on this interface.
+	 * Figure out if there is a pseudo interface (vlan, lagg, etc.)
+	 * involved.  Don't offload if the SYN had a VLAN tag and the vid
+	 * doesn't match anything on this interface.
+	 *
+	 * XXX: lagg support, lagg + vlan support.
 	 */
 	vid = EVL_VLANOFTAG(be16toh(cpl->vlan));
 	if (vid != 0xfff) {
-		ifp_vlan = VLAN_DEVAT(ifp, vid);
-		if (ifp_vlan == NULL)
+		ifp = VLAN_DEVAT(hw_ifp, vid);
+		if (ifp == NULL)
 			REJECT_PASS_ACCEPT();
-	}
+	} else
+		ifp = hw_ifp;
 
 	/*
 	 * Don't offload if the peer requested a TCP option that's not known to
@@ -1050,31 +1247,36 @@ do_pass_accept_req(struct sge_iq *iq, const struct rss_header *rss,
 	if (cpl->tcpopt.unknown)
 		REJECT_PASS_ACCEPT();
 
-	/*
-	 * Don't offload if the outgoing interface for the route back to the
-	 * peer is not the same as the interface that received the SYN.
-	 * XXX: too restrictive.
-	 */
-	nam.sin_len = sizeof(nam);
-	nam.sin_family = AF_INET;
-	nam.sin_addr = inc.inc_faddr;
-	rt = rtalloc1((struct sockaddr *)&nam, 0, 0);
-	if (rt == NULL)
-		REJECT_PASS_ACCEPT();
-	else {
-		struct sockaddr *nexthop;
+	if (inc.inc_flags & INC_ISIPV6) {
 
-		RT_UNLOCK(rt);
-		nexthop = rt->rt_flags & RTF_GATEWAY ? rt->rt_gateway :
-		    (struct sockaddr *)&nam;
-		if (rt->rt_ifp == ifp ||
-		    (ifp_vlan != NULL && rt->rt_ifp == ifp_vlan))
-			e = t4_l2t_get(pi, rt->rt_ifp, nexthop);
-		RTFREE(rt);
-		if (e == NULL)
-			REJECT_PASS_ACCEPT();	/* no l2te, or ifp mismatch */
+		/* Don't offload if the ifcap isn't enabled */
+		if ((ifp->if_capenable & IFCAP_TOE6) == 0)
+			REJECT_PASS_ACCEPT();
+
+		/*
+		 * SYN must be directed to an IP6 address on this ifnet.  This
+		 * is more restrictive than in6_localip.
+		 */
+		if (!ifnet_has_ip6(ifp, &inc.inc6_laddr))
+			REJECT_PASS_ACCEPT();
+	} else {
+
+		/* Don't offload if the ifcap isn't enabled */
+		if ((ifp->if_capenable & IFCAP_TOE4) == 0)
+			REJECT_PASS_ACCEPT();
+
+		/*
+		 * SYN must be directed to an IP address on this ifnet.  This
+		 * is more restrictive than in_localip.
+		 */
+		if (!ifnet_has_ip(ifp, inc.inc_laddr))
+			REJECT_PASS_ACCEPT();
 	}
 
+	e = get_l2te_for_nexthop(pi, ifp, &inc);
+	if (e == NULL)
+		REJECT_PASS_ACCEPT();
+
 	synqe = mbuf_to_synqe(m);
 	if (synqe == NULL)
 		REJECT_PASS_ACCEPT();
@@ -1133,7 +1335,7 @@ do_pass_accept_req(struct sge_iq *iq, const struct rss_header *rss,
 	synqe->lctx = lctx;
 	synqe->syn = m;
 	m = NULL;
-	refcount_init(&synqe->refcnt, 0);
+	refcount_init(&synqe->refcnt, 1);	/* 1 means extra hold */
 	synqe->l2e_idx = e->idx;
 	synqe->rcv_bufsize = rx_credits;
 	atomic_store_rel_ptr(&synqe->wr, (uintptr_t)wr);
@@ -1166,7 +1368,7 @@ do_pass_accept_req(struct sge_iq *iq, const struct rss_header *rss,
 		 */
 		m = m_dup(synqe->syn, M_NOWAIT);
 		if (m)
-			m->m_pkthdr.rcvif = ifp;
+			m->m_pkthdr.rcvif = hw_ifp;
 
 		remove_tid(sc, synqe->tid);
 		free(wr, M_CXGBE);
@@ -1179,6 +1381,7 @@ do_pass_accept_req(struct sge_iq *iq, const struct rss_header *rss,
 		if (inp)
 			INP_WUNLOCK(inp);
 
+		release_synqe(synqe);	/* extra hold */
 		REJECT_PASS_ACCEPT();
 	}
 
@@ -1193,15 +1396,19 @@ do_pass_accept_req(struct sge_iq *iq, const struct rss_header *rss,
 		 * this tid because there was no L2T entry for the tid at that
 		 * time.  Abort it now.  The reply to the abort will clean up.
 		 */
-		CTR5(KTR_CXGBE, "%s: stid %u, tid %u, lctx %p, synqe %p, ABORT",
-		    __func__, stid, tid, lctx, synqe);
-		send_reset_synqe(tod, synqe);
+		CTR6(KTR_CXGBE,
+		    "%s: stid %u, tid %u, lctx %p, synqe %p (0x%x), ABORT",
+		    __func__, stid, tid, lctx, synqe, synqe->flags);
+		if (!(synqe->flags & TPF_SYNQE_EXPANDED))
+			send_reset_synqe(tod, synqe);
 		INP_WUNLOCK(inp);
 
+		release_synqe(synqe);	/* extra hold */
 		return (__LINE__);
 	}
 	INP_WUNLOCK(inp);
 
+	release_synqe(synqe);	/* extra hold */
 	return (0);
 reject:
 	CTR4(KTR_CXGBE, "%s: stid %u, tid %u, REJECT (%d)", __func__, stid, tid,
@@ -1216,7 +1423,7 @@ reject:
 		m->m_pkthdr.csum_flags |= (CSUM_IP_CHECKED | CSUM_IP_VALID |
 		    CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
 		m->m_pkthdr.csum_data = 0xffff;
-		ifp->if_input(ifp, m);
+		hw_ifp->if_input(hw_ifp, m);
 	}
 
 	return (reject_reason);
diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c
index 330172d..64e8b26 100644
--- a/sys/dev/cxgbe/tom/t4_tom.c
+++ b/sys/dev/cxgbe/tom/t4_tom.c
@@ -29,6 +29,7 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
+#include "opt_inet6.h"
 
 #include <sys/param.h>
 #include <sys/types.h>
@@ -40,10 +41,14 @@ __FBSDID("$FreeBSD$");
 #include <sys/domain.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
+#include <net/if.h>
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
+#include <netinet/in_var.h>
 #include <netinet/ip.h>
+#include <netinet/ip6.h>
 #include <netinet/tcp_var.h>
+#include <netinet6/scope6_var.h>
 #define TCPSTATES
 #include <netinet/tcp_fsm.h>
 #include <netinet/toecore.h>
@@ -58,6 +63,9 @@ __FBSDID("$FreeBSD$");
 static struct protosw ddp_protosw;
 static struct pr_usrreqs ddp_usrreqs;
 
+static struct protosw ddp6_protosw;
+static struct pr_usrreqs ddp6_usrreqs;
+
 /* Module ops */
 static int t4_tom_mod_load(void);
 static int t4_tom_mod_unload(void);
@@ -77,6 +85,11 @@ static void queue_tid_release(struct adapter *, int);
 static void release_offload_resources(struct toepcb *);
 static int alloc_tid_tabs(struct tid_info *);
 static void free_tid_tabs(struct tid_info *);
+static int add_lip(struct adapter *, struct in6_addr *);
+static int delete_lip(struct adapter *, struct in6_addr *);
+static struct clip_entry *search_lip(struct tom_data *, struct in6_addr *);
+static void init_clip_table(struct adapter *, struct tom_data *);
+static void destroy_clip_table(struct adapter *, struct tom_data *);
 static void free_tom_data(struct adapter *, struct tom_data *);
 
 struct toepcb *
@@ -170,8 +183,12 @@ offload_socket(struct socket *so, struct toepcb *toep)
 	sb = &so->so_rcv;
 	SOCKBUF_LOCK(sb);
 	sb->sb_flags |= SB_NOCOALESCE;
-	if (toep->ulp_mode == ULP_MODE_TCPDDP)
-		so->so_proto = &ddp_protosw;
+	if (toep->ulp_mode == ULP_MODE_TCPDDP) {
+		if (inp->inp_vflag & INP_IPV6)
+			so->so_proto = &ddp6_protosw;
+		else
+			so->so_proto = &ddp_protosw;
+	}
 	SOCKBUF_UNLOCK(sb);
 
 	/* Update TCP PCB */
@@ -237,8 +254,8 @@ release_offload_resources(struct toepcb *toep)
 	KASSERT(!(toep->flags & TPF_ATTACHED),
 	    ("%s: %p is still attached.", __func__, toep));
 
-	CTR4(KTR_CXGBE, "%s: toep %p (tid %d, l2te %p)",
-	    __func__, toep, tid, toep->l2te);
+	CTR5(KTR_CXGBE, "%s: toep %p (tid %d, l2te %p, ce %p)",
+	    __func__, toep, tid, toep->l2te, toep->ce);
 
 	if (toep->ulp_mode == ULP_MODE_TCPDDP)
 		release_ddp_resources(toep);
@@ -251,6 +268,9 @@ release_offload_resources(struct toepcb *toep)
 		release_tid(sc, tid, toep->ctrlq);
 	}
 
+	if (toep->ce)
+		release_lip(td, toep->ce);
+
 	mtx_lock(&td->toep_list_lock);
 	TAILQ_REMOVE(&td->toep_list, toep, link);
 	mtx_unlock(&td->toep_list_lock);
@@ -394,7 +414,7 @@ int
 find_best_mtu_idx(struct adapter *sc, struct in_conninfo *inc, int pmss)
 {
 	unsigned short *mtus = &sc->params.mtus[0];
-	int i = 0, mss;
+	int i, mss, n;
 
 	KASSERT(inc != NULL || pmss > 0,
 	    ("%s: at least one of inc/pmss must be specified", __func__));
@@ -403,8 +423,13 @@ find_best_mtu_idx(struct adapter *sc, struct in_conninfo *inc, int pmss)
 	if (pmss > 0 && mss > pmss)
 		mss = pmss;
 
-	while (i < NMTUS - 1 && mtus[i + 1] <= mss + 40)
-		++i;
+	if (inc->inc_flags & INC_ISIPV6)
+		n = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
+	else
+		n = sizeof(struct ip) + sizeof(struct tcphdr);
+
+	for (i = 0; i < NMTUS - 1 && mtus[i + 1] <= mss + n; i++)
+		continue;
 
 	return (i);
 }
@@ -513,6 +538,24 @@ select_ntuple(struct port_info *pi, struct l2t_entry *e, uint32_t filter_mode)
 	return (htobe32(ntuple));
 }
 
+void
+set_tcpddp_ulp_mode(struct toepcb *toep)
+{
+
+	toep->ulp_mode = ULP_MODE_TCPDDP;
+	toep->ddp_flags = DDP_OK;
+	toep->ddp_score = DDP_LOW_SCORE;
+}
+
+int
+negative_advice(int status)
+{
+
+	return (status == CPL_ERR_RTX_NEG_ADVICE ||
+	    status == CPL_ERR_PERSIST_NEG_ADVICE ||
+	    status == CPL_ERR_KEEPALV_NEG_ADVICE);
+}
+
 static int
 alloc_tid_tabs(struct tid_info *t)
 {
@@ -536,12 +579,10 @@ alloc_tid_tabs(struct tid_info *t)
 	t->atid_tab[t->natids - 1].next = NULL;
 
 	mtx_init(&t->stid_lock, "stid lock", NULL, MTX_DEF);
-	t->stid_tab = (union serv_entry *)&t->atid_tab[t->natids];
-	t->sfree = t->stid_tab;
+	t->stid_tab = (struct listen_ctx **)&t->atid_tab[t->natids];
 	t->stids_in_use = 0;
-	for (i = 1; i < t->nstids; i++)
-		t->stid_tab[i - 1].next = &t->stid_tab[i];
-	t->stid_tab[t->nstids - 1].next = NULL;
+	TAILQ_INIT(&t->stids);
+	t->nstids_free_head = t->nstids;
 
 	atomic_store_rel_int(&t->tids_in_use, 0);
 
@@ -567,9 +608,157 @@ free_tid_tabs(struct tid_info *t)
 		mtx_destroy(&t->stid_lock);
 }
 
+static int
+add_lip(struct adapter *sc, struct in6_addr *lip)
+{
+        struct fw_clip_cmd c;
+
+	ASSERT_SYNCHRONIZED_OP(sc);
+	/* mtx_assert(&td->clip_table_lock, MA_OWNED); */
+
+        memset(&c, 0, sizeof(c));
+	c.op_to_write = htonl(V_FW_CMD_OP(FW_CLIP_CMD) | F_FW_CMD_REQUEST |
+	    F_FW_CMD_WRITE);
+        c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_ALLOC | FW_LEN16(c));
+        c.ip_hi = *(uint64_t *)&lip->s6_addr[0];
+        c.ip_lo = *(uint64_t *)&lip->s6_addr[8];
+
+	return (t4_wr_mbox_ns(sc, sc->mbox, &c, sizeof(c), &c));
+}
+
+static int
+delete_lip(struct adapter *sc, struct in6_addr *lip)
+{
+	struct fw_clip_cmd c;
+
+	ASSERT_SYNCHRONIZED_OP(sc);
+	/* mtx_assert(&td->clip_table_lock, MA_OWNED); */
+
+	memset(&c, 0, sizeof(c));
+	c.op_to_write = htonl(V_FW_CMD_OP(FW_CLIP_CMD) | F_FW_CMD_REQUEST |
+	    F_FW_CMD_READ);
+        c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_FREE | FW_LEN16(c));
+        c.ip_hi = *(uint64_t *)&lip->s6_addr[0];
+        c.ip_lo = *(uint64_t *)&lip->s6_addr[8];
+
+	return (t4_wr_mbox_ns(sc, sc->mbox, &c, sizeof(c), &c));
+}
+
+static struct clip_entry *
+search_lip(struct tom_data *td, struct in6_addr *lip)
+{
+	struct clip_entry *ce;
+
+	mtx_assert(&td->clip_table_lock, MA_OWNED);
+
+	TAILQ_FOREACH(ce, &td->clip_table, link) {
+		if (IN6_ARE_ADDR_EQUAL(&ce->lip, lip))
+			return (ce);
+	}
+
+	return (NULL);
+}
+
+struct clip_entry *
+hold_lip(struct tom_data *td, struct in6_addr *lip)
+{
+	struct clip_entry *ce;
+
+	mtx_lock(&td->clip_table_lock);
+	ce = search_lip(td, lip);
+	if (ce != NULL)
+		ce->refcount++;
+	mtx_unlock(&td->clip_table_lock);
+
+	return (ce);
+}
+
+void
+release_lip(struct tom_data *td, struct clip_entry *ce)
+{
+
+	mtx_lock(&td->clip_table_lock);
+	KASSERT(search_lip(td, &ce->lip) == ce,
+	    ("%s: CLIP entry %p p not in CLIP table.", __func__, ce));
+	KASSERT(ce->refcount > 0,
+	    ("%s: CLIP entry %p has refcount 0", __func__, ce));
+	--ce->refcount;
+	mtx_unlock(&td->clip_table_lock);
+}
+
+static void
+init_clip_table(struct adapter *sc, struct tom_data *td)
+{
+	struct in6_ifaddr *ia;
+	struct in6_addr *lip, tlip;
+	struct clip_entry *ce;
+
+	ASSERT_SYNCHRONIZED_OP(sc);
+
+	mtx_init(&td->clip_table_lock, "CLIP table lock", NULL, MTX_DEF);
+	TAILQ_INIT(&td->clip_table);
+
+	IN6_IFADDR_RLOCK();
+	TAILQ_FOREACH(ia, &V_in6_ifaddrhead, ia_link) {
+		lip = &ia->ia_addr.sin6_addr;
+
+		KASSERT(!IN6_IS_ADDR_MULTICAST(lip),
+		    ("%s: mcast address in in6_ifaddr list", __func__));
+
+		if (IN6_IS_ADDR_LOOPBACK(lip))
+			continue;
+		if (IN6_IS_SCOPE_EMBED(lip)) {
+			/* Remove the embedded scope */
+			tlip = *lip;
+			lip = &tlip;
+			in6_clearscope(lip);
+		}
+		/*
+		 * XXX: how to weed out the link local address for the loopback
+		 * interface?  It's fe80::1 usually (always?).
+		 */
+
+		mtx_lock(&td->clip_table_lock);
+		if (search_lip(td, lip) == NULL) {
+			ce = malloc(sizeof(*ce), M_CXGBE, M_NOWAIT);
+			memcpy(&ce->lip, lip, sizeof(ce->lip));
+			ce->refcount = 0;
+			if (add_lip(sc, lip) == 0)
+				TAILQ_INSERT_TAIL(&td->clip_table, ce, link);
+			else
+				free(ce, M_CXGBE);
+		}
+		mtx_unlock(&td->clip_table_lock);
+	}
+	IN6_IFADDR_RUNLOCK();
+}
+
+static void
+destroy_clip_table(struct adapter *sc, struct tom_data *td)
+{
+	struct clip_entry *ce, *ce_temp;
+
+	if (mtx_initialized(&td->clip_table_lock)) {
+		mtx_lock(&td->clip_table_lock);
+		TAILQ_FOREACH_SAFE(ce, &td->clip_table, link, ce_temp) {
+			KASSERT(ce->refcount == 0,
+			    ("%s: CLIP entry %p still in use (%d)", __func__,
+			    ce, ce->refcount));
+			TAILQ_REMOVE(&td->clip_table, ce, link);
+			delete_lip(sc, &ce->lip);
+			free(ce, M_CXGBE);
+		}
+		mtx_unlock(&td->clip_table_lock);
+		mtx_destroy(&td->clip_table_lock);
+	}
+}
+
 static void
 free_tom_data(struct adapter *sc, struct tom_data *td)
 {
+
+	ASSERT_SYNCHRONIZED_OP(sc);
+
 	KASSERT(TAILQ_EMPTY(&td->toep_list),
 	    ("%s: TOE PCB list is not empty.", __func__));
 	KASSERT(td->lctx_count == 0,
@@ -578,6 +767,7 @@ free_tom_data(struct adapter *sc, struct tom_data *td)
 	t4_uninit_l2t_cpl_handlers(sc);
 	t4_uninit_cpl_io_handlers(sc);
 	t4_uninit_ddp(sc, td);
+	destroy_clip_table(sc, td);
 
 	if (td->listen_mask != 0)
 		hashdestroy(td->listen_hash, M_CXGBE, td->listen_mask);
@@ -602,7 +792,7 @@ t4_tom_activate(struct adapter *sc)
 	struct toedev *tod;
 	int i, rc;
 
-	ADAPTER_LOCK_ASSERT_OWNED(sc);	/* for sc->flags */
+	ASSERT_SYNCHRONIZED_OP(sc);
 
 	/* per-adapter softc for TOM */
 	td = malloc(sizeof(*td), M_CXGBE, M_ZERO | M_NOWAIT);
@@ -623,8 +813,12 @@ t4_tom_activate(struct adapter *sc)
 	if (rc != 0)
 		goto done;
 
+	/* DDP page pods and CPL handlers */
 	t4_init_ddp(sc, td);
 
+	/* CLIP table for IPv6 offload */
+	init_clip_table(sc, td);
+
 	/* CPL handlers */
 	t4_init_connect_cpl_handlers(sc);
 	t4_init_l2t_cpl_handlers(sc);
@@ -668,7 +862,7 @@ t4_tom_deactivate(struct adapter *sc)
 	int rc = 0;
 	struct tom_data *td = sc->tom_softc;
 
-	ADAPTER_LOCK_ASSERT_OWNED(sc);	/* for sc->flags */
+	ASSERT_SYNCHRONIZED_OP(sc);
 
 	if (td == NULL)
 		return (0);	/* XXX. KASSERT? */
@@ -700,17 +894,24 @@ static int
 t4_tom_mod_load(void)
 {
 	int rc;
-	struct protosw *tcp_protosw;
+	struct protosw *tcp_protosw, *tcp6_protosw;
 
 	tcp_protosw = pffindproto(PF_INET, IPPROTO_TCP, SOCK_STREAM);
 	if (tcp_protosw == NULL)
 		return (ENOPROTOOPT);
-
 	bcopy(tcp_protosw, &ddp_protosw, sizeof(ddp_protosw));
 	bcopy(tcp_protosw->pr_usrreqs, &ddp_usrreqs, sizeof(ddp_usrreqs));
 	ddp_usrreqs.pru_soreceive = t4_soreceive_ddp;
 	ddp_protosw.pr_usrreqs = &ddp_usrreqs;
 
+	tcp6_protosw = pffindproto(PF_INET6, IPPROTO_TCP, SOCK_STREAM);
+	if (tcp6_protosw == NULL)
+		return (ENOPROTOOPT);
+	bcopy(tcp6_protosw, &ddp6_protosw, sizeof(ddp6_protosw));
+	bcopy(tcp6_protosw->pr_usrreqs, &ddp6_usrreqs, sizeof(ddp6_usrreqs));
+	ddp6_usrreqs.pru_soreceive = t4_soreceive_ddp;
+	ddp6_protosw.pr_usrreqs = &ddp6_usrreqs;
+
 	rc = t4_register_uld(&tom_uld_info);
 	if (rc != 0)
 		t4_tom_mod_unload();
@@ -721,11 +922,14 @@ t4_tom_mod_load(void)
 static void
 tom_uninit(struct adapter *sc, void *arg __unused)
 {
+	if (begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4tomun"))
+		return;
+
 	/* Try to free resources (works only if no port has IFCAP_TOE) */
-	ADAPTER_LOCK(sc);
 	if (sc->flags & TOM_INIT_DONE)
 		t4_deactivate_uld(sc, ULD_TOM);
-	ADAPTER_UNLOCK(sc);
+
+	end_synchronized_op(sc, LOCK_HELD);
 }
 
 static int
diff --git a/sys/dev/cxgbe/tom/t4_tom.h b/sys/dev/cxgbe/tom/t4_tom.h
index 9549b0b..d0fbbd2 100644
--- a/sys/dev/cxgbe/tom/t4_tom.h
+++ b/sys/dev/cxgbe/tom/t4_tom.h
@@ -109,6 +109,7 @@ struct toepcb {
 	struct sge_ofld_rxq *ofld_rxq;
 	struct sge_wrq *ctrlq;
 	struct l2t_entry *l2te;	/* L2 table entry used by this connection */
+	struct clip_entry *ce;	/* CLIP table entry used by this tid */
 	int tid;		/* Connection identifier */
 	unsigned int tx_credits;/* tx WR credits (in 16 byte units) remaining */
 	unsigned int sb_cc;	/* last noted value of so_rcv->sb_cc */
@@ -140,15 +141,6 @@ struct flowc_tx_params {
 #define	DDP_LOW_SCORE	1
 #define	DDP_HIGH_SCORE	3
 
-static inline void
-set_tcpddp_ulp_mode(struct toepcb *toep)
-{
-
-	toep->ulp_mode = ULP_MODE_TCPDDP;
-	toep->ddp_flags = DDP_OK;
-	toep->ddp_score = DDP_LOW_SCORE;
-}
-
 /*
  * Compressed state for embryonic connections for a listener.  Barely fits in
  * 64B, try not to grow it further.
@@ -174,6 +166,7 @@ struct listen_ctx {
 	LIST_ENTRY(listen_ctx) link;	/* listen hash linkage */
 	volatile int refcount;
 	int stid;
+	struct stid_region stid_region;
 	int flags;
 	struct inpcb *inp;		/* listening socket's inp */
 	struct sge_wrq *ctrlq;
@@ -183,6 +176,12 @@ struct listen_ctx {
 
 TAILQ_HEAD(ppod_head, ppod_region);
 
+struct clip_entry {
+	TAILQ_ENTRY(clip_entry) link;
+	struct in6_addr lip;	/* local IPv6 address */
+	u_int refcount;
+};
+
 struct tom_data {
 	struct toedev tod;
 
@@ -200,6 +199,9 @@ struct tom_data {
 	int nppods_free;	/* # of available ppods */
 	int nppods_free_head;	/* # of available ppods at the begining */
 	struct ppod_head ppods;
+
+	struct mtx clip_table_lock;
+	TAILQ_HEAD(, clip_entry) clip_table;
 };
 
 static inline struct tom_data *
@@ -233,6 +235,10 @@ int select_rcv_wscale(void);
 uint64_t calc_opt0(struct socket *, struct port_info *, struct l2t_entry *,
     int, int, int, int);
 uint32_t select_ntuple(struct port_info *, struct l2t_entry *, uint32_t);
+void set_tcpddp_ulp_mode(struct toepcb *);
+int negative_advice(int);
+struct clip_entry *hold_lip(struct tom_data *, struct in6_addr *);
+void release_lip(struct tom_data *, struct clip_entry *);
 
 /* t4_connect.c */
 void t4_init_connect_cpl_handlers(struct adapter *);
diff --git a/sys/dev/cxgbe/tom/t4_tom_l2t.c b/sys/dev/cxgbe/tom/t4_tom_l2t.c
index ffe64c5..7a75394 100644
--- a/sys/dev/cxgbe/tom/t4_tom_l2t.c
+++ b/sys/dev/cxgbe/tom/t4_tom_l2t.c
@@ -27,6 +27,7 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
+#include "opt_inet6.h"
 
 #ifdef TCP_OFFLOAD
 #include <sys/param.h>
@@ -34,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/bus.h>
+#include <sys/fnv_hash.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/rwlock.h>
@@ -48,28 +50,89 @@ __FBSDID("$FreeBSD$");
 #include <netinet/toecore.h>
 
 #include "common/common.h"
-#include "common/jhash.h"
 #include "common/t4_msg.h"
 #include "tom/t4_tom_l2t.h"
 #include "tom/t4_tom.h"
 
 #define VLAN_NONE	0xfff
 
-#define SA(x)           ((struct sockaddr *)(x))
-#define SIN(x)          ((struct sockaddr_in *)(x))
-#define SINADDR(x)      (SIN(x)->sin_addr.s_addr)
-
 static inline void
 l2t_hold(struct l2t_data *d, struct l2t_entry *e)
 {
+
 	if (atomic_fetchadd_int(&e->refcnt, 1) == 0)  /* 0 -> 1 transition */
 		atomic_subtract_int(&d->nfree, 1);
 }
 
-static inline unsigned int
-arp_hash(const uint32_t key, int ifindex)
+static inline u_int
+l2_hash(struct l2t_data *d, const struct sockaddr *sa, int ifindex)
 {
-	return jhash_2words(key, ifindex, 0) & (L2T_SIZE - 1);
+	u_int hash, half = d->l2t_size / 2, start = 0;
+	const void *key;
+	size_t len;
+
+	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
+	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
+	    sa->sa_family));
+
+	if (sa->sa_family == AF_INET) {
+		const struct sockaddr_in *sin = (const void *)sa;
+
+		key = &sin->sin_addr;
+		len = sizeof(sin->sin_addr);
+	} else {
+		const struct sockaddr_in6 *sin6 = (const void *)sa;
+
+		key = &sin6->sin6_addr;
+		len = sizeof(sin6->sin6_addr);
+		start = half;
+	}
+
+	hash = fnv_32_buf(key, len, FNV1_32_INIT);
+	hash = fnv_32_buf(&ifindex, sizeof(ifindex), hash);
+	hash %= half;
+
+	return (hash + start);
+}
+
+static inline int
+l2_cmp(const struct sockaddr *sa, struct l2t_entry *e)
+{
+
+	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
+	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
+	    sa->sa_family));
+
+	if (sa->sa_family == AF_INET) {
+		const struct sockaddr_in *sin = (const void *)sa;
+
+		return (e->addr[0] != sin->sin_addr.s_addr);
+	} else {
+		const struct sockaddr_in6 *sin6 = (const void *)sa;
+
+		return (memcmp(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr)));
+	}
+}
+
+static inline void
+l2_store(const struct sockaddr *sa, struct l2t_entry *e)
+{
+
+	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
+	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
+	    sa->sa_family));
+
+	if (sa->sa_family == AF_INET) {
+		const struct sockaddr_in *sin = (const void *)sa;
+
+		e->addr[0] = sin->sin_addr.s_addr;
+		e->ipv6 = 0;
+	} else {
+		const struct sockaddr_in6 *sin6 = (const void *)sa;
+
+		memcpy(&e->addr[0], &sin6->sin6_addr, sizeof(e->addr));
+		e->ipv6 = 1;
+	}
 }
 
 /*
@@ -100,7 +163,7 @@ send_pending(struct adapter *sc, struct l2t_entry *e)
 static void
 resolution_failed_for_wr(struct wrqe *wr)
 {
-	log(LOG_ERR, "%s: leaked work request %p, wr_len %d", __func__, wr,
+	log(LOG_ERR, "%s: leaked work request %p, wr_len %d\n", __func__, wr,
 	    wr->wr_len);
 
 	/* free(wr, M_CXGBE); */
@@ -175,15 +238,25 @@ resolve_entry(struct adapter *sc, struct l2t_entry *e)
 	struct tom_data *td = sc->tom_softc;
 	struct toedev *tod = &td->tod;
 	struct sockaddr_in sin = {0};
+	struct sockaddr_in6 sin6 = {0};
+	struct sockaddr *sa;
 	uint8_t dmac[ETHER_ADDR_LEN];
 	uint16_t vtag = VLAN_NONE;
 	int rc;
 
-	sin.sin_family = AF_INET;
-	sin.sin_len = sizeof(struct sockaddr_in);
-	SINADDR(&sin) = e->addr;
+	if (e->ipv6 == 0) {
+		sin.sin_family = AF_INET;
+		sin.sin_len = sizeof(struct sockaddr_in);
+		sin.sin_addr.s_addr = e->addr[0];
+		sa = (void *)&sin;
+	} else {
+		sin6.sin6_family = AF_INET6;
+		sin6.sin6_len = sizeof(struct sockaddr_in6);
+		memcpy(&sin6.sin6_addr, &e->addr[0], sizeof(e->addr));
+		sa = (void *)&sin6;
+	}
 
-	rc = toe_l2_resolve(tod, e->ifp, SA(&sin), dmac, &vtag);
+	rc = toe_l2_resolve(tod, e->ifp, sa, dmac, &vtag);
 	if (rc == EWOULDBLOCK)
 		return (rc);
 
@@ -263,7 +336,7 @@ do_l2t_write_rpl2(struct sge_iq *iq, const struct rss_header *rss,
 	struct adapter *sc = iq->adapter;
 	const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1);
 	unsigned int tid = GET_TID(rpl);
-	unsigned int idx = tid & (L2T_SIZE - 1);
+	unsigned int idx = tid % L2T_SIZE;
 	int rc;
 
 	rc = do_l2t_write_rpl(iq, rss, m);
@@ -271,7 +344,7 @@ do_l2t_write_rpl2(struct sge_iq *iq, const struct rss_header *rss,
 		return (rc);
 
 	if (tid & F_SYNC_WR) {
-		struct l2t_entry *e = &sc->l2t->l2tab[idx];
+		struct l2t_entry *e = &sc->l2t->l2tab[idx - sc->vres.l2t.start];
 
 		mtx_lock(&e->lock);
 		if (e->state != L2T_STATE_SWITCHING) {
@@ -310,21 +383,22 @@ t4_l2t_get(struct port_info *pi, struct ifnet *ifp, struct sockaddr *sa)
 {
 	struct l2t_entry *e;
 	struct l2t_data *d = pi->adapter->l2t;
-	uint32_t addr = SINADDR(sa);
-	int hash = arp_hash(addr, ifp->if_index);
-	unsigned int smt_idx = pi->port_id;
+	u_int hash, smt_idx = pi->port_id;
 
-	if (sa->sa_family != AF_INET)
-		return (NULL);	/* XXX: no IPv6 support right now */
+	KASSERT(sa->sa_family == AF_INET || sa->sa_family == AF_INET6,
+	    ("%s: sa %p has unexpected sa_family %d", __func__, sa,
+	    sa->sa_family));
 
 #ifndef VLAN_TAG
 	if (ifp->if_type == IFT_L2VLAN)
 		return (NULL);
 #endif
 
+	hash = l2_hash(d, sa, ifp->if_index);
 	rw_wlock(&d->lock);
 	for (e = d->l2tab[hash].first; e; e = e->next) {
-		if (e->addr == addr && e->ifp == ifp && e->smt_idx == smt_idx) {
+		if (l2_cmp(sa, e) == 0 && e->ifp == ifp &&
+		    e->smt_idx == smt_idx) {
 			l2t_hold(d, e);
 			goto done;
 		}
@@ -338,7 +412,7 @@ t4_l2t_get(struct port_info *pi, struct ifnet *ifp, struct sockaddr *sa)
 		d->l2tab[hash].first = e;
 
 		e->state = L2T_STATE_RESOLVING;
-		e->addr = addr;
+		l2_store(sa, e);
 		e->ifp = ifp;
 		e->smt_idx = smt_idx;
 		e->hash = hash;
@@ -368,14 +442,14 @@ t4_l2_update(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa,
 	struct adapter *sc = tod->tod_softc;
 	struct l2t_entry *e;
 	struct l2t_data *d = sc->l2t;
-	uint32_t addr = SINADDR(sa);
-	int hash = arp_hash(addr, ifp->if_index);
+	u_int hash;
 
 	KASSERT(d != NULL, ("%s: no L2 table", __func__));
 
+	hash = l2_hash(d, sa, ifp->if_index);
 	rw_rlock(&d->lock);
 	for (e = d->l2tab[hash].first; e; e = e->next) {
-		if (e->addr == addr && e->ifp == ifp) {
+		if (l2_cmp(sa, e) == 0 && e->ifp == ifp) {
 			mtx_lock(&e->lock);
 			if (atomic_load_acq_int(&e->refcnt))
 				goto found;
diff --git a/sys/dev/digi/digi_isa.c b/sys/dev/digi/digi_isa.c
index f1f4a5f..0a0a2a0 100644
--- a/sys/dev/digi/digi_isa.c
+++ b/sys/dev/digi/digi_isa.c
@@ -462,7 +462,8 @@ static device_method_t digi_isa_methods[] = {
 	DEVMETHOD(device_attach, digi_isa_attach),
 	DEVMETHOD(device_detach, digi_detach),
 	DEVMETHOD(device_shutdown, digi_shutdown),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t digi_isa_drv = {
diff --git a/sys/dev/digi/digi_pci.c b/sys/dev/digi/digi_pci.c
index 342858e..55814cd 100644
--- a/sys/dev/digi/digi_pci.c
+++ b/sys/dev/digi/digi_pci.c
@@ -219,7 +219,8 @@ static device_method_t digi_pci_methods[] = {
 	DEVMETHOD(device_attach, digi_pci_attach),
 	DEVMETHOD(device_detach, digi_detach),
 	DEVMETHOD(device_shutdown, digi_shutdown),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t digi_pci_drv = {
diff --git a/sys/dev/e1000/if_em.c b/sys/dev/e1000/if_em.c
index 8ba32e0..3344697 100644
--- a/sys/dev/e1000/if_em.c
+++ b/sys/dev/e1000/if_em.c
@@ -309,7 +309,7 @@ static device_method_t em_methods[] = {
 	DEVMETHOD(device_shutdown, em_shutdown),
 	DEVMETHOD(device_suspend, em_suspend),
 	DEVMETHOD(device_resume, em_resume),
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t em_driver = {
diff --git a/sys/dev/e1000/if_igb.c b/sys/dev/e1000/if_igb.c
index 8eb3011..2921a91 100644
--- a/sys/dev/e1000/if_igb.c
+++ b/sys/dev/e1000/if_igb.c
@@ -294,7 +294,7 @@ static device_method_t igb_methods[] = {
 	DEVMETHOD(device_shutdown, igb_shutdown),
 	DEVMETHOD(device_suspend, igb_suspend),
 	DEVMETHOD(device_resume, igb_resume),
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t igb_driver = {
@@ -4330,8 +4330,8 @@ fail:
 	 * the rings that completed, the failing case will have
 	 * cleaned up for itself. 'i' is the endpoint.
 	 */
-	for (int j = 0; j > i; ++j) {
-		rxr = &adapter->rx_rings[i];
+	for (int j = 0; j < i; ++j) {
+		rxr = &adapter->rx_rings[j];
 		IGB_RX_LOCK(rxr);
 		igb_free_receive_ring(rxr);
 		IGB_RX_UNLOCK(rxr);
diff --git a/sys/dev/e1000/if_lem.c b/sys/dev/e1000/if_lem.c
index 58e6c7e..6b2b4f7 100644
--- a/sys/dev/e1000/if_lem.c
+++ b/sys/dev/e1000/if_lem.c
@@ -262,7 +262,7 @@ static device_method_t lem_methods[] = {
 	DEVMETHOD(device_shutdown, lem_shutdown),
 	DEVMETHOD(device_suspend, lem_suspend),
 	DEVMETHOD(device_resume, lem_resume),
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t lem_driver = {
diff --git a/sys/dev/ep/if_ep_eisa.c b/sys/dev/ep/if_ep_eisa.c
index e25a531..bc77ec8 100644
--- a/sys/dev/ep/if_ep_eisa.c
+++ b/sys/dev/ep/if_ep_eisa.c
@@ -238,7 +238,7 @@ static device_method_t ep_eisa_methods[] = {
 	DEVMETHOD(device_attach, ep_eisa_attach),
 	DEVMETHOD(device_detach, ep_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t ep_eisa_driver = {
diff --git a/sys/dev/ep/if_ep_isa.c b/sys/dev/ep/if_ep_isa.c
index 0455c40..e125daf 100644
--- a/sys/dev/ep/if_ep_isa.c
+++ b/sys/dev/ep/if_ep_isa.c
@@ -391,7 +391,7 @@ static device_method_t ep_isa_methods[] = {
 	DEVMETHOD(device_attach, ep_isa_attach),
 	DEVMETHOD(device_detach, ep_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t ep_isa_driver = {
diff --git a/sys/dev/ep/if_ep_mca.c b/sys/dev/ep/if_ep_mca.c
index 3c1645c..17097ab 100644
--- a/sys/dev/ep/if_ep_mca.c
+++ b/sys/dev/ep/if_ep_mca.c
@@ -145,7 +145,7 @@ static device_method_t ep_mca_methods[] = {
 	DEVMETHOD(device_attach, ep_mca_attach),
 	DEVMETHOD(device_detach, ep_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t ep_mca_driver = {
diff --git a/sys/dev/ep/if_ep_pccard.c b/sys/dev/ep/if_ep_pccard.c
index 98ebaf4..1b445f9 100644
--- a/sys/dev/ep/if_ep_pccard.c
+++ b/sys/dev/ep/if_ep_pccard.c
@@ -223,7 +223,7 @@ static device_method_t ep_pccard_methods[] = {
 	DEVMETHOD(device_attach, ep_pccard_attach),
 	DEVMETHOD(device_detach, ep_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t ep_pccard_driver = {
diff --git a/sys/dev/fdc/fdc_acpi.c b/sys/dev/fdc/fdc_acpi.c
index 7769a57..fd5bf90 100644
--- a/sys/dev/fdc/fdc_acpi.c
+++ b/sys/dev/fdc/fdc_acpi.c
@@ -258,7 +258,7 @@ static device_method_t fdc_acpi_methods[] = {
 	DEVMETHOD(bus_read_ivar,	fdc_read_ivar),
 	DEVMETHOD(bus_write_ivar,	fdc_write_ivar),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t fdc_acpi_driver = {
diff --git a/sys/dev/fdt/fdt_mips.c b/sys/dev/fdt/fdt_mips.c
index 0e6828e..4df31d8 100644
--- a/sys/dev/fdt/fdt_mips.c
+++ b/sys/dev/fdt/fdt_mips.c
@@ -49,8 +49,26 @@ struct fdt_fixup_entry fdt_fixup_table[] = {
 	{ NULL, NULL }
 };
 
+/*
+ * For PIC-free boards, provide a PIC decoder to be used with mips4k CP0
+ * interrupt control directly.
+ */
+static int
+fdt_pic_decode_mips4k_cp0(phandle_t node, pcell_t *intr, int *interrupt,
+    int *trig, int *pol)
+{
+
+	if (!fdt_is_compatible(node, "mips,mips4k"))
+		return (ENXIO);
+
+	*interrupt = fdt32_to_cpu(intr[0]);
+	*trig = INTR_TRIGGER_CONFORM;
+	*pol = INTR_POLARITY_CONFORM;
+
+	return (0);
+}
+
 fdt_pic_decode_t fdt_pic_table[] = {
-	NULL,
-	NULL,
+	&fdt_pic_decode_mips4k_cp0,
 	NULL
 };
diff --git a/sys/dev/hpt27xx/hpt27xx_config.c b/sys/dev/hpt27xx/hpt27xx_config.c
index f87c7ee..5203f4a 100644
--- a/sys/dev/hpt27xx/hpt27xx_config.c
+++ b/sys/dev/hpt27xx/hpt27xx_config.c
@@ -58,9 +58,9 @@ int init_config(void)
 	return 0;
 }
 
-char driver_name[] = "hpt27xx";
-char driver_name_long[] = "RocketRAID 27xx controller driver";
-char driver_ver[] = "v1.0 (" __DATE__ " " __TIME__ ")";
+const char driver_name[] = "hpt27xx";
+const char driver_name_long[] = "RocketRAID 27xx controller driver";
+const char driver_ver[] = "v1.0";
 int  osm_max_targets = 0xff;
 
 
diff --git a/sys/dev/hpt27xx/os_bsd.h b/sys/dev/hpt27xx/os_bsd.h
index 3895f40..e3f8e7d 100644
--- a/sys/dev/hpt27xx/os_bsd.h
+++ b/sys/dev/hpt27xx/os_bsd.h
@@ -135,9 +135,9 @@ INQUIRYDATA, *PINQUIRYDATA;
 #include <dev/hpt27xx/ldm.h>
 
 /* driver parameters */
-extern char driver_name[];
-extern char driver_name_long[];
-extern char driver_ver[];
+extern const char driver_name[];
+extern const char driver_name_long[];
+extern const char driver_ver[];
 extern int  osm_max_targets;
 
 /*
diff --git a/sys/dev/hpt27xx/osm_bsd.c b/sys/dev/hpt27xx/osm_bsd.c
index ac7e9d8..fab1c71d 100644
--- a/sys/dev/hpt27xx/osm_bsd.c
+++ b/sys/dev/hpt27xx/osm_bsd.c
@@ -167,7 +167,8 @@ static int hpt_alloc_mem(PVBUS_EXT vbus_ext)
 
 		HPT_ASSERT((f->size & (f->alignment-1))==0);
 
-		for (order=0, size=PAGE_SIZE; size<f->size; order++, size<<=1) ;
+		for (order=0, size=PAGE_SIZE; size<f->size; order++, size<<=1)
+			;
 
 		KdPrint(("%s: %d*%d=%d bytes, order %d",
 			f->tag, f->count, f->size, f->count*f->size, order));
@@ -1036,6 +1037,7 @@ static void hpt_final_init(void *dummy)
 	}
 
 	if (!i) {
+		if (bootverbose)
 			os_printk("no controller detected.");
 		return;
 	}
@@ -1177,7 +1179,7 @@ static void hpt_final_init(void *dummy)
 	}	
 
 	make_dev(&hpt_cdevsw, DRIVER_MINOR, UID_ROOT, GID_OPERATOR,
-	    S_IRUSR | S_IWUSR, driver_name);
+	    S_IRUSR | S_IWUSR, "%s", driver_name);
 }
 
 #if defined(KLD_MODULE) && (__FreeBSD_version >= 503000)
@@ -1224,6 +1226,7 @@ static void override_kernel_driver(void)
 
 static void hpt_init(void *dummy)
 {
+	if (bootverbose)
 		os_printk("%s %s", driver_name_long, driver_ver);
 
 	override_kernel_driver();
diff --git a/sys/dev/hwpmc/hwpmc_core.c b/sys/dev/hwpmc/hwpmc_core.c
index 71d8d8a..77eb93b 100644
--- a/sys/dev/hwpmc/hwpmc_core.c
+++ b/sys/dev/hwpmc/hwpmc_core.c
@@ -560,7 +560,8 @@ struct iap_event_descr {
 #define	IAP_F_SB	(1 << 6)	/* CPU: Sandy Bridge */
 #define	IAP_F_IB	(1 << 7)	/* CPU: Ivy Bridge */
 #define	IAP_F_SBX	(1 << 8)	/* CPU: Sandy Bridge Xeon */
-#define	IAP_F_FM	(1 << 9)	/* Fixed mask */
+#define	IAP_F_IBX	(1 << 9)	/* CPU: Ivy Bridge */
+#define	IAP_F_FM	(1 << 10)	/* Fixed mask */
 
 #define	IAP_F_ALLCPUSCORE2					\
     (IAP_F_CC | IAP_F_CC2 | IAP_F_CC2E | IAP_F_CA)
@@ -603,7 +604,7 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(03H_01H, 0x03, 0x01, IAP_F_FM | IAP_F_I7O | IAP_F_SB |
 	IAP_F_SBX),
     IAPDESCR(03H_02H, 0x03, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | 
-	IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(03H_04H, 0x03, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7O),
     IAPDESCR(03H_08H, 0x03, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB |
 	IAP_F_SBX),
@@ -619,9 +620,9 @@ static struct iap_event_descr iap_events[] = {
 
     IAPDESCR(05H_00H, 0x05, 0x00, IAP_F_FM | IAP_F_CC),
     IAPDESCR(05H_01H, 0x05, 0x01, IAP_F_FM | IAP_F_I7O | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(05H_02H, 0x05, 0x02, IAP_F_FM | IAP_F_I7O | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(05H_03H, 0x05, 0x03, IAP_F_FM | IAP_F_I7O),
 
     IAPDESCR(06H_00H, 0x06, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2 |
@@ -634,7 +635,7 @@ static struct iap_event_descr iap_events[] = {
 
     IAPDESCR(07H_00H, 0x07, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2),
     IAPDESCR(07H_01H, 0x07, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 | 
-	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(07H_02H, 0x07, 0x02, IAP_F_FM | IAP_F_ALLCPUSCORE2),
     IAPDESCR(07H_03H, 0x07, 0x03, IAP_F_FM | IAP_F_ALLCPUSCORE2),
     IAPDESCR(07H_06H, 0x07, 0x06, IAP_F_FM | IAP_F_CA),
@@ -657,9 +658,9 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(08H_20H, 0x08, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(08H_40H, 0x08, 0x40, IAP_F_FM | IAP_F_I7O),
     IAPDESCR(08H_80H, 0x08, 0x80, IAP_F_FM | IAP_F_I7),
-    IAPDESCR(08H_81H, 0x08, 0x81, IAP_F_FM | IAP_F_IB),
-    IAPDESCR(08H_82H, 0x08, 0x82, IAP_F_FM | IAP_F_IB),
-    IAPDESCR(08H_84H, 0x08, 0x84, IAP_F_FM | IAP_F_IB),
+    IAPDESCR(08H_81H, 0x08, 0x81, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
+    IAPDESCR(08H_82H, 0x08, 0x82, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
+    IAPDESCR(08H_84H, 0x08, 0x84, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
 
     IAPDESCR(09H_01H, 0x09, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7O),
     IAPDESCR(09H_02H, 0x09, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7O),
@@ -679,11 +680,11 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(0DH_40H, 0x0D, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
 
     IAPDESCR(0EH_01H, 0x0E, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(0EH_02H, 0x0E, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
-    IAPDESCR(0EH_10H, 0x0E, 0x10, IAP_F_FM | IAP_F_IB),
-    IAPDESCR(0EH_20H, 0x0E, 0x20, IAP_F_FM | IAP_F_IB),
-    IAPDESCR(0EH_40H, 0x0E, 0x40, IAP_F_FM | IAP_F_IB),
+    IAPDESCR(0EH_10H, 0x0E, 0x10, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
+    IAPDESCR(0EH_20H, 0x0E, 0x20, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
+    IAPDESCR(0EH_40H, 0x0E, 0x40, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
 
     IAPDESCR(0FH_01H, 0x0F, 0x01, IAP_F_FM | IAP_F_I7),
     IAPDESCR(0FH_02H, 0x0F, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
@@ -733,7 +734,7 @@ static struct iap_event_descr iap_events[] = {
 
     IAPDESCR(14H_00H, 0x14, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2),
     IAPDESCR(14H_01H, 0x14, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 | 
-	 IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	 IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(14H_02H, 0x14, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
 
     IAPDESCR(17H_01H, 0x17, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
@@ -760,28 +761,28 @@ static struct iap_event_descr iap_events[] = {
 
     IAPDESCR(24H, 0x24, IAP_M_CORE | IAP_M_PREFETCH, IAP_F_ALLCPUSCORE2),
     IAPDESCR(24H_01H, 0x24, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(24H_02H, 0x24, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(24H_03H, 0x24, 0x03, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(24H_04H, 0x24, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(24H_08H, 0x24, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(24H_0CH, 0x24, 0x0C, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(24H_10H, 0x24, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(24H_20H, 0x24, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(24H_30H, 0x24, 0x30, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(24H_40H, 0x24, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(24H_80H, 0x24, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(24H_C0H, 0x24, 0xC0, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(24H_AAH, 0x24, 0xAA, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(24H_FFH, 0x24, 0xFF, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
 
@@ -802,15 +803,15 @@ static struct iap_event_descr iap_events[] = {
 
     IAPDESCR(27H, 0x27, IAP_M_CORE | IAP_M_PREFETCH, IAP_F_ALLCPUSCORE2),
     IAPDESCR(27H_01H, 0x27, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(27H_02H, 0x27, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(27H_04H, 0x27, 0x04, IAP_F_FM | IAP_F_I7O | IAP_F_SB |
 	IAP_F_SBX),
     IAPDESCR(27H_08H, 0x27, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(27H_0EH, 0x27, 0x0E, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(27H_0FH, 0x27, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(27H_10H, 0x27, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(27H_20H, 0x27, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(27H_40H, 0x27, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
@@ -819,13 +820,15 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(27H_F0H, 0x27, 0xF0, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
 
     IAPDESCR(28H, 0x28, IAP_M_CORE | IAP_M_MESI, IAP_F_ALLCPUSCORE2),
-    IAPDESCR(28H_01H, 0x28, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB | IAP_F_SBX),
+    IAPDESCR(28H_01H, 0x28, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB | IAP_F_SBX |
+	IAP_F_IBX),
     IAPDESCR(28H_02H, 0x28, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SBX),
     IAPDESCR(28H_04H, 0x28, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(28H_08H, 0x28, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
-    IAPDESCR(28H_0FH, 0x28, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
+    IAPDESCR(28H_0FH, 0x28, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB | IAP_F_SBX |
+	IAP_F_IBX),
 
     IAPDESCR(29H, 0x29, IAP_M_CORE | IAP_M_MESI, IAP_F_CC),
     IAPDESCR(29H, 0x29, IAP_M_CORE | IAP_M_MESI | IAP_M_PREFETCH,
@@ -838,9 +841,9 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(2EH_01H, 0x2E, 0x01, IAP_F_FM | IAP_F_WM),
     IAPDESCR(2EH_02H, 0x2E, 0x02, IAP_F_FM | IAP_F_WM),
     IAPDESCR(2EH_41H, 0x2E, 0x41, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7 |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(2EH_4FH, 0x2E, 0x4F, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7 |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(30H, 0x30, IAP_M_CORE | IAP_M_MESI | IAP_M_PREFETCH,
 	IAP_F_ALLCPUSCORE2),
@@ -853,9 +856,9 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(3BH_C0H, 0x3B, 0xC0, IAP_F_FM | IAP_F_ALLCPUSCORE2),
 
     IAPDESCR(3CH_00H, 0x3C, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
-        IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+        IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(3CH_01H, 0x3C, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
-        IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+        IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(3CH_02H, 0x3C, 0x02, IAP_F_FM | IAP_F_ALLCPUSCORE2),
 
     IAPDESCR(3DH_01H, 0x3D, 0x01, IAP_F_FM | IAP_F_I7O),
@@ -897,18 +900,18 @@ static struct iap_event_descr iap_events[] = {
 
     IAPDESCR(48H_00H, 0x48, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
     IAPDESCR(48H_01H, 0x48, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(48H_02H, 0x48, 0x02, IAP_F_FM | IAP_F_I7O),
 
     IAPDESCR(49H_00H, 0x49, 0x00, IAP_F_FM | IAP_F_CC),
     IAPDESCR(49H_01H, 0x49, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-        IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+        IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX  | IAP_F_IBX),
     IAPDESCR(49H_02H, 0x49, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-        IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+        IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(49H_04H, 0x49, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(49H_10H, 0x49, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-        IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+        IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(49H_20H, 0x49, 0x20, IAP_F_FM | IAP_F_I7),
     IAPDESCR(49H_40H, 0x49, 0x40, IAP_F_FM | IAP_F_I7O),
     IAPDESCR(49H_80H, 0x49, 0x80, IAP_F_FM | IAP_F_WM | IAP_F_I7),
@@ -921,9 +924,9 @@ static struct iap_event_descr iap_events[] = {
 
     IAPDESCR(4CH_00H, 0x4C, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
     IAPDESCR(4CH_01H, 0x4C, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(4CH_02H, 0x4C, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(4DH_01H, 0x4D, 0x01, IAP_F_FM | IAP_F_I7O),
 
@@ -940,7 +943,7 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(4FH_10H, 0x4F, 0x10, IAP_F_FM | IAP_F_WM),
 
     IAPDESCR(51H_01H, 0x51, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(51H_02H, 0x51, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
 	IAP_F_SB | IAP_F_SBX),
     IAPDESCR(51H_04H, 0x51, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
@@ -952,10 +955,10 @@ static struct iap_event_descr iap_events[] = {
 
     IAPDESCR(53H_01H, 0x53, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
 
-    IAPDESCR(58H_01H, 0x58, 0x01, IAP_F_FM | IAP_F_IB),
-    IAPDESCR(58H_02H, 0x58, 0x02, IAP_F_FM | IAP_F_IB),
-    IAPDESCR(58H_04H, 0x58, 0x04, IAP_F_FM | IAP_F_IB),
-    IAPDESCR(58H_08H, 0x58, 0x08, IAP_F_FM | IAP_F_IB),
+    IAPDESCR(58H_01H, 0x58, 0x01, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
+    IAPDESCR(58H_02H, 0x58, 0x02, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
+    IAPDESCR(58H_04H, 0x58, 0x04, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
+    IAPDESCR(58H_08H, 0x58, 0x08, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
 
     IAPDESCR(59H_20H, 0x59, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
     IAPDESCR(59H_40H, 0x59, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
@@ -967,23 +970,25 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(5BH_4FH, 0x5B, 0x4F, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
 
     IAPDESCR(5CH_01H, 0x5C, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(5CH_02H, 0x5C, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     
     IAPDESCR(5EH_01H, 0x5E, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(5FH_01H, 0x5F, 0x01, IAP_F_FM | IAP_F_IB),
+    IAPDESCR(5FH_04H, 0x5F, 0x04, IAP_F_IBX),
 
     IAPDESCR(60H, 0x60, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2),
     IAPDESCR(60H_01H, 0x60, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
-    IAPDESCR(60H_02H, 0x60, 0x02, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_IB),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
+    IAPDESCR(60H_02H, 0x60, 0x02, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_IB |
+	IAP_F_IBX),
     IAPDESCR(60H_04H, 0x60, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(60H_08H, 0x60, 0x08, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(61H, 0x61, IAP_M_AGENT, IAP_F_CA | IAP_F_CC2),
     IAPDESCR(61H_00H, 0x61, 0x00, IAP_F_FM | IAP_F_CC),
@@ -995,9 +1000,9 @@ static struct iap_event_descr iap_events[] = {
 	IAP_F_CA | IAP_F_CC2),
     IAPDESCR(63H, 0x63, IAP_M_CORE, IAP_F_CC),
     IAPDESCR(63H_01H, 0x63, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(63H_02H, 0x63, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(64H, 0x64, IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
     IAPDESCR(64H_40H, 0x64, 0x40, IAP_F_FM | IAP_F_CC),
@@ -1039,20 +1044,20 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(78H, 0x78, IAP_M_CORE | IAP_M_SNOOPTYPE, IAP_F_CA | IAP_F_CC2),
 
     IAPDESCR(79H_02H, 0x79, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(79H_04H, 0x79, 0x04, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(79H_08H, 0x79, 0x08, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(79H_10H, 0x79, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(79H_20H, 0x79, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(79H_30H, 0x79, 0x30, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
-    IAPDESCR(79H_18H, 0x79, 0x18, IAP_F_FM | IAP_F_IB),
-    IAPDESCR(79H_24H, 0x79, 0x24, IAP_F_FM | IAP_F_IB),
-    IAPDESCR(79H_3CH, 0x79, 0x3C, IAP_F_FM | IAP_F_IB),
+	IAP_F_SBX | IAP_F_IBX),
+    IAPDESCR(79H_18H, 0x79, 0x18, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
+    IAPDESCR(79H_24H, 0x79, 0x24, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
+    IAPDESCR(79H_3CH, 0x79, 0x3C, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
  
     IAPDESCR(7AH, 0x7A, IAP_M_AGENT, IAP_F_CA | IAP_F_CC2),
 
@@ -1068,7 +1073,7 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(80H_00H, 0x80, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
     IAPDESCR(80H_01H, 0x80, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(80H_02H, 0x80, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
-	IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(80H_03H, 0x80, 0x03, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
 	IAP_F_WM),
     IAPDESCR(80H_04H, 0x80, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
@@ -1089,13 +1094,13 @@ static struct iap_event_descr iap_events[] = {
 
     IAPDESCR(85H_00H, 0x85, 0x00, IAP_F_FM | IAP_F_CC),
     IAPDESCR(85H_01H, 0x85, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(85H_02H, 0x85, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(85H_04H, 0x85, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(85H_10H, 0x85, 0x10, IAP_F_FM | IAP_F_I7O | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(85H_20H, 0x85, 0x20, IAP_F_FM | IAP_F_I7O),
     IAPDESCR(85H_40H, 0x85, 0x40, IAP_F_FM | IAP_F_I7O),
     IAPDESCR(85H_80H, 0x85, 0x80, IAP_F_FM | IAP_F_WM | IAP_F_I7O),
@@ -1104,57 +1109,57 @@ static struct iap_event_descr iap_events[] = {
 
     IAPDESCR(87H_00H, 0x87, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
     IAPDESCR(87H_01H, 0x87, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(87H_02H, 0x87, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(87H_04H, 0x87, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(87H_08H, 0x87, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(87H_0FH, 0x87, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
 
     IAPDESCR(88H_00H, 0x88, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
     IAPDESCR(88H_01H, 0x88, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(88H_02H, 0x88, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(88H_04H, 0x88, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(88H_07H, 0x88, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(88H_08H, 0x88, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(88H_10H, 0x88, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(88H_20H, 0x88, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(88H_30H, 0x88, 0x30, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(88H_40H, 0x88, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(88H_7FH, 0x88, 0x7F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(88H_80H, 0x88, 0x80, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(88H_FFH, 0x88, 0xFF, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(89H_00H, 0x89, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
     IAPDESCR(89H_01H, 0x89, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(89H_02H, 0x89, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(89H_04H, 0x89, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(89H_07H, 0x89, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(89H_08H, 0x89, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(89H_10H, 0x89, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(89H_20H, 0x89, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(89H_30H, 0x89, 0x30, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(89H_40H, 0x89, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(89H_7FH, 0x89, 0x7F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(89H_80H, 0x89, 0x80, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(89H_FFH, 0x89, 0xFF, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(8AH_00H, 0x8A, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
     IAPDESCR(8BH_00H, 0x8B, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
@@ -1170,44 +1175,44 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(94H_00H, 0x94, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
 
     IAPDESCR(9CH_01H, 0x9C, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX), 
+	IAP_F_SBX | IAP_F_IBX), 
 
     IAPDESCR(97H_00H, 0x97, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
     IAPDESCR(98H_00H, 0x98, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
     IAPDESCR(A0H_00H, 0xA0, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
 
     IAPDESCR(A1H_01H, 0xA1, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(A1H_02H, 0xA1, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(A1H_04H, 0xA1, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(A1H_08H, 0xA1, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(A1H_0CH, 0xA1, 0x0C, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(A1H_10H, 0xA1, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(A1H_20H, 0xA1, 0x20, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(A1H_30H, 0xA1, 0x30, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(A1H_40H, 0xA1, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(A1H_80H, 0xA1, 0x80, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     
     IAPDESCR(A2H_00H, 0xA2, 0x00, IAP_F_FM | IAP_F_CC),
     IAPDESCR(A2H_01H, 0xA2, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(A2H_02H, 0xA2, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
 	IAP_F_SB | IAP_F_SBX),
     IAPDESCR(A2H_04H, 0xA2, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(A2H_08H, 0xA2, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(A2H_10H, 0xA2, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(A2H_20H, 0xA2, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
 	IAP_F_SB | IAP_F_SBX),
     IAPDESCR(A2H_40H, 0xA2, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
@@ -1215,9 +1220,10 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(A2H_80H, 0xA2, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
 	IAP_F_SB | IAP_F_SBX),
 
-    IAPDESCR(A3H_01H, 0xA3, 0x01, IAP_F_FM | IAP_F_SBX),
-    IAPDESCR(A3H_02H, 0xA3, 0x02, IAP_F_FM | IAP_F_SBX),
-    IAPDESCR(A3H_04H, 0xA3, 0x04, IAP_F_FM | IAP_F_SBX),
+    IAPDESCR(A3H_01H, 0xA3, 0x01, IAP_F_FM | IAP_F_SBX | IAP_F_IBX),
+    IAPDESCR(A3H_02H, 0xA3, 0x02, IAP_F_FM | IAP_F_SBX | IAP_F_IBX),
+    IAPDESCR(A3H_04H, 0xA3, 0x04, IAP_F_FM | IAP_F_SBX | IAP_F_IBX),
+    IAPDESCR(A3H_08H, 0xA3, 0x08, IAP_F_IBX),
 
     IAPDESCR(A6H_01H, 0xA6, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(A7H_01H, 0xA7, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
@@ -1229,26 +1235,27 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(AAH_08H, 0xAA, 0x08, IAP_F_FM | IAP_F_CC2),
 
     IAPDESCR(ABH_01H, 0xAB, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(ABH_02H, 0xAB, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(ACH_02H, 0xAC, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
     IAPDESCR(ACH_08H, 0xAC, 0x08, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(ACH_0AH, 0xAC, 0x0A, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
 
     IAPDESCR(AEH_01H, 0xAE, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IB),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(B0H_00H, 0xB0, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
     IAPDESCR(B0H_01H, 0xB0, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
-    IAPDESCR(B0H_02H, 0xB0, 0x02, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_IB),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
+    IAPDESCR(B0H_02H, 0xB0, 0x02, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_IB |
+	IAP_F_IBX),
     IAPDESCR(B0H_04H, 0xB0, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(B0H_08H, 0xB0, 0x08, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(B0H_10H, 0xB0, 0x10, IAP_F_FM | IAP_F_WM | IAP_F_I7O),
     IAPDESCR(B0H_20H, 0xB0, 0x20, IAP_F_FM | IAP_F_I7O),
     IAPDESCR(B0H_40H, 0xB0, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
@@ -1256,9 +1263,9 @@ static struct iap_event_descr iap_events[] = {
 
     IAPDESCR(B1H_00H, 0xB1, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
     IAPDESCR(B1H_01H, 0xB1, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(B1H_02H, 0xB1, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(B1H_04H, 0xB1, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(B1H_08H, 0xB1, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(B1H_10H, 0xB1, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
@@ -1295,7 +1302,7 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(B6H_01H, 0xB6, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
 
     IAPDESCR(B7H_01H, 0xB7, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(B8H_01H, 0xB8, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(B8H_02H, 0xB8, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
@@ -1305,19 +1312,20 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(BAH_02H, 0xBA, 0x02, IAP_F_FM | IAP_F_I7O),
 
     IAPDESCR(BBH_01H, 0xBB, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(BDH_01H, 0xBD, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(BDH_20H, 0xBD, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(BFH_05H, 0xBF, 0x05, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
 
     IAPDESCR(C0H_00H, 0xC0, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(C0H_01H, 0xC0, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
+	IAP_F_IBX),
     IAPDESCR(C0H_02H, 0xC0, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
 	IAP_F_I7 | IAP_F_WM | IAP_F_SB),
     IAPDESCR(C0H_04H, 0xC0, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
@@ -1328,18 +1336,20 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(C1H_01H, 0xC1, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
     IAPDESCR(C1H_02H, 0xC1, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
     IAPDESCR(C1H_08H, 0xC1, 0x08, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(C1H_10H, 0xC1, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(C1H_20H, 0xC1, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(C1H_FEH, 0xC1, 0xFE, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
 
     IAPDESCR(C2H_00H, 0xC2, 0x00, IAP_F_FM | IAP_F_CC),
     IAPDESCR(C2H_01H, 0xC2, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
+	IAP_F_IBX),
     IAPDESCR(C2H_02H, 0xC2, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
+	IAP_F_IBX),
     IAPDESCR(C2H_04H, 0xC2, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
 	IAP_F_I7 | IAP_F_WM),
     IAPDESCR(C2H_07H, 0xC2, 0x07, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
@@ -1351,44 +1361,50 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(C3H_01H, 0xC3, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
 	IAP_F_I7 | IAP_F_WM),
     IAPDESCR(C3H_02H, 0xC3, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(C3H_04H, 0xC3, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
+	IAP_F_IBX),
     IAPDESCR(C3H_10H, 0xC3, 0x10, IAP_F_FM | IAP_F_I7O),
     IAPDESCR(C3H_20H, 0xC3, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(C4H_00H, 0xC4, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
-	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
+	IAP_F_IBX),
     IAPDESCR(C4H_01H, 0xC4, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
+	IAP_F_IBX),
     IAPDESCR(C4H_02H, 0xC4, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
+	IAP_F_IBX),
     IAPDESCR(C4H_04H, 0xC4, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
+	IAP_F_IBX),
     IAPDESCR(C4H_08H, 0xC4, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(C4H_0CH, 0xC4, 0x0C, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
     IAPDESCR(C4H_0FH, 0xC4, 0x0F, IAP_F_FM | IAP_F_CA),
     IAPDESCR(C4H_10H, 0xC4, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(C4H_20H, 0xC4, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(C4H_40H, 0xC4, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(C5H_00H, 0xC5, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
-	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
+	IAP_F_IBX),
     IAPDESCR(C5H_01H, 0xC5, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_SB |
-	IAP_F_IB | IAP_F_SBX),
+	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(C5H_02H, 0xC5, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(C5H_04H, 0xC5, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_SB |
-	IAP_F_IB | IAP_F_SBX),
+	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(C5H_10H, 0xC5, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(C5H_20H, 0xC5, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(C6H_00H, 0xC6, 0x00, IAP_F_FM | IAP_F_CC),
     IAPDESCR(C6H_01H, 0xC6, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
@@ -1415,15 +1431,15 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(CAH_00H, 0xCA, 0x00, IAP_F_FM | IAP_F_CC),
     IAPDESCR(CAH_01H, 0xCA, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
     IAPDESCR(CAH_02H, 0xCA, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(CAH_04H, 0xCA, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(CAH_08H, 0xCA, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(CAH_10H, 0xCA, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(CAH_1EH, 0xCA, 0x1E, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(CBH_01H, 0xCB, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
 	IAP_F_I7 | IAP_F_WM),
@@ -1445,56 +1461,59 @@ static struct iap_event_descr iap_events[] = {
 	IAP_F_I7 | IAP_F_WM),
     IAPDESCR(CCH_03H, 0xCC, 0x03, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(CCH_20H, 0xCC, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(CDH_00H, 0xCD, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
     IAPDESCR(CDH_01H, 0xCD, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(CDH_02H, 0xCD, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX), 
+	IAP_F_SBX | IAP_F_IBX), 
 
     IAPDESCR(CEH_00H, 0xCE, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
     IAPDESCR(CFH_00H, 0xCF, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
 
     IAPDESCR(D0H_00H, 0xD0, 0x00, IAP_F_FM | IAP_F_CC),
     IAPDESCR(D0H_01H, 0xD0, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(D0H_02H, 0xD0, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(D0H_10H, 0xD0, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(D0H_20H, 0xD0, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(D0H_40H, 0xD0, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(D0H_80H, 0xD0, 0X80, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
  
     IAPDESCR(D1H_01H, 0xD1, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_SB |
-	IAP_F_IB | IAP_F_SBX),
+	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(D1H_02H, 0xD1, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(D1H_04H, 0xD1, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(D1H_08H, 0xD1, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
-    IAPDESCR(D1H_20H, 0xD1, 0x20, IAP_F_FM | IAP_F_SBX),
+    IAPDESCR(D1H_20H, 0xD1, 0x20, IAP_F_FM | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(D1H_40H, 0xD1, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(D2H_01H, 0xD2, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB),
+	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_IBX),
     IAPDESCR(D2H_02H, 0xD2, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB),
+	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_IBX),
     IAPDESCR(D2H_04H, 0xD2, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB),
+	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_IBX),
     IAPDESCR(D2H_08H, 0xD2, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
-	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB),
+	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_IBX),
     IAPDESCR(D2H_0FH, 0xD2, 0x0F, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
 	IAP_F_I7 | IAP_F_WM),
     IAPDESCR(D2H_10H, 0xD2, 0x10, IAP_F_FM | IAP_F_CC2E),
 
-    IAPDESCR(D3H_01H, 0xD3, 0x01, IAP_F_FM | IAP_F_IB | IAP_F_SBX),
-    IAPDESCR(D3H_04H, 0xD3, 0x04, IAP_F_FM | IAP_F_SBX),
+    IAPDESCR(D3H_01H, 0xD3, 0x01, IAP_F_FM | IAP_F_IB | IAP_F_SBX |
+	IAP_F_IBX),
+    IAPDESCR(D3H_04H, 0xD3, 0x04, IAP_F_FM | IAP_F_SBX | IAP_F_IBX),
+    IAPDESCR(D3H_10H, 0xD3, 0x10, IAP_F_IBX),
+    IAPDESCR(D3H_20H, 0xD3, 0x20, IAP_F_IBX),
 
     IAPDESCR(D4H_01H, 0xD4, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
 	IAP_F_I7 | IAP_F_WM),
@@ -1553,6 +1572,7 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(E6H_01H, 0xE6, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
 	IAP_F_WM | IAP_F_SBX),
     IAPDESCR(E6H_02H, 0xE6, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
+    IAPDESCR(E6H_1FH, 0xE6, 0x1F, IAP_F_IBX),
 
     IAPDESCR(E8H_01H, 0xE8, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(E8H_02H, 0xE8, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
@@ -1562,40 +1582,41 @@ static struct iap_event_descr iap_events[] = {
 
     IAPDESCR(F0H_00H, 0xF0, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
     IAPDESCR(F0H_01H, 0xF0, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(F0H_02H, 0xF0, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(F0H_04H, 0xF0, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(F0H_08H, 0xF0, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(F0H_10H, 0xF0, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(F0H_20H, 0xF0, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(F0H_40H, 0xF0, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(F0H_80H, 0xF0, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(F1H_01H, 0xF1, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
-	IAP_F_SBX),
+	IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(F1H_02H, 0xF1, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(F1H_04H, 0xF1, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(F1H_07H, 0xF1, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
 
     IAPDESCR(F2H_01H, 0xF2, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(F2H_02H, 0xF2, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(F2H_04H, 0xF2, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
     IAPDESCR(F2H_08H, 0xF2, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX),
-    IAPDESCR(F2H_0AH, 0xF2, 0x0A, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
+    IAPDESCR(F2H_0AH, 0xF2, 0x0A, IAP_F_FM | IAP_F_SB | IAP_F_SBX |
+	IAP_F_IBX),
     IAPDESCR(F2H_0FH, 0xF2, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
 
     IAPDESCR(F3H_01H, 0xF3, 0x01, IAP_F_FM | IAP_F_I7O),
@@ -1782,7 +1803,7 @@ iap_event_westmere_ok_on_counter(enum pmc_event pe, int ri)
 }
 
 static int
-iap_event_sb_sbx_ib_ok_on_counter(enum pmc_event pe, int ri)
+iap_event_sb_sbx_ib_ibx_ok_on_counter(enum pmc_event pe, int ri)
 {
 	uint32_t mask;
 
@@ -1801,6 +1822,7 @@ iap_event_sb_sbx_ib_ok_on_counter(enum pmc_event pe, int ri)
 		mask = 0x4;
 		break;
 		/* Events valid only on counter 3. */
+	case PMC_EV_IAP_EVENT_A3H_08H:
 	case PMC_EV_IAP_EVENT_BBH_01H:
 	case PMC_EV_IAP_EVENT_CDH_01H:
 	case PMC_EV_IAP_EVENT_CDH_02H:
@@ -1892,7 +1914,8 @@ iap_allocate_pmc(int cpu, int ri, struct pmc *pm,
 	case PMC_CPU_INTEL_SANDYBRIDGE:
 	case PMC_CPU_INTEL_SANDYBRIDGE_XEON:
 	case PMC_CPU_INTEL_IVYBRIDGE:
-		if (iap_event_sb_sbx_ib_ok_on_counter(ev, ri) == 0)
+	case PMC_CPU_INTEL_IVYBRIDGE_XEON:
+		if (iap_event_sb_sbx_ib_ibx_ok_on_counter(ev, ri) == 0)
 			return (EINVAL);
 		break;
 	case PMC_CPU_INTEL_WESTMERE:
@@ -1929,6 +1952,9 @@ iap_allocate_pmc(int cpu, int ri, struct pmc *pm,
 	case PMC_CPU_INTEL_IVYBRIDGE:
 		cpuflag = IAP_F_IB;
 		break;
+	case PMC_CPU_INTEL_IVYBRIDGE_XEON:
+		cpuflag = IAP_F_IBX;
+		break;
 	case PMC_CPU_INTEL_SANDYBRIDGE:	
 		cpuflag = IAP_F_SB;
 		break;
@@ -2043,8 +2069,9 @@ iap_allocate_pmc(int cpu, int ri, struct pmc *pm,
 		    a->pm_md.pm_iap.pm_iap_rsp & ~IA_OFFCORE_RSP_MASK_I7WM)
 			return (EINVAL);
 		else if ((core_cputype == PMC_CPU_INTEL_SANDYBRIDGE ||
-		    core_cputype == PMC_CPU_INTEL_SANDYBRIDGE_XEON ||
-			core_cputype == PMC_CPU_INTEL_IVYBRIDGE) &&
+			core_cputype == PMC_CPU_INTEL_SANDYBRIDGE_XEON ||
+			core_cputype == PMC_CPU_INTEL_IVYBRIDGE ||
+			core_cputype == PMC_CPU_INTEL_IVYBRIDGE_XEON) &&
 		    a->pm_md.pm_iap.pm_iap_rsp & ~IA_OFFCORE_RSP_MASK_SBIB)
 			return (EINVAL);
 		pm->pm_md.pm_iap.pm_iap_rsp = a->pm_md.pm_iap.pm_iap_rsp;
diff --git a/sys/dev/hwpmc/hwpmc_intel.c b/sys/dev/hwpmc/hwpmc_intel.c
index f633ac9..00ec29e 100644
--- a/sys/dev/hwpmc/hwpmc_intel.c
+++ b/sys/dev/hwpmc/hwpmc_intel.c
@@ -154,6 +154,10 @@ pmc_intel_initialize(void)
 			cputype = PMC_CPU_INTEL_IVYBRIDGE;
 			nclasses = 3;
 			break;
+		case 0x3E:	/* Per Intel document 325462-045US 01/2013. */
+			cputype = PMC_CPU_INTEL_IVYBRIDGE_XEON;
+			nclasses = 3;
+			break;
 		}
 		break;
 #if	defined(__i386__) || defined(__amd64__)
@@ -196,6 +200,7 @@ pmc_intel_initialize(void)
 	case PMC_CPU_INTEL_SANDYBRIDGE:
 	case PMC_CPU_INTEL_WESTMERE:
 	case PMC_CPU_INTEL_SANDYBRIDGE_XEON:
+	case PMC_CPU_INTEL_IVYBRIDGE_XEON:
 		error = pmc_core_initialize(pmc_mdep, ncpus);
 		break;
 
@@ -280,6 +285,7 @@ pmc_intel_finalize(struct pmc_mdep *md)
 	case PMC_CPU_INTEL_SANDYBRIDGE:
 	case PMC_CPU_INTEL_WESTMERE:
 	case PMC_CPU_INTEL_SANDYBRIDGE_XEON:
+	case PMC_CPU_INTEL_IVYBRIDGE_XEON:
 		pmc_core_finalize(md);
 		break;
 
diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c
index 9e82a34..2f2f05a 100644
--- a/sys/dev/hwpmc/hwpmc_mod.c
+++ b/sys/dev/hwpmc/hwpmc_mod.c
@@ -3022,7 +3022,7 @@ pmc_syscall_handler(struct thread *td, void *syscall_args)
 		}
 
 		nevent = 0;
-		for (ev = PMC_EV_SOFT_FIRST; ev <= PMC_EV_SOFT_LAST; ev++) {
+		for (ev = PMC_EV_SOFT_FIRST; (int)ev <= PMC_EV_SOFT_LAST; ev++) {
 			ps = pmc_soft_ev_acquire(ev);
 			if (ps == NULL)
 				continue;
diff --git a/sys/dev/hwpmc/hwpmc_soft.c b/sys/dev/hwpmc/hwpmc_soft.c
index c3d2dec..dac3612 100644
--- a/sys/dev/hwpmc/hwpmc_soft.c
+++ b/sys/dev/hwpmc/hwpmc_soft.c
@@ -116,7 +116,7 @@ soft_allocate_pmc(int cpu, int ri, struct pmc *pm,
 		return (EPERM);
 
 	ev = pm->pm_event;
-	if (ev < PMC_EV_SOFT_FIRST || ev > PMC_EV_SOFT_LAST)
+	if ((int)ev < PMC_EV_SOFT_FIRST || (int)ev > PMC_EV_SOFT_LAST)
 		return (EINVAL);
 
 	/* Check if event is registered. */
diff --git a/sys/dev/hwpmc/pmc_events.h b/sys/dev/hwpmc/pmc_events.h
index cb22324..8f246f6 100644
--- a/sys/dev/hwpmc/pmc_events.h
+++ b/sys/dev/hwpmc/pmc_events.h
@@ -722,6 +722,7 @@ __PMC_EV(IAP, EVENT_5CH_01H)			\
 __PMC_EV(IAP, EVENT_5CH_02H)			\
 __PMC_EV(IAP, EVENT_5EH_01H)			\
 __PMC_EV(IAP, EVENT_5FH_01H)			\
+__PMC_EV(IAP, EVENT_5FH_04H)			\
 __PMC_EV(IAP, EVENT_60H)			\
 __PMC_EV(IAP, EVENT_60H_01H)			\
 __PMC_EV(IAP, EVENT_60H_02H)			\
@@ -861,6 +862,7 @@ __PMC_EV(IAP, EVENT_A2H_80H)			\
 __PMC_EV(IAP, EVENT_A3H_01H)                  	\
 __PMC_EV(IAP, EVENT_A3H_02H)                  	\
 __PMC_EV(IAP, EVENT_A3H_04H)                  	\
+__PMC_EV(IAP, EVENT_A3H_08H)                  	\
 __PMC_EV(IAP, EVENT_A6H_01H)			\
 __PMC_EV(IAP, EVENT_A7H_01H)			\
 __PMC_EV(IAP, EVENT_A8H_01H)			\
@@ -1021,6 +1023,8 @@ __PMC_EV(IAP, EVENT_D2H_0FH)			\
 __PMC_EV(IAP, EVENT_D2H_10H)			\
 __PMC_EV(IAP, EVENT_D3H_01H)			\
 __PMC_EV(IAP, EVENT_D3H_04H)			\
+__PMC_EV(IAP, EVENT_D3H_10H)			\
+__PMC_EV(IAP, EVENT_D3H_20H)			\
 __PMC_EV(IAP, EVENT_D4H_01H)			\
 __PMC_EV(IAP, EVENT_D4H_02H)			\
 __PMC_EV(IAP, EVENT_D4H_04H)			\
@@ -1061,6 +1065,7 @@ __PMC_EV(IAP, EVENT_E5H_01H)			\
 __PMC_EV(IAP, EVENT_E6H_00H)			\
 __PMC_EV(IAP, EVENT_E6H_01H)			\
 __PMC_EV(IAP, EVENT_E6H_02H)			\
+__PMC_EV(IAP, EVENT_E6H_1FH)			\
 __PMC_EV(IAP, EVENT_E8H_01H)			\
 __PMC_EV(IAP, EVENT_E8H_02H)			\
 __PMC_EV(IAP, EVENT_E8H_03H)			\
@@ -2584,6 +2589,205 @@ __PMC_EV_ALIAS("L2_LINES_OUT.PF_CLEAN", IAP_EVENT_F2H_04H)	\
 __PMC_EV_ALIAS("L2_LINES_OUT.PF_DIRTY", IAP_EVENT_F2H_08H)
 
 /*
+ * Aliases for Ivy Bridge Xeon PMC events (325462-045US January 2013)
+ */
+
+#define __PMC_EV_ALIAS_IVYBRIDGE_XEON()                                     		\
+__PMC_EV_ALIAS("LD_BLOCKS.STORE_FORWARD", IAP_EVENT_03H_02H)				\
+__PMC_EV_ALIAS("MISALIGN_MEM_REF.LOADS", IAP_EVENT_05H_01H)				\
+__PMC_EV_ALIAS("MISALIGN_MEM_REF.STORES", IAP_EVENT_05H_02H)				\
+__PMC_EV_ALIAS("LD_BLOCKS_PARTIAL.ADDRESS_ALIAS", IAP_EVENT_07H_01H)			\
+__PMC_EV_ALIAS("DTLB_LOAD_MISSES.DEMAND_LD_MISS_CAUSES_A_WALK", IAP_EVENT_08H_81H)	\
+__PMC_EV_ALIAS("DTLB_LOAD_MISSES.DEMAND_LD_WALK_COMPLETED", IAP_EVENT_08H_82H)		\
+__PMC_EV_ALIAS("DTLB_LOAD_MISSES.DEMAND_LD_WALK_DURATION", IAP_EVENT_08H_84H)		\
+__PMC_EV_ALIAS("UOPS_ISSUED.ANY", IAP_EVENT_0EH_01H)					\
+__PMC_EV_ALIAS("UOPS_ISSUED.FLAGS_MERGE", IAP_EVENT_0EH_10H)				\
+__PMC_EV_ALIAS("UOPS_ISSUED.SLOW_LEA", IAP_EVENT_0EH_20H)				\
+__PMC_EV_ALIAS("UOPS_ISSUED.SINGLE_MUL", IAP_EVENT_0EH_40H)				\
+__PMC_EV_ALIAS("ARITH.FPU_DIV_ACTIVE", IAP_EVENT_14H_01H)				\
+__PMC_EV_ALIAS("L2_RQSTS.DEMAND_DATA_RD_HIT", IAP_EVENT_24H_01H)			\
+__PMC_EV_ALIAS("L2_RQSTS.ALL_DEMAND_DATA_RD", IAP_EVENT_24H_03H)			\
+__PMC_EV_ALIAS("L2_RQSTS.RFO_HITS", IAP_EVENT_24H_04H)					\
+__PMC_EV_ALIAS("L2_RQSTS.RFO_MISS", IAP_EVENT_24H_08H)					\
+__PMC_EV_ALIAS("L2_RQSTS.ALL_RFO", IAP_EVENT_24H_0CH)					\
+__PMC_EV_ALIAS("L2_RQSTS.CODE_RD_HIT", IAP_EVENT_24H_10H)				\
+__PMC_EV_ALIAS("L2_RQSTS.CODE_RD_MISS", IAP_EVENT_24H_20H)				\
+__PMC_EV_ALIAS("L2_RQSTS.ALL_CODE_RD", IAP_EVENT_24H_30H)				\
+__PMC_EV_ALIAS("L2_RQSTS.PF_HIT", IAP_EVENT_24H_40H)					\
+__PMC_EV_ALIAS("L2_RQSTS.PF_MISS", IAP_EVENT_24H_80H)					\
+__PMC_EV_ALIAS("L2_RQSTS.ALL_PF", IAP_EVENT_24H_C0H)					\
+__PMC_EV_ALIAS("L2_STORE_LOCK_RQSTS.MISS", IAP_EVENT_27H_01H)				\
+__PMC_EV_ALIAS("L2_STORE_LOCK_RQSTS.HIT_M", IAP_EVENT_27H_08H)				\
+__PMC_EV_ALIAS("L2_STORE_LOCK_RQSTS.ALL", IAP_EVENT_27H_0FH)				\
+__PMC_EV_ALIAS("L2_L1D_WB_RQSTS.MISS", IAP_EVENT_28H_01H)				\
+__PMC_EV_ALIAS("L2_L1D_WB_RQSTS.HIT_E", IAP_EVENT_28H_04H)				\
+__PMC_EV_ALIAS("L2_L1D_WB_RQSTS.HIT_M", IAP_EVENT_28H_08H)				\
+__PMC_EV_ALIAS("L2_L1D_WB_RQSTS.ALL", IAP_EVENT_28H_0FH)				\
+__PMC_EV_ALIAS("LONGEST_LAT_CACHE.REFERENCE", IAP_EVENT_2EH_4FH)			\
+__PMC_EV_ALIAS("LONGEST_LAT_CACHE.MISS", IAP_EVENT_2EH_41H)				\
+__PMC_EV_ALIAS("CPU_CLK_UNHALTED.THREAD_P", IAP_EVENT_3CH_00H)				\
+__PMC_EV_ALIAS("CPU_CLK_THREAD_UNHALTED.REF_XCLK", IAP_EVENT_3CH_01H)			\
+__PMC_EV_ALIAS("L1D_PEND_MISS.PENDING", IAP_EVENT_48H_01H)				\
+__PMC_EV_ALIAS("DTLB_STORE_MISSES.MISS_CAUSES_A_WALK", IAP_EVENT_49H_01H)		\
+__PMC_EV_ALIAS("DTLB_STORE_MISSES.WALK_COMPLETED", IAP_EVENT_49H_02H)			\
+__PMC_EV_ALIAS("DTLB_STORE_MISSES.WALK_DURATION", IAP_EVENT_49H_04H)			\
+__PMC_EV_ALIAS("DTLB_STORE_MISSES.STLB_HIT", IAP_EVENT_49H_10H)				\
+__PMC_EV_ALIAS("LOAD_HIT_PRE.SW_PF", IAP_EVENT_4CH_01H)					\
+__PMC_EV_ALIAS("LOAD_HIT_PRE.HW_PF", IAP_EVENT_4CH_02H)					\
+__PMC_EV_ALIAS("L1D.REPLACEMENT", IAP_EVENT_51H_01H)					\
+__PMC_EV_ALIAS("MOVE_ELIMINATION.INT_NOT_ELIMINATED", IAP_EVENT_58H_01H)		\
+__PMC_EV_ALIAS("MOVE_ELIMINATION.SIMD_NOT_ELIMINATED", IAP_EVENT_58H_02H)		\
+__PMC_EV_ALIAS("MOVE_ELIMINATION.INT_ELIMINATED", IAP_EVENT_58H_04H)			\
+__PMC_EV_ALIAS("MOVE_ELIMINATION.SIMD_ELIMINATED", IAP_EVENT_58H_08H)			\
+__PMC_EV_ALIAS("CPL_CYCLES.RING0", IAP_EVENT_5CH_01H)					\
+__PMC_EV_ALIAS("CPL_CYCLES.RING123", IAP_EVENT_5CH_02H)					\
+__PMC_EV_ALIAS("RS_EVENTS.EMPTY_CYCLES", IAP_EVENT_5EH_01H)				\
+__PMC_EV_ALIAS("DTLB_LOAD_MISSES.STLB_HIT", IAP_EVENT_5FH_04H)				\
+__PMC_EV_ALIAS("OFFCORE_REQUESTS_OUTSTANDING.DEMAND_DATA_RD", IAP_EVENT_60H_01H)	\
+__PMC_EV_ALIAS("OFFCORE_REQUESTS_OUTSTANDING.DEMAND_CODE_RD", IAP_EVENT_60H_02H)	\
+__PMC_EV_ALIAS("OFFCORE_REQUESTS_OUTSTANDING.DEMAND_RFO", IAP_EVENT_60H_04H)		\
+__PMC_EV_ALIAS("OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD", IAP_EVENT_60H_08H)		\
+__PMC_EV_ALIAS("LOCK_CYCLES.SPLIT_LOCK_UC_LOCK_DURATION", IAP_EVENT_63H_01H)		\
+__PMC_EV_ALIAS("LOCK_CYCLES.CACHE_LOCK_DURATION", IAP_EVENT_63H_02H)			\
+__PMC_EV_ALIAS("IDQ.EMPTY", IAP_EVENT_79H_02H)						\
+__PMC_EV_ALIAS("IDQ.MITE_UOPS", IAP_EVENT_79H_04H)					\
+__PMC_EV_ALIAS("IDQ.DSB_UOPS", IAP_EVENT_79H_08H)					\
+__PMC_EV_ALIAS("IDQ.MS_DSB_UOPS", IAP_EVENT_79H_10H)					\
+__PMC_EV_ALIAS("IDQ.MS_MITE_UOPS", IAP_EVENT_79H_20H)					\
+__PMC_EV_ALIAS("IDQ.MS_UOPS", IAP_EVENT_79H_30H)					\
+__PMC_EV_ALIAS("IDQ.ALL_DSB_CYCLES_ANY_UOPS", IAP_EVENT_79H_18H)			\
+__PMC_EV_ALIAS("IDQ.ALL_DSB_CYCLES_4_UOPS", IAP_EVENT_79H_18H)				\
+__PMC_EV_ALIAS("IDQ.ALL_MITE_CYCLES_ANY_UOPS", IAP_EVENT_79H_24H)			\
+__PMC_EV_ALIAS("IDQ.ALL_MITE_CYCLES_4_UOPS", IAP_EVENT_79H_24H)				\
+__PMC_EV_ALIAS("IDQ.MITE_ALL_UOPS", IAP_EVENT_79H_3CH)					\
+__PMC_EV_ALIAS("ICACHE.MISSES", IAP_EVENT_80H_02H)					\
+__PMC_EV_ALIAS("ITLB_MISSES.MISS_CAUSES_A_WALK", IAP_EVENT_85H_01H)			\
+__PMC_EV_ALIAS("ITLB_MISSES.WALK_COMPLETED", IAP_EVENT_85H_02H)				\
+__PMC_EV_ALIAS("ITLB_MISSES.WALK_DURATION", IAP_EVENT_85H_04H)				\
+__PMC_EV_ALIAS("ITLB_MISSES.STLB_HIT", IAP_EVENT_85H_10H)				\
+__PMC_EV_ALIAS("ILD_STALL.LCP", IAP_EVENT_87H_01H)					\
+__PMC_EV_ALIAS("ILD_STALL.IQ_FULL", IAP_EVENT_87H_04H)					\
+__PMC_EV_ALIAS("BR_INST_EXEC.COND", IAP_EVENT_88H_01H)					\
+__PMC_EV_ALIAS("BR_INST_EXEC.DIRECT_JMP", IAP_EVENT_88H_02H)				\
+__PMC_EV_ALIAS("BR_INST_EXEC.INDIRECT_JMP_NON_CALL_RET", IAP_EVENT_88H_04H)		\
+__PMC_EV_ALIAS("BR_INST_EXEC.RETURN_NEAR", IAP_EVENT_88H_08H)				\
+__PMC_EV_ALIAS("BR_INST_EXEC.DIRECT_NEAR_CALL", IAP_EVENT_88H_10H)			\
+__PMC_EV_ALIAS("BR_INST_EXEC.INDIRECT_NEAR_CALL", IAP_EVENT_88H_20H)			\
+__PMC_EV_ALIAS("BR_INST_EXEC.NONTAKEN", IAP_EVENT_88H_40H)				\
+__PMC_EV_ALIAS("BR_INST_EXEC.TAKEN", IAP_EVENT_88H_80H)					\
+__PMC_EV_ALIAS("BR_INST_EXEC.ALL_BRANCHES", IAP_EVENT_88H_FFH)				\
+__PMC_EV_ALIAS("BR_MISP_EXEC.COND", IAP_EVENT_89H_01H)					\
+__PMC_EV_ALIAS("BR_MISP_EXEC.INDIRECT_JMP_NON_CALL_RET", IAP_EVENT_89H_04H)		\
+__PMC_EV_ALIAS("BR_MISP_EXEC.RETURN_NEAR", IAP_EVENT_89H_08H)				\
+__PMC_EV_ALIAS("BR_MISP_EXEC.DIRECT_NEAR_CALL", IAP_EVENT_89H_10H)			\
+__PMC_EV_ALIAS("BR_MISP_EXEC.INDIRECT_NEAR_CALL", IAP_EVENT_89H_20H)			\
+__PMC_EV_ALIAS("BR_MISP_EXEC.NONTAKEN", IAP_EVENT_89H_40H)				\
+__PMC_EV_ALIAS("BR_MISP_EXEC.TAKEN", IAP_EVENT_89H_80H)					\
+__PMC_EV_ALIAS("BR_MISP_EXEC.ALL_BRANCHES", IAP_EVENT_89H_FFH)				\
+__PMC_EV_ALIAS("IDQ_UOPS_NOT_DELIVERED.CORE", IAP_EVENT_9CH_01H)			\
+__PMC_EV_ALIAS("UOPS_DISPATCHED_PORT.PORT_0", IAP_EVENT_A1H_01H)			\
+__PMC_EV_ALIAS("UOPS_DISPATCHED_PORT.PORT_1", IAP_EVENT_A1H_02H)			\
+__PMC_EV_ALIAS("UOPS_DISPATCHED_PORT.PORT_2_LD", IAP_EVENT_A1H_04H)			\
+__PMC_EV_ALIAS("UOPS_DISPATCHED_PORT.PORT_2_STA", IAP_EVENT_A1H_08H)			\
+__PMC_EV_ALIAS("UOPS_DISPATCHED_PORT.PORT_2", IAP_EVENT_A1H_0CH)			\
+__PMC_EV_ALIAS("UOPS_DISPATCHED_PORT.PORT_3_LD", IAP_EVENT_A1H_10H)			\
+__PMC_EV_ALIAS("UOPS_DISPATCHED_PORT.PORT_3_STA", IAP_EVENT_A1H_20H)			\
+__PMC_EV_ALIAS("UOPS_DISPATCHED_PORT.PORT_3", IAP_EVENT_A1H_30H)			\
+__PMC_EV_ALIAS("UOPS_DISPATCHED_PORT.PORT_4", IAP_EVENT_A1H_40H)			\
+__PMC_EV_ALIAS("UOPS_DISPATCHED_PORT.PORT_5", IAP_EVENT_A1H_80H)			\
+__PMC_EV_ALIAS("RESOURCE_STALLS.ANY", IAP_EVENT_A2H_01H)				\
+__PMC_EV_ALIAS("RESOURCE_STALLS.RS", IAP_EVENT_A2H_04H)					\
+__PMC_EV_ALIAS("RESOURCE_STALLS.SB", IAP_EVENT_A2H_08H)					\
+__PMC_EV_ALIAS("RESOURCE_STALLS.ROB", IAP_EVENT_A2H_10H)				\
+__PMC_EV_ALIAS("CYCLE_ACTIVITY.CYCLES_L2_PENDING", IAP_EVENT_A3H_01H)			\
+__PMC_EV_ALIAS("CYCLE_ACTIVITY.CYCLES_LDM_PENDING", IAP_EVENT_A3H_02H)			\
+__PMC_EV_ALIAS("CYCLE_ACTIVITY.CYCLES_NO_EXECUTE", IAP_EVENT_A3H_04H)			\
+__PMC_EV_ALIAS("CYCLE_ACTIVITY.CYCLES_L1D_PENDING", IAP_EVENT_A3H_08H)			\
+__PMC_EV_ALIAS("DSB2MITE_SWITCHES.COUNT", IAP_EVENT_ABH_01H)				\
+__PMC_EV_ALIAS("DSB2MITE_SWITCHES.PENALTY_CYCLES", IAP_EVENT_ABH_02H)			\
+__PMC_EV_ALIAS("DSB_FILL.EXCEED_DSB_LINES", IAP_EVENT_ACH_08H)				\
+__PMC_EV_ALIAS("ITLB.ITLB_FLUSH", IAP_EVENT_AEH_01H)					\
+__PMC_EV_ALIAS("OFFCORE_REQUESTS.DEMAND_DATA_RD", IAP_EVENT_B0H_01H)			\
+__PMC_EV_ALIAS("OFFCORE_REQUESTS.DEMAND_CODE_RD", IAP_EVENT_B0H_02H)			\
+__PMC_EV_ALIAS("OFFCORE_REQUESTS.DEMAND_RFO", IAP_EVENT_B0H_04H)			\
+__PMC_EV_ALIAS("OFFCORE_REQUESTS.ALL_DATA_RD", IAP_EVENT_B0H_08H)			\
+__PMC_EV_ALIAS("UOPS_EXECUTED.THREAD", IAP_EVENT_B1H_01H)				\
+__PMC_EV_ALIAS("UOPS_EXECUTED.CORE", IAP_EVENT_B1H_02H)					\
+__PMC_EV_ALIAS("OFF_CORE_RESPONSE_0", IAP_EVENT_B7H_01H)				\
+__PMC_EV_ALIAS("OFF_CORE_RESPONSE_1", IAP_EVENT_BBH_01H)				\
+__PMC_EV_ALIAS("TLB_FLUSH.DTLB_THREAD", IAP_EVENT_BDH_01H)				\
+__PMC_EV_ALIAS("TLB_FLUSH.STLB_ANY", IAP_EVENT_BDH_20H)					\
+__PMC_EV_ALIAS("INST_RETIRED.ANY_P", IAP_EVENT_C0H_00H)					\
+__PMC_EV_ALIAS("INST_RETIRED.ALL", IAP_EVENT_C0H_01H)					\
+__PMC_EV_ALIAS("OTHER_ASSISTS.AVX_STORE", IAP_EVENT_C1H_08H)				\
+__PMC_EV_ALIAS("OTHER_ASSISTS.AVX_TO_SSE", IAP_EVENT_C1H_10H)				\
+__PMC_EV_ALIAS("OTHER_ASSISTS.SSE_TO_AVX", IAP_EVENT_C1H_20H)				\
+__PMC_EV_ALIAS("UOPS_RETIRED.ALL", IAP_EVENT_C2H_01H)					\
+__PMC_EV_ALIAS("UOPS_RETIRED.RETIRE_SLOTS", IAP_EVENT_C2H_02H)				\
+__PMC_EV_ALIAS("MACHINE_CLEARS.MEMORY_ORDERING", IAP_EVENT_C3H_02H)			\
+__PMC_EV_ALIAS("MACHINE_CLEARS.SMC", IAP_EVENT_C3H_04H)					\
+__PMC_EV_ALIAS("MACHINE_CLEARS.MASKMOV", IAP_EVENT_C3H_20H)				\
+__PMC_EV_ALIAS("BR_INST_RETIRED.ALL_BRANCHES", IAP_EVENT_C4H_00H)			\
+__PMC_EV_ALIAS("BR_INST_RETIRED.CONDITIONAL", IAP_EVENT_C4H_01H)			\
+__PMC_EV_ALIAS("BR_INST_RETIRED.NEAR_CALL", IAP_EVENT_C4H_02H)				\
+__PMC_EV_ALIAS("BR_INST_RETIRED.ALL_BRANCHES", IAP_EVENT_C4H_04H)			\
+__PMC_EV_ALIAS("BR_INST_RETIRED.NEAR_RETURN", IAP_EVENT_C4H_08H)			\
+__PMC_EV_ALIAS("BR_INST_RETIRED.NOT_TAKEN", IAP_EVENT_C4H_10H)				\
+__PMC_EV_ALIAS("BR_INST_RETIRED.NEAR_TAKEN", IAP_EVENT_C4H_20H)				\
+__PMC_EV_ALIAS("BR_INST_RETIRED.FAR_BRANCH", IAP_EVENT_C4H_40H)				\
+__PMC_EV_ALIAS("BR_MISP_RETIRED.ALL_BRANCHES", IAP_EVENT_C5H_00H)			\
+__PMC_EV_ALIAS("BR_MISP_RETIRED.CONDITIONAL", IAP_EVENT_C5H_01H)			\
+__PMC_EV_ALIAS("BR_MISP_RETIRED.NEAR_CALL", IAP_EVENT_C5H_02H)				\
+__PMC_EV_ALIAS("BR_MISP_RETIRED.ALL_BRANCHES", IAP_EVENT_C5H_04H)			\
+__PMC_EV_ALIAS("BR_MISP_RETIRED.NOT_TAKEN", IAP_EVENT_C5H_10H)				\
+__PMC_EV_ALIAS("BR_MISP_RETIRED.TAKEN", IAP_EVENT_C5H_20H)				\
+__PMC_EV_ALIAS("FP_ASSIST.X87_OUTPUT", IAP_EVENT_CAH_02H)				\
+__PMC_EV_ALIAS("FP_ASSIST.X87_INPUT", IAP_EVENT_CAH_04H)				\
+__PMC_EV_ALIAS("FP_ASSIST.SIMD_OUTPUT", IAP_EVENT_CAH_08H)				\
+__PMC_EV_ALIAS("FP_ASSIST.SIMD_INPUT", IAP_EVENT_CAH_10H)				\
+__PMC_EV_ALIAS("FP_ASSIST.ANY", IAP_EVENT_CAH_1EH)					\
+__PMC_EV_ALIAS("ROB_MISC_EVENTS.LBR_INSERTS", IAP_EVENT_CCH_20H)			\
+__PMC_EV_ALIAS("MEM_TRANS_RETIRED.LOAD_LATENCY", IAP_EVENT_CDH_01H)			\
+__PMC_EV_ALIAS("MEM_TRANS_RETIRED.PRECISE_STORE", IAP_EVENT_CDH_02H)			\
+__PMC_EV_ALIAS("MEM_UOP_RETIRED.LOADS", IAP_EVENT_D0H_01H)				\
+__PMC_EV_ALIAS("MEM_UOP_RETIRED.STORES", IAP_EVENT_D0H_02H)				\
+__PMC_EV_ALIAS("MEM_UOP_RETIRED.STLB_MISS", IAP_EVENT_D0H_10H)				\
+__PMC_EV_ALIAS("MEM_UOP_RETIRED.LOCK", IAP_EVENT_D0H_20H)				\
+__PMC_EV_ALIAS("MEM_UOP_RETIRED.SPLIT", IAP_EVENT_D0H_40H)				\
+__PMC_EV_ALIAS("MEM_UOP_RETIRED.ALL", IAP_EVENT_D0H_80H)				\
+__PMC_EV_ALIAS("MEM_LOAD_UOPS_RETIRED.L1_HIT", IAP_EVENT_D1H_01H)			\
+__PMC_EV_ALIAS("MEM_LOAD_UOPS_RETIRED.L2_HIT", IAP_EVENT_D1H_02H)			\
+__PMC_EV_ALIAS("MEM_LOAD_UOPS_RETIRED.LLC_HIT", IAP_EVENT_D1H_04H)			\
+__PMC_EV_ALIAS("MEM_LOAD_UOPS_RETIRED.LLC_MISS", IAP_EVENT_D1H_20H)			\
+__PMC_EV_ALIAS("MEM_LOAD_UOPS_RETIRED.HIT_LFB", IAP_EVENT_D1H_40H)			\
+__PMC_EV_ALIAS("MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS", IAP_EVENT_D2H_01H)		\
+__PMC_EV_ALIAS("MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT", IAP_EVENT_D2H_02H)		\
+__PMC_EV_ALIAS("MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM", IAP_EVENT_D2H_04H)		\
+__PMC_EV_ALIAS("MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_NONE", IAP_EVENT_D2H_08H)		\
+__PMC_EV_ALIAS("MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM", IAP_EVENT_D3H_01H)		\
+__PMC_EV_ALIAS("MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM", IAP_EVENT_D3H_04H)		\
+__PMC_EV_ALIAS("MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM", IAP_EVENT_D3H_10H)		\
+__PMC_EV_ALIAS("MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD", IAP_EVENT_D3H_20H)		\
+__PMC_EV_ALIAS("BACLEARS.ANY", IAP_EVENT_E6H_1FH)					\
+__PMC_EV_ALIAS("L2_TRANS.DEMAND_DATA_RD", IAP_EVENT_F0H_01H)				\
+__PMC_EV_ALIAS("L2_TRANS.RFO", IAP_EVENT_F0H_02H)					\
+__PMC_EV_ALIAS("L2_TRANS.CODE_RD", IAP_EVENT_F0H_04H)					\
+__PMC_EV_ALIAS("L2_TRANS.ALL_PF", IAP_EVENT_F0H_08H)					\
+__PMC_EV_ALIAS("L2_TRANS.L1D_WB", IAP_EVENT_F0H_10H)					\
+__PMC_EV_ALIAS("L2_TRANS.L2_FILL", IAP_EVENT_F0H_20H)					\
+__PMC_EV_ALIAS("L2_TRANS.L2_WB", IAP_EVENT_F0H_40H)					\
+__PMC_EV_ALIAS("L2_TRANS.ALL_REQUESTS", IAP_EVENT_F0H_80H)				\
+__PMC_EV_ALIAS("L2_LINES_IN.I", IAP_EVENT_F1H_01H)					\
+__PMC_EV_ALIAS("L2_LINES_IN.S", IAP_EVENT_F1H_02H)					\
+__PMC_EV_ALIAS("L2_LINES_IN.E", IAP_EVENT_F1H_04H)					\
+__PMC_EV_ALIAS("L2_LINES_IN.ALL", IAP_EVENT_F1H_07H)					\
+__PMC_EV_ALIAS("L2_LINES_OUT.DEMAND_CLEAN", IAP_EVENT_F2H_01H)				\
+__PMC_EV_ALIAS("L2_LINES_OUT.DEMAND_DIRTY", IAP_EVENT_F2H_02H)				\
+__PMC_EV_ALIAS("L2_LINES_OUT.PF_CLEAN", IAP_EVENT_F2H_04H)				\
+__PMC_EV_ALIAS("L2_LINES_OUT.PF_DIRTY", IAP_EVENT_F2H_08H)				\
+__PMC_EV_ALIAS("L2_LINES_OUT.DIRTY_ALL", IAP_EVENT_F2H_0AH)
+
+/*
  *  Aliases for Sandy Bridge PMC events (253669-039US May 2011) 
  */
 
diff --git a/sys/dev/iicbus/ad7418.c b/sys/dev/iicbus/ad7418.c
index 3875307..dcb8b95 100644
--- a/sys/dev/iicbus/ad7418.c
+++ b/sys/dev/iicbus/ad7418.c
@@ -219,7 +219,7 @@ static device_method_t ad7418_methods[] = {
 	DEVMETHOD(device_probe,		ad7418_probe),
 	DEVMETHOD(device_attach,	ad7418_attach),
 
-	{0, 0},
+	DEVMETHOD_END
 };
 
 static driver_t ad7418_driver = {
diff --git a/sys/dev/iicbus/ds133x.c b/sys/dev/iicbus/ds133x.c
index 572384f..20f1d40 100644
--- a/sys/dev/iicbus/ds133x.c
+++ b/sys/dev/iicbus/ds133x.c
@@ -347,7 +347,7 @@ static device_method_t ds133x_methods[] = {
 	DEVMETHOD(clock_gettime,	ds133x_gettime),
 	DEVMETHOD(clock_settime,	ds133x_settime),
 
-	{0, 0},
+	DEVMETHOD_END
 };
 
 static driver_t ds133x_driver = {
diff --git a/sys/dev/iicbus/ds1672.c b/sys/dev/iicbus/ds1672.c
index a9209bd..1dd0ab0 100644
--- a/sys/dev/iicbus/ds1672.c
+++ b/sys/dev/iicbus/ds1672.c
@@ -167,7 +167,7 @@ static device_method_t ds1672_methods[] = {
 	DEVMETHOD(clock_gettime,	ds1672_gettime),
 	DEVMETHOD(clock_settime,	ds1672_settime),
 
-	{0, 0},
+	DEVMETHOD_END
 };
 
 static driver_t ds1672_driver = {
diff --git a/sys/dev/iicbus/icee.c b/sys/dev/iicbus/icee.c
index 93c03ad..2e26d0e 100644
--- a/sys/dev/iicbus/icee.c
+++ b/sys/dev/iicbus/icee.c
@@ -262,7 +262,7 @@ static device_method_t icee_methods[] = {
 	DEVMETHOD(device_probe,		icee_probe),
 	DEVMETHOD(device_attach,	icee_attach),
 
-	{0, 0},
+	DEVMETHOD_END
 };
 
 static driver_t icee_driver = {
diff --git a/sys/dev/isf/isf.c b/sys/dev/isf/isf.c
index e537799..eaaad15 100644
--- a/sys/dev/isf/isf.c
+++ b/sys/dev/isf/isf.c
@@ -150,6 +150,7 @@ static void	isf_task(void *arg);
  * physical package, due to variable block size support in the StrataFlash
  * part.
  */
+devclass_t	isf_devclass;
 
 static uint16_t
 isf_read_reg(struct isf_softc *sc, uint16_t reg)
diff --git a/sys/dev/isf/isf.h b/sys/dev/isf/isf.h
index f5cdc08..bfcca07 100644
--- a/sys/dev/isf/isf.h
+++ b/sys/dev/isf/isf.h
@@ -89,6 +89,8 @@ struct isf_softc {
 
 int	isf_attach(struct isf_softc *sc);
 void	isf_detach(struct isf_softc *sc);
+
+extern devclass_t	isf_devclass;
 #endif /* _KERNEL */
 
 #endif	/* _DEV_ISF_H_ */
diff --git a/sys/dev/isf/isf_fdt.c b/sys/dev/isf/isf_fdt.c
new file mode 100644
index 0000000..f4c20a2
--- /dev/null
+++ b/sys/dev/isf/isf_fdt.c
@@ -0,0 +1,124 @@
+/*-
+ * Copyright (c) 2012 Robert N. M. Watson
+ * Copyright (c) 2012 SRI International
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
+ * ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/condvar.h>
+#include <sys/conf.h>
+#include <sys/bio.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/rman.h>
+#include <sys/systm.h>
+#include <sys/taskqueue.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+
+#include <geom/geom_disk.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <dev/isf/isf.h>
+
+/*
+ * FDT bus attachment for the Intel Strata Flash devices.
+ */
+static int
+isf_fdt_probe(device_t dev)
+{
+
+	if (ofw_bus_is_compatible(dev, "intel,strataflash")) {
+		device_set_desc(dev, "Intel StrataFlash NOR flash device");
+		return (BUS_PROBE_DEFAULT);
+	}
+	return (ENXIO);
+}
+
+static int
+isf_fdt_attach(device_t dev)
+{
+	int			 error;
+	struct isf_softc	*sc;
+
+	sc = device_get_softc(dev);
+	sc->isf_dev = dev;
+	sc->isf_unit = device_get_unit(dev);
+	sc->isf_rid = 0;
+	sc->isf_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+	    &sc->isf_rid, RF_ACTIVE);
+	if (sc->isf_res == NULL) {
+		device_printf(dev, "couldn't map memory\n");
+		return (ENXIO);
+	}
+	error = isf_attach(sc);
+	if (error)
+		bus_release_resource(dev, SYS_RES_MEMORY, sc->isf_rid,
+		    sc->isf_res);
+	return (error);
+}
+
+static int
+isf_fdt_detach(device_t dev)
+{
+	struct isf_softc *sc;
+
+	sc = device_get_softc(dev);
+	KASSERT(sc->isf_res != NULL, ("%s: resources not allocated",
+	    __func__));
+	isf_detach(sc);
+	bus_release_resource(dev, SYS_RES_MEMORY, sc->isf_rid, sc->isf_res);
+	return (0);
+}
+
+static device_method_t isf_fdt_methods[] = {
+	DEVMETHOD(device_probe,		isf_fdt_probe),
+	DEVMETHOD(device_attach,	isf_fdt_attach),
+	DEVMETHOD(device_detach,	isf_fdt_detach),
+	{ 0, 0 }
+};
+
+static driver_t isf_fdt_driver = {
+	"isf",
+	isf_fdt_methods,
+	sizeof(struct isf_softc),
+};
+
+DRIVER_MODULE(isf, simplebus, isf_fdt_driver, isf_devclass, 0, 0);
diff --git a/sys/dev/isf/isf_nexus.c b/sys/dev/isf/isf_nexus.c
index af2f874..7631453 100644
--- a/sys/dev/isf/isf_nexus.c
+++ b/sys/dev/isf/isf_nexus.c
@@ -115,6 +115,4 @@ static driver_t isf_nexus_driver = {
 	sizeof(struct isf_softc),
 };
 
-static devclass_t isf_devclass;
-
 DRIVER_MODULE(isf, nexus, isf_nexus_driver, isf_devclass, 0, 0);
diff --git a/sys/dev/ixgb/if_ixgb.c b/sys/dev/ixgb/if_ixgb.c
index e9acae0..69a8fb9 100644
--- a/sys/dev/ixgb/if_ixgb.c
+++ b/sys/dev/ixgb/if_ixgb.c
@@ -159,7 +159,8 @@ static device_method_t ixgb_methods[] = {
 	DEVMETHOD(device_attach, ixgb_attach),
 	DEVMETHOD(device_detach, ixgb_detach),
 	DEVMETHOD(device_shutdown, ixgb_shutdown),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t ixgb_driver = {
diff --git a/sys/dev/ixgbe/ixgbe.c b/sys/dev/ixgbe/ixgbe.c
index 1d97343..cfcc342 100644
--- a/sys/dev/ixgbe/ixgbe.c
+++ b/sys/dev/ixgbe/ixgbe.c
@@ -216,7 +216,8 @@ static device_method_t ixgbe_methods[] = {
 	DEVMETHOD(device_attach, ixgbe_attach),
 	DEVMETHOD(device_detach, ixgbe_detach),
 	DEVMETHOD(device_shutdown, ixgbe_shutdown),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t ixgbe_driver = {
diff --git a/sys/dev/ixgbe/ixgbe_82598.h b/sys/dev/ixgbe/ixgbe_82598.h
index a195b15..a195b15 100755..100644
--- a/sys/dev/ixgbe/ixgbe_82598.h
+++ b/sys/dev/ixgbe/ixgbe_82598.h
diff --git a/sys/dev/ixgbe/ixgbe_82599.h b/sys/dev/ixgbe/ixgbe_82599.h
index dca39b7..dca39b7 100755..100644
--- a/sys/dev/ixgbe/ixgbe_82599.h
+++ b/sys/dev/ixgbe/ixgbe_82599.h
diff --git a/sys/dev/ixgbe/ixgbe_x540.c b/sys/dev/ixgbe/ixgbe_x540.c
index 0635cda..0635cda 100755..100644
--- a/sys/dev/ixgbe/ixgbe_x540.c
+++ b/sys/dev/ixgbe/ixgbe_x540.c
diff --git a/sys/dev/ixgbe/ixgbe_x540.h b/sys/dev/ixgbe/ixgbe_x540.h
index 29cf8bb..29cf8bb 100755..100644
--- a/sys/dev/ixgbe/ixgbe_x540.h
+++ b/sys/dev/ixgbe/ixgbe_x540.h
diff --git a/sys/dev/ixgbe/ixv.c b/sys/dev/ixgbe/ixv.c
index b9aff80..ef5f753 100644
--- a/sys/dev/ixgbe/ixv.c
+++ b/sys/dev/ixgbe/ixv.c
@@ -169,7 +169,8 @@ static device_method_t ixv_methods[] = {
 	DEVMETHOD(device_attach, ixv_attach),
 	DEVMETHOD(device_detach, ixv_detach),
 	DEVMETHOD(device_shutdown, ixv_shutdown),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t ixv_driver = {
diff --git a/sys/dev/mfi/mfi_cam.c b/sys/dev/mfi/mfi_cam.c
index 599ff3b..2bbfafe 100644
--- a/sys/dev/mfi/mfi_cam.c
+++ b/sys/dev/mfi/mfi_cam.c
@@ -99,7 +99,8 @@ static device_method_t	mfip_methods[] = {
 	DEVMETHOD(device_probe,		mfip_probe),
 	DEVMETHOD(device_attach,	mfip_attach),
 	DEVMETHOD(device_detach,	mfip_detach),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 static driver_t mfip_driver = {
 	"mfip",
diff --git a/sys/dev/mmc/mmc.c b/sys/dev/mmc/mmc.c
index 27447ad..e7f6648 100644
--- a/sys/dev/mmc/mmc.c
+++ b/sys/dev/mmc/mmc.c
@@ -353,7 +353,8 @@ mmc_highest_voltage(uint32_t ocr)
 {
 	int i;
 
-	for (i = 30; i >= 0; i--)
+	for (i = MMC_OCR_MAX_VOLTAGE_SHIFT;
+	    i >= MMC_OCR_MIN_VOLTAGE_SHIFT; i--)
 		if (ocr & (1 << i))
 			return (i);
 	return (-1);
diff --git a/sys/dev/mmc/mmcreg.h b/sys/dev/mmc/mmcreg.h
index 3d2b569..f454ddb 100644
--- a/sys/dev/mmc/mmcreg.h
+++ b/sys/dev/mmc/mmcreg.h
@@ -353,6 +353,7 @@ struct mmc_request {
 #define	MMC_OCR_VOLTAGE	0x3fffffffU	/* Vdd Voltage mask */
 #define	MMC_OCR_LOW_VOLTAGE (1u << 7)	/* Low Voltage Range -- tbd */
 #define	MMC_OCR_200_210	(1U << 8)	/* Vdd voltage 2.00 ~ 2.10 */
+#define	MMC_OCR_MIN_VOLTAGE_SHIFT	8
 #define	MMC_OCR_210_220	(1U << 9)	/* Vdd voltage 2.10 ~ 2.20 */
 #define	MMC_OCR_220_230	(1U << 10)	/* Vdd voltage 2.20 ~ 2.30 */
 #define	MMC_OCR_230_240	(1U << 11)	/* Vdd voltage 2.30 ~ 2.40 */
@@ -368,6 +369,7 @@ struct mmc_request {
 #define	MMC_OCR_330_340	(1U << 21)	/* Vdd voltage 3.30 ~ 3.40 */
 #define	MMC_OCR_340_350	(1U << 22)	/* Vdd voltage 3.40 ~ 3.50 */
 #define	MMC_OCR_350_360	(1U << 23)	/* Vdd voltage 3.50 ~ 3.60 */
+#define	MMC_OCR_MAX_VOLTAGE_SHIFT	23
 #define	MMC_OCR_CCS	(1u << 30)	/* Card Capacity status (SD vs SDHC) */
 #define	MMC_OCR_CARD_BUSY (1U << 31)	/* Card Power up status */
 
diff --git a/sys/dev/mn/if_mn.c b/sys/dev/mn/if_mn.c
index 48fc3dc..239aa0f 100644
--- a/sys/dev/mn/if_mn.c
+++ b/sys/dev/mn/if_mn.c
@@ -1418,7 +1418,7 @@ static device_method_t mn_methods[] = {
         DEVMETHOD(device_resume,        bus_generic_resume),
         DEVMETHOD(device_shutdown,      bus_generic_shutdown),
 
-        {0, 0}
+	DEVMETHOD_END
 };
  
 static driver_t mn_driver = {
diff --git a/sys/dev/mxge/if_mxge.c b/sys/dev/mxge/if_mxge.c
index c04e0df2..feefc3b 100644
--- a/sys/dev/mxge/if_mxge.c
+++ b/sys/dev/mxge/if_mxge.c
@@ -124,7 +124,8 @@ static device_method_t mxge_methods[] =
   DEVMETHOD(device_attach, mxge_attach),
   DEVMETHOD(device_detach, mxge_detach),
   DEVMETHOD(device_shutdown, mxge_shutdown),
-  {0, 0}
+
+  DEVMETHOD_END
 };
 
 static driver_t mxge_driver =
diff --git a/sys/dev/my/if_my.c b/sys/dev/my/if_my.c
index 9648412..a487d90 100644
--- a/sys/dev/my/if_my.c
+++ b/sys/dev/my/if_my.c
@@ -153,7 +153,7 @@ static device_method_t my_methods[] = {
 	DEVMETHOD(device_detach, my_detach),
 	DEVMETHOD(device_shutdown, my_shutdown),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t my_driver = {
diff --git a/sys/dev/nand/nand_id.c b/sys/dev/nand/nand_id.c
index 75c5834..76c5f9f 100644
--- a/sys/dev/nand/nand_id.c
+++ b/sys/dev/nand/nand_id.c
@@ -37,6 +37,8 @@ struct nand_params nand_ids[] = {
 	    0x20, 0x200, 0x10, 0x20, 0 },
 	{ { NAND_MAN_SAMSUNG, 0xd3 }, "Samsung NAND 1GiB 3,3V 8-bit",
 	    0x400, 0x800, 0x40, 0x40, 0 },
+	{ { NAND_MAN_SAMSUNG, 0xdc }, "Samsung NAND 512MiB 3,3V 8-bit",
+	    0x200, 0x800, 0x40, 0x40, 0 },
 	{ { NAND_MAN_HYNIX, 0x76 }, "Hynix NAND 64MiB 3,3V 8-bit",
 	    0x40, 0x200, 0x10, 0x20, 0 },
 	{ { NAND_MAN_HYNIX, 0xdc }, "Hynix NAND 512MiB 3,3V 8-bit",
diff --git a/sys/dev/netmap/if_em_netmap.h b/sys/dev/netmap/if_em_netmap.h
index b4b268a..776f0e0 100644
--- a/sys/dev/netmap/if_em_netmap.h
+++ b/sys/dev/netmap/if_em_netmap.h
@@ -292,6 +292,8 @@ em_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 	l = rxr->next_to_check;
 	j = netmap_idx_n2k(kring, l);
 	if (netmap_no_pendintr || force_update) {
+		uint16_t slot_flags = kring->nkr_slot_flags;
+
 		for (n = 0; ; n++) {
 			struct e1000_rx_desc *curr = &rxr->rx_base[l];
 			uint32_t staterr = le32toh(curr->status);
@@ -299,6 +301,7 @@ em_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 			if ((staterr & E1000_RXD_STAT_DD) == 0)
 				break;
 			ring->slot[j].len = le16toh(curr->length);
+			ring->slot[j].flags = slot_flags;
 			bus_dmamap_sync(rxr->rxtag, rxr->rx_buffers[l].map,
 				BUS_DMASYNC_POSTREAD);
 			j = (j == lim) ? 0 : j + 1;
diff --git a/sys/dev/netmap/if_igb_netmap.h b/sys/dev/netmap/if_igb_netmap.h
index bd83563..e817341 100644
--- a/sys/dev/netmap/if_igb_netmap.h
+++ b/sys/dev/netmap/if_igb_netmap.h
@@ -263,6 +263,8 @@ igb_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 	l = rxr->next_to_check;
 	j = netmap_idx_n2k(kring, l);
 	if (netmap_no_pendintr || force_update) {
+		uint16_t slot_flags = kring->nkr_slot_flags;
+
 		for (n = 0; ; n++) {
 			union e1000_adv_rx_desc *curr = &rxr->rx_base[l];
 			uint32_t staterr = le32toh(curr->wb.upper.status_error);
@@ -270,6 +272,7 @@ igb_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 			if ((staterr & E1000_RXD_STAT_DD) == 0)
 				break;
 			ring->slot[j].len = le16toh(curr->wb.upper.length);
+			ring->slot[j].flags = slot_flags;
 			bus_dmamap_sync(rxr->ptag,
 				rxr->rx_buffers[l].pmap, BUS_DMASYNC_POSTREAD);
 			j = (j == lim) ? 0 : j + 1;
diff --git a/sys/dev/netmap/if_lem_netmap.h b/sys/dev/netmap/if_lem_netmap.h
index 9f24580..d3be700 100644
--- a/sys/dev/netmap/if_lem_netmap.h
+++ b/sys/dev/netmap/if_lem_netmap.h
@@ -253,6 +253,8 @@ lem_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 	l = adapter->next_rx_desc_to_check;
 	j = netmap_idx_n2k(kring, l);
 	if (netmap_no_pendintr || force_update) {
+		uint16_t slot_flags = kring->nkr_slot_flags;
+
 		for (n = 0; ; n++) {
 			struct e1000_rx_desc *curr = &adapter->rx_desc_base[l];
 			uint32_t staterr = le32toh(curr->status);
@@ -266,6 +268,7 @@ lem_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 				len = 0;
 			}
 			ring->slot[j].len = len;
+			ring->slot[j].flags = slot_flags;
 			bus_dmamap_sync(adapter->rxtag,
 				adapter->rx_buffer_area[l].map,
 				    BUS_DMASYNC_POSTREAD);
diff --git a/sys/dev/netmap/if_re_netmap.h b/sys/dev/netmap/if_re_netmap.h
index bdd5703..1c747b7 100644
--- a/sys/dev/netmap/if_re_netmap.h
+++ b/sys/dev/netmap/if_re_netmap.h
@@ -245,6 +245,8 @@ re_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 	l = sc->rl_ldata.rl_rx_prodidx; /* next pkt to check */
 	j = netmap_idx_n2k(kring, l); /* the kring index */
 	if (netmap_no_pendintr || force_update) {
+		uint16_t slot_flags = kring->nkr_slot_flags;
+
 		for (n = kring->nr_hwavail; n < lim ; n++) {
 			struct rl_desc *cur_rx = &sc->rl_ldata.rl_rx_list[l];
 			uint32_t rxstat = le32toh(cur_rx->rl_cmdstat);
@@ -256,6 +258,7 @@ re_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 			/* XXX subtract crc */
 			total_len = (total_len < 4) ? 0 : total_len - 4;
 			kring->ring->slot[j].len = total_len;
+			kring->ring->slot[j].flags = slot_flags;
 			/*  sync was in re_newbuf() */
 			bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag,
 			    rxd[l].rx_dmamap, BUS_DMASYNC_POSTREAD);
diff --git a/sys/dev/netmap/ixgbe_netmap.h b/sys/dev/netmap/ixgbe_netmap.h
index 55bd21f..55660ec 100644
--- a/sys/dev/netmap/ixgbe_netmap.h
+++ b/sys/dev/netmap/ixgbe_netmap.h
@@ -63,9 +63,6 @@
  *	This is tricky, much better to use TDH for now.
  */
 SYSCTL_DECL(_dev_netmap);
-static int ix_write_len;
-SYSCTL_INT(_dev_netmap, OID_AUTO, ix_write_len,
-    CTLFLAG_RW, &ix_write_len, 0, "write rx len");
 static int ix_rx_miss, ix_rx_miss_bufs, ix_use_dd, ix_crcstrip;
 SYSCTL_INT(_dev_netmap, OID_AUTO, ix_crcstrip,
     CTLFLAG_RW, &ix_crcstrip, 0, "strip CRC on rx frames");
@@ -485,12 +482,9 @@ ixgbe_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 	 * rxr->next_to_check is set to 0 on a ring reinit
 	 */
 	if (netmap_no_pendintr || force_update) {
-		/* XXX apparently the length field in advanced descriptors
-		 * does not include the CRC irrespective of the setting
-		 * of CRCSTRIP. The data sheets say differently.
-		 * Very strange.
-		 */
 		int crclen = ix_crcstrip ? 0 : 4;
+		uint16_t slot_flags = kring->nkr_slot_flags;
+
 		l = rxr->next_to_check;
 		j = netmap_idx_n2k(kring, l);
 
@@ -501,8 +495,7 @@ ixgbe_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 			if ((staterr & IXGBE_RXD_STAT_DD) == 0)
 				break;
 			ring->slot[j].len = le16toh(curr->wb.upper.length) - crclen;
-			if (ix_write_len)
-				D("rx[%d] len %d", j, ring->slot[j].len);
+			ring->slot[j].flags = slot_flags;
 			bus_dmamap_sync(rxr->ptag,
 			    rxr->rx_buffers[l].pmap, BUS_DMASYNC_POSTREAD);
 			j = (j == lim) ? 0 : j + 1;
diff --git a/sys/dev/netmap/netmap.c b/sys/dev/netmap/netmap.c
index 4f5d486..35d5303 100644
--- a/sys/dev/netmap/netmap.c
+++ b/sys/dev/netmap/netmap.c
@@ -120,10 +120,12 @@ SYSCTL_INT(_dev_netmap, OID_AUTO, no_pendintr,
 
 int netmap_drop = 0;	/* debugging */
 int netmap_flags = 0;	/* debug flags */
+int netmap_fwd = 0;	/* force transparent mode */
 int netmap_copy = 0;	/* debugging, copy content */
 
 SYSCTL_INT(_dev_netmap, OID_AUTO, drop, CTLFLAG_RW, &netmap_drop, 0 , "");
 SYSCTL_INT(_dev_netmap, OID_AUTO, flags, CTLFLAG_RW, &netmap_flags, 0 , "");
+SYSCTL_INT(_dev_netmap, OID_AUTO, fwd, CTLFLAG_RW, &netmap_fwd, 0 , "");
 SYSCTL_INT(_dev_netmap, OID_AUTO, copy, CTLFLAG_RW, &netmap_copy, 0 , "");
 
 #ifdef NM_BRIDGE /* support for netmap bridge */
@@ -275,6 +277,51 @@ nm_find_bridge(const char *name)
 }
 #endif /* NM_BRIDGE */
 
+
+/*
+ * Fetch configuration from the device, to cope with dynamic
+ * reconfigurations after loading the module.
+ */
+static int
+netmap_update_config(struct netmap_adapter *na)
+{
+	struct ifnet *ifp = na->ifp;
+	u_int txr, txd, rxr, rxd;
+
+	txr = txd = rxr = rxd = 0;
+	if (na->nm_config) {
+		na->nm_config(ifp, &txr, &txd, &rxr, &rxd);
+	} else {
+		/* take whatever we had at init time */
+		txr = na->num_tx_rings;
+		txd = na->num_tx_desc;
+		rxr = na->num_rx_rings;
+		rxd = na->num_rx_desc;
+	}	
+
+	if (na->num_tx_rings == txr && na->num_tx_desc == txd &&
+	    na->num_rx_rings == rxr && na->num_rx_desc == rxd)
+		return 0; /* nothing changed */
+	if (netmap_verbose || na->refcount > 0) {
+		D("stored config %s: txring %d x %d, rxring %d x %d",
+			ifp->if_xname,
+			na->num_tx_rings, na->num_tx_desc,
+			na->num_rx_rings, na->num_rx_desc);
+		D("new config %s: txring %d x %d, rxring %d x %d",
+			ifp->if_xname, txr, txd, rxr, rxd);
+	}
+	if (na->refcount == 0) {
+		D("configuration changed (but fine)");
+		na->num_tx_rings = txr;
+		na->num_tx_desc = txd;
+		na->num_rx_rings = rxr;
+		na->num_rx_desc = rxd;
+		return 0;
+	}
+	D("configuration changed while active, this is bad...");
+	return 1;
+}
+
 /*------------- memory allocator -----------------*/
 #ifdef NETMAP_MEM2
 #include "netmap_mem2.c"
@@ -351,7 +398,8 @@ netmap_dtor_locked(void *data)
 	if (na->refcount <= 0) {	/* last instance */
 		u_int i, j, lim;
 
-		D("deleting last netmap instance for %s", ifp->if_xname);
+		if (netmap_verbose)
+			D("deleting last instance for %s", ifp->if_xname);
 		/*
 		 * there is a race here with *_netmap_task() and
 		 * netmap_poll(), which don't run under NETMAP_REG_LOCK.
@@ -482,7 +530,8 @@ static int
 netmap_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
     vm_ooffset_t foff, struct ucred *cred, u_short *color)
 {
-	D("first mmap for %p", handle);
+	if (netmap_verbose)
+		D("first mmap for %p", handle);
 	return saved_cdev_pager_ops.cdev_pg_ctor(handle,
 			size, prot, foff, cred, color);
 }
@@ -491,7 +540,7 @@ static void
 netmap_dev_pager_dtor(void *handle)
 {
 	saved_cdev_pager_ops.cdev_pg_dtor(handle);
-	D("ready to release memory for %p", handle);
+	ND("ready to release memory for %p", handle);
 }
 
 
@@ -507,7 +556,7 @@ netmap_mmap_single(struct cdev *cdev, vm_ooffset_t *foff,
 {
 	vm_object_t obj;
 
-	D("cdev %p foff %jd size %jd objp %p prot %d", cdev,
+	ND("cdev %p foff %jd size %jd objp %p prot %d", cdev,
 	    (intmax_t )*foff, (intmax_t )objsize, objp, prot);
 	obj = vm_pager_allocate(OBJT_DEVICE, cdev, objsize, prot, *foff,
             curthread->td_ucred);
@@ -515,7 +564,7 @@ netmap_mmap_single(struct cdev *cdev, vm_ooffset_t *foff,
 	if (obj == NULL)
 		return EINVAL;
 	if (saved_cdev_pager_ops.cdev_pg_fault == NULL) {
-		D("initialize cdev_pager_ops");
+		ND("initialize cdev_pager_ops");
 		saved_cdev_pager_ops = *(obj->un_pager.devp.ops);
 		netmap_cdev_pager_ops.cdev_pg_fault =
 			saved_cdev_pager_ops.cdev_pg_fault;
@@ -572,7 +621,9 @@ netmap_mmap(__unused struct cdev *dev,
 static int
 netmap_close(struct cdev *dev, int fflag, int devtype, struct thread *td)
 {
-	D("dev %p fflag 0x%x devtype %d td %p", dev, fflag, devtype, td);
+	if (netmap_verbose)
+		D("dev %p fflag 0x%x devtype %d td %p",
+			dev, fflag, devtype, td);
 	return 0;
 }
 
@@ -598,63 +649,170 @@ netmap_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
 
 /*
  * Handlers for synchronization of the queues from/to the host.
- *
- * netmap_sync_to_host() passes packets up. We are called from a
- * system call in user process context, and the only contention
- * can be among multiple user threads erroneously calling
- * this routine concurrently. In principle we should not even
- * need to lock.
+ * Netmap has two operating modes:
+ * - in the default mode, the rings connected to the host stack are
+ *   just another ring pair managed by userspace;
+ * - in transparent mode (XXX to be defined) incoming packets
+ *   (from the host or the NIC) are marked as NS_FORWARD upon
+ *   arrival, and the user application has a chance to reset the
+ *   flag for packets that should be dropped.
+ *   On the RXSYNC or poll(), packets in RX rings between
+ *   kring->nr_kcur and ring->cur with NS_FORWARD still set are moved
+ *   to the other side.
+ * The transfer NIC --> host is relatively easy, just encapsulate
+ * into mbufs and we are done. The host --> NIC side is slightly
+ * harder because there might not be room in the tx ring so it
+ * might take a while before releasing the buffer.
+ */
+
+/*
+ * pass a chain of buffers to the host stack as coming from 'dst'
  */
 static void
-netmap_sync_to_host(struct netmap_adapter *na)
+netmap_send_up(struct ifnet *dst, struct mbuf *head)
 {
-	struct netmap_kring *kring = &na->tx_rings[na->num_tx_rings];
-	struct netmap_ring *ring = kring->ring;
-	struct mbuf *head = NULL, *tail = NULL, *m;
-	u_int k, n, lim = kring->nkr_num_slots - 1;
+	struct mbuf *m;
 
-	k = ring->cur;
-	if (k > lim) {
-		netmap_ring_reinit(kring);
-		return;
+	/* send packets up, outside the lock */
+	while ((m = head) != NULL) {
+		head = head->m_nextpkt;
+		m->m_nextpkt = NULL;
+		if (netmap_verbose & NM_VERB_HOST)
+			D("sending up pkt %p size %d", m, MBUF_LEN(m));
+		NM_SEND_UP(dst, m);
 	}
-	// na->nm_lock(na->ifp, NETMAP_CORE_LOCK, 0);
+}
 
-	/* Take packets from hwcur to cur and pass them up.
+struct mbq {
+	struct mbuf *head;
+	struct mbuf *tail;
+	int count;
+};
+
+/*
+ * put a copy of the buffers marked NS_FORWARD into an mbuf chain.
+ * Run from hwcur to cur - reserved
+ */
+static void
+netmap_grab_packets(struct netmap_kring *kring, struct mbq *q, int force)
+{
+	/* Take packets from hwcur to cur-reserved and pass them up.
 	 * In case of no buffers we give up. At the end of the loop,
 	 * the queue is drained in all cases.
+	 * XXX handle reserved
 	 */
+	int k = kring->ring->cur - kring->ring->reserved;
+	u_int n, lim = kring->nkr_num_slots - 1;
+	struct mbuf *m, *tail = q->tail;
+
+	if (k < 0)
+		k = k + kring->nkr_num_slots;
 	for (n = kring->nr_hwcur; n != k;) {
-		struct netmap_slot *slot = &ring->slot[n];
+		struct netmap_slot *slot = &kring->ring->slot[n];
 
 		n = (n == lim) ? 0 : n + 1;
+		if ((slot->flags & NS_FORWARD) == 0 && !force)
+			continue;
 		if (slot->len < 14 || slot->len > NETMAP_BUF_SIZE) {
 			D("bad pkt at %d len %d", n, slot->len);
 			continue;
 		}
-		m = m_devget(NMB(slot), slot->len, 0, na->ifp, NULL);
+		slot->flags &= ~NS_FORWARD; // XXX needed ?
+		m = m_devget(NMB(slot), slot->len, 0, kring->na->ifp, NULL);
 
 		if (m == NULL)
 			break;
 		if (tail)
 			tail->m_nextpkt = m;
 		else
-			head = m;
+			q->head = m;
 		tail = m;
+		q->count++;
 		m->m_nextpkt = NULL;
 	}
+	q->tail = tail;
+}
+
+/*
+ * called under main lock to send packets from the host to the NIC
+ * The host ring has packets from nr_hwcur to (cur - reserved)
+ * to be sent down. We scan the tx rings, which have just been
+ * flushed so nr_hwcur == cur. Pushing packets down means
+ * increment cur and decrement avail.
+ * XXX to be verified
+ */
+static void
+netmap_sw_to_nic(struct netmap_adapter *na)
+{
+	struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings];
+	struct netmap_kring *k1 = &na->tx_rings[0];
+	int i, howmany, src_lim, dst_lim;
+
+	howmany = kring->nr_hwavail;	/* XXX otherwise cur - reserved - nr_hwcur */
+
+	src_lim = kring->nkr_num_slots;
+	for (i = 0; howmany > 0 && i < na->num_tx_rings; i++, k1++) {
+		ND("%d packets left to ring %d (space %d)", howmany, i, k1->nr_hwavail);
+		dst_lim = k1->nkr_num_slots;
+		while (howmany > 0 && k1->ring->avail > 0) {
+			struct netmap_slot *src, *dst, tmp;
+			src = &kring->ring->slot[kring->nr_hwcur];
+			dst = &k1->ring->slot[k1->ring->cur];
+			tmp = *src;
+			src->buf_idx = dst->buf_idx;
+			src->flags = NS_BUF_CHANGED;
+
+			dst->buf_idx = tmp.buf_idx;
+			dst->len = tmp.len;
+			dst->flags = NS_BUF_CHANGED;
+			ND("out len %d buf %d from %d to %d",
+				dst->len, dst->buf_idx,
+				kring->nr_hwcur, k1->ring->cur);
+
+			if (++kring->nr_hwcur >= src_lim)
+				kring->nr_hwcur = 0;
+			howmany--;
+			kring->nr_hwavail--;
+			if (++k1->ring->cur >= dst_lim)
+				k1->ring->cur = 0;
+			k1->ring->avail--;
+		}
+		kring->ring->cur = kring->nr_hwcur; // XXX
+		k1++;
+	}
+}
+
+/*
+ * netmap_sync_to_host() passes packets up. We are called from a
+ * system call in user process context, and the only contention
+ * can be among multiple user threads erroneously calling
+ * this routine concurrently.
+ */
+static void
+netmap_sync_to_host(struct netmap_adapter *na)
+{
+	struct netmap_kring *kring = &na->tx_rings[na->num_tx_rings];
+	struct netmap_ring *ring = kring->ring;
+	u_int k, lim = kring->nkr_num_slots - 1;
+	struct mbq q = { NULL, NULL };
+
+	k = ring->cur;
+	if (k > lim) {
+		netmap_ring_reinit(kring);
+		return;
+	}
+	// na->nm_lock(na->ifp, NETMAP_CORE_LOCK, 0);
+
+	/* Take packets from hwcur to cur and pass them up.
+	 * In case of no buffers we give up. At the end of the loop,
+	 * the queue is drained in all cases.
+	 */
+	netmap_grab_packets(kring, &q, 1);
 	kring->nr_hwcur = k;
 	kring->nr_hwavail = ring->avail = lim;
 	// na->nm_lock(na->ifp, NETMAP_CORE_UNLOCK, 0);
 
-	/* send packets up, outside the lock */
-	while ((m = head) != NULL) {
-		head = head->m_nextpkt;
-		m->m_nextpkt = NULL;
-		if (netmap_verbose & NM_VERB_HOST)
-			D("sending up pkt %p size %d", m, MBUF_LEN(m));
-		NM_SEND_UP(na->ifp, m);
-	}
+	netmap_send_up(na->ifp, q.head);
 }
 
 /*
@@ -877,6 +1035,7 @@ netmap_set_ringid(struct netmap_priv_d *priv, u_int ringid)
 	priv->np_txpoll = (ringid & NETMAP_NO_TX_POLL) ? 0 : 1;
 	if (need_lock)
 		na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0);
+    if (netmap_verbose) {
 	if (ringid & NETMAP_SW_RING)
 		D("ringid %s set to SW RING", ifp->if_xname);
 	else if (ringid & NETMAP_HW_RING)
@@ -884,6 +1043,7 @@ netmap_set_ringid(struct netmap_priv_d *priv, u_int ringid)
 			priv->np_qfirst);
 	else
 		D("ringid %s set to all %d HW RINGS", ifp->if_xname, lim);
+    }
 	return 0;
 }
 
@@ -965,6 +1125,7 @@ netmap_ioctl(struct cdev *dev, u_long cmd, caddr_t data,
 		if (error)
 			break;
 		na = NA(ifp); /* retrieve netmap_adapter */
+		netmap_update_config(na);
 		nmr->nr_rx_rings = na->num_rx_rings;
 		nmr->nr_tx_rings = na->num_tx_rings;
 		nmr->nr_rx_slots = na->num_rx_desc;
@@ -1014,6 +1175,8 @@ netmap_ioctl(struct cdev *dev, u_long cmd, caddr_t data,
 			break;
 		}
 
+		/* ring configuration may have changed, fetch from the card */
+		netmap_update_config(na);
 		priv->np_ifp = ifp;	/* store the reference */
 		error = netmap_set_ringid(priv, nmr->nr_ringid);
 		if (error)
@@ -1182,7 +1345,8 @@ netmap_poll(struct cdev *dev, int events, struct thread *td)
 	struct ifnet *ifp;
 	struct netmap_kring *kring;
 	u_int core_lock, i, check_all, want_tx, want_rx, revents = 0;
-	u_int lim_tx, lim_rx;
+	u_int lim_tx, lim_rx, host_forwarded = 0;
+	struct mbq q = { NULL, NULL, 0 };
 	enum {NO_CL, NEED_CL, LOCKED_CL }; /* see below */
 	void *pwait = dev;	/* linux compatibility */
 
@@ -1230,6 +1394,17 @@ netmap_poll(struct cdev *dev, int events, struct thread *td)
 		return (revents);
 	}
 
+	/* if we are in transparent mode, check also the host rx ring */
+	kring = &na->rx_rings[lim_rx];
+	if ( (priv->np_qlast == NETMAP_HW_RING) // XXX check_all
+			&& want_rx
+			&& (netmap_fwd || kring->ring->flags & NR_FORWARD) ) {
+		if (kring->ring->avail == 0)
+			netmap_sync_from_host(na, td, dev);
+		if (kring->ring->avail > 0)
+			revents |= want_rx;
+	}
+
 	/*
 	 * check_all is set if the card has more than one queue and
 	 * the client is polling all of them. If true, we sleep on
@@ -1305,6 +1480,7 @@ netmap_poll(struct cdev *dev, int events, struct thread *td)
 	 * to avoid that the tx rings stall).
 	 */
 	if (priv->np_txpoll || want_tx) {
+flush_tx:
 		for (i = priv->np_qfirst; i < lim_tx; i++) {
 			kring = &na->tx_rings[i];
 			/*
@@ -1357,6 +1533,11 @@ netmap_poll(struct cdev *dev, int events, struct thread *td)
 			}
 			if (na->separate_locks)
 				na->nm_lock(ifp, NETMAP_RX_LOCK, i);
+			if (netmap_fwd ||kring->ring->flags & NR_FORWARD) {
+				ND(10, "forwarding some buffers up %d to %d",
+				    kring->nr_hwcur, kring->ring->cur);
+				netmap_grab_packets(kring, &q, netmap_fwd);
+			}
 
 			if (na->nm_rxsync(ifp, i, 0 /* no lock */))
 				revents |= POLLERR;
@@ -1379,8 +1560,28 @@ netmap_poll(struct cdev *dev, int events, struct thread *td)
 		if (want_rx)
 			selrecord(td, &na->rx_si);
 	}
+
+	/* forward host to the netmap ring */
+	kring = &na->rx_rings[lim_rx];
+	if (kring->nr_hwavail > 0)
+		ND("host rx %d has %d packets", lim_rx, kring->nr_hwavail);
+	if ( (priv->np_qlast == NETMAP_HW_RING) // XXX check_all
+			&& (netmap_fwd || kring->ring->flags & NR_FORWARD)
+			 && kring->nr_hwavail > 0 && !host_forwarded) {
+		if (core_lock == NEED_CL) {
+			na->nm_lock(ifp, NETMAP_CORE_LOCK, 0);
+			core_lock = LOCKED_CL;
+		}
+		netmap_sw_to_nic(na);
+		host_forwarded = 1; /* prevent another pass */
+		want_rx = 0;
+		goto flush_tx;
+	}
+
 	if (core_lock == LOCKED_CL)
 		na->nm_lock(ifp, NETMAP_CORE_UNLOCK, 0);
+	if (q.head)
+		netmap_send_up(na->ifp, q.head);
 
 	return (revents);
 }
@@ -1444,46 +1645,28 @@ netmap_lock_wrapper(struct ifnet *dev, int what, u_int queueid)
  * setups.
  */
 int
-netmap_attach(struct netmap_adapter *na, int num_queues)
+netmap_attach(struct netmap_adapter *arg, int num_queues)
 {
-	int n, size;
-	void *buf;
-	struct ifnet *ifp = na->ifp;
+	struct netmap_adapter *na = NULL;
+	struct ifnet *ifp = arg ? arg->ifp : NULL;
 
-	if (ifp == NULL) {
-		D("ifp not set, giving up");
-		return EINVAL;
-	}
-	/* clear other fields ? */
-	na->refcount = 0;
+	if (arg == NULL || ifp == NULL)
+		goto fail;
+	na = malloc(sizeof(*na), M_DEVBUF, M_NOWAIT | M_ZERO);
+	if (na == NULL)
+		goto fail;
+	WNA(ifp) = na;
+	*na = *arg; /* copy everything, trust the driver to not pass junk */
+	NETMAP_SET_CAPABLE(ifp);
 	if (na->num_tx_rings == 0)
 		na->num_tx_rings = num_queues;
 	na->num_rx_rings = num_queues;
-	/* on each direction we have N+1 resources
-	 * 0..n-1	are the hardware rings
-	 * n		is the ring attached to the stack.
-	 */
-	n = na->num_rx_rings + na->num_tx_rings + 2;
-	size = sizeof(*na) + n * sizeof(struct netmap_kring);
-
-	buf = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
-	if (buf) {
-		WNA(ifp) = buf;
-		na->tx_rings = (void *)((char *)buf + sizeof(*na));
-		na->rx_rings = na->tx_rings + na->num_tx_rings + 1;
-		bcopy(na, buf, sizeof(*na));
-		NETMAP_SET_CAPABLE(ifp);
-
-		na = buf;
-		/* Core lock initialized here.  Others are initialized after
-		 * netmap_if_new.
-		 */
-		mtx_init(&na->core_lock, "netmap core lock", MTX_NETWORK_LOCK,
-		    MTX_DEF);
-		if (na->nm_lock == NULL) {
-			ND("using default locks for %s", ifp->if_xname);
-			na->nm_lock = netmap_lock_wrapper;
-		}
+	na->refcount = na->na_single = na->na_multi = 0;
+	/* Core lock initialized here, others after netmap_if_new. */
+	mtx_init(&na->core_lock, "netmap core lock", MTX_NETWORK_LOCK, MTX_DEF);
+	if (na->nm_lock == NULL) {
+		ND("using default locks for %s", ifp->if_xname);
+		na->nm_lock = netmap_lock_wrapper;
 	}
 #ifdef linux
 	if (ifp->netdev_ops) {
@@ -1493,9 +1676,12 @@ netmap_attach(struct netmap_adapter *na, int num_queues)
 	}
 	na->nm_ndo.ndo_start_xmit = linux_netmap_start;
 #endif
-	D("%s for %s", buf ? "ok" : "failed", ifp->if_xname);
+	D("success for %s", ifp->if_xname);
+	return 0;
 
-	return (buf ? 0 : ENOMEM);
+fail:
+	D("fail, arg %p ifp %p na %p", arg, ifp, na);
+	return (na ? EINVAL : ENOMEM);
 }
 
 
@@ -1513,6 +1699,10 @@ netmap_detach(struct ifnet *ifp)
 
 	mtx_destroy(&na->core_lock);
 
+	if (na->tx_rings) { /* XXX should not happen */
+		D("freeing leftover tx_rings");
+		free(na->tx_rings, M_DEVBUF);
+	}
 	bzero(na, sizeof(*na));
 	WNA(ifp) = NULL;
 	free(na, M_DEVBUF);
@@ -1543,7 +1733,8 @@ netmap_start(struct ifnet *ifp, struct mbuf *m)
 		goto done;	/* no space */
 	}
 	if (len > NETMAP_BUF_SIZE) {
-		D("drop packet size %d > %d", len, NETMAP_BUF_SIZE);
+		D("%s from_host, drop packet size %d > %d", ifp->if_xname,
+			len, NETMAP_BUF_SIZE);
 		goto done;	/* too long for us */
 	}
 
@@ -1554,6 +1745,7 @@ netmap_start(struct ifnet *ifp, struct mbuf *m)
 	slot = &kring->ring->slot[i];
 	m_copydata(m, 0, len, NMB(slot));
 	slot->len = len;
+	slot->flags = kring->nkr_slot_flags;
 	kring->nr_hwavail++;
 	if (netmap_verbose  & NM_VERB_HOST)
 		D("wake up host ring %s %d", na->ifp->if_xname, na->num_rx_rings);
diff --git a/sys/dev/netmap/netmap_kern.h b/sys/dev/netmap/netmap_kern.h
index bb0d3fa..86a26fb 100644
--- a/sys/dev/netmap/netmap_kern.h
+++ b/sys/dev/netmap/netmap_kern.h
@@ -119,6 +119,10 @@ struct netmap_adapter;
  * RX rings: the next empty buffer (hwcur + hwavail + hwofs) coincides with
  * 	the next empty buffer as known by the hardware (next_to_check or so).
  * TX rings: hwcur + hwofs coincides with next_to_send
+ *
+ * For received packets, slot->flags is set to nkr_slot_flags
+ * so we can provide a proper initial value (e.g. set NS_FORWARD
+ * when operating in 'transparent' mode).
  */
 struct netmap_kring {
 	struct netmap_ring *ring;
@@ -128,6 +132,7 @@ struct netmap_kring {
 #define NKR_PENDINTR	0x1	// Pending interrupt.
 	u_int nkr_num_slots;
 
+	uint16_t	nkr_slot_flags;	/* initial value for flags */
 	int	nkr_hwofs;	/* offset between NIC and netmap ring */
 	struct netmap_adapter *na;
 	NM_SELINFO_T si;	/* poll/select wait queue */
@@ -198,6 +203,9 @@ struct netmap_adapter {
 	void (*nm_lock)(struct ifnet *, int what, u_int ringid);
 	int (*nm_txsync)(struct ifnet *, u_int ring, int lock);
 	int (*nm_rxsync)(struct ifnet *, u_int ring, int lock);
+	/* return configuration information */
+	int (*nm_config)(struct ifnet *, u_int *txr, u_int *txd,
+					u_int *rxr, u_int *rxd);
 
 	int bdg_port;
 #ifdef linux
diff --git a/sys/dev/netmap/netmap_mem1.c b/sys/dev/netmap/netmap_mem1.c
deleted file mode 100644
index 521855a..0000000
--- a/sys/dev/netmap/netmap_mem1.c
+++ /dev/null
@@ -1,521 +0,0 @@
-/*
- * Copyright (C) 2011 Matteo Landi, Luigi Rizzo. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *   1. Redistributions of source code must retain the above copyright
- *      notice, this list of conditions and the following disclaimer.
- *   2. Redistributions in binary form must reproduce the above copyright
- *      notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-/*
- * $FreeBSD$
- *
- * The original netmap memory allocator, using a single large
- * chunk of memory allocated with contigmalloc.
- */
-
-/*
- * Default amount of memory pre-allocated by the module.
- * We start with a large size and then shrink our demand
- * according to what is avalable when the module is loaded.
- */
-#define NETMAP_MEMORY_SIZE (64 * 1024 * PAGE_SIZE)
-static void * netmap_malloc(size_t size, const char *msg);
-static void netmap_free(void *addr, const char *msg);
-
-#define netmap_if_malloc(len)   netmap_malloc(len, "nifp")
-#define netmap_if_free(v)	netmap_free((v), "nifp")
-
-#define netmap_ring_malloc(len) netmap_malloc(len, "ring")
-#define netmap_free_rings(na)		\
-	netmap_free((na)->tx_rings[0].ring, "shadow rings");
-
-/*
- * Allocator for a pool of packet buffers. For each buffer we have
- * one entry in the bitmap to signal the state. Allocation scans
- * the bitmap, but since this is done only on attach, we are not
- * too worried about performance
- * XXX if we need to allocate small blocks, a translation
- * table is used both for kernel virtual address and physical
- * addresses.
- */
-struct netmap_buf_pool {
-	u_int total_buffers;	/* total buffers. */
-	u_int free;
-	u_int bufsize;
-	char *base;		/* buffer base address */
-	uint32_t *bitmap;	/* one bit per buffer, 1 means free */
-};
-struct netmap_buf_pool nm_buf_pool;
-SYSCTL_INT(_dev_netmap, OID_AUTO, total_buffers,
-    CTLFLAG_RD, &nm_buf_pool.total_buffers, 0, "total_buffers");
-SYSCTL_INT(_dev_netmap, OID_AUTO, free_buffers,
-    CTLFLAG_RD, &nm_buf_pool.free, 0, "free_buffers");
-
-
-/*
- * Allocate n buffers from the ring, and fill the slot.
- * Buffer 0 is the 'junk' buffer.
- */
-static void
-netmap_new_bufs(struct netmap_if *nifp __unused,
-		struct netmap_slot *slot, u_int n)
-{
-	struct netmap_buf_pool *p = &nm_buf_pool;
-	uint32_t bi = 0;		/* index in the bitmap */
-	uint32_t mask, j, i = 0;	/* slot counter */
-
-	if (n > p->free) {
-		D("only %d out of %d buffers available", i, n);
-		return;
-	}
-	/* termination is guaranteed by p->free */
-	while (i < n && p->free > 0) {
-		uint32_t cur = p->bitmap[bi];
-		if (cur == 0) { /* bitmask is fully used */
-			bi++;
-			continue;
-		}
-		/* locate a slot */
-		for (j = 0, mask = 1; (cur & mask) == 0; j++, mask <<= 1) ;
-		p->bitmap[bi] &= ~mask;		/* slot in use */
-		p->free--;
-		slot[i].buf_idx = bi*32+j;
-		slot[i].len = p->bufsize;
-		slot[i].flags = NS_BUF_CHANGED;
-		i++;
-	}
-	ND("allocated %d buffers, %d available", n, p->free);
-}
-
-
-static void
-netmap_free_buf(struct netmap_if *nifp __unused, uint32_t i)
-{
-	struct netmap_buf_pool *p = &nm_buf_pool;
-
-	uint32_t pos, mask;
-	if (i >= p->total_buffers) {
-		D("invalid free index %d", i);
-		return;
-	}
-	pos = i / 32;
-	mask = 1 << (i % 32);
-	if (p->bitmap[pos] & mask) {
-		D("slot %d already free", i);
-		return;
-	}
-	p->bitmap[pos] |= mask;
-	p->free++;
-}
-
-
-/* Descriptor of the memory objects handled by our memory allocator. */
-struct netmap_mem_obj {
-	TAILQ_ENTRY(netmap_mem_obj) nmo_next; /* next object in the
-						 chain. */
-	int nmo_used; /* flag set on used memory objects. */
-	size_t nmo_size; /* size of the memory area reserved for the
-			    object. */
-	void *nmo_data; /* pointer to the memory area. */
-};
-
-/* Wrap our memory objects to make them ``chainable``. */
-TAILQ_HEAD(netmap_mem_obj_h, netmap_mem_obj);
-
-
-/* Descriptor of our custom memory allocator. */
-struct netmap_mem_d {
-	struct mtx nm_mtx; /* lock used to handle the chain of memory
-			      objects. */
-	struct netmap_mem_obj_h nm_molist; /* list of memory objects */
-	size_t nm_size; /* total amount of memory used for rings etc. */
-	size_t nm_totalsize; /* total amount of allocated memory
-		(the difference is used for buffers) */
-	size_t nm_buf_start; /* offset of packet buffers.
-			This is page-aligned. */
-	size_t nm_buf_len; /* total memory for buffers */
-	void *nm_buffer; /* pointer to the whole pre-allocated memory
-			    area. */
-};
-
-/* Shorthand to compute a netmap interface offset. */
-#define netmap_if_offset(v)                                     \
-    ((char *) (v) - (char *) nm_mem->nm_buffer)
-/* .. and get a physical address given a memory offset */
-#define netmap_ofstophys(o)                                     \
-    (vtophys(nm_mem->nm_buffer) + (o))
-
-
-/*------ netmap memory allocator -------*/
-/*
- * Request for a chunk of memory.
- *
- * Memory objects are arranged into a list, hence we need to walk this
- * list until we find an object with the needed amount of data free.
- * This sounds like a completely inefficient implementation, but given
- * the fact that data allocation is done once, we can handle it
- * flawlessly.
- *
- * Return NULL on failure.
- */
-static void *
-netmap_malloc(size_t size, __unused const char *msg)
-{
-	struct netmap_mem_obj *mem_obj, *new_mem_obj;
-	void *ret = NULL;
-
-	NMA_LOCK();
-	TAILQ_FOREACH(mem_obj, &nm_mem->nm_molist, nmo_next) {
-		if (mem_obj->nmo_used != 0 || mem_obj->nmo_size < size)
-			continue;
-
-		new_mem_obj = malloc(sizeof(struct netmap_mem_obj), M_NETMAP,
-				     M_WAITOK | M_ZERO);
-		TAILQ_INSERT_BEFORE(mem_obj, new_mem_obj, nmo_next);
-
-		new_mem_obj->nmo_used = 1;
-		new_mem_obj->nmo_size = size;
-		new_mem_obj->nmo_data = mem_obj->nmo_data;
-		memset(new_mem_obj->nmo_data, 0, new_mem_obj->nmo_size);
-
-		mem_obj->nmo_size -= size;
-		mem_obj->nmo_data = (char *) mem_obj->nmo_data + size;
-		if (mem_obj->nmo_size == 0) {
-			TAILQ_REMOVE(&nm_mem->nm_molist, mem_obj,
-				     nmo_next);
-			free(mem_obj, M_NETMAP);
-		}
-
-		ret = new_mem_obj->nmo_data;
-
-		break;
-	}
-	NMA_UNLOCK();
-	ND("%s: %d bytes at %p", msg, size, ret);
-
-	return (ret);
-}
-
-/*
- * Return the memory to the allocator.
- *
- * While freeing a memory object, we try to merge adjacent chunks in
- * order to reduce memory fragmentation.
- */
-static void
-netmap_free(void *addr, const char *msg)
-{
-	size_t size;
-	struct netmap_mem_obj *cur, *prev, *next;
-
-	if (addr == NULL) {
-		D("NULL addr for %s", msg);
-		return;
-	}
-
-	NMA_LOCK();
-	TAILQ_FOREACH(cur, &nm_mem->nm_molist, nmo_next) {
-		if (cur->nmo_data == addr && cur->nmo_used)
-			break;
-	}
-	if (cur == NULL) {
-		NMA_UNLOCK();
-		D("invalid addr %s %p", msg, addr);
-		return;
-	}
-
-	size = cur->nmo_size;
-	cur->nmo_used = 0;
-
-	/* merge current chunk of memory with the previous one,
-	   if present. */
-	prev = TAILQ_PREV(cur, netmap_mem_obj_h, nmo_next);
-	if (prev && prev->nmo_used == 0) {
-		TAILQ_REMOVE(&nm_mem->nm_molist, cur, nmo_next);
-		prev->nmo_size += cur->nmo_size;
-		free(cur, M_NETMAP);
-		cur = prev;
-	}
-
-	/* merge with the next one */
-	next = TAILQ_NEXT(cur, nmo_next);
-	if (next && next->nmo_used == 0) {
-		TAILQ_REMOVE(&nm_mem->nm_molist, next, nmo_next);
-		cur->nmo_size += next->nmo_size;
-		free(next, M_NETMAP);
-	}
-	NMA_UNLOCK();
-	ND("freed %s %d bytes at %p", msg, size, addr);
-}
-
-
-/*
- * Create and return a new ``netmap_if`` object, and possibly also
- * rings and packet buffors.
- *
- * Return NULL on failure.
- */
-static void *
-netmap_if_new(const char *ifname, struct netmap_adapter *na)
-{
-	struct netmap_if *nifp;
-	struct netmap_ring *ring;
-	struct netmap_kring *kring;
-	char *buff;
-	u_int i, len, ofs, numdesc;
-	u_int nrx = na->num_rx_rings + 1; /* shorthand, include stack queue */
-	u_int ntx = na->num_tx_rings + 1; /* shorthand, include stack queue */
-
-	/*
-	 * the descriptor is followed inline by an array of offsets
-	 * to the tx and rx rings in the shared memory region.
-	 */
-	len = sizeof(struct netmap_if) + (nrx + ntx) * sizeof(ssize_t);
-	nifp = netmap_if_malloc(len);
-	if (nifp == NULL)
-		return (NULL);
-
-	/* initialize base fields */
-	*(int *)(uintptr_t)&nifp->ni_rx_rings = na->num_rx_rings;
-	*(int *)(uintptr_t)&nifp->ni_tx_rings = na->num_tx_rings;
-	strncpy(nifp->ni_name, ifname, IFNAMSIZ);
-
-	(na->refcount)++;	/* XXX atomic ? we are under lock */
-	if (na->refcount > 1)
-		goto final;
-
-	/*
-	 * First instance. Allocate the netmap rings
-	 * (one for each hw queue, one pair for the host).
-	 * The rings are contiguous, but have variable size.
-	 * The entire block is reachable at
-	 *	na->tx_rings[0]
-	 */
-	len = (ntx + nrx) * sizeof(struct netmap_ring) +
-	      (ntx * na->num_tx_desc + nrx * na->num_rx_desc) *
-		   sizeof(struct netmap_slot);
-	buff = netmap_ring_malloc(len);
-	if (buff == NULL) {
-		D("failed to allocate %d bytes for %s shadow ring",
-			len, ifname);
-error:
-		(na->refcount)--;
-		netmap_if_free(nifp);
-		return (NULL);
-	}
-	/* Check whether we have enough buffers */
-	len = ntx * na->num_tx_desc + nrx * na->num_rx_desc;
-	NMA_LOCK();
-	if (nm_buf_pool.free < len) {
-		NMA_UNLOCK();
-		netmap_free(buff, "not enough bufs");
-		goto error;
-	}
-	/*
-	 * in the kring, store the pointers to the shared rings
-	 * and initialize the rings. We are under NMA_LOCK().
-	 */
-	ofs = 0;
-	for (i = 0; i < ntx; i++) { /* Transmit rings */
-		kring = &na->tx_rings[i];
-		numdesc = na->num_tx_desc;
-		bzero(kring, sizeof(*kring));
-		kring->na = na;
-
-		ring = kring->ring = (struct netmap_ring *)(buff + ofs);
-		*(ssize_t *)(uintptr_t)&ring->buf_ofs =
-			nm_buf_pool.base - (char *)ring;
-		ND("txring[%d] at %p ofs %d", i, ring, ring->buf_ofs);
-		*(uint32_t *)(uintptr_t)&ring->num_slots =
-			kring->nkr_num_slots = numdesc;
-
-		/*
-		 * IMPORTANT:
-		 * Always keep one slot empty, so we can detect new
-		 * transmissions comparing cur and nr_hwcur (they are
-		 * the same only if there are no new transmissions).
-		 */
-		ring->avail = kring->nr_hwavail = numdesc - 1;
-		ring->cur = kring->nr_hwcur = 0;
-		*(uint16_t *)(uintptr_t)&ring->nr_buf_size = NETMAP_BUF_SIZE;
-		netmap_new_bufs(nifp, ring->slot, numdesc);
-
-		ofs += sizeof(struct netmap_ring) +
-			numdesc * sizeof(struct netmap_slot);
-	}
-
-	for (i = 0; i < nrx; i++) { /* Receive rings */
-		kring = &na->rx_rings[i];
-		numdesc = na->num_rx_desc;
-		bzero(kring, sizeof(*kring));
-		kring->na = na;
-
-		ring = kring->ring = (struct netmap_ring *)(buff + ofs);
-		*(ssize_t *)(uintptr_t)&ring->buf_ofs =
-			nm_buf_pool.base - (char *)ring;
-		ND("rxring[%d] at %p offset %d", i, ring, ring->buf_ofs);
-		*(uint32_t *)(uintptr_t)&ring->num_slots =
-			kring->nkr_num_slots = numdesc;
-		ring->cur = kring->nr_hwcur = 0;
-		ring->avail = kring->nr_hwavail = 0; /* empty */
-		*(uint16_t *)(uintptr_t)&ring->nr_buf_size = NETMAP_BUF_SIZE;
-		netmap_new_bufs(nifp, ring->slot, numdesc);
-		ofs += sizeof(struct netmap_ring) +
-			numdesc * sizeof(struct netmap_slot);
-	}
-	NMA_UNLOCK();
-	// XXX initialize the selrecord structs.
-
-final:
-	/*
-	 * fill the slots for the rx and tx queues. They contain the offset
-	 * between the ring and nifp, so the information is usable in
-	 * userspace to reach the ring from the nifp.
-	 */
-	for (i = 0; i < ntx; i++) {
-		*(ssize_t *)(uintptr_t)&nifp->ring_ofs[i] =
-			(char *)na->tx_rings[i].ring - (char *)nifp;
-	}
-	for (i = 0; i < nrx; i++) {
-		*(ssize_t *)(uintptr_t)&nifp->ring_ofs[i+ntx] =
-			(char *)na->rx_rings[i].ring - (char *)nifp;
-	}
-	return (nifp);
-}
-
-/*
- * Initialize the memory allocator.
- *
- * Create the descriptor for the memory , allocate the pool of memory
- * and initialize the list of memory objects with a single chunk
- * containing the whole pre-allocated memory marked as free.
- *
- * Start with a large size, then halve as needed if we fail to
- * allocate the block. While halving, always add one extra page
- * because buffers 0 and 1 are used for special purposes.
- * Return 0 on success, errno otherwise.
- */
-static int
-netmap_memory_init(void)
-{
-	struct netmap_mem_obj *mem_obj;
-	void *buf = NULL;
-	int i, n, sz = NETMAP_MEMORY_SIZE;
-	int extra_sz = 0; // space for rings and two spare buffers
-
-	for (; sz >= 1<<20; sz >>=1) {
-		extra_sz = sz/200;
-		extra_sz = (extra_sz + 2*PAGE_SIZE - 1) & ~(PAGE_SIZE-1);
-	        buf = contigmalloc(sz + extra_sz,
-			     M_NETMAP,
-			     M_WAITOK | M_ZERO,
-			     0, /* low address */
-			     -1UL, /* high address */
-			     PAGE_SIZE, /* alignment */
-			     0 /* boundary */
-			    );
-		if (buf)
-			break;
-	}
-	if (buf == NULL)
-		return (ENOMEM);
-	sz += extra_sz;
-	nm_mem = malloc(sizeof(struct netmap_mem_d), M_NETMAP,
-			      M_WAITOK | M_ZERO);
-	mtx_init(&nm_mem->nm_mtx, "netmap memory allocator lock", NULL,
-		 MTX_DEF);
-	TAILQ_INIT(&nm_mem->nm_molist);
-	nm_mem->nm_buffer = buf;
-	nm_mem->nm_totalsize = sz;
-
-	/*
-	 * A buffer takes 2k, a slot takes 8 bytes + ring overhead,
-	 * so the ratio is 200:1. In other words, we can use 1/200 of
-	 * the memory for the rings, and the rest for the buffers,
-	 * and be sure we never run out.
-	 */
-	nm_mem->nm_size = sz/200;
-	nm_mem->nm_buf_start =
-		(nm_mem->nm_size + PAGE_SIZE - 1) & ~(PAGE_SIZE-1);
-	nm_mem->nm_buf_len = sz - nm_mem->nm_buf_start;
-
-	nm_buf_pool.base = nm_mem->nm_buffer;
-	nm_buf_pool.base += nm_mem->nm_buf_start;
-	netmap_buffer_base = nm_buf_pool.base;
-	D("netmap_buffer_base %p (offset %d)",
-		netmap_buffer_base, (int)nm_mem->nm_buf_start);
-	/* number of buffers, they all start as free */
-
-	netmap_total_buffers = nm_buf_pool.total_buffers =
-		nm_mem->nm_buf_len / NETMAP_BUF_SIZE;
-	nm_buf_pool.bufsize = NETMAP_BUF_SIZE;
-
-	D("Have %d MB, use %dKB for rings, %d buffers at %p",
-		(sz >> 20), (int)(nm_mem->nm_size >> 10),
-		nm_buf_pool.total_buffers, nm_buf_pool.base);
-
-	/* allocate and initialize the bitmap. Entry 0 is considered
-	 * always busy (used as default when there are no buffers left).
-	 */
-	n = (nm_buf_pool.total_buffers + 31) / 32;
-	nm_buf_pool.bitmap = malloc(sizeof(uint32_t) * n, M_NETMAP,
-			 M_WAITOK | M_ZERO);
-	nm_buf_pool.bitmap[0] = ~3; /* slot 0 and 1 always busy */
-	for (i = 1; i < n; i++)
-		nm_buf_pool.bitmap[i] = ~0;
-	nm_buf_pool.free = nm_buf_pool.total_buffers - 2;
-	
-	mem_obj = malloc(sizeof(struct netmap_mem_obj), M_NETMAP,
-			 M_WAITOK | M_ZERO);
-	TAILQ_INSERT_HEAD(&nm_mem->nm_molist, mem_obj, nmo_next);
-	mem_obj->nmo_used = 0;
-	mem_obj->nmo_size = nm_mem->nm_size;
-	mem_obj->nmo_data = nm_mem->nm_buffer;
-
-	return (0);
-}
-
-
-/*
- * Finalize the memory allocator.
- *
- * Free all the memory objects contained inside the list, and deallocate
- * the pool of memory; finally free the memory allocator descriptor.
- */
-static void
-netmap_memory_fini(void)
-{
-	struct netmap_mem_obj *mem_obj;
-
-	while (!TAILQ_EMPTY(&nm_mem->nm_molist)) {
-		mem_obj = TAILQ_FIRST(&nm_mem->nm_molist);
-		TAILQ_REMOVE(&nm_mem->nm_molist, mem_obj, nmo_next);
-		if (mem_obj->nmo_used == 1) {
-			printf("netmap: leaked %d bytes at %p\n",
-			       (int)mem_obj->nmo_size,
-			       mem_obj->nmo_data);
-		}
-		free(mem_obj, M_NETMAP);
-	}
-	contigfree(nm_mem->nm_buffer, nm_mem->nm_totalsize, M_NETMAP);
-	// XXX mutex_destroy(nm_mtx);
-	free(nm_mem, M_NETMAP);
-}
-/*------------- end of memory allocator -----------------*/
diff --git a/sys/dev/netmap/netmap_mem2.c b/sys/dev/netmap/netmap_mem2.c
index 7472178..03a52b6 100644
--- a/sys/dev/netmap/netmap_mem2.c
+++ b/sys/dev/netmap/netmap_mem2.c
@@ -388,7 +388,7 @@ netmap_obj_free_va(struct netmap_obj_pool *p, void *vaddr)
 		netmap_obj_free(p, j);
 		return;
 	}
-	ND("address %p is not contained inside any cluster (%s)",
+	D("address %p is not contained inside any cluster (%s)",
 	    vaddr, p->name);
 }
 
@@ -559,8 +559,9 @@ netmap_config_obj_allocator(struct netmap_obj_pool *p, u_int objtotal, u_int obj
 	i =  (clustsize & (PAGE_SIZE - 1));
 	if (i)
 		clustsize += PAGE_SIZE - i;
-	D("objsize %d clustsize %d objects %d",
-		objsize, clustsize, clustentries);
+	if (netmap_verbose)
+		D("objsize %d clustsize %d objects %d",
+			objsize, clustsize, clustentries);
 
 	/*
 	 * The number of clusters is n = ceil(objtotal/clustentries)
@@ -649,9 +650,10 @@ netmap_finalize_obj_allocator(struct netmap_obj_pool *p)
 		}
 	}
 	p->bitmap[0] = ~3; /* objs 0 and 1 is always busy */
-	D("Pre-allocated %d clusters (%d/%dKB) for '%s'",
-	    p->_numclusters, p->_clustsize >> 10,
-	    p->_memtotal >> 10, p->name);
+	if (netmap_verbose)
+		D("Pre-allocated %d clusters (%d/%dKB) for '%s'",
+		    p->_numclusters, p->_clustsize >> 10,
+		    p->_memtotal >> 10, p->name);
 
 	return 0;
 
@@ -721,7 +723,7 @@ netmap_memory_finalize(void)
 
 	nm_mem.refcount++;
 	if (nm_mem.refcount > 1) {
-		D("busy (refcount %d)", nm_mem.refcount);
+		ND("busy (refcount %d)", nm_mem.refcount);
 		goto out;
 	}
 
@@ -796,6 +798,8 @@ static void
 netmap_free_rings(struct netmap_adapter *na)
 {
 	int i;
+	if (!na->tx_rings)
+		return;
 	for (i = 0; i < na->num_tx_rings + 1; i++) {
 		netmap_ring_free(na->tx_rings[i].ring);
 		na->tx_rings[i].ring = NULL;
@@ -804,22 +808,32 @@ netmap_free_rings(struct netmap_adapter *na)
 		netmap_ring_free(na->rx_rings[i].ring);
 		na->rx_rings[i].ring = NULL;
 	}
+	free(na->tx_rings, M_DEVBUF);
+	na->tx_rings = na->rx_rings = NULL;
 }
 
 
 
 /* call with NMA_LOCK held */
+/*
+ * Allocate the per-fd structure netmap_if.
+ * If this is the first instance, also allocate the krings, rings etc.
+ */
 static void *
 netmap_if_new(const char *ifname, struct netmap_adapter *na)
 {
 	struct netmap_if *nifp;
 	struct netmap_ring *ring;
 	ssize_t base; /* handy for relative offsets between rings and nifp */
-	u_int i, len, ndesc;
-	u_int ntx = na->num_tx_rings + 1; /* shorthand, include stack ring */
-	u_int nrx = na->num_rx_rings + 1; /* shorthand, include stack ring */
+	u_int i, len, ndesc, ntx, nrx;
 	struct netmap_kring *kring;
 
+	if (netmap_update_config(na)) {
+		/* configuration mismatch, report and fail */
+		return NULL;
+	}
+	ntx = na->num_tx_rings + 1; /* shorthand, include stack ring */
+	nrx = na->num_rx_rings + 1; /* shorthand, include stack ring */
 	/*
 	 * the descriptor is followed inline by an array of offsets
 	 * to the tx and rx rings in the shared memory region.
@@ -840,6 +854,14 @@ netmap_if_new(const char *ifname, struct netmap_adapter *na)
 		goto final;
 	}
 
+	len = (ntx + nrx) * sizeof(struct netmap_kring);
+	na->tx_rings = malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO);
+	if (na->tx_rings == NULL) {
+		D("Cannot allocate krings for %s", ifname);
+		goto cleanup;
+	}
+	na->rx_rings = na->tx_rings + ntx;
+
 	/*
 	 * First instance, allocate netmap rings and buffers for this card
 	 * The rings are contiguous, but have variable size.
@@ -947,5 +969,6 @@ static void
 netmap_memory_deref(void)
 {
 	nm_mem.refcount--;
-	D("refcount = %d", nm_mem.refcount);
+	if (netmap_verbose)
+		D("refcount = %d", nm_mem.refcount);
 }
diff --git a/sys/dev/nvram2env/nvram2env.c b/sys/dev/nvram2env/nvram2env.c
index 32f17f9..5de4f87 100644
--- a/sys/dev/nvram2env/nvram2env.c
+++ b/sys/dev/nvram2env/nvram2env.c
@@ -303,7 +303,8 @@ static device_method_t nvram2env_methods[] = {
 	DEVMETHOD(device_identify, 	nvram2env_identify),
 	DEVMETHOD(device_probe,		nvram2env_probe),
 	DEVMETHOD(device_attach,	nvram2env_attach),
-	{0, 0},
+
+	DEVMETHOD_END
 };
 
 static driver_t nvram2env_driver = {
diff --git a/sys/dev/nxge/if_nxge.c b/sys/dev/nxge/if_nxge.c
index c7dbeb6..df1b9ec 100644
--- a/sys/dev/nxge/if_nxge.c
+++ b/sys/dev/nxge/if_nxge.c
@@ -3507,7 +3507,8 @@ static device_method_t xge_methods[] = {
 	DEVMETHOD(device_attach,    xge_attach),
 	DEVMETHOD(device_detach,    xge_detach),
 	DEVMETHOD(device_shutdown,  xge_shutdown),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t xge_driver = {
diff --git a/sys/dev/oce/oce_if.c b/sys/dev/oce/oce_if.c
index 7d15fcc..cba57bf 100644
--- a/sys/dev/oce/oce_if.c
+++ b/sys/dev/oce/oce_if.c
@@ -108,7 +108,8 @@ static device_method_t oce_dispatch[] = {
 	DEVMETHOD(device_attach, oce_attach),
 	DEVMETHOD(device_detach, oce_detach),
 	DEVMETHOD(device_shutdown, oce_shutdown),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t oce_driver = {
diff --git a/sys/dev/puc/puc.c b/sys/dev/puc/puc.c
index 7b93306..d7bfb63 100644
--- a/sys/dev/puc/puc.c
+++ b/sys/dev/puc/puc.c
@@ -622,7 +622,7 @@ puc_bus_setup_intr(device_t dev, device_t child, struct resource *res,
 	if (cookiep == NULL || res != port->p_ires)
 		return (EINVAL);
 	/* We demand that serdev devices use filter_only interrupts. */
-	if (ihand != NULL)
+	if (port->p_type == PUC_TYPE_SERIAL && ihand != NULL)
 		return (ENXIO);
 	if (rman_get_device(port->p_ires) != originator)
 		return (ENXIO);
diff --git a/sys/dev/sdhci/sdhci_pci.c b/sys/dev/sdhci/sdhci_pci.c
index 02dc2a1..b8d8530 100644
--- a/sys/dev/sdhci/sdhci_pci.c
+++ b/sys/dev/sdhci/sdhci_pci.c
@@ -426,7 +426,7 @@ static device_method_t sdhci_methods[] = {
 	DEVMETHOD(sdhci_write_4,	sdhci_pci_write_4),
 	DEVMETHOD(sdhci_write_multi_4,	sdhci_pci_write_multi_4),
 
-	{0, 0},
+	DEVMETHOD_END
 };
 
 static driver_t sdhci_pci_driver = {
diff --git a/sys/dev/sound/pci/emu10kx-midi.c b/sys/dev/sound/pci/emu10kx-midi.c
index c85800e..0d5fa30 100644
--- a/sys/dev/sound/pci/emu10kx-midi.c
+++ b/sys/dev/sound/pci/emu10kx-midi.c
@@ -239,7 +239,7 @@ static device_method_t emu_midi_methods[] = {
 	DEVMETHOD(device_attach, emu_midi_attach),
 	DEVMETHOD(device_detach, emu_midi_detach),
 
-	{0, 0},
+	DEVMETHOD_END
 };
 
 static driver_t emu_midi_driver = {
diff --git a/sys/dev/sound/pci/emu10kx-pcm.c b/sys/dev/sound/pci/emu10kx-pcm.c
index ab20dc6..406a85f 100644
--- a/sys/dev/sound/pci/emu10kx-pcm.c
+++ b/sys/dev/sound/pci/emu10kx-pcm.c
@@ -1520,7 +1520,8 @@ static device_method_t emu_pcm_methods[] = {
 	DEVMETHOD(device_probe, emu_pcm_probe),
 	DEVMETHOD(device_attach, emu_pcm_attach),
 	DEVMETHOD(device_detach, emu_pcm_detach),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t emu_pcm_driver = {
diff --git a/sys/dev/sound/pci/emu10kx.c b/sys/dev/sound/pci/emu10kx.c
index 5f217c1..35a4957 100644
--- a/sys/dev/sound/pci/emu10kx.c
+++ b/sys/dev/sound/pci/emu10kx.c
@@ -3529,7 +3529,7 @@ static device_method_t emu_methods[] = {
 	DEVMETHOD(bus_read_ivar, emu_read_ivar),
 	DEVMETHOD(bus_write_ivar, emu_write_ivar),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 
diff --git a/sys/dev/sound/sbus/cs4231.c b/sys/dev/sound/sbus/cs4231.c
index e643a4e..83f70f3 100644
--- a/sys/dev/sound/sbus/cs4231.c
+++ b/sys/dev/sound/sbus/cs4231.c
@@ -231,7 +231,8 @@ static device_method_t cs4231_sbus_methods[] = {
 	DEVMETHOD(device_detach,	cs4231_bus_detach),
 	DEVMETHOD(device_suspend,	cs4231_bus_suspend),
 	DEVMETHOD(device_resume,	cs4231_bus_resume),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t cs4231_sbus_driver = {
@@ -249,7 +250,8 @@ static device_method_t cs4231_ebus_methods[] = {
 	DEVMETHOD(device_detach,	cs4231_bus_detach),
 	DEVMETHOD(device_suspend,	cs4231_bus_suspend),
 	DEVMETHOD(device_resume,	cs4231_bus_resume),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t cs4231_ebus_driver = {
diff --git a/sys/dev/sound/usb/uaudio_pcm.c b/sys/dev/sound/usb/uaudio_pcm.c
index b57d0fb..bd00a01 100644
--- a/sys/dev/sound/usb/uaudio_pcm.c
+++ b/sys/dev/sound/usb/uaudio_pcm.c
@@ -227,7 +227,7 @@ static device_method_t ua_pcm_methods[] = {
 	DEVMETHOD(device_attach, ua_attach),
 	DEVMETHOD(device_detach, ua_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t ua_pcm_driver = {
diff --git a/sys/dev/sym/sym_hipd.c b/sys/dev/sym/sym_hipd.c
index b644f94..7d0450c 100644
--- a/sys/dev/sym/sym_hipd.c
+++ b/sys/dev/sym/sym_hipd.c
@@ -86,6 +86,7 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/bus.h>
 #include <machine/resource.h>
+#include <machine/atomic.h>
 
 #ifdef __sparc64__
 #include <dev/ofw/openfirm.h>
@@ -136,7 +137,7 @@ typedef	u_int32_t u32;
 #elif	defined	__sparc64__
 #define MEMORY_BARRIER()	__asm__ volatile("membar #Sync" : : : "memory")
 #elif	defined	__arm__
-#define MEMORY_BARRIER()	__do_dmb()
+#define MEMORY_BARRIER()	dmb()
 #else
 #error	"Not supported platform"
 #endif
@@ -3426,7 +3427,6 @@ sym_getsync(hcb_p np, u_char dt, u_char sfac, u_char *divp, u_char *fakp)
 	/*
 	 *  Check against our hardware limits, or bugs :).
 	 */
-	if (fak < 0)	{fak = 0; ret = -1;}
 	if (fak > 2)	{fak = 2; ret = -1;}
 
 	/*
diff --git a/sys/dev/terasic/de4led/terasic_de4led.c b/sys/dev/terasic/de4led/terasic_de4led.c
index 1fb92d2..bcef7af 100644
--- a/sys/dev/terasic/de4led/terasic_de4led.c
+++ b/sys/dev/terasic/de4led/terasic_de4led.c
@@ -43,6 +43,8 @@ __FBSDID("$FreeBSD$");
 #include <dev/led/led.h>
 #include <dev/terasic/de4led/terasic_de4led.h>
 
+devclass_t	terasic_de4led_devclass;
+
 static void
 terasic_de4led_update(struct terasic_de4led_softc *sc)
 {
diff --git a/sys/dev/terasic/de4led/terasic_de4led.h b/sys/dev/terasic/de4led/terasic_de4led.h
index 89461d1..4475e9b 100644
--- a/sys/dev/terasic/de4led/terasic_de4led.h
+++ b/sys/dev/terasic/de4led/terasic_de4led.h
@@ -72,4 +72,6 @@ struct terasic_de4led_softc {
 void	terasic_de4led_attach(struct terasic_de4led_softc *sc);
 void	terasic_de4led_detach(struct terasic_de4led_softc *sc);
 
+extern devclass_t	terasic_de4led_devclass;
+
 #endif /* _DEV_TERASIC_DE4LED_H_ */
diff --git a/sys/dev/terasic/de4led/terasic_de4led_fdt.c b/sys/dev/terasic/de4led/terasic_de4led_fdt.c
new file mode 100644
index 0000000..0888d6d
--- /dev/null
+++ b/sys/dev/terasic/de4led/terasic_de4led_fdt.c
@@ -0,0 +1,118 @@
+/*-
+ * Copyright (c) 2012 Robert N. M. Watson
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
+ * ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/condvar.h>
+#include <sys/conf.h>
+#include <sys/bio.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/rman.h>
+#include <sys/systm.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <dev/terasic/de4led/terasic_de4led.h>
+
+/*
+ * Nexus bus attachment for the 8-element LED on the Terasic DE-4 FPGA board,
+ * which is hooked up to the processor via a memory-mapped Avalon bus.
+ */
+static int
+terasic_de4led_fdt_probe(device_t dev)
+{
+
+	if (ofw_bus_is_compatible(dev, "sri-cambridge,de4led")) {
+		device_set_desc(dev, "Terasic DE4 8-element LED");
+		return (BUS_PROBE_DEFAULT);
+	}
+	return (ENXIO);
+}
+
+static int
+terasic_de4led_fdt_attach(device_t dev)
+{
+	struct terasic_de4led_softc *sc;
+
+	sc = device_get_softc(dev);
+	sc->tdl_dev = dev;
+	sc->tdl_unit = device_get_unit(dev);
+	sc->tdl_rid = 0;
+	sc->tdl_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+	    &sc->tdl_rid, RF_ACTIVE);
+	if (sc->tdl_res == NULL) {
+		device_printf(dev, "couldn't map memory\n");
+		return (ENXIO);
+	}
+	terasic_de4led_attach(sc);
+	return (0);
+}
+
+static int
+terasic_de4led_fdt_detach(device_t dev)
+{
+	struct terasic_de4led_softc *sc;
+
+	sc = device_get_softc(dev);
+	KASSERT(sc->tdl_res != NULL, ("%s: resources not allocated",
+	    __func__));
+	terasic_de4led_detach(sc);
+	bus_release_resource(dev, SYS_RES_MEMORY, sc->tdl_rid, sc->tdl_res);
+	return (0);
+}
+
+static device_method_t terasic_de4led_fdt_methods[] = {
+	DEVMETHOD(device_probe,		terasic_de4led_fdt_probe),
+	DEVMETHOD(device_attach,	terasic_de4led_fdt_attach),
+	DEVMETHOD(device_detach,	terasic_de4led_fdt_detach),
+	{ 0, 0 }
+};
+
+static driver_t terasic_de4led_fdt_driver = {
+	"terasic_de4led",
+	terasic_de4led_fdt_methods,
+	sizeof(struct terasic_de4led_softc),
+};
+
+DRIVER_MODULE(terasic_de4led, simplebus, terasic_de4led_fdt_driver,
+    terasic_de4led_devclass, 0, 0);
diff --git a/sys/dev/terasic/de4led/terasic_de4led_nexus.c b/sys/dev/terasic/de4led/terasic_de4led_nexus.c
index 92da503..870d412 100644
--- a/sys/dev/terasic/de4led/terasic_de4led_nexus.c
+++ b/sys/dev/terasic/de4led/terasic_de4led_nexus.c
@@ -106,7 +106,5 @@ static driver_t terasic_de4led_nexus_driver = {
 	sizeof(struct terasic_de4led_softc),
 };
 
-static devclass_t terasic_de4led_devclass;
-
 DRIVER_MODULE(terasic_de4led, nexus, terasic_de4led_nexus_driver,
     terasic_de4led_devclass, 0, 0);
diff --git a/sys/dev/terasic/mtl/terasic_mtl.c b/sys/dev/terasic/mtl/terasic_mtl.c
index 48e60c7..5305172 100644
--- a/sys/dev/terasic/mtl/terasic_mtl.c
+++ b/sys/dev/terasic/mtl/terasic_mtl.c
@@ -63,6 +63,9 @@ __FBSDID("$FreeBSD$");
  * attached as they may be called even if the attach routine hasn't been, on
  * an error.
  */
+
+devclass_t	terasic_mtl_devclass;
+
 int
 terasic_mtl_attach(struct terasic_mtl_softc *sc)
 {
diff --git a/sys/dev/terasic/mtl/terasic_mtl.h b/sys/dev/terasic/mtl/terasic_mtl.h
index ae41067..e07bbf4 100644
--- a/sys/dev/terasic/mtl/terasic_mtl.h
+++ b/sys/dev/terasic/mtl/terasic_mtl.h
@@ -159,6 +159,8 @@ struct terasic_mtl_softc {
 int	terasic_mtl_attach(struct terasic_mtl_softc *sc);
 void	terasic_mtl_detach(struct terasic_mtl_softc *sc);
 
+extern devclass_t	terasic_mtl_devclass;
+
 /*
  * Sub-driver setup routines.
  */
diff --git a/sys/dev/terasic/mtl/terasic_mtl_fdt.c b/sys/dev/terasic/mtl/terasic_mtl_fdt.c
new file mode 100644
index 0000000..7ffffb2
--- /dev/null
+++ b/sys/dev/terasic/mtl/terasic_mtl_fdt.c
@@ -0,0 +1,200 @@
+/*-
+ * Copyright (c) 2012-2013 Robert N. M. Watson
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
+ * ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/condvar.h>
+#include <sys/conf.h>
+#include <sys/consio.h>				/* struct vt_mode */
+#include <sys/fbio.h>				/* video_adapter_t */
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/rman.h>
+#include <sys/systm.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <dev/terasic/mtl/terasic_mtl.h>
+
+static int
+terasic_mtl_fdt_probe(device_t dev)
+{
+
+	if (ofw_bus_is_compatible(dev, "sri-cambridge,mtl")) {
+		device_set_desc(dev, "Terasic Multi-touch LCD (MTL)");
+		return (BUS_PROBE_DEFAULT);
+	}
+        return (ENXIO);
+}
+
+static int
+terasic_mtl_fdt_attach(device_t dev)
+{
+	struct terasic_mtl_softc *sc;
+	int error;
+
+	sc = device_get_softc(dev);
+	sc->mtl_dev = dev;
+	sc->mtl_unit = device_get_unit(dev);
+
+	/*
+	 * FDT allows multiple memory resources to be defined for a device;
+	 * query them in the order registers, pixel buffer, text buffer.
+	 * However, we need to sanity-check that they are page-aligned and
+	 * page-sized, so we may still abort.
+	 */
+	sc->mtl_reg_rid = 0;
+	sc->mtl_reg_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+	    &sc->mtl_reg_rid, RF_ACTIVE);
+	if (sc->mtl_reg_res == NULL) {
+		device_printf(dev, "couldn't map register memory\n");
+		error = ENXIO;
+		goto error;
+	}
+	if (rman_get_start(sc->mtl_reg_res) % PAGE_SIZE != 0) {
+		device_printf(dev, "improper register address");
+		error = ENXIO;
+		goto error;
+	}
+	if (rman_get_size(sc->mtl_reg_res) % PAGE_SIZE != 0) {
+		device_printf(dev, "improper register size");
+		error = ENXIO;
+		goto error;
+	}
+	device_printf(sc->mtl_dev, "registers at mem %p-%p\n",
+            (void *)rman_get_start(sc->mtl_reg_res),
+	    (void *)(rman_get_start(sc->mtl_reg_res) +
+	      rman_get_size(sc->mtl_reg_res)));
+
+	sc->mtl_pixel_rid = 1;
+	sc->mtl_pixel_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+	    &sc->mtl_pixel_rid, RF_ACTIVE);
+	if (sc->mtl_pixel_res == NULL) {
+		device_printf(dev, "couldn't map pixel memory\n");
+		error = ENXIO;
+		goto error;
+	}
+	if (rman_get_start(sc->mtl_pixel_res) % PAGE_SIZE != 0) {
+		device_printf(dev, "improper pixel address");
+		error = ENXIO;
+		goto error;
+	}
+	if (rman_get_size(sc->mtl_pixel_res) % PAGE_SIZE != 0) {
+		device_printf(dev, "improper pixel size");
+		error = ENXIO;
+		goto error;
+	}
+	device_printf(sc->mtl_dev, "pixel frame buffer at mem %p-%p\n",
+            (void *)rman_get_start(sc->mtl_pixel_res),
+	    (void *)(rman_get_start(sc->mtl_pixel_res) +
+	      rman_get_size(sc->mtl_pixel_res)));
+
+	sc->mtl_text_rid = 2;
+	sc->mtl_text_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+	    &sc->mtl_text_rid, RF_ACTIVE);
+	if (sc->mtl_text_res == NULL) {
+		device_printf(dev, "couldn't map text memory\n");
+		error = ENXIO;
+		goto error;
+	}
+	if (rman_get_start(sc->mtl_text_res) % PAGE_SIZE != 0) {
+		device_printf(dev, "improper text address");
+		error = ENXIO;
+		goto error;
+	}
+	if (rman_get_size(sc->mtl_text_res) % PAGE_SIZE != 0) {
+		device_printf(dev, "improper text size");
+		error = ENXIO;
+		goto error;
+	}
+	device_printf(sc->mtl_dev, "text frame buffer at mem %p-%p\n",
+            (void *)rman_get_start(sc->mtl_text_res),
+	    (void *)(rman_get_start(sc->mtl_text_res) +
+	      rman_get_size(sc->mtl_text_res)));
+
+	error = terasic_mtl_attach(sc);
+	if (error == 0)
+		return (0);
+error:
+	if (sc->mtl_text_res != NULL)
+		bus_release_resource(dev, SYS_RES_MEMORY, sc->mtl_text_rid,
+		    sc->mtl_text_res);
+	if (sc->mtl_pixel_res != NULL)
+		bus_release_resource(dev, SYS_RES_MEMORY, sc->mtl_pixel_rid,
+		    sc->mtl_pixel_res);
+	if (sc->mtl_reg_res != NULL)
+		bus_release_resource(dev, SYS_RES_MEMORY, sc->mtl_reg_rid,
+		    sc->mtl_reg_res);
+	return (error);
+}
+
+static int
+terasic_mtl_fdt_detach(device_t dev)
+{
+	struct terasic_mtl_softc *sc;
+
+	sc = device_get_softc(dev);
+	terasic_mtl_detach(sc);
+	bus_release_resource(dev, SYS_RES_MEMORY, sc->mtl_text_rid,
+	    sc->mtl_text_res);
+	bus_release_resource(dev, SYS_RES_MEMORY, sc->mtl_pixel_rid,
+	    sc->mtl_pixel_res);
+	bus_release_resource(dev, SYS_RES_MEMORY, sc->mtl_reg_rid,
+	    sc->mtl_reg_res);
+	return (0);
+}
+
+static device_method_t terasic_mtl_fdt_methods[] = {
+	DEVMETHOD(device_probe,		terasic_mtl_fdt_probe),
+	DEVMETHOD(device_attach,	terasic_mtl_fdt_attach),
+	DEVMETHOD(device_detach,	terasic_mtl_fdt_detach),
+	{ 0, 0 }
+};
+
+static driver_t terasic_mtl_fdt_driver = {
+	"terasic_mtl",
+	terasic_mtl_fdt_methods,
+	sizeof(struct terasic_mtl_softc),
+};
+
+DRIVER_MODULE(mtl, simplebus, terasic_mtl_fdt_driver, terasic_mtl_devclass, 0,
+    0);
diff --git a/sys/dev/terasic/mtl/terasic_mtl_nexus.c b/sys/dev/terasic/mtl/terasic_mtl_nexus.c
index 4a6f2ba..2dff7db 100644
--- a/sys/dev/terasic/mtl/terasic_mtl_nexus.c
+++ b/sys/dev/terasic/mtl/terasic_mtl_nexus.c
@@ -190,7 +190,5 @@ static driver_t terasic_mtl_nexus_driver = {
 	sizeof(struct terasic_mtl_softc),
 };
 
-static devclass_t terasic_mtl_devclass;
-
 DRIVER_MODULE(mtl, nexus, terasic_mtl_nexus_driver, terasic_mtl_devclass, 0,
     0);
diff --git a/sys/dev/uart/uart_bus.h b/sys/dev/uart/uart_bus.h
index 01752d0..322e9a8 100644
--- a/sys/dev/uart/uart_bus.h
+++ b/sys/dev/uart/uart_bus.h
@@ -138,6 +138,7 @@ extern char uart_driver_name[];
 
 int uart_bus_attach(device_t dev);
 int uart_bus_detach(device_t dev);
+int uart_bus_resume(device_t dev);
 serdev_intr_t *uart_bus_ihand(device_t dev, int ipend);
 int uart_bus_ipend(device_t dev);
 int uart_bus_probe(device_t dev, int regshft, int rclk, int rid, int chan);
diff --git a/sys/dev/uart/uart_bus_acpi.c b/sys/dev/uart/uart_bus_acpi.c
index 756f7f2..ccd213d 100644
--- a/sys/dev/uart/uart_bus_acpi.c
+++ b/sys/dev/uart/uart_bus_acpi.c
@@ -47,6 +47,7 @@ static device_method_t uart_acpi_methods[] = {
 	DEVMETHOD(device_probe,		uart_acpi_probe),
 	DEVMETHOD(device_attach,	uart_bus_attach),
 	DEVMETHOD(device_detach,	uart_bus_detach),
+	DEVMETHOD(device_resume,	uart_bus_resume),
 	{ 0, 0 }
 };
 
diff --git a/sys/dev/uart/uart_bus_isa.c b/sys/dev/uart/uart_bus_isa.c
index 1836663..1fda15f 100644
--- a/sys/dev/uart/uart_bus_isa.c
+++ b/sys/dev/uart/uart_bus_isa.c
@@ -50,6 +50,7 @@ static device_method_t uart_isa_methods[] = {
 	DEVMETHOD(device_probe,		uart_isa_probe),
 	DEVMETHOD(device_attach,	uart_bus_attach),
 	DEVMETHOD(device_detach,	uart_bus_detach),
+	DEVMETHOD(device_resume,	uart_bus_resume),
 	{ 0, 0 }
 };
 
diff --git a/sys/dev/uart/uart_bus_pci.c b/sys/dev/uart/uart_bus_pci.c
index 307ee340..4237fa4 100644
--- a/sys/dev/uart/uart_bus_pci.c
+++ b/sys/dev/uart/uart_bus_pci.c
@@ -51,6 +51,7 @@ static device_method_t uart_pci_methods[] = {
 	DEVMETHOD(device_probe,		uart_pci_probe),
 	DEVMETHOD(device_attach,	uart_bus_attach),
 	DEVMETHOD(device_detach,	uart_bus_detach),
+	DEVMETHOD(device_resume,	uart_bus_resume),
 	{ 0, 0 }
 };
 
diff --git a/sys/dev/uart/uart_core.c b/sys/dev/uart/uart_core.c
index 47d5f3d..f5a9a1c 100644
--- a/sys/dev/uart/uart_core.c
+++ b/sys/dev/uart/uart_core.c
@@ -590,3 +590,12 @@ uart_bus_detach(device_t dev)
 
 	return (0);
 }
+
+int
+uart_bus_resume(device_t dev)
+{
+	struct uart_softc *sc;
+
+	sc = device_get_softc(dev);
+	return (UART_ATTACH(sc));
+}
diff --git a/sys/dev/uart/uart_dev_ns8250.c b/sys/dev/uart/uart_dev_ns8250.c
index 5d56a68..eb36da1 100644
--- a/sys/dev/uart/uart_dev_ns8250.c
+++ b/sys/dev/uart/uart_dev_ns8250.c
@@ -31,6 +31,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/bus.h>
 #include <sys/conf.h>
+#include <sys/kernel.h>
+#include <sys/sysctl.h>
 #include <machine/bus.h>
 
 #include <dev/uart/uart.h>
@@ -845,6 +847,11 @@ ns8250_bus_setsig(struct uart_softc *sc, int sig)
 	return (0);
 }
 
+static int broken_txfifo = 0;
+SYSCTL_INT(_hw, OID_AUTO, broken_txfifo, CTLFLAG_RW | CTLFLAG_TUN,
+	&broken_txfifo, 0, "UART FIFO has QEMU emulation bug");
+TUNABLE_INT("hw.broken_txfifo", &broken_txfifo);
+
 static int
 ns8250_bus_transmit(struct uart_softc *sc)
 {
@@ -862,7 +869,12 @@ ns8250_bus_transmit(struct uart_softc *sc)
 		uart_setreg(bas, REG_DATA, sc->sc_txbuf[i]);
 		uart_barrier(bas);
 	}
-	sc->sc_txbusy = 1;
+	if (broken_txfifo)
+		ns8250_drain(bas, UART_DRAIN_TRANSMITTER);
+	else
+		sc->sc_txbusy = 1;
 	uart_unlock(sc->sc_hwmtx);
+	if (broken_txfifo)
+		uart_sched_softih(sc, SER_INT_TXIDLE);
 	return (0);
 }
diff --git a/sys/dev/usb/controller/at91dci.c b/sys/dev/usb/controller/at91dci.c
index 55688d9..6ebd37c 100644
--- a/sys/dev/usb/controller/at91dci.c
+++ b/sys/dev/usb/controller/at91dci.c
@@ -1,6 +1,4 @@
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2007-2008 Hans Petter Selasky. All rights reserved.
  *
@@ -44,6 +42,9 @@ __FBSDID("$FreeBSD$");
  * endpoints, Function-address and more.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -79,6 +80,8 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
+
 #include <dev/usb/controller/at91dci.h>
 
 #define	AT9100_DCI_BUS2SC(bus) \
@@ -1737,18 +1740,12 @@ static const struct usb_hub_descriptor_min at91dci_hubd = {
 	.DeviceRemovable = {0},		/* port is removable */
 };
 
-#define	STRING_LANG \
-  0x09, 0x04,				/* American English */
-
 #define	STRING_VENDOR \
-  'A', 0, 'T', 0, 'M', 0, 'E', 0, 'L', 0
+  "A\0T\0M\0E\0L"
 
 #define	STRING_PRODUCT \
-  'D', 0, 'C', 0, 'I', 0, ' ', 0, 'R', 0, \
-  'o', 0, 'o', 0, 't', 0, ' ', 0, 'H', 0, \
-  'U', 0, 'B', 0,
+  "D\0C\0I\0 \0R\0o\0o\0t\0 \0H\0U\0B"
 
-USB_MAKE_STRING_DESC(STRING_LANG, at91dci_langtab);
 USB_MAKE_STRING_DESC(STRING_VENDOR, at91dci_vendor);
 USB_MAKE_STRING_DESC(STRING_PRODUCT, at91dci_product);
 
@@ -1950,8 +1947,8 @@ tr_handle_get_descriptor:
 	case UDESC_STRING:
 		switch (value & 0xff) {
 		case 0:		/* Language table */
-			len = sizeof(at91dci_langtab);
-			ptr = (const void *)&at91dci_langtab;
+			len = sizeof(usb_string_lang_en);
+			ptr = (const void *)&usb_string_lang_en;
 			goto tr_valid;
 
 		case 1:		/* Vendor */
diff --git a/sys/dev/usb/controller/atmegadci.c b/sys/dev/usb/controller/atmegadci.c
index 132fd48..0dc726c 100644
--- a/sys/dev/usb/controller/atmegadci.c
+++ b/sys/dev/usb/controller/atmegadci.c
@@ -1,6 +1,4 @@
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2009 Hans Petter Selasky. All rights reserved.
  *
@@ -36,6 +34,9 @@ __FBSDID("$FreeBSD$");
  * endpoints, Function-address and more.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -71,6 +72,8 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
+
 #include <dev/usb/controller/atmegadci.h>
 
 #define	ATMEGA_BUS2SC(bus) \
@@ -1562,18 +1565,12 @@ static const struct usb_hub_descriptor_min atmegadci_hubd = {
 	.DeviceRemovable = {0},		/* port is removable */
 };
 
-#define	STRING_LANG \
-  0x09, 0x04,				/* American English */
-
 #define	STRING_VENDOR \
-  'A', 0, 'T', 0, 'M', 0, 'E', 0, 'G', 0, 'A', 0
+  "A\0T\0M\0E\0G\0A"
 
 #define	STRING_PRODUCT \
-  'D', 0, 'C', 0, 'I', 0, ' ', 0, 'R', 0, \
-  'o', 0, 'o', 0, 't', 0, ' ', 0, 'H', 0, \
-  'U', 0, 'B', 0,
+  "D\0C\0I\0 \0R\0o\0o\0t\0 \0H\0U\0B"
 
-USB_MAKE_STRING_DESC(STRING_LANG, atmegadci_langtab);
 USB_MAKE_STRING_DESC(STRING_VENDOR, atmegadci_vendor);
 USB_MAKE_STRING_DESC(STRING_PRODUCT, atmegadci_product);
 
@@ -1776,8 +1773,8 @@ tr_handle_get_descriptor:
 	case UDESC_STRING:
 		switch (value & 0xff) {
 		case 0:		/* Language table */
-			len = sizeof(atmegadci_langtab);
-			ptr = (const void *)&atmegadci_langtab;
+			len = sizeof(usb_string_lang_en);
+			ptr = (const void *)&usb_string_lang_en;
 			goto tr_valid;
 
 		case 1:		/* Vendor */
diff --git a/sys/dev/usb/controller/avr32dci.c b/sys/dev/usb/controller/avr32dci.c
index 14b4f64..1d2ce87 100644
--- a/sys/dev/usb/controller/avr32dci.c
+++ b/sys/dev/usb/controller/avr32dci.c
@@ -1,6 +1,4 @@
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2009 Hans Petter Selasky. All rights reserved.
  *
@@ -35,7 +33,9 @@ __FBSDID("$FreeBSD$");
  * NOTE: When the chip detects BUS-reset it will also reset the
  * endpoints, Function-address and more.
  */
-
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -71,6 +71,8 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
+
 #include <dev/usb/controller/avr32dci.h>
 
 #define	AVR32_BUS2SC(bus) \
@@ -1503,18 +1505,12 @@ static const struct usb_hub_descriptor_min avr32dci_hubd = {
 	.DeviceRemovable = {0},		/* port is removable */
 };
 
-#define	STRING_LANG \
-  0x09, 0x04,				/* American English */
-
 #define	STRING_VENDOR \
-  'A', 0, 'V', 0, 'R', 0, '3', 0, '2', 0
+  "A\0V\0R\0003\0002"
 
 #define	STRING_PRODUCT \
-  'D', 0, 'C', 0, 'I', 0, ' ', 0, 'R', 0, \
-  'o', 0, 'o', 0, 't', 0, ' ', 0, 'H', 0, \
-  'U', 0, 'B', 0,
+  "D\0C\0I\0 \0R\0o\0o\0t\0 \0H\0U\0B"
 
-USB_MAKE_STRING_DESC(STRING_LANG, avr32dci_langtab);
 USB_MAKE_STRING_DESC(STRING_VENDOR, avr32dci_vendor);
 USB_MAKE_STRING_DESC(STRING_PRODUCT, avr32dci_product);
 
@@ -1717,8 +1713,8 @@ tr_handle_get_descriptor:
 	case UDESC_STRING:
 		switch (value & 0xff) {
 		case 0:		/* Language table */
-			len = sizeof(avr32dci_langtab);
-			ptr = (const void *)&avr32dci_langtab;
+			len = sizeof(usb_string_lang_en);
+			ptr = (const void *)&usb_string_lang_en;
 			goto tr_valid;
 
 		case 1:		/* Vendor */
diff --git a/sys/dev/usb/controller/dwc_otg.c b/sys/dev/usb/controller/dwc_otg.c
index 89e312a..d08b3d9 100644
--- a/sys/dev/usb/controller/dwc_otg.c
+++ b/sys/dev/usb/controller/dwc_otg.c
@@ -1,3 +1,4 @@
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2012 Hans Petter Selasky. All rights reserved.
  * Copyright (c) 2010-2011 Aleksandr Rybalko. All rights reserved.
@@ -41,9 +42,9 @@
  * internal reset.
  */
 
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -79,6 +80,7 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 #include <dev/usb/controller/dwc_otg.h>
 #include <dev/usb/controller/dwc_otgreg.h>
@@ -3489,18 +3491,12 @@ static const struct usb_hub_descriptor_min dwc_otg_hubd = {
 	.DeviceRemovable = {0},		/* port is removable */
 };
 
-#define	STRING_LANG \
-  0x09, 0x04,				/* American English */
-
 #define	STRING_VENDOR \
-  'D', 0, 'W', 0, 'C', 0, 'O', 0, 'T', 0, 'G', 0
+  "D\0W\0C\0O\0T\0G"
 
 #define	STRING_PRODUCT \
-  'O', 0, 'T', 0, 'G', 0, ' ', 0, 'R', 0, \
-  'o', 0, 'o', 0, 't', 0, ' ', 0, 'H', 0, \
-  'U', 0, 'B', 0,
+  "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
 
-USB_MAKE_STRING_DESC(STRING_LANG, dwc_otg_langtab);
 USB_MAKE_STRING_DESC(STRING_VENDOR, dwc_otg_vendor);
 USB_MAKE_STRING_DESC(STRING_PRODUCT, dwc_otg_product);
 
@@ -3702,8 +3698,8 @@ tr_handle_get_descriptor:
 	case UDESC_STRING:
 		switch (value & 0xff) {
 		case 0:		/* Language table */
-			len = sizeof(dwc_otg_langtab);
-			ptr = (const void *)&dwc_otg_langtab;
+			len = sizeof(usb_string_lang_en);
+			ptr = (const void *)&usb_string_lang_en;
 			goto tr_valid;
 
 		case 1:		/* Vendor */
diff --git a/sys/dev/usb/controller/ehci.c b/sys/dev/usb/controller/ehci.c
index b301ea4..51b012b 100644
--- a/sys/dev/usb/controller/ehci.c
+++ b/sys/dev/usb/controller/ehci.c
@@ -1,3 +1,4 @@
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
  * Copyright (c) 2004 The NetBSD Foundation, Inc. All rights reserved.
@@ -43,9 +44,9 @@
  * 1) command failures are not recovered correctly
  */
 
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -81,6 +82,8 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
+
 #include <dev/usb/controller/ehci.h>
 #include <dev/usb/controller/ehcireg.h>
 
diff --git a/sys/dev/usb/controller/musb_otg.c b/sys/dev/usb/controller/musb_otg.c
index 627b40b..10dc90b 100644
--- a/sys/dev/usb/controller/musb_otg.c
+++ b/sys/dev/usb/controller/musb_otg.c
@@ -36,6 +36,9 @@
  * NOTE: The current implementation only supports Device Side Mode!
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -71,6 +74,8 @@
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
+
 #include <dev/usb/controller/musb_otg.h>
 
 #define	MUSBOTG_INTR_ENDPT 1
@@ -2206,19 +2211,12 @@ static const struct usb_hub_descriptor_min musbotg_hubd = {
 	.DeviceRemovable = {0},		/* port is removable */
 };
 
-#define	STRING_LANG \
-  0x09, 0x04,				/* American English */
-
 #define	STRING_VENDOR \
-  'M', 0, 'e', 0, 'n', 0, 't', 0, 'o', 0, 'r', 0, ' ', 0, \
-  'G', 0, 'r', 0, 'a', 0, 'p', 0, 'h', 0, 'i', 0, 'c', 0, 's', 0
+  "M\0e\0n\0t\0o\0r\0 \0G\0r\0a\0p\0h\0i\0c\0s"
 
 #define	STRING_PRODUCT \
-  'O', 0, 'T', 0, 'G', 0, ' ', 0, 'R', 0, \
-  'o', 0, 'o', 0, 't', 0, ' ', 0, 'H', 0, \
-  'U', 0, 'B', 0,
+  "O\0T\0G\0 \0R\0o\0o\0t\0 \0H\0U\0B"
 
-USB_MAKE_STRING_DESC(STRING_LANG, musbotg_langtab);
 USB_MAKE_STRING_DESC(STRING_VENDOR, musbotg_vendor);
 USB_MAKE_STRING_DESC(STRING_PRODUCT, musbotg_product);
 
@@ -2420,8 +2418,8 @@ tr_handle_get_descriptor:
 	case UDESC_STRING:
 		switch (value & 0xff) {
 		case 0:		/* Language table */
-			len = sizeof(musbotg_langtab);
-			ptr = (const void *)&musbotg_langtab;
+			len = sizeof(usb_string_lang_en);
+			ptr = (const void *)&usb_string_lang_en;
 			goto tr_valid;
 
 		case 1:		/* Vendor */
diff --git a/sys/dev/usb/controller/ohci.c b/sys/dev/usb/controller/ohci.c
index d499506..e78c9e9 100644
--- a/sys/dev/usb/controller/ohci.c
+++ b/sys/dev/usb/controller/ohci.c
@@ -1,3 +1,4 @@
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
  * Copyright (c) 1998 The NetBSD Foundation, Inc. All rights reserved.
@@ -25,9 +26,6 @@
  * SUCH DAMAGE.
  */
 
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
 /*
  * USB Open Host Controller driver.
  *
@@ -35,6 +33,9 @@ __FBSDID("$FreeBSD$");
  * USB spec:  http://www.usb.org/developers/docs/usbspec.zip
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -70,6 +71,8 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
+
 #include <dev/usb/controller/ohci.h>
 #include <dev/usb/controller/ohcireg.h>
 
diff --git a/sys/dev/usb/controller/uhci.c b/sys/dev/usb/controller/uhci.c
index 78399cc..fe3cf93 100644
--- a/sys/dev/usb/controller/uhci.c
+++ b/sys/dev/usb/controller/uhci.c
@@ -1,3 +1,4 @@
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
  * Copyright (c) 1998 The NetBSD Foundation, Inc. All rights reserved.
@@ -25,9 +26,6 @@
  * SUCH DAMAGE.
  */
 
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
 /*
  * USB Universal Host Controller driver.
  * Handles e.g. PIIX3 and PIIX4.
@@ -38,6 +36,9 @@ __FBSDID("$FreeBSD$");
  *             ftp://download.intel.com/design/intarch/datashts/29056201.pdf
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -73,6 +74,8 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
+
 #include <dev/usb/controller/uhci.h>
 #include <dev/usb/controller/uhcireg.h>
 
diff --git a/sys/dev/usb/controller/usb_controller.c b/sys/dev/usb/controller/usb_controller.c
index 5f28640..204569f 100644
--- a/sys/dev/usb/controller/usb_controller.c
+++ b/sys/dev/usb/controller/usb_controller.c
@@ -24,6 +24,9 @@
  * SUCH DAMAGE.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include "opt_ddb.h"
 
 #include <sys/stdint.h>
@@ -62,6 +65,7 @@
 #include <dev/usb/usb_bus.h>
 #include <dev/usb/usb_pf.h>
 #include "usb_if.h"
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 /* function prototypes  */
 
@@ -84,10 +88,12 @@ SYSCTL_INT(_hw_usb_ctrl, OID_AUTO, debug, CTLFLAG_RW, &usb_ctrl_debug, 0,
     "Debug level");
 #endif
 
+#if USB_HAVE_ROOT_MOUNT_HOLD
 static int usb_no_boot_wait = 0;
 TUNABLE_INT("hw.usb.no_boot_wait", &usb_no_boot_wait);
 SYSCTL_INT(_hw_usb, OID_AUTO, no_boot_wait, CTLFLAG_RD|CTLFLAG_TUN, &usb_no_boot_wait, 0,
     "No USB device enumerate waiting at boot.");
+#endif
 
 static int usb_no_suspend_wait = 0;
 TUNABLE_INT("hw.usb.no_suspend_wait", &usb_no_suspend_wait);
@@ -108,7 +114,8 @@ static device_method_t usb_methods[] = {
 	DEVMETHOD(device_suspend, usb_suspend),
 	DEVMETHOD(device_resume, usb_resume),
 	DEVMETHOD(device_shutdown, usb_shutdown),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t usb_driver = {
@@ -144,6 +151,7 @@ usb_probe(device_t dev)
 	return (0);
 }
 
+#if USB_HAVE_ROOT_MOUNT_HOLD
 static void
 usb_root_mount_rel(struct usb_bus *bus)
 {
@@ -153,6 +161,7 @@ usb_root_mount_rel(struct usb_bus *bus)
 		bus->bus_roothold = NULL;
 	}
 }
+#endif
 
 /*------------------------------------------------------------------------*
  *	usb_attach
@@ -169,11 +178,12 @@ usb_attach(device_t dev)
 		return (ENXIO);
 	}
 
+#if USB_HAVE_ROOT_MOUNT_HOLD
 	if (usb_no_boot_wait == 0) {
 		/* delay vfs_mountroot until the bus is explored */
 		bus->bus_roothold = root_mount_hold(device_get_nameunit(dev));
 	}
-
+#endif
 	usb_attach_sub(dev, bus);
 
 	return (0);			/* return success */
@@ -196,8 +206,10 @@ usb_detach(device_t dev)
 	/* Stop power watchdog */
 	usb_callout_drain(&bus->power_wdog);
 
+#if USB_HAVE_ROOT_MOUNT_HOLD
 	/* Let the USB explore process detach all devices. */
 	usb_root_mount_rel(bus);
+#endif
 
 	USB_BUS_LOCK(bus);
 
@@ -363,7 +375,9 @@ usb_bus_explore(struct usb_proc_msg *pm)
 		(udev->hub->explore) (udev);
 		USB_BUS_LOCK(bus);
 	}
+#if USB_HAVE_ROOT_MOUNT_HOLD
 	usb_root_mount_rel(bus);
+#endif
 }
 
 /*------------------------------------------------------------------------*
@@ -630,7 +644,9 @@ usb_bus_attach(struct usb_proc_msg *pm)
 
 	default:
 		device_printf(bus->bdev, "Unsupported USB revision\n");
+#if USB_HAVE_ROOT_MOUNT_HOLD
 		usb_root_mount_rel(bus);
+#endif
 		return;
 	}
 
@@ -672,7 +688,9 @@ usb_bus_attach(struct usb_proc_msg *pm)
 	if (err) {
 		device_printf(bus->bdev, "Root HUB problem, error=%s\n",
 		    usbd_errstr(err));
+#if USB_HAVE_ROOT_MOUNT_HOLD
 		usb_root_mount_rel(bus);
+#endif
 	}
 
 	/* set softc - we are ready */
diff --git a/sys/dev/usb/controller/uss820dci.c b/sys/dev/usb/controller/uss820dci.c
index bbb9f75..6750f32 100644
--- a/sys/dev/usb/controller/uss820dci.c
+++ b/sys/dev/usb/controller/uss820dci.c
@@ -32,6 +32,9 @@
  * NOTE: The datasheet does not document everything.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -67,6 +70,8 @@
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
+
 #include <dev/usb/controller/uss820dci.h>
 
 #define	USS820_DCI_BUS2SC(bus) \
@@ -1803,18 +1808,12 @@ static const struct usb_hub_descriptor_min uss820dci_hubd = {
 	.DeviceRemovable = {0},		/* port is removable */
 };
 
-#define	STRING_LANG \
-  0x09, 0x04,				/* American English */
-
 #define	STRING_VENDOR \
-  'A', 0, 'G', 0, 'E', 0, 'R', 0, 'E', 0
+  "A\0G\0E\0R\0E"
 
 #define	STRING_PRODUCT \
-  'D', 0, 'C', 0, 'I', 0, ' ', 0, 'R', 0, \
-  'o', 0, 'o', 0, 't', 0, ' ', 0, 'H', 0, \
-  'U', 0, 'B', 0,
+  "D\0C\0I\0 \0R\0o\0o\0t\0 \0H\0U\0B"
 
-USB_MAKE_STRING_DESC(STRING_LANG, uss820dci_langtab);
 USB_MAKE_STRING_DESC(STRING_VENDOR, uss820dci_vendor);
 USB_MAKE_STRING_DESC(STRING_PRODUCT, uss820dci_product);
 
@@ -2016,8 +2015,8 @@ tr_handle_get_descriptor:
 	case UDESC_STRING:
 		switch (value & 0xff) {
 		case 0:		/* Language table */
-			len = sizeof(uss820dci_langtab);
-			ptr = (const void *)&uss820dci_langtab;
+			len = sizeof(usb_string_lang_en);
+			ptr = (const void *)&usb_string_lang_en;
 			goto tr_valid;
 
 		case 1:		/* Vendor */
diff --git a/sys/dev/usb/controller/xhci.c b/sys/dev/usb/controller/xhci.c
index c62f781..4023ecc 100644
--- a/sys/dev/usb/controller/xhci.c
+++ b/sys/dev/usb/controller/xhci.c
@@ -1,3 +1,4 @@
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2010 Hans Petter Selasky. All rights reserved.
  *
@@ -38,9 +39,9 @@
  * way we avoid too much diveration among USB drivers.
  */
 
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -76,6 +77,8 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
+
 #include <dev/usb/controller/xhci.h>
 #include <dev/usb/controller/xhcireg.h>
 
@@ -886,6 +889,12 @@ xhci_check_transfer(struct xhci_softc *sc, struct xhci_trb *trb)
 			 * a short packet also makes the transfer done
 			 */
 			if (td->remainder > 0) {
+				if (td->alt_next == NULL) {
+					DPRINTF(
+					    "short TD has no alternate next\n");
+					xhci_generic_done(xfer);
+					break;
+				}
 				DPRINTF("TD has short pkt\n");
 				if (xfer->flags_int.short_frames_ok ||
 				    xfer->flags_int.isochronous_xfr ||
diff --git a/sys/dev/usb/input/uhid.c b/sys/dev/usb/input/uhid.c
index 333d437..48d9c4f 100644
--- a/sys/dev/usb/input/uhid.c
+++ b/sys/dev/usb/input/uhid.c
@@ -852,7 +852,8 @@ static device_method_t uhid_methods[] = {
 	DEVMETHOD(device_probe, uhid_probe),
 	DEVMETHOD(device_attach, uhid_attach),
 	DEVMETHOD(device_detach, uhid_detach),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t uhid_driver = {
diff --git a/sys/dev/usb/input/ukbd.c b/sys/dev/usb/input/ukbd.c
index 45ac8fc..0c56ce0 100644
--- a/sys/dev/usb/input/ukbd.c
+++ b/sys/dev/usb/input/ukbd.c
@@ -995,13 +995,12 @@ ukbd_probe(device_t dev)
 	if (uaa->info.bInterfaceClass != UICLASS_HID)
 		return (ENXIO);
 
+	if (usb_test_quirk(uaa, UQ_KBD_IGNORE))
+		return (ENXIO);
+
 	if ((uaa->info.bInterfaceSubClass == UISUBCLASS_BOOT) &&
-	    (uaa->info.bInterfaceProtocol == UIPROTO_BOOT_KEYBOARD)) {
-		if (usb_test_quirk(uaa, UQ_KBD_IGNORE))
-			return (ENXIO);
-		else
-			return (BUS_PROBE_DEFAULT);
-	}
+	    (uaa->info.bInterfaceProtocol == UIPROTO_BOOT_KEYBOARD))
+		return (BUS_PROBE_DEFAULT);
 
 	error = usbd_req_get_hid_desc(uaa->device, NULL,
 	    &d_ptr, &d_len, M_TEMP, uaa->info.bIfaceIndex);
@@ -1009,23 +1008,20 @@ ukbd_probe(device_t dev)
 	if (error)
 		return (ENXIO);
 
-	/*
-	 * NOTE: we currently don't support USB mouse and USB keyboard
-	 * on the same USB endpoint.
-	 */
-	if (hid_is_collection(d_ptr, d_len,
-	    HID_USAGE2(HUP_GENERIC_DESKTOP, HUG_MOUSE))) {
-		/* most likely a mouse */
-		error = ENXIO;
-	} else if (hid_is_collection(d_ptr, d_len,
-	    HID_USAGE2(HUP_GENERIC_DESKTOP, HUG_KEYBOARD))) {
-		if (usb_test_quirk(uaa, UQ_KBD_IGNORE))
+	if (hid_is_keyboard(d_ptr, d_len)) {
+		if (hid_is_mouse(d_ptr, d_len)) {
+			/*
+			 * NOTE: We currently don't support USB mouse
+			 * and USB keyboard on the same USB endpoint.
+			 * Let "ums" driver win.
+			 */
 			error = ENXIO;
-		else
+		} else {
 			error = BUS_PROBE_DEFAULT;
-	} else
+		}
+	} else {
 		error = ENXIO;
-
+	}
 	free(d_ptr, M_TEMP);
 	return (error);
 }
@@ -2124,7 +2120,8 @@ static device_method_t ukbd_methods[] = {
 	DEVMETHOD(device_attach, ukbd_attach),
 	DEVMETHOD(device_detach, ukbd_detach),
 	DEVMETHOD(device_resume, ukbd_resume),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t ukbd_driver = {
diff --git a/sys/dev/usb/input/ums.c b/sys/dev/usb/input/ums.c
index c20c986..cc52b99 100644
--- a/sys/dev/usb/input/ums.c
+++ b/sys/dev/usb/input/ums.c
@@ -368,9 +368,7 @@ ums_probe(device_t dev)
 {
 	struct usb_attach_arg *uaa = device_get_ivars(dev);
 	void *d_ptr;
-	struct hid_data *hd;
-	struct hid_item hi;
-	int error, mdepth, found;
+	int error;
 	uint16_t d_len;
 
 	DPRINTFN(11, "\n");
@@ -394,44 +392,13 @@ ums_probe(device_t dev)
 	if (error)
 		return (ENXIO);
 
-	hd = hid_start_parse(d_ptr, d_len, 1 << hid_input);
-	if (hd == NULL)
-		return (0);
-	mdepth = 0;
-	found = 0;
-	while (hid_get_item(hd, &hi)) {
-		switch (hi.kind) {
-		case hid_collection:
-			if (mdepth != 0)
-				mdepth++;
-			else if (hi.collection == 1 &&
-			     hi.usage ==
-			      HID_USAGE2(HUP_GENERIC_DESKTOP, HUG_MOUSE))
-				mdepth++;
-			break;
-		case hid_endcollection:
-			if (mdepth != 0)
-				mdepth--;
-			break;
-		case hid_input:
-			if (mdepth == 0)
-				break;
-			if (hi.usage ==
-			     HID_USAGE2(HUP_GENERIC_DESKTOP, HUG_X) &&
-			    (hi.flags & MOUSE_FLAGS_MASK) == MOUSE_FLAGS)
-				found++;
-			if (hi.usage ==
-			     HID_USAGE2(HUP_GENERIC_DESKTOP, HUG_Y) &&
-			    (hi.flags & MOUSE_FLAGS_MASK) == MOUSE_FLAGS)
-				found++;
-			break;
-		default:
-			break;
-		}
-	}
-	hid_end_parse(hd);
+	if (hid_is_mouse(d_ptr, d_len))
+		error = BUS_PROBE_DEFAULT;
+	else
+		error = ENXIO;
+
 	free(d_ptr, M_TEMP);
-	return (found ? BUS_PROBE_DEFAULT : ENXIO);
+	return (error);
 }
 
 static void
@@ -1072,7 +1039,8 @@ static device_method_t ums_methods[] = {
 	DEVMETHOD(device_probe, ums_probe),
 	DEVMETHOD(device_attach, ums_attach),
 	DEVMETHOD(device_detach, ums_detach),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t ums_driver = {
diff --git a/sys/dev/usb/misc/udbp.c b/sys/dev/usb/misc/udbp.c
index 56f86ae..8b949ef 100644
--- a/sys/dev/usb/misc/udbp.c
+++ b/sys/dev/usb/misc/udbp.c
@@ -248,7 +248,8 @@ static device_method_t udbp_methods[] = {
 	DEVMETHOD(device_probe, udbp_probe),
 	DEVMETHOD(device_attach, udbp_attach),
 	DEVMETHOD(device_detach, udbp_detach),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t udbp_driver = {
diff --git a/sys/dev/usb/misc/ufm.c b/sys/dev/usb/misc/ufm.c
index 032b635..0612dd2 100644
--- a/sys/dev/usb/misc/ufm.c
+++ b/sys/dev/usb/misc/ufm.c
@@ -105,7 +105,8 @@ static device_method_t ufm_methods[] = {
 	DEVMETHOD(device_probe, ufm_probe),
 	DEVMETHOD(device_attach, ufm_attach),
 	DEVMETHOD(device_detach, ufm_detach),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t ufm_driver = {
diff --git a/sys/dev/usb/net/if_axe.c b/sys/dev/usb/net/if_axe.c
index 694e5a5..f0d1044 100644
--- a/sys/dev/usb/net/if_axe.c
+++ b/sys/dev/usb/net/if_axe.c
@@ -159,6 +159,7 @@ static const STRUCT_USB_HOST_ID axe_devs[] = {
 	AXE_DEV(COREGA, FETHER_USB2_TX, 0),
 	AXE_DEV(DLINK, DUBE100, 0),
 	AXE_DEV(DLINK, DUBE100B1, AXE_FLAG_772),
+	AXE_DEV(DLINK, DUBE100C1, AXE_FLAG_772B),
 	AXE_DEV(GOODWAY, GWUSB2E, 0),
 	AXE_DEV(IODATA, ETGUS2, AXE_FLAG_178),
 	AXE_DEV(JVC, MP_PRX1, 0),
diff --git a/sys/dev/usb/net/if_cdce.c b/sys/dev/usb/net/if_cdce.c
index 6e0ce97..6649bc0 100644
--- a/sys/dev/usb/net/if_cdce.c
+++ b/sys/dev/usb/net/if_cdce.c
@@ -237,7 +237,7 @@ static device_method_t cdce_methods[] = {
 	DEVMETHOD(device_suspend, cdce_suspend),
 	DEVMETHOD(device_resume, cdce_resume),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t cdce_driver = {
@@ -500,6 +500,7 @@ cdce_attach(device_t dev)
 	const struct usb_interface_descriptor *id;
 	const struct usb_cdc_ethernet_descriptor *ued;
 	const struct usb_config *pcfg;
+	uint32_t seed;
 	int error;
 	uint8_t i;
 	uint8_t data_iface_no;
@@ -612,8 +613,9 @@ alloc_transfers:
 		/* fake MAC address */
 
 		device_printf(dev, "faking MAC address\n");
+		seed = ticks;
 		sc->sc_ue.ue_eaddr[0] = 0x2a;
-		memcpy(&sc->sc_ue.ue_eaddr[1], &ticks, sizeof(uint32_t));
+		memcpy(&sc->sc_ue.ue_eaddr[1], &seed, sizeof(uint32_t));
 		sc->sc_ue.ue_eaddr[5] = device_get_unit(dev);
 
 	} else {
diff --git a/sys/dev/usb/net/if_cue.c b/sys/dev/usb/net/if_cue.c
index 30c5247a..6414e7a 100644
--- a/sys/dev/usb/net/if_cue.c
+++ b/sys/dev/usb/net/if_cue.c
@@ -157,7 +157,7 @@ static device_method_t cue_methods[] = {
 	DEVMETHOD(device_attach, cue_attach),
 	DEVMETHOD(device_detach, cue_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t cue_driver = {
diff --git a/sys/dev/usb/net/if_ipheth.c b/sys/dev/usb/net/if_ipheth.c
index 28f941c..3eb2582 100644
--- a/sys/dev/usb/net/if_ipheth.c
+++ b/sys/dev/usb/net/if_ipheth.c
@@ -116,7 +116,7 @@ static device_method_t ipheth_methods[] = {
 	DEVMETHOD(device_attach, ipheth_attach),
 	DEVMETHOD(device_detach, ipheth_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t ipheth_driver = {
diff --git a/sys/dev/usb/net/if_kue.c b/sys/dev/usb/net/if_kue.c
index 791c73a..ec0c5f0 100644
--- a/sys/dev/usb/net/if_kue.c
+++ b/sys/dev/usb/net/if_kue.c
@@ -199,7 +199,7 @@ static device_method_t kue_methods[] = {
 	DEVMETHOD(device_attach, kue_attach),
 	DEVMETHOD(device_detach, kue_detach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t kue_driver = {
diff --git a/sys/dev/usb/net/if_smsc.c b/sys/dev/usb/net/if_smsc.c
index 5ce45bc..5c3db72 100644
--- a/sys/dev/usb/net/if_smsc.c
+++ b/sys/dev/usb/net/if_smsc.c
@@ -1009,6 +1009,10 @@ smsc_bulk_read_callback(struct usb_xfer *xfer, usb_error_t error)
 
 				/* Check if RX TCP/UDP checksumming is being offloaded */
 				if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) {
+
+					struct ether_header *eh;
+
+					eh = mtod(m, struct ether_header *);
 				
 					/* Remove the extra 2 bytes of the csum */
 					pktlen -= 2;
@@ -1020,8 +1024,10 @@ smsc_bulk_read_callback(struct usb_xfer *xfer, usb_error_t error)
 					 * the padding bytes as well. Therefore to be safe we
 					 * ignore the H/W csum on frames less than or equal to
 					 * 64 bytes.
+					 *
+					 * Ignore H/W csum for non-IPv4 packets.
 					 */
-					if (pktlen > ETHER_MIN_LEN) {
+					if (be16toh(eh->ether_type) == ETHERTYPE_IP && pktlen > ETHER_MIN_LEN) {
 					
 						/* Indicate the UDP/TCP csum has been calculated */
 						m->m_pkthdr.csum_flags |= CSUM_DATA_VALID;
@@ -1810,7 +1816,7 @@ static device_method_t smsc_methods[] = {
 	DEVMETHOD(miibus_writereg, smsc_miibus_writereg),
 	DEVMETHOD(miibus_statchg, smsc_miibus_statchg),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t smsc_driver = {
diff --git a/sys/dev/usb/quirk/usb_quirk.c b/sys/dev/usb/quirk/usb_quirk.c
index b8f07aa..52f511d 100644
--- a/sys/dev/usb/quirk/usb_quirk.c
+++ b/sys/dev/usb/quirk/usb_quirk.c
@@ -111,6 +111,7 @@ static struct usb_quirk_entry usb_quirks[USB_DEV_QUIRKS_MAX] = {
 	USB_QUIRK(ITUNERNET, USBLCD2X20, 0x0000, 0xffff, UQ_HID_IGNORE),
 	USB_QUIRK(ITUNERNET, USBLCD4X20, 0x0000, 0xffff, UQ_HID_IGNORE),
 	USB_QUIRK(LIEBERT, POWERSURE_PXT, 0x0000, 0xffff, UQ_HID_IGNORE),
+	USB_QUIRK(LIEBERT2, PSI1000, 0x0000, 0xffff, UQ_HID_IGNORE),
 	USB_QUIRK(MGE, UPS1, 0x0000, 0xffff, UQ_HID_IGNORE),
 	USB_QUIRK(MGE, UPS2, 0x0000, 0xffff, UQ_HID_IGNORE),
 	USB_QUIRK(APPLE, IPHONE, 0x0000, 0xffff, UQ_HID_IGNORE),
@@ -302,7 +303,7 @@ static struct usb_quirk_entry usb_quirks[USB_DEV_QUIRKS_MAX] = {
 	    UQ_MSC_FORCE_PROTO_SCSI, UQ_MSC_NO_INQUIRY),
 	USB_QUIRK(ONSPEC, READER, 0x0000, 0xffff, UQ_MSC_FORCE_PROTO_SCSI),
 	USB_QUIRK(ONSPEC, UCF100, 0x0000, 0xffff, UQ_MSC_FORCE_WIRE_BBB,
-	    UQ_MSC_FORCE_PROTO_ATAPI, UQ_MSC_NO_INQUIRY | UQ_MSC_NO_GETMAXLUN),
+	    UQ_MSC_FORCE_PROTO_ATAPI, UQ_MSC_NO_INQUIRY, UQ_MSC_NO_GETMAXLUN),
 	USB_QUIRK(ONSPEC2, IMAGEMATE_SDDR55, 0x0000, 0xffff,
 	    UQ_MSC_FORCE_PROTO_SCSI, UQ_MSC_NO_GETMAXLUN),
 	USB_QUIRK(PANASONIC, KXL840AN, 0x0000, 0xffff, UQ_MSC_FORCE_WIRE_BBB,
@@ -338,6 +339,8 @@ static struct usb_quirk_entry usb_quirks[USB_DEV_QUIRKS_MAX] = {
 	    UQ_MSC_FORCE_PROTO_SCSI, UQ_MSC_IGNORE_RESIDUE),
 	USB_QUIRK(SANDISK, SDDR31, 0x0000, 0xffff, UQ_MSC_FORCE_WIRE_BBB,
 	    UQ_MSC_FORCE_PROTO_SCSI, UQ_MSC_READ_CAP_OFFBY1),
+	USB_QUIRK(SANDISK, IMAGEMATE_SDDR289, 0x0000, 0xffff,
+	    UQ_MSC_NO_SYNC_CACHE, UQ_MSC_NO_GETMAXLUN),
 	USB_QUIRK(SCANLOGIC, SL11R, 0x0000, 0xffff, UQ_MSC_FORCE_WIRE_BBB,
 	    UQ_MSC_FORCE_PROTO_ATAPI, UQ_MSC_NO_INQUIRY),
 	USB_QUIRK(SHUTTLE, EUSB, 0x0000, 0xffff, UQ_MSC_FORCE_WIRE_CBI_I,
@@ -799,7 +802,7 @@ usb_quirk_ioctl(unsigned long cmd, caddr_t data,
 		}
 		if (x == USB_SUB_QUIRKS_MAX) {
 			/* all quirk entries are unused - release */
-			memset(pqe, 0, sizeof(pqe));
+			memset(pqe, 0, sizeof(*pqe));
 		}
 		mtx_unlock(&usb_quirk_mtx);
 		return (0);		/* success */
diff --git a/sys/dev/usb/serial/u3g.c b/sys/dev/usb/serial/u3g.c
index dc670d66..f2c3a45 100644
--- a/sys/dev/usb/serial/u3g.c
+++ b/sys/dev/usb/serial/u3g.c
@@ -356,6 +356,7 @@ static const STRUCT_USB_HOST_ID u3g_devs[] = {
 	U3G_DEV(NOVATEL, V740, 0),
 	U3G_DEV(NOVATEL, X950D, 0),
 	U3G_DEV(NOVATEL, XU870, 0),
+	U3G_DEV(MOTOROLA2, MB886, U3GINIT_SCSIEJECT),
 	U3G_DEV(OPTION, E6500, 0),
 	U3G_DEV(OPTION, E6501, 0),
 	U3G_DEV(OPTION, E6601, 0),
diff --git a/sys/dev/usb/storage/umass.c b/sys/dev/usb/storage/umass.c
index 48ae389..e2ff0ef 100644
--- a/sys/dev/usb/storage/umass.c
+++ b/sys/dev/usb/storage/umass.c
@@ -698,7 +698,8 @@ static device_method_t umass_methods[] = {
 	DEVMETHOD(device_probe, umass_probe),
 	DEVMETHOD(device_attach, umass_attach),
 	DEVMETHOD(device_detach, umass_detach),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t umass_driver = {
@@ -2251,8 +2252,11 @@ umass_cam_action(struct cam_sim *sim, union ccb *ccb)
 							/*ascq*/ 0x00,
 							/*extra args*/ SSD_ELEM_NONE);
 						ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
-						ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR |
-						    CAM_AUTOSNS_VALID;
+						ccb->ccb_h.status =
+						    CAM_SCSI_STATUS_ERROR |
+						    CAM_AUTOSNS_VALID |
+						    CAM_DEV_QFRZN;
+						xpt_freeze_devq(ccb->ccb_h.path, 1);
 						xpt_done(ccb);
 						goto done;
 					}
@@ -2512,7 +2516,8 @@ umass_cam_cb(struct umass_softc *sc, union ccb *ccb, uint32_t residue,
 		 * recovered. We return an error to CAM and let CAM
 		 * retry the command if necessary.
 		 */
-		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
+		xpt_freeze_devq(ccb->ccb_h.path, 1);
+		ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 		xpt_done(ccb);
 		break;
 	}
@@ -2575,8 +2580,9 @@ umass_cam_sense_cb(struct umass_softc *sc, union ccb *ccb, uint32_t residue,
 			 * usual.
 			 */
 
+			xpt_freeze_devq(ccb->ccb_h.path, 1);
 			ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR
-			    | CAM_AUTOSNS_VALID;
+			    | CAM_AUTOSNS_VALID | CAM_DEV_QFRZN;
 			ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
 
 #if 0
@@ -2587,18 +2593,23 @@ umass_cam_sense_cb(struct umass_softc *sc, union ccb *ccb, uint32_t residue,
 
 			/* the rest of the command was filled in at attach */
 
-			if (umass_std_transform(sc, ccb,
+			if ((sc->sc_transform)(sc,
 			    &sc->cam_scsi_test_unit_ready.opcode,
-			    sizeof(sc->cam_scsi_test_unit_ready))) {
+			    sizeof(sc->cam_scsi_test_unit_ready)) == 1) {
 				umass_command_start(sc, DIR_NONE, NULL, 0,
 				    ccb->ccb_h.timeout,
 				    &umass_cam_quirk_cb, ccb);
+				break;
 			}
-			break;
 		} else {
-			ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR
-			    | CAM_AUTOSNS_VALID;
-			ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
+			xpt_freeze_devq(ccb->ccb_h.path, 1);
+			if (key >= 0) {
+				ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR
+				    | CAM_AUTOSNS_VALID | CAM_DEV_QFRZN;
+				ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
+			} else
+				ccb->ccb_h.status = CAM_AUTOSENSE_FAIL
+				    | CAM_DEV_QFRZN;
 		}
 		xpt_done(ccb);
 		break;
@@ -2606,15 +2617,16 @@ umass_cam_sense_cb(struct umass_softc *sc, union ccb *ccb, uint32_t residue,
 	default:
 		DPRINTF(sc, UDMASS_SCSI, "Autosense failed, "
 		    "status %d\n", status);
-		ccb->ccb_h.status = CAM_AUTOSENSE_FAIL;
+		xpt_freeze_devq(ccb->ccb_h.path, 1);
+		ccb->ccb_h.status = CAM_AUTOSENSE_FAIL | CAM_DEV_QFRZN;
 		xpt_done(ccb);
 	}
 }
 
 /*
  * This completion code just handles the fact that we sent a test-unit-ready
- * after having previously failed a READ CAPACITY with CHECK_COND.  Even
- * though this command succeeded, we have to tell CAM to retry.
+ * after having previously failed a READ CAPACITY with CHECK_COND.  The CCB
+ * status for CAM is already set earlier.
  */
 static void
 umass_cam_quirk_cb(struct umass_softc *sc, union ccb *ccb, uint32_t residue,
@@ -2623,9 +2635,6 @@ umass_cam_quirk_cb(struct umass_softc *sc, union ccb *ccb, uint32_t residue,
 	DPRINTF(sc, UDMASS_SCSI, "Test unit ready "
 	    "returned status %d\n", status);
 
-	ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR
-	    | CAM_AUTOSNS_VALID;
-	ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
 	xpt_done(ccb);
 }
 
@@ -2914,7 +2923,8 @@ umass_std_transform(struct umass_softc *sc, union ccb *ccb,
 		xpt_done(ccb);
 		return (0);
 	} else if (retval == 0) {
-		ccb->ccb_h.status = CAM_REQ_INVALID;
+		xpt_freeze_devq(ccb->ccb_h.path, 1);
+		ccb->ccb_h.status = CAM_REQ_INVALID | CAM_DEV_QFRZN;
 		xpt_done(ccb);
 		return (0);
 	}
diff --git a/sys/dev/usb/storage/urio.c b/sys/dev/usb/storage/urio.c
index 39fcbe9..6cbb596 100644
--- a/sys/dev/usb/storage/urio.c
+++ b/sys/dev/usb/storage/urio.c
@@ -185,7 +185,8 @@ static device_method_t urio_methods[] = {
 	DEVMETHOD(device_probe, urio_probe),
 	DEVMETHOD(device_attach, urio_attach),
 	DEVMETHOD(device_detach, urio_detach),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t urio_driver = {
diff --git a/sys/dev/usb/storage/ustorage_fs.c b/sys/dev/usb/storage/ustorage_fs.c
index df89619..15a5ba9 100644
--- a/sys/dev/usb/storage/ustorage_fs.c
+++ b/sys/dev/usb/storage/ustorage_fs.c
@@ -36,6 +36,9 @@
  * Linux USB gadget stack.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -62,6 +65,7 @@
 
 #define	USB_DEBUG_VAR ustorage_fs_debug
 #include <dev/usb/usb_debug.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 #ifdef USB_DEBUG
 static int ustorage_fs_debug = 0;
@@ -252,7 +256,7 @@ static device_method_t ustorage_fs_methods[] = {
 	DEVMETHOD(device_suspend, ustorage_fs_suspend),
 	DEVMETHOD(device_resume, ustorage_fs_resume),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t ustorage_fs_driver = {
diff --git a/sys/dev/usb/template/usb_template.c b/sys/dev/usb/template/usb_template.c
index 9934202..36c11d9 100644
--- a/sys/dev/usb/template/usb_template.c
+++ b/sys/dev/usb/template/usb_template.c
@@ -29,6 +29,9 @@
  * USB templates.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -67,6 +70,7 @@
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
 #include <dev/usb/template/usb_template.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 MODULE_DEPEND(usb_template, usb, 1, 1, 1);
 MODULE_VERSION(usb_template, 1);
diff --git a/sys/dev/usb/template/usb_template_audio.c b/sys/dev/usb/template/usb_template_audio.c
index 8e9e7f0..7c7cebe 100644
--- a/sys/dev/usb/template/usb_template_audio.c
+++ b/sys/dev/usb/template/usb_template_audio.c
@@ -1,6 +1,4 @@
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2010 Hans Petter Selasky. All rights reserved.
  *
@@ -30,6 +28,9 @@ __FBSDID("$FreeBSD$");
  * This file contains the USB template for an USB Audio Device.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -51,9 +52,11 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
+#include <dev/usb/usb_core.h>
 #include <dev/usb/usb_cdc.h>
 
 #include <dev/usb/template/usb_template.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 enum {
 	INDEX_AUDIO_LANG,
@@ -64,30 +67,21 @@ enum {
 	INDEX_AUDIO_MAX,
 };
 
-#define	STRING_LANG \
-  0x09, 0x04,				/* American English */
-
 #define	STRING_AUDIO_PRODUCT \
-  'A', 0, 'u', 0, 'd', 0, 'i', 0, 'o', 0, ' ', 0, \
-  'T', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, \
-  'D', 0, 'e', 0, 'v', 0, 'i', 0, 'c', 0, 'e', 0, ' ', 0, 
+  "A\0u\0d\0i\0o\0 \0T\0e\0s\0t\0 \0D\0e\0v\0i\0c\0e"
 
 #define	STRING_AUDIO_MIXER \
-  'M', 0, 'i', 0, 'x', 0, 'e', 0, 'r', 0, ' ', 0, \
-  'i', 0, 'n', 0, 't', 0, 'e', 0, 'r', 0, 'f', 0, 'a', 0, 'c', 0, 'e', 0,
+  "M\0i\0x\0e\0r\0 \0i\0n\0t\0e\0r\0f\0a\0c\0e"
 
 #define	STRING_AUDIO_RECORD \
-  'R', 0, 'e', 0, 'c', 0, 'o', 0, 'r', 0, 'd', 0, ' ', 0, \
-  'i', 0, 'n', 0, 't', 0, 'e', 0, 'r', 0, 'f', 0, 'a', 0, 'c', 0, 'e', 0,
+  "R\0e\0c\0o\0r\0d\0 \0i\0n\0t\0e\0r\0f\0a\0c\0e"
 
 #define	STRING_AUDIO_PLAYBACK \
-  'P', 0, 'l', 0, 'a', 0, 'y', 0, 'b', 0, 'a', 0, 'c', 0, 'k', 0, ' ', 0, \
-  'i', 0, 'n', 0, 't', 0, 'e', 0, 'r', 0, 'f', 0, 'a', 0, 'c', 0, 'e', 0,
+  "P\0l\0a\0y\0b\0a\0c\0k\0 \0i\0n\0t\0e\0r\0f\0a\0c\0e"
 
 
 /* make the real string descriptors */
 
-USB_MAKE_STRING_DESC(STRING_LANG, string_lang);
 USB_MAKE_STRING_DESC(STRING_AUDIO_MIXER, string_audio_mixer);
 USB_MAKE_STRING_DESC(STRING_AUDIO_RECORD, string_audio_record);
 USB_MAKE_STRING_DESC(STRING_AUDIO_PLAYBACK, string_audio_playback);
@@ -385,7 +379,7 @@ static const void *
 audio_get_string_desc(uint16_t lang_id, uint8_t string_index)
 {
 	static const void *ptr[INDEX_AUDIO_MAX] = {
-		[INDEX_AUDIO_LANG] = &string_lang,
+		[INDEX_AUDIO_LANG] = &usb_string_lang_en,
 		[INDEX_AUDIO_MIXER] = &string_audio_mixer,
 		[INDEX_AUDIO_RECORD] = &string_audio_record,
 		[INDEX_AUDIO_PLAYBACK] = &string_audio_playback,
@@ -393,7 +387,7 @@ audio_get_string_desc(uint16_t lang_id, uint8_t string_index)
 	};
 
 	if (string_index == 0) {
-		return (&string_lang);
+		return (&usb_string_lang_en);
 	}
 	if (lang_id != 0x0409) {
 		return (NULL);
diff --git a/sys/dev/usb/template/usb_template_cdce.c b/sys/dev/usb/template/usb_template_cdce.c
index 481a69c..68f69d0 100644
--- a/sys/dev/usb/template/usb_template_cdce.c
+++ b/sys/dev/usb/template/usb_template_cdce.c
@@ -1,6 +1,4 @@
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2007 Hans Petter Selasky <hselasky@FreeBSD.org>
  * All rights reserved.
@@ -31,6 +29,9 @@ __FBSDID("$FreeBSD$");
  * This file contains the USB templates for a CDC USB ethernet device.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -52,9 +53,11 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
+#include <dev/usb/usb_core.h>
 #include <dev/usb/usb_cdc.h>
 
 #include <dev/usb/template/usb_template.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 enum {
 	STRING_LANG_INDEX,
@@ -68,61 +71,31 @@ enum {
 	STRING_ETH_MAX,
 };
 
-#define	STRING_LANG \
-  0x09, 0x04,				/* American English */
-
 #define	STRING_MAC \
-  '2', 0, 'A', 0, '2', 0, '3', 0, \
-  '4', 0, '5', 0, '6', 0, '7', 0, \
-  '8', 0, '9', 0, 'A', 0, 'B', 0,
+  "2\0A\0002\0003\0004\0005\0006\0007\08\09\0A\0B"
 
 #define	STRING_ETH_CONTROL \
-  'U', 0, 'S', 0, 'B', 0, ' ', 0, \
-  'E', 0, 't', 0, 'h', 0, 'e', 0, \
-  'r', 0, 'n', 0, 'e', 0, 't', 0, \
-  ' ', 0, 'C', 0, 'o', 0, 'm', 0, \
-  'm', 0, ' ', 0, 'i', 0, 'n', 0, \
-  't', 0, 'e', 0, 'r', 0, 'f', 0, \
-  'a', 0, 'c', 0, 'e', 0,
+  "U\0S\0B\0 \0E\0t\0h\0e\0r\0n\0e\0t\0 " \
+  "\0C\0o\0m\0m\0 \0I\0n\0t\0e\0r\0f\0a\0c\0e"
 
 #define	STRING_ETH_DATA \
-  'U', 0, 'S', 0, 'B', 0, ' ', 0, \
-  'E', 0, 't', 0, 'h', 0, 'e', 0, \
-  'r', 0, 'n', 0, 'e', 0, 't', 0, \
-  ' ', 0, 'D', 0, 'a', 0, 't', 0, \
-  'a', 0, ' ', 0, 'i', 0, 'n', 0, \
-  't', 0, 'e', 0, 'r', 0, 'f', 0, \
-  'a', 0, 'c', 0, 'e', 0,
+  "U\0S\0B\0 \0E\0t\0h\0e\0r\0n\0e\0t\0 \0D\0a\0t\0a\0 " \
+  "\0I\0n\0t\0e\0r\0f\0a\0c\0e"
 
 #define	STRING_ETH_CONFIG \
-  'D', 0, 'e', 0, 'f', 0, 'a', 0, \
-  'u', 0, 'l', 0, 't', 0, ' ', 0, \
-  'c', 0, 'o', 0, 'n', 0, 'f', 0, \
-  'i', 0, 'g', 0,
+  "D\0e\0f\0a\0u\0l\0t\0 \0c\0o\0n\0f\0i\0g"
 
 #define	STRING_ETH_VENDOR \
-  'F', 0, 'r', 0, 'e', 0, 'e', 0, \
-  'B', 0, 'S', 0, 'D', 0, ' ', 0, \
-  'f', 0, 'o', 0, 'u', 0, 'n', 0, \
-  'd', 0, 'a', 0, 't', 0, 'i', 0, \
-  'o', 0, 'n', 0,
+  "F\0r\0e\0e\0B\0S\0D\0 \0f\0o\0u\0n\0d\0a\0t\0i\0o\0n"
 
 #define	STRING_ETH_PRODUCT \
-  'U', 0, 'S', 0, 'B', 0, ' ', 0, \
-  'E', 0, 't', 0, 'h', 0, 'e', 0, \
-  'r', 0, 'n', 0, 'e', 0, 't', 0, \
-  ' ', 0, 'A', 0, 'd', 0, 'a', 0, \
-  'p', 0, 't', 0, 'e', 0, 'r', 0,
+  "U\0S\0B\0 \0E\0t\0h\0e\0r\0n\0e\0t\0 \0A\0d\0a\0p\0t\0e\0r"
 
 #define	STRING_ETH_SERIAL \
-  'D', 0, 'e', 0, 'c', 0, 'e', 0, \
-  'm', 0, 'b', 0, 'e', 0, 'r', 0, \
-  ' ', 0, '2', 0, '0', 0, '0', 0, \
-  '7', 0,
+  "D\0e\0c\0e\0m\0b\0e\0r\0 \0002\0000\0000\0007"
 
 /* make the real string descriptors */
 
-USB_MAKE_STRING_DESC(STRING_LANG, string_lang);
 USB_MAKE_STRING_DESC(STRING_MAC, string_mac);
 USB_MAKE_STRING_DESC(STRING_ETH_CONTROL, string_eth_control);
 USB_MAKE_STRING_DESC(STRING_ETH_DATA, string_eth_data);
@@ -286,7 +259,7 @@ static const void *
 eth_get_string_desc(uint16_t lang_id, uint8_t string_index)
 {
 	static const void *ptr[STRING_ETH_MAX] = {
-		[STRING_LANG_INDEX] = &string_lang,
+		[STRING_LANG_INDEX] = &usb_string_lang_en,
 		[STRING_MAC_INDEX] = &string_mac,
 		[STRING_ETH_CONTROL_INDEX] = &string_eth_control,
 		[STRING_ETH_DATA_INDEX] = &string_eth_data,
@@ -297,7 +270,7 @@ eth_get_string_desc(uint16_t lang_id, uint8_t string_index)
 	};
 
 	if (string_index == 0) {
-		return (&string_lang);
+		return (&usb_string_lang_en);
 	}
 	if (lang_id != 0x0409) {
 		return (NULL);
diff --git a/sys/dev/usb/template/usb_template_kbd.c b/sys/dev/usb/template/usb_template_kbd.c
index 6295825..306b2fe 100644
--- a/sys/dev/usb/template/usb_template_kbd.c
+++ b/sys/dev/usb/template/usb_template_kbd.c
@@ -1,6 +1,4 @@
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2010 Hans Petter Selasky. All rights reserved.
  *
@@ -30,6 +28,9 @@ __FBSDID("$FreeBSD$");
  * This file contains the USB template for an USB Keyboard Device.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -51,9 +52,11 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
+#include <dev/usb/usb_core.h>
 #include <dev/usb/usb_cdc.h>
 
 #include <dev/usb/template/usb_template.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 enum {
 	INDEX_LANG,
@@ -62,21 +65,14 @@ enum {
 	INDEX_MAX,
 };
 
-#define	STRING_LANG \
-  0x09, 0x04,				/* American English */
-
 #define	STRING_PRODUCT \
-  'K', 0, 'e', 0, 'y', 0, 'b', 0, 'o', 0, 'a', 0, 'r', 0, 'd', 0, ' ', 0, \
-  'T', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, \
-  'D', 0, 'e', 0, 'v', 0, 'i', 0, 'c', 0, 'e', 0, ' ', 0, 
+  "K\0e\0y\0b\0o\0a\0r\0d\0 \0T\0e\0s\0t\0 \0D\0e\0v\0i\0c\0e"
 
 #define	STRING_KEYBOARD \
-  'K', 0, 'e', 0, 'y', 0, 'b', 0, 'o', 0, 'a', 0, 'r', 0, 'd', 0, ' ', 0, \
-  'i', 0, 'n', 0, 't', 0, 'e', 0, 'r', 0, 'f', 0, 'a', 0, 'c', 0, 'e', 0,
+  "K\0e\0y\0b\0o\0a\0r\0d\0 \0i\0n\0t\0e\0r\0f\0a\0c\0e"
 
 /* make the real string descriptors */
 
-USB_MAKE_STRING_DESC(STRING_LANG, string_lang);
 USB_MAKE_STRING_DESC(STRING_KEYBOARD, string_keyboard);
 USB_MAKE_STRING_DESC(STRING_PRODUCT, string_product);
 
@@ -206,13 +202,13 @@ static const void *
 keyboard_get_string_desc(uint16_t lang_id, uint8_t string_index)
 {
 	static const void *ptr[INDEX_MAX] = {
-		[INDEX_LANG] = &string_lang,
+		[INDEX_LANG] = &usb_string_lang_en,
 		[INDEX_KEYBOARD] = &string_keyboard,
 		[INDEX_PRODUCT] = &string_product,
 	};
 
 	if (string_index == 0) {
-		return (&string_lang);
+		return (&usb_string_lang_en);
 	}
 	if (lang_id != 0x0409) {
 		return (NULL);
diff --git a/sys/dev/usb/template/usb_template_modem.c b/sys/dev/usb/template/usb_template_modem.c
index abc6f26..6a976f7 100644
--- a/sys/dev/usb/template/usb_template_modem.c
+++ b/sys/dev/usb/template/usb_template_modem.c
@@ -1,6 +1,4 @@
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2010 Hans Petter Selasky. All rights reserved.
  *
@@ -30,6 +28,9 @@ __FBSDID("$FreeBSD$");
  * This file contains the USB template for an USB Modem Device.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -51,9 +52,11 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
+#include <dev/usb/usb_core.h>
 #include <dev/usb/usb_cdc.h>
 
 #include <dev/usb/template/usb_template.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 enum {
 	INDEX_LANG,
@@ -62,21 +65,14 @@ enum {
 	INDEX_MAX,
 };
 
-#define	STRING_LANG \
-  0x09, 0x04,				/* American English */
-
 #define	STRING_PRODUCT \
-  'M', 0, 'o', 0, 'd', 0, 'e', 0, 'm', 0, ' ', 0, \
-  'T', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, \
-  'D', 0, 'e', 0, 'v', 0, 'i', 0, 'c', 0, 'e', 0, ' ', 0, 
+  "M\0o\0d\0e\0m\0 \0T\0e\0s\0t\0 \0D\0e\0v\0i\0c\0e"
 
 #define	STRING_MODEM \
-  'M', 0, 'o', 0, 'd', 0, 'e', 0, 'm', 0, ' ', 0, \
-  'i', 0, 'n', 0, 't', 0, 'e', 0, 'r', 0, 'f', 0, 'a', 0, 'c', 0, 'e', 0,
+  "M\0o\0d\0e\0m\0 \0i\0n\0t\0e\0r\0f\0a\0c\0e"
 
 /* make the real string descriptors */
 
-USB_MAKE_STRING_DESC(STRING_LANG, string_lang);
 USB_MAKE_STRING_DESC(STRING_MODEM, string_modem);
 USB_MAKE_STRING_DESC(STRING_PRODUCT, string_product);
 
@@ -234,13 +230,13 @@ static const void *
 modem_get_string_desc(uint16_t lang_id, uint8_t string_index)
 {
 	static const void *ptr[INDEX_MAX] = {
-		[INDEX_LANG] = &string_lang,
+		[INDEX_LANG] = &usb_string_lang_en,
 		[INDEX_MODEM] = &string_modem,
 		[INDEX_PRODUCT] = &string_product,
 	};
 
 	if (string_index == 0) {
-		return (&string_lang);
+		return (&usb_string_lang_en);
 	}
 	if (lang_id != 0x0409) {
 		return (NULL);
diff --git a/sys/dev/usb/template/usb_template_mouse.c b/sys/dev/usb/template/usb_template_mouse.c
index 628c9a5..a48ccf4 100644
--- a/sys/dev/usb/template/usb_template_mouse.c
+++ b/sys/dev/usb/template/usb_template_mouse.c
@@ -1,6 +1,4 @@
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2010 Hans Petter Selasky. All rights reserved.
  *
@@ -30,6 +28,9 @@ __FBSDID("$FreeBSD$");
  * This file contains the USB template for an USB Mouse Device.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -51,9 +52,11 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
+#include <dev/usb/usb_core.h>
 #include <dev/usb/usb_cdc.h>
 
 #include <dev/usb/template/usb_template.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 enum {
 	INDEX_LANG,
@@ -62,21 +65,14 @@ enum {
 	INDEX_MAX,
 };
 
-#define	STRING_LANG \
-  0x09, 0x04,				/* American English */
-
 #define	STRING_PRODUCT \
-  'M', 0, 'o', 0, 'u', 0, 's', 0, 'e', 0, ' ', 0, \
-  'T', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, \
-  'D', 0, 'e', 0, 'v', 0, 'i', 0, 'c', 0, 'e', 0,
+  "M\0o\0u\0s\0e\0 \0T\0e\0s\0t\0 \0D\0e\0v\0i\0c\0e"
 
 #define	STRING_MOUSE \
-  'M', 0, 'o', 0, 'u', 0, 's', 0, 'e', 0, ' ', 0, \
-  'i', 0, 'n', 0, 't', 0, 'e', 0, 'r', 0, 'f', 0, 'a', 0, 'c', 0, 'e', 0,
+  "M\0o\0u\0s\0e\0 \0i\0n\0t\0e\0r\0f\0a\0c\0e"
 
 /* make the real string descriptors */
 
-USB_MAKE_STRING_DESC(STRING_LANG, string_lang);
 USB_MAKE_STRING_DESC(STRING_MOUSE, string_mouse);
 USB_MAKE_STRING_DESC(STRING_PRODUCT, string_product);
 
@@ -204,13 +200,13 @@ static const void *
 mouse_get_string_desc(uint16_t lang_id, uint8_t string_index)
 {
 	static const void *ptr[INDEX_MAX] = {
-		[INDEX_LANG] = &string_lang,
+		[INDEX_LANG] = &usb_string_lang_en,
 		[INDEX_MOUSE] = &string_mouse,
 		[INDEX_PRODUCT] = &string_product,
 	};
 
 	if (string_index == 0) {
-		return (&string_lang);
+		return (&usb_string_lang_en);
 	}
 	if (lang_id != 0x0409) {
 		return (NULL);
diff --git a/sys/dev/usb/template/usb_template_msc.c b/sys/dev/usb/template/usb_template_msc.c
index 5c05ffe..6809e63 100644
--- a/sys/dev/usb/template/usb_template_msc.c
+++ b/sys/dev/usb/template/usb_template_msc.c
@@ -1,6 +1,4 @@
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2008 Hans Petter Selasky <hselasky@FreeBSD.org>
  * All rights reserved.
@@ -31,6 +29,9 @@ __FBSDID("$FreeBSD$");
  * This file contains the USB templates for an USB Mass Storage Device.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -52,8 +53,10 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
+#include <dev/usb/usb_core.h>
 
 #include <dev/usb/template/usb_template.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 enum {
 	STRING_LANG_INDEX,
@@ -65,45 +68,24 @@ enum {
 	STRING_MSC_MAX,
 };
 
-#define	STRING_LANG \
-  0x09, 0x04,				/* American English */
-
 #define	STRING_MSC_DATA	\
-  'U', 0, 'S', 0, 'B', 0, ' ', 0, \
-  'M', 0, 'a', 0, 's', 0, 's', 0, \
-  ' ', 0, 'S', 0, 't', 0, 'o', 0, \
-  'r', 0, 'a', 0, 'g', 0, 'e', 0, \
-  ' ', 0, 'I', 0, 'n', 0, 't', 0, \
-  'e', 0, 'r', 0, 'f', 0, 'a', 0, \
-  'c', 0, 'e', 0,
+  "U\0S\0B\0 \0M\0a\0s\0s\0 \0S\0t\0o\0r\0a\0g\0e\0 " \
+  "\0I\0n\0t\0e\0r\0f\0a\0c\0e"
 
 #define	STRING_MSC_CONFIG \
-  'D', 0, 'e', 0, 'f', 0, 'a', 0, \
-  'u', 0, 'l', 0, 't', 0, ' ', 0, \
-  'c', 0, 'o', 0, 'n', 0, 'f', 0, \
-  'i', 0, 'g', 0,
+  "D\0e\0f\0a\0u\0l\0t\0 \0c\0o\0n\0f\0i\0g"
 
 #define	STRING_MSC_VENDOR \
-  'F', 0, 'r', 0, 'e', 0, 'e', 0, \
-  'B', 0, 'S', 0, 'D', 0, ' ', 0, \
-  'f', 0, 'o', 0, 'u', 0, 'n', 0, \
-  'd', 0, 'a', 0, 't', 0, 'i', 0, \
-  'o', 0, 'n', 0,
+  "F\0r\0e\0e\0B\0S\0D\0 \0f\0o\0u\0n\0d\0a\0t\0i\0o\0n"
 
 #define	STRING_MSC_PRODUCT \
-  'U', 0, 'S', 0, 'B', 0, ' ', 0, \
-  'M', 0, 'e', 0, 'm', 0, 'o', 0, \
-  'r', 0, 'y', 0, ' ', 0, 'S', 0, \
-  't', 0, 'i', 0, 'c', 0, 'k', 0
+  "U\0S\0B\0 \0M\0e\0m\0o\0r\0y\0 \0S\0t\0i\0c\0k"
 
 #define	STRING_MSC_SERIAL \
-  'M', 0, 'a', 0, 'r', 0, 'c', 0, \
-  'h', 0, ' ', 0, '2', 0, '0', 0, \
-  '0', 0, '8', 0,
+  "M\0a\0r\0c\0h\0 \0002\0000\0000\08"
 
 /* make the real string descriptors */
 
-USB_MAKE_STRING_DESC(STRING_LANG, string_lang);
 USB_MAKE_STRING_DESC(STRING_MSC_DATA, string_msc_data);
 USB_MAKE_STRING_DESC(STRING_MSC_CONFIG, string_msc_config);
 USB_MAKE_STRING_DESC(STRING_MSC_VENDOR, string_msc_vendor);
@@ -195,7 +177,7 @@ static const void *
 msc_get_string_desc(uint16_t lang_id, uint8_t string_index)
 {
 	static const void *ptr[STRING_MSC_MAX] = {
-		[STRING_LANG_INDEX] = &string_lang,
+		[STRING_LANG_INDEX] = &usb_string_lang_en,
 		[STRING_MSC_DATA_INDEX] = &string_msc_data,
 		[STRING_MSC_CONFIG_INDEX] = &string_msc_config,
 		[STRING_MSC_VENDOR_INDEX] = &string_msc_vendor,
@@ -204,7 +186,7 @@ msc_get_string_desc(uint16_t lang_id, uint8_t string_index)
 	};
 
 	if (string_index == 0) {
-		return (&string_lang);
+		return (&usb_string_lang_en);
 	}
 	if (lang_id != 0x0409) {
 		return (NULL);
diff --git a/sys/dev/usb/template/usb_template_mtp.c b/sys/dev/usb/template/usb_template_mtp.c
index f48fbf4..f0528a5 100644
--- a/sys/dev/usb/template/usb_template_mtp.c
+++ b/sys/dev/usb/template/usb_template_mtp.c
@@ -1,6 +1,4 @@
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2008 Hans Petter Selasky <hselasky@FreeBSD.org>
  * All rights reserved.
@@ -38,6 +36,9 @@ __FBSDID("$FreeBSD$");
  * operating system the VID and PID of your device.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -59,7 +60,10 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
+#include <dev/usb/usb_core.h>
+
 #include <dev/usb/template/usb_template.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 #define	MTP_BREQUEST 0x08
 
@@ -73,41 +77,23 @@ enum {
 	STRING_MTP_MAX,
 };
 
-#define	STRING_LANG \
-  0x09, 0x04,				/* American English */
-
 #define	STRING_MTP_DATA	\
-  'U', 0, 'S', 0, 'B', 0, ' ', 0, \
-  'M', 0, 'T', 0, 'P', 0, \
-  ' ', 0, 'I', 0, 'n', 0, 't', 0, \
-  'e', 0, 'r', 0, 'f', 0, 'a', 0, \
-  'c', 0, 'e', 0,
+  "U\0S\0B\0 \0M\0T\0P\0 \0I\0n\0t\0e\0r\0f\0a\0c\0e"
 
 #define	STRING_MTP_CONFIG \
-  'D', 0, 'e', 0, 'f', 0, 'a', 0, \
-  'u', 0, 'l', 0, 't', 0, ' ', 0, \
-  'c', 0, 'o', 0, 'n', 0, 'f', 0, \
-  'i', 0, 'g', 0,
+  "D\0e\0f\0a\0u\0l\0t\0 \0c\0o\0n\0f\0i\0g"
 
 #define	STRING_MTP_VENDOR \
-  'F', 0, 'r', 0, 'e', 0, 'e', 0, \
-  'B', 0, 'S', 0, 'D', 0, ' ', 0, \
-  'f', 0, 'o', 0, 'u', 0, 'n', 0, \
-  'd', 0, 'a', 0, 't', 0, 'i', 0, \
-  'o', 0, 'n', 0,
+  "F\0r\0e\0e\0B\0S\0D\0 \0f\0o\0u\0n\0d\0a\0t\0i\0o\0n"
 
 #define	STRING_MTP_PRODUCT \
-  'U', 0, 'S', 0, 'B', 0, ' ', 0, \
-  'M', 0, 'T', 0, 'P', 0,
+  "U\0S\0B\0 \0M\0T\0P"
 
 #define	STRING_MTP_SERIAL \
-  'J', 0, 'u', 0, 'n', 0, 'e', 0, \
-  ' ', 0, '2', 0, '0', 0, '0', 0, \
-  '8', 0,
+  "J\0u\0n\0e\0 \0002\0000\0000\08"
 
 /* make the real string descriptors */
 
-USB_MAKE_STRING_DESC(STRING_LANG, string_lang);
 USB_MAKE_STRING_DESC(STRING_MTP_DATA, string_mtp_data);
 USB_MAKE_STRING_DESC(STRING_MTP_CONFIG, string_mtp_config);
 USB_MAKE_STRING_DESC(STRING_MTP_VENDOR, string_mtp_vendor);
@@ -244,7 +230,7 @@ static const void *
 mtp_get_string_desc(uint16_t lang_id, uint8_t string_index)
 {
 	static const void *ptr[STRING_MTP_MAX] = {
-		[STRING_LANG_INDEX] = &string_lang,
+		[STRING_LANG_INDEX] = &usb_string_lang_en,
 		[STRING_MTP_DATA_INDEX] = &string_mtp_data,
 		[STRING_MTP_CONFIG_INDEX] = &string_mtp_config,
 		[STRING_MTP_VENDOR_INDEX] = &string_mtp_vendor,
@@ -266,7 +252,7 @@ mtp_get_string_desc(uint16_t lang_id, uint8_t string_index)
 		return (dummy_desc);
 	}
 	if (string_index == 0) {
-		return (&string_lang);
+		return (&usb_string_lang_en);
 	}
 	if (lang_id != 0x0409) {
 		return (NULL);
diff --git a/sys/dev/usb/ufm_ioctl.h b/sys/dev/usb/ufm_ioctl.h
index 921b3d4..5a23388 100644
--- a/sys/dev/usb/ufm_ioctl.h
+++ b/sys/dev/usb/ufm_ioctl.h
@@ -1,3 +1,4 @@
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 2001 M. Warner Losh
  * All rights reserved.
@@ -28,7 +29,8 @@
  * its contributors.
  */
 
-/*  $FreeBSD$ */
+#ifndef _UFM_IOCTL_H_
+#define	_UFM_IOCTL_H_
 
 #include <sys/ioccom.h>
 
@@ -37,3 +39,5 @@
 #define	FM_START	_IOWR('U', 202, int)
 #define	FM_STOP		_IOWR('U', 203, int)
 #define	FM_GET_STAT	_IOWR('U', 204, int)
+
+#endif			/* _UFM_IOCTL_H_ */
diff --git a/sys/dev/usb/usb.h b/sys/dev/usb/usb.h
index a47c9d8..b8b52d5 100644
--- a/sys/dev/usb/usb.h
+++ b/sys/dev/usb/usb.h
@@ -40,22 +40,28 @@
 #define	_USB_STANDARD_H_
 
 #if defined(_KERNEL)
+#ifndef USB_GLOBAL_INCLUDE_FILE
 #include "opt_usb.h"
+#endif
 
 /* Declare parent SYSCTL USB node. */
 #ifdef SYSCTL_DECL
 SYSCTL_DECL(_hw_usb);
 #endif
 
+#ifndef USB_GLOBAL_INCLUDE_FILE
 #include <sys/malloc.h>
+#endif
 
 MALLOC_DECLARE(M_USB);
 MALLOC_DECLARE(M_USBDEV);
 MALLOC_DECLARE(M_USBHC);
 #endif /* _KERNEL */
 
+#ifndef USB_GLOBAL_INCLUDE_FILE
 #include <dev/usb/usb_endian.h>
 #include <dev/usb/usb_freebsd.h>
+#endif
 
 #define	USB_STACK_VERSION 2000		/* 2.0 */
 
@@ -560,17 +566,23 @@ struct usb_string_descriptor {
 typedef struct usb_string_descriptor usb_string_descriptor_t;
 
 #define	USB_MAKE_STRING_DESC(m,name)	\
-struct name {				\
+static const struct {			\
   uByte bLength;			\
   uByte bDescriptorType;		\
   uByte bData[sizeof((uint8_t []){m})];	\
-} __packed;				\
-static const struct name name = {	\
-  .bLength = sizeof(struct name),	\
+} __packed name = {			\
+  .bLength = sizeof(name),		\
   .bDescriptorType = UDESC_STRING,	\
   .bData = { m },			\
 }
 
+struct usb_string_lang {
+	uByte bLength;
+	uByte bDescriptorType;
+	uByte bData[2];
+} __packed;
+typedef struct usb_string_lang usb_string_lang_t;
+
 struct usb_hub_descriptor {
 	uByte	bDescLength;
 	uByte	bDescriptorType;
diff --git a/sys/dev/usb/usb_busdma.c b/sys/dev/usb/usb_busdma.c
index f3c4833..1808ab4 100644
--- a/sys/dev/usb/usb_busdma.c
+++ b/sys/dev/usb/usb_busdma.c
@@ -24,6 +24,9 @@
  * SUCH DAMAGE.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -59,6 +62,7 @@
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 #if USB_HAVE_BUSDMA
 static void	usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t);
@@ -440,7 +444,6 @@ usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
 	rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
 	pc->page_offset_buf = rem;
 	pc->page_offset_end += rem;
-	nseg--;
 #ifdef USB_DEBUG
 	if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) {
 		/*
@@ -451,7 +454,7 @@ usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
 		goto done;
 	}
 #endif
-	while (nseg > 0) {
+	while (1) {
 		off += USB_PAGE_SIZE;
 		if (off >= (segs->ds_len + rem)) {
 			/* page crossing */
@@ -459,6 +462,8 @@ usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
 			segs++;
 			off = 0;
 			rem = 0;
+			if (nseg == 0)
+				break;
 		}
 		pg++;
 		pg->physaddr = (segs->ds_addr + off) & ~(USB_PAGE_SIZE - 1);
diff --git a/sys/dev/usb/usb_busdma.h b/sys/dev/usb/usb_busdma.h
index 6b6e403..1050af5 100644
--- a/sys/dev/usb/usb_busdma.h
+++ b/sys/dev/usb/usb_busdma.h
@@ -27,10 +27,12 @@
 #ifndef _USB_BUSDMA_H_
 #define	_USB_BUSDMA_H_
 
+#ifndef USB_GLOBAL_INCLUDE_FILE
 #include <sys/uio.h>
 #include <sys/mbuf.h>
 
 #include <machine/bus.h>
+#endif
 
 /* defines */
 
diff --git a/sys/dev/usb/usb_compat_linux.c b/sys/dev/usb/usb_compat_linux.c
index 22bf1f4..a42f636 100644
--- a/sys/dev/usb/usb_compat_linux.c
+++ b/sys/dev/usb/usb_compat_linux.c
@@ -25,6 +25,9 @@
  * SUCH DAMAGE.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -61,6 +64,7 @@
 #include <dev/usb/usb_hub.h>
 #include <dev/usb/usb_request.h>
 #include <dev/usb/usb_debug.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 struct usb_linux_softc {
 	LIST_ENTRY(usb_linux_softc) sc_attached_list;
@@ -109,7 +113,7 @@ static device_method_t usb_linux_methods[] = {
 	DEVMETHOD(device_suspend, usb_linux_suspend),
 	DEVMETHOD(device_resume, usb_linux_resume),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t usb_linux_driver = {
diff --git a/sys/dev/usb/usb_core.c b/sys/dev/usb/usb_core.c
index 1bd05c8..8a5b93c 100644
--- a/sys/dev/usb/usb_core.c
+++ b/sys/dev/usb/usb_core.c
@@ -30,6 +30,9 @@
  * http://www.usb.org/developers/devclass_docs/
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -51,6 +54,12 @@
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
+
+const struct usb_string_lang usb_string_lang_en = {
+	sizeof(usb_string_lang_en), UDESC_STRING,
+	{ 0x09, 0x04 } /* American English */
+};
 
 MALLOC_DEFINE(M_USB, "USB", "USB");
 MALLOC_DEFINE(M_USBDEV, "USBdev", "USB device");
diff --git a/sys/dev/usb/usb_core.h b/sys/dev/usb/usb_core.h
index 776cf17..afd74ed 100644
--- a/sys/dev/usb/usb_core.h
+++ b/sys/dev/usb/usb_core.h
@@ -69,6 +69,7 @@ struct usb_page;
 struct usb_page_cache;
 struct usb_xfer;
 struct usb_xfer_root;
+struct usb_string_lang;
 
 /* typedefs */
 
@@ -174,6 +175,7 @@ struct usb_xfer {
 /* external variables */
 
 extern struct mtx usb_ref_lock;
+extern const struct usb_string_lang usb_string_lang_en;
 
 /* typedefs */
 
diff --git a/sys/dev/usb/usb_debug.c b/sys/dev/usb/usb_debug.c
index 7ca4daa..3382988 100644
--- a/sys/dev/usb/usb_debug.c
+++ b/sys/dev/usb/usb_debug.c
@@ -24,6 +24,9 @@
  * SUCH DAMAGE.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -55,6 +58,7 @@
 
 #include <ddb/ddb.h>
 #include <ddb/db_sym.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 /*
  * Define this unconditionally in case a kernel module is loaded that
diff --git a/sys/dev/usb/usb_dev.c b/sys/dev/usb/usb_dev.c
index a8adbf6..c4264c5 100644
--- a/sys/dev/usb/usb_dev.c
+++ b/sys/dev/usb/usb_dev.c
@@ -27,6 +27,9 @@
  * usb_dev.c - An abstraction layer for creating devices under /dev/...
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -75,6 +78,7 @@
 #include <sys/syscallsubr.h>
 
 #include <machine/stdarg.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 #if USB_HAVE_UGEN
 
diff --git a/sys/dev/usb/usb_dev.h b/sys/dev/usb/usb_dev.h
index aa9197f..3c75701 100644
--- a/sys/dev/usb/usb_dev.h
+++ b/sys/dev/usb/usb_dev.h
@@ -27,11 +27,13 @@
 #ifndef _USB_DEV_H_
 #define	_USB_DEV_H_
 
+#ifndef USB_GLOBAL_INCLUDE_FILE
 #include <sys/file.h>
 #include <sys/selinfo.h>
 #include <sys/poll.h>
 #include <sys/signalvar.h>
 #include <sys/proc.h>
+#endif
 
 struct usb_fifo;
 struct usb_mbuf;
diff --git a/sys/dev/usb/usb_device.c b/sys/dev/usb/usb_device.c
index 05c0a4c..8d88bf5 100644
--- a/sys/dev/usb/usb_device.c
+++ b/sys/dev/usb/usb_device.c
@@ -24,6 +24,9 @@
  * SUCH DAMAGE.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -78,6 +81,7 @@
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 /* function prototypes  */
 
diff --git a/sys/dev/usb/usb_dynamic.c b/sys/dev/usb/usb_dynamic.c
index 1358b30..f8bb03f 100644
--- a/sys/dev/usb/usb_dynamic.c
+++ b/sys/dev/usb/usb_dynamic.c
@@ -24,6 +24,9 @@
  * SUCH DAMAGE.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -50,6 +53,7 @@
 #include <dev/usb/usb_process.h>
 #include <dev/usb/usb_device.h>
 #include <dev/usb/usb_dynamic.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 /* function prototypes */
 static usb_handle_req_t usb_temp_get_desc_w;
diff --git a/sys/dev/usb/usb_endian.h b/sys/dev/usb/usb_endian.h
index 29479f1..0bbcb9b 100644
--- a/sys/dev/usb/usb_endian.h
+++ b/sys/dev/usb/usb_endian.h
@@ -27,8 +27,10 @@
 #ifndef _USB_ENDIAN_H_
 #define	_USB_ENDIAN_H_
 
+#ifndef USB_GLOBAL_INCLUDE_FILE
 #include <sys/stdint.h>
 #include <sys/endian.h>
+#endif
 
 /*
  * Declare the basic USB record types. USB records have an alignment
diff --git a/sys/dev/usb/usb_error.c b/sys/dev/usb/usb_error.c
index 119c617..5987d81 100644
--- a/sys/dev/usb/usb_error.c
+++ b/sys/dev/usb/usb_error.c
@@ -24,6 +24,9 @@
  * SUCH DAMAGE.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -45,6 +48,7 @@
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 static const char* usb_errstr_table[USB_ERR_MAX] = {
 	[USB_ERR_NORMAL_COMPLETION]	= "USB_ERR_NORMAL_COMPLETION",
diff --git a/sys/dev/usb/usb_freebsd.h b/sys/dev/usb/usb_freebsd.h
index 42a54c5..e93bee7 100644
--- a/sys/dev/usb/usb_freebsd.h
+++ b/sys/dev/usb/usb_freebsd.h
@@ -42,6 +42,8 @@
 #define	USB_HAVE_POWERD 1
 #define	USB_HAVE_MSCTEST 1
 #define	USB_HAVE_PF 1
+#define	USB_HAVE_ROOT_MOUNT_HOLD 1
+#define	USB_HAVE_ID_SECTION 1
 
 #define	USB_TD_GET_PROC(td) (td)->td_proc
 #define	USB_PROC_GET_GID(td) (td)->p_pgid
diff --git a/sys/dev/usb/usb_freebsd_loader.h b/sys/dev/usb/usb_freebsd_loader.h
new file mode 100644
index 0000000..566f922
--- /dev/null
+++ b/sys/dev/usb/usb_freebsd_loader.h
@@ -0,0 +1,84 @@
+/* $FreeBSD$ */
+/*-
+ * Copyright (c) 2013 Hans Petter Selasky. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Including this file is mandatory for all USB related c-files in the loader.
+ */
+
+#ifndef _USB_FREEBSD_LOADER_H_
+#define	_USB_FREEBSD_LOADER_H_
+
+/* Default USB configuration */
+#define	USB_HAVE_UGEN 0
+#define	USB_HAVE_DEVCTL 0
+#define	USB_HAVE_BUSDMA 1
+#define	USB_HAVE_COMPAT_LINUX 0
+#define	USB_HAVE_USER_IO 0
+#define	USB_HAVE_MBUF 0
+#define	USB_HAVE_TT_SUPPORT 1
+#define	USB_HAVE_POWERD 1
+#define	USB_HAVE_MSCTEST 0
+#define	USB_HAVE_PF 0
+#define	USB_HAVE_ROOT_MOUNT_HOLD 0
+#define	USB_HAVE_ID_SECTION 0
+
+#define	USB_TD_GET_PROC(td) (td)->td_proc
+#define	USB_PROC_GET_GID(td) (td)->p_pgid
+
+#if (!defined(USB_HOST_ALIGN)) || (USB_HOST_ALIGN <= 0)
+/* Use default value. */
+#undef USB_HOST_ALIGN
+#define	USB_HOST_ALIGN    8		/* bytes, must be power of two */
+#endif
+/* Sanity check for USB_HOST_ALIGN: Verify power of two. */
+#if ((-USB_HOST_ALIGN) & USB_HOST_ALIGN) != USB_HOST_ALIGN
+#error "USB_HOST_ALIGN is not power of two."
+#endif
+#define	USB_FS_ISOC_UFRAME_MAX 4	/* exclusive unit */
+#define	USB_BUS_MAX 256			/* units */
+#define	USB_MAX_DEVICES 128		/* units */
+#define	USB_IFACE_MAX 32		/* units */
+#define	USB_FIFO_MAX 128		/* units */
+#define	USB_MAX_EP_STREAMS 8		/* units */
+
+#define	USB_MAX_FS_ISOC_FRAMES_PER_XFER (120)	/* units */
+#define	USB_MAX_HS_ISOC_FRAMES_PER_XFER (8*120)	/* units */
+
+#define	USB_HUB_MAX_DEPTH	5
+#define	USB_EP0_BUFSIZE		1024	/* bytes */
+#define	USB_CS_RESET_LIMIT	20	/* failures = 20 * 50 ms = 1sec */
+
+#define	USB_MAX_AUTO_QUIRK	4	/* maximum number of dynamic quirks */
+
+typedef uint32_t usb_timeout_t;		/* milliseconds */
+typedef uint32_t usb_frlength_t;	/* bytes */
+typedef uint32_t usb_frcount_t;		/* units */
+typedef uint32_t usb_size_t;		/* bytes */
+typedef uint32_t usb_ticks_t;		/* system defined */
+typedef uint16_t usb_power_mask_t;	/* see "USB_HW_POWER_XXX" */
+typedef uint16_t usb_stream_t;		/* stream ID */
+
+#endif					/* _USB_FREEBSD_LOADER_H_ */
diff --git a/sys/dev/usb/usb_generic.c b/sys/dev/usb/usb_generic.c
index f8376bf..13f5bd6 100644
--- a/sys/dev/usb/usb_generic.c
+++ b/sys/dev/usb/usb_generic.c
@@ -24,6 +24,9 @@
  * SUCH DAMAGE.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -67,6 +70,7 @@
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 #if USB_HAVE_UGEN
 
diff --git a/sys/dev/usb/usb_handle_request.c b/sys/dev/usb/usb_handle_request.c
index f5b226e..13fff91 100644
--- a/sys/dev/usb/usb_handle_request.c
+++ b/sys/dev/usb/usb_handle_request.c
@@ -24,6 +24,9 @@
  * SUCH DAMAGE.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -61,6 +64,7 @@
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 /* function prototypes */
 
diff --git a/sys/dev/usb/usb_hid.c b/sys/dev/usb/usb_hid.c
index 6552546..e6361a6 100644
--- a/sys/dev/usb/usb_hid.c
+++ b/sys/dev/usb/usb_hid.c
@@ -1,8 +1,5 @@
+/* $FreeBSD$ */
 /*	$NetBSD: hid.c,v 1.17 2001/11/13 06:24:53 lukem Exp $	*/
-
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
 /*-
  * Copyright (c) 1998 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -33,6 +30,9 @@ __FBSDID("$FreeBSD$");
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb_process.h>
 #include <dev/usb/usb_device.h>
 #include <dev/usb/usb_request.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 static void hid_clear_local(struct hid_item *);
 static uint8_t hid_get_byte(struct hid_data *s, const uint16_t wSize);
@@ -845,3 +846,79 @@ usbd_req_get_hid_desc(struct usb_device *udev, struct mtx *mtx,
 	}
 	return (USB_ERR_NORMAL_COMPLETION);
 }
+
+/*------------------------------------------------------------------------*
+ *	hid_is_mouse
+ *
+ * This function will decide if a USB descriptor belongs to a USB mouse.
+ *
+ * Return values:
+ * Zero: Not a USB mouse.
+ * Else: Is a USB mouse.
+ *------------------------------------------------------------------------*/
+int
+hid_is_mouse(const void *d_ptr, uint16_t d_len)
+{
+	struct hid_data *hd;
+	struct hid_item hi;
+	int mdepth;
+	int found;
+
+	hd = hid_start_parse(d_ptr, d_len, 1 << hid_input);
+	if (hd == NULL)
+		return (0);
+
+	mdepth = 0;
+	found = 0;
+
+	while (hid_get_item(hd, &hi)) {
+		switch (hi.kind) {
+		case hid_collection:
+			if (mdepth != 0)
+				mdepth++;
+			else if (hi.collection == 1 &&
+			     hi.usage ==
+			      HID_USAGE2(HUP_GENERIC_DESKTOP, HUG_MOUSE))
+				mdepth++;
+			break;
+		case hid_endcollection:
+			if (mdepth != 0)
+				mdepth--;
+			break;
+		case hid_input:
+			if (mdepth == 0)
+				break;
+			if (hi.usage ==
+			     HID_USAGE2(HUP_GENERIC_DESKTOP, HUG_X) &&
+			    (hi.flags & (HIO_CONST|HIO_RELATIVE)) == HIO_RELATIVE)
+				found++;
+			if (hi.usage ==
+			     HID_USAGE2(HUP_GENERIC_DESKTOP, HUG_Y) &&
+			    (hi.flags & (HIO_CONST|HIO_RELATIVE)) == HIO_RELATIVE)
+				found++;
+			break;
+		default:
+			break;
+		}
+	}
+	hid_end_parse(hd);
+	return (found);
+}
+
+/*------------------------------------------------------------------------*
+ *	hid_is_keyboard
+ *
+ * This function will decide if a USB descriptor belongs to a USB keyboard.
+ *
+ * Return values:
+ * Zero: Not a USB keyboard.
+ * Else: Is a USB keyboard.
+ *------------------------------------------------------------------------*/
+int
+hid_is_keyboard(const void *d_ptr, uint16_t d_len)
+{
+	if (hid_is_collection(d_ptr, d_len,
+	    HID_USAGE2(HUP_GENERIC_DESKTOP, HUG_KEYBOARD)))
+		return (1);
+	return (0);
+}
diff --git a/sys/dev/usb/usb_hub.c b/sys/dev/usb/usb_hub.c
index e3a673d..5e372c8 100644
--- a/sys/dev/usb/usb_hub.c
+++ b/sys/dev/usb/usb_hub.c
@@ -30,6 +30,9 @@
  * USB spec: http://www.usb.org/developers/docs/usbspec.zip 
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -69,6 +72,7 @@
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 #define	UHUB_INTR_INTERVAL 250		/* ms */
 #define	UHUB_N_TRANSFER 1
diff --git a/sys/dev/usb/usb_ioctl.h b/sys/dev/usb/usb_ioctl.h
index 62edb99..1d9b223 100644
--- a/sys/dev/usb/usb_ioctl.h
+++ b/sys/dev/usb/usb_ioctl.h
@@ -29,12 +29,14 @@
 #ifndef _USB_IOCTL_H_
 #define	_USB_IOCTL_H_
 
+#ifndef USB_GLOBAL_INCLUDE_FILE
 #include <sys/ioccom.h>
 
 /* Building "kdump" depends on these includes */
 
 #include <dev/usb/usb_endian.h>
 #include <dev/usb/usb.h>
+#endif
 
 #define	USB_DEVICE_NAME "usbctl"
 #define	USB_DEVICE_DIR "usb"
diff --git a/sys/dev/usb/usb_lookup.c b/sys/dev/usb/usb_lookup.c
index e03f9b6..f3dc551 100644
--- a/sys/dev/usb/usb_lookup.c
+++ b/sys/dev/usb/usb_lookup.c
@@ -24,6 +24,9 @@
  * SUCH DAMAGE.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -47,6 +50,7 @@
 
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 /*------------------------------------------------------------------------*
  *	usbd_lookup_id_by_info
@@ -174,7 +178,7 @@ usbd_lookup_id_by_uaa(const struct usb_device_id *id, usb_size_t sizeof_id,
 #define	MFL_SIZE "0"
 #endif
 
-#ifdef KLD_MODULE
+#if defined(KLD_MODULE) && (USB_HAVE_ID_SECTION != 0)
 static const char __section("bus_autoconf_format") __used usb_id_format[] = {
 
 	/* Declare that three different sections use the same format */
diff --git a/sys/dev/usb/usb_mbuf.c b/sys/dev/usb/usb_mbuf.c
index 65b2a91..ff2f6da 100644
--- a/sys/dev/usb/usb_mbuf.c
+++ b/sys/dev/usb/usb_mbuf.c
@@ -24,6 +24,9 @@
  * SUCH DAMAGE.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -47,6 +50,7 @@
 #include <dev/usb/usbdi.h>
 #include <dev/usb/usb_dev.h>
 #include <dev/usb/usb_mbuf.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 /*------------------------------------------------------------------------*
  *      usb_alloc_mbufs - allocate mbufs to an usbd interface queue
diff --git a/sys/dev/usb/usb_msctest.c b/sys/dev/usb/usb_msctest.c
index c5e8f49..497e382 100644
--- a/sys/dev/usb/usb_msctest.c
+++ b/sys/dev/usb/usb_msctest.c
@@ -32,6 +32,9 @@
  * mass storage quirks for not supported SCSI commands!
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -66,6 +69,7 @@
 #include <dev/usb/usb_request.h>
 #include <dev/usb/usb_util.h>
 #include <dev/usb/quirk/usb_quirk.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 enum {
 	ST_COMMAND,
diff --git a/sys/dev/usb/usb_parse.c b/sys/dev/usb/usb_parse.c
index 66ab5fa..d6e54ad 100644
--- a/sys/dev/usb/usb_parse.c
+++ b/sys/dev/usb/usb_parse.c
@@ -24,6 +24,9 @@
  * SUCH DAMAGE.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -46,7 +49,7 @@
 #include <dev/usb/usb.h>
 #include <dev/usb/usbdi.h>
 #include <dev/usb/usbdi_util.h>
-
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 /*------------------------------------------------------------------------*
  *	usb_desc_foreach
diff --git a/sys/dev/usb/usb_pci.h b/sys/dev/usb/usb_pci.h
index 071dfc3..6eebc55 100644
--- a/sys/dev/usb/usb_pci.h
+++ b/sys/dev/usb/usb_pci.h
@@ -31,9 +31,11 @@
  * We don't want the following files included everywhere, that's why
  * they are in a separate file.
  */
+#ifndef USB_GLOBAL_INCLUDE_FILE
 #include <dev/pci/pcireg.h>
 #include <dev/pci/pcivar.h>
 
 #include <sys/rman.h>
+#endif
 
 #endif					/* _USB_PCI_H_ */
diff --git a/sys/dev/usb/usb_pf.c b/sys/dev/usb/usb_pf.c
index d9c8dac..98705eb 100644
--- a/sys/dev/usb/usb_pf.c
+++ b/sys/dev/usb/usb_pf.c
@@ -1,3 +1,4 @@
+/* $FreeBSD$ */
 /*-
  * Copyright (c) 1990, 1991, 1993
  *	The Regents of the University of California.  All rights reserved.
@@ -32,8 +33,9 @@
  * SUCH DAMAGE.
  */
 
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/param.h>
 #include <sys/kernel.h>
 #include <sys/bus.h>
@@ -59,6 +61,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/usb/usb_bus.h>
 #include <dev/usb/usb_pf.h>
 #include <dev/usb/usb_transfer.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 static void usbpf_init(void);
 static void usbpf_uninit(void);
diff --git a/sys/dev/usb/usb_process.c b/sys/dev/usb/usb_process.c
index c006f51..34314f4 100644
--- a/sys/dev/usb/usb_process.c
+++ b/sys/dev/usb/usb_process.c
@@ -26,6 +26,9 @@
 
 #define	USB_DEBUG_VAR usb_proc_debug
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -55,6 +58,7 @@
 #include <sys/proc.h>
 #include <sys/kthread.h>
 #include <sys/sched.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 #if (__FreeBSD_version < 700000)
 #define	thread_lock(td) mtx_lock_spin(&sched_lock)
diff --git a/sys/dev/usb/usb_process.h b/sys/dev/usb/usb_process.h
index 23cf660..923f645 100644
--- a/sys/dev/usb/usb_process.h
+++ b/sys/dev/usb/usb_process.h
@@ -27,9 +27,11 @@
 #ifndef _USB_PROCESS_H_
 #define	_USB_PROCESS_H_
 
+#ifndef USB_GLOBAL_INCLUDE_FILE
 #include <sys/interrupt.h>
 #include <sys/priority.h>
 #include <sys/runq.h>
+#endif
 
 /* defines */
 #define	USB_PRI_HIGH	PI_SWI(SWI_NET)
diff --git a/sys/dev/usb/usb_request.c b/sys/dev/usb/usb_request.c
index ce50d9b..6a81945 100644
--- a/sys/dev/usb/usb_request.c
+++ b/sys/dev/usb/usb_request.c
@@ -26,6 +26,9 @@
  * SUCH DAMAGE.
  */ 
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -66,6 +69,7 @@
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
 #include <sys/ctype.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 static int usb_no_cs_fail;
 
diff --git a/sys/dev/usb/usb_transfer.c b/sys/dev/usb/usb_transfer.c
index 6e48a3e..397b0b7 100644
--- a/sys/dev/usb/usb_transfer.c
+++ b/sys/dev/usb/usb_transfer.c
@@ -24,6 +24,9 @@
  * SUCH DAMAGE.
  */ 
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -61,6 +64,7 @@
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
 #include <dev/usb/usb_pf.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 struct usb_std_packet_size {
 	struct {
diff --git a/sys/dev/usb/usb_util.c b/sys/dev/usb/usb_util.c
index 4fe79c5..f9fec76 100644
--- a/sys/dev/usb/usb_util.c
+++ b/sys/dev/usb/usb_util.c
@@ -24,6 +24,9 @@
  * SUCH DAMAGE.
  */
 
+#ifdef USB_GLOBAL_INCLUDE_FILE
+#include USB_GLOBAL_INCLUDE_FILE
+#else
 #include <sys/stdint.h>
 #include <sys/stddef.h>
 #include <sys/param.h>
@@ -56,6 +59,7 @@
 
 #include <dev/usb/usb_controller.h>
 #include <dev/usb/usb_bus.h>
+#endif			/* USB_GLOBAL_INCLUDE_FILE */
 
 /*------------------------------------------------------------------------*
  *	device_set_usb_desc
diff --git a/sys/dev/usb/usbdevs b/sys/dev/usb/usbdevs
index 0cd43ef..aff55fd 100644
--- a/sys/dev/usb/usbdevs
+++ b/sys/dev/usb/usbdevs
@@ -581,6 +581,7 @@ vendor CCYU		0x1065	CCYU Technology
 vendor CURITEL		0x106c	Curitel Communications Inc
 vendor SILABS2		0x10a6	SILABS2
 vendor USI		0x10ab	USI
+vendor LIEBERT2		0x10af	Liebert
 vendor PLX		0x10b5	PLX
 vendor ASANTE		0x10bd	Asante
 vendor SILABS		0x10c4	Silicon Labs
@@ -1486,6 +1487,7 @@ product DIGIANSWER ZIGBEE802154	0x000a	ZigBee/802.15.4 MAC
 /* D-Link products */
 /*product DLINK DSBS25		0x0100	DSB-S25 serial*/
 product DLINK DUBE100		0x1a00	10/100 Ethernet
+product DLINK DUBE100C1		0x1a02	DUB-E100 rev C1
 product DLINK DSB650TX4		0x200c	10/100 Ethernet
 product DLINK DWL120E		0x3200	DWL-120 rev E
 product DLINK DWL122		0x3700	DWL-122
@@ -2426,6 +2428,7 @@ product LEXMARK S2450		0x0009	Optra S 2450
 
 /* Liebert products */
 product LIEBERT POWERSURE_PXT	0xffff	PowerSure Personal XT
+product LIEBERT2 PSI1000	0x0004	UPS PSI 1000 FW:08
 
 /* Link Instruments Inc. products */
 product	LINKINSTRUMENTS MSO19	0xf190	Link Instruments MSO-19
@@ -2939,6 +2942,7 @@ product MOTOROLA2 A41XV32X	0x2a22	A41x/V32x Mobile Phones
 product MOTOROLA2 E398		0x4810	E398 Mobile Phone
 product MOTOROLA2 USBLAN	0x600c	USBLAN
 product MOTOROLA2 USBLAN2	0x6027	USBLAN
+product MOTOROLA2 MB886		0x710f	MB886 Mobile Phone (Atria HD)
 product MOTOROLA4 RT2770	0x9031	RT2770
 product MOTOROLA4 RT3070	0x9032	RT3070
 
@@ -3560,6 +3564,7 @@ product SANDISK SDDR75		0x0810	ImageMate SDDR-75
 product SANDISK SDCZ2_256	0x7104	Cruzer Mini 256MB
 product SANDISK SDCZ4_128	0x7112	Cruzer Micro 128MB
 product SANDISK SDCZ4_256	0x7113	Cruzer Micro 256MB
+product SANDISK IMAGEMATE_SDDR289	0xb6ba	ImageMate SDDR-289
 
 /* Sanwa Electric Instrument Co., Ltd. products */
 product SANWA KB_USB2 		0x0701	KB-USB2 multimeter cable
diff --git a/sys/dev/usb/usbdi.h b/sys/dev/usb/usbdi.h
index d753fe1..26f7769 100644
--- a/sys/dev/usb/usbdi.h
+++ b/sys/dev/usb/usbdi.h
@@ -102,7 +102,9 @@ typedef void (usb_fifo_filter_t)(struct usb_fifo *fifo, struct usb_mbuf *m);
 
 
 /* USB events */
+#ifndef USB_GLOBAL_INCLUDE_FILE
 #include <sys/eventhandler.h>
+#endif
 typedef void (*usb_dev_configured_t)(void *, struct usb_device *,
     struct usb_attach_arg *);
 EVENTHANDLER_DECLARE(usb_dev_configured, usb_dev_configured_t);
@@ -239,12 +241,21 @@ struct usb_config {
  * have your driver module automatically loaded in host, device or
  * both modes respectivly:
  */
+#if USB_HAVE_ID_SECTION
 #define	STRUCT_USB_HOST_ID \
     struct usb_device_id __section("usb_host_id")
 #define	STRUCT_USB_DEVICE_ID \
     struct usb_device_id __section("usb_device_id")
 #define	STRUCT_USB_DUAL_ID \
     struct usb_device_id __section("usb_dual_id")
+#else
+#define	STRUCT_USB_HOST_ID \
+    struct usb_device_id
+#define	STRUCT_USB_DEVICE_ID \
+    struct usb_device_id
+#define	STRUCT_USB_DUAL_ID \
+    struct usb_device_id
+#endif			/* USB_HAVE_ID_SECTION */
 
 /*
  * The following structure is used when looking up an USB driver for
diff --git a/sys/dev/usb/usbhid.h b/sys/dev/usb/usbhid.h
index f40232a..28dfede 100644
--- a/sys/dev/usb/usbhid.h
+++ b/sys/dev/usb/usbhid.h
@@ -29,7 +29,9 @@
 #ifndef _USB_HID_H_
 #define	_USB_HID_H_
 
+#ifndef USB_GLOBAL_INCLUDE_FILE
 #include <dev/usb/usb_endian.h>
+#endif
 
 #define	UR_GET_HID_DESCRIPTOR	0x06
 #define	UDESC_HID		0x21
@@ -242,5 +244,7 @@ struct usb_hid_descriptor *hid_get_descriptor_from_usb(
 usb_error_t usbd_req_get_hid_desc(struct usb_device *udev, struct mtx *mtx,
 	    void **descp, uint16_t *sizep, struct malloc_type *mem,
 	    uint8_t iface_index);
+int	hid_is_mouse(const void *d_ptr, uint16_t d_len);
+int	hid_is_keyboard(const void *d_ptr, uint16_t d_len);
 #endif					/* _KERNEL */
 #endif					/* _USB_HID_H_ */
diff --git a/sys/dev/vx/if_vx_eisa.c b/sys/dev/vx/if_vx_eisa.c
index 80fb5c6..24dc2dd 100644
--- a/sys/dev/vx/if_vx_eisa.c
+++ b/sys/dev/vx/if_vx_eisa.c
@@ -177,7 +177,7 @@ static device_method_t vx_eisa_methods[] = {
 	DEVMETHOD(device_probe, vx_eisa_probe),
 	DEVMETHOD(device_attach, vx_eisa_attach),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t vx_eisa_driver = {
diff --git a/sys/dev/vx/if_vx_pci.c b/sys/dev/vx/if_vx_pci.c
index 0412778..38037fa 100644
--- a/sys/dev/vx/if_vx_pci.c
+++ b/sys/dev/vx/if_vx_pci.c
@@ -61,7 +61,7 @@ static device_method_t vx_methods[] = {
 	DEVMETHOD(device_attach, vx_pci_attach),
 	DEVMETHOD(device_shutdown, vx_pci_shutdown),
 
-	{0, 0}
+	DEVMETHOD_END
 };
 
 static driver_t vx_driver = {
diff --git a/sys/dev/vxge/vxge.c b/sys/dev/vxge/vxge.c
index c35d84b..3650843 100644
--- a/sys/dev/vxge/vxge.c
+++ b/sys/dev/vxge/vxge.c
@@ -4188,7 +4188,8 @@ static device_method_t vxge_methods[] = {
 	DEVMETHOD(device_attach, vxge_attach),
 	DEVMETHOD(device_detach, vxge_detach),
 	DEVMETHOD(device_shutdown, vxge_shutdown),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t vxge_driver = {
diff --git a/sys/dev/wbwd/wbwd.c b/sys/dev/wbwd/wbwd.c
index dfd8f3c..484b5bf 100644
--- a/sys/dev/wbwd/wbwd.c
+++ b/sys/dev/wbwd/wbwd.c
@@ -179,6 +179,12 @@ struct winbond_vendor_device_id {
 		.device_rev	= 0x73,
 		.descr		= "Winbond 83627DHG-P",   
 	},
+	{
+		.vendor_id	= 0x5ca3,
+		.device_id	= 0xc3,
+		.device_rev	= 0x33,
+		.descr		= "Nuvoton WPCM450RA0BX",   
+	},
 };
 
 static void
@@ -596,6 +602,9 @@ wb_probe_enable(device_t dev, int probe)
 			goto cleanup;
 		}
 
+		if (dev_id == 0xff && dev_rev == 0xff)
+			goto cleanup;
+
 		for (j = 0; j < sizeof(wb_devs) / sizeof(*wb_devs); j++) {
 			if (wb_devs[j].device_id == dev_id &&
 			    wb_devs[j].device_rev == dev_rev) {
@@ -605,6 +614,16 @@ wb_probe_enable(device_t dev, int probe)
 				break;
 			}
 		}
+
+		if (!found) {
+			if (probe && dev != NULL) {
+				device_set_desc(dev, "Unknown Winbond/Nuvoton model");
+				device_printf(dev, "DevID 0x%02x DevRev 0x%02x, "
+				    "please report this.\n", dev_id, dev_rev);
+			}
+			found++;
+		}
+
 		if (probe && found && bootverbose && dev != NULL)
 			device_printf(dev, "%s EFER 0x%02x ID 0x%02x Rev 0x%02x"
 			     " CR26 0x%02x (probing)\n", device_get_desc(dev),
diff --git a/sys/dev/xen/console/console.c b/sys/dev/xen/console/console.c
index 6a845ca..6281bf2 100644
--- a/sys/dev/xen/console/console.c
+++ b/sys/dev/xen/console/console.c
@@ -410,7 +410,8 @@ static device_method_t xc_methods[] = {
 	DEVMETHOD(device_identify, xc_identify),
 	DEVMETHOD(device_probe, xc_probe),
 	DEVMETHOD(device_attach, xc_attach),
-	{0, 0}
+
+	DEVMETHOD_END
 };
 
 static driver_t xc_driver = {
diff --git a/sys/fs/ext2fs/ext2_alloc.c b/sys/fs/ext2fs/ext2_alloc.c
index a701087..cafd32d 100644
--- a/sys/fs/ext2fs/ext2_alloc.c
+++ b/sys/fs/ext2fs/ext2_alloc.c
@@ -151,11 +151,11 @@ nospace:
 
 static SYSCTL_NODE(_vfs, OID_AUTO, ext2fs, CTLFLAG_RW, 0, "EXT2FS filesystem");
 
-static int doasyncfree = 1;
+static int doasyncfree = 0;
 SYSCTL_INT(_vfs_ext2fs, OID_AUTO, doasyncfree, CTLFLAG_RW, &doasyncfree, 0,
     "Use asychronous writes to update block pointers when freeing blocks");
 
-static int doreallocblks = 1;
+static int doreallocblks = 0;
 SYSCTL_INT(_vfs_ext2fs, OID_AUTO, doreallocblks, CTLFLAG_RW, &doreallocblks, 0, "");
 
 int
@@ -169,7 +169,7 @@ ext2_reallocblks(ap)
 	struct inode *ip;
 	struct vnode *vp;
 	struct buf *sbp, *ebp;
-	int32_t *bap, *sbap, *ebap = 0;
+	uint32_t *bap, *sbap, *ebap = 0;
 	struct ext2mount *ump;
 	struct cluster_save *buflist;
 	struct indir start_ap[NIADDR + 1], end_ap[NIADDR + 1], *idp;
@@ -242,7 +242,7 @@ ext2_reallocblks(ap)
 	} else {
 #ifdef DIAGNOSTIC
 		if (start_ap[start_lvl-1].in_lbn == idp->in_lbn)
-			panic("ext2_reallocblk: start == end");
+			panic("ext2_reallocblks: start == end");
 #endif
 		ssize = len - (idp->in_off + 1);
 		if (bread(vp, idp->in_lbn, (int)fs->e2fs_bsize, NOCRED, &ebp))
@@ -1004,7 +1004,7 @@ ext2_blkfree(ip, bno, size)
         if (isclr(bbp, bno)) {
                 printf("block = %lld, fs = %s\n",
                      (long long)bno, fs->e2fs_fsmnt);
-                panic("blkfree: freeing free block");
+                panic("ext2_blkfree: freeing free block");
         }
         clrbit(bbp, bno);
 	EXT2_LOCK(ump);
@@ -1032,7 +1032,6 @@ ext2_vfree(pvp, ino, mode)
 	struct ext2mount *ump;
 	int error, cg;
 	char * ibp;
-/*	mode_t save_i_mode; */
 
 	pip = VTOI(pvp);
 	fs = pip->i_e2fs;
@@ -1055,7 +1054,7 @@ ext2_vfree(pvp, ino, mode)
 		printf("ino = %llu, fs = %s\n",
 			 (unsigned long long)ino, fs->e2fs_fsmnt);
 		if (fs->e2fs_ronly == 0)
-			panic("ifree: freeing free inode");
+			panic("ext2_vfree: freeing free inode");
 	}
 	clrbit(ibp, ino);
 	EXT2_LOCK(ump);
@@ -1100,7 +1099,7 @@ ext2_mapsearch(struct m_ext2fs *fs, char *bbp, daddr_t bpref)
 		if (loc == NULL) {
 			printf("start = %d, len = %d, fs = %s\n",
 				start, len, fs->e2fs_fsmnt);
-			panic("ext2fs_alloccg: map corrupted");
+			panic("ext2_mapsearch: map corrupted");
 			/* NOTREACHED */
 		}
 	}
diff --git a/sys/fs/ext2fs/ext2_balloc.c b/sys/fs/ext2fs/ext2_balloc.c
index 6e60c6e..1861f69 100644
--- a/sys/fs/ext2fs/ext2_balloc.c
+++ b/sys/fs/ext2fs/ext2_balloc.c
@@ -69,7 +69,7 @@ ext2_balloc(ip, lbn, size, cred, bpp, flags)
 	struct buf *bp, *nbp;
 	struct vnode *vp = ITOV(ip);
 	struct indir indirs[NIADDR + 2];
-	int32_t newb, *bap, pref;
+	uint32_t newb, *bap, pref;
 	int osize, nsize, num, i, error;
 
 	*bpp = NULL;
diff --git a/sys/fs/ext2fs/ext2_dinode.h b/sys/fs/ext2fs/ext2_dinode.h
index 7d97bb8..7d97bb8 100755..100644
--- a/sys/fs/ext2fs/ext2_dinode.h
+++ b/sys/fs/ext2fs/ext2_dinode.h
diff --git a/sys/fs/ext2fs/ext2_dir.h b/sys/fs/ext2fs/ext2_dir.h
index 5e11166..2bad4cd 100755..100644
--- a/sys/fs/ext2fs/ext2_dir.h
+++ b/sys/fs/ext2fs/ext2_dir.h
@@ -37,7 +37,7 @@
 struct	ext2fs_direct {
 	uint32_t e2d_ino;		/* inode number of entry */
 	uint16_t e2d_reclen;		/* length of this record */
-	uint16_t e2d_namlen;		/* length of string in d_name */
+	uint16_t e2d_namlen;		/* length of string in e2d_name */
 	char e2d_name[EXT2FS_MAXNAMLEN];/* name with length<=EXT2FS_MAXNAMLEN */
 };
 /*
@@ -49,7 +49,7 @@ struct	ext2fs_direct {
 struct	ext2fs_direct_2 {
 	uint32_t e2d_ino;		/* inode number of entry */
 	uint16_t e2d_reclen;		/* length of this record */
-	uint8_t e2d_namlen;		/* length of string in d_name */
+	uint8_t e2d_namlen;		/* length of string in e2d_name */
 	uint8_t e2d_type;		/* file type */
 	char e2d_name[EXT2FS_MAXNAMLEN];/* name with length<=EXT2FS_MAXNAMLEN */
 };
diff --git a/sys/fs/ext2fs/ext2_inode.c b/sys/fs/ext2fs/ext2_inode.c
index e7fc9a8..fe9b50d 100644
--- a/sys/fs/ext2fs/ext2_inode.c
+++ b/sys/fs/ext2fs/ext2_inode.c
@@ -119,7 +119,7 @@ ext2_truncate(vp, length, flags, cred, td)
 	int32_t lastblock;
 	struct inode *oip;
 	int32_t bn, lbn, lastiblock[NIADDR], indir_lbn[NIADDR];
-	int32_t oldblks[NDADDR + NIADDR], newblks[NDADDR + NIADDR];
+	uint32_t oldblks[NDADDR + NIADDR], newblks[NDADDR + NIADDR];
 	struct bufobj *bo;
 	struct m_ext2fs *fs;
 	struct buf *bp;
@@ -180,7 +180,7 @@ ext2_truncate(vp, length, flags, cred, td)
 		else
 			bawrite(bp);
 		oip->i_flag |= IN_CHANGE | IN_UPDATE;
-		return (ext2_update(ovp, 1));
+		return (ext2_update(ovp, !DOINGASYNC(ovp)));
 	}
 	/*
 	 * Shorten the size of the file. If the file is not being
@@ -238,7 +238,7 @@ ext2_truncate(vp, length, flags, cred, td)
 	for (i = NDADDR - 1; i > lastblock; i--)
 		oip->i_db[i] = 0;
 	oip->i_flag |= IN_CHANGE | IN_UPDATE;
-	allerror = ext2_update(ovp, 1);
+	allerror = ext2_update(ovp, !DOINGASYNC(ovp));
 
 	/*
 	 * Having written the new inode to disk, save its new configuration
@@ -311,7 +311,7 @@ ext2_truncate(vp, length, flags, cred, td)
 		oip->i_size = length;
 		newspace = blksize(fs, oip, lastblock);
 		if (newspace == 0)
-			panic("itrunc: newspace");
+			panic("ext2_truncate: newspace");
 		if (oldspace - newspace > 0) {
 			/*
 			 * Block number of space to be free'd is
@@ -341,8 +341,9 @@ done:
 	 * Put back the real size.
 	 */
 	oip->i_size = length;
-	oip->i_blocks -= blocksreleased;
-	if (oip->i_blocks < 0)			/* sanity */
+	if (oip->i_blocks >= blocksreleased)
+		oip->i_blocks -= blocksreleased;
+	else				/* sanity */
 		oip->i_blocks = 0;
 	oip->i_flag |= IN_CHANGE;
 	vnode_pager_setsize(ovp, length);
@@ -420,9 +421,13 @@ ext2_indirtrunc(ip, lbn, dbn, lastbn, level, countp)
 	  (u_int)(NINDIR(fs) - (last + 1)) * sizeof(int32_t));
 	if (last == -1)
 		bp->b_flags |= B_INVAL;
-	error = bwrite(bp);
-	if (error)
-		allerror = error;
+	if (DOINGASYNC(vp)) {
+		bdwrite(bp);
+	} else {
+		error = bwrite(bp);
+		if (error)
+			allerror = error;
+	}
 	bap = copy;
 
 	/*
diff --git a/sys/fs/ext2fs/ext2_lookup.c b/sys/fs/ext2fs/ext2_lookup.c
index c279998..9aadb6a 100644
--- a/sys/fs/ext2fs/ext2_lookup.c
+++ b/sys/fs/ext2fs/ext2_lookup.c
@@ -812,7 +812,7 @@ ext2_direnter(ip, dvp, cnp)
 
 #ifdef DIAGNOSTIC
 	if ((cnp->cn_flags & SAVENAME) == 0)
-		panic("direnter: missing name");
+		panic("ext2_direnter: missing name");
 #endif
 	dp = VTOI(dvp);
 	newdir.e2d_ino = ip->i_number;
diff --git a/sys/fs/ext2fs/ext2_vfsops.c b/sys/fs/ext2fs/ext2_vfsops.c
index 3f181fd..136f01c 100644
--- a/sys/fs/ext2fs/ext2_vfsops.c
+++ b/sys/fs/ext2fs/ext2_vfsops.c
@@ -127,7 +127,7 @@ ext2_mount(struct mount *mp)
 
 	vfs_getopt(opts, "fspath", (void **)&path, NULL);
 	/* Double-check the length of path.. */
-	if (strlen(path) >= MAXMNTLEN - 1)
+	if (strlen(path) >= MAXMNTLEN)
 		return (ENAMETOOLONG);
 
 	fspec = NULL;
@@ -318,12 +318,12 @@ compute_sb_data(struct vnode *devvp, struct ext2fs *es,
 	int i;
 	int logic_sb_block = 1;	/* XXX for now */
 	struct buf *bp;
+	uint32_t e2fs_descpb;
 
 	fs->e2fs_bsize = EXT2_MIN_BLOCK_SIZE << es->e2fs_log_bsize;
 	fs->e2fs_bshift = EXT2_MIN_BLOCK_LOG_SIZE + es->e2fs_log_bsize;
 	fs->e2fs_fsbtodb = es->e2fs_log_bsize + 1;
 	fs->e2fs_qbmask = fs->e2fs_bsize - 1;
-	fs->e2fs_blocksize_bits = es->e2fs_log_bsize + 10;
 	fs->e2fs_fsize = EXT2_MIN_FRAG_SIZE << es->e2fs_log_fsize;
 	if (fs->e2fs_fsize)
 		fs->e2fs_fpb = fs->e2fs_bsize / fs->e2fs_fsize;
@@ -331,10 +331,8 @@ compute_sb_data(struct vnode *devvp, struct ext2fs *es,
 	fs->e2fs_fpg = es->e2fs_fpg;
 	fs->e2fs_ipg = es->e2fs_ipg;
 	if (es->e2fs_rev == E2FS_REV0) {
-		fs->e2fs_first_inode = EXT2_FIRSTINO;
 		fs->e2fs_isize = E2FS_REV0_INODE_SIZE ;
 	} else {
-		fs->e2fs_first_inode = es->e2fs_first_ino;
 		fs->e2fs_isize = es->e2fs_inode_size;
 
 		/*
@@ -357,12 +355,11 @@ compute_sb_data(struct vnode *devvp, struct ext2fs *es,
 
 	fs->e2fs_ipb = fs->e2fs_bsize / EXT2_INODE_SIZE(fs);
 	fs->e2fs_itpg = fs->e2fs_ipg /fs->e2fs_ipb;
-	fs->e2fs_descpb = fs->e2fs_bsize / sizeof(struct ext2_gd);
 	/* s_resuid / s_resgid ? */
 	fs->e2fs_gcount = (es->e2fs_bcount - es->e2fs_first_dblock +
 	    EXT2_BLOCKS_PER_GROUP(fs) - 1) / EXT2_BLOCKS_PER_GROUP(fs);
-	db_count = (fs->e2fs_gcount + EXT2_DESC_PER_BLOCK(fs) - 1) /
-	    EXT2_DESC_PER_BLOCK(fs);
+	e2fs_descpb = fs->e2fs_bsize / sizeof(struct ext2_gd);
+	db_count = (fs->e2fs_gcount + e2fs_descpb - 1) / e2fs_descpb;
 	fs->e2fs_gdbcount = db_count;
 	fs->e2fs_gd = malloc(db_count * fs->e2fs_bsize,
 	    M_EXT2MNT, M_WAITOK);
@@ -766,7 +763,7 @@ ext2_statfs(struct mount *mp, struct statfs *sbp)
 	ump = VFSTOEXT2(mp);
 	fs = ump->um_e2fs;
 	if (fs->e2fs->e2fs_magic != E2FS_MAGIC)
-		panic("ext2fs_statfs");
+		panic("ext2_statfs");
 
 	/*
 	 * Compute the overhead (FS structures)
diff --git a/sys/fs/ext2fs/ext2fs.h b/sys/fs/ext2fs/ext2fs.h
index 5cafc7b..c5a4c15 100755..100644
--- a/sys/fs/ext2fs/ext2fs.h
+++ b/sys/fs/ext2fs/ext2fs.h
@@ -153,7 +153,6 @@ struct m_ext2fs {
 	char     e2fs_fmod;       /* super block modified flag */
 	uint32_t e2fs_bsize;      /* Block size */
 	uint32_t e2fs_bshift;     /* calc of logical block no */
-	int32_t  e2fs_bmask;      /* calc of block offset */
 	int32_t  e2fs_bpg;	  /* Number of blocks per group */
 	int64_t  e2fs_qbmask;     /* = s_blocksize -1 */
 	uint32_t e2fs_fsbtodb;    /* Shift to get disk block */
@@ -163,14 +162,9 @@ struct m_ext2fs {
 	uint32_t e2fs_fsize;      /* Size of fragments per block */
 	uint32_t e2fs_fpb;	  /* Number of fragments per block */
 	uint32_t e2fs_fpg;	  /* Number of fragments per group */
-	uint32_t e2fs_dbpg;       /* Number of descriptor blocks per group */
-	uint32_t e2fs_descpb;     /* Number of group descriptors per block */
 	uint32_t e2fs_gdbcount;   /* Number of group descriptors */
 	uint32_t e2fs_gcount;     /* Number of groups */
-	uint32_t e2fs_first_inode;/* First inode on fs */
 	int32_t  e2fs_isize;      /* Size of inode */
-	uint32_t e2fs_mount_opt;
-	uint32_t e2fs_blocksize_bits;
 	uint32_t e2fs_total_dir;  /* Total number of directories */
 	uint8_t	*e2fs_contigdirs; /* (u) # of contig. allocated dirs */
 	char     e2fs_wasvalid;   /* valid at mount time */
@@ -313,27 +307,9 @@ struct csum {
 #define EXT2_MIN_BLOCK_SIZE		1024
 #define	EXT2_MAX_BLOCK_SIZE		4096
 #define EXT2_MIN_BLOCK_LOG_SIZE		  10
-#if defined(_KERNEL)
-# define EXT2_BLOCK_SIZE(s)		((s)->e2fs_bsize)
-#else
-# define EXT2_BLOCK_SIZE(s)		(EXT2_MIN_BLOCK_SIZE << (s)->e2fs_log_bsize)
-#endif
+#define EXT2_BLOCK_SIZE(s)		((s)->e2fs_bsize)
 #define	EXT2_ADDR_PER_BLOCK(s)		(EXT2_BLOCK_SIZE(s) / sizeof(uint32_t))
-#if defined(_KERNEL)
-# define EXT2_BLOCK_SIZE_BITS(s)	((s)->e2fs_blocksize_bits)
-#else
-# define EXT2_BLOCK_SIZE_BITS(s)	((s)->e2fs_log_bsize + 10)
-#endif
-#if defined(_KERNEL)
-#define	EXT2_ADDR_PER_BLOCK_BITS(s)	(EXT2_SB(s)->s_addr_per_block_bits)
 #define EXT2_INODE_SIZE(s)		(EXT2_SB(s)->e2fs_isize)
-#define EXT2_FIRST_INO(s)		(EXT2_SB(s)->e2fs_first_inode)
-#else
-#define EXT2_INODE_SIZE(s)	(((s)->s_rev_level == E2FS_REV0) ? \
-				 E2FS_REV0 : (s)->s_inode_size)
-#define EXT2_FIRST_INO(s)	(((s)->s_rev_level == E2FS_REV0) ? \
-				 E2FS_REV0 : (s)->e2fs_first_ino)
-#endif
 
 /*
  * Macro-instructions used to manage fragments
@@ -341,25 +317,12 @@ struct csum {
 #define EXT2_MIN_FRAG_SIZE		1024
 #define	EXT2_MAX_FRAG_SIZE		4096
 #define EXT2_MIN_FRAG_LOG_SIZE		  10
-#if defined(_KERNEL)
-# define EXT2_FRAG_SIZE(s)		(EXT2_SB(s)->e2fs_fsize)
-# define EXT2_FRAGS_PER_BLOCK(s)	(EXT2_SB(s)->e2fs_fpb)
-#else
-# define EXT2_FRAG_SIZE(s)		(EXT2_MIN_FRAG_SIZE << (s)->e2fs_log_fsize)
-# define EXT2_FRAGS_PER_BLOCK(s)	(EXT2_BLOCK_SIZE(s) / EXT2_FRAG_SIZE(s))
-#endif
+#define EXT2_FRAG_SIZE(s)		(EXT2_SB(s)->e2fs_fsize)
+#define EXT2_FRAGS_PER_BLOCK(s)		(EXT2_SB(s)->e2fs_fpb)
 
 /*
  * Macro-instructions used to manage group descriptors
  */
-#if defined(_KERNEL)
-# define EXT2_BLOCKS_PER_GROUP(s)	(EXT2_SB(s)->e2fs_bpg)
-# define EXT2_DESC_PER_BLOCK(s)		(EXT2_SB(s)->e2fs_descpb)
-# define EXT2_DESC_PER_BLOCK_BITS(s)	(EXT2_SB(s)->s_desc_per_block_bits)
-#else
-# define EXT2_BLOCKS_PER_GROUP(s)	((s)->e2fs_bpg)
-# define EXT2_DESC_PER_BLOCK(s)		(EXT2_BLOCK_SIZE(s) / sizeof(struct ext2_gd))
-
-#endif
+#define EXT2_BLOCKS_PER_GROUP(s)	(EXT2_SB(s)->e2fs_bpg)
 
 #endif	/* !_FS_EXT2FS_EXT2FS_H_ */
diff --git a/sys/fs/ext2fs/fs.h b/sys/fs/ext2fs/fs.h
index 880c500..7068e79 100644
--- a/sys/fs/ext2fs/fs.h
+++ b/sys/fs/ext2fs/fs.h
@@ -149,7 +149,4 @@
  */
 #define	NINDIR(fs)	(EXT2_ADDR_PER_BLOCK(fs))
 
-extern int inside[], around[];
-extern u_char *fragtbl[];
-
 #endif /* !_FS_EXT2FS_FS_H_ */
diff --git a/sys/fs/ext2fs/inode.h b/sys/fs/ext2fs/inode.h
index d8aaac9..ec47216 100644
--- a/sys/fs/ext2fs/inode.h
+++ b/sys/fs/ext2fs/inode.h
@@ -90,11 +90,11 @@ struct inode {
 	int32_t		i_atimensec;	/* Last access time. */
 	int32_t		i_ctimensec;	/* Last inode change time. */
 	int32_t		i_birthnsec;	/* Inode creation time. */
-	int32_t		i_db[NDADDR];	/* Direct disk blocks. */
-	int32_t		i_ib[NIADDR];	/* Indirect disk blocks. */
+	uint32_t	i_db[NDADDR];	/* Direct disk blocks. */
+	uint32_t	i_ib[NIADDR];	/* Indirect disk blocks. */
 	uint32_t	i_flags;	/* Status flags (chflags). */
-	int32_t		i_blocks;	/* Blocks actually held. */
-	int32_t		i_gen;		/* Generation number. */
+	uint32_t	i_blocks;	/* Blocks actually held. */
+	uint32_t	i_gen;		/* Generation number. */
 	uint32_t	i_uid;		/* File owner. */
 	uint32_t	i_gid;		/* File group. */
 };
@@ -162,7 +162,7 @@ struct ufid {
 	uint16_t ufid_len;	/* Length of structure. */
 	uint16_t ufid_pad;	/* Force 32-bit alignment. */
 	ino_t	 ufid_ino;	/* File number (ino). */
-	int32_t	 ufid_gen;	/* Generation number. */
+	uint32_t ufid_gen;	/* Generation number. */
 };
 #endif /* _KERNEL */
 
diff --git a/sys/fs/msdosfs/msdosfs_fat.c b/sys/fs/msdosfs/msdosfs_fat.c
index 0b1899f..3994042 100644
--- a/sys/fs/msdosfs/msdosfs_fat.c
+++ b/sys/fs/msdosfs/msdosfs_fat.c
@@ -321,8 +321,8 @@ updatefats(pmp, bp, fatbn)
 	struct buf *bp;
 	u_long fatbn;
 {
-	int i;
 	struct buf *bpn;
+	int cleanfat, i;
 
 #ifdef MSDOSFS_DEBUG
 	printf("updatefats(pmp %p, bp %p, fatbn %lu)\n", pmp, bp, fatbn);
@@ -362,13 +362,24 @@ updatefats(pmp, bp, fatbn)
 		 * filesystem was mounted.  If synch is asked for then use
 		 * bwrite()'s and really slow things down.
 		 */
+		if (fatbn != pmp->pm_fatblk || FAT12(pmp))
+			cleanfat = 0;
+		else if (FAT16(pmp))
+			cleanfat = 16;
+		else
+			cleanfat = 32;
 		for (i = 1; i < pmp->pm_FATs; i++) {
 			fatbn += pmp->pm_FATsecs;
 			/* getblk() never fails */
 			bpn = getblk(pmp->pm_devvp, fatbn, bp->b_bcount,
 			    0, 0, 0);
 			bcopy(bp->b_data, bpn->b_data, bp->b_bcount);
-			if (pmp->pm_flags & MSDOSFSMNT_WAITONFAT)
+			/* Force the clean bit on in the other copies. */
+			if (cleanfat == 16)
+				((u_int8_t *)bpn->b_data)[3] |= 0x80;
+			else if (cleanfat == 32)
+				((u_int8_t *)bpn->b_data)[7] |= 0x08;
+			if (pmp->pm_mountp->mnt_flag & MNT_SYNCHRONOUS)
 				bwrite(bpn);
 			else
 				bdwrite(bpn);
@@ -378,7 +389,7 @@ updatefats(pmp, bp, fatbn)
 	/*
 	 * Write out the first (or current) fat last.
 	 */
-	if (pmp->pm_flags & MSDOSFSMNT_WAITONFAT)
+	if (pmp->pm_mountp->mnt_flag & MNT_SYNCHRONOUS)
 		bwrite(bp);
 	else
 		bdwrite(bp);
@@ -1114,7 +1125,7 @@ extendfile(dep, count, bpp, ncp, flags)
  * Routine to mark a FAT16 or FAT32 volume as "clean" or "dirty" by
  * manipulating the upper bit of the FAT entry for cluster 1.  Note that
  * this bit is not defined for FAT12 volumes, which are always assumed to
- * be dirty.
+ * be clean.
  *
  * The fatentry() routine only works on cluster numbers that a file could
  * occupy, so it won't manipulate the entry for cluster 1.  So we have to do
diff --git a/sys/fs/msdosfs/msdosfs_vfsops.c b/sys/fs/msdosfs/msdosfs_vfsops.c
index e99fff3..a209970 100644
--- a/sys/fs/msdosfs/msdosfs_vfsops.c
+++ b/sys/fs/msdosfs/msdosfs_vfsops.c
@@ -553,8 +553,7 @@ mountmsdosfs(struct vnode *devvp, struct mount *mp)
 	}
 
 	if (pmp->pm_RootDirEnts == 0) {
-		if (pmp->pm_Sectors
-		    || pmp->pm_FATsecs
+		if (pmp->pm_FATsecs
 		    || getushort(b710->bpbFSVers)) {
 			error = EINVAL;
 #ifdef MSDOSFS_DEBUG
diff --git a/sys/fs/msdosfs/msdosfs_vnops.c b/sys/fs/msdosfs/msdosfs_vnops.c
index a90ee8d..8e045cb 100644
--- a/sys/fs/msdosfs/msdosfs_vnops.c
+++ b/sys/fs/msdosfs/msdosfs_vnops.c
@@ -973,7 +973,7 @@ msdosfs_rename(ap)
 	u_char to_count;
 	int doingdirectory = 0, newparent = 0;
 	int error;
-	u_long cn;
+	u_long cn, pcl;
 	daddr_t bn;
 	struct denode *fddep;	/* from file's parent directory	 */
 	struct msdosfsmount *pmp;
@@ -1246,9 +1246,12 @@ abortit:
 			goto bad;
 		}
 		dotdotp = (struct direntry *)bp->b_data + 1;
-		putushort(dotdotp->deStartCluster, dp->de_StartCluster);
+		pcl = dp->de_StartCluster;
+		if (FAT32(pmp) && pcl == pmp->pm_rootdirblk)
+			pcl = MSDOSFSROOT;
+		putushort(dotdotp->deStartCluster, pcl);
 		if (FAT32(pmp))
-			putushort(dotdotp->deHighClust, dp->de_StartCluster >> 16);
+			putushort(dotdotp->deHighClust, pcl >> 16);
 		if (DOINGASYNC(fvp))
 			bdwrite(bp);
 		else if ((error = bwrite(bp)) != 0) {
@@ -1369,8 +1372,13 @@ msdosfs_mkdir(ap)
 	putushort(denp[0].deMDate, ndirent.de_MDate);
 	putushort(denp[0].deMTime, ndirent.de_MTime);
 	pcl = pdep->de_StartCluster;
+	/*
+	 * Although the root directory has a non-magic starting cluster
+	 * number for FAT32, chkdsk and fsck_msdosfs still require
+	 * references to it in dotdot entries to be magic.
+	 */
 	if (FAT32(pmp) && pcl == pmp->pm_rootdirblk)
-		pcl = 0;
+		pcl = MSDOSFSROOT;
 	putushort(denp[1].deStartCluster, pcl);
 	putushort(denp[1].deCDate, ndirent.de_CDate);
 	putushort(denp[1].deCTime, ndirent.de_CTime);
@@ -1380,7 +1388,7 @@ msdosfs_mkdir(ap)
 	putushort(denp[1].deMTime, ndirent.de_MTime);
 	if (FAT32(pmp)) {
 		putushort(denp[0].deHighClust, newcluster >> 16);
-		putushort(denp[1].deHighClust, pdep->de_StartCluster >> 16);
+		putushort(denp[1].deHighClust, pcl >> 16);
 	}
 
 	if (DOINGASYNC(ap->a_dvp))
diff --git a/sys/fs/nfs/nfs.h b/sys/fs/nfs/nfs.h
index 3cc8c8f..febb631 100644
--- a/sys/fs/nfs/nfs.h
+++ b/sys/fs/nfs/nfs.h
@@ -523,7 +523,6 @@ struct nfsrv_descript {
 	int			*nd_errp;	/* Pointer to ret status */
 	u_int32_t		nd_retxid;	/* Reply xid */
 	struct nfsrvcache	*nd_rp;		/* Assoc. cache entry */
-	struct timeval		nd_starttime;	/* Time RPC initiated */
 	fhandle_t		nd_fh;		/* File handle */
 	struct ucred		*nd_cred;	/* Credentials */
 	uid_t			nd_saveduid;	/* Saved uid */
diff --git a/sys/fs/nfs/nfs_commonkrpc.c b/sys/fs/nfs/nfs_commonkrpc.c
index 6640c1f..23d8954 100644
--- a/sys/fs/nfs/nfs_commonkrpc.c
+++ b/sys/fs/nfs/nfs_commonkrpc.c
@@ -459,18 +459,17 @@ nfs_feedback(int type, int proc, void *arg)
 {
 	struct nfs_feedback_arg *nf = (struct nfs_feedback_arg *) arg;
 	struct nfsmount *nmp = nf->nf_mount;
-	struct timeval now;
-
-	getmicrouptime(&now);
+	time_t now;
 
 	switch (type) {
 	case FEEDBACK_REXMIT2:
 	case FEEDBACK_RECONNECT:
-		if (nf->nf_lastmsg + nmp->nm_tprintf_delay < now.tv_sec) {
+		now = NFSD_MONOSEC;
+		if (nf->nf_lastmsg + nmp->nm_tprintf_delay < now) {
 			nfs_down(nmp, nf->nf_td,
 			    "not responding", 0, NFSSTA_TIMEO);
 			nf->nf_tprintfmsg = TRUE;
-			nf->nf_lastmsg = now.tv_sec;
+			nf->nf_lastmsg = now;
 		}
 		break;
 
@@ -501,7 +500,7 @@ newnfs_request(struct nfsrv_descript *nd, struct nfsmount *nmp,
 	u_int16_t procnum;
 	u_int trylater_delay = 1;
 	struct nfs_feedback_arg nf;
-	struct timeval timo, now;
+	struct timeval timo;
 	AUTH *auth;
 	struct rpc_callextra ext;
 	enum clnt_stat stat;
@@ -617,8 +616,7 @@ newnfs_request(struct nfsrv_descript *nd, struct nfsmount *nmp,
 		bzero(&nf, sizeof(struct nfs_feedback_arg));
 		nf.nf_mount = nmp;
 		nf.nf_td = td;
-		getmicrouptime(&now);
-		nf.nf_lastmsg = now.tv_sec -
+		nf.nf_lastmsg = NFSD_MONOSEC -
 		    ((nmp->nm_tprintf_delay)-(nmp->nm_tprintf_initial_delay));
 	}
 
@@ -767,12 +765,18 @@ tryagain:
 	if (stat == RPC_SUCCESS) {
 		error = 0;
 	} else if (stat == RPC_TIMEDOUT) {
+		NFSINCRGLOBAL(newnfsstats.rpctimeouts);
 		error = ETIMEDOUT;
 	} else if (stat == RPC_VERSMISMATCH) {
+		NFSINCRGLOBAL(newnfsstats.rpcinvalid);
 		error = EOPNOTSUPP;
 	} else if (stat == RPC_PROGVERSMISMATCH) {
+		NFSINCRGLOBAL(newnfsstats.rpcinvalid);
 		error = EPROTONOSUPPORT;
+	} else if (stat == RPC_INTR) {
+		error = EINTR;
 	} else {
+		NFSINCRGLOBAL(newnfsstats.rpcinvalid);
 		error = EACCES;
 	}
 	if (error) {
diff --git a/sys/fs/nfs/nfs_commonsubs.c b/sys/fs/nfs/nfs_commonsubs.c
index 95aa7bd..b2d74c7 100644
--- a/sys/fs/nfs/nfs_commonsubs.c
+++ b/sys/fs/nfs/nfs_commonsubs.c
@@ -218,6 +218,7 @@ nfsm_mbufuio(struct nfsrv_descript *nd, struct uio *uiop, int siz)
 				}
 				mbufcp = NFSMTOD(mp, caddr_t);
 				len = mbuf_len(mp);
+				KASSERT(len > 0, ("len %d", len));
 			}
 			xfer = (left > len) ? len : left;
 #ifdef notdef
@@ -1998,7 +1999,6 @@ nfsv4_fillattr(struct nfsrv_descript *nd, struct mount *mp, vnode_t vp,
 	struct statfs fs;
 	struct nfsfsinfo fsinf;
 	struct timespec temptime;
-	struct timeval curtime;
 	NFSACL_T *aclp, *naclp = NULL;
 #ifdef QUOTA
 	struct dqblk dqb;
@@ -2412,8 +2412,7 @@ nfsv4_fillattr(struct nfsrv_descript *nd, struct mount *mp, vnode_t vp,
 			retnum += NFSX_V4TIME;
 			break;
 		case NFSATTRBIT_TIMEACCESSSET:
-			NFSGETTIME(&curtime);
-			if (vap->va_atime.tv_sec != curtime.tv_sec) {
+			if ((vap->va_vaflags & VA_UTIMES_NULL) == 0) {
 				NFSM_BUILD(tl, u_int32_t *, NFSX_V4SETTIME);
 				*tl++ = txdr_unsigned(NFSV4SATTRTIME_TOCLIENT);
 				txdr_nfsv4time(&vap->va_atime, tl);
@@ -2442,8 +2441,7 @@ nfsv4_fillattr(struct nfsrv_descript *nd, struct mount *mp, vnode_t vp,
 			retnum += NFSX_V4TIME;
 			break;
 		case NFSATTRBIT_TIMEMODIFYSET:
-			NFSGETTIME(&curtime);
-			if (vap->va_mtime.tv_sec != curtime.tv_sec) {
+			if ((vap->va_vaflags & VA_UTIMES_NULL) == 0) {
 				NFSM_BUILD(tl, u_int32_t *, NFSX_V4SETTIME);
 				*tl++ = txdr_unsigned(NFSV4SATTRTIME_TOCLIENT);
 				txdr_nfsv4time(&vap->va_mtime, tl);
diff --git a/sys/fs/nfs/nfsport.h b/sys/fs/nfs/nfsport.h
index 3f22b7a..ea04525 100644
--- a/sys/fs/nfs/nfsport.h
+++ b/sys/fs/nfs/nfsport.h
@@ -588,12 +588,6 @@ void nfsrvd_rcv(struct socket *, void *, int);
 #define	NCHNAMLEN	9999999
 
 /*
- * Define these to use the time of day clock.
- */
-#define	NFSGETTIME(t)		(getmicrotime(t))
-#define	NFSGETNANOTIME(t)	(getnanotime(t))
-
-/*
  * These macros are defined to initialize and set the timer routine.
  */
 #define	NFS_TIMERINIT \
diff --git a/sys/fs/nfsclient/nfs_clport.c b/sys/fs/nfsclient/nfs_clport.c
index 145ff63..1179eb5 100644
--- a/sys/fs/nfsclient/nfs_clport.c
+++ b/sys/fs/nfsclient/nfs_clport.c
@@ -726,7 +726,6 @@ nfscl_fillsattr(struct nfsrv_descript *nd, struct vattr *vap,
 	u_int32_t *tl;
 	struct nfsv2_sattr *sp;
 	nfsattrbit_t attrbits;
-	struct timeval curtime;
 
 	switch (nd->nd_flag & (ND_NFSV2 | ND_NFSV3 | ND_NFSV4)) {
 	case ND_NFSV2:
@@ -755,7 +754,6 @@ nfscl_fillsattr(struct nfsrv_descript *nd, struct vattr *vap,
 		txdr_nfsv2time(&vap->va_mtime, &sp->sa_mtime);
 		break;
 	case ND_NFSV3:
-		getmicrotime(&curtime);
 		if (vap->va_mode != (mode_t)VNOVAL) {
 			NFSM_BUILD(tl, u_int32_t *, 2 * NFSX_UNSIGNED);
 			*tl++ = newnfs_true;
@@ -789,7 +787,7 @@ nfscl_fillsattr(struct nfsrv_descript *nd, struct vattr *vap,
 			*tl = newnfs_false;
 		}
 		if (vap->va_atime.tv_sec != VNOVAL) {
-			if (vap->va_atime.tv_sec != curtime.tv_sec) {
+			if ((vap->va_vaflags & VA_UTIMES_NULL) == 0) {
 				NFSM_BUILD(tl, u_int32_t *, 3 * NFSX_UNSIGNED);
 				*tl++ = txdr_unsigned(NFSV3SATTRTIME_TOCLIENT);
 				txdr_nfsv3time(&vap->va_atime, tl);
@@ -802,7 +800,7 @@ nfscl_fillsattr(struct nfsrv_descript *nd, struct vattr *vap,
 			*tl = txdr_unsigned(NFSV3SATTRTIME_DONTCHANGE);
 		}
 		if (vap->va_mtime.tv_sec != VNOVAL) {
-			if (vap->va_mtime.tv_sec != curtime.tv_sec) {
+			if ((vap->va_vaflags & VA_UTIMES_NULL) == 0) {
 				NFSM_BUILD(tl, u_int32_t *, 3 * NFSX_UNSIGNED);
 				*tl++ = txdr_unsigned(NFSV3SATTRTIME_TOCLIENT);
 				txdr_nfsv3time(&vap->va_mtime, tl);
diff --git a/sys/fs/nfsclient/nfs_clrpcops.c b/sys/fs/nfsclient/nfs_clrpcops.c
index be0476a..a89b907 100644
--- a/sys/fs/nfsclient/nfs_clrpcops.c
+++ b/sys/fs/nfsclient/nfs_clrpcops.c
@@ -1444,7 +1444,7 @@ nfsrpc_readrpc(vnode_t vp, struct uio *uiop, struct ucred *cred,
 			NFSM_DISSECT(tl, u_int32_t *, NFSX_UNSIGNED);
 			eof = fxdr_unsigned(int, *tl);
 		}
-		NFSM_STRSIZ(retlen, rsize);
+		NFSM_STRSIZ(retlen, len);
 		error = nfsm_mbufuio(nd, uiop, retlen);
 		if (error)
 			goto nfsmout;
diff --git a/sys/fs/nfsclient/nfs_clstate.c b/sys/fs/nfsclient/nfs_clstate.c
index 8b5acb9..a774103 100644
--- a/sys/fs/nfsclient/nfs_clstate.c
+++ b/sys/fs/nfsclient/nfs_clstate.c
@@ -2447,7 +2447,7 @@ nfscl_renewthread(struct nfsclclient *clp, NFSPROC_T *p)
 	u_int32_t clidrev;
 	int error, cbpathdown, islept, igotlock, ret, clearok;
 	uint32_t recover_done_time = 0;
-	struct timespec mytime;
+	time_t mytime;
 	static time_t prevsec = 0;
 	struct nfscllockownerfh *lfhp, *nlfhp;
 	struct nfscllockownerfhhead lfh;
@@ -2720,9 +2720,9 @@ tryagain2:
 		 * Call nfscl_cleanupkext() once per second to check for
 		 * open/lock owners where the process has exited.
 		 */
-		NFSGETNANOTIME(&mytime);
-		if (prevsec != mytime.tv_sec) {
-			prevsec = mytime.tv_sec;
+		mytime = NFSD_MONOSEC;
+		if (prevsec != mytime) {
+			prevsec = mytime;
 			nfscl_cleanupkext(clp, &lfh);
 		}
 
@@ -4611,7 +4611,7 @@ nfscl_delegmodtime(vnode_t vp)
 	}
 	dp = nfscl_finddeleg(clp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len);
 	if (dp != NULL && (dp->nfsdl_flags & NFSCLDL_WRITE)) {
-		NFSGETNANOTIME(&dp->nfsdl_modtime);
+		nanotime(&dp->nfsdl_modtime);
 		dp->nfsdl_flags |= NFSCLDL_MODTIMESET;
 	}
 	NFSUNLOCKCLSTATE();
diff --git a/sys/fs/nfsclient/nfs_clvnops.c b/sys/fs/nfsclient/nfs_clvnops.c
index 3a898f2..f778009 100644
--- a/sys/fs/nfsclient/nfs_clvnops.c
+++ b/sys/fs/nfsclient/nfs_clvnops.c
@@ -3247,7 +3247,7 @@ nfsfifo_read(struct vop_read_args *ap)
 	 */
 	mtx_lock(&np->n_mtx);
 	np->n_flag |= NACC;
-	getnanotime(&np->n_atim);
+	vfs_timestamp(&np->n_atim);
 	mtx_unlock(&np->n_mtx);
 	error = fifo_specops.vop_read(ap);
 	return error;	
@@ -3266,7 +3266,7 @@ nfsfifo_write(struct vop_write_args *ap)
 	 */
 	mtx_lock(&np->n_mtx);
 	np->n_flag |= NUPD;
-	getnanotime(&np->n_mtim);
+	vfs_timestamp(&np->n_mtim);
 	mtx_unlock(&np->n_mtx);
 	return(fifo_specops.vop_write(ap));
 }
@@ -3286,7 +3286,7 @@ nfsfifo_close(struct vop_close_args *ap)
 
 	mtx_lock(&np->n_mtx);
 	if (np->n_flag & (NACC | NUPD)) {
-		getnanotime(&ts);
+		vfs_timestamp(&ts);
 		if (np->n_flag & NACC)
 			np->n_atim = ts;
 		if (np->n_flag & NUPD)
diff --git a/sys/fs/nfsserver/nfs_nfsdkrpc.c b/sys/fs/nfsserver/nfs_nfsdkrpc.c
index 337951b..cffb46f 100644
--- a/sys/fs/nfsserver/nfs_nfsdkrpc.c
+++ b/sys/fs/nfsserver/nfs_nfsdkrpc.c
@@ -310,7 +310,6 @@ nfs_proc(struct nfsrv_descript *nd, u_int32_t xid, struct socket *so,
 	} else {
 		isdgram = 1;
 	}
-	NFSGETTIME(&nd->nd_starttime);
 
 	/*
 	 * Two cases:
diff --git a/sys/fs/nfsserver/nfs_nfsdport.c b/sys/fs/nfsserver/nfs_nfsdport.c
index f56a292..1731c72 100644
--- a/sys/fs/nfsserver/nfs_nfsdport.c
+++ b/sys/fs/nfsserver/nfs_nfsdport.c
@@ -1476,7 +1476,7 @@ nfsvno_updfilerev(struct vnode *vp, struct nfsvattr *nvap,
 	struct vattr va;
 
 	VATTR_NULL(&va);
-	getnanotime(&va.va_mtime);
+	vfs_timestamp(&va.va_mtime);
 	(void) VOP_SETATTR(vp, &va, cred);
 	(void) nfsvno_getattr(vp, nvap, cred, p, 1);
 }
@@ -2248,7 +2248,6 @@ nfsrv_sattr(struct nfsrv_descript *nd, struct nfsvattr *nvap,
 {
 	u_int32_t *tl;
 	struct nfsv2_sattr *sp;
-	struct timeval curtime;
 	int error = 0, toclient = 0;
 
 	switch (nd->nd_flag & (ND_NFSV2 | ND_NFSV3 | ND_NFSV4)) {
@@ -2307,9 +2306,7 @@ nfsrv_sattr(struct nfsrv_descript *nd, struct nfsvattr *nvap,
 			toclient = 1;
 			break;
 		case NFSV3SATTRTIME_TOSERVER:
-			NFSGETTIME(&curtime);
-			nvap->na_atime.tv_sec = curtime.tv_sec;
-			nvap->na_atime.tv_nsec = curtime.tv_usec * 1000;
+			vfs_timestamp(&nvap->na_atime);
 			nvap->na_vaflags |= VA_UTIMES_NULL;
 			break;
 		};
@@ -2321,9 +2318,7 @@ nfsrv_sattr(struct nfsrv_descript *nd, struct nfsvattr *nvap,
 			nvap->na_vaflags &= ~VA_UTIMES_NULL;
 			break;
 		case NFSV3SATTRTIME_TOSERVER:
-			NFSGETTIME(&curtime);
-			nvap->na_mtime.tv_sec = curtime.tv_sec;
-			nvap->na_mtime.tv_nsec = curtime.tv_usec * 1000;
+			vfs_timestamp(&nvap->na_mtime);
 			if (!toclient)
 				nvap->na_vaflags |= VA_UTIMES_NULL;
 			break;
@@ -2353,7 +2348,6 @@ nfsv4_sattr(struct nfsrv_descript *nd, struct nfsvattr *nvap,
 	u_char *cp, namestr[NFSV4_SMALLSTR + 1];
 	uid_t uid;
 	gid_t gid;
-	struct timeval curtime;
 
 	error = nfsrv_getattrbits(nd, attrbitp, NULL, &retnotsup);
 	if (error)
@@ -2488,9 +2482,7 @@ nfsv4_sattr(struct nfsrv_descript *nd, struct nfsvattr *nvap,
 			    toclient = 1;
 			    attrsum += NFSX_V4TIME;
 			} else {
-			    NFSGETTIME(&curtime);
-			    nvap->na_atime.tv_sec = curtime.tv_sec;
-			    nvap->na_atime.tv_nsec = curtime.tv_usec * 1000;
+			    vfs_timestamp(&nvap->na_atime);
 			    nvap->na_vaflags |= VA_UTIMES_NULL;
 			}
 			break;
@@ -2515,9 +2507,7 @@ nfsv4_sattr(struct nfsrv_descript *nd, struct nfsvattr *nvap,
 			    nvap->na_vaflags &= ~VA_UTIMES_NULL;
 			    attrsum += NFSX_V4TIME;
 			} else {
-			    NFSGETTIME(&curtime);
-			    nvap->na_mtime.tv_sec = curtime.tv_sec;
-			    nvap->na_mtime.tv_nsec = curtime.tv_usec * 1000;
+			    vfs_timestamp(&nvap->na_mtime);
 			    if (!toclient)
 				nvap->na_vaflags |= VA_UTIMES_NULL;
 			}
diff --git a/sys/fs/nfsserver/nfs_nfsdserv.c b/sys/fs/nfsserver/nfs_nfsdserv.c
index 85fbbd2..307b1f6 100644
--- a/sys/fs/nfsserver/nfs_nfsdserv.c
+++ b/sys/fs/nfsserver/nfs_nfsdserv.c
@@ -55,6 +55,11 @@ extern int nfs_rootfhset;
 extern int nfsrv_enable_crossmntpt;
 #endif	/* !APPLEKEXT */
 
+static int	nfs_async = 0;
+SYSCTL_DECL(_vfs_nfsd);
+SYSCTL_INT(_vfs_nfsd, OID_AUTO, async, CTLFLAG_RW, &nfs_async, 0,
+    "Tell client that writes were synced even though they were not");
+
 /*
  * This list defines the GSS mechanisms supported.
  * (Don't ask me how you get these strings from the RFC stuff like
@@ -912,7 +917,13 @@ nfsrvd_write(struct nfsrv_descript *nd, __unused int isdgram,
 			goto out;
 		NFSM_BUILD(tl, u_int32_t *, 4 * NFSX_UNSIGNED);
 		*tl++ = txdr_unsigned(retlen);
-		if (stable == NFSWRITE_UNSTABLE)
+		/*
+		 * If nfs_async is set, then pretend the write was FILESYNC.
+		 * Warning: Doing this violates RFC1813 and runs a risk
+		 * of data written by a client being lost when the server
+		 * crashes/reboots.
+		 */
+		if (stable == NFSWRITE_UNSTABLE && nfs_async == 0)
 			*tl++ = txdr_unsigned(stable);
 		else
 			*tl++ = txdr_unsigned(NFSWRITE_FILESYNC);
diff --git a/sys/fs/nfsserver/nfs_nfsdstate.c b/sys/fs/nfsserver/nfs_nfsdstate.c
index 8b2f8b8..d74a5a9 100644
--- a/sys/fs/nfsserver/nfs_nfsdstate.c
+++ b/sys/fs/nfsserver/nfs_nfsdstate.c
@@ -3967,7 +3967,6 @@ nfsrv_setupstable(NFSPROC_T *p)
 	int error, i, tryagain;
 	off_t off = 0;
 	ssize_t aresid, len;
-	struct timeval curtime;
 
 	/*
 	 * If NFSNSF_UPDATEDONE is set, this is a restart of the nfsds without
@@ -3978,8 +3977,7 @@ nfsrv_setupstable(NFSPROC_T *p)
 	/*
 	 * Set Grace over just until the file reads successfully.
 	 */
-	NFSGETTIME(&curtime);
-	nfsrvboottime = curtime.tv_sec;
+	nfsrvboottime = time_second;
 	LIST_INIT(&sf->nsf_head);
 	sf->nsf_flags = (NFSNSF_GRACEOVER | NFSNSF_NEEDLOCK);
 	sf->nsf_eograce = NFSD_MONOSEC + NFSRV_LEASEDELTA;
@@ -4650,8 +4648,7 @@ out:
 APPLESTATIC void
 nfsd_recalldelegation(vnode_t vp, NFSPROC_T *p)
 {
-	struct timespec mytime;
-	int32_t starttime;
+	time_t starttime;
 	int error;
 
 	/*
@@ -4675,8 +4672,7 @@ nfsd_recalldelegation(vnode_t vp, NFSPROC_T *p)
 	 * Now, call nfsrv_checkremove() in a loop while it returns
 	 * NFSERR_DELAY. Return upon any other error or when timed out.
 	 */
-	NFSGETNANOTIME(&mytime);
-	starttime = (u_int32_t)mytime.tv_sec;
+	starttime = NFSD_MONOSEC;
 	do {
 		if (NFSVOPLOCK(vp, LK_EXCLUSIVE) == 0) {
 			error = nfsrv_checkremove(vp, 0, p);
@@ -4684,11 +4680,7 @@ nfsd_recalldelegation(vnode_t vp, NFSPROC_T *p)
 		} else
 			error = EPERM;
 		if (error == NFSERR_DELAY) {
-			NFSGETNANOTIME(&mytime);
-			if (((u_int32_t)mytime.tv_sec - starttime) >
-			    NFS_REMOVETIMEO &&
-			    ((u_int32_t)mytime.tv_sec - starttime) <
-			    100000)
+			if (NFSD_MONOSEC - starttime > NFS_REMOVETIMEO)
 				break;
 			/* Sleep for a short period of time */
 			(void) nfs_catnap(PZERO, 0, "nfsremove");
@@ -4949,9 +4941,7 @@ nfsrv_notsamecredname(struct nfsrv_descript *nd, struct nfsclient *clp)
 static time_t
 nfsrv_leaseexpiry(void)
 {
-	struct timeval curtime;
 
-	NFSGETTIME(&curtime);
 	if (nfsrv_stablefirst.nsf_eograce > NFSD_MONOSEC)
 		return (NFSD_MONOSEC + 2 * (nfsrv_lease + NFSRV_LEASEDELTA));
 	return (NFSD_MONOSEC + nfsrv_lease + NFSRV_LEASEDELTA);
diff --git a/sys/fs/nullfs/null_subr.c b/sys/fs/nullfs/null_subr.c
index 0b72249..90e56ea 100644
--- a/sys/fs/nullfs/null_subr.c
+++ b/sys/fs/nullfs/null_subr.c
@@ -46,9 +46,6 @@
 
 #include <fs/nullfs/null.h>
 
-#define LOG2_SIZEVNODE 8		/* log2(sizeof struct vnode) */
-#define	NNULLNODECACHE 16
-
 /*
  * Null layer cache:
  * Each cache entry holds a reference to the lower vnode
@@ -57,12 +54,11 @@
  * alias is removed the lower vnode is vrele'd.
  */
 
-#define	NULL_NHASH(vp) \
-	(&null_node_hashtbl[(((uintptr_t)vp)>>LOG2_SIZEVNODE) & null_node_hash])
+#define	NULL_NHASH(vp) (&null_node_hashtbl[vfs_hash_index(vp) & null_hash_mask])
 
 static LIST_HEAD(null_node_hashhead, null_node) *null_node_hashtbl;
-static u_long null_node_hash;
-struct mtx null_hashmtx;
+static struct mtx null_hashmtx;
+static u_long null_hash_mask;
 
 static MALLOC_DEFINE(M_NULLFSHASH, "nullfs_hash", "NULLFS hash table");
 MALLOC_DEFINE(M_NULLFSNODE, "nullfs_node", "NULLFS vnode private part");
@@ -77,8 +73,8 @@ nullfs_init(vfsp)
 	struct vfsconf *vfsp;
 {
 
-	NULLFSDEBUG("nullfs_init\n");		/* printed during system boot */
-	null_node_hashtbl = hashinit(NNULLNODECACHE, M_NULLFSHASH, &null_node_hash);
+	null_node_hashtbl = hashinit(desiredvnodes, M_NULLFSHASH,
+	    &null_hash_mask);
 	mtx_init(&null_hashmtx, "nullhs", NULL, MTX_DEF);
 	return (0);
 }
@@ -89,7 +85,7 @@ nullfs_uninit(vfsp)
 {
 
 	mtx_destroy(&null_hashmtx);
-	hashdestroy(null_node_hashtbl, M_NULLFSHASH, null_node_hash);
+	hashdestroy(null_node_hashtbl, M_NULLFSHASH, null_hash_mask);
 	return (0);
 }
 
diff --git a/sys/fs/nullfs/null_vfsops.c b/sys/fs/nullfs/null_vfsops.c
index 5abfa49..3724e0a 100644
--- a/sys/fs/nullfs/null_vfsops.c
+++ b/sys/fs/nullfs/null_vfsops.c
@@ -67,15 +67,6 @@ static vfs_vget_t	nullfs_vget;
 static vfs_extattrctl_t	nullfs_extattrctl;
 static vfs_reclaim_lowervp_t nullfs_reclaim_lowervp;
 
-/* Mount options that we support. */
-static const char *nullfs_opts[] = {
-	"cache",
-	"export",
-	"from",
-	"target",
-	NULL
-};
-
 /*
  * Mount null layer
  */
@@ -97,8 +88,6 @@ nullfs_mount(struct mount *mp)
 		return (EPERM);
 	if (mp->mnt_flag & MNT_ROOTFS)
 		return (EOPNOTSUPP);
-	if (vfs_filteropt(mp->mnt_optnew, nullfs_opts))
-		return (EINVAL);
 
 	/*
 	 * Update is a no-op
diff --git a/sys/fs/nullfs/null_vnops.c b/sys/fs/nullfs/null_vnops.c
index cc35d81..f59865f 100644
--- a/sys/fs/nullfs/null_vnops.c
+++ b/sys/fs/nullfs/null_vnops.c
@@ -740,6 +740,14 @@ null_reclaim(struct vop_reclaim_args *ap)
 	vp->v_object = NULL;
 	vp->v_vnlock = &vp->v_lock;
 	VI_UNLOCK(vp);
+
+	/*
+	 * If we were opened for write, we leased one write reference
+	 * to the lower vnode.  If this is a reclamation due to the
+	 * forced unmount, undo the reference now.
+	 */
+	if (vp->v_writecount > 0)
+		VOP_ADD_WRITECOUNT(lowervp, -1);
 	vput(lowervp);
 	free(xp, M_NULLFSNODE);
 
diff --git a/sys/geom/journal/g_journal.c b/sys/geom/journal/g_journal.c
index 4581430..a3c996c 100644
--- a/sys/geom/journal/g_journal.c
+++ b/sys/geom/journal/g_journal.c
@@ -2976,7 +2976,7 @@ g_journal_do_switch(struct g_class *classp)
 		g_journal_switch_wait(sc);
 		mtx_unlock(&sc->sc_mtx);
 
-		vfs_write_resume(mp);
+		vfs_write_resume(mp, 0);
 next:
 		mtx_lock(&mountlist_mtx);
 		vfs_unbusy(mp);
diff --git a/sys/geom/mirror/g_mirror.c b/sys/geom/mirror/g_mirror.c
index 17271dd..0a8aeb0 100644
--- a/sys/geom/mirror/g_mirror.c
+++ b/sys/geom/mirror/g_mirror.c
@@ -81,7 +81,8 @@ SYSCTL_UINT(_kern_geom_mirror, OID_AUTO, sync_requests, CTLFLAG_RDTUN,
 	G_MIRROR_DEBUG(4, "%s: Woken up %p.", __func__, (ident));	\
 } while (0)
 
-static eventhandler_tag g_mirror_pre_sync = NULL;
+static eventhandler_tag g_mirror_post_sync = NULL;
+static int g_mirror_shutdown = 0;
 
 static int g_mirror_destroy_geom(struct gctl_req *req, struct g_class *mp,
     struct g_geom *gp);
@@ -456,9 +457,7 @@ g_mirror_init_disk(struct g_mirror_softc *sc, struct g_provider *pp,
 	disk->d_priority = md->md_priority;
 	disk->d_flags = md->md_dflags;
 	error = g_getattr("GEOM::candelete", disk->d_consumer, &i);
-	if (error != 0)
-		goto fail;
-	if (i)
+	if (error == 0 && i != 0)
 		disk->d_flags |= G_MIRROR_DISK_FLAG_CANDELETE;
 	if (md->md_provider[0] != '\0')
 		disk->d_flags |= G_MIRROR_DISK_FLAG_HARDCODED;
@@ -815,7 +814,7 @@ g_mirror_idle(struct g_mirror_softc *sc, int acw)
 		return (0);
 	if (acw > 0 || (acw == -1 && sc->sc_provider->acw > 0)) {
 		timeout = g_mirror_idletime - (time_uptime - sc->sc_last_write);
-		if (timeout > 0)
+		if (!g_mirror_shutdown && timeout > 0)
 			return (timeout);
 	}
 	sc->sc_idle = 1;
@@ -2821,7 +2820,7 @@ g_mirror_access(struct g_provider *pp, int acr, int acw, int ace)
 			error = ENXIO;
 		goto end;
 	}
-	if (dcw == 0 && !sc->sc_idle)
+	if (dcw == 0)
 		g_mirror_idle(sc, dcw);
 	if ((sc->sc_flags & G_MIRROR_DEVICE_FLAG_DESTROYING) != 0) {
 		if (acr > 0 || acw > 0 || ace > 0) {
@@ -3232,7 +3231,7 @@ g_mirror_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
 }
 
 static void
-g_mirror_shutdown_pre_sync(void *arg, int howto)
+g_mirror_shutdown_post_sync(void *arg, int howto)
 {
 	struct g_class *mp;
 	struct g_geom *gp, *gp2;
@@ -3242,6 +3241,7 @@ g_mirror_shutdown_pre_sync(void *arg, int howto)
 	mp = arg;
 	DROP_GIANT();
 	g_topology_lock();
+	g_mirror_shutdown = 1;
 	LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
 		if ((sc = gp->softc) == NULL)
 			continue;
@@ -3250,6 +3250,7 @@ g_mirror_shutdown_pre_sync(void *arg, int howto)
 			continue;
 		g_topology_unlock();
 		sx_xlock(&sc->sc_lock);
+		g_mirror_idle(sc, -1);
 		g_cancel_event(sc);
 		error = g_mirror_destroy(sc, G_MIRROR_DESTROY_DELAYED);
 		if (error != 0)
@@ -3264,9 +3265,9 @@ static void
 g_mirror_init(struct g_class *mp)
 {
 
-	g_mirror_pre_sync = EVENTHANDLER_REGISTER(shutdown_pre_sync,
-	    g_mirror_shutdown_pre_sync, mp, SHUTDOWN_PRI_FIRST);
-	if (g_mirror_pre_sync == NULL)
+	g_mirror_post_sync = EVENTHANDLER_REGISTER(shutdown_post_sync,
+	    g_mirror_shutdown_post_sync, mp, SHUTDOWN_PRI_FIRST);
+	if (g_mirror_post_sync == NULL)
 		G_MIRROR_DEBUG(0, "Warning! Cannot register shutdown event.");
 }
 
@@ -3274,8 +3275,8 @@ static void
 g_mirror_fini(struct g_class *mp)
 {
 
-	if (g_mirror_pre_sync != NULL)
-		EVENTHANDLER_DEREGISTER(shutdown_pre_sync, g_mirror_pre_sync);
+	if (g_mirror_post_sync != NULL)
+		EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_mirror_post_sync);
 }
 
 DECLARE_GEOM_CLASS(g_mirror_class, g_mirror);
diff --git a/sys/geom/raid/g_raid.c b/sys/geom/raid/g_raid.c
index 19b6d9b..91d14c3 100644
--- a/sys/geom/raid/g_raid.c
+++ b/sys/geom/raid/g_raid.c
@@ -163,6 +163,8 @@ g_raid_disk_state2str(int state)
 		return ("NONE");
 	case G_RAID_DISK_S_OFFLINE:
 		return ("OFFLINE");
+	case G_RAID_DISK_S_DISABLED:
+		return ("DISABLED");
 	case G_RAID_DISK_S_FAILED:
 		return ("FAILED");
 	case G_RAID_DISK_S_STALE_FAILED:
@@ -535,7 +537,9 @@ g_raid_report_disk_state(struct g_raid_disk *disk)
 
 	if (disk->d_consumer == NULL)
 		return;
-	if (disk->d_state == G_RAID_DISK_S_FAILED ||
+	if (disk->d_state == G_RAID_DISK_S_DISABLED) {
+		s = G_STATE_ACTIVE; /* XXX */
+	} else if (disk->d_state == G_RAID_DISK_S_FAILED ||
 	    disk->d_state == G_RAID_DISK_S_STALE_FAILED) {
 		s = G_STATE_FAILED;
 	} else {
diff --git a/sys/geom/raid/g_raid.h b/sys/geom/raid/g_raid.h
index 20b0822..183edff6 100644
--- a/sys/geom/raid/g_raid.h
+++ b/sys/geom/raid/g_raid.h
@@ -140,11 +140,12 @@ struct g_raid_event {
 };
 #define G_RAID_DISK_S_NONE		0x00	/* State is unknown. */
 #define G_RAID_DISK_S_OFFLINE		0x01	/* Missing disk placeholder. */
-#define G_RAID_DISK_S_FAILED		0x02	/* Failed. */
-#define G_RAID_DISK_S_STALE_FAILED	0x03	/* Old failed. */
-#define G_RAID_DISK_S_SPARE		0x04	/* Hot-spare. */
-#define G_RAID_DISK_S_STALE		0x05	/* Old disk, unused now. */
-#define G_RAID_DISK_S_ACTIVE		0x06	/* Operational. */
+#define G_RAID_DISK_S_DISABLED		0x02	/* Disabled. */
+#define G_RAID_DISK_S_FAILED		0x03	/* Failed. */
+#define G_RAID_DISK_S_STALE_FAILED	0x04	/* Old failed. */
+#define G_RAID_DISK_S_SPARE		0x05	/* Hot-spare. */
+#define G_RAID_DISK_S_STALE		0x06	/* Old disk, unused now. */
+#define G_RAID_DISK_S_ACTIVE		0x07	/* Operational. */
 
 #define G_RAID_DISK_E_DISCONNECTED	0x01
 
diff --git a/sys/geom/raid/md_intel.c b/sys/geom/raid/md_intel.c
index 12122b0..eeb42d5 100644
--- a/sys/geom/raid/md_intel.c
+++ b/sys/geom/raid/md_intel.c
@@ -98,6 +98,9 @@ struct intel_raid_vol {
 	uint8_t		cng_master_disk;
 	uint16_t	cache_policy;
 	uint8_t		cng_state;
+#define INTEL_CNGST_UPDATED		0
+#define INTEL_CNGST_NEEDS_UPDATE	1
+#define INTEL_CNGST_MASTER_MISSING	2
 	uint8_t		cng_sub_state;
 	uint32_t	filler_0[10];
 
@@ -130,6 +133,7 @@ struct intel_raid_disk {
 #define INTEL_F_ASSIGNED	0x02
 #define INTEL_F_FAILED		0x04
 #define INTEL_F_ONLINE		0x08
+#define INTEL_F_DISABLED	0x80
 	uint32_t	owner_cfg_num;
 	uint32_t	sectors_hi;
 	uint32_t	filler[3];
@@ -162,18 +166,33 @@ struct intel_raid_conf {
 #define INTEL_ATTR_RAID1E	0x00000008
 #define INTEL_ATTR_RAID5	0x00000010
 #define INTEL_ATTR_RAIDCNG	0x00000020
+#define INTEL_ATTR_EXT_STRIP	0x00000040
+#define INTEL_ATTR_NVM_CACHE	0x02000000
+#define INTEL_ATTR_2TB_DISK	0x04000000
+#define INTEL_ATTR_BBM		0x08000000
+#define INTEL_ATTR_NVM_CACHE2	0x10000000
 #define INTEL_ATTR_2TB		0x20000000
 #define INTEL_ATTR_PM		0x40000000
 #define INTEL_ATTR_CHECKSUM	0x80000000
 
 	uint8_t		total_disks;
 	uint8_t		total_volumes;
-	uint8_t		dummy_2[2];
-	uint32_t	filler_0[39];
+	uint8_t		error_log_pos;
+	uint8_t		dummy_2[1];
+	uint32_t	cache_size;
+	uint32_t	orig_config_id;
+	uint32_t	pwr_cycle_count;
+	uint32_t	bbm_log_size;
+	uint32_t	filler_0[35];
 	struct intel_raid_disk	disk[1];	/* total_disks entries. */
 	/* Here goes total_volumes of struct intel_raid_vol. */
 } __packed;
 
+#define INTEL_ATTR_SUPPORTED	( INTEL_ATTR_RAID0 | INTEL_ATTR_RAID1 |	\
+    INTEL_ATTR_RAID10 | INTEL_ATTR_RAID1E | INTEL_ATTR_RAID5 |		\
+    INTEL_ATTR_RAIDCNG | INTEL_ATTR_EXT_STRIP | INTEL_ATTR_2TB_DISK |	\
+    INTEL_ATTR_2TB | INTEL_ATTR_PM | INTEL_ATTR_CHECKSUM )
+
 #define INTEL_MAX_MD_SIZE(ndisks)				\
     (sizeof(struct intel_raid_conf) +				\
      sizeof(struct intel_raid_disk) * (ndisks - 1) +		\
@@ -187,9 +206,17 @@ struct g_raid_md_intel_perdisk {
 	struct intel_raid_disk	 pd_disk_meta;
 };
 
+struct g_raid_md_intel_pervolume {
+	int			 pv_volume_pos;
+	int			 pv_cng;
+	int			 pv_cng_man_sync;
+	int			 pv_cng_master_disk;
+};
+
 struct g_raid_md_intel_object {
 	struct g_raid_md_object	 mdio_base;
 	uint32_t		 mdio_config_id;
+	uint32_t		 mdio_orig_config_id;
 	uint32_t		 mdio_generation;
 	struct intel_raid_conf	*mdio_meta;
 	struct callout		 mdio_start_co;	/* STARTING state timer. */
@@ -206,6 +233,7 @@ static g_raid_md_ctl_t g_raid_md_ctl_intel;
 static g_raid_md_write_t g_raid_md_write_intel;
 static g_raid_md_fail_disk_t g_raid_md_fail_disk_intel;
 static g_raid_md_free_disk_t g_raid_md_free_disk_intel;
+static g_raid_md_free_volume_t g_raid_md_free_volume_intel;
 static g_raid_md_free_t g_raid_md_free_intel;
 
 static kobj_method_t g_raid_md_intel_methods[] = {
@@ -216,6 +244,7 @@ static kobj_method_t g_raid_md_intel_methods[] = {
 	KOBJMETHOD(g_raid_md_write,	g_raid_md_write_intel),
 	KOBJMETHOD(g_raid_md_fail_disk,	g_raid_md_fail_disk_intel),
 	KOBJMETHOD(g_raid_md_free_disk,	g_raid_md_free_disk_intel),
+	KOBJMETHOD(g_raid_md_free_volume,	g_raid_md_free_volume_intel),
 	KOBJMETHOD(g_raid_md_free,	g_raid_md_free_intel),
 	{ 0, 0 }
 };
@@ -354,29 +383,45 @@ g_raid_md_intel_print(struct intel_raid_conf *meta)
 	printf("config_size         0x%08x\n", meta->config_size);
 	printf("config_id           0x%08x\n", meta->config_id);
 	printf("generation          0x%08x\n", meta->generation);
+	printf("error_log_size      %d\n", meta->error_log_size);
 	printf("attributes          0x%08x\n", meta->attributes);
 	printf("total_disks         %u\n", meta->total_disks);
 	printf("total_volumes       %u\n", meta->total_volumes);
-	printf("DISK#   serial disk_sectors disk_sectors_hi disk_id flags\n");
+	printf("error_log_pos       %u\n", meta->error_log_pos);
+	printf("cache_size          %u\n", meta->cache_size);
+	printf("orig_config_id      0x%08x\n", meta->orig_config_id);
+	printf("pwr_cycle_count     %u\n", meta->pwr_cycle_count);
+	printf("bbm_log_size        %u\n", meta->bbm_log_size);
+	printf("DISK#   serial disk_sectors disk_sectors_hi disk_id flags owner\n");
 	for (i = 0; i < meta->total_disks; i++ ) {
-		printf("    %d   <%.16s> %u %u 0x%08x 0x%08x\n", i,
+		printf("    %d   <%.16s> %u %u 0x%08x 0x%08x %08x\n", i,
 		    meta->disk[i].serial, meta->disk[i].sectors,
-		    meta->disk[i].sectors_hi,
-		    meta->disk[i].id, meta->disk[i].flags);
+		    meta->disk[i].sectors_hi, meta->disk[i].id,
+		    meta->disk[i].flags, meta->disk[i].owner_cfg_num);
 	}
 	for (i = 0; i < meta->total_volumes; i++) {
 		mvol = intel_get_volume(meta, i);
 		printf(" ****** Volume %d ******\n", i);
 		printf(" name               %.16s\n", mvol->name);
 		printf(" total_sectors      %ju\n", mvol->total_sectors);
-		printf(" state              %u\n", mvol->state);
+		printf(" state              0x%08x\n", mvol->state);
 		printf(" reserved           %u\n", mvol->reserved);
+		printf(" migr_priority      %u\n", mvol->migr_priority);
+		printf(" num_sub_vols       %u\n", mvol->num_sub_vols);
+		printf(" tid                %u\n", mvol->tid);
+		printf(" cng_master_disk    %u\n", mvol->cng_master_disk);
+		printf(" cache_policy       %u\n", mvol->cache_policy);
+		printf(" cng_state          %u\n", mvol->cng_state);
+		printf(" cng_sub_state      %u\n", mvol->cng_sub_state);
 		printf(" curr_migr_unit     %u\n", mvol->curr_migr_unit);
 		printf(" curr_migr_unit_hi  %u\n", mvol->curr_migr_unit_hi);
 		printf(" checkpoint_id      %u\n", mvol->checkpoint_id);
 		printf(" migr_state         %u\n", mvol->migr_state);
 		printf(" migr_type          %u\n", mvol->migr_type);
 		printf(" dirty              %u\n", mvol->dirty);
+		printf(" fs_state           %u\n", mvol->fs_state);
+		printf(" verify_errors      %u\n", mvol->verify_errors);
+		printf(" bad_blocks         %u\n", mvol->bad_blocks);
 
 		for (j = 0; j < (mvol->migr_state ? 2 : 1); j++) {
 			printf("  *** Map %d ***\n", j);
@@ -432,7 +477,7 @@ intel_meta_read(struct g_consumer *cp)
 	struct g_provider *pp;
 	struct intel_raid_conf *meta;
 	struct intel_raid_vol *mvol;
-	struct intel_raid_map *mmap;
+	struct intel_raid_map *mmap, *mmap1;
 	char *buf;
 	int error, i, j, k, left, size;
 	uint32_t checksum, *ptr;
@@ -525,6 +570,23 @@ badsize:
 		}
 	}
 
+	g_raid_md_intel_print(meta);
+
+	if (strncmp(meta->version, INTEL_VERSION_1300, 6) > 0) {
+		G_RAID_DEBUG(1, "Intel unsupported version: '%.6s'",
+		    meta->version);
+		free(meta, M_MD_INTEL);
+		return (NULL);
+	}
+
+	if (strncmp(meta->version, INTEL_VERSION_1300, 6) >= 0 &&
+	    (meta->attributes & ~INTEL_ATTR_SUPPORTED) != 0) {
+		G_RAID_DEBUG(1, "Intel unsupported attributes: 0x%08x",
+		    meta->attributes & ~INTEL_ATTR_SUPPORTED);
+		free(meta, M_MD_INTEL);
+		return (NULL);
+	}
+
 	/* Validate disk indexes. */
 	for (i = 0; i < meta->total_volumes; i++) {
 		mvol = intel_get_volume(meta, i);
@@ -547,16 +609,39 @@ badsize:
 	/* Validate migration types. */
 	for (i = 0; i < meta->total_volumes; i++) {
 		mvol = intel_get_volume(meta, i);
+		/* Deny unknown migration types. */
 		if (mvol->migr_state &&
 		    mvol->migr_type != INTEL_MT_INIT &&
 		    mvol->migr_type != INTEL_MT_REBUILD &&
 		    mvol->migr_type != INTEL_MT_VERIFY &&
+		    mvol->migr_type != INTEL_MT_GEN_MIGR &&
 		    mvol->migr_type != INTEL_MT_REPAIR) {
 			G_RAID_DEBUG(1, "Intel metadata has unsupported"
 			    " migration type %d", mvol->migr_type);
 			free(meta, M_MD_INTEL);
 			return (NULL);
 		}
+		/* Deny general migrations except SINGLE->RAID1. */
+		if (mvol->migr_state &&
+		    mvol->migr_type == INTEL_MT_GEN_MIGR) {
+			mmap = intel_get_map(mvol, 0);
+			mmap1 = intel_get_map(mvol, 1);
+			if (mmap1->total_disks != 1 ||
+			    mmap->type != INTEL_T_RAID1 ||
+			    mmap->total_disks != 2 ||
+			    mmap->offset != mmap1->offset ||
+			    mmap->disk_sectors != mmap1->disk_sectors ||
+			    mmap->total_domains != mmap->total_disks ||
+			    mmap->offset_hi != mmap1->offset_hi ||
+			    mmap->disk_sectors_hi != mmap1->disk_sectors_hi ||
+			    (mmap->disk_idx[0] != mmap1->disk_idx[0] &&
+			     mmap->disk_idx[0] != mmap1->disk_idx[1])) {
+				G_RAID_DEBUG(1, "Intel metadata has unsupported"
+				    " variant of general migration");
+				free(meta, M_MD_INTEL);
+				return (NULL);
+			}
+		}
 	}
 
 	return (meta);
@@ -633,7 +718,7 @@ intel_meta_write_spare(struct g_consumer *cp, struct intel_raid_disk *d)
 	memcpy(&meta->version[0], INTEL_VERSION_1000,
 	    sizeof(INTEL_VERSION_1000) - 1);
 	meta->config_size = INTEL_MAX_MD_SIZE(1);
-	meta->config_id = arc4random();
+	meta->config_id = meta->orig_config_id = arc4random();
 	meta->generation = 1;
 	meta->total_disks = 1;
 	meta->disk[0] = *d;
@@ -699,9 +784,11 @@ static struct g_raid_volume *
 g_raid_md_intel_get_volume(struct g_raid_softc *sc, int id)
 {
 	struct g_raid_volume	*mvol;
+	struct g_raid_md_intel_pervolume *pv;
 
 	TAILQ_FOREACH(mvol, &sc->sc_volumes, v_next) {
-		if ((intptr_t)(mvol->v_md_data) == id)
+		pv = mvol->v_md_data;
+		if (pv->pv_volume_pos == id)
 			break;
 	}
 	return (mvol);
@@ -715,11 +802,12 @@ g_raid_md_intel_start_disk(struct g_raid_disk *disk)
 	struct g_raid_disk *olddisk, *tmpdisk;
 	struct g_raid_md_object *md;
 	struct g_raid_md_intel_object *mdi;
+	struct g_raid_md_intel_pervolume *pv;
 	struct g_raid_md_intel_perdisk *pd, *oldpd;
 	struct intel_raid_conf *meta;
 	struct intel_raid_vol *mvol;
 	struct intel_raid_map *mmap0, *mmap1;
-	int disk_pos, resurrection = 0;
+	int disk_pos, resurrection = 0, migr_global, i;
 
 	sc = disk->d_softc;
 	md = sc->sc_md;
@@ -733,7 +821,8 @@ g_raid_md_intel_start_disk(struct g_raid_disk *disk)
 	if (disk_pos < 0) {
 		G_RAID_DEBUG1(1, sc, "Unknown, probably new or stale disk");
 		/* Failed stale disk is useless for us. */
-		if (pd->pd_disk_meta.flags & INTEL_F_FAILED) {
+		if ((pd->pd_disk_meta.flags & INTEL_F_FAILED) &&
+		    !(pd->pd_disk_meta.flags & INTEL_F_DISABLED)) {
 			g_raid_change_disk_state(disk, G_RAID_DISK_S_STALE_FAILED);
 			return (0);
 		}
@@ -824,7 +913,10 @@ nofit:
 	}
 
 	/* Welcome the new disk. */
-	if (resurrection)
+	if ((meta->disk[disk_pos].flags & INTEL_F_DISABLED) &&
+	    !(pd->pd_disk_meta.flags & INTEL_F_SPARE))
+		g_raid_change_disk_state(disk, G_RAID_DISK_S_DISABLED);
+	else if (resurrection)
 		g_raid_change_disk_state(disk, G_RAID_DISK_S_ACTIVE);
 	else if (meta->disk[disk_pos].flags & INTEL_F_FAILED)
 		g_raid_change_disk_state(disk, G_RAID_DISK_S_FAILED);
@@ -833,15 +925,27 @@ nofit:
 	else
 		g_raid_change_disk_state(disk, G_RAID_DISK_S_ACTIVE);
 	TAILQ_FOREACH(sd, &disk->d_subdisks, sd_next) {
-		mvol = intel_get_volume(meta,
-		    (uintptr_t)(sd->sd_volume->v_md_data));
+		pv = sd->sd_volume->v_md_data;
+		mvol = intel_get_volume(meta, pv->pv_volume_pos);
 		mmap0 = intel_get_map(mvol, 0);
 		if (mvol->migr_state)
 			mmap1 = intel_get_map(mvol, 1);
 		else
 			mmap1 = mmap0;
 
-		if (resurrection) {
+		migr_global = 1;
+		for (i = 0; i < mmap0->total_disks; i++) {
+			if ((mmap0->disk_idx[i] & INTEL_DI_RBLD) == 0 &&
+			    (mmap1->disk_idx[i] & INTEL_DI_RBLD) != 0)
+				migr_global = 0;
+		}
+
+		if ((meta->disk[disk_pos].flags & INTEL_F_DISABLED) &&
+		    !(pd->pd_disk_meta.flags & INTEL_F_SPARE)) {
+			/* Disabled disk, useless. */
+			g_raid_change_subdisk_state(sd,
+			    G_RAID_SUBDISK_S_NONE);
+		} else if (resurrection) {
 			/* Stale disk, almost same as new. */
 			g_raid_change_subdisk_state(sd,
 			    G_RAID_SUBDISK_S_NEW);
@@ -850,7 +954,8 @@ nofit:
 			g_raid_change_subdisk_state(sd,
 			    G_RAID_SUBDISK_S_FAILED);
 		} else if (mvol->migr_state == 0) {
-			if (mmap0->status == INTEL_S_UNINITIALIZED) {
+			if (mmap0->status == INTEL_S_UNINITIALIZED &&
+			    (!pv->pv_cng || pv->pv_cng_master_disk != disk_pos)) {
 				/* Freshly created uninitialized volume. */
 				g_raid_change_subdisk_state(sd,
 				    G_RAID_SUBDISK_S_UNINITIALIZED);
@@ -858,7 +963,8 @@ nofit:
 				/* Freshly inserted disk. */
 				g_raid_change_subdisk_state(sd,
 				    G_RAID_SUBDISK_S_NEW);
-			} else if (mvol->dirty) {
+			} else if (mvol->dirty && (!pv->pv_cng ||
+			    pv->pv_cng_master_disk != disk_pos)) {
 				/* Dirty volume (unclean shutdown). */
 				g_raid_change_subdisk_state(sd,
 				    G_RAID_SUBDISK_S_STALE);
@@ -885,7 +991,13 @@ nofit:
 					    sd->sd_volume->v_strip_size *
 					    mmap0->total_domains;
 				}
-			} else if (mvol->dirty) {
+			} else if (mvol->migr_type == INTEL_MT_INIT &&
+			    migr_global) {
+				/* Freshly created uninitialized volume. */
+				g_raid_change_subdisk_state(sd,
+				    G_RAID_SUBDISK_S_UNINITIALIZED);
+			} else if (mvol->dirty && (!pv->pv_cng ||
+			    pv->pv_cng_master_disk != disk_pos)) {
 				/* Dirty volume (unclean shutdown). */
 				g_raid_change_subdisk_state(sd,
 				    G_RAID_SUBDISK_S_STALE);
@@ -900,7 +1012,8 @@ nofit:
 				/* Freshly inserted disk. */
 				g_raid_change_subdisk_state(sd,
 				    G_RAID_SUBDISK_S_NEW);
-			} else if (mmap1->disk_idx[sd->sd_pos] & INTEL_DI_RBLD) {
+			} else if ((mmap1->disk_idx[sd->sd_pos] & INTEL_DI_RBLD) ||
+			    migr_global) {
 				/* Resyncing disk. */
 				g_raid_change_subdisk_state(sd,
 				    G_RAID_SUBDISK_S_RESYNC);
@@ -921,6 +1034,16 @@ nofit:
 				g_raid_change_subdisk_state(sd,
 				    G_RAID_SUBDISK_S_ACTIVE);
 			}
+		} else if (mvol->migr_type == INTEL_MT_GEN_MIGR) {
+			if ((mmap1->disk_idx[0] & INTEL_DI_IDX) != disk_pos) {
+				/* Freshly inserted disk. */
+				g_raid_change_subdisk_state(sd,
+				    G_RAID_SUBDISK_S_NEW);
+			} else {
+				/* Up to date disk. */
+				g_raid_change_subdisk_state(sd,
+				    G_RAID_SUBDISK_S_ACTIVE);
+			}
 		}
 		g_raid_event_send(sd, G_RAID_SUBDISK_E_NEW,
 		    G_RAID_EVENT_SUBDISK);
@@ -929,7 +1052,8 @@ nofit:
 	/* Update status of our need for spare. */
 	if (mdi->mdio_started) {
 		mdi->mdio_incomplete =
-		    (g_raid_ndisks(sc, G_RAID_DISK_S_ACTIVE) <
+		    (g_raid_ndisks(sc, G_RAID_DISK_S_ACTIVE) +
+		     g_raid_ndisks(sc, G_RAID_DISK_S_DISABLED) <
 		     meta->total_disks);
 	}
 
@@ -961,7 +1085,8 @@ g_raid_md_intel_refill(struct g_raid_softc *sc)
 	update = 0;
 	do {
 		/* Make sure we miss anything. */
-		na = g_raid_ndisks(sc, G_RAID_DISK_S_ACTIVE);
+		na = g_raid_ndisks(sc, G_RAID_DISK_S_ACTIVE) +
+		    g_raid_ndisks(sc, G_RAID_DISK_S_DISABLED);
 		if (na == meta->total_disks)
 			break;
 
@@ -973,7 +1098,8 @@ g_raid_md_intel_refill(struct g_raid_softc *sc)
 		TAILQ_FOREACH(disk, &sc->sc_disks, d_next) {
 			if (disk->d_state == G_RAID_DISK_S_STALE) {
 				update += g_raid_md_intel_start_disk(disk);
-				if (disk->d_state == G_RAID_DISK_S_ACTIVE)
+				if (disk->d_state == G_RAID_DISK_S_ACTIVE ||
+				    disk->d_state == G_RAID_DISK_S_DISABLED)
 					break;
 			}
 		}
@@ -997,8 +1123,8 @@ g_raid_md_intel_refill(struct g_raid_softc *sc)
 	}
 
 	/* Update status of our need for spare. */
-	mdi->mdio_incomplete = (g_raid_ndisks(sc, G_RAID_DISK_S_ACTIVE) <
-	    meta->total_disks);
+	mdi->mdio_incomplete = (g_raid_ndisks(sc, G_RAID_DISK_S_ACTIVE) +
+	    g_raid_ndisks(sc, G_RAID_DISK_S_DISABLED) < meta->total_disks);
 
 	/* Request retaste hoping to find spare. */
 	if (mdi->mdio_incomplete) {
@@ -1014,6 +1140,7 @@ g_raid_md_intel_start(struct g_raid_softc *sc)
 {
 	struct g_raid_md_object *md;
 	struct g_raid_md_intel_object *mdi;
+	struct g_raid_md_intel_pervolume *pv;
 	struct g_raid_md_intel_perdisk *pd;
 	struct intel_raid_conf *meta;
 	struct intel_raid_vol *mvol;
@@ -1031,8 +1158,14 @@ g_raid_md_intel_start(struct g_raid_softc *sc)
 	for (i = 0; i < meta->total_volumes; i++) {
 		mvol = intel_get_volume(meta, i);
 		mmap = intel_get_map(mvol, 0);
-		vol = g_raid_create_volume(sc, mvol->name, -1);
-		vol->v_md_data = (void *)(intptr_t)i;
+		vol = g_raid_create_volume(sc, mvol->name, mvol->tid - 1);
+		pv = malloc(sizeof(*pv), M_MD_INTEL, M_WAITOK | M_ZERO);
+		pv->pv_volume_pos = i;
+		pv->pv_cng = (mvol->state & INTEL_ST_CLONE_N_GO) != 0;
+		pv->pv_cng_man_sync = (mvol->state & INTEL_ST_CLONE_MAN_SYNC) != 0;
+		if (mvol->cng_master_disk < mmap->total_disks)
+			pv->pv_cng_master_disk = mvol->cng_master_disk;
+		vol->v_md_data = pv;
 		vol->v_raid_level_qualifier = G_RAID_VOLUME_RLQ_NONE;
 		if (mmap->type == INTEL_T_RAID0)
 			vol->v_raid_level = G_RAID_VOLUME_RL_RAID0;
@@ -1186,7 +1319,7 @@ g_raid_md_create_intel(struct g_raid_md_object *md, struct g_class *mp,
 	char name[16];
 
 	mdi = (struct g_raid_md_intel_object *)md;
-	mdi->mdio_config_id = arc4random();
+	mdi->mdio_config_id = mdi->mdio_orig_config_id = arc4random();
 	mdi->mdio_generation = 0;
 	snprintf(name, sizeof(name), "Intel-%08x", mdi->mdio_config_id);
 	sc = g_raid_create_node(mp, name, md);
@@ -1294,8 +1427,6 @@ g_raid_md_taste_intel(struct g_raid_md_object *md, struct g_class *mp,
 		goto fail1;
 	}
 
-	/* Metadata valid. Print it. */
-	g_raid_md_intel_print(meta);
 	G_RAID_DEBUG(1, "Intel disk position %d", disk_pos);
 	spare = meta->disk[disk_pos].flags & INTEL_F_SPARE;
 
@@ -1333,6 +1464,7 @@ search:
 	} else { /* Not found matching node -- create one. */
 		result = G_RAID_MD_TASTE_NEW;
 		mdi->mdio_config_id = meta->config_id;
+		mdi->mdio_orig_config_id = meta->orig_config_id;
 		snprintf(name, sizeof(name), "Intel-%08x", meta->config_id);
 		sc = g_raid_create_node(mp, name, md);
 		md->mdo_softc = sc;
@@ -1450,6 +1582,7 @@ g_raid_md_ctl_intel(struct g_raid_md_object *md,
 	struct g_raid_subdisk *sd;
 	struct g_raid_disk *disk;
 	struct g_raid_md_intel_object *mdi;
+	struct g_raid_md_intel_pervolume *pv;
 	struct g_raid_md_intel_perdisk *pd;
 	struct g_consumer *cp;
 	struct g_provider *pp;
@@ -1621,7 +1754,9 @@ g_raid_md_ctl_intel(struct g_raid_md_object *md,
 		/* We have all we need, create things: volume, ... */
 		mdi->mdio_started = 1;
 		vol = g_raid_create_volume(sc, volname, -1);
-		vol->v_md_data = (void *)(intptr_t)0;
+		pv = malloc(sizeof(*pv), M_MD_INTEL, M_WAITOK | M_ZERO);
+		pv->pv_volume_pos = 0;
+		vol->v_md_data = pv;
 		vol->v_raid_level = level;
 		vol->v_raid_level_qualifier = qual;
 		vol->v_strip_size = strip;
@@ -1814,7 +1949,9 @@ g_raid_md_ctl_intel(struct g_raid_md_object *md,
 
 		/* We have all we need, create things: volume, ... */
 		vol = g_raid_create_volume(sc, volname, -1);
-		vol->v_md_data = (void *)(intptr_t)i;
+		pv = malloc(sizeof(*pv), M_MD_INTEL, M_WAITOK | M_ZERO);
+		pv->pv_volume_pos = i;
+		vol->v_md_data = pv;
 		vol->v_raid_level = level;
 		vol->v_raid_level_qualifier = qual;
 		vol->v_strip_size = strip;
@@ -2105,6 +2242,7 @@ g_raid_md_write_intel(struct g_raid_md_object *md, struct g_raid_volume *tvol,
 	struct g_raid_subdisk *sd;
 	struct g_raid_disk *disk;
 	struct g_raid_md_intel_object *mdi;
+	struct g_raid_md_intel_pervolume *pv;
 	struct g_raid_md_intel_perdisk *pd;
 	struct intel_raid_conf *meta;
 	struct intel_raid_vol *mvol;
@@ -2133,9 +2271,14 @@ g_raid_md_write_intel(struct g_raid_md_object *md, struct g_raid_volume *tvol,
 			pd->pd_disk_meta.flags =
 			    INTEL_F_ONLINE | INTEL_F_ASSIGNED;
 		} else if (disk->d_state == G_RAID_DISK_S_FAILED) {
-			pd->pd_disk_meta.flags = INTEL_F_FAILED | INTEL_F_ASSIGNED;
+			pd->pd_disk_meta.flags = INTEL_F_FAILED |
+			    INTEL_F_ASSIGNED;
+		} else if (disk->d_state == G_RAID_DISK_S_DISABLED) {
+			pd->pd_disk_meta.flags = INTEL_F_FAILED |
+			    INTEL_F_ASSIGNED | INTEL_F_DISABLED;
 		} else {
-			pd->pd_disk_meta.flags = INTEL_F_ASSIGNED;
+			if (!(pd->pd_disk_meta.flags & INTEL_F_DISABLED))
+				pd->pd_disk_meta.flags = INTEL_F_ASSIGNED;
 			if (pd->pd_disk_meta.id != 0xffffffff) {
 				pd->pd_disk_meta.id = 0xffffffff;
 				len = strlen(pd->pd_disk_meta.serial);
@@ -2151,6 +2294,7 @@ g_raid_md_write_intel(struct g_raid_md_object *md, struct g_raid_volume *tvol,
 	memcpy(&meta->intel_id[0], INTEL_MAGIC, sizeof(INTEL_MAGIC) - 1);
 	meta->config_size = INTEL_MAX_MD_SIZE(numdisks);
 	meta->config_id = mdi->mdio_config_id;
+	meta->orig_config_id = mdi->mdio_orig_config_id;
 	meta->generation = mdi->mdio_generation;
 	meta->attributes = INTEL_ATTR_CHECKSUM;
 	meta->total_disks = numdisks;
@@ -2159,18 +2303,21 @@ g_raid_md_write_intel(struct g_raid_md_object *md, struct g_raid_volume *tvol,
 		if (pd->pd_disk_pos < 0)
 			continue;
 		meta->disk[pd->pd_disk_pos] = pd->pd_disk_meta;
+		if (pd->pd_disk_meta.sectors_hi != 0)
+			meta->attributes |= INTEL_ATTR_2TB_DISK;
 	}
 
 	/* Fill volumes and maps. */
 	vi = 0;
 	version = INTEL_VERSION_1000;
 	TAILQ_FOREACH(vol, &sc->sc_volumes, v_next) {
+		pv = vol->v_md_data;
 		if (vol->v_stopping)
 			continue;
 		mvol = intel_get_volume(meta, vi);
 
 		/* New metadata may have different volumes order. */
-		vol->v_md_data = (void *)(intptr_t)vi;
+		pv->pv_volume_pos = vi;
 
 		for (sdi = 0; sdi < vol->v_disks_count; sdi++) {
 			sd = &vol->v_subdisks[sdi];
@@ -2187,21 +2334,23 @@ g_raid_md_write_intel(struct g_raid_md_object *md, struct g_raid_volume *tvol,
 			meta->attributes |= INTEL_ATTR_RAID1;
 		else if (vol->v_raid_level == G_RAID_VOLUME_RL_RAID5)
 			meta->attributes |= INTEL_ATTR_RAID5;
-		else
+		else if ((vol->v_disks_count & 1) == 0)
 			meta->attributes |= INTEL_ATTR_RAID10;
-
-		if (meta->attributes & INTEL_ATTR_2TB)
-			cv = INTEL_VERSION_1300;
-//		else if (dev->status == DEV_CLONE_N_GO)
-//			cv = INTEL_VERSION_1206;
+		else
+			meta->attributes |= INTEL_ATTR_RAID1E;
+		if (pv->pv_cng)
+			meta->attributes |= INTEL_ATTR_RAIDCNG;
+		if (vol->v_strip_size > 131072)
+			meta->attributes |= INTEL_ATTR_EXT_STRIP;
+
+		if (pv->pv_cng)
+			cv = INTEL_VERSION_1206;
 		else if (vol->v_disks_count > 4)
 			cv = INTEL_VERSION_1204;
 		else if (vol->v_raid_level == G_RAID_VOLUME_RL_RAID5)
 			cv = INTEL_VERSION_1202;
 		else if (vol->v_disks_count > 2)
 			cv = INTEL_VERSION_1201;
-		else if (vi > 0)
-			cv = INTEL_VERSION_1200;
 		else if (vol->v_raid_level == G_RAID_VOLUME_RL_RAID1)
 			cv = INTEL_VERSION_1100;
 		else
@@ -2211,6 +2360,22 @@ g_raid_md_write_intel(struct g_raid_md_object *md, struct g_raid_volume *tvol,
 
 		strlcpy(&mvol->name[0], vol->v_name, sizeof(mvol->name));
 		mvol->total_sectors = vol->v_mediasize / sectorsize;
+		mvol->state = (INTEL_ST_READ_COALESCING |
+		    INTEL_ST_WRITE_COALESCING);
+		mvol->tid = vol->v_global_id + 1;
+		if (pv->pv_cng) {
+			mvol->state |= INTEL_ST_CLONE_N_GO;
+			if (pv->pv_cng_man_sync)
+				mvol->state |= INTEL_ST_CLONE_MAN_SYNC;
+			mvol->cng_master_disk = pv->pv_cng_master_disk;
+			if (vol->v_subdisks[pv->pv_cng_master_disk].sd_state ==
+			    G_RAID_SUBDISK_S_NONE)
+				mvol->cng_state = INTEL_CNGST_MASTER_MISSING;
+			else if (vol->v_state != G_RAID_VOLUME_S_OPTIMAL)
+				mvol->cng_state = INTEL_CNGST_NEEDS_UPDATE;
+			else
+				mvol->cng_state = INTEL_CNGST_UPDATED;
+		}
 
 		/* Check for any recovery in progress. */
 		state = G_RAID_SUBDISK_S_ACTIVE;
@@ -2305,7 +2470,8 @@ g_raid_md_write_intel(struct g_raid_md_object *md, struct g_raid_volume *tvol,
 					mmap1->disk_idx[sdi] |= INTEL_DI_RBLD;
 			}
 			if ((sd->sd_state == G_RAID_SUBDISK_S_NONE ||
-			     sd->sd_state == G_RAID_SUBDISK_S_FAILED) &&
+			     sd->sd_state == G_RAID_SUBDISK_S_FAILED ||
+			     sd->sd_state == G_RAID_SUBDISK_S_REBUILD) &&
 			    mmap0->failed_disk_num == 0xff) {
 				mmap0->failed_disk_num = sdi;
 				if (mvol->migr_state)
@@ -2315,7 +2481,10 @@ g_raid_md_write_intel(struct g_raid_md_object *md, struct g_raid_volume *tvol,
 		vi++;
 	}
 	meta->total_volumes = vi;
-	if (strcmp(version, INTEL_VERSION_1300) != 0)
+	if (vi > 1 || meta->attributes &
+	     (INTEL_ATTR_EXT_STRIP | INTEL_ATTR_2TB_DISK | INTEL_ATTR_2TB))
+		version = INTEL_VERSION_1300;
+	if (strcmp(version, INTEL_VERSION_1300) < 0)
 		meta->attributes &= INTEL_ATTR_CHECKSUM;
 	memcpy(&meta->version[0], version, sizeof(INTEL_VERSION_1000) - 1);
 
@@ -2403,6 +2572,18 @@ g_raid_md_free_disk_intel(struct g_raid_md_object *md,
 }
 
 static int
+g_raid_md_free_volume_intel(struct g_raid_md_object *md,
+    struct g_raid_volume *vol)
+{
+	struct g_raid_md_intel_pervolume *pv;
+
+	pv = (struct g_raid_md_intel_pervolume *)vol->v_md_data;
+	free(pv, M_MD_INTEL);
+	vol->v_md_data = NULL;
+	return (0);
+}
+
+static int
 g_raid_md_free_intel(struct g_raid_md_object *md)
 {
 	struct g_raid_md_intel_object *mdi;
diff --git a/sys/geom/raid/md_promise.c b/sys/geom/raid/md_promise.c
index 5512b5a..0007b20 100644
--- a/sys/geom/raid/md_promise.c
+++ b/sys/geom/raid/md_promise.c
@@ -84,7 +84,7 @@ struct promise_raid_conf {
 	struct promise_raid_disk	disk;	/* This subdisk info. */
 	uint32_t	disk_offset;		/* Subdisk offset. */
 	uint32_t	disk_sectors;		/* Subdisk size */
-	uint32_t	rebuild_lba;		/* Rebuild position. */
+	uint32_t	disk_rebuild;		/* Rebuild position. */
 	uint16_t	generation;		/* Generation number. */
 	uint8_t		status;			/* Volume status. */
 #define PROMISE_S_VALID		0x01
@@ -123,7 +123,18 @@ struct promise_raid_conf {
 	uint32_t	magic_4;
 	uint32_t	magic_5;
 	uint32_t	total_sectors_high;
-	uint32_t	filler3[324];
+	uint8_t		magic_6;
+	uint8_t		sector_size;
+	uint16_t	magic_7;
+	uint32_t	magic_8[31];
+	uint32_t	backup_time;
+	uint16_t	magic_9;
+	uint32_t	disk_offset_high;
+	uint32_t	disk_sectors_high;
+	uint32_t	disk_rebuild_high;
+	uint16_t	magic_10;
+	uint32_t	magic_11[3];
+	uint32_t	filler3[284];
 	uint32_t	checksum;
 } __packed;
 
@@ -191,7 +202,7 @@ g_raid_md_promise_print(struct promise_raid_conf *meta)
 	    meta->disk.device, meta->disk.id);
 	printf("disk_offset         %u\n", meta->disk_offset);
 	printf("disk_sectors        %u\n", meta->disk_sectors);
-	printf("rebuild_lba         %u\n", meta->rebuild_lba);
+	printf("disk_rebuild        %u\n", meta->disk_rebuild);
 	printf("generation          %u\n", meta->generation);
 	printf("status              0x%02x\n", meta->status);
 	printf("type                %u\n", meta->type);
@@ -217,6 +228,11 @@ g_raid_md_promise_print(struct promise_raid_conf *meta)
 	printf("magic_4             0x%08x\n", meta->magic_4);
 	printf("magic_5             0x%08x\n", meta->magic_5);
 	printf("total_sectors_high  0x%08x\n", meta->total_sectors_high);
+	printf("sector_size         %u\n", meta->sector_size);
+	printf("backup_time         %d\n", meta->backup_time);
+	printf("disk_offset_high    0x%08x\n", meta->disk_offset_high);
+	printf("disk_sectors_high   0x%08x\n", meta->disk_sectors_high);
+	printf("disk_rebuild_high   0x%08x\n", meta->disk_rebuild_high);
 	printf("=================================================\n");
 }
 
@@ -244,9 +260,9 @@ promise_meta_find_disk(struct promise_raid_conf *meta, uint64_t id)
 
 static int
 promise_meta_unused_range(struct promise_raid_conf **metaarr, int nsd,
-    uint32_t sectors, uint32_t *off, uint32_t *size)
+    off_t sectors, off_t *off, off_t *size)
 {
-	uint32_t coff, csize;
+	off_t coff, csize, tmp;
 	int i, j;
 
 	sectors -= 131072;
@@ -257,10 +273,10 @@ promise_meta_unused_range(struct promise_raid_conf **metaarr, int nsd,
 	i = 0;
 	while (1) {
 		for (j = 0; j < nsd; j++) {
-			if (metaarr[j]->disk_offset >= coff) {
-				csize = MIN(csize,
-				    metaarr[j]->disk_offset - coff);
-			}
+			tmp = ((off_t)metaarr[j]->disk_offset_high << 32) +
+			    metaarr[j]->disk_offset;
+			if (tmp >= coff)
+				csize = MIN(csize, tmp - coff);
 		}
 		if (csize > *size) {
 			*off = coff;
@@ -268,7 +284,10 @@ promise_meta_unused_range(struct promise_raid_conf **metaarr, int nsd,
 		}
 		if (i >= nsd)
 			break;
-		coff = metaarr[i]->disk_offset + metaarr[i]->disk_sectors;
+		coff = ((off_t)metaarr[i]->disk_offset_high << 32) +
+		     metaarr[i]->disk_offset +
+		    ((off_t)metaarr[i]->disk_sectors_high << 32) +
+		     metaarr[i]->disk_sectors;
 		csize = sectors - coff;
 		i++;
 	};
@@ -369,6 +388,26 @@ next:
 		return (subdisks);
 	}
 
+	/* Remove filler garbage from fields used in newer metadata. */
+	if (meta->disk_offset_high == 0x8b8c8d8e &&
+	    meta->disk_sectors_high == 0x8788898a &&
+	    meta->disk_rebuild_high == 0x83848586) {
+		meta->disk_offset_high = 0;
+		meta->disk_sectors_high = 0;
+		if (meta->disk_rebuild == UINT32_MAX)
+			meta->disk_rebuild_high = UINT32_MAX;
+		else
+			meta->disk_rebuild_high = 0;
+		if (meta->total_sectors_high == 0x15161718) {
+			meta->total_sectors_high = 0;
+			meta->backup_time = 0;
+			if (meta->rebuild_lba64 == 0x2122232425262728)
+				meta->rebuild_lba64 = UINT64_MAX;
+		}
+	}
+	if (meta->sector_size < 1 || meta->sector_size > 8)
+		meta->sector_size = 1;
+
 	/* Save this part and look for next. */
 	*metaarr = meta;
 	metaarr++;
@@ -386,8 +425,9 @@ promise_meta_write(struct g_consumer *cp,
 	struct g_provider *pp;
 	struct promise_raid_conf *meta;
 	char *buf;
+	off_t off, size;
 	int error, i, subdisk, fake;
-	uint32_t checksum, *ptr, off, size;
+	uint32_t checksum, *ptr;
 
 	pp = cp->provider;
 	subdisk = 0;
@@ -409,9 +449,12 @@ next:
 		meta->disk.flags = PROMISE_F_ONLINE | PROMISE_F_VALID;
 		meta->disk.number = 0xff;
 		arc4rand(&meta->disk.id, sizeof(meta->disk.id), 0);
-		meta->disk_offset = off;
-		meta->disk_sectors = size;
-		meta->rebuild_lba = UINT32_MAX;
+		meta->disk_offset_high = off >> 32;
+		meta->disk_offset = (uint32_t)off;
+		meta->disk_sectors_high = size >> 32;
+		meta->disk_sectors = (uint32_t)size;
+		meta->disk_rebuild_high = UINT32_MAX;
+		meta->disk_rebuild = UINT32_MAX;
 		fake = 1;
 	}
 	if (meta != NULL) {
@@ -464,6 +507,7 @@ static int
 promise_meta_write_spare(struct g_consumer *cp)
 {
 	struct promise_raid_conf *meta;
+	off_t tmp;
 	int error;
 
 	meta = malloc(sizeof(*meta), M_MD_PROMISE, M_WAITOK | M_ZERO);
@@ -473,9 +517,11 @@ promise_meta_write_spare(struct g_consumer *cp)
 	meta->disk.flags = PROMISE_F_SPARE | PROMISE_F_ONLINE | PROMISE_F_VALID;
 	meta->disk.number = 0xff;
 	arc4rand(&meta->disk.id, sizeof(meta->disk.id), 0);
-	meta->disk_sectors = cp->provider->mediasize / cp->provider->sectorsize;
-	meta->disk_sectors -= 131072;
-	meta->rebuild_lba = UINT32_MAX;
+	tmp = cp->provider->mediasize / cp->provider->sectorsize - 131072;
+	meta->disk_sectors_high = tmp >> 32;
+	meta->disk_sectors = (uint32_t)tmp;
+	meta->disk_rebuild_high = UINT32_MAX;
+	meta->disk_rebuild = UINT32_MAX;
 	error = promise_meta_write(cp, &meta, 1);
 	free(meta, M_MD_PROMISE);
 	return (error);
@@ -617,9 +663,8 @@ g_raid_md_promise_start_disk(struct g_raid_disk *disk, int sdn,
 	struct g_raid_md_promise_perdisk *pd;
 	struct g_raid_md_promise_pervolume *pv;
 	struct promise_raid_conf *meta;
-	off_t size;
+	off_t eoff, esize, size;
 	int disk_pos, md_disk_pos, i, resurrection = 0;
-	uint32_t eoff, esize;
 
 	sc = disk->d_softc;
 	pd = (struct g_raid_md_promise_perdisk *)disk->d_md_data;
@@ -729,8 +774,10 @@ nofit:
 		sd->sd_offset = (off_t)eoff * 512;
 		sd->sd_size = (off_t)esize * 512;
 	} else {
-		sd->sd_offset = (off_t)pd->pd_meta[sdn]->disk_offset * 512;
-		sd->sd_size = (off_t)pd->pd_meta[sdn]->disk_sectors * 512;
+		sd->sd_offset = (((off_t)pd->pd_meta[sdn]->disk_offset_high
+		    << 32) + pd->pd_meta[sdn]->disk_offset) * 512;
+		sd->sd_size = (((off_t)pd->pd_meta[sdn]->disk_sectors_high
+		    << 32) + pd->pd_meta[sdn]->disk_sectors) * 512;
 	}
 
 	if (resurrection) {
@@ -749,7 +796,8 @@ nofit:
 			sd->sd_rebuild_pos = 0;
 		else {
 			sd->sd_rebuild_pos =
-			    (off_t)pd->pd_meta[sdn]->rebuild_lba * 512;
+			    (((off_t)pd->pd_meta[sdn]->disk_rebuild_high << 32) +
+			     pd->pd_meta[sdn]->disk_rebuild) * 512;
 		}
 	} else if (!(meta->disks[md_disk_pos].flags & PROMISE_F_ONLINE)) {
 		/* Rebuilding disk. */
@@ -875,13 +923,15 @@ g_raid_md_promise_start(struct g_raid_volume *vol)
 	vol->v_disks_count = meta->total_disks;
 	vol->v_mediasize = (off_t)meta->total_sectors * 512; //ZZZ
 	if (meta->total_sectors_high < 256) /* If value looks sane. */
-		vol->v_mediasize |=
+		vol->v_mediasize +=
 		    ((off_t)meta->total_sectors_high << 32) * 512; //ZZZ
-	vol->v_sectorsize = 512; //ZZZ
+	vol->v_sectorsize = 512 * meta->sector_size;
 	for (i = 0; i < vol->v_disks_count; i++) {
 		sd = &vol->v_subdisks[i];
-		sd->sd_offset = (off_t)meta->disk_offset * 512; //ZZZ
-		sd->sd_size = (off_t)meta->disk_sectors * 512; //ZZZ
+		sd->sd_offset = (((off_t)meta->disk_offset_high << 32) +
+		    meta->disk_offset) * 512;
+		sd->sd_size = (((off_t)meta->disk_sectors_high << 32) +
+		    meta->disk_sectors) * 512;
 	}
 	g_raid_start_volume(vol);
 
@@ -1213,9 +1263,8 @@ g_raid_md_ctl_promise(struct g_raid_md_object *md,
 	const char *nodename, *verb, *volname, *levelname, *diskname;
 	char *tmp;
 	int *nargs, *force;
-	off_t size, sectorsize, strip;
+	off_t esize, offs[PROMISE_MAX_DISKS], size, sectorsize, strip;
 	intmax_t *sizearg, *striparg;
-	uint32_t offs[PROMISE_MAX_DISKS], esize;
 	int numdisks, i, len, level, qual;
 	int error;
 
@@ -1323,13 +1372,6 @@ g_raid_md_ctl_promise(struct g_raid_md_object *md,
 			cp->private = disk;
 			g_topology_unlock();
 
-			if (pp->mediasize / pp->sectorsize > UINT32_MAX) {
-				gctl_error(req,
-				    "Disk '%s' is too big.", diskname);
-				error = -8;
-				break;
-			}
-
 			g_raid_get_disk_info(disk);
 
 			/* Reserve some space for metadata. */
@@ -1394,10 +1436,6 @@ g_raid_md_ctl_promise(struct g_raid_md_object *md,
 			gctl_error(req, "Size too small.");
 			return (-13);
 		}
-		if (size > 0xffffffffllu * sectorsize) {
-			gctl_error(req, "Size too big.");
-			return (-14);
-		}
 
 		/* We have all we need, create things: volume, ... */
 		pv = malloc(sizeof(*pv), M_MD_PROMISE, M_WAITOK | M_ZERO);
@@ -1629,14 +1667,6 @@ g_raid_md_ctl_promise(struct g_raid_md_object *md,
 			pp = cp->provider;
 			g_topology_unlock();
 
-			if (pp->mediasize / pp->sectorsize > UINT32_MAX) {
-				gctl_error(req,
-				    "Disk '%s' is too big.", diskname);
-				g_raid_kill_consumer(sc, cp);
-				error = -8;
-				break;
-			}
-
 			pd = malloc(sizeof(*pd), M_MD_PROMISE, M_WAITOK | M_ZERO);
 
 			disk = g_raid_create_disk(sc);
@@ -1733,9 +1763,9 @@ g_raid_md_write_promise(struct g_raid_md_object *md, struct g_raid_volume *tvol,
 		    vol->v_raid_level == G_RAID_VOLUME_RL_RAID1E)
 			meta->array_width /= 2;
 		meta->array_number = vol->v_global_id;
-		meta->total_sectors = vol->v_mediasize / vol->v_sectorsize;
-		meta->total_sectors_high =
-		    (vol->v_mediasize / vol->v_sectorsize) >> 32;
+		meta->total_sectors = vol->v_mediasize / 512;
+		meta->total_sectors_high = (vol->v_mediasize / 512) >> 32;
+		meta->sector_size = vol->v_sectorsize / 512;
 		meta->cylinders = meta->total_sectors / (255 * 63) - 1;
 		meta->heads = 254;
 		meta->sectors = 63;
@@ -1828,15 +1858,24 @@ g_raid_md_write_promise(struct g_raid_md_object *md, struct g_raid_volume *tvol,
 			pd->pd_meta[j] = promise_meta_copy(meta);
 			pd->pd_meta[j]->disk = meta->disks[pos];
 			pd->pd_meta[j]->disk.number = pos;
+			pd->pd_meta[j]->disk_offset_high =
+			    (sd->sd_offset / 512) >> 32;
 			pd->pd_meta[j]->disk_offset = sd->sd_offset / 512;
+			pd->pd_meta[j]->disk_sectors_high =
+			    (sd->sd_size / 512) >> 32;
 			pd->pd_meta[j]->disk_sectors = sd->sd_size / 512;
 			if (sd->sd_state == G_RAID_SUBDISK_S_REBUILD) {
-				pd->pd_meta[j]->rebuild_lba =
+				pd->pd_meta[j]->disk_rebuild_high =
+				    (sd->sd_rebuild_pos / 512) >> 32;
+				pd->pd_meta[j]->disk_rebuild =
 				    sd->sd_rebuild_pos / 512;
-			} else if (sd->sd_state < G_RAID_SUBDISK_S_REBUILD)
-				pd->pd_meta[j]->rebuild_lba = 0;
-			else
-				pd->pd_meta[j]->rebuild_lba = UINT32_MAX;
+			} else if (sd->sd_state < G_RAID_SUBDISK_S_REBUILD) {
+				pd->pd_meta[j]->disk_rebuild_high = 0;
+				pd->pd_meta[j]->disk_rebuild = 0;
+			} else {
+				pd->pd_meta[j]->disk_rebuild_high = UINT32_MAX;
+				pd->pd_meta[j]->disk_rebuild = UINT32_MAX;
+			}
 			pd->pd_updated = 1;
 		}
 	}
diff --git a/sys/geom/raid/tr_concat.c b/sys/geom/raid/tr_concat.c
index f3935d7..60db472 100644
--- a/sys/geom/raid/tr_concat.c
+++ b/sys/geom/raid/tr_concat.c
@@ -124,7 +124,8 @@ g_raid_tr_update_state_concat(struct g_raid_volume *vol)
 		 * Some metadata modules may not know CONCAT volume
 		 * mediasize until all disks connected. Recalculate.
 		 */
-		if (G_RAID_VOLUME_S_ALIVE(s) &&
+		if (vol->v_raid_level == G_RAID_VOLUME_RL_CONCAT &&
+		    G_RAID_VOLUME_S_ALIVE(s) &&
 		    !G_RAID_VOLUME_S_ALIVE(vol->v_state)) {
 			size = 0;
 			for (i = 0; i < vol->v_disks_count; i++) {
diff --git a/sys/geom/raid3/g_raid3.c b/sys/geom/raid3/g_raid3.c
index 711e9e1..927cb31 100644
--- a/sys/geom/raid3/g_raid3.c
+++ b/sys/geom/raid3/g_raid3.c
@@ -104,7 +104,8 @@ SYSCTL_UINT(_kern_geom_raid3_stat, OID_AUTO, parity_mismatch, CTLFLAG_RD,
 	G_RAID3_DEBUG(4, "%s: Woken up %p.", __func__, (ident));	\
 } while (0)
 
-static eventhandler_tag g_raid3_pre_sync = NULL;
+static eventhandler_tag g_raid3_post_sync = NULL;
+static int g_raid3_shutdown = 0;
 
 static int g_raid3_destroy_geom(struct gctl_req *req, struct g_class *mp,
     struct g_geom *gp);
@@ -876,7 +877,7 @@ g_raid3_idle(struct g_raid3_softc *sc, int acw)
 		return (0);
 	if (acw > 0 || (acw == -1 && sc->sc_provider->acw > 0)) {
 		timeout = g_raid3_idletime - (time_uptime - sc->sc_last_write);
-		if (timeout > 0)
+		if (!g_raid3_shutdown && timeout > 0)
 			return (timeout);
 	}
 	sc->sc_idle = 1;
@@ -3098,7 +3099,7 @@ g_raid3_access(struct g_provider *pp, int acr, int acw, int ace)
 			error = ENXIO;
 		goto end;
 	}
-	if (dcw == 0 && !sc->sc_idle)
+	if (dcw == 0)
 		g_raid3_idle(sc, dcw);
 	if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROYING) != 0) {
 		if (acr > 0 || acw > 0 || ace > 0) {
@@ -3544,7 +3545,7 @@ g_raid3_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
 }
 
 static void
-g_raid3_shutdown_pre_sync(void *arg, int howto)
+g_raid3_shutdown_post_sync(void *arg, int howto)
 {
 	struct g_class *mp;
 	struct g_geom *gp, *gp2;
@@ -3554,6 +3555,7 @@ g_raid3_shutdown_pre_sync(void *arg, int howto)
 	mp = arg;
 	DROP_GIANT();
 	g_topology_lock();
+	g_raid3_shutdown = 1;
 	LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
 		if ((sc = gp->softc) == NULL)
 			continue;
@@ -3562,6 +3564,7 @@ g_raid3_shutdown_pre_sync(void *arg, int howto)
 			continue;
 		g_topology_unlock();
 		sx_xlock(&sc->sc_lock);
+		g_raid3_idle(sc, -1);
 		g_cancel_event(sc);
 		error = g_raid3_destroy(sc, G_RAID3_DESTROY_DELAYED);
 		if (error != 0)
@@ -3576,9 +3579,9 @@ static void
 g_raid3_init(struct g_class *mp)
 {
 
-	g_raid3_pre_sync = EVENTHANDLER_REGISTER(shutdown_pre_sync,
-	    g_raid3_shutdown_pre_sync, mp, SHUTDOWN_PRI_FIRST);
-	if (g_raid3_pre_sync == NULL)
+	g_raid3_post_sync = EVENTHANDLER_REGISTER(shutdown_post_sync,
+	    g_raid3_shutdown_post_sync, mp, SHUTDOWN_PRI_FIRST);
+	if (g_raid3_post_sync == NULL)
 		G_RAID3_DEBUG(0, "Warning! Cannot register shutdown event.");
 }
 
@@ -3586,8 +3589,8 @@ static void
 g_raid3_fini(struct g_class *mp)
 {
 
-	if (g_raid3_pre_sync != NULL)
-		EVENTHANDLER_DEREGISTER(shutdown_pre_sync, g_raid3_pre_sync);
+	if (g_raid3_post_sync != NULL)
+		EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_raid3_post_sync);
 }
 
 DECLARE_GEOM_CLASS(g_raid3_class, g_raid3);
diff --git a/sys/geom/raid3/g_raid3_ctl.c b/sys/geom/raid3/g_raid3_ctl.c
index 952ac2b..b3a42ff 100644
--- a/sys/geom/raid3/g_raid3_ctl.c
+++ b/sys/geom/raid3/g_raid3_ctl.c
@@ -404,7 +404,7 @@ g_raid3_ctl_insert(struct gctl_req *req, struct g_class *mp)
 	u_char *sector;
 	off_t compsize;
 	intmax_t *no;
-	int *hardcode, *nargs, error;
+	int *hardcode, *nargs, error, autono;
 
 	nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs));
 	if (nargs == NULL) {
@@ -425,11 +425,10 @@ g_raid3_ctl_insert(struct gctl_req *req, struct g_class *mp)
 		gctl_error(req, "No 'arg%u' argument.", 1);
 		return;
 	}
-	no = gctl_get_paraml(req, "number", sizeof(*no));
-	if (no == NULL) {
-		gctl_error(req, "No '%s' argument.", "no");
-		return;
-	}
+	if (gctl_get_param(req, "number", NULL) != NULL)
+		no = gctl_get_paraml(req, "number", sizeof(*no));
+	else
+		no = NULL;
 	if (strncmp(name, "/dev/", 5) == 0)
 		name += 5;
 	g_topology_lock();
@@ -465,16 +464,30 @@ g_raid3_ctl_insert(struct gctl_req *req, struct g_class *mp)
 		gctl_error(req, "No such device: %s.", name);
 		goto end;
 	}
-	if (*no >= sc->sc_ndisks) {
-		sx_xunlock(&sc->sc_lock);
-		gctl_error(req, "Invalid component number.");
-		goto end;
-	}
-	disk = &sc->sc_disks[*no];
-	if (disk->d_state != G_RAID3_DISK_STATE_NODISK) {
-		sx_xunlock(&sc->sc_lock);
-		gctl_error(req, "Component %jd is already connected.", *no);
-		goto end;
+	if (no != NULL) {
+		if (*no < 0 || *no >= sc->sc_ndisks) {
+			sx_xunlock(&sc->sc_lock);
+			gctl_error(req, "Invalid component number.");
+			goto end;
+		}
+		disk = &sc->sc_disks[*no];
+		if (disk->d_state != G_RAID3_DISK_STATE_NODISK) {
+			sx_xunlock(&sc->sc_lock);
+			gctl_error(req, "Component %jd is already connected.",
+			    *no);
+			goto end;
+		}
+	} else {
+		disk = NULL;
+		for (autono = 0; autono < sc->sc_ndisks && disk == NULL; autono++)
+			if (sc->sc_disks[autono].d_state ==
+			    G_RAID3_DISK_STATE_NODISK)
+				disk = &sc->sc_disks[autono];
+		if (disk == NULL) {
+			sx_xunlock(&sc->sc_lock);
+			gctl_error(req, "No disconnected components.");
+			goto end;
+		}
 	}
 	if (((sc->sc_sectorsize / (sc->sc_ndisks - 1)) % pp->sectorsize) != 0) {
 		sx_xunlock(&sc->sc_lock);
diff --git a/sys/i386/conf/GENERIC b/sys/i386/conf/GENERIC
index 3a808ba..47af43b 100644
--- a/sys/i386/conf/GENERIC
+++ b/sys/i386/conf/GENERIC
@@ -208,7 +208,6 @@ device		uart		# Generic UART driver
 device		ppc
 device		ppbus		# Parallel port bus (required)
 device		lpt		# Printer
-device		plip		# TCP/IP over parallel
 device		ppi		# Parallel port interface device
 #device		vpo		# Requires scbus and da
 
@@ -346,3 +345,11 @@ device		snd_via8233	# VIA VT8233x Audio
 device		mmc		# MMC/SD bus
 device		mmcsd		# MMC/SD memory card
 device		sdhci		# Generic PCI SD Host Controller
+
+# VirtIO support
+device		virtio		# Generic VirtIO bus (required)
+device		virtio_pci	# VirtIO PCI device
+device		vtnet		# VirtIO Ethernet device
+device		virtio_blk	# VirtIO Block device
+device		virtio_scsi	# VirtIO SCSI device
+device		virtio_balloon	# VirtIO Memory Balloon device
diff --git a/sys/i386/conf/NOTES b/sys/i386/conf/NOTES
index 1868595..95ccd4e 100644
--- a/sys/i386/conf/NOTES
+++ b/sys/i386/conf/NOTES
@@ -771,6 +771,15 @@ device		glxiic		# AMD Geode LX CS5536 System Management Bus
 #
 device		glxsb		# AMD Geode LX Security Block
 
+#
+# VirtIO support
+device		virtio		# Generic VirtIO bus (required)
+device		virtio_pci	# VirtIO PCI Interface
+device		vtnet		# VirtIO Ethernet device
+device		virtio_blk	# VirtIO Block device
+device		virtio_scsi	# VirtIO SCSI device
+device		virtio_balloon	# VirtIO Memory Balloon device
+
 #####################################################################
 
 #
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index 89c0430..a7046d8 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -1528,11 +1528,11 @@ cpususpend_handler(void)
 	while (!CPU_ISSET(cpu, &started_cpus))
 		ia32_pause();
 
-	CPU_CLR_ATOMIC(cpu, &started_cpus);
-
 	/* Resume MCA and local APIC */
 	mca_resume();
 	lapic_setup(0);
+
+	CPU_CLR_ATOMIC(cpu, &started_cpus);
 }
 /*
  * This is called once the rest of the system is up and running and we're
diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c
index d719e80..f6ce067 100644
--- a/sys/i386/i386/pmap.c
+++ b/sys/i386/i386/pmap.c
@@ -96,6 +96,7 @@ __FBSDID("$FreeBSD$");
  *	and to when physical maps must be made correct.
  */
 
+#include "opt_apic.h"
 #include "opt_cpu.h"
 #include "opt_pmap.h"
 #include "opt_smp.h"
@@ -135,6 +136,11 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_reserv.h>
 #include <vm/uma.h>
 
+#ifdef DEV_APIC
+#include <sys/bus.h>
+#include <machine/intr_machdep.h>
+#include <machine/apicvar.h>
+#endif
 #include <machine/cpu.h>
 #include <machine/cputypes.h>
 #include <machine/md_var.h>
@@ -1176,6 +1182,16 @@ pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
 	else if ((cpu_feature & CPUID_CLFSH) != 0 &&
 	    eva - sva < PMAP_CLFLUSH_THRESHOLD) {
 
+#ifdef DEV_APIC
+		/*
+		 * XXX: Some CPUs fault, hang, or trash the local APIC
+		 * registers if we use CLFLUSH on the local APIC
+		 * range.  The local APIC is always uncached, so we
+		 * don't need to flush for that range anyway.
+		 */
+		if (pmap_kextract(sva) == lapic_paddr)
+			return;
+#endif
 		/*
 		 * Otherwise, do per-cache line flush.  Use the mfence
 		 * instruction to insure that previous stores are
diff --git a/sys/i386/include/intr_machdep.h b/sys/i386/include/intr_machdep.h
index 004ed52..b3dd122 100644
--- a/sys/i386/include/intr_machdep.h
+++ b/sys/i386/include/intr_machdep.h
@@ -94,7 +94,7 @@ struct pic {
 	int (*pic_config_intr)(struct intsrc *, enum intr_trigger,
 	    enum intr_polarity);
 	int (*pic_assign_cpu)(struct intsrc *, u_int apic_id);
-	STAILQ_ENTRY(pic) pics;
+	TAILQ_ENTRY(pic) pics;
 };
 
 /* Flags for pic_disable_source() */
diff --git a/sys/i386/include/vmparam.h b/sys/i386/include/vmparam.h
index 9cfd692..f974495 100644
--- a/sys/i386/include/vmparam.h
+++ b/sys/i386/include/vmparam.h
@@ -206,9 +206,4 @@
 #define VM_MAX_AUTOTUNE_MAXUSERS 384
 #endif
 
-#ifndef VM_MAX_AUTOTUNE_NMBCLUSTERS
-/* old maxusers max value. */
-#define VM_MAX_AUTOTUNE_NMBCLUSTERS (1024 + VM_MAX_AUTOTUNE_MAXUSERS * 64)
-#endif
-
 #endif /* _MACHINE_VMPARAM_H_ */
diff --git a/sys/i386/linux/linux.h b/sys/i386/linux/linux.h
index 799259c..87d419f 100644
--- a/sys/i386/linux/linux.h
+++ b/sys/i386/linux/linux.h
@@ -101,11 +101,6 @@ typedef struct {
 /*
  * Miscellaneous
  */
-#define	LINUX_NAME_MAX		255
-#define	LINUX_MAX_UTSNAME	65
-
-#define	LINUX_CTL_MAXNAME	10
-
 #define LINUX_AT_COUNT		16	/* Count of used aux entry types.
 					 * Keep this synchronized with
 					 * elf_linux_fixup() code.
@@ -121,11 +116,6 @@ struct l___sysctl_args
 	l_ulong		__spare[4];
 };
 
-/* Scheduling policies */
-#define	LINUX_SCHED_OTHER	0
-#define	LINUX_SCHED_FIFO	1
-#define	LINUX_SCHED_RR		2
-
 /* Resource limits */
 #define	LINUX_RLIMIT_CPU	0
 #define	LINUX_RLIMIT_FSIZE	1
@@ -240,15 +230,6 @@ struct l_statfs64 {
         l_int           f_spare[6];
 };
 
-struct l_new_utsname {
-	char	sysname[LINUX_MAX_UTSNAME];
-	char	nodename[LINUX_MAX_UTSNAME];
-	char	release[LINUX_MAX_UTSNAME];
-	char	version[LINUX_MAX_UTSNAME];
-	char	machine[LINUX_MAX_UTSNAME];
-	char	domainname[LINUX_MAX_UTSNAME];
-};
-
 /*
  * Signalling
  */
@@ -510,28 +491,10 @@ struct l_rt_sigframe {
 	l_handler_t 		sf_handler;
 };
 
-extern int bsd_to_linux_signal[];
-extern int linux_to_bsd_signal[];
 extern struct sysentvec linux_sysvec;
 extern struct sysentvec elf_linux_sysvec;
 
 /*
- * Pluggable ioctl handlers
- */
-struct linux_ioctl_args;
-struct thread;
-
-typedef int linux_ioctl_function_t(struct thread *, struct linux_ioctl_args *);
-
-struct linux_ioctl_handler {
-	linux_ioctl_function_t *func;
-	int	low, high;
-};
-
-int	linux_ioctl_register_handler(struct linux_ioctl_handler *h);
-int	linux_ioctl_unregister_handler(struct linux_ioctl_handler *h);
-
-/*
  * open/fcntl flags
  */
 #define	LINUX_O_RDONLY		00000000
@@ -573,65 +536,6 @@ int	linux_ioctl_unregister_handler(struct linux_ioctl_handler *h);
 #define	LINUX_F_WRLCK		1
 #define	LINUX_F_UNLCK		2
 
-/*
- * posix_fadvise advice
- */
-#define	LINUX_POSIX_FADV_NORMAL		0
-#define	LINUX_POSIX_FADV_RANDOM		1
-#define	LINUX_POSIX_FADV_SEQUENTIAL    	2
-#define	LINUX_POSIX_FADV_WILLNEED      	3
-#define	LINUX_POSIX_FADV_DONTNEED      	4
-#define	LINUX_POSIX_FADV_NOREUSE       	5
-
-/*
- * mount flags
- */
-#define	LINUX_MS_RDONLY		0x0001
-#define	LINUX_MS_NOSUID		0x0002
-#define	LINUX_MS_NODEV		0x0004
-#define	LINUX_MS_NOEXEC		0x0008
-#define	LINUX_MS_REMOUNT	0x0020
-
-/*
- * SystemV IPC defines
- */
-#define	LINUX_SEMOP		1
-#define	LINUX_SEMGET		2
-#define	LINUX_SEMCTL		3
-#define	LINUX_MSGSND		11
-#define	LINUX_MSGRCV		12
-#define	LINUX_MSGGET		13
-#define	LINUX_MSGCTL		14
-#define	LINUX_SHMAT		21
-#define	LINUX_SHMDT		22
-#define	LINUX_SHMGET		23
-#define	LINUX_SHMCTL		24
-
-#define	LINUX_IPC_RMID		0
-#define	LINUX_IPC_SET		1
-#define	LINUX_IPC_STAT		2
-#define	LINUX_IPC_INFO		3
-
-#define	LINUX_SHM_LOCK		11
-#define	LINUX_SHM_UNLOCK	12
-#define	LINUX_SHM_STAT		13
-#define	LINUX_SHM_INFO		14
-
-#define	LINUX_SHM_RDONLY	0x1000
-#define	LINUX_SHM_RND		0x2000
-#define	LINUX_SHM_REMAP		0x4000
-
-/* semctl commands */
-#define	LINUX_GETPID		11
-#define	LINUX_GETVAL		12
-#define	LINUX_GETALL		13
-#define	LINUX_GETNCNT		14
-#define	LINUX_GETZCNT		15
-#define	LINUX_SETVAL		16
-#define	LINUX_SETALL		17
-#define	LINUX_SEM_STAT		18
-#define	LINUX_SEM_INFO		19
-
 union l_semun {
 	l_int		val;
 	struct l_semid_ds	*buf;
@@ -643,25 +547,6 @@ union l_semun {
 /*
  * Socket defines
  */
-#define	LINUX_SOCKET 		1
-#define	LINUX_BIND		2
-#define	LINUX_CONNECT 		3
-#define	LINUX_LISTEN 		4
-#define	LINUX_ACCEPT 		5
-#define	LINUX_GETSOCKNAME	6
-#define	LINUX_GETPEERNAME	7
-#define	LINUX_SOCKETPAIR	8
-#define	LINUX_SEND		9
-#define	LINUX_RECV		10
-#define	LINUX_SENDTO 		11
-#define	LINUX_RECVFROM 		12
-#define	LINUX_SHUTDOWN 		13
-#define	LINUX_SETSOCKOPT	14
-#define	LINUX_GETSOCKOPT	15
-#define	LINUX_SENDMSG		16
-#define	LINUX_RECVMSG		17
-#define	LINUX_ACCEPT4		18
-
 #define	LINUX_SOL_SOCKET	1
 #define	LINUX_SOL_IP		0
 #define	LINUX_SOL_IPX		256
@@ -690,17 +575,6 @@ union l_semun {
 #define	LINUX_SO_TIMESTAMP	29
 #define	LINUX_SO_ACCEPTCONN	30
 
-#define	LINUX_IP_TOS		1
-#define	LINUX_IP_TTL		2
-#define	LINUX_IP_HDRINCL	3
-#define	LINUX_IP_OPTIONS	4
-
-#define	LINUX_IP_MULTICAST_IF		32
-#define	LINUX_IP_MULTICAST_TTL		33
-#define	LINUX_IP_MULTICAST_LOOP		34
-#define	LINUX_IP_ADD_MEMBERSHIP		35
-#define	LINUX_IP_DROP_MEMBERSHIP	36
-
 struct l_sockaddr {
 	l_ushort	sa_family;
 	char		sa_data[14];
@@ -860,30 +734,6 @@ struct l_desc_struct {
 #define	LINUX_GET_USEABLE(desc)		\
 	(((desc)->b >> LINUX_ENTRY_B_USEABLE) & 1)
 
-#define	LINUX_CLOCK_REALTIME		0
-#define	LINUX_CLOCK_MONOTONIC		1
-#define	LINUX_CLOCK_PROCESS_CPUTIME_ID	2
-#define	LINUX_CLOCK_THREAD_CPUTIME_ID	3
-#define	LINUX_CLOCK_REALTIME_HR		4
-#define	LINUX_CLOCK_MONOTONIC_HR	5
-
-#define	LINUX_CLONE_VM			0x00000100
-#define	LINUX_CLONE_FS			0x00000200
-#define	LINUX_CLONE_FILES		0x00000400
-#define	LINUX_CLONE_SIGHAND		0x00000800
-#define	LINUX_CLONE_PID			0x00001000	/* No longer exist in Linux */
-#define	LINUX_CLONE_VFORK		0x00004000
-#define	LINUX_CLONE_PARENT		0x00008000
-#define	LINUX_CLONE_THREAD		0x00010000
-#define	LINUX_CLONE_SETTLS		0x00080000
-#define	LINUX_CLONE_PARENT_SETTID	0x00100000
-#define	LINUX_CLONE_CHILD_CLEARTID	0x00200000
-#define	LINUX_CLONE_CHILD_SETTID	0x01000000
-
-#define	LINUX_THREADING_FLAGS					\
-	(LINUX_CLONE_VM | LINUX_CLONE_FS | LINUX_CLONE_FILES |	\
-	LINUX_CLONE_SIGHAND | LINUX_CLONE_THREAD)
-
 /* robust futexes */
 struct linux_robust_list {
 	struct linux_robust_list	*next;
@@ -895,7 +745,4 @@ struct linux_robust_list_head {
 	struct linux_robust_list	*pending_list;
 };
 
-int linux_set_upcall_kse(struct thread *td, register_t stack);
-int linux_set_cloned_tls(struct thread *td, void *desc);
-
 #endif /* !_I386_LINUX_H_ */
diff --git a/sys/i386/linux/linux_ptrace.c b/sys/i386/linux/linux_ptrace.c
index e9559f8..1074d21 100644
--- a/sys/i386/linux/linux_ptrace.c
+++ b/sys/i386/linux/linux_ptrace.c
@@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$");
 
 #include <i386/linux/linux.h>
 #include <i386/linux/linux_proto.h>
+#include <compat/linux/linux_signal.h>
 
 #if !defined(CPU_DISABLE_SSE) && defined(I686_CPU)
 #define CPU_ENABLE_SSE
diff --git a/sys/i386/linux/linux_sysvec.c b/sys/i386/linux/linux_sysvec.c
index d4e23e1..71417e0 100644
--- a/sys/i386/linux/linux_sysvec.c
+++ b/sys/i386/linux/linux_sysvec.c
@@ -66,6 +66,7 @@ __FBSDID("$FreeBSD$");
 #include <i386/linux/linux_proto.h>
 #include <compat/linux/linux_emul.h>
 #include <compat/linux/linux_futex.h>
+#include <compat/linux/linux_ioctl.h>
 #include <compat/linux/linux_mib.h>
 #include <compat/linux/linux_misc.h>
 #include <compat/linux/linux_signal.h>
diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index 00b1c3f..a92b6db 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -325,6 +325,7 @@ print_version(void *data __unused)
 	while (len > 0 && version[len - 1] == '\n')
 		len--;
 	printf("%.*s %s\n", len, version, machine);
+	printf("%s\n", compiler_version);
 }
 
 SYSINIT(announce, SI_SUB_COPYRIGHT, SI_ORDER_FIRST, print_caddr_t,
diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c
index 5e10200..9d62c58 100644
--- a/sys/kern/kern_clock.c
+++ b/sys/kern/kern_clock.c
@@ -382,7 +382,7 @@ static void watchdog_config(void *, u_int, int *);
 int	stathz;
 int	profhz;
 int	profprocs;
-int	ticks;
+volatile int	ticks;
 int	psratio;
 
 static DPCPU_DEFINE(int, pcputicks);	/* Per-CPU version of ticks. */
@@ -469,7 +469,7 @@ void
 hardclock(int usermode, uintfptr_t pc)
 {
 
-	atomic_add_int((volatile int *)&ticks, 1);
+	atomic_add_int(&ticks, 1);
 	hardclock_cpu(usermode);
 	tc_ticktock(1);
 	cpu_tick_calibration();
diff --git a/sys/kern/kern_clocksource.c b/sys/kern/kern_clocksource.c
index 05e47e0..6fd40a8 100644
--- a/sys/kern/kern_clocksource.c
+++ b/sys/kern/kern_clocksource.c
@@ -317,14 +317,16 @@ getnextevent(struct bintime *event)
 	nonidle = !state->idle;
 	if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) {
 #ifdef SMP
-		CPU_FOREACH(cpu) {
-			if (curcpu == cpu)
-				continue;
-			state = DPCPU_ID_PTR(cpu, timerstate);
-			nonidle += !state->idle;
-			if (bintime_cmp(event, &state->nextevent, >)) {
-				*event = state->nextevent;
-				c = cpu;
+		if (smp_started) {
+			CPU_FOREACH(cpu) {
+				if (curcpu == cpu)
+					continue;
+				state = DPCPU_ID_PTR(cpu, timerstate);
+				nonidle += !state->idle;
+				if (bintime_cmp(event, &state->nextevent, >)) {
+					*event = state->nextevent;
+					c = cpu;
+				}
 			}
 		}
 #endif
diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index 6dae173..87d75ea 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -779,8 +779,10 @@ kern_fcntl(struct thread *td, int fd, int cmd, intptr_t arg)
 		if (arg != 0) {
 			vp = fp->f_vnode;
 			error = vn_lock(vp, LK_SHARED);
-			if (error != 0)
-				goto readahead_vnlock_fail;
+			if (error != 0) {
+				fdrop(fp, td);
+				break;
+			}
 			bsize = fp->f_vnode->v_mount->mnt_stat.f_iosize;
 			VOP_UNLOCK(vp, 0);
 			fp->f_seqcount = (arg + bsize - 1) / bsize;
@@ -788,7 +790,6 @@ kern_fcntl(struct thread *td, int fd, int cmd, intptr_t arg)
 				new = old = fp->f_flag;
 				new |= FRDAHEAD;
 			} while (!atomic_cmpset_rel_int(&fp->f_flag, old, new));
-		readahead_vnlock_fail:;
 		} else {
 			do {
 				new = old = fp->f_flag;
diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c
index 50b47fb..3a9a3a4 100644
--- a/sys/kern/kern_exit.c
+++ b/sys/kern/kern_exit.c
@@ -1024,20 +1024,18 @@ kern_wait(struct thread *td, pid_t pid, int *status, int options,
     struct rusage *rusage)
 {
 	struct __wrusage wru, *wrup;
-	struct proc *q;
 	idtype_t idtype;
 	id_t id;
 	int ret;
 
+	/*
+	 * Translate the special pid values into the (idtype, pid)
+	 * pair for kern_wait6.  The WAIT_MYPGRP case is handled by
+	 * kern_wait6() on its own.
+	 */
 	if (pid == WAIT_ANY) {
 		idtype = P_ALL;
 		id = 0;
-	} else if (pid == WAIT_MYPGRP) {
-		idtype = P_PGID;
-		q = td->td_proc;
-		PROC_LOCK(q);
-		id = (id_t)q->p_pgid;
-		PROC_UNLOCK(q);
 	} else if (pid < 0) {
 		idtype = P_PGID;
 		id = (id_t)-pid;
@@ -1045,10 +1043,12 @@ kern_wait(struct thread *td, pid_t pid, int *status, int options,
 		idtype = P_PID;
 		id = (id_t)pid;
 	}
+
 	if (rusage != NULL)
 		wrup = &wru;
 	else
 		wrup = NULL;
+
 	/*
 	 * For backward compatibility we implicitly add flags WEXITED
 	 * and WTRAPPED here.
diff --git a/sys/kern/kern_ktr.c b/sys/kern/kern_ktr.c
index a83cedf..5b9d7aa 100644
--- a/sys/kern/kern_ktr.c
+++ b/sys/kern/kern_ktr.c
@@ -66,6 +66,10 @@ __FBSDID("$FreeBSD$");
 #include <ddb/db_output.h>
 #endif
 
+#ifndef KTR_BOOT_ENTRIES
+#define	KTR_BOOT_ENTRIES	1024
+#endif
+
 #ifndef KTR_ENTRIES
 #define	KTR_ENTRIES	1024
 #endif
@@ -96,9 +100,9 @@ FEATURE(ktr, "Kernel support for KTR kernel tracing facility");
 volatile int	ktr_idx = 0;
 int	ktr_mask = KTR_MASK;
 int	ktr_compile = KTR_COMPILE;
-int	ktr_entries = KTR_ENTRIES;
+int	ktr_entries = KTR_BOOT_ENTRIES;
 int	ktr_version = KTR_VERSION;
-struct	ktr_entry ktr_buf_init[KTR_ENTRIES];
+struct	ktr_entry ktr_buf_init[KTR_BOOT_ENTRIES];
 struct	ktr_entry *ktr_buf = ktr_buf_init;
 cpuset_t ktr_cpumask = CPUSET_T_INITIALIZER(KTR_CPUMASK);
 static char ktr_cpumask_str[CPUSETBUFSIZ];
@@ -194,6 +198,28 @@ SYSCTL_PROC(_debug_ktr, OID_AUTO, mask, CTLTYPE_UINT|CTLFLAG_RW, 0, 0,
     sysctl_debug_ktr_mask, "IU",
     "Bitmask of KTR event classes for which logging is enabled");
 
+#if KTR_ENTRIES != KTR_BOOT_ENTRIES
+/*
+ * A simplified version of sysctl_debug_ktr_entries.
+ * No need to care about SMP, scheduling, etc.
+ */
+static void
+ktr_entries_initializer(void *dummy __unused)
+{
+	int mask;
+
+	/* Temporarily disable ktr in case malloc() is being traced. */
+	mask = ktr_mask;
+	ktr_mask = 0;
+	ktr_buf = malloc(sizeof(*ktr_buf) * KTR_ENTRIES, M_KTR,
+	    M_WAITOK | M_ZERO);
+	ktr_entries = KTR_ENTRIES;
+	ktr_mask = mask;
+}
+SYSINIT(ktr_entries_initializer, SI_SUB_KMEM, SI_ORDER_ANY,
+    ktr_entries_initializer, NULL);
+#endif
+
 static int
 sysctl_debug_ktr_entries(SYSCTL_HANDLER_ARGS)
 {
diff --git a/sys/kern/kern_mbuf.c b/sys/kern/kern_mbuf.c
index 3bdfd88..2259aa2 100644
--- a/sys/kern/kern_mbuf.c
+++ b/sys/kern/kern_mbuf.c
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_extern.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_page.h>
+#include <vm/vm_map.h>
 #include <vm/uma.h>
 #include <vm/uma_int.h>
 #include <vm/uma_dbg.h>
@@ -104,15 +105,24 @@ int nmbjumbo16;			/* limits number of 16k jumbo clusters */
 struct mbstat mbstat;
 
 /*
- * tunable_mbinit() has to be run before init_maxsockets() thus
- * the SYSINIT order below is SI_ORDER_MIDDLE while init_maxsockets()
- * runs at SI_ORDER_ANY.
- *
- * NB: This has to be done before VM init.
+ * tunable_mbinit() has to be run before any mbuf allocations are done.
  */
 static void
 tunable_mbinit(void *dummy)
 {
+	quad_t realmem, maxmbufmem;
+
+	/*
+	 * The default limit for all mbuf related memory is 1/2 of all
+	 * available kernel memory (physical or kmem).
+	 * At most it can be 3/4 of available kernel memory.
+	 */
+	realmem = qmin((quad_t)physmem * PAGE_SIZE,
+	    vm_map_max(kernel_map) - vm_map_min(kernel_map));
+	maxmbufmem = realmem / 2;
+	TUNABLE_QUAD_FETCH("kern.maxmbufmem", &maxmbufmem);
+	if (maxmbufmem > realmem / 4 * 3)
+		maxmbufmem = realmem / 4 * 3;
 
 	TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
 	if (nmbclusters == 0)
@@ -139,7 +149,7 @@ tunable_mbinit(void *dummy)
 		nmbufs = lmax(maxmbufmem / MSIZE / 5,
 		    nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16);
 }
-SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_MIDDLE, tunable_mbinit, NULL);
+SYSINIT(tunable_mbinit, SI_SUB_KMEM, SI_ORDER_MIDDLE, tunable_mbinit, NULL);
 
 static int
 sysctl_nmbclusters(SYSCTL_HANDLER_ARGS)
@@ -279,16 +289,14 @@ static int	mb_zinit_pack(void *, int, int);
 static void	mb_zfini_pack(void *, int);
 
 static void	mb_reclaim(void *);
-static void	mbuf_init(void *);
 static void    *mbuf_jumbo_alloc(uma_zone_t, int, uint8_t *, int);
 
-/* Ensure that MSIZE must be a power of 2. */
+/* Ensure that MSIZE is a power of 2. */
 CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
 
 /*
  * Initialize FreeBSD Network buffer allocation.
  */
-SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
 static void
 mbuf_init(void *dummy)
 {
@@ -396,6 +404,7 @@ mbuf_init(void *dummy)
 	mbstat.sf_iocnt = 0;
 	mbstat.sf_allocwait = mbstat.sf_allocfail = 0;
 }
+SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
 
 /*
  * UMA backend page allocator for the jumbo frame zones.
diff --git a/sys/kern/kern_mib.c b/sys/kern/kern_mib.c
index 29864a8..c84d4b2 100644
--- a/sys/kern/kern_mib.c
+++ b/sys/kern/kern_mib.c
@@ -99,6 +99,9 @@ SYSCTL_INT(_kern, KERN_OSREV, osrevision, CTLFLAG_RD|CTLFLAG_CAPRD,
 SYSCTL_STRING(_kern, KERN_VERSION, version, CTLFLAG_RD|CTLFLAG_MPSAFE,
     version, 0, "Kernel version");
 
+SYSCTL_STRING(_kern, OID_AUTO, compiler_version, CTLFLAG_RD|CTLFLAG_MPSAFE,
+    compiler_version, 0, "Version of compiler used to compile kernel");
+
 SYSCTL_STRING(_kern, KERN_OSTYPE, ostype, CTLFLAG_RD|CTLFLAG_MPSAFE|
     CTLFLAG_CAPRD, ostype, 0, "Operating system type");
 
diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c
index a08b218..2ffa2f9 100644
--- a/sys/kern/kern_tc.c
+++ b/sys/kern/kern_tc.c
@@ -103,8 +103,8 @@ static struct timecounter *timecounters = &dummy_timecounter;
 
 int tc_min_ticktock_freq = 1;
 
-time_t time_second = 1;
-time_t time_uptime = 1;
+volatile time_t time_second = 1;
+volatile time_t time_uptime = 1;
 
 struct bintime boottimebin;
 struct timeval boottime;
diff --git a/sys/kern/subr_bus.c b/sys/kern/subr_bus.c
index c41fba0..1d45f28 100644
--- a/sys/kern/subr_bus.c
+++ b/sys/kern/subr_bus.c
@@ -3318,7 +3318,7 @@ resource_list_release(struct resource_list *rl, device_t bus, device_t child,
 /**
  * @brief Fully release a reserved resource
  *
- * Fully releases a resouce reserved via resource_list_reserve().
+ * Fully releases a resource reserved via resource_list_reserve().
  *
  * @param rl		the resource list which was allocated from
  * @param bus		the parent device of @p child
diff --git a/sys/kern/subr_param.c b/sys/kern/subr_param.c
index fe10a87..f36c769 100644
--- a/sys/kern/subr_param.c
+++ b/sys/kern/subr_param.c
@@ -93,7 +93,6 @@ int	ncallout;			/* maximum # of timer events */
 int	nbuf;
 int	ngroups_max;			/* max # groups per process */
 int	nswbuf;
-quad_t	maxmbufmem;			/* max mbuf memory */
 pid_t	pid_max = PID_MAX;
 long	maxswzone;			/* max swmeta KVA storage */
 long	maxbcache;			/* max buffer cache KVA storage */
@@ -161,6 +160,7 @@ static const char *const vm_bnames[] = {
 	"Bochs",			/* Bochs */
 	"Xen",				/* Xen */
 	"BHYVE",			/* bhyve */
+	"Seabios",			/* KVM */
 	NULL
 };
 
@@ -169,6 +169,7 @@ static const char *const vm_pnames[] = {
 	"Virtual Machine",		/* Microsoft VirtualPC */
 	"VirtualBox",			/* Sun xVM VirtualBox */
 	"Parallels Virtual Platform",	/* Parallels VM */
+	"KVM",				/* KVM */
 	NULL
 };
 
@@ -272,7 +273,6 @@ init_param1(void)
 void
 init_param2(long physpages)
 {
-	quad_t realmem;
 
 	/* Base parameters */
 	maxusers = MAXUSERS;
@@ -324,23 +324,14 @@ init_param2(long physpages)
 
 	/*
 	 * XXX: Does the callout wheel have to be so big?
+	 *
+	 * Clip callout to result of previous function of maxusers maximum
+	 * 384.  This is still huge, but acceptable.
 	 */
-	ncallout = 16 + maxproc + maxfiles;
+	ncallout = imin(16 + maxproc + maxfiles, 18508);
 	TUNABLE_INT_FETCH("kern.ncallout", &ncallout);
 
 	/*
-	 * The default limit for all mbuf related memory is 1/2 of all
-	 * available kernel memory (physical or kmem).
-	 * At most it can be 3/4 of available kernel memory.
-	 */
-	realmem = qmin((quad_t)physpages * PAGE_SIZE,
-	    VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS);
-	maxmbufmem = realmem / 2;
-	TUNABLE_QUAD_FETCH("kern.maxmbufmem", &maxmbufmem);
-	if (maxmbufmem > (realmem / 4) * 3)
-		maxmbufmem = (realmem / 4) * 3;
-
-	/*
 	 * The default for maxpipekva is min(1/64 of the kernel address space,
 	 * max(1/64 of main memory, 512KB)).  See sys_pipe.c for more details.
 	 */
diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c
index ab6163d..4606f37 100644
--- a/sys/kern/uipc_mbuf.c
+++ b/sys/kern/uipc_mbuf.c
@@ -85,6 +85,79 @@ SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW,
 #endif
 
 /*
+ * m_get2() allocates minimum mbuf that would fit "size" argument.
+ */
+struct mbuf *
+m_get2(int how, short type, int flags, int size)
+{
+	struct mb_args args;
+	struct mbuf *m, *n;
+	uma_zone_t zone;
+
+	args.flags = flags;
+	args.type = type;
+
+	if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0))
+		return (uma_zalloc_arg(zone_mbuf, &args, how));
+	if (size <= MCLBYTES)
+		return (uma_zalloc_arg(zone_pack, &args, how));
+	if (size > MJUM16BYTES)
+		return (NULL);
+
+	m = uma_zalloc_arg(zone_mbuf, &args, how);
+	if (m == NULL)
+		return (NULL);
+
+#if MJUMPAGESIZE != MCLBYTES
+	if (size <= MJUMPAGESIZE)
+		zone = zone_jumbop;
+	else
+#endif
+	if (size <= MJUM9BYTES)
+		zone = zone_jumbo9;
+	else
+		zone = zone_jumbo16;
+
+	n = uma_zalloc_arg(zone, m, how);
+	if (n == NULL) {
+		uma_zfree(zone_mbuf, m);
+		return (NULL);
+	}
+
+	return (m);
+}
+
+/*
+ * m_getjcl() returns an mbuf with a cluster of the specified size attached.
+ * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
+ */
+struct mbuf *
+m_getjcl(int how, short type, int flags, int size)
+{
+	struct mb_args args;
+	struct mbuf *m, *n;
+	uma_zone_t zone;
+
+	if (size == MCLBYTES)
+		return m_getcl(how, type, flags);
+
+	args.flags = flags;
+	args.type = type;
+
+	m = uma_zalloc_arg(zone_mbuf, &args, how);
+	if (m == NULL)
+		return (NULL);
+
+	zone = m_getzone(size);
+	n = uma_zalloc_arg(zone, m, how);
+	if (n == NULL) {
+		uma_zfree(zone_mbuf, m);
+		return (NULL);
+	}
+	return (m);
+}
+
+/*
  * Allocate a given length worth of mbufs and/or clusters (whatever fits
  * best) and return a pointer to the top of the allocated chain.  If an
  * existing mbuf chain is provided, then we will append the new chain
diff --git a/sys/kern/vfs_hash.c b/sys/kern/vfs_hash.c
index aad22e0..0271e49 100644
--- a/sys/kern/vfs_hash.c
+++ b/sys/kern/vfs_hash.c
@@ -54,11 +54,18 @@ vfs_hashinit(void *dummy __unused)
 /* Must be SI_ORDER_SECOND so desiredvnodes is available */
 SYSINIT(vfs_hash, SI_SUB_VFS, SI_ORDER_SECOND, vfs_hashinit, NULL);
 
+u_int
+vfs_hash_index(struct vnode *vp)
+{
+
+	return (vp->v_hash + vp->v_mount->mnt_hashseed);
+}
+
 static struct vfs_hash_head *
-vfs_hash_index(const struct mount *mp, u_int hash)
+vfs_hash_bucket(const struct mount *mp, u_int hash)
 {
 
-	return(&vfs_hash_tbl[(hash + mp->mnt_hashseed) & vfs_hash_mask]);
+	return (&vfs_hash_tbl[(hash + mp->mnt_hashseed) & vfs_hash_mask]);
 }
 
 int
@@ -69,7 +76,7 @@ vfs_hash_get(const struct mount *mp, u_int hash, int flags, struct thread *td, s
 
 	while (1) {
 		mtx_lock(&vfs_hash_mtx);
-		LIST_FOREACH(vp, vfs_hash_index(mp, hash), v_hashlist) {
+		LIST_FOREACH(vp, vfs_hash_bucket(mp, hash), v_hashlist) {
 			if (vp->v_hash != hash)
 				continue;
 			if (vp->v_mount != mp)
@@ -113,7 +120,7 @@ vfs_hash_insert(struct vnode *vp, u_int hash, int flags, struct thread *td, stru
 	while (1) {
 		mtx_lock(&vfs_hash_mtx);
 		LIST_FOREACH(vp2,
-		    vfs_hash_index(vp->v_mount, hash), v_hashlist) {
+		    vfs_hash_bucket(vp->v_mount, hash), v_hashlist) {
 			if (vp2->v_hash != hash)
 				continue;
 			if (vp2->v_mount != vp->v_mount)
@@ -138,7 +145,7 @@ vfs_hash_insert(struct vnode *vp, u_int hash, int flags, struct thread *td, stru
 			
 	}
 	vp->v_hash = hash;
-	LIST_INSERT_HEAD(vfs_hash_index(vp->v_mount, hash), vp, v_hashlist);
+	LIST_INSERT_HEAD(vfs_hash_bucket(vp->v_mount, hash), vp, v_hashlist);
 	mtx_unlock(&vfs_hash_mtx);
 	return (0);
 }
@@ -149,7 +156,7 @@ vfs_hash_rehash(struct vnode *vp, u_int hash)
 
 	mtx_lock(&vfs_hash_mtx);
 	LIST_REMOVE(vp, v_hashlist);
-	LIST_INSERT_HEAD(vfs_hash_index(vp->v_mount, hash), vp, v_hashlist);
+	LIST_INSERT_HEAD(vfs_hash_bucket(vp->v_mount, hash), vp, v_hashlist);
 	vp->v_hash = hash;
 	mtx_unlock(&vfs_hash_mtx);
 }
diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index 7c243b6..1c26368 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -279,6 +279,8 @@ SYSCTL_INT(_debug, OID_AUTO, vnlru_nowhere, CTLFLAG_RW,
 #define VSHOULDFREE(vp) (!((vp)->v_iflag & VI_FREE) && !(vp)->v_holdcnt)
 #define VSHOULDBUSY(vp) (((vp)->v_iflag & VI_FREE) && (vp)->v_holdcnt)
 
+/* Shift count for (uintptr_t)vp to initialize vp->v_hash. */
+static int vnsz2log;
 
 /*
  * Initialize the vnode management data structures.
@@ -293,6 +295,7 @@ SYSCTL_INT(_debug, OID_AUTO, vnlru_nowhere, CTLFLAG_RW,
 static void
 vntblinit(void *dummy __unused)
 {
+	u_int i;
 	int physvnodes, virtvnodes;
 
 	/*
@@ -332,6 +335,9 @@ vntblinit(void *dummy __unused)
 	syncer_maxdelay = syncer_mask + 1;
 	mtx_init(&sync_mtx, "Syncer mtx", NULL, MTX_DEF);
 	cv_init(&sync_wakeup, "syncer");
+	for (i = 1; i <= sizeof(struct vnode); i <<= 1)
+		vnsz2log++;
+	vnsz2log--;
 }
 SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL);
 
@@ -1067,6 +1073,14 @@ alloc:
 	}
 	rangelock_init(&vp->v_rl);
 
+	/*
+	 * For the filesystems which do not use vfs_hash_insert(),
+	 * still initialize v_hash to have vfs_hash_index() useful.
+	 * E.g., nullfs uses vfs_hash_index() on the lower vnode for
+	 * its own hashing.
+	 */
+	vp->v_hash = (uintptr_t)vp >> vnsz2log;
+
 	*vpp = vp;
 	return (0);
 }
diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c
index 3f2e523..1a5f2ae 100644
--- a/sys/kern/vfs_syscalls.c
+++ b/sys/kern/vfs_syscalls.c
@@ -187,8 +187,8 @@ sys_quotactl(td, uap)
 	AUDIT_ARG_UID(uap->uid);
 	if (!prison_allow(td->td_ucred, PR_ALLOW_QUOTAS))
 		return (EPERM);
-	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNODE1,
-	   UIO_USERSPACE, uap->path, td);
+	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNODE1, UIO_USERSPACE,
+	    uap->path, td);
 	if ((error = namei(&nd)) != 0)
 		return (error);
 	NDFREE(&nd, NDF_ONLY_PNBUF);
@@ -293,8 +293,8 @@ kern_statfs(struct thread *td, char *path, enum uio_seg pathseg,
 	int error;
 	struct nameidata nd;
 
-	NDINIT(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF |
-	    AUDITVNODE1, pathseg, path, td);
+	NDINIT(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF | AUDITVNODE1,
+	    pathseg, path, td);
 	error = namei(&nd);
 	if (error)
 		return (error);
@@ -871,8 +871,8 @@ sys_chroot(td, uap)
 	error = priv_check(td, PRIV_VFS_CHROOT);
 	if (error)
 		return (error);
-	NDINIT(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF |
-	    AUDITVNODE1, UIO_USERSPACE, uap->path, td);
+	NDINIT(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF | AUDITVNODE1,
+	    UIO_USERSPACE, uap->path, td);
 	error = namei(&nd);
 	if (error)
 		goto error;
@@ -1077,8 +1077,8 @@ kern_openat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
 	/* Set the flags early so the finit in devfs can pick them up. */
 	fp->f_flag = flags & FMASK;
 	cmode = ((mode &~ fdp->fd_cmask) & ALLPERMS) &~ S_ISTXT;
-	NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | AUDITVNODE1, pathseg,
-	    path, fd, rights_needed, td);
+	NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | AUDITVNODE1, pathseg, path, fd,
+	    rights_needed, td);
 	td->td_dupfd = -1;		/* XXX check for fdopen */
 	error = vn_open(&nd, &flags, cmode, fp);
 	if (error) {
@@ -1108,7 +1108,7 @@ kern_openat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
 
 		if (error == ERESTART)
 			error = EINTR;
-		goto bad_unlocked;
+		goto bad;
 	}
 	td->td_dupfd = 0;
 	NDFREE(&nd, NDF_ONLY_PNBUF);
@@ -1150,12 +1150,11 @@ success:
 			 */
 			if ((error = kern_capwrap(td, fp, nd.ni_baserights,
 			    &indx)) != 0)
-				goto bad_unlocked;
+				goto bad;
 		} else
 #endif
 			if ((error = finstall(td, fp, &indx, flags)) != 0)
-				goto bad_unlocked;
-
+				goto bad;
 	}
 
 	/*
@@ -1166,7 +1165,6 @@ success:
 	td->td_retval[0] = indx;
 	return (0);
 bad:
-bad_unlocked:
 	KASSERT(indx == -1, ("indx=%d, should be -1", indx));
 	fdrop(fp, td);
 	return (error);
@@ -1279,9 +1277,8 @@ kern_mknodat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
 		return (error);
 restart:
 	bwillwrite();
-	NDINIT_ATRIGHTS(&nd, CREATE,
-	    LOCKPARENT | SAVENAME | AUDITVNODE1, pathseg, path, fd,
-	    CAP_MKNOD, td);
+	NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT | SAVENAME | AUDITVNODE1,
+	    pathseg, path, fd, CAP_MKNOD, td);
 	if ((error = namei(&nd)) != 0)
 		return (error);
 	vp = nd.ni_vp;
@@ -1400,9 +1397,8 @@ kern_mkfifoat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
 	AUDIT_ARG_MODE(mode);
 restart:
 	bwillwrite();
-	NDINIT_ATRIGHTS(&nd, CREATE,
-	    LOCKPARENT | SAVENAME | AUDITVNODE1, pathseg, path, fd,
-	    CAP_MKFIFO, td);
+	NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT | SAVENAME | AUDITVNODE1,
+	    pathseg, path, fd, CAP_MKFIFO, td);
 	if ((error = namei(&nd)) != 0)
 		return (error);
 	if (nd.ni_vp != NULL) {
@@ -1541,8 +1537,7 @@ kern_linkat(struct thread *td, int fd1, int fd2, char *path1, char *path2,
 	int error;
 
 	bwillwrite();
-	NDINIT_AT(&nd, LOOKUP, follow | AUDITVNODE1, segflg, path1,
-	    fd1, td);
+	NDINIT_AT(&nd, LOOKUP, follow | AUDITVNODE1, segflg, path1, fd1, td);
 
 	if ((error = namei(&nd)) != 0)
 		return (error);
@@ -1556,8 +1551,8 @@ kern_linkat(struct thread *td, int fd1, int fd2, char *path1, char *path2,
 		vrele(vp);
 		return (error);
 	}
-	NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT | SAVENAME |
-	    AUDITVNODE2, segflg, path2, fd2, CAP_CREATE, td);
+	NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT | SAVENAME | AUDITVNODE2,
+	    segflg, path2, fd2, CAP_CREATE, td);
 	if ((error = namei(&nd)) == 0) {
 		if (nd.ni_vp != NULL) {
 			if (nd.ni_dvp == nd.ni_vp)
@@ -1649,8 +1644,8 @@ kern_symlinkat(struct thread *td, char *path1, int fd, char *path2,
 	AUDIT_ARG_TEXT(syspath);
 restart:
 	bwillwrite();
-	NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT | SAVENAME |
-	    AUDITVNODE1, segflg, path2, fd, CAP_CREATE, td);
+	NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT | SAVENAME | AUDITVNODE1,
+	    segflg, path2, fd, CAP_CREATE, td);
 	if ((error = namei(&nd)) != 0)
 		goto out;
 	if (nd.ni_vp) {
@@ -1801,8 +1796,8 @@ kern_unlinkat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
 
 restart:
 	bwillwrite();
-	NDINIT_ATRIGHTS(&nd, DELETE, LOCKPARENT | LOCKLEAF |
-	    AUDITVNODE1, pathseg, path, fd, CAP_DELETE, td);
+	NDINIT_ATRIGHTS(&nd, DELETE, LOCKPARENT | LOCKLEAF | AUDITVNODE1,
+	    pathseg, path, fd, CAP_DELETE, td);
 	if ((error = namei(&nd)) != 0)
 		return (error == EINVAL ? EPERM : error);
 	vp = nd.ni_vp;
@@ -2309,8 +2304,8 @@ kern_statat_vnhook(struct thread *td, int flag, int fd, char *path,
 		return (EINVAL);
 
 	NDINIT_ATRIGHTS(&nd, LOOKUP, ((flag & AT_SYMLINK_NOFOLLOW) ? NOFOLLOW :
-	    FOLLOW) | LOCKSHARED | LOCKLEAF | AUDITVNODE1, pathseg,
-	    path, fd, CAP_FSTAT, td);
+	    FOLLOW) | LOCKSHARED | LOCKLEAF | AUDITVNODE1, pathseg, path, fd,
+	    CAP_FSTAT, td);
 
 	if ((error = namei(&nd)) != 0)
 		return (error);
@@ -2486,7 +2481,8 @@ sys_lpathconf(td, uap)
 	} */ *uap;
 {
 
-	return (kern_pathconf(td, uap->path, UIO_USERSPACE, uap->name, NOFOLLOW));
+	return (kern_pathconf(td, uap->path, UIO_USERSPACE, uap->name,
+	    NOFOLLOW));
 }
 
 int
@@ -2572,8 +2568,8 @@ kern_readlinkat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
 	if (count > IOSIZE_MAX)
 		return (EINVAL);
 
-	NDINIT_AT(&nd, LOOKUP, NOFOLLOW | LOCKSHARED | LOCKLEAF |
-	    AUDITVNODE1, pathseg, path, fd, td);
+	NDINIT_AT(&nd, LOOKUP, NOFOLLOW | LOCKSHARED | LOCKLEAF | AUDITVNODE1,
+	    pathseg, path, fd, td);
 
 	if ((error = namei(&nd)) != 0)
 		return (error);
@@ -2670,8 +2666,7 @@ sys_chflags(td, uap)
 	struct nameidata nd;
 
 	AUDIT_ARG_FFLAGS(uap->flags);
-	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE,
-	    uap->path, td);
+	NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->path, td);
 	if ((error = namei(&nd)) != 0)
 		return (error);
 	NDFREE(&nd, NDF_ONLY_PNBUF);
@@ -2695,8 +2690,8 @@ sys_lchflags(td, uap)
 	struct nameidata nd;
 
 	AUDIT_ARG_FFLAGS(uap->flags);
-	NDINIT(&nd, LOOKUP, NOFOLLOW | AUDITVNODE1, UIO_USERSPACE,
-	    uap->path, td);
+	NDINIT(&nd, LOOKUP, NOFOLLOW | AUDITVNODE1, UIO_USERSPACE, uap->path,
+	    td);
 	if ((error = namei(&nd)) != 0)
 		return (error);
 	NDFREE(&nd, NDF_ONLY_PNBUF);
@@ -2852,8 +2847,8 @@ kern_fchmodat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
 
 	AUDIT_ARG_MODE(mode);
 	follow = (flag & AT_SYMLINK_NOFOLLOW) ? NOFOLLOW : FOLLOW;
-	NDINIT_ATRIGHTS(&nd, LOOKUP,  follow | AUDITVNODE1, pathseg,
-	    path, fd, CAP_FCHMOD, td);
+	NDINIT_ATRIGHTS(&nd, LOOKUP,  follow | AUDITVNODE1, pathseg, path, fd,
+	    CAP_FCHMOD, td);
 	if ((error = namei(&nd)) != 0)
 		return (error);
 	NDFREE(&nd, NDF_ONLY_PNBUF);
@@ -2982,8 +2977,8 @@ kern_fchownat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
 
 	AUDIT_ARG_OWNER(uid, gid);
 	follow = (flag & AT_SYMLINK_NOFOLLOW) ? NOFOLLOW : FOLLOW;
-	NDINIT_ATRIGHTS(&nd, LOOKUP, follow | AUDITVNODE1, pathseg,
-	    path, fd, CAP_FCHOWN, td);
+	NDINIT_ATRIGHTS(&nd, LOOKUP, follow | AUDITVNODE1, pathseg, path, fd,
+	    CAP_FCHOWN, td);
 
 	if ((error = namei(&nd)) != 0)
 		return (error);
@@ -3188,8 +3183,8 @@ kern_utimesat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
 
 	if ((error = getutimes(tptr, tptrseg, ts)) != 0)
 		return (error);
-	NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | AUDITVNODE1, pathseg,
-	    path, fd, CAP_FUTIMES, td);
+	NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | AUDITVNODE1, pathseg, path, fd,
+	    CAP_FUTIMES, td);
 
 	if ((error = namei(&nd)) != 0)
 		return (error);
@@ -3510,8 +3505,8 @@ kern_renameat(struct thread *td, int oldfd, char *old, int newfd, char *new,
 	NDINIT_ATRIGHTS(&fromnd, DELETE, LOCKPARENT | LOCKLEAF | SAVESTART |
 	    AUDITVNODE1, pathseg, old, oldfd, CAP_DELETE, td);
 #else
-	NDINIT_ATRIGHTS(&fromnd, DELETE, WANTPARENT | SAVESTART |
-	    AUDITVNODE1, pathseg, old, oldfd, CAP_DELETE, td);
+	NDINIT_ATRIGHTS(&fromnd, DELETE, WANTPARENT | SAVESTART | AUDITVNODE1,
+	    pathseg, old, oldfd, CAP_DELETE, td);
 #endif
 
 	if ((error = namei(&fromnd)) != 0)
@@ -3656,8 +3651,8 @@ kern_mkdirat(struct thread *td, int fd, char *path, enum uio_seg segflg,
 	AUDIT_ARG_MODE(mode);
 restart:
 	bwillwrite();
-	NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT | SAVENAME |
-	    AUDITVNODE1, segflg, path, fd, CAP_MKDIR, td);
+	NDINIT_ATRIGHTS(&nd, CREATE, LOCKPARENT | SAVENAME | AUDITVNODE1,
+	    segflg, path, fd, CAP_MKDIR, td);
 	nd.ni_cnd.cn_flags |= WILLBEDIR;
 	if ((error = namei(&nd)) != 0)
 		return (error);
@@ -3740,8 +3735,8 @@ kern_rmdirat(struct thread *td, int fd, char *path, enum uio_seg pathseg)
 
 restart:
 	bwillwrite();
-	NDINIT_ATRIGHTS(&nd, DELETE, LOCKPARENT | LOCKLEAF |
-	    AUDITVNODE1, pathseg, path, fd, CAP_RMDIR, td);
+	NDINIT_ATRIGHTS(&nd, DELETE, LOCKPARENT | LOCKLEAF | AUDITVNODE1,
+	    pathseg, path, fd, CAP_RMDIR, td);
 	if ((error = namei(&nd)) != 0)
 		return (error);
 	vp = nd.ni_vp;
@@ -4123,8 +4118,8 @@ sys_revoke(td, uap)
 	int error;
 	struct nameidata nd;
 
-	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNODE1,
-	    UIO_USERSPACE, uap->path, td);
+	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNODE1, UIO_USERSPACE,
+	    uap->path, td);
 	if ((error = namei(&nd)) != 0)
 		return (error);
 	vp = nd.ni_vp;
@@ -4230,8 +4225,8 @@ sys_lgetfh(td, uap)
 	error = priv_check(td, PRIV_VFS_GETFH);
 	if (error)
 		return (error);
-	NDINIT(&nd, LOOKUP, NOFOLLOW | LOCKLEAF | AUDITVNODE1,
-	    UIO_USERSPACE, uap->fname, td);
+	NDINIT(&nd, LOOKUP, NOFOLLOW | LOCKLEAF | AUDITVNODE1, UIO_USERSPACE,
+	    uap->fname, td);
 	error = namei(&nd);
 	if (error)
 		return (error);
@@ -4266,8 +4261,8 @@ sys_getfh(td, uap)
 	error = priv_check(td, PRIV_VFS_GETFH);
 	if (error)
 		return (error);
-	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNODE1,
-	    UIO_USERSPACE, uap->fname, td);
+	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | AUDITVNODE1, UIO_USERSPACE,
+	    uap->fname, td);
 	error = namei(&nd);
 	if (error)
 		return (error);
diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c
index bbe837a..4ebf4b5 100644
--- a/sys/kern/vfs_vnops.c
+++ b/sys/kern/vfs_vnops.c
@@ -1642,7 +1642,7 @@ vfs_write_suspend(mp)
 	else
 		MNT_IUNLOCK(mp);
 	if ((error = VFS_SYNC(mp, MNT_SUSPEND)) != 0)
-		vfs_write_resume(mp);
+		vfs_write_resume(mp, 0);
 	return (error);
 }
 
@@ -1650,7 +1650,7 @@ vfs_write_suspend(mp)
  * Request a filesystem to resume write operations.
  */
 void
-vfs_write_resume_flags(struct mount *mp, int flags)
+vfs_write_resume(struct mount *mp, int flags)
 {
 
 	MNT_ILOCK(mp);
@@ -1677,23 +1677,14 @@ vfs_write_resume_flags(struct mount *mp, int flags)
 	}
 }
 
-void
-vfs_write_resume(struct mount *mp)
-{
-
-	vfs_write_resume_flags(mp, 0);
-}
-
 /*
  * Implement kqueues for files by translating it to vnode operation.
  */
 static int
 vn_kqfilter(struct file *fp, struct knote *kn)
 {
-	int error;
 
-	error = VOP_KQFILTER(fp->f_vnode, kn);
-	return (error);
+	return (VOP_KQFILTER(fp->f_vnode, kn));
 }
 
 /*
diff --git a/sys/libkern/arm/divsi3.S b/sys/libkern/arm/divsi3.S
index 00bacfa..700ae37 100644
--- a/sys/libkern/arm/divsi3.S
+++ b/sys/libkern/arm/divsi3.S
@@ -49,6 +49,10 @@ ENTRY_NP(__modsi3)
 #endif
 	RET
 
+#ifdef __ARM_EABI__
+ENTRY_NP(__aeabi_uidiv)
+ENTRY_NP(__aeabi_uidivmod)
+#endif
 ENTRY_NP(__udivsi3)
 .L_udivide:				/* r0 = r0 / r1; r1 = r0 % r1 */
 	eor     r0, r1, r0 
@@ -71,6 +75,10 @@ ENTRY_NP(__udivsi3)
 	mov	r1, #0
 	RET
 
+#ifdef __ARM_EABI__
+ENTRY_NP(__aeabi_idiv)
+ENTRY_NP(__aeabi_idivmod)
+#endif
 ENTRY_NP(__divsi3)
 .L_divide:				/* r0 = r0 / r1; r1 = r0 % r1 */
 	eor     r0, r1, r0 
diff --git a/sys/libkern/arm/ldivmod.S b/sys/libkern/arm/ldivmod.S
new file mode 100644
index 0000000..a88db54
--- /dev/null
+++ b/sys/libkern/arm/ldivmod.S
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2012 Andrew Turner
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$");
+
+#ifdef __ARM_EABI__
+
+/*
+ * These calculate:
+ * q = n / m 
+ * With a remainer r.
+ *
+ * They take n in {r0, r1} and m in {r2, r3} then pass them into the
+ * helper function. The hepler functions return q in {r0, r1} as
+ * required by the API spec however r is returned on the stack. The
+ * ABI required us to return r in {r2, r3}.
+ *
+ * We need to allocate 8 bytes on the stack to store r, the link
+ * register, and a pointer to the space where the helper function
+ * will write r to. After returning from the helper fuinction we load
+ * the old link register and r from the stack and return.
+ */
+ENTRY_NP(__aeabi_ldivmod)
+	sub	sp, sp, #8	/* Space for the remainder */
+	stmfd	sp!, {sp, lr}	/* Save a pointer to the above space and lr */
+	bl	PIC_SYM(_C_LABEL(__kern_ldivmod), PLT)
+	ldr	lr, [sp, #4]	/* Restore lr */
+	add	sp, sp, #8	/* Move sp to the remainder value */
+	ldmfd	sp!, {r2, r3}	/* Load the remainder */
+	RET
+
+ENTRY_NP(__aeabi_uldivmod)
+	sub	sp, sp, #8	/* Space for the remainder */
+	stmfd	sp!, {sp, lr}	/* Save a pointer to the above space and lr */
+	bl	PIC_SYM(_C_LABEL(__qdivrem), PLT)
+	ldr	lr, [sp, #4]	/* Restore lr */
+	add	sp, sp, #8	/* Move sp to the remainder value */
+	ldmfd	sp!, {r2, r3}	/* Load the remainder */
+	RET
+
+#endif
+
diff --git a/sys/arm/mv/kirkwood/sheevaplug.c b/sys/libkern/arm/ldivmod_helper.c
index 3463106..9061ae4 100644
--- a/sys/arm/mv/kirkwood/sheevaplug.c
+++ b/sys/libkern/arm/ldivmod_helper.c
@@ -1,10 +1,7 @@
-/*-
- * Copyright (c) 2010 The FreeBSD Foundation
+/*
+ * Copyright (C) 2012 Andrew Turner
  * All rights reserved.
  *
- * This software was developed by Semihalf under sponsorship from
- * the FreeBSD Foundation.
- *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -17,7 +14,7 @@
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
@@ -25,20 +22,32 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <dev/fdt/fdt_common.h>
-#include <dev/ofw/openfirm.h>
+#ifdef __ARM_EABI__
+#include <libkern/quad.h>
 
-#include <machine/fdt.h>
+/*
+ * Helper for __aeabi_ldivmod.
+ * TODO: __divdi3 calls __qdivrem. We should do the same and use the
+ * remainder value rather than re-calculating it.
+ */
+long long __kern_ldivmod(long long, long long, long long *);
 
-int
-fdt_pci_devmap(phandle_t node, struct pmap_devmap *devmap, vm_offset_t io_va,
-    vm_offset_t mem_va)
+long long
+__kern_ldivmod(long long n, long long m, long long *rem)
 {
+	long long q;
 
-	return (0);
+	q = __divdi3(n, m);	/* q = n / m */
+	*rem = n - m * q;
+
+	return q;
 }
+
+#endif
+
diff --git a/sys/libkern/quad.h b/sys/libkern/quad.h
index 5d4f844..4850d89 100644
--- a/sys/libkern/quad.h
+++ b/sys/libkern/quad.h
@@ -110,4 +110,9 @@ u_quad_t	__udivdi3(u_quad_t a, u_quad_t b);
 u_quad_t	__umoddi3(u_quad_t a, u_quad_t b);
 int		__ucmpdi2(u_quad_t a, u_quad_t b);
 
+/* ARM EABI support functions. */
+#ifdef __ARM_EABI__
+int		__aeabi_ulcmp(unsigned long long, unsigned long long);
+#endif
+
 #endif /* !_LIBKERN_QUAD_H_ */
diff --git a/sys/libkern/ucmpdi2.c b/sys/libkern/ucmpdi2.c
index 376d9c4..cd41c69 100644
--- a/sys/libkern/ucmpdi2.c
+++ b/sys/libkern/ucmpdi2.c
@@ -51,3 +51,15 @@ __ucmpdi2(a, b)
 	return (aa.ul[H] < bb.ul[H] ? 0 : aa.ul[H] > bb.ul[H] ? 2 :
 	    aa.ul[L] < bb.ul[L] ? 0 : aa.ul[L] > bb.ul[L] ? 2 : 1);
 }
+
+#ifdef __ARM_EABI__
+/*
+ * Return -1, 0 or 1 as a <, =, > b respectively.
+ */
+int
+__aeabi_ulcmp(unsigned long long a, unsigned long long b)
+{
+	return __ucmpdi2(a, b) - 1;
+}
+#endif
+
diff --git a/sys/mips/beri/beri_machdep.c b/sys/mips/beri/beri_machdep.c
index f2ef5a7..98fcb85a 100644
--- a/sys/mips/beri/beri_machdep.c
+++ b/sys/mips/beri/beri_machdep.c
@@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/cpu.h>
 #include <sys/cons.h>
 #include <sys/exec.h>
+#include <sys/linker.h>
 #include <sys/ucontext.h>
 #include <sys/proc.h>
 #include <sys/kdb.h>
@@ -70,6 +71,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/cpuregs.h>
 #include <machine/hwfunc.h>
 #include <machine/md_var.h>
+#include <machine/metadata.h>
 #include <machine/pmap.h>
 #include <machine/trap.h>
 
@@ -87,17 +89,6 @@ mips_init(void)
 {
 	int i;
 
-#ifdef FDT
-#ifndef FDT_DTB_STATIC
-#error	"mips_init with FDT requires FDT_DTB_STATIC"
-#endif
-
-	if (OF_install(OFW_FDT, 0) == FALSE)
-		while (1);
-	if (OF_init(&fdt_static_dtb) != 0)
-		while (1);
-#endif
-
 	for (i = 0; i < 10; i++) {
 		phys_avail[i] = 0;
 	}
@@ -146,6 +137,10 @@ platform_start(__register_t a0, __register_t a1,  __register_t a2,
 	char **argv = (char **)a1;
 	char **envp = (char **)a2;
 	unsigned int memsize = a3;
+#ifdef FDT
+	vm_offset_t dtbp;
+	void *kmdp;
+#endif
 	int i;
 
 	/* clear the BSS and SBSS segments */
@@ -156,6 +151,33 @@ platform_start(__register_t a0, __register_t a1,  __register_t a2,
 
 	mips_pcpu0_init();
 
+#ifdef FDT
+	/*
+	 * Find the dtb passed in by the boot loader (currently fictional).
+	 */
+	kmdp = preload_search_by_type("elf kernel");
+	if (kmdp != NULL)
+		dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t);
+	else
+		dtbp = (vm_offset_t)NULL;
+
+#if defined(FDT_DTB_STATIC)
+	/*
+	 * In case the device tree blob was not retrieved (from metadata) try
+	 * to use the statically embedded one.
+	 */
+	if (dtbp == (vm_offset_t)NULL)
+		dtbp = (vm_offset_t)&fdt_static_dtb;
+#else
+#error	"Non-static FDT not yet supported on BERI"
+#endif
+
+	if (OF_install(OFW_FDT, 0) == FALSE)
+		while (1);
+	if (OF_init(&fdt_static_dtb) != 0)
+		while (1);
+#endif
+
 	/*
 	 * XXXRW: We have no way to compare wallclock time to cycle rate on
 	 * BERI, so for now assume we run at the MALTA default (100MHz).
diff --git a/sys/mips/beri/files.beri b/sys/mips/beri/files.beri
index 02a27f8..6794793 100644
--- a/sys/mips/beri/files.beri
+++ b/sys/mips/beri/files.beri
@@ -1,10 +1,13 @@
 # $FreeBSD$
 dev/altera/jtag_uart/altera_jtag_uart_cons.c	optional altera_jtag_uart
 dev/altera/jtag_uart/altera_jtag_uart_tty.c	optional altera_jtag_uart
+dev/altera/jtag_uart/altera_jtag_uart_fdt.c	optional altera_jtag_uart fdt
 dev/altera/jtag_uart/altera_jtag_uart_nexus.c	optional altera_jtag_uart
 dev/terasic/de4led/terasic_de4led.c	optional terasic_de4led
+dev/terasic/de4led/terasic_de4led_fdt.c	optional terasic_de4led fdt
 dev/terasic/de4led/terasic_de4led_nexus.c	optional terasic_de4led
 dev/terasic/mtl/terasic_mtl.c		optional terasic_mtl
+dev/terasic/mtl/terasic_mtl_fdt.c	optional terasic_mtl fdt
 dev/terasic/mtl/terasic_mtl_nexus.c	optional terasic_mtl
 dev/terasic/mtl/terasic_mtl_pixel.c	optional terasic_mtl
 dev/terasic/mtl/terasic_mtl_reg.c	optional terasic_mtl
diff --git a/sys/mips/include/bus.h b/sys/mips/include/bus.h
index b904a5d..6825b23 100644
--- a/sys/mips/include/bus.h
+++ b/sys/mips/include/bus.h
@@ -728,6 +728,8 @@ void	__bs_c(f,_bs_c_8) (void *t, bus_space_handle_t bsh1,	\
  */
 DECLARE_BUS_SPACE_PROTOTYPES(generic);
 extern bus_space_tag_t mips_bus_space_generic;
+extern bus_space_tag_t mips_bus_space_fdt;
+
 /* Special bus space for RMI processors */
 #if defined(CPU_RMI) || defined (CPU_NLM)
 extern bus_space_tag_t rmi_bus_space;
diff --git a/sys/mips/include/fdt.h b/sys/mips/include/fdt.h
index 3b20a72..2f1fda5 100644
--- a/sys/mips/include/fdt.h
+++ b/sys/mips/include/fdt.h
@@ -51,7 +51,7 @@
 #if defined(CPU_RMI) || defined(CPU_NLM)
 #define fdtbus_bs_tag	rmi_uart_bus_space
 #else
-#define fdtbus_bs_tag	NULL
+#define fdtbus_bs_tag	mips_bus_space_fdt
 #endif
 
 #endif /* _MACHINE_FDT_H_ */
diff --git a/sys/mips/include/metadata.h b/sys/mips/include/metadata.h
index 84e6f87..779c2f6 100644
--- a/sys/mips/include/metadata.h
+++ b/sys/mips/include/metadata.h
@@ -30,5 +30,6 @@
 #define	_MACHINE_METADATA_H_
 
 #define	MODINFOMD_SMAP		0x1001
+#define	MODINFOMD_DTBP	0x1002
 
 #endif /* !_MACHINE_METADATA_H_ */
diff --git a/sys/mips/include/vmparam.h b/sys/mips/include/vmparam.h
index aa0a5d7..ef97336 100644
--- a/sys/mips/include/vmparam.h
+++ b/sys/mips/include/vmparam.h
@@ -130,10 +130,11 @@
 #endif
 
 /*
- * Ceiling on amount of kmem_map kva space.
+ * Ceiling on the amount of kmem_map KVA space: 40% of the entire KVA space.
  */
 #ifndef VM_KMEM_SIZE_MAX
-#define	VM_KMEM_SIZE_MAX	(200 * 1024 * 1024)
+#define	VM_KMEM_SIZE_MAX	((VM_MAX_KERNEL_ADDRESS - \
+    VM_MIN_KERNEL_ADDRESS + 1) * 2 / 5)
 #endif
 
 /* initial pagein size of beginning of executable file */
diff --git a/sys/mips/mips/bus_space_fdt.c b/sys/mips/mips/bus_space_fdt.c
new file mode 100644
index 0000000..38efea1
--- /dev/null
+++ b/sys/mips/mips/bus_space_fdt.c
@@ -0,0 +1,212 @@
+/*      $NetBSD: bus.h,v 1.12 1997/10/01 08:25:15 fvdl Exp $    */
+/*-
+ * $Id: bus.h,v 1.6 2007/08/09 11:23:32 katta Exp $
+ *
+ * Copyright (c) 1996, 1997 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
+ * NASA Ames Research Center.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the NetBSD
+ *	Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright (c) 1996 Charles M. Hannum.  All rights reserved.
+ * Copyright (c) 1996 Christopher G. Demetriou.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *      This product includes software developed by Christopher G. Demetriou
+ *	for the NetBSD Project.
+ * 4. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *	from: src/sys/alpha/include/bus.h,v 1.5 1999/08/28 00:38:40 peter
+ * $FreeBSD$
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/ktr.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_extern.h>
+
+#include <machine/bus.h>
+#include <machine/cache.h>
+
+static int	fdt_bs_map(void *, bus_addr_t, bus_size_t, int,
+		    bus_space_handle_t *);
+
+static struct bus_space fdt_space = {
+	/* cookie */
+	(void *) 0,
+
+	/* mapping/unmapping */
+	fdt_bs_map,
+	generic_bs_unmap,
+	generic_bs_subregion,
+
+	/* allocation/deallocation */
+	generic_bs_alloc,
+	generic_bs_free,
+
+	/* barrier */
+	generic_bs_barrier,
+
+	/* read (single) */
+	generic_bs_r_1,
+	generic_bs_r_2,
+	generic_bs_r_4,
+	generic_bs_r_8,
+
+	/* read multiple */
+	generic_bs_rm_1,
+	generic_bs_rm_2,
+	generic_bs_rm_4,
+	generic_bs_rm_8,
+
+	/* read region */
+	generic_bs_rr_1,
+	generic_bs_rr_2,
+	generic_bs_rr_4,
+	generic_bs_rr_8,
+
+	/* write (single) */
+	generic_bs_w_1,
+	generic_bs_w_2,
+	generic_bs_w_4,
+	generic_bs_w_8,
+
+	/* write multiple */
+	generic_bs_wm_1,
+	generic_bs_wm_2,
+	generic_bs_wm_4,
+	generic_bs_wm_8,
+
+	/* write region */
+	generic_bs_wr_1,
+	generic_bs_wr_2,
+	generic_bs_wr_4,
+	generic_bs_wr_8,
+
+	/* set multiple */
+	generic_bs_sm_1,
+	generic_bs_sm_2,
+	generic_bs_sm_4,
+	generic_bs_sm_8,
+
+	/* set region */
+	generic_bs_sr_1,
+	generic_bs_sr_2,
+	generic_bs_sr_4,
+	generic_bs_sr_8,
+
+	/* copy */
+	generic_bs_c_1,
+	generic_bs_c_2,
+	generic_bs_c_4,
+	generic_bs_c_8,
+
+	/* read (single) stream */
+	generic_bs_r_1,
+	generic_bs_r_2,
+	generic_bs_r_4,
+	generic_bs_r_8,
+
+	/* read multiple stream */
+	generic_bs_rm_1,
+	generic_bs_rm_2,
+	generic_bs_rm_4,
+	generic_bs_rm_8,
+
+	/* read region stream */
+	generic_bs_rr_1,
+	generic_bs_rr_2,
+	generic_bs_rr_4,
+	generic_bs_rr_8,
+
+	/* write (single) stream */
+	generic_bs_w_1,
+	generic_bs_w_2,
+	generic_bs_w_4,
+	generic_bs_w_8,
+
+	/* write multiple stream */
+	generic_bs_wm_1,
+	generic_bs_wm_2,
+	generic_bs_wm_4,
+	generic_bs_wm_8,
+
+	/* write region stream */
+	generic_bs_wr_1,
+	generic_bs_wr_2,
+	generic_bs_wr_4,
+	generic_bs_wr_8,
+};
+
+/* generic bus_space tag */
+bus_space_tag_t	mips_bus_space_fdt = &fdt_space;
+
+static int
+fdt_bs_map(void *t __unused, bus_addr_t addr, bus_size_t size __unused,
+    int flags __unused, bus_space_handle_t *bshp)
+{
+
+	*bshp = MIPS_PHYS_TO_DIRECT_UNCACHED(addr);
+	return (0);
+}
diff --git a/sys/mips/nlm/dev/net/mdio.c b/sys/mips/nlm/dev/net/mdio.c
index ed2abe4..0845339 100644
--- a/sys/mips/nlm/dev/net/mdio.c
+++ b/sys/mips/nlm/dev/net/mdio.c
@@ -149,7 +149,8 @@ nlm_int_gmac_mdio_reset(uint64_t nae_base, int bus, int block,
 	uint32_t val;
 
 	val = (7 << INT_MDIO_CTRL_XDIV_POS) | 
-	    (1 << INT_MDIO_CTRL_MCDIV_POS);
+	    (1 << INT_MDIO_CTRL_MCDIV_POS) |
+	    (INT_MDIO_CTRL_SMP);
 
 	nlm_write_nae_reg(nae_base,
 	    NAE_REG(block, intf_type, (INT_MDIO_CTRL + bus * 4)), 
@@ -302,10 +303,13 @@ nlm_gmac_mdio_reset(uint64_t nae_base, int bus, int block,
 {
 	uint32_t ctrlval;
 
+	ctrlval = nlm_read_nae_reg(nae_base,
+	    NAE_REG(block, intf_type, (EXT_G0_MDIO_CTRL+bus*4)));
+
 	if (nlm_is_xlp8xx_ax() || nlm_is_xlp8xx_b0() || nlm_is_xlp3xx_ax())
-	    	ctrlval = EXT_G_MDIO_DIV;
+		ctrlval |= EXT_G_MDIO_DIV;
 	else
-		ctrlval = EXT_G_MDIO_DIV_WITH_HW_DIV64;
+		ctrlval |= EXT_G_MDIO_DIV_WITH_HW_DIV64;
 
 	nlm_write_nae_reg(nae_base,
 	    NAE_REG(block, intf_type, (EXT_G0_MDIO_CTRL + bus * 4)), 
@@ -314,3 +318,16 @@ nlm_gmac_mdio_reset(uint64_t nae_base, int bus, int block,
 	    NAE_REG(block, intf_type, (EXT_G0_MDIO_CTRL + bus * 4)), ctrlval);
 	return (0);
 }
+
+/*
+ * nlm_mdio_reset_all : reset all internal and external MDIO
+ */
+void
+nlm_mdio_reset_all(uint64_t nae_base)
+{
+	/* reset internal MDIO */
+	nlm_int_gmac_mdio_reset(nae_base, 0, BLOCK_7, LANE_CFG);
+	/* reset external MDIO */
+	nlm_gmac_mdio_reset(nae_base, 0, BLOCK_7, LANE_CFG);
+	nlm_gmac_mdio_reset(nae_base, 1, BLOCK_7, LANE_CFG);
+}
diff --git a/sys/mips/nlm/dev/net/nae.c b/sys/mips/nlm/dev/net/nae.c
index dce4836..d97b290 100644
--- a/sys/mips/nlm/dev/net/nae.c
+++ b/sys/mips/nlm/dev/net/nae.c
@@ -1427,9 +1427,8 @@ nlm_nae_open_if(uint64_t nae_base, int nblock, int port_type,
     int port, uint32_t desc_size)
 {
 	uint32_t netwk_inf;
-	uint32_t mac_cfg1, mac_cfg2, netior_ctrl3;
-	int iface, speed, duplex, ifmode;
-	int iface_ctrl_reg, iface_ctrl3_reg, conf1_reg, conf2_reg;
+	uint32_t mac_cfg1, netior_ctrl3;
+	int iface, iface_ctrl_reg, iface_ctrl3_reg, conf1_reg, conf2_reg;
 
 	switch (port_type) {
 	case XAUIC:
@@ -1487,6 +1486,7 @@ nlm_nae_open_if(uint64_t nae_base, int nblock, int port_type,
 		/* clear gmac reset */
 		mac_cfg1 = nlm_read_nae_reg(nae_base, conf1_reg);
 		nlm_write_nae_reg(nae_base, conf1_reg, mac_cfg1 & ~(1 << 31));
+
 		/* clear speed debug bit */
 		iface_ctrl3_reg = SGMII_NET_IFACE_CTRL3(nblock, iface);
 		netior_ctrl3 = nlm_read_nae_reg(nae_base, iface_ctrl3_reg);
@@ -1500,33 +1500,21 @@ nlm_nae_open_if(uint64_t nae_base, int nblock, int port_type,
 		nlm_write_nae_reg(nae_base, iface_ctrl_reg,
 		    netwk_inf & ~(0x1 << 2));
 
-		/* setup defaults */	/* XXXJC: take defaults from sc? */
-		speed = 2;
-		duplex = 1;
-		ifmode = 0x2;
-		netwk_inf = nlm_read_nae_reg(nae_base, iface_ctrl_reg);
-		netwk_inf &= ~(0x3);
-		nlm_write_nae_reg(nae_base, iface_ctrl_reg,
-		    netwk_inf | (speed & 0x3));
-		mac_cfg2 = nlm_read_nae_reg(nae_base, conf2_reg);
-		mac_cfg2 &= ~(0x3 << 8);
-		nlm_write_nae_reg(nae_base, conf2_reg, 
-		    mac_cfg2			| 
-		    ((ifmode & 0x3) << 8)	| /* interface mode */
-		    (duplex & 0x1));
-
 		/* clear stats counters */
 		netwk_inf = nlm_read_nae_reg(nae_base, iface_ctrl_reg);
 		nlm_write_nae_reg(nae_base, iface_ctrl_reg,
 		    netwk_inf | (1 << 15));
+
 		/* enable stats counters */
 		netwk_inf = nlm_read_nae_reg(nae_base, iface_ctrl_reg);
 		nlm_write_nae_reg(nae_base, iface_ctrl_reg,
 		    (netwk_inf & ~(1 << 15)) | (1 << 16));
+
+		/* flow control? */
 		mac_cfg1 = nlm_read_nae_reg(nae_base, conf1_reg);
 		nlm_write_nae_reg(nae_base, conf1_reg,
 		    mac_cfg1 | (0x3 << 4));
-		break;
+ 		break;
 	}
 
 	nlm_nae_init_ingress(nae_base, desc_size);
diff --git a/sys/mips/nlm/dev/net/sgmii.c b/sys/mips/nlm/dev/net/sgmii.c
index 91f5f7a..18b1411 100644
--- a/sys/mips/nlm/dev/net/sgmii.c
+++ b/sys/mips/nlm/dev/net/sgmii.c
@@ -69,13 +69,6 @@ nlm_configure_sgmii_interface(uint64_t nae_base, int block, int port,
 void
 nlm_sgmii_pcs_init(uint64_t nae_base, uint32_t cplx_mask)
 {
-	/* reset internal MDIO */
-	nlm_int_gmac_mdio_reset(nae_base, 0, BLOCK_7, LANE_CFG);
-
-	/* reset external MDIO */
-	nlm_gmac_mdio_reset(nae_base, 0, BLOCK_7, LANE_CFG);
-	nlm_gmac_mdio_reset(nae_base, 1, BLOCK_7, LANE_CFG);
-
 	xlp_nae_config_lane_gmac(nae_base, cplx_mask);
 }
 
diff --git a/sys/mips/nlm/dev/net/xlpge.c b/sys/mips/nlm/dev/net/xlpge.c
index 0e41559..6d221e4 100644
--- a/sys/mips/nlm/dev/net/xlpge.c
+++ b/sys/mips/nlm/dev/net/xlpge.c
@@ -444,6 +444,8 @@ nlm_xlpnae_init(int node, struct nlm_xlpnae_softc *sc)
 	val = nlm_set_device_frequency(node, DFS_DEVICE_NAE, sc->freq);
 	printf("Setup NAE frequency to %dMHz\n", val);
 
+	nlm_mdio_reset_all(nae_base);
+
 	printf("Initialze SGMII PCS for blocks 0x%x\n", sc->sgmiimask);
 	nlm_sgmii_pcs_init(nae_base, sc->sgmiimask);
 
@@ -797,8 +799,9 @@ nlm_xlpge_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 		if (ifp->if_flags & IFF_UP) {
 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 				nlm_xlpge_init(sc);
-		    	nlm_xlpge_mac_set_rx_mode(sc);
-			nlm_xlpge_port_enable(sc);
+			else
+				nlm_xlpge_port_enable(sc);
+			nlm_xlpge_mac_set_rx_mode(sc);
 			sc->link = NLM_LINK_UP;
 		} else {
 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
@@ -866,6 +869,7 @@ xlpge_tx(struct ifnet *ifp, struct mbuf *mbuf_chain)
 		vm_offset_t buf = (vm_offset_t) m->m_data;
 		int	len = m->m_len;
 		int	frag_sz;
+		uint64_t desc;
 
 		/*printf("m_data = %p len %d\n", m->m_data, len); */
 		while (len) {
@@ -880,8 +884,9 @@ xlpge_tx(struct ifnet *ifp, struct mbuf *mbuf_chain)
 			frag_sz = PAGE_SIZE - (buf & PAGE_MASK);
 			if (len < frag_sz)
 				frag_sz = len;
-			p2p->frag[pos] = nae_tx_desc(P2D_NEOP, 0, 127,
+			desc = nae_tx_desc(P2D_NEOP, 0, 127,
 			    frag_sz, paddr);
+			p2p->frag[pos] = htobe64(desc);
 			pos++;
 			len -= frag_sz;
 			buf += frag_sz;
@@ -891,7 +896,7 @@ xlpge_tx(struct ifnet *ifp, struct mbuf *mbuf_chain)
 	KASSERT(pos != 0, ("Zero-length mbuf chain?\n"));
 
 	/* Make the last one P2D EOP */
-	p2p->frag[pos-1] |= (uint64_t)P2D_EOP << 62;
+	p2p->frag[pos-1] |= htobe64((uint64_t)P2D_EOP << 62);
 
 	/* stash useful pointers in the desc */
 	p2p->frag[XLP_NTXFRAGS-3] = 0xf00bad;
@@ -1128,7 +1133,8 @@ get_buf(void)
 	if ((m_new = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR)) == NULL)
 		return (NULL);
 	m_new->m_len = m_new->m_pkthdr.len = MCLBYTES;
-	m_adj(m_new, NAE_CACHELINE_SIZE - ((uintptr_t)m_new->m_data & 0x1f));
+	KASSERT(((uintptr_t)m_new->m_data & (NAE_CACHELINE_SIZE - 1)) == 0,
+	    ("m_new->m_data is not cacheline aligned"));
 	md = (uint64_t *)m_new->m_data;
 	md[0] = (intptr_t)m_new;        /* Back Ptr */
 	md[1] = 0xf00bad;
@@ -1137,10 +1143,9 @@ get_buf(void)
 #ifdef INVARIANTS
 	temp1 = vtophys((vm_offset_t) m_new->m_data);
 	temp2 = vtophys((vm_offset_t) m_new->m_data + 1536);
-	KASSERT(temp1 + 1536) != temp2,
+	KASSERT((temp1 + 1536) != temp2,
 	    ("Alloced buffer is not contiguous"));
 #endif
-
 	return ((void *)m_new->m_data);
 }
 
@@ -1288,6 +1293,7 @@ nlm_xlpge_attach(device_t dev)
 	nlm_xlpge_ifinit(sc);
 	ifp_ports[port].xlpge_sc = sc;
 	nlm_xlpge_mii_init(dev, sc);
+
 	nlm_xlpge_setup_stats_sysctl(dev, sc);
 
 	return (0);
@@ -1364,13 +1370,13 @@ nlm_xlpge_mii_statchg(device_t dev)
 	if (mii->mii_media_status & IFM_ACTIVE) {
 		if (IFM_SUBTYPE(mii->mii_media_active) ==  IFM_10_T) {
 			sc->speed = NLM_SGMII_SPEED_10;
-			speed =  "10-Mbps";
+			speed =  "10Mbps";
 		} else if (IFM_SUBTYPE(mii->mii_media_active) == IFM_100_TX) {
 			sc->speed = NLM_SGMII_SPEED_100;
-			speed = "100-Mbps";
+			speed = "100Mbps";
 		} else { /* default to 1G */
 			sc->speed = NLM_SGMII_SPEED_1000;
-			speed =  "1-Gbps";
+			speed =  "1Gbps";
 		}
 
 		if ((mii->mii_media_active & IFM_GMASK) == IFM_FDX) {
@@ -1381,11 +1387,11 @@ nlm_xlpge_mii_statchg(device_t dev)
 			duplexity = "half";
 		}
 
-		printf("Setup [complex=%d, port=%d] with speed=%s duplex=%s\n",
+		printf("Port [%d, %d] setup with speed=%s duplex=%s\n",
 		    sc->block, sc->port, speed, duplexity);
 
 		nlm_nae_setup_mac(sc->base_addr, sc->block, sc->port, 0, 1, 1,
-			sc->speed, sc->duplexity);
+		    sc->speed, sc->duplexity);
 	}
 }
 
@@ -1548,7 +1554,7 @@ nlm_xlpge_msgring_handler(int vc, int size, int code, int src_id,
 		ifp->if_opackets++;
 
 	} else if (size > 1) { /* Recieve packet */
-		phys_addr = msg->msg[1] & 0xffffffffe0ULL;
+		phys_addr = msg->msg[1] & 0xffffffffc0ULL;
 		length = (msg->msg[1] >> 40) & 0x3fff;
 		length -= MAC_CRC_LEN;
 
diff --git a/sys/mips/nlm/hal/fmn.c b/sys/mips/nlm/hal/fmn.c
index 6d40134..8ebc7537 100644
--- a/sys/mips/nlm/hal/fmn.c
+++ b/sys/mips/nlm/hal/fmn.c
@@ -138,9 +138,9 @@ uint32_t nlm_cms_total_stations = 18 * 4 /*xlp_num_nodes*/;
 
 void nlm_cms_setup_credits(uint64_t base, int destid, int srcid, int credit)
 {
-	uint32_t val;
+	uint64_t val;
 
-	val = ((credit << 24) | (destid << 12) | (srcid << 0));
+	val = (((uint64_t)credit << 24) | (destid << 12) | (srcid << 0));
 	nlm_write_cms_reg(base, CMS_OUTPUTQ_CREDIT_CFG, val);
 
 }
diff --git a/sys/mips/nlm/hal/mdio.h b/sys/mips/nlm/hal/mdio.h
index a1e720b..31b2aa0 100644
--- a/sys/mips/nlm/hal/mdio.h
+++ b/sys/mips/nlm/hal/mdio.h
@@ -100,6 +100,7 @@ int nlm_int_gmac_mdio_reset(uint64_t, int, int, int);
 int nlm_gmac_mdio_read(uint64_t, int, int, int, int, int);
 int nlm_gmac_mdio_write(uint64_t, int, int, int, int, int, uint16_t);
 int nlm_gmac_mdio_reset(uint64_t, int, int, int);
+void nlm_mdio_reset_all(uint64_t);
 
 #endif /* !(LOCORE) && !(__ASSEMBLY__) */
 #endif
diff --git a/sys/mips/nlm/hal/sys.h b/sys/mips/nlm/hal/sys.h
index adfd8a8..2092d37 100644
--- a/sys/mips/nlm/hal/sys.h
+++ b/sys/mips/nlm/hal/sys.h
@@ -140,5 +140,18 @@ enum {
 	INVALID_DFS_DEVICE = 0xFF
 };
 
+static __inline
+void nlm_sys_enable_block(uint64_t sys_base, int block)
+{
+	uint32_t dfsdis, mask;
+
+	mask = 1 << block;
+	dfsdis = nlm_read_sys_reg(sys_base, SYS_DFS_DIS_CTRL);
+	if ((dfsdis & mask) == 0)
+		return;			/* already enabled, nothing to do */
+	dfsdis &= ~mask;
+	nlm_write_sys_reg(sys_base, SYS_DFS_DIS_CTRL, dfsdis);
+}
+
 #endif
 #endif
diff --git a/sys/mips/nlm/hal/ucore_loader.h b/sys/mips/nlm/hal/ucore_loader.h
index 8298c82..cace077 100644
--- a/sys/mips/nlm/hal/ucore_loader.h
+++ b/sys/mips/nlm/hal/ucore_loader.h
@@ -49,7 +49,7 @@ nlm_ucore_load_image(uint64_t nae_base, int ucore)
 
 	size = sizeof(ucore_app_bin)/sizeof(uint32_t);
 	for (i = 0; i < size; i++, addr += 4)
-		nlm_store_word_daddr(addr, p[i]);
+		nlm_store_word_daddr(addr, htobe32(p[i]));
 
 	/* add a 'nop' if number of instructions are odd */
 	if (size & 0x1) 
diff --git a/sys/mips/nlm/xlp_machdep.c b/sys/mips/nlm/xlp_machdep.c
index a4ffc73..f95592e 100644
--- a/sys/mips/nlm/xlp_machdep.c
+++ b/sys/mips/nlm/xlp_machdep.c
@@ -157,6 +157,20 @@ xlp_setup_mmu(void)
 }
 
 static void
+xlp_enable_blocks(void)
+{
+	uint64_t sysbase;
+	int i;
+
+	for (i = 0; i < XLP_MAX_NODES; i++) {
+		if (!nlm_dev_exists(XLP_IO_SYS_OFFSET(i)))
+			continue;
+		sysbase = nlm_get_sys_regbase(i);
+		nlm_sys_enable_block(sysbase, DFS_DEVICE_RSA);
+	}
+}
+
+static void
 xlp_parse_mmu_options(void)
 {
 	uint64_t sysbase;
@@ -420,9 +434,8 @@ xlp_pic_init(void)
 #define	XLP_MEM_LIM	0xfffff000UL
 #endif
 static vm_paddr_t xlp_mem_excl[] = {
-	0,          0,		/* entry for kernel image, set by xlp_mem_init*/
-	0x0c000000, 0x0d000000,	/* uboot mess */
-	0x10000000, 0x14000000,	/* cms queue and other stuff */
+	0,          0,		/* for kernel image region, see xlp_mem_init */
+	0x0c000000, 0x14000000,	/* uboot area, cms queue and other stuff */
 	0x1fc00000, 0x1fd00000,	/* reset vec */
 	0x1e000000, 0x1e200000,	/* poe buffers */
 };
@@ -559,6 +572,8 @@ platform_start(__register_t a0 __unused,
 	/* setup for the startup core */
 	xlp_setup_mmu();
 
+	xlp_enable_blocks();
+
 	/* Read/Guess/setup board information */
 	nlm_board_info_setup();
 
diff --git a/sys/mips/nlm/xlp_pci.c b/sys/mips/nlm/xlp_pci.c
index 0610a15..23bd46c 100644
--- a/sys/mips/nlm/xlp_pci.c
+++ b/sys/mips/nlm/xlp_pci.c
@@ -487,12 +487,14 @@ xlp_pcib_write_config(device_t dev, u_int b, u_int s, u_int f,
 }
 
 /*
- * Enable byte swap in hardware. Program a link's PCIe SWAP regions
- * from the link's IO and MEM address ranges.
+ * Enable byte swap in hardware when compiled big-endian.
+ * Programs a link's PCIe SWAP regions from the link's IO and MEM address
+ * ranges.
  */
 static void
 xlp_pcib_hardware_swap_enable(int node, int link)
 {
+#if BYTE_ORDER == BIG_ENDIAN
 	uint64_t bbase, linkpcibase;
 	uint32_t bar;
 	int pcieoffset;
@@ -514,6 +516,7 @@ xlp_pcib_hardware_swap_enable(int node, int link)
 
 	bar = nlm_read_bridge_reg(bbase, BRIDGE_PCIEIO_LIMIT0 + link);
 	nlm_write_pci_reg(linkpcibase, PCIE_BYTE_SWAP_IO_LIM, bar | 0xFFF);
+#endif
 }
 
 static int 
diff --git a/sys/mips/rmi/msgring_xls.cfg b/sys/mips/rmi/msgring_xls.cfg
index 35bfb17..35bfb17 100755..100644
--- a/sys/mips/rmi/msgring_xls.cfg
+++ b/sys/mips/rmi/msgring_xls.cfg
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
index 1344297..28cb56c 100644
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -335,6 +335,7 @@ SUBDIR=	\
 	vge \
 	${_viawd} \
 	vkbd \
+	${_vmm} \
 	${_vpo} \
 	vr \
 	vte \
@@ -720,6 +721,7 @@ _twa=		twa
 _vesa=		vesa
 _viawd=		viawd
 _virtio=	virtio
+_vmm=		vmm
 _vxge=  	vxge
 _x86bios=	x86bios
 _wbwd=		wbwd
diff --git a/sys/modules/cxgbe/tom/Makefile b/sys/modules/cxgbe/tom/Makefile
index 72721be..d02afd4 100644
--- a/sys/modules/cxgbe/tom/Makefile
+++ b/sys/modules/cxgbe/tom/Makefile
@@ -10,15 +10,20 @@ CXGBE = ${.CURDIR}/../../../dev/cxgbe
 KMOD = t4_tom
 SRCS = t4_tom.c t4_connect.c t4_listen.c t4_cpl_io.c t4_tom_l2t.c t4_ddp.c
 SRCS+= device_if.h bus_if.h pci_if.h
-SRCS+= opt_inet.h
+SRCS+= opt_inet.h opt_inet6.h
 
 CFLAGS+= -I${CXGBE}
 
 .if !defined(KERNBUILDDIR)
 .if ${MK_INET_SUPPORT} != "no"
 opt_inet.h:
-	echo "#define INET 1" > ${.TARGET}
-	echo "#define TCP_OFFLOAD 1" >> ${.TARGET}
+	@echo "#define INET 1" > ${.TARGET}
+	@echo "#define TCP_OFFLOAD 1" >> ${.TARGET}
+.endif
+
+.if ${MK_INET6_SUPPORT} != "no"
+opt_inet6.h:
+	@echo "#define INET6 1" > ${.TARGET}
 .endif
 .endif
 
diff --git a/sys/modules/digi/Makefile b/sys/modules/digi/Makefile
index 9d9aea6..9d9aea6 100755..100644
--- a/sys/modules/digi/Makefile
+++ b/sys/modules/digi/Makefile
diff --git a/sys/modules/digi/Makefile.inc b/sys/modules/digi/Makefile.inc
index 265f86d..265f86d 100755..100644
--- a/sys/modules/digi/Makefile.inc
+++ b/sys/modules/digi/Makefile.inc
diff --git a/sys/modules/hpt27xx/Makefile b/sys/modules/hpt27xx/Makefile
index 883229b..82b51408 100644
--- a/sys/modules/hpt27xx/Makefile
+++ b/sys/modules/hpt27xx/Makefile
@@ -12,6 +12,3 @@ hpt27xx_lib.o:
 	uudecode -p < ${HPT27XX}/$(MACHINE_ARCH)-elf.hpt27xx_lib.o.uu > ${.TARGET}
 
 .include <bsd.kmod.mk>
-
-CWARNFLAGS.osm_bsd.c=	${NO_WFORMAT_SECURITY}
-CWARNFLAGS+=		${CWARNFLAGS.${.IMPSRC:T}}
diff --git a/sys/modules/isci/Makefile b/sys/modules/isci/Makefile
index 82d4018..82d4018 100755..100644
--- a/sys/modules/isci/Makefile
+++ b/sys/modules/isci/Makefile
diff --git a/sys/modules/sound/driver/ich/Makefile b/sys/modules/sound/driver/ich/Makefile
index e0f2ce7..e0f2ce7 100755..100644
--- a/sys/modules/sound/driver/ich/Makefile
+++ b/sys/modules/sound/driver/ich/Makefile
diff --git a/sys/modules/usb/Makefile b/sys/modules/usb/Makefile
index 4492755..6481838 100644
--- a/sys/modules/usb/Makefile
+++ b/sys/modules/usb/Makefile
@@ -36,7 +36,7 @@ SUBDIR += ${_rum} run ${_uath} upgt usie ural ${_zyd} ${_urtw}
 SUBDIR += atp uhid ukbd ums udbp ufm uep
 SUBDIR += ucom u3g uark ubsa ubser uchcom ucycom ufoma uftdi ugensa uipaq ulpt \
 	  umct umcs umodem umoscom uplcom uslcom uvisor uvscom
-SUBDIR += uether aue axe cdce cue ${_kue} mos rue udav uhso ipheth
+SUBDIR += uether aue axe cdce cue ${_kue} mos rue smsc udav uhso ipheth
 SUBDIR += usfs umass urio
 SUBDIR += quirk template
 
diff --git a/sys/modules/usb/smsc/Makefile b/sys/modules/usb/smsc/Makefile
index a97e9ba..c6f2de8 100644
--- a/sys/modules/usb/smsc/Makefile
+++ b/sys/modules/usb/smsc/Makefile
@@ -31,7 +31,7 @@ S=     ${.CURDIR}/../../..
 
 KMOD=	if_smsc
 SRCS=	opt_bus.h opt_usb.h device_if.h bus_if.h usb_if.h usbdevs.h \
-	miibus_if.h opt_inet.h \
+	miibus_if.h opt_inet.h opt_platform.h \
 	if_smsc.c
 
 .include <bsd.kmod.mk>
diff --git a/sys/modules/vmm/Makefile b/sys/modules/vmm/Makefile
new file mode 100644
index 0000000..8b565da
--- /dev/null
+++ b/sys/modules/vmm/Makefile
@@ -0,0 +1,62 @@
+# $FreeBSD$
+
+KMOD=	vmm
+
+SRCS=	opt_ddb.h device_if.h bus_if.h pci_if.h
+
+CFLAGS+= -DVMM_KEEP_STATS -DSMP
+CFLAGS+= -I${.CURDIR}/../../amd64/vmm	
+CFLAGS+= -I${.CURDIR}/../../amd64/vmm/io	
+CFLAGS+= -I${.CURDIR}/../../amd64/vmm/intel
+
+# generic vmm support
+.PATH: ${.CURDIR}/../../amd64/vmm
+SRCS+=	vmm.c		\
+	vmm_dev.c	\
+	vmm_host.c	\
+	vmm_instruction_emul.c	\
+	vmm_ipi.c	\
+	vmm_lapic.c	\
+	vmm_mem.c	\
+	vmm_msr.c	\
+	vmm_stat.c	\
+	vmm_util.c	\
+	x86.c		\
+	vmm_support.S
+
+.PATH: ${.CURDIR}/../../amd64/vmm/io
+SRCS+=	iommu.c		\
+	ppt.c           \
+	vdev.c		\
+	vlapic.c
+
+# intel-specific files
+.PATH: ${.CURDIR}/../../amd64/vmm/intel
+SRCS+=	ept.c		\
+	vmcs.c		\
+	vmx_msr.c	\
+	vmx.c		\
+	vtd.c
+
+# amd-specific files
+.PATH: ${.CURDIR}/../../amd64/vmm/amd
+SRCS+=	amdv.c
+
+OBJS=	vmx_support.o
+
+CLEANFILES=	vmx_assym.s vmx_genassym.o
+
+vmx_assym.s:    vmx_genassym.o
+.if exists(@)
+vmx_assym.s:    @/kern/genassym.sh
+.endif
+	sh @/kern/genassym.sh vmx_genassym.o > ${.TARGET}
+
+vmx_support.o:	vmx_support.S vmx_assym.s
+	${CC} -c -x assembler-with-cpp -DLOCORE ${CFLAGS} \
+	    ${.IMPSRC} -o ${.TARGET}
+
+vmx_genassym.o: vmx_genassym.c @ machine x86
+	${CC} -c ${CFLAGS:N-fno-common} ${.IMPSRC}
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/wlan/Makefile b/sys/modules/wlan/Makefile
index 7205e28..30aa745 100644
--- a/sys/modules/wlan/Makefile
+++ b/sys/modules/wlan/Makefile
@@ -12,7 +12,7 @@ SRCS=	ieee80211.c ieee80211_action.c ieee80211_ageq.c \
 	ieee80211_ratectl_none.c ieee80211_regdomain.c \
 	ieee80211_ht.c ieee80211_hwmp.c ieee80211_adhoc.c ieee80211_hostap.c \
 	ieee80211_monitor.c ieee80211_sta.c ieee80211_wds.c ieee80211_ddb.c \
-	ieee80211_tdma.c
+	ieee80211_tdma.c ieee80211_superg.c
 SRCS+=	bus_if.h device_if.h opt_ddb.h opt_inet.h opt_inet6.h opt_ipx.h \
 	opt_tdma.h opt_wlan.h
 
diff --git a/sys/net/bpf.c b/sys/net/bpf.c
index 9ee1964..ba31c66 100644
--- a/sys/net/bpf.c
+++ b/sys/net/bpf.c
@@ -522,32 +522,15 @@ bpf_movein(struct uio *uio, int linktype, struct ifnet *ifp, struct mbuf **mp,
 	}
 
 	len = uio->uio_resid;
-
-	if (len - hlen > ifp->if_mtu)
+	if (len < hlen || len - hlen > ifp->if_mtu)
 		return (EMSGSIZE);
 
-	if ((unsigned)len > MJUM16BYTES)
+	m = m_get2(M_WAITOK, MT_DATA, M_PKTHDR, len);
+	if (m == NULL)
 		return (EIO);
-
-	if (len <= MHLEN)
-		MGETHDR(m, M_WAITOK, MT_DATA);
-	else if (len <= MCLBYTES)
-		m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR);
-	else
-		m = m_getjcl(M_WAITOK, MT_DATA, M_PKTHDR,
-#if (MJUMPAGESIZE > MCLBYTES)
-		    len <= MJUMPAGESIZE ? MJUMPAGESIZE :
-#endif
-		    (len <= MJUM9BYTES ? MJUM9BYTES : MJUM16BYTES));
 	m->m_pkthdr.len = m->m_len = len;
-	m->m_pkthdr.rcvif = NULL;
 	*mp = m;
 
-	if (m->m_len < hlen) {
-		error = EPERM;
-		goto bad;
-	}
-
 	error = uiomove(mtod(m, u_char *), len, uio);
 	if (error)
 		goto bad;
diff --git a/sys/net/if_lagg.c b/sys/net/if_lagg.c
index 0428cbe..3634d0b 100644
--- a/sys/net/if_lagg.c
+++ b/sys/net/if_lagg.c
@@ -789,7 +789,7 @@ lagg_port_output(struct ifnet *ifp, struct mbuf *m,
 
 	/* drop any other frames */
 	m_freem(m);
-	return (EBUSY);
+	return (ENETDOWN);
 }
 
 static void
@@ -1862,7 +1862,7 @@ lagg_lacp_start(struct lagg_softc *sc, struct mbuf *m)
 	lp = lacp_select_tx_port(sc, m);
 	if (lp == NULL) {
 		m_freem(m);
-		return (EBUSY);
+		return (ENETDOWN);
 	}
 
 	/* Send mbuf */
diff --git a/sys/net/if_llatbl.c b/sys/net/if_llatbl.c
index 5812623..84ea6c6 100644
--- a/sys/net/if_llatbl.c
+++ b/sys/net/if_llatbl.c
@@ -273,10 +273,9 @@ lla_rt_output(struct rt_msghdr *rtm, struct rt_addrinfo *info)
 	u_int laflags = 0, flags = 0;
 	int error = 0;
 
-	if (dl == NULL || dl->sdl_family != AF_LINK) {
-		log(LOG_INFO, "%s: invalid dl\n", __func__);
-		return EINVAL;
-	}
+	KASSERT(dl != NULL && dl->sdl_family == AF_LINK,
+	    ("%s: invalid dl\n", __func__));
+
 	ifp = ifnet_byindex(dl->sdl_index);
 	if (ifp == NULL) {
 		log(LOG_INFO, "%s: invalid ifp (sdl_index %d)\n",
@@ -286,28 +285,8 @@ lla_rt_output(struct rt_msghdr *rtm, struct rt_addrinfo *info)
 
 	switch (rtm->rtm_type) {
 	case RTM_ADD:
-		if (rtm->rtm_flags & RTF_ANNOUNCE) {
+		if (rtm->rtm_flags & RTF_ANNOUNCE)
 			flags |= LLE_PUB;
-#ifdef INET
-			if (dst->sa_family == AF_INET &&
-			    ((struct sockaddr_inarp *)dst)->sin_other != 0) {
-				struct rtentry *rt;
-				((struct sockaddr_inarp *)dst)->sin_other = 0;
-				rt = rtalloc1(dst, 0, 0);
-				if (rt == NULL || !(rt->rt_flags & RTF_HOST)) {
-					log(LOG_INFO, "%s: RTM_ADD publish "
-					    "(proxy only) is invalid\n",
-					    __func__);
-					if (rt)
-						RTFREE_LOCKED(rt);
-					return EINVAL;
-				}
-				RTFREE_LOCKED(rt);
-
-				flags |= LLE_PROXY;
-			}
-#endif
-		}
 		flags |= LLE_CREATE;
 		break;
 
@@ -346,7 +325,7 @@ lla_rt_output(struct rt_msghdr *rtm, struct rt_addrinfo *info)
 			 * LLE_DELETED flag, and reset the expiration timer
 			 */
 			bcopy(LLADDR(dl), &lle->ll_addr, ifp->if_addrlen);
-			lle->la_flags |= (flags & (LLE_PUB | LLE_PROXY));
+			lle->la_flags |= (flags & LLE_PUB);
 			lle->la_flags |= LLE_VALID;
 			lle->la_flags &= ~LLE_DELETED;
 #ifdef INET6
@@ -368,15 +347,12 @@ lla_rt_output(struct rt_msghdr *rtm, struct rt_addrinfo *info)
 			laflags = lle->la_flags;
 			LLE_WUNLOCK(lle);
 #ifdef INET
-			/*  gratuitous ARP */
-			if ((laflags & LLE_PUB) && dst->sa_family == AF_INET) {
+			/* gratuitous ARP */
+			if ((laflags & LLE_PUB) && dst->sa_family == AF_INET)
 				arprequest(ifp,
 				    &((struct sockaddr_in *)dst)->sin_addr,
 				    &((struct sockaddr_in *)dst)->sin_addr,
-				    ((laflags & LLE_PROXY) ?
-					(u_char *)IF_LLADDR(ifp) :
-					(u_char *)LLADDR(dl)));
-			}
+				    (u_char *)LLADDR(dl));
 #endif
 		} else {
 			if (flags & LLE_EXCLUSIVE)
diff --git a/sys/net/if_llatbl.h b/sys/net/if_llatbl.h
index 8da08ba..693ccd5 100644
--- a/sys/net/if_llatbl.h
+++ b/sys/net/if_llatbl.h
@@ -172,7 +172,6 @@ MALLOC_DECLARE(M_LLTABLE);
 #define	LLE_STATIC	0x0002	/* entry is static */
 #define	LLE_IFADDR	0x0004	/* entry is interface addr */
 #define	LLE_VALID	0x0008	/* ll_addr is valid */
-#define	LLE_PROXY	0x0010	/* proxy entry ??? */
 #define	LLE_PUB		0x0020	/* publish entry ??? */
 #define	LLE_LINKED	0x0040	/* linked to lookup structure */
 #define	LLE_EXCLUSIVE	0x2000	/* return lle xlocked  */
@@ -205,4 +204,14 @@ lla_lookup(struct lltable *llt, u_int flags, const struct sockaddr *l3addr)
 }
 
 int		lla_rt_output(struct rt_msghdr *, struct rt_addrinfo *);
+
+#include <sys/eventhandler.h>
+enum {
+	LLENTRY_RESOLVED,
+	LLENTRY_TIMEDOUT,
+	LLENTRY_DELETED,
+	LLENTRY_EXPIRED,
+};
+typedef void (*lle_event_fn)(void *, struct llentry *, int);
+EVENTHANDLER_DECLARE(lle_event, lle_event_fn);
 #endif  /* _NET_IF_LLATBL_H_ */
diff --git a/sys/net/netmap.h b/sys/net/netmap.h
index 1aa5e29..638cb18 100644
--- a/sys/net/netmap.h
+++ b/sys/net/netmap.h
@@ -32,7 +32,7 @@
 
 /*
  * $FreeBSD$
- * $Id: netmap.h 10601 2012-02-21 16:40:14Z luigi $
+ * $Id: netmap.h 11997 2013-01-17 21:59:12Z luigi $
  *
  * Definitions of constants and the structures used by the netmap
  * framework, for the part visible to both kernel and userspace.
@@ -113,15 +113,28 @@
  * In the kernel, buffers do not necessarily need to be contiguous,
  * and the virtual and physical addresses are derived through
  * a lookup table.
- * To associate a different buffer to a slot, applications must
- * write the new index in buf_idx, and set NS_BUF_CHANGED flag to
- * make sure that the kernel updates the hardware ring as needed.
  *
- * Normally the driver is not requested to report the result of
- * transmissions (this can dramatically speed up operation).
- * However the user may request to report completion by setting
- * NS_REPORT.
+ * struct netmap_slot:
+ *
+ * buf_idx	is the index of the buffer associated to the slot.
+ * len		is the length of the payload
+ * NS_BUF_CHANGED	must be set whenever userspace wants
+ *		to change buf_idx (it might be necessary to
+ *		reprogram the NIC slot)
+ * NS_REPORT	must be set if we want the NIC to generate an interrupt
+ *		when this slot is used. Leaving it to 0 improves
+ *		performance.
+ * NS_FORWARD	if set on a receive ring, and the device is in
+ *		transparent mode, buffers released with the flag set
+ *		will be forwarded to the 'other' side (host stack
+ *		or NIC, respectively) on the next select() or ioctl()
+ * NS_NO_LEARN	on a VALE switch, do not 'learn' the source port for
+ *		this packet.
+ * NS_PORT_MASK	the high 8 bits of the flag, if not zero, indicate the
+ *		destination port for the VALE switch, overriding
+ *		the lookup table.
  */
+
 struct netmap_slot {
 	uint32_t buf_idx; /* buffer index */
 	uint16_t len;	/* packet length, to be copied to/from the hw ring */
@@ -130,6 +143,12 @@ struct netmap_slot {
 #define	NS_REPORT	0x0002	/* ask the hardware to report results
 				 * e.g. by generating an interrupt
 				 */
+#define	NS_FORWARD	0x0004	/* pass packet to the other endpoint
+				 * (host stack or device)
+				 */
+#define	NS_NO_LEARN	0x0008
+#define	NS_PORT_SHIFT	8
+#define	NS_PORT_MASK	(0xff << NS_PORT_SHIFT)
 };
 
 /*
@@ -186,6 +205,18 @@ struct netmap_slot {
  *	a system call.
  *
  *	The netmap_kring is only modified by the upper half of the kernel.
+ *
+ * FLAGS
+ *	NR_TIMESTAMP	updates the 'ts' field on each syscall. This is
+ *			a global timestamp for all packets.
+ *	NR_RX_TSTMP	if set, the last 64 byte in each buffer will
+ *			contain a timestamp for the frame supplied by
+ *			the hardware (if supported)
+ *	NR_FORWARD	if set, the NS_FORWARD flag in each slot of the
+ *			RX ring is checked, and if set the packet is
+ *			passed to the other side (host stack or device,
+ *			respectively). This permits bpf-like behaviour
+ *			or transparency for selected packets.
  */
 struct netmap_ring {
 	/*
@@ -202,6 +233,8 @@ struct netmap_ring {
 	const uint16_t	nr_buf_size;
 	uint16_t	flags;
 #define	NR_TIMESTAMP	0x0002		/* set timestamp on *sync() */
+#define	NR_FORWARD	0x0004		/* enable NS_FORWARD for ring */
+#define	NR_RX_TSTMP	0x0008		/* set rx timestamp in slots */
 
 	struct timeval	ts;		/* time of last *sync() */
 
diff --git a/sys/net80211/ieee80211_adhoc.c b/sys/net80211/ieee80211_adhoc.c
index 0220474..b0998f5 100644
--- a/sys/net80211/ieee80211_adhoc.c
+++ b/sys/net80211/ieee80211_adhoc.c
@@ -688,6 +688,7 @@ adhoc_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0,
 	struct ieee80211_frame *wh;
 	uint8_t *frm, *efrm, *sfrm;
 	uint8_t *ssid, *rates, *xrates;
+	int ht_state_change = 0;
 
 	wh = mtod(m0, struct ieee80211_frame *);
 	frm = (uint8_t *)&wh[1];
@@ -748,10 +749,27 @@ adhoc_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m0,
 				memcpy(ni->ni_tstamp.data, scan.tstamp,
 					sizeof(ni->ni_tstamp));
 			}
+			/*
+			 * This isn't enabled yet - otherwise it would
+			 * update the HT parameters and channel width
+			 * from any node, which could lead to lots of
+			 * strange behaviour if the 11n nodes aren't
+			 * exactly configured to match.
+			 */
+#if 0
+			if (scan.htcap != NULL && scan.htinfo != NULL &&
+			    (vap->iv_flags_ht & IEEE80211_FHT_HT)) {
+				if (ieee80211_ht_updateparams(ni,
+				    scan.htcap, scan.htinfo))
+					ht_state_change = 1;
+			}
+#endif
 			if (ni != NULL) {
 				IEEE80211_RSSI_LPF(ni->ni_avgrssi, rssi);
 				ni->ni_noise = nf;
 			}
+			if (ht_state_change)
+				ieee80211_update_chw(ic);
 		}
 		break;
 	}
diff --git a/sys/net80211/ieee80211_node.c b/sys/net80211/ieee80211_node.c
index 7ed3523..a81c481 100644
--- a/sys/net80211/ieee80211_node.c
+++ b/sys/net80211/ieee80211_node.c
@@ -771,6 +771,7 @@ ieee80211_sta_join(struct ieee80211vap *vap, struct ieee80211_channel *chan,
 		/* XXX msg */
 		return 0;
 	}
+
 	/*
 	 * Expand scan state into node's format.
 	 * XXX may not need all this stuff
@@ -821,6 +822,29 @@ ieee80211_sta_join(struct ieee80211vap *vap, struct ieee80211_channel *chan,
 		IEEE80211_F_DOSORT);
 	if (ieee80211_iserp_rateset(&ni->ni_rates))
 		ni->ni_flags |= IEEE80211_NODE_ERP;
+
+	/*
+	 * Setup HT state for this node if it's available, otherwise
+	 * non-STA modes won't pick this state up.
+	 *
+	 * For IBSS and related modes that don't go through an
+	 * association request/response, the only appropriate place
+	 * to setup the HT state is here.
+	 */
+	if (ni->ni_ies.htinfo_ie != NULL &&
+	    ni->ni_ies.htcap_ie != NULL &&
+	    vap->iv_flags_ht & IEEE80211_FHT_HT) {
+		ieee80211_ht_node_init(ni);
+		ieee80211_ht_updateparams(ni,
+		    ni->ni_ies.htcap_ie,
+		    ni->ni_ies.htinfo_ie);
+		ieee80211_setup_htrates(ni, ni->ni_ies.htcap_ie,
+		    IEEE80211_F_JOIN | IEEE80211_F_DOBRS);
+		ieee80211_setup_basic_htrates(ni, ni->ni_ies.htinfo_ie);
+	}
+	/* XXX else check for ath FF? */
+	/* XXX QoS? Difficult given that WME config is specific to a master */
+
 	ieee80211_node_setuptxparms(ni);
 	ieee80211_ratectl_node_init(ni);
 
@@ -938,6 +962,9 @@ ieee80211_ies_expand(struct ieee80211_ies *ies)
 		case IEEE80211_ELEMID_HTCAP:
 			ies->htcap_ie = ie;
 			break;
+		case IEEE80211_ELEMID_HTINFO:
+			ies->htinfo_ie = ie;
+			break;
 #ifdef IEEE80211_SUPPORT_MESH
 		case IEEE80211_ELEMID_MESHID:
 			ies->meshid_ie = ie;
@@ -1404,7 +1431,7 @@ ieee80211_fakeup_adhoc_node(struct ieee80211vap *vap,
 {
 	struct ieee80211_node *ni;
 
-	IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE,
+	IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE | IEEE80211_MSG_ASSOC,
 	    "%s: mac<%s>\n", __func__, ether_sprintf(macaddr));
 	ni = ieee80211_dup_bss(vap, macaddr);
 	if (ni != NULL) {
@@ -1444,6 +1471,8 @@ ieee80211_init_neighbor(struct ieee80211_node *ni,
 	const struct ieee80211_frame *wh,
 	const struct ieee80211_scanparams *sp)
 {
+	int do_ht_setup = 0;
+
 	ni->ni_esslen = sp->ssid[1];
 	memcpy(ni->ni_essid, sp->ssid + 2, sp->ssid[1]);
 	IEEE80211_ADDR_COPY(ni->ni_bssid, wh->i_addr3);
@@ -1469,12 +1498,41 @@ ieee80211_init_neighbor(struct ieee80211_node *ni,
 		if (ni->ni_ies.ath_ie != NULL)
 			ieee80211_parse_ath(ni, ni->ni_ies.ath_ie);
 #endif
+		if (ni->ni_ies.htcap_ie != NULL)
+			ieee80211_parse_htcap(ni, ni->ni_ies.htcap_ie);
+		if (ni->ni_ies.htinfo_ie != NULL)
+			ieee80211_parse_htinfo(ni, ni->ni_ies.htinfo_ie);
+
+		if ((ni->ni_ies.htcap_ie != NULL) &&
+		    (ni->ni_ies.htinfo_ie != NULL) &&
+		    (ni->ni_vap->iv_flags_ht & IEEE80211_FHT_HT)) {
+			do_ht_setup = 1;
+		}
 	}
 
 	/* NB: must be after ni_chan is setup */
 	ieee80211_setup_rates(ni, sp->rates, sp->xrates,
 		IEEE80211_F_DOSORT | IEEE80211_F_DOFRATE |
 		IEEE80211_F_DONEGO | IEEE80211_F_DODEL);
+
+	/*
+	 * If the neighbor is HT compatible, flip that on.
+	 */
+	if (do_ht_setup) {
+		IEEE80211_DPRINTF(ni->ni_vap, IEEE80211_MSG_ASSOC,
+		    "%s: doing HT setup\n", __func__);
+		ieee80211_ht_node_init(ni);
+		ieee80211_ht_updateparams(ni,
+		    ni->ni_ies.htcap_ie,
+		    ni->ni_ies.htinfo_ie);
+		ieee80211_setup_htrates(ni,
+		    ni->ni_ies.htcap_ie,
+		    IEEE80211_F_JOIN | IEEE80211_F_DOBRS);
+		ieee80211_setup_basic_htrates(ni,
+		    ni->ni_ies.htinfo_ie);
+		ieee80211_node_setuptxparms(ni);
+		ieee80211_ratectl_node_init(ni);
+	}
 }
 
 /*
@@ -1490,7 +1548,7 @@ ieee80211_add_neighbor(struct ieee80211vap *vap,
 {
 	struct ieee80211_node *ni;
 
-	IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE,
+	IEEE80211_DPRINTF(vap, IEEE80211_MSG_ASSOC,
 	    "%s: mac<%s>\n", __func__, ether_sprintf(wh->i_addr2));
 	ni = ieee80211_dup_bss(vap, wh->i_addr2);/* XXX alloc_node? */
 	if (ni != NULL) {
diff --git a/sys/net80211/ieee80211_power.c b/sys/net80211/ieee80211_power.c
index 558e1f3..31bc578 100644
--- a/sys/net80211/ieee80211_power.c
+++ b/sys/net80211/ieee80211_power.c
@@ -416,6 +416,8 @@ pwrsave_flushq(struct ieee80211_node *ni)
 	struct ieee80211vap *vap = ni->ni_vap;
 	struct ieee80211_psq_head *qhead;
 	struct ifnet *parent, *ifp;
+	struct mbuf *parent_q = NULL, *ifp_q = NULL;
+	struct mbuf *m;
 
 	IEEE80211_NOTE(vap, IEEE80211_MSG_POWER, ni,
 	    "flush ps queue, %u packets queued", psq->psq_len);
@@ -427,8 +429,7 @@ pwrsave_flushq(struct ieee80211_node *ni)
 		parent = vap->iv_ic->ic_ifp;
 		/* XXX need different driver interface */
 		/* XXX bypasses q max and OACTIVE */
-		IF_PREPEND_LIST(&parent->if_snd, qhead->head, qhead->tail,
-		    qhead->len);
+		parent_q = qhead->head;
 		qhead->head = qhead->tail = NULL;
 		qhead->len = 0;
 	} else
@@ -439,8 +440,7 @@ pwrsave_flushq(struct ieee80211_node *ni)
 		ifp = vap->iv_ifp;
 		/* XXX need different driver interface */
 		/* XXX bypasses q max and OACTIVE */
-		IF_PREPEND_LIST(&ifp->if_snd, qhead->head, qhead->tail,
-		    qhead->len);
+		ifp_q = qhead->head;
 		qhead->head = qhead->tail = NULL;
 		qhead->len = 0;
 	} else
@@ -450,10 +450,34 @@ pwrsave_flushq(struct ieee80211_node *ni)
 
 	/* NB: do this outside the psq lock */
 	/* XXX packets might get reordered if parent is OACTIVE */
-	if (parent != NULL)
-		if_start(parent);
-	if (ifp != NULL)
-		if_start(ifp);
+	/* parent frames, should be encapsulated */
+	if (parent != NULL) {
+		while (parent_q != NULL) {
+			m = parent_q;
+			parent_q = m->m_nextpkt;
+			/* must be encapsulated */
+			KASSERT((m->m_flags & M_ENCAP),
+			    ("%s: parentq with non-M_ENCAP frame!\n",
+			    __func__));
+			/*
+			 * For encaped frames, we need to free the node
+			 * reference upon failure.
+			 */
+			if (parent->if_transmit(parent, m) != 0)
+				ieee80211_free_node(ni);
+		}
+	}
+
+	/* VAP frames, aren't encapsulated */
+	if (ifp != NULL) {
+		while (ifp_q != NULL) {
+			m = ifp_q;
+			ifp_q = m->m_nextpkt;
+			KASSERT((!(m->m_flags & M_ENCAP)),
+			    ("%s: vapq with M_ENCAP frame!\n", __func__));
+			(void) ifp->if_transmit(ifp, m);
+		}
+	}
 }
 
 /*
diff --git a/sys/net80211/ieee80211_scan_sta.c b/sys/net80211/ieee80211_scan_sta.c
index 6f5e9d2..381faca 100644
--- a/sys/net80211/ieee80211_scan_sta.c
+++ b/sys/net80211/ieee80211_scan_sta.c
@@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
 #ifdef IEEE80211_SUPPORT_MESH
 #include <net80211/ieee80211_mesh.h>
 #endif
+#include <net80211/ieee80211_ratectl.h>
 
 #include <net/bpf.h>
 
@@ -1567,6 +1568,7 @@ adhoc_pick_bss(struct ieee80211_scan_state *ss, struct ieee80211vap *vap)
 	struct sta_table *st = ss->ss_priv;
 	struct sta_entry *selbs;
 	struct ieee80211_channel *chan;
+	struct ieee80211com *ic = vap->iv_ic;
 
 	KASSERT(vap->iv_opmode == IEEE80211_M_IBSS ||
 		vap->iv_opmode == IEEE80211_M_AHDEMO ||
@@ -1612,15 +1614,19 @@ notfound:
 			 */
 			if (vap->iv_des_chan == IEEE80211_CHAN_ANYC ||
 			    IEEE80211_IS_CHAN_RADAR(vap->iv_des_chan)) {
-				struct ieee80211com *ic = vap->iv_ic;
-
 				chan = adhoc_pick_channel(ss, 0);
-				if (chan != NULL)
-					chan = ieee80211_ht_adjust_channel(ic,
-					    chan, vap->iv_flags_ht);
 			} else
 				chan = vap->iv_des_chan;
 			if (chan != NULL) {
+				struct ieee80211com *ic = vap->iv_ic;
+				/*
+				 * Create a HT capable IBSS; the per-node
+				 * probe request/response will result in
+				 * "correct" rate control capabilities being
+				 * negotiated.
+				 */
+				chan = ieee80211_ht_adjust_channel(ic,
+				    chan, vap->iv_flags_ht);
 				ieee80211_create_ibss(vap, chan);
 				return 1;
 			}
@@ -1644,6 +1650,14 @@ notfound:
 	chan = selbs->base.se_chan;
 	if (selbs->se_flags & STA_DEMOTE11B)
 		chan = demote11b(vap, chan);
+	/*
+	 * If HT is available, make it a possibility here.
+	 * The intent is to enable HT20/HT40 when joining a non-HT
+	 * IBSS node; we can then advertise HT IEs and speak HT
+	 * to any subsequent nodes that support it.
+	 */
+	chan = ieee80211_ht_adjust_channel(ic,
+	    chan, vap->iv_flags_ht);
 	if (!ieee80211_sta_join(vap, chan, &selbs->base))
 		goto notfound;
 	return 1;				/* terminate scan */
diff --git a/sys/net80211/ieee80211_superg.c b/sys/net80211/ieee80211_superg.c
index 0599f45..9ac5878 100644
--- a/sys/net80211/ieee80211_superg.c
+++ b/sys/net80211/ieee80211_superg.c
@@ -28,6 +28,8 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_wlan.h"
 
+#ifdef	IEEE80211_SUPPORT_SUPERG
+
 #include <sys/param.h>
 #include <sys/systm.h> 
 #include <sys/mbuf.h>   
@@ -921,3 +923,5 @@ superg_ioctl_set80211(struct ieee80211vap *vap, struct ieee80211req *ireq)
 	return 0;
 }
 IEEE80211_IOCTL_SET(superg, superg_ioctl_set80211);
+
+#endif	/* IEEE80211_SUPPORT_SUPERG */
diff --git a/sys/netgraph/ng_ether.c b/sys/netgraph/ng_ether.c
index 05d2e6e..6266f40 100644
--- a/sys/netgraph/ng_ether.c
+++ b/sys/netgraph/ng_ether.c
@@ -117,6 +117,8 @@ static ng_rcvdata_t	ng_ether_rcvdata;
 static ng_disconnect_t	ng_ether_disconnect;
 static int		ng_ether_mod_event(module_t mod, int event, void *data);
 
+static eventhandler_tag	ng_ether_ifnet_arrival_cookie;
+
 /* List of commands and how to convert arguments to/from ASCII */
 static const struct ng_cmdlist ng_ether_cmdlist[] = {
 	{
@@ -214,6 +216,24 @@ static struct ng_type ng_ether_typestruct = {
 NETGRAPH_INIT(ether, &ng_ether_typestruct);
 
 /******************************************************************
+		    UTILITY FUNCTIONS
+******************************************************************/
+static void
+ng_ether_sanitize_ifname(const char *ifname, char *name)
+{
+	int i;
+
+	for (i = 0; i < IFNAMSIZ; i++) {
+		if (ifname[i] == '.' || ifname[i] == ':')
+			name[i] = '_';
+		else
+			name[i] = ifname[i];
+		if (name[i] == '\0')
+			break;
+	}
+}
+
+/******************************************************************
 		    ETHERNET FUNCTION HOOKS
 ******************************************************************/
 
@@ -282,6 +302,7 @@ ng_ether_output(struct ifnet *ifp, struct mbuf **mp)
 static void
 ng_ether_attach(struct ifnet *ifp)
 {
+	char name[IFNAMSIZ];
 	priv_p priv;
 	node_p node;
 
@@ -319,10 +340,9 @@ ng_ether_attach(struct ifnet *ifp)
 	priv->hwassist = ifp->if_hwassist;
 
 	/* Try to give the node the same name as the interface */
-	if (ng_name_node(node, ifp->if_xname) != 0) {
-		log(LOG_WARNING, "%s: can't name node %s\n",
-		    __func__, ifp->if_xname);
-	}
+	ng_ether_sanitize_ifname(ifp->if_xname, name);
+	if (ng_name_node(node, name) != 0)
+		log(LOG_WARNING, "%s: can't name node %s\n", __func__, name);
 }
 
 /*
@@ -378,6 +398,32 @@ ng_ether_link_state(struct ifnet *ifp, int state)
 	}
 }
 
+/*
+ * Interface arrival notification handler.
+ * The notification is produced in two cases:
+ *  o a new interface arrives
+ *  o an existing interface got renamed
+ * Currently the first case is handled by ng_ether_attach via special
+ * hook ng_ether_attach_p.
+ */
+static void
+ng_ether_ifnet_arrival_event(void *arg __unused, struct ifnet *ifp)
+{
+	char name[IFNAMSIZ];
+	node_p node = IFP2NG(ifp);
+
+	/*
+	 * Just return if it's a new interface without an ng_ether companion.
+	 */
+	if (node == NULL)
+		return;
+
+	/* Try to give the node the same name as the new interface name */
+	ng_ether_sanitize_ifname(ifp->if_xname, name);
+	if (ng_name_node(node, name) != 0)
+		log(LOG_WARNING, "%s: can't re-name node %s\n", __func__, name);
+}
+
 /******************************************************************
 		    NETGRAPH NODE METHODS
 ******************************************************************/
@@ -771,6 +817,9 @@ ng_ether_mod_event(module_t mod, int event, void *data)
 		ng_ether_input_orphan_p = ng_ether_input_orphan;
 		ng_ether_link_state_p = ng_ether_link_state;
 
+		ng_ether_ifnet_arrival_cookie =
+		    EVENTHANDLER_REGISTER(ifnet_arrival_event,
+		    ng_ether_ifnet_arrival_event, NULL, EVENTHANDLER_PRI_ANY);
 		break;
 
 	case MOD_UNLOAD:
@@ -783,6 +832,9 @@ ng_ether_mod_event(module_t mod, int event, void *data)
 		 * is MOD_UNLOAD, so there's no need to detach any nodes.
 		 */
 
+		EVENTHANDLER_DEREGISTER(ifnet_arrival_event,
+		    ng_ether_ifnet_arrival_cookie);
+
 		/* Unregister function hooks */
 		ng_ether_attach_p = NULL;
 		ng_ether_detach_p = NULL;
diff --git a/sys/netinet/if_ether.h b/sys/netinet/if_ether.h
index e37a964..6b47912 100644
--- a/sys/netinet/if_ether.h
+++ b/sys/netinet/if_ether.h
@@ -89,6 +89,7 @@ struct	ether_arp {
 #define	arp_pln	ea_hdr.ar_pln
 #define	arp_op	ea_hdr.ar_op
 
+#ifndef BURN_BRIDGES	/* Can be used by third party software. */
 struct sockaddr_inarp {
 	u_char	sin_len;
 	u_char	sin_family;
@@ -99,6 +100,8 @@ struct sockaddr_inarp {
 	u_short	sin_other;
 #define SIN_PROXY 1
 };
+#endif /* !BURN_BRIDGES  */
+
 /*
  * IP and ethernet specific routing flags
  */
@@ -120,17 +123,6 @@ void	arprequest(struct ifnet *, struct in_addr *, struct in_addr *,
 void	arp_ifinit(struct ifnet *, struct ifaddr *);
 void	arp_ifinit2(struct ifnet *, struct ifaddr *, u_char *);
 void	arp_ifscrub(struct ifnet *, uint32_t);
-
-#include <sys/eventhandler.h>
-enum {
-	LLENTRY_RESOLVED,
-	LLENTRY_TIMEDOUT,
-	LLENTRY_DELETED,
-	LLENTRY_EXPIRED,
-};
-typedef void (*lle_event_fn)(void *, struct llentry *, int);
-EVENTHANDLER_DECLARE(lle_event, lle_event_fn);
-
 #endif
 
 #endif
diff --git a/sys/netinet/in.c b/sys/netinet/in.c
index 2b805c6..3ddb46c 100644
--- a/sys/netinet/in.c
+++ b/sys/netinet/in.c
@@ -1494,7 +1494,7 @@ in_lltable_dump(struct lltable *llt, struct sysctl_req *wr)
 	/* XXX stack use */
 	struct {
 		struct rt_msghdr	rtm;
-		struct sockaddr_inarp	sin;
+		struct sockaddr_in	sin;
 		struct sockaddr_dl	sdl;
 	} arpc;
 	int error, i;
@@ -1515,7 +1515,7 @@ in_lltable_dump(struct lltable *llt, struct sysctl_req *wr)
 			/*
 			 * produce a msg made of:
 			 *  struct rt_msghdr;
-			 *  struct sockaddr_inarp; (IPv4)
+			 *  struct sockaddr_in; (IPv4)
 			 *  struct sockaddr_dl;
 			 */
 			bzero(&arpc, sizeof(arpc));
@@ -1529,12 +1529,8 @@ in_lltable_dump(struct lltable *llt, struct sysctl_req *wr)
 			arpc.sin.sin_addr.s_addr = SIN(lle)->sin_addr.s_addr;
 
 			/* publish */
-			if (lle->la_flags & LLE_PUB) {
+			if (lle->la_flags & LLE_PUB)
 				arpc.rtm.rtm_flags |= RTF_ANNOUNCE;
-				/* proxy only */
-				if (lle->la_flags & LLE_PROXY)
-					arpc.sin.sin_other = SIN_PROXY;
-			}
 
 			sdl = &arpc.sdl;
 			sdl->sdl_family = AF_LINK;
diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
index 6706bc4..f5e2ef2 100644
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -334,8 +334,7 @@ in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
 
 	if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 		return (EINVAL);
-	anonport = inp->inp_lport == 0 && (nam == NULL ||
-	    ((struct sockaddr_in *)nam)->sin_port == 0);
+	anonport = nam == NULL || ((struct sockaddr_in *)nam)->sin_port == 0;
 	error = in_pcbbind_setup(inp, nam, &inp->inp_laddr.s_addr,
 	    &inp->inp_lport, cred);
 	if (error)
diff --git a/sys/netinet/libalias/alias.c b/sys/netinet/libalias/alias.c
index da28b22..3b2068b 100644
--- a/sys/netinet/libalias/alias.c
+++ b/sys/netinet/libalias/alias.c
@@ -1760,21 +1760,7 @@ m_megapullup(struct mbuf *m, int len) {
 	if (m->m_next == NULL && M_WRITABLE(m) && M_TRAILINGSPACE(m) >= RESERVE)
 		return (m);
 
-	if (len <= MCLBYTES - RESERVE) {
-		mcl = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
-	} else if (len < MJUM16BYTES) {
-		int size;
-		if (len <= MJUMPAGESIZE - RESERVE) {
-			size = MJUMPAGESIZE;
-		} else if (len <= MJUM9BYTES - RESERVE) {
-			size = MJUM9BYTES;
-		} else {
-			size = MJUM16BYTES;
-		};
-		mcl = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, size);
-	} else {
-		goto bad;
-	}
+	mcl = m_get2(M_NOWAIT, MT_DATA, M_PKTHDR, len + RESERVE);
 	if (mcl == NULL)
 		goto bad;
  
diff --git a/sys/netinet/libalias/libalias.3 b/sys/netinet/libalias/libalias.3
index 1837c29..4d02cc4 100644
--- a/sys/netinet/libalias/libalias.3
+++ b/sys/netinet/libalias/libalias.3
@@ -901,39 +901,6 @@ is provided for error checking purposes.
 This function can be used if an already-aliased packet needs to have its
 original IP header restored for further processing (e.g.\& logging).
 .Ed
-.Sh AUTHORS
-.An Charles Mott Aq cm@linktel.net ,
-versions 1.0 - 1.8, 2.0 - 2.4.
-.An Eivind Eklund Aq eivind@FreeBSD.org ,
-versions 1.8b, 1.9 and 2.5.
-Added IRC DCC support as well as contributing a number of architectural
-improvements; added the firewall bypass for FTP/IRC DCC.
-.An Erik Salander Aq erik@whistle.com
-added support for PPTP and RTSP.
-.An Junichi Satoh Aq junichi@junichi.org
-added support for RTSP/PNA.
-.An Ruslan Ermilov Aq ru@FreeBSD.org
-added support for PPTP and LSNAT as well as general hacking.
-.An Paolo Pisati Aq piso@FreeBSD.org
-made the library modular, moving support for all
-protocols (except for IP, TCP and UDP) to external modules.
-.Sh ACKNOWLEDGEMENTS
-Listed below, in approximate chronological order, are individuals who
-have provided valuable comments and/or debugging assistance.
-.Bd -ragged -offset indent
-.An -split
-.An Gary Roberts
-.An Tom Torrance
-.An Reto Burkhalter
-.An Martin Renters
-.An Brian Somers
-.An Paul Traina
-.An Ari Suutari
-.An Dave Remien
-.An J. Fortes
-.An Andrzej Bialecki
-.An Gordon Burditt
-.Ed
 .Sh CONCEPTUAL BACKGROUND
 This section is intended for those who are planning to modify the source
 code or want to create somewhat esoteric applications using the packet
@@ -1475,3 +1442,38 @@ with the
 facility and the
 .Dv LOG_INFO
 level.
+.Sh AUTHORS
+.An Charles Mott Aq cm@linktel.net ,
+versions 1.0 - 1.8, 2.0 - 2.4.
+.An Eivind Eklund Aq eivind@FreeBSD.org ,
+versions 1.8b, 1.9 and 2.5.
+Added IRC DCC support as well as contributing a number of architectural
+improvements; added the firewall bypass for FTP/IRC DCC.
+.An Erik Salander Aq erik@whistle.com
+added support for PPTP and RTSP.
+.An Junichi Satoh Aq junichi@junichi.org
+added support for RTSP/PNA.
+.An Ruslan Ermilov Aq ru@FreeBSD.org
+added support for PPTP and LSNAT as well as general hacking.
+.An Gleb Smirnoff Aq glebius@FreeBSD.org
+ported the library to kernel space.
+.An Paolo Pisati Aq piso@FreeBSD.org
+made the library modular, moving support for all
+protocols (except for IP, TCP and UDP) to external modules.
+.Sh ACKNOWLEDGEMENTS
+Listed below, in approximate chronological order, are individuals who
+have provided valuable comments and/or debugging assistance.
+.Bd -ragged -offset indent
+.An -split
+.An Gary Roberts
+.An Tom Torrance
+.An Reto Burkhalter
+.An Martin Renters
+.An Brian Somers
+.An Paul Traina
+.An Ari Suutari
+.An Dave Remien
+.An J. Fortes
+.An Andrzej Bialecki
+.An Gordon Burditt
+.Ed
diff --git a/sys/netinet/tcp.h b/sys/netinet/tcp.h
index c714360..fb2f810 100644
--- a/sys/netinet/tcp.h
+++ b/sys/netinet/tcp.h
@@ -149,20 +149,25 @@ struct tcphdr {
 #endif /* __BSD_VISIBLE */
 
 /*
- * User-settable options (used with setsockopt).
+ * User-settable options (used with setsockopt).  These are discrete
+ * values and are not masked together.  Some values appear to be
+ * bitmasks for historical reasons.
  */
-#define	TCP_NODELAY	0x01	/* don't delay send to coalesce packets */
+#define	TCP_NODELAY	1	/* don't delay send to coalesce packets */
 #if __BSD_VISIBLE
-#define	TCP_MAXSEG	0x02	/* set maximum segment size */
-#define TCP_NOPUSH	0x04	/* don't push last block of write */
-#define TCP_NOOPT	0x08	/* don't use TCP options */
-#define TCP_MD5SIG	0x10	/* use MD5 digests (RFC2385) */
-#define	TCP_INFO	0x20	/* retrieve tcp_info structure */
-#define	TCP_CONGESTION	0x40	/* get/set congestion control algorithm */
-#define	TCP_KEEPINIT	0x80	/* N, time to establish connection */
-#define	TCP_KEEPIDLE	0x100	/* L,N,X start keeplives after this period */
-#define	TCP_KEEPINTVL	0x200	/* L,N interval between keepalives */
-#define	TCP_KEEPCNT	0x400	/* L,N number of keepalives before close */
+#define	TCP_MAXSEG	2	/* set maximum segment size */
+#define TCP_NOPUSH	4	/* don't push last block of write */
+#define TCP_NOOPT	8	/* don't use TCP options */
+#define TCP_MD5SIG	16	/* use MD5 digests (RFC2385) */
+#define	TCP_INFO	32	/* retrieve tcp_info structure */
+#define	TCP_CONGESTION	64	/* get/set congestion control algorithm */
+#define	TCP_KEEPINIT	128	/* N, time to establish connection */
+#define	TCP_KEEPIDLE	256	/* L,N,X start keeplives after this period */
+#define	TCP_KEEPINTVL	512	/* L,N interval between keepalives */
+#define	TCP_KEEPCNT	1024	/* L,N number of keepalives before close */
+
+/* Start of reserved space for third-party user-settable options. */
+#define	TCP_VENDOR	SO_VENDOR
 
 #define	TCP_CA_NAME_MAX	16	/* max congestion control name length */
 
diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c
index 846887d2..e37de24 100644
--- a/sys/netinet/tcp_input.c
+++ b/sys/netinet/tcp_input.c
@@ -285,7 +285,7 @@ cc_ack_received(struct tcpcb *tp, struct tcphdr *th, uint16_t type)
 	INP_WLOCK_ASSERT(tp->t_inpcb);
 
 	tp->ccv->bytes_this_ack = BYTES_THIS_ACK(tp, th);
-	if (tp->snd_cwnd == min(tp->snd_cwnd, tp->snd_wnd))
+	if (tp->snd_cwnd <= tp->snd_wnd)
 		tp->ccv->flags |= CCF_CWND_LIMITED;
 	else
 		tp->ccv->flags &= ~CCF_CWND_LIMITED;
diff --git a/sys/netinet/tcp_reass.c b/sys/netinet/tcp_reass.c
index 289b58a..5fd29b8 100644
--- a/sys/netinet/tcp_reass.c
+++ b/sys/netinet/tcp_reass.c
@@ -106,8 +106,8 @@ tcp_reass_zone_change(void *tag)
 
 	/* Set the zone limit and read back the effective value. */
 	V_tcp_reass_maxseg = nmbclusters / 16;
-	uma_zone_set_max(V_tcp_reass_zone, V_tcp_reass_maxseg);
-	V_tcp_reass_maxseg = uma_zone_get_max(V_tcp_reass_zone);
+	V_tcp_reass_maxseg = uma_zone_set_max(V_tcp_reass_zone,
+	    V_tcp_reass_maxseg);
 }
 
 void
@@ -120,8 +120,8 @@ tcp_reass_init(void)
 	V_tcp_reass_zone = uma_zcreate("tcpreass", sizeof (struct tseg_qent),
 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
 	/* Set the zone limit and read back the effective value. */
-	uma_zone_set_max(V_tcp_reass_zone, V_tcp_reass_maxseg);
-	V_tcp_reass_maxseg = uma_zone_get_max(V_tcp_reass_zone);
+	V_tcp_reass_maxseg = uma_zone_set_max(V_tcp_reass_zone,
+	    V_tcp_reass_maxseg);
 	EVENTHANDLER_REGISTER(nmbclusters_change,
 	    tcp_reass_zone_change, NULL, EVENTHANDLER_PRI_ANY);
 }
diff --git a/sys/netinet/tcp_syncache.c b/sys/netinet/tcp_syncache.c
index 5baf3b8..441c269 100644
--- a/sys/netinet/tcp_syncache.c
+++ b/sys/netinet/tcp_syncache.c
@@ -268,8 +268,8 @@ syncache_init(void)
 	/* Create the syncache entry zone. */
 	V_tcp_syncache.zone = uma_zcreate("syncache", sizeof(struct syncache),
 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
-	uma_zone_set_max(V_tcp_syncache.zone, V_tcp_syncache.cache_limit);
-	V_tcp_syncache.cache_limit = uma_zone_get_max(V_tcp_syncache.zone);
+	V_tcp_syncache.cache_limit = uma_zone_set_max(V_tcp_syncache.zone,
+	    V_tcp_syncache.cache_limit);
 }
 
 #ifdef VIMAGE
@@ -1493,6 +1493,15 @@ syncache_respond(struct syncache *sc)
 		th->th_sum = in6_cksum_pseudo(ip6, tlen + optlen - hlen,
 		    IPPROTO_TCP, 0);
 		ip6->ip6_hlim = in6_selecthlim(NULL, NULL);
+#ifdef TCP_OFFLOAD
+		if (ADDED_BY_TOE(sc)) {
+			struct toedev *tod = sc->sc_tod;
+
+			error = tod->tod_syncache_respond(tod, sc->sc_todctx, m);
+
+			return (error);
+		}
+#endif
 		error = ip6_output(m, NULL, NULL, 0, NULL, NULL, NULL);
 	}
 #endif
diff --git a/sys/netinet/tcp_timer.c b/sys/netinet/tcp_timer.c
index b943eb6..48444c1 100644
--- a/sys/netinet/tcp_timer.c
+++ b/sys/netinet/tcp_timer.c
@@ -119,6 +119,11 @@ SYSCTL_INT(_net_inet_tcp, OID_AUTO, keepcnt, CTLFLAG_RW, &tcp_keepcnt, 0,
 	/* max idle probes */
 int	tcp_maxpersistidle;
 
+static int	tcp_rexmit_drop_options = 0;
+SYSCTL_INT(_net_inet_tcp, OID_AUTO, rexmit_drop_options, CTLFLAG_RW,
+    &tcp_rexmit_drop_options, 0,
+    "Drop TCP options from 3rd and later retransmitted SYN");
+
 static int	per_cpu_timers = 0;
 SYSCTL_INT(_net_inet_tcp, OID_AUTO, per_cpu_timers, CTLFLAG_RW,
     &per_cpu_timers , 0, "run tcp timers on all cpus");
@@ -595,7 +600,8 @@ tcp_timer_rexmt(void * xtp)
 	 * header compression code which trashes TCP segments containing
 	 * unknown-to-them TCP options.
 	 */
-	if ((tp->t_state == TCPS_SYN_SENT) && (tp->t_rxtshift == 3))
+	if (tcp_rexmit_drop_options && (tp->t_state == TCPS_SYN_SENT) &&
+	    (tp->t_rxtshift == 3))
 		tp->t_flags &= ~(TF_REQ_SCALE|TF_REQ_TSTMP|TF_SACK_PERMIT);
 	/*
 	 * If we backed off this far, our srtt estimate is probably bogus.
diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c
index 85597d9..ef0aad1 100644
--- a/sys/netinet/tcp_usrreq.c
+++ b/sys/netinet/tcp_usrreq.c
@@ -370,7 +370,8 @@ tcp_usr_listen(struct socket *so, int backlog, struct thread *td)
 		tp->t_state = TCPS_LISTEN;
 		solisten_proto(so, backlog);
 #ifdef TCP_OFFLOAD
-		tcp_offload_listen_start(tp);
+		if ((so->so_options & SO_NO_OFFLOAD) == 0)
+			tcp_offload_listen_start(tp);
 #endif
 	}
 	SOCK_UNLOCK(so);
@@ -414,7 +415,8 @@ tcp6_usr_listen(struct socket *so, int backlog, struct thread *td)
 		tp->t_state = TCPS_LISTEN;
 		solisten_proto(so, backlog);
 #ifdef TCP_OFFLOAD
-		tcp_offload_listen_start(tp);
+		if ((so->so_options & SO_NO_OFFLOAD) == 0)
+			tcp_offload_listen_start(tp);
 #endif
 	}
 	SOCK_UNLOCK(so);
@@ -468,6 +470,7 @@ tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 		goto out;
 #ifdef TCP_OFFLOAD
 	if (registered_toedevs > 0 &&
+	    (so->so_options & SO_NO_OFFLOAD) == 0 &&
 	    (error = tcp_offload_connect(so, nam)) == 0)
 		goto out;
 #endif
@@ -534,6 +537,7 @@ tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 			goto out;
 #ifdef TCP_OFFLOAD
 		if (registered_toedevs > 0 &&
+		    (so->so_options & SO_NO_OFFLOAD) == 0 &&
 		    (error = tcp_offload_connect(so, nam)) == 0)
 			goto out;
 #endif
@@ -550,6 +554,7 @@ tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 		goto out;
 #ifdef TCP_OFFLOAD
 	if (registered_toedevs > 0 &&
+	    (so->so_options & SO_NO_OFFLOAD) == 0 &&
 	    (error = tcp_offload_connect(so, nam)) == 0)
 		goto out;
 #endif
@@ -766,6 +771,7 @@ tcp_usr_rcvd(struct socket *so, int flags)
 #ifdef TCP_OFFLOAD
 	if (tp->t_flags & TF_TOE)
 		tcp_offload_rcvd(tp);
+	else
 #endif
 	tcp_output(tp);
 
diff --git a/sys/netinet/toecore.c b/sys/netinet/toecore.c
index 7e498dd..53d4778 100644
--- a/sys/netinet/toecore.c
+++ b/sys/netinet/toecore.c
@@ -52,6 +52,8 @@ __FBSDID("$FreeBSD$");
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
 #include <netinet/in_var.h>
+#include <netinet6/in6_var.h>
+#include <netinet6/in6_pcb.h>
 #include <netinet6/nd6.h>
 #define TCPSTATES
 #include <netinet/tcp.h>
@@ -355,11 +357,14 @@ toe_4tuple_check(struct in_conninfo *inc, struct tcphdr *th, struct ifnet *ifp)
 {
 	struct inpcb *inp;
 
-	if (inc->inc_flags & INC_ISIPV6)
-		return (ENOSYS);	/* XXX: implement */
-
-	inp = in_pcblookup(&V_tcbinfo, inc->inc_faddr, inc->inc_fport,
-	    inc->inc_laddr, inc->inc_lport, INPLOOKUP_WLOCKPCB, ifp);
+	if (inc->inc_flags & INC_ISIPV6) {
+		inp = in6_pcblookup(&V_tcbinfo, &inc->inc6_faddr,
+		    inc->inc_fport, &inc->inc6_laddr, inc->inc_lport,
+		    INPLOOKUP_WLOCKPCB, ifp);
+	} else {
+		inp = in_pcblookup(&V_tcbinfo, inc->inc_faddr, inc->inc_fport,
+		    inc->inc_laddr, inc->inc_lport, INPLOOKUP_WLOCKPCB, ifp);
+	}
 	if (inp != NULL) {
 		INP_WLOCK_ASSERT(inp);
 
@@ -440,6 +445,68 @@ toe_route_redirect_event(void *arg __unused, struct rtentry *rt0,
 	return;
 }
 
+#ifdef INET6
+/*
+ * XXX: no checks to verify that sa is really a neighbor because we assume it is
+ * the result of a route lookup and is on-link on the given ifp.
+ */
+static int
+toe_nd6_resolve(struct ifnet *ifp, struct sockaddr *sa, uint8_t *lladdr)
+{
+	struct llentry *lle;
+	struct sockaddr_in6 *sin6 = (void *)sa;
+	int rc, flags = 0;
+
+restart:
+	IF_AFDATA_RLOCK(ifp);
+	lle = lla_lookup(LLTABLE6(ifp), flags, sa);
+	IF_AFDATA_RUNLOCK(ifp);
+	if (lle == NULL) {
+		IF_AFDATA_LOCK(ifp);
+		lle = nd6_lookup(&sin6->sin6_addr, ND6_CREATE | ND6_EXCLUSIVE,
+		    ifp);
+		IF_AFDATA_UNLOCK(ifp);
+		if (lle == NULL)
+			return (ENOMEM); /* Couldn't create entry in cache. */
+		lle->ln_state = ND6_LLINFO_INCOMPLETE;
+		nd6_llinfo_settimer_locked(lle,
+		    (long)ND_IFINFO(ifp)->retrans * hz / 1000);
+		LLE_WUNLOCK(lle);
+
+		nd6_ns_output(ifp, NULL, &sin6->sin6_addr, NULL, 0);
+
+		return (EWOULDBLOCK);
+	}
+
+	if (lle->ln_state == ND6_LLINFO_STALE) {
+		if ((flags & LLE_EXCLUSIVE) == 0) {
+			LLE_RUNLOCK(lle);
+			flags |= LLE_EXCLUSIVE;
+			goto restart;
+		}
+
+		LLE_WLOCK_ASSERT(lle);
+
+		lle->la_asked = 0;
+		lle->ln_state = ND6_LLINFO_DELAY;
+		nd6_llinfo_settimer_locked(lle, (long)V_nd6_delay * hz);
+	}
+
+	if (lle->la_flags & LLE_VALID) {
+		memcpy(lladdr, &lle->ll_addr, ifp->if_addrlen);
+		rc = 0;
+	} else
+		rc = EWOULDBLOCK;
+
+	if (flags & LLE_EXCLUSIVE)
+		LLE_WUNLOCK(lle);
+	else
+		LLE_RUNLOCK(lle);
+
+	return (rc);
+}
+#endif
+
 /*
  * Returns 0 or EWOULDBLOCK on success (any other value is an error).  0 means
  * lladdr and vtag are valid on return, EWOULDBLOCK means the TOE driver's
@@ -449,7 +516,9 @@ int
 toe_l2_resolve(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa,
     uint8_t *lladdr, uint16_t *vtag)
 {
+#ifdef INET
 	struct llentry *lle;
+#endif
 	int rc;
 
 	switch (sa->sa_family) {
@@ -460,7 +529,7 @@ toe_l2_resolve(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa,
 #endif
 #ifdef INET6
 	case AF_INET6:
-		rc = nd6_storelladdr(ifp, NULL, sa, lladdr, &lle);
+		rc = toe_nd6_resolve(ifp, sa, lladdr);
 		break;
 #endif
 	default:
diff --git a/sys/netinet/udp.h b/sys/netinet/udp.h
index 2dbcaca..c2d638d 100644
--- a/sys/netinet/udp.h
+++ b/sys/netinet/udp.h
@@ -48,8 +48,10 @@ struct udphdr {
 /* 
  * User-settable options (used with setsockopt).
  */
-#define	UDP_ENCAP			0x01
+#define	UDP_ENCAP			1
 
+/* Start of reserved space for third-party user-settable options. */
+#define	UDP_VENDOR			SO_VENDOR
 
 /*
  * UDP Encapsulation of IPsec Packets options.
diff --git a/sys/netinet6/in6_src.c b/sys/netinet6/in6_src.c
index fb7e6f0..8d04c46 100644
--- a/sys/netinet6/in6_src.c
+++ b/sys/netinet6/in6_src.c
@@ -383,10 +383,12 @@ in6_selectsrc(struct sockaddr_in6 *dstsock, struct ip6_pktopts *opts,
 		 */
 
 		/* Rule 5: Prefer outgoing interface */
-		if (ia_best->ia_ifp == ifp && ia->ia_ifp != ifp)
-			NEXT(5);
-		if (ia_best->ia_ifp != ifp && ia->ia_ifp == ifp)
-			REPLACE(5);
+		if (!(ND_IFINFO(ifp)->flags & ND6_IFF_NO_PREFER_IFACE)) {
+			if (ia_best->ia_ifp == ifp && ia->ia_ifp != ifp)
+				NEXT(5);
+			if (ia_best->ia_ifp != ifp && ia->ia_ifp == ifp)
+				REPLACE(5);
+		}
 
 		/*
 		 * Rule 6: Prefer matching label
diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c
index 5107f1d..fd420d8 100644
--- a/sys/netinet6/nd6.c
+++ b/sys/netinet6/nd6.c
@@ -509,6 +509,7 @@ nd6_llinfo_timer(void *arg)
 				ln->la_hold = m0;
 				clear_llinfo_pqueue(ln);
 			}
+			EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_TIMEDOUT);
 			(void)nd6_free(ln, 0);
 			ln = NULL;
 			if (m != NULL)
@@ -526,6 +527,7 @@ nd6_llinfo_timer(void *arg)
 	case ND6_LLINFO_STALE:
 		/* Garbage Collection(RFC 2461 5.3) */
 		if (!ND6_LLINFO_PERMANENT(ln)) {
+			EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_EXPIRED);
 			(void)nd6_free(ln, 1);
 			ln = NULL;
 		}
@@ -553,6 +555,7 @@ nd6_llinfo_timer(void *arg)
 			nd6_ns_output(ifp, dst, dst, ln, 0);
 			LLE_WLOCK(ln);
 		} else {
+			EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_EXPIRED);
 			(void)nd6_free(ln, 0);
 			ln = NULL;
 		}
@@ -1601,6 +1604,7 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr,
 		 */
 		bcopy(lladdr, &ln->ll_addr, ifp->if_addrlen);
 		ln->la_flags |= LLE_VALID;
+		EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_RESOLVED);
 	}
 
 	if (!is_newentry) {
@@ -2166,7 +2170,7 @@ nd6_storelladdr(struct ifnet *ifp, struct mbuf *m,
 
 	*lle = NULL;
 	IF_AFDATA_UNLOCK_ASSERT(ifp);
-	if (m->m_flags & M_MCAST) {
+	if (m != NULL && m->m_flags & M_MCAST) {
 		int i;
 
 		switch (ifp->if_type) {
diff --git a/sys/netinet6/nd6.h b/sys/netinet6/nd6.h
index 1655a6c..94202e1 100644
--- a/sys/netinet6/nd6.h
+++ b/sys/netinet6/nd6.h
@@ -86,6 +86,7 @@ struct nd_ifinfo {
 #define ND6_IFF_DONT_SET_IFROUTE	0x10
 #define ND6_IFF_AUTO_LINKLOCAL	0x20
 #define	ND6_IFF_NO_RADR		0x40
+#define ND6_IFF_NO_PREFER_IFACE	0x80 /* XXX: not related to ND. */
 
 #define	ND6_CREATE		LLE_CREATE
 #define	ND6_EXCLUSIVE		LLE_EXCLUSIVE
diff --git a/sys/netinet6/nd6_nbr.c b/sys/netinet6/nd6_nbr.c
index 31d0ccb..7fe75bf 100644
--- a/sys/netinet6/nd6_nbr.c
+++ b/sys/netinet6/nd6_nbr.c
@@ -762,6 +762,7 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len)
 		 */
 		bcopy(lladdr, &ln->ll_addr, ifp->if_addrlen);
 		ln->la_flags |= LLE_VALID;
+		EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_RESOLVED);
 		if (is_solicited) {
 			ln->ln_state = ND6_LLINFO_REACHABLE;
 			ln->ln_byhint = 0;
@@ -837,6 +838,8 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len)
 			if (lladdr != NULL) {
 				bcopy(lladdr, &ln->ll_addr, ifp->if_addrlen);
 				ln->la_flags |= LLE_VALID;
+				EVENTHANDLER_INVOKE(lle_event, ln,
+				    LLENTRY_RESOLVED);
 			}
 
 			/*
diff --git a/sys/netinet6/scope6.c b/sys/netinet6/scope6.c
index 74d034e..b6479a1 100644
--- a/sys/netinet6/scope6.c
+++ b/sys/netinet6/scope6.c
@@ -336,7 +336,6 @@ scope6_addr2default(struct in6_addr *addr)
 int
 sa6_embedscope(struct sockaddr_in6 *sin6, int defaultok)
 {
-	struct ifnet *ifp;
 	u_int32_t zoneid;
 
 	if ((zoneid = sin6->sin6_scope_id) == 0 && defaultok)
@@ -351,15 +350,11 @@ sa6_embedscope(struct sockaddr_in6 *sin6, int defaultok)
 		 * zone IDs assuming a one-to-one mapping between interfaces
 		 * and links.
 		 */
-		if (V_if_index < zoneid)
-			return (ENXIO);
-		ifp = ifnet_byindex(zoneid);
-		if (ifp == NULL) /* XXX: this can happen for some OS */
+		if (V_if_index < zoneid || ifnet_byindex(zoneid) == NULL)
 			return (ENXIO);
 
 		/* XXX assignment to 16bit from 32bit variable */
 		sin6->sin6_addr.s6_addr16[1] = htons(zoneid & 0xffff);
-
 		sin6->sin6_scope_id = 0;
 	}
 
@@ -425,59 +420,30 @@ in6_setscope(struct in6_addr *in6, struct ifnet *ifp, u_int32_t *ret_id)
 	 * interface.
 	 */
 	if (IN6_IS_ADDR_LOOPBACK(in6)) {
-		if (!(ifp->if_flags & IFF_LOOPBACK)) {
+		if (!(ifp->if_flags & IFF_LOOPBACK))
 			return (EINVAL);
-		} else {
-			if (ret_id != NULL)
-				*ret_id = 0; /* there's no ambiguity */
-			return (0);
+	} else {
+		scope = in6_addrscope(in6);
+		if (scope == IPV6_ADDR_SCOPE_INTFACELOCAL ||
+		    scope == IPV6_ADDR_SCOPE_LINKLOCAL) {
+			/*
+			 * Currently we use interface indeces as the
+			 * zone IDs for interface-local and link-local
+			 * scopes.
+			 */
+			zoneid = ifp->if_index;
+			in6->s6_addr16[1] = htons(zoneid & 0xffff); /* XXX */
+		} else if (scope != IPV6_ADDR_SCOPE_GLOBAL) {
+			IF_AFDATA_RLOCK(ifp);
+			sid = SID(ifp);
+			zoneid = sid->s6id_list[scope];
+			IF_AFDATA_RUNLOCK(ifp);
 		}
 	}
 
-	if (ret_id == NULL && !IN6_IS_SCOPE_EMBED(in6))
-		return (0);
-
-	IF_AFDATA_RLOCK(ifp);
-
-	sid = SID(ifp);
-
-#ifdef DIAGNOSTIC
-	if (sid == NULL) { /* should not happen */
-		panic("in6_setscope: scope array is NULL");
-		/* NOTREACHED */
-	}
-#endif
-
-	scope = in6_addrscope(in6);
-	switch (scope) {
-	case IPV6_ADDR_SCOPE_INTFACELOCAL: /* should be interface index */
-		zoneid = sid->s6id_list[IPV6_ADDR_SCOPE_INTFACELOCAL];
-		break;
-
-	case IPV6_ADDR_SCOPE_LINKLOCAL:
-		zoneid = sid->s6id_list[IPV6_ADDR_SCOPE_LINKLOCAL];
-		break;
-
-	case IPV6_ADDR_SCOPE_SITELOCAL:
-		zoneid = sid->s6id_list[IPV6_ADDR_SCOPE_SITELOCAL];
-		break;
-
-	case IPV6_ADDR_SCOPE_ORGLOCAL:
-		zoneid = sid->s6id_list[IPV6_ADDR_SCOPE_ORGLOCAL];
-		break;
-
-	default:
-		zoneid = 0;	/* XXX: treat as global. */
-		break;
-	}
-	IF_AFDATA_RUNLOCK(ifp);
-
 	if (ret_id != NULL)
 		*ret_id = zoneid;
 
-	if (IN6_IS_SCOPE_EMBED(in6))
-		in6->s6_addr16[1] = htons(zoneid & 0xffff); /* XXX */
-
 	return (0);
 }
 
diff --git a/sys/nfs/nfs_common.c b/sys/nfs/nfs_common.c
index 6f03948..83e7125 100644
--- a/sys/nfs/nfs_common.c
+++ b/sys/nfs/nfs_common.c
@@ -90,15 +90,6 @@ static int nfs_realign_count;
 SYSCTL_INT(_vfs_nfs_common, OID_AUTO, realign_count, CTLFLAG_RD,
     &nfs_realign_count, 0, "Number of mbuf realignments done");
 
-u_quad_t
-nfs_curusec(void)
-{
-	struct timeval tv;
-
-	getmicrotime(&tv);
-	return ((u_quad_t)tv.tv_sec * 1000000 + (u_quad_t)tv.tv_usec);
-}
-
 /*
  * copies mbuf chain to the uio scatter/gather list
  */
diff --git a/sys/nfs/nfs_common.h b/sys/nfs/nfs_common.h
index 1c0b315..c3420b3 100644
--- a/sys/nfs/nfs_common.h
+++ b/sys/nfs/nfs_common.h
@@ -46,7 +46,6 @@ extern nfstype nfsv3_type[];
 #define	vtonfsv3_type(a)	txdr_unsigned(nfsv3_type[((int32_t)(a))])
 
 int	nfs_adv(struct mbuf **, caddr_t *, int, int);
-u_quad_t nfs_curusec(void);
 void	*nfsm_disct(struct mbuf **, caddr_t *, int, int, int);
 int	nfs_realign(struct mbuf **, int);
 
diff --git a/sys/nfsclient/nfs_krpc.c b/sys/nfsclient/nfs_krpc.c
index 0e4a83e..dd441ae 100644
--- a/sys/nfsclient/nfs_krpc.c
+++ b/sys/nfsclient/nfs_krpc.c
@@ -394,18 +394,17 @@ nfs_feedback(int type, int proc, void *arg)
 {
 	struct nfs_feedback_arg *nf = (struct nfs_feedback_arg *) arg;
 	struct nfsmount *nmp = nf->nf_mount;
-	struct timeval now;
-
-	getmicrouptime(&now);
+	time_t now;
 
 	switch (type) {
 	case FEEDBACK_REXMIT2:
 	case FEEDBACK_RECONNECT:
-		if (nf->nf_lastmsg + nmp->nm_tprintf_delay < now.tv_sec) {
+		now = time_uptime;
+		if (nf->nf_lastmsg + nmp->nm_tprintf_delay < now) {
 			nfs_down(nmp, nf->nf_td,
 			    "not responding", 0, NFSSTA_TIMEO);
 			nf->nf_tprintfmsg = TRUE;
-			nf->nf_lastmsg = now.tv_sec;
+			nf->nf_lastmsg = now;
 		}
 		break;
 
@@ -438,7 +437,6 @@ nfs_request(struct vnode *vp, struct mbuf *mreq, int procnum,
 	time_t waituntil;
 	caddr_t dpos;
 	int error = 0, timeo;
-	struct timeval now;
 	AUTH *auth = NULL;
 	enum nfs_rto_timer_t timer;
 	struct nfs_feedback_arg nf;
@@ -455,8 +453,7 @@ nfs_request(struct vnode *vp, struct mbuf *mreq, int procnum,
 	bzero(&nf, sizeof(struct nfs_feedback_arg));
 	nf.nf_mount = nmp;
 	nf.nf_td = td;
-	getmicrouptime(&now);
-	nf.nf_lastmsg = now.tv_sec -
+	nf.nf_lastmsg = time_uptime -
 	    ((nmp->nm_tprintf_delay) - (nmp->nm_tprintf_initial_delay));
 
 	/*
@@ -549,14 +546,21 @@ tryagain:
 	 */
 	if (stat == RPC_SUCCESS)
 		error = 0;
-	else if (stat == RPC_TIMEDOUT)
+	else if (stat == RPC_TIMEDOUT) {
+		nfsstats.rpctimeouts++;
 		error = ETIMEDOUT;
-	else if (stat == RPC_VERSMISMATCH)
+	} else if (stat == RPC_VERSMISMATCH) {
+		nfsstats.rpcinvalid++;
 		error = EOPNOTSUPP;
-	else if (stat == RPC_PROGVERSMISMATCH)
+	} else if (stat == RPC_PROGVERSMISMATCH) {
+		nfsstats.rpcinvalid++;
 		error = EPROTONOSUPPORT;
-	else
+	} else if (stat == RPC_INTR) {
+		error = EINTR;
+	} else {
+		nfsstats.rpcinvalid++;
 		error = EACCES;
+	}
 	if (error)
 		goto nfsmout;
 
@@ -572,6 +576,7 @@ tryagain:
 	if (error == ENOMEM) {
 		m_freem(mrep);
 		AUTH_DESTROY(auth);
+		nfsstats.rpcinvalid++;
 		return (error);
 	}
 
diff --git a/sys/nfsclient/nfs_subs.c b/sys/nfsclient/nfs_subs.c
index fd83794..3723223 100644
--- a/sys/nfsclient/nfs_subs.c
+++ b/sys/nfsclient/nfs_subs.c
@@ -1110,7 +1110,7 @@ nfsm_v3attrbuild_xx(struct vattr *va, int full, struct mbuf **mb,
 		*tl = nfs_false;
 	}
 	if (va->va_atime.tv_sec != VNOVAL) {
-		if (va->va_atime.tv_sec != time_second) {
+		if ((va->va_vaflags & VA_UTIMES_NULL) == 0) {
 			tl = nfsm_build_xx(3 * NFSX_UNSIGNED, mb, bpos);
 			*tl++ = txdr_unsigned(NFSV3SATTRTIME_TOCLIENT);
 			txdr_nfsv3time(&va->va_atime, tl);
@@ -1123,7 +1123,7 @@ nfsm_v3attrbuild_xx(struct vattr *va, int full, struct mbuf **mb,
 		*tl = txdr_unsigned(NFSV3SATTRTIME_DONTCHANGE);
 	}
 	if (va->va_mtime.tv_sec != VNOVAL) {
-		if (va->va_mtime.tv_sec != time_second) {
+		if ((va->va_vaflags & VA_UTIMES_NULL) == 0) {
 			tl = nfsm_build_xx(3 * NFSX_UNSIGNED, mb, bpos);
 			*tl++ = txdr_unsigned(NFSV3SATTRTIME_TOCLIENT);
 			txdr_nfsv3time(&va->va_mtime, tl);
diff --git a/sys/nfsclient/nfs_vnops.c b/sys/nfsclient/nfs_vnops.c
index ca9edad..5026daf 100644
--- a/sys/nfsclient/nfs_vnops.c
+++ b/sys/nfsclient/nfs_vnops.c
@@ -3458,7 +3458,7 @@ nfsfifo_read(struct vop_read_args *ap)
 	 */
 	mtx_lock(&np->n_mtx);
 	np->n_flag |= NACC;
-	getnanotime(&np->n_atim);
+	vfs_timestamp(&np->n_atim);
 	mtx_unlock(&np->n_mtx);
 	error = fifo_specops.vop_read(ap);
 	return error;	
@@ -3477,7 +3477,7 @@ nfsfifo_write(struct vop_write_args *ap)
 	 */
 	mtx_lock(&np->n_mtx);
 	np->n_flag |= NUPD;
-	getnanotime(&np->n_mtim);
+	vfs_timestamp(&np->n_mtim);
 	mtx_unlock(&np->n_mtx);
 	return(fifo_specops.vop_write(ap));
 }
@@ -3497,7 +3497,7 @@ nfsfifo_close(struct vop_close_args *ap)
 
 	mtx_lock(&np->n_mtx);
 	if (np->n_flag & (NACC | NUPD)) {
-		getnanotime(&ts);
+		vfs_timestamp(&ts);
 		if (np->n_flag & NACC)
 			np->n_atim = ts;
 		if (np->n_flag & NUPD)
diff --git a/sys/nfsserver/nfs_srvsubs.c b/sys/nfsserver/nfs_srvsubs.c
index eedac6c..04371da 100644
--- a/sys/nfsserver/nfs_srvsubs.c
+++ b/sys/nfsserver/nfs_srvsubs.c
@@ -1393,7 +1393,7 @@ nfsm_srvsattr_xx(struct vattr *a, struct mbuf **md, caddr_t *dpos)
 		toclient = 1;
 		break;
 	case NFSV3SATTRTIME_TOSERVER:
-		getnanotime(&(a)->va_atime);
+		vfs_timestamp(&a->va_atime);
 		a->va_vaflags |= VA_UTIMES_NULL;
 		break;
 	}
@@ -1409,7 +1409,7 @@ nfsm_srvsattr_xx(struct vattr *a, struct mbuf **md, caddr_t *dpos)
 		a->va_vaflags &= ~VA_UTIMES_NULL;
 		break;
 	case NFSV3SATTRTIME_TOSERVER:
-		getnanotime(&(a)->va_mtime);
+		vfs_timestamp(&a->va_mtime);
 		if (toclient == 0)
 			a->va_vaflags |= VA_UTIMES_NULL;
 		break;
diff --git a/sys/pc98/cbus/pckbd.c b/sys/pc98/cbus/pckbd.c
index 5efb983..e424294 100644
--- a/sys/pc98/cbus/pckbd.c
+++ b/sys/pc98/cbus/pckbd.c
@@ -77,9 +77,9 @@ DRIVER_MODULE(pckbd, isa, pckbd_driver, pckbd_devclass, 0, 0);
 
 static bus_addr_t pckbd_iat[] = {0, 2};
 
-static int		pckbd_probe_unit(int unit, int port, int irq,
+static int		pckbd_probe_unit(device_t dev, int port, int irq,
 					 int flags);
-static int		pckbd_attach_unit(int unit, keyboard_t **kbd,
+static int		pckbd_attach_unit(device_t dev, keyboard_t **kbd,
 					  int port, int irq, int flags);
 static timeout_t	pckbd_timeout;
 
@@ -103,7 +103,7 @@ pckbdprobe(device_t dev)
 		return ENXIO;
 	isa_load_resourcev(res, pckbd_iat, 2);
 
-	error = pckbd_probe_unit(device_get_unit(dev),
+	error = pckbd_probe_unit(dev,
 				 isa_get_port(dev),
 				 (1 << isa_get_irq(dev)),
 				 device_get_flags(dev));
@@ -128,7 +128,7 @@ pckbdattach(device_t dev)
 		return ENXIO;
 	isa_load_resourcev(res, pckbd_iat, 2);
 
-	error = pckbd_attach_unit(device_get_unit(dev), &kbd,
+	error = pckbd_attach_unit(dev, &kbd,
 				  isa_get_port(dev),
 				  (1 << isa_get_irq(dev)),
 				  device_get_flags(dev));
@@ -164,7 +164,7 @@ pckbd_isa_intr(void *arg)
 }
 
 static int
-pckbd_probe_unit(int unit, int port, int irq, int flags)
+pckbd_probe_unit(device_t dev, int port, int irq, int flags)
 {
 	keyboard_switch_t *sw;
 	int args[2];
@@ -176,24 +176,26 @@ pckbd_probe_unit(int unit, int port, int irq, int flags)
 
 	args[0] = port;
 	args[1] = irq;
-	error = (*sw->probe)(unit, args, flags);
+	error = (*sw->probe)(device_get_unit(dev), args, flags);
 	if (error)
 		return error;
 	return 0;
 }
 
 static int
-pckbd_attach_unit(int unit, keyboard_t **kbd, int port, int irq, int flags)
+pckbd_attach_unit(device_t dev, keyboard_t **kbd, int port, int irq, int flags)
 {
 	keyboard_switch_t *sw;
 	int args[2];
 	int error;
+	int unit;
 
 	sw = kbd_get_switch(DRIVER_NAME);
 	if (sw == NULL)
 		return ENXIO;
 
 	/* reset, initialize and enable the device */
+	unit = device_get_unit(dev);
 	args[0] = port;
 	args[1] = irq;
 	*kbd = NULL;
diff --git a/sys/pc98/conf/GENERIC b/sys/pc98/conf/GENERIC
index 2b048a9..eda1d14 100644
--- a/sys/pc98/conf/GENERIC
+++ b/sys/pc98/conf/GENERIC
@@ -151,7 +151,6 @@ device		mse
 device		ppc
 device		ppbus		# Parallel port bus (required)
 device		lpt		# Printer
-device		plip		# TCP/IP over parallel
 device		ppi		# Parallel port interface device
 #device		vpo		# Requires scbus and da
 # OLD Parallel port
diff --git a/sys/pci/if_rl.c b/sys/pci/if_rl.c
index 98e3352..4f91d1f 100644
--- a/sys/pci/if_rl.c
+++ b/sys/pci/if_rl.c
@@ -148,6 +148,8 @@ static const struct rl_type rl_devs[] = {
 		"Delta Electronics 8139 10/100BaseTX" },
 	{ ADDTRON_VENDORID, ADDTRON_DEVICEID_8139, RL_8139,
 		"Addtron Technology 8139 10/100BaseTX" },
+	{ DLINK_VENDORID, DLINK_DEVICEID_520TX_REVC1, RL_8139,
+		"D-Link DFE-520TX (rev. C1) 10/100BaseTX" },
 	{ DLINK_VENDORID, DLINK_DEVICEID_530TXPLUS, RL_8139,
 		"D-Link DFE-530TX+ 10/100BaseTX" },
 	{ DLINK_VENDORID, DLINK_DEVICEID_690TXD, RL_8139,
diff --git a/sys/pci/if_rlreg.h b/sys/pci/if_rlreg.h
index 7822ce3..be89c4f5 100644
--- a/sys/pci/if_rlreg.h
+++ b/sys/pci/if_rlreg.h
@@ -1048,6 +1048,11 @@ struct rl_softc {
 #define	DLINK_DEVICEID_530TXPLUS		0x1300
 
 /*
+ * D-Link DFE-520TX rev. C1 device ID
+ */
+#define	DLINK_DEVICEID_520TX_REVC1		0x4200
+
+/*
  * D-Link DFE-5280T device ID
  */
 #define	DLINK_DEVICEID_528T			0x4300
diff --git a/sys/pci/ncr.c b/sys/pci/ncr.c
index 793ae80..ead7d43 100644
--- a/sys/pci/ncr.c
+++ b/sys/pci/ncr.c
@@ -1386,7 +1386,7 @@ static char *ncr_name (ncb_p np)
  * Kernel variables referenced in the scripts.
  * THESE MUST ALL BE ALIGNED TO A 4-BYTE BOUNDARY.
  */
-static void *script_kvars[] =
+static volatile void *script_kvars[] =
 	{ &time_second, &ticks, &ncr_cache };
 
 static	struct script script0 = {
diff --git a/sys/security/audit/audit.c b/sys/security/audit/audit.c
index 2063c9d..cb3406d 100644
--- a/sys/security/audit/audit.c
+++ b/sys/security/audit/audit.c
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/filedesc.h>
 #include <sys/fcntl.h>
 #include <sys/ipc.h>
+#include <sys/jail.h>
 #include <sys/kernel.h>
 #include <sys/kthread.h>
 #include <sys/malloc.h>
@@ -211,6 +212,7 @@ audit_record_ctor(void *mem, int size, void *arg, int flags)
 	struct kaudit_record *ar;
 	struct thread *td;
 	struct ucred *cred;
+	struct prison *pr;
 
 	KASSERT(sizeof(*ar) == size, ("audit_record_ctor: wrong size"));
 
@@ -233,6 +235,17 @@ audit_record_ctor(void *mem, int size, void *arg, int flags)
 	ar->k_ar.ar_subj_pid = td->td_proc->p_pid;
 	ar->k_ar.ar_subj_amask = cred->cr_audit.ai_mask;
 	ar->k_ar.ar_subj_term_addr = cred->cr_audit.ai_termid;
+	/*
+	 * If this process is jailed, make sure we capture the name of the
+	 * jail so we can use it to generate a zonename token when we covert
+	 * this record to BSM.
+	 */
+	if (jailed(cred)) {
+		pr = cred->cr_prison;
+		(void) strlcpy(ar->k_ar.ar_jailname, pr->pr_name,
+		    sizeof(ar->k_ar.ar_jailname));
+	} else
+		ar->k_ar.ar_jailname[0] = '\0';
 	return (0);
 }
 
diff --git a/sys/security/audit/audit_bsm.c b/sys/security/audit/audit_bsm.c
index 6e49b51..b02d677 100644
--- a/sys/security/audit/audit_bsm.c
+++ b/sys/security/audit/audit_bsm.c
@@ -462,7 +462,7 @@ audit_sys_auditon(struct audit_record *ar, struct au_record *rec)
 int
 kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 {
-	struct au_token *tok, *subj_tok;
+	struct au_token *tok, *subj_tok, *jail_tok;
 	struct au_record *rec;
 	au_tid_t tid;
 	struct audit_record *ar;
@@ -475,8 +475,13 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 	rec = kau_open();
 
 	/*
-	 * Create the subject token.
+	 * Create the subject token.  If this credential was jailed be sure to
+	 * generate a zonename token.
 	 */
+	if (ar->ar_jailname[0] != '\0')
+		jail_tok = au_to_zonename(ar->ar_jailname);
+	else
+		jail_tok = NULL;
 	switch (ar->ar_subj_term_addr.at_type) {
 	case AU_IPv4:
 		tid.port = ar->ar_subj_term_addr.at_port;
@@ -1623,11 +1628,15 @@ kaudit_to_bsm(struct kaudit_record *kar, struct au_record **pau)
 		/*
 		 * Write the subject token so it is properly freed here.
 		 */
+		if (jail_tok != NULL)
+			kau_write(rec, jail_tok);
 		kau_write(rec, subj_tok);
 		kau_free(rec);
 		return (BSM_NOAUDIT);
 	}
 
+	if (jail_tok != NULL)
+		kau_write(rec, jail_tok);
 	kau_write(rec, subj_tok);
 	tok = au_to_return32(au_errno_to_bsm(ar->ar_errno), ar->ar_retval);
 	kau_write(rec, tok);  /* Every record gets a return token */
diff --git a/sys/security/audit/audit_private.h b/sys/security/audit/audit_private.h
index ad931c0..a5716d0 100644
--- a/sys/security/audit/audit_private.h
+++ b/sys/security/audit/audit_private.h
@@ -230,6 +230,7 @@ struct audit_record {
 	int			ar_arg_exitretval;
 	struct sockaddr_storage ar_arg_sockaddr;
 	cap_rights_t		ar_arg_rights;
+	char			ar_jailname[MAXHOSTNAMELEN];
 };
 
 /*
diff --git a/sys/sparc64/conf/GENERIC b/sys/sparc64/conf/GENERIC
index f9d3b93..79124ab 100644
--- a/sys/sparc64/conf/GENERIC
+++ b/sys/sparc64/conf/GENERIC
@@ -161,7 +161,6 @@ device		uart		# Multi-uart driver
 #device		ppc
 #device		ppbus		# Parallel port bus (required)
 #device		lpt		# Printer
-#device		plip		# TCP/IP over parallel
 #device		ppi		# Parallel port interface device
 #device		vpo		# Requires scbus and da
 
diff --git a/sys/sparc64/include/smp.h b/sys/sparc64/include/smp.h
index db17309..2e1110b 100644
--- a/sys/sparc64/include/smp.h
+++ b/sys/sparc64/include/smp.h
@@ -42,6 +42,7 @@
 #include <sys/cpuset.h>
 #include <sys/proc.h>
 #include <sys/sched.h>
+#include <sys/smp.h>
 
 #include <machine/intr_machdep.h>
 #include <machine/pcb.h>
@@ -202,6 +203,7 @@ ipi_rd(u_int cpu, void *func, u_long *val)
 		return (NULL);
 	sched_pin();
 	ira = &ipi_rd_args;
+	mtx_lock_spin(&smp_ipi_mtx);
 	CPU_SETOF(cpu, &ira->ira_mask);
 	ira->ira_val = val;
 	cpu_ipi_single(cpu, 0, (u_long)func, (u_long)ira);
@@ -298,18 +300,6 @@ ipi_wait(void *cookie)
 	}
 }
 
-static __inline void
-ipi_wait_unlocked(void *cookie)
-{
-	volatile cpuset_t *mask;
-
-	if ((mask = cookie) != NULL) {
-		while (!CPU_EMPTY(mask))
-			;
-		sched_unpin();
-	}
-}
-
 #endif /* _MACHINE_PMAP_H_ && _SYS_MUTEX_H_ */
 
 #endif /* !LOCORE */
@@ -368,12 +358,6 @@ ipi_wait(void *cookie __unused)
 }
 
 static __inline void
-ipi_wait_unlocked(void *cookie __unused)
-{
-
-}
-
-static __inline void
 tl_ipi_cheetah_dcache_page_inval(void)
 {
 
diff --git a/sys/sparc64/sparc64/tick.c b/sys/sparc64/sparc64/tick.c
index 69062ec..cfd5776 100644
--- a/sys/sparc64/sparc64/tick.c
+++ b/sys/sparc64/sparc64/tick.c
@@ -334,7 +334,7 @@ stick_get_timecount_mp(struct timecounter *tc)
 	if (curcpu == 0)
 		stick = rdstick();
 	else
-		ipi_wait_unlocked(ipi_rd(0, tl_ipi_stick_rd, &stick));
+		ipi_wait(ipi_rd(0, tl_ipi_stick_rd, &stick));
 	sched_unpin();
 	return (stick);
 }
@@ -348,7 +348,7 @@ tick_get_timecount_mp(struct timecounter *tc)
 	if (curcpu == 0)
 		tick = rd(tick);
 	else
-		ipi_wait_unlocked(ipi_rd(0, tl_ipi_tick_rd, &tick));
+		ipi_wait(ipi_rd(0, tl_ipi_tick_rd, &tick));
 	sched_unpin();
 	return (tick);
 }
diff --git a/sys/sys/bufobj.h b/sys/sys/bufobj.h
index 3934553..79bef84 100644
--- a/sys/sys/bufobj.h
+++ b/sys/sys/bufobj.h
@@ -89,12 +89,7 @@ struct buf_ops {
 
 struct bufobj {
 	struct mtx	bo_mtx;		/* Mutex which protects "i" things */
-	struct bufv	bo_clean;	/* i Clean buffers */
-	struct bufv	bo_dirty;	/* i Dirty buffers */
-	long		bo_numoutput;	/* i Writes in progress */
-	u_int		bo_flag;	/* i Flags */
 	struct buf_ops	*bo_ops;	/* - Buffer operations */
-	int		bo_bsize;	/* - Block size for i/o */
 	struct vm_object *bo_object;	/* v Place to store VM object */
 	LIST_ENTRY(bufobj) bo_synclist;	/* S dirty vnode list */
 	void		*bo_private;	/* private pointer */
@@ -103,6 +98,11 @@ struct bufobj {
 					 * XXX: only to keep the syncer working
 					 * XXX: for now.
 					 */
+	struct bufv	bo_clean;	/* i Clean buffers */
+	struct bufv	bo_dirty;	/* i Dirty buffers */
+	long		bo_numoutput;	/* i Writes in progress */
+	u_int		bo_flag;	/* i Flags */
+	int		bo_bsize;	/* - Block size for i/o */
 };
 
 /*
diff --git a/sys/sys/kernel.h b/sys/sys/kernel.h
index 441bcb3..c54677a 100644
--- a/sys/sys/kernel.h
+++ b/sys/sys/kernel.h
@@ -63,7 +63,7 @@ extern int psratio;			/* ratio: prof / stat */
 extern int stathz;			/* statistics clock's frequency */
 extern int profhz;			/* profiling clock's frequency */
 extern int profprocs;			/* number of process's profiling */
-extern int ticks;
+extern volatile int ticks;
 
 #endif /* _KERNEL */
 
diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h
index 22742c9..af7f92b 100644
--- a/sys/sys/mbuf.h
+++ b/sys/sys/mbuf.h
@@ -52,11 +52,14 @@
  * stored.  Additionally, it is possible to allocate a separate buffer
  * externally and attach it to the mbuf in a way similar to that of mbuf
  * clusters.
+ *
+ * MLEN is data length in a normal mbuf.
+ * MHLEN is data length in an mbuf with pktheader.
+ * MINCLSIZE is a smallest amount of data that should be put into cluster.
  */
-#define	MLEN		(MSIZE - sizeof(struct m_hdr))	/* normal data len */
-#define	MHLEN		(MLEN - sizeof(struct pkthdr))	/* data len w/pkthdr */
-#define	MINCLSIZE	(MHLEN + 1)	/* smallest amount to put in cluster */
-#define	M_MAXCOMPRESS	(MHLEN / 2)	/* max amount to copy for compression */
+#define	MLEN		((int)(MSIZE - sizeof(struct m_hdr)))
+#define	MHLEN		((int)(MLEN - sizeof(struct pkthdr)))
+#define	MINCLSIZE	(MHLEN + 1)
 
 #ifdef _KERNEL
 /*-
@@ -384,7 +387,6 @@ struct mbstat {
  *
  * The rest of it is defined in kern/kern_mbuf.c
  */
-extern quad_t		maxmbufmem;
 extern uma_zone_t	zone_mbuf;
 extern uma_zone_t	zone_clust;
 extern uma_zone_t	zone_pack;
@@ -393,23 +395,8 @@ extern uma_zone_t	zone_jumbo9;
 extern uma_zone_t	zone_jumbo16;
 extern uma_zone_t	zone_ext_refcnt;
 
-static __inline struct mbuf	*m_getcl(int how, short type, int flags);
-static __inline struct mbuf	*m_get(int how, short type);
-static __inline struct mbuf	*m_get2(int how, short type, int flags,
-				    u_int size);
-static __inline struct mbuf	*m_gethdr(int how, short type);
-static __inline struct mbuf	*m_getjcl(int how, short type, int flags,
-				    int size);
-static __inline struct mbuf	*m_getclr(int how, short type);	/* XXX */
-static __inline int		 m_init(struct mbuf *m, uma_zone_t zone,
-				    int size, int how, short type, int flags);
-static __inline struct mbuf	*m_free(struct mbuf *m);
-static __inline void		 m_clget(struct mbuf *m, int how);
-static __inline void		*m_cljget(struct mbuf *m, int how, int size);
-static __inline void		 m_chtype(struct mbuf *m, short new_type);
-void				 mb_free_ext(struct mbuf *);
-static __inline struct mbuf	*m_last(struct mbuf *m);
-int				 m_pkthdr_init(struct mbuf *m, int how);
+void		 mb_free_ext(struct mbuf *);
+int		 m_pkthdr_init(struct mbuf *, int);
 
 static __inline int
 m_gettype(int size)
@@ -502,7 +489,7 @@ m_get(int how, short type)
 
 	args.flags = 0;
 	args.type = type;
-	return ((struct mbuf *)(uma_zalloc_arg(zone_mbuf, &args, how)));
+	return (uma_zalloc_arg(zone_mbuf, &args, how));
 }
 
 /*
@@ -529,7 +516,7 @@ m_gethdr(int how, short type)
 
 	args.flags = M_PKTHDR;
 	args.type = type;
-	return ((struct mbuf *)(uma_zalloc_arg(zone_mbuf, &args, how)));
+	return (uma_zalloc_arg(zone_mbuf, &args, how));
 }
 
 static __inline struct mbuf *
@@ -539,85 +526,7 @@ m_getcl(int how, short type, int flags)
 
 	args.flags = flags;
 	args.type = type;
-	return ((struct mbuf *)(uma_zalloc_arg(zone_pack, &args, how)));
-}
-
-/*
- * m_get2() allocates minimum mbuf that would fit "size" argument.
- *
- * XXX: This is rather large, should be real function maybe.
- */
-static __inline struct mbuf *
-m_get2(int how, short type, int flags, u_int size)
-{
-	struct mb_args args;
-	struct mbuf *m, *n;
-	uma_zone_t zone;
-
-	args.flags = flags;
-	args.type = type;
-
-	if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0))
-		return ((struct mbuf *)(uma_zalloc_arg(zone_mbuf, &args, how)));
-	if (size <= MCLBYTES)
-		return ((struct mbuf *)(uma_zalloc_arg(zone_pack, &args, how)));
-
-	if (size > MJUM16BYTES)
-		return (NULL);
-
-	m = uma_zalloc_arg(zone_mbuf, &args, how);
-	if (m == NULL)
-		return (NULL);
-
-#if MJUMPAGESIZE != MCLBYTES
-	if (size <= MJUMPAGESIZE)
-		zone = zone_jumbop;
-	else
-#endif
-	if (size <= MJUM9BYTES)
-		zone = zone_jumbo9;
-	else
-		zone = zone_jumbo16;
-
-	n = uma_zalloc_arg(zone, m, how);
-	if (n == NULL) {
-		uma_zfree(zone_mbuf, m);
-		return (NULL);
-	}
-
-	return (m);
-}
-
-/*
- * m_getjcl() returns an mbuf with a cluster of the specified size attached.
- * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
- *
- * XXX: This is rather large, should be real function maybe.
- */
-static __inline struct mbuf *
-m_getjcl(int how, short type, int flags, int size)
-{
-	struct mb_args args;
-	struct mbuf *m, *n;
-	uma_zone_t zone;
-
-	if (size == MCLBYTES)
-		return m_getcl(how, type, flags);
-
-	args.flags = flags;
-	args.type = type;
-
-	m = uma_zalloc_arg(zone_mbuf, &args, how);
-	if (m == NULL)
-		return (NULL);
-
-	zone = m_getzone(size);
-	n = uma_zalloc_arg(zone, m, how);
-	if (n == NULL) {
-		uma_zfree(zone_mbuf, m);
-		return (NULL);
-	}
-	return (m);
+	return (uma_zalloc_arg(zone_pack, &args, how));
 }
 
 static __inline void
@@ -881,7 +790,7 @@ struct mbuf	*m_copymdata(struct mbuf *, struct mbuf *,
 		    int, int, int, int);
 struct mbuf	*m_copypacket(struct mbuf *, int);
 void		 m_copy_pkthdr(struct mbuf *, struct mbuf *);
-struct mbuf	*m_copyup(struct mbuf *n, int len, int dstoff);
+struct mbuf	*m_copyup(struct mbuf *, int, int);
 struct mbuf	*m_defrag(struct mbuf *, int);
 void		 m_demote(struct mbuf *, int);
 struct mbuf	*m_devget(char *, int, int, struct ifnet *,
@@ -891,6 +800,8 @@ int		 m_dup_pkthdr(struct mbuf *, struct mbuf *, int);
 u_int		 m_fixhdr(struct mbuf *);
 struct mbuf	*m_fragment(struct mbuf *, int, int);
 void		 m_freem(struct mbuf *);
+struct mbuf	*m_get2(int, short, int, int);
+struct mbuf	*m_getjcl(int, short, int, int);
 struct mbuf	*m_getm2(struct mbuf *, int, int, short, int);
 struct mbuf	*m_getptr(struct mbuf *, int, int *);
 u_int		 m_length(struct mbuf *, struct mbuf **);
@@ -900,10 +811,10 @@ struct mbuf	*m_prepend(struct mbuf *, int, int);
 void		 m_print(const struct mbuf *, int);
 struct mbuf	*m_pulldown(struct mbuf *, int, int, int *);
 struct mbuf	*m_pullup(struct mbuf *, int);
-int		m_sanity(struct mbuf *, int);
+int		 m_sanity(struct mbuf *, int);
 struct mbuf	*m_split(struct mbuf *, int, int);
 struct mbuf	*m_uiotombuf(struct uio *, int, int, int, int);
-struct mbuf	*m_unshare(struct mbuf *, int how);
+struct mbuf	*m_unshare(struct mbuf *, int);
 
 /*-
  * Network packets may have annotations attached by affixing a list of
diff --git a/sys/sys/mount.h b/sys/sys/mount.h
index 992227c..bbbc569 100644
--- a/sys/sys/mount.h
+++ b/sys/sys/mount.h
@@ -762,8 +762,9 @@ extern	struct nfs_public nfs_pub;
 
 /*
  * Declarations for these vfs default operations are located in
- * kern/vfs_default.c, they should be used instead of making "dummy"
- * functions or casting entries in the VFS op table to "enopnotsupp()".
+ * kern/vfs_default.c.  They will be automatically used to replace
+ * null entries in VFS ops tables when registering a new filesystem
+ * type in the global table.
  */
 vfs_root_t		vfs_stdroot;
 vfs_quotactl_t		vfs_stdquotactl;
diff --git a/sys/sys/param.h b/sys/sys/param.h
index d7a74f2..c290466 100644
--- a/sys/sys/param.h
+++ b/sys/sys/param.h
@@ -48,9 +48,9 @@
  * __FreeBSD_version numbers are documented in the Porter's Handbook.
  * If you bump the version for any reason, you should update the documentation
  * there.
- * Currently this lives here:
+ * Currently this lives here in the doc/ repository:
  *
- *	doc/en_US.ISO8859-1/books/porters-handbook/book.sgml
+ *	head/en_US.ISO8859-1/books/porters-handbook/book.xml
  *
  * scheme is:  <major><two digit minor>Rxx
  *		'R' is in the range 0 to 4 if this is a release branch or
@@ -58,7 +58,7 @@
  *		in the range 5 to 9.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 1000025	/* Master, propagated to newvers */
+#define __FreeBSD_version 1000027	/* Master, propagated to newvers */
 
 /*
  * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,
@@ -156,8 +156,8 @@
  * MCLBYTES must be no larger than PAGE_SIZE.
  */
 #ifndef	MSIZE
-#define MSIZE		256		/* size of an mbuf */
-#endif	/* MSIZE */
+#define	MSIZE		256		/* size of an mbuf */
+#endif
 
 #ifndef	MCLSHIFT
 #define MCLSHIFT	11		/* convert bytes to mbuf clusters */
diff --git a/sys/sys/pmc.h b/sys/sys/pmc.h
index 7db7dff..2d7665a 100644
--- a/sys/sys/pmc.h
+++ b/sys/sys/pmc.h
@@ -89,6 +89,7 @@
 	__PMC_CPU(INTEL_SANDYBRIDGE, 0x8D,   "Intel Sandy Bridge")	\
 	__PMC_CPU(INTEL_IVYBRIDGE, 0x8E,   "Intel Ivy Bridge")	\
 	__PMC_CPU(INTEL_SANDYBRIDGE_XEON, 0x8F,   "Intel Sandy Bridge Xeon")	\
+	__PMC_CPU(INTEL_IVYBRIDGE_XEON, 0x90,   "Intel Ivy Bridge Xeon")	\
 	__PMC_CPU(INTEL_XSCALE,	0x100,	"Intel XScale")		\
 	__PMC_CPU(MIPS_24K,     0x200,  "MIPS 24K")		\
 	__PMC_CPU(MIPS_OCTEON,  0x201,  "Cavium Octeon")	\
diff --git a/sys/sys/socket.h b/sys/sys/socket.h
index d7c6c79..c29a689 100644
--- a/sys/sys/socket.h
+++ b/sys/sys/socket.h
@@ -143,6 +143,15 @@ typedef	__uid_t		uid_t;
 #endif
 
 /*
+ * Space reserved for new socket options added by third-party vendors.
+ * This range applies to all socket option levels.  New socket options
+ * in FreeBSD should always use an option value less than SO_VENDOR.
+ */
+#if __BSD_VISIBLE
+#define	SO_VENDOR	0x80000000
+#endif
+
+/*
  * Structure used for manipulating linger option.
  */
 struct linger {
diff --git a/sys/sys/systm.h b/sys/sys/systm.h
index 7c3f13e..dd96d70 100644
--- a/sys/sys/systm.h
+++ b/sys/sys/systm.h
@@ -49,6 +49,7 @@ extern int cold;		/* nonzero if we are doing a cold boot */
 extern int rebooting;		/* kern_reboot() has been called. */
 extern const char *panicstr;	/* panic message */
 extern char version[];		/* system version */
+extern char compiler_version[];	/* compiler version */
 extern char copyright[];	/* system copyright */
 extern int kstack_pages;	/* number of kernel stack pages */
 
diff --git a/sys/sys/time.h b/sys/sys/time.h
index 12d43cf..80878c0 100644
--- a/sys/sys/time.h
+++ b/sys/sys/time.h
@@ -287,8 +287,8 @@ struct clockinfo {
 void	inittodr(time_t base);
 void	resettodr(void);
 
-extern time_t	time_second;
-extern time_t	time_uptime;
+extern volatile time_t	time_second;
+extern volatile time_t	time_uptime;
 extern struct bintime boottimebin;
 extern struct timeval boottime;
 
diff --git a/sys/sys/un.h b/sys/sys/un.h
index 1f4731e..7837e76 100644
--- a/sys/sys/un.h
+++ b/sys/sys/un.h
@@ -53,9 +53,12 @@ struct sockaddr_un {
 #if __BSD_VISIBLE
 
 /* Socket options. */
-#define	LOCAL_PEERCRED		0x001	/* retrieve peer credentials */
-#define	LOCAL_CREDS		0x002	/* pass credentials to receiver */
-#define	LOCAL_CONNWAIT		0x004	/* connects block until accepted */
+#define	LOCAL_PEERCRED		1	/* retrieve peer credentials */
+#define	LOCAL_CREDS		2	/* pass credentials to receiver */
+#define	LOCAL_CONNWAIT		4	/* connects block until accepted */
+
+/* Start of reserved space for third-party socket options. */
+#define	LOCAL_VENDOR		SO_VENDOR
 
 #ifndef _KERNEL
 
diff --git a/sys/sys/vdso.h b/sys/sys/vdso.h
index 653a606..f584315 100644
--- a/sys/sys/vdso.h
+++ b/sys/sys/vdso.h
@@ -61,13 +61,9 @@ struct timeval;
 struct timezone;
 
 int __vdso_clock_gettime(clockid_t clock_id, struct timespec *ts);
-#pragma weak __vdso_clock_gettime
-
 int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz);
-#pragma weak __vdso_gettimeofday
-
 u_int __vdso_gettc(const struct vdso_timehands *vdso_th);
-#pragma weak __vdso_gettc
+int __vdso_gettimekeep(struct vdso_timekeep **tk);
 
 #endif
 
diff --git a/sys/sys/vmmeter.h b/sys/sys/vmmeter.h
index 1a29ec1..59b430d 100644
--- a/sys/sys/vmmeter.h
+++ b/sys/sys/vmmeter.h
@@ -61,6 +61,7 @@ struct vmmeter {
 	 * Virtual memory activity.
 	 */
 	u_int v_vm_faults;	/* (p) address memory faults */
+	u_int v_io_faults;	/* (p) page faults requiring I/O */
 	u_int v_cow_faults;	/* (p) copy-on-writes faults */
 	u_int v_cow_optim;	/* (p) optimized copy-on-writes faults */
 	u_int v_zfod;		/* (p) pages zero filled on demand */
diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h
index f487f37..b54dc04 100644
--- a/sys/sys/vnode.h
+++ b/sys/sys/vnode.h
@@ -99,7 +99,6 @@ struct vnode {
 	 * Fields which define the identity of the vnode.  These fields are
 	 * owned by the filesystem (XXX: and vgone() ?)
 	 */
-	enum	vtype v_type;			/* u vnode type */
 	const char *v_tag;			/* u type of underlying data */
 	struct	vop_vector *v_op;		/* u vnode operations vector */
 	void	*v_data;			/* u private data for fs */
@@ -122,10 +121,10 @@ struct vnode {
 	} v_un;
 
 	/*
-	 * vfs_hash:  (mount + inode) -> vnode hash.
+	 * vfs_hash: (mount + inode) -> vnode hash.  The hash value
+	 * itself is grouped with other int fields, to avoid padding.
 	 */
 	LIST_ENTRY(vnode)	v_hashlist;
-	u_int			v_hash;
 
 	/*
 	 * VFS_namecache stuff
@@ -135,24 +134,11 @@ struct vnode {
 	struct namecache *v_cache_dd;		/* c Cache entry for .. vnode */
 
 	/*
-	 * clustering stuff
-	 */
-	daddr_t	v_cstart;			/* v start block of cluster */
-	daddr_t	v_lasta;			/* v last allocation  */
-	daddr_t	v_lastw;			/* v last write  */
-	int	v_clen;				/* v length of cur. cluster */
-
-	/*
 	 * Locking
 	 */
 	struct	lock v_lock;			/* u (if fs don't have one) */
 	struct	mtx v_interlock;		/* lock for "i" things */
 	struct	lock *v_vnlock;			/* u pointer to vnode lock */
-	int	v_holdcnt;			/* i prevents recycling. */
-	int	v_usecount;			/* i ref count of users */
-	u_int	v_iflag;			/* i vnode flags (see below) */
-	u_int	v_vflag;			/* v vnode flags */
-	int	v_writecount;			/* v ref count of writers */
 
 	/*
 	 * The machinery of being a vnode
@@ -167,6 +153,22 @@ struct vnode {
 	struct label *v_label;			/* MAC label for vnode */
 	struct lockf *v_lockf;		/* Byte-level advisory lock list */
 	struct rangelock v_rl;			/* Byte-range lock */
+
+	/*
+	 * clustering stuff
+	 */
+	daddr_t	v_cstart;			/* v start block of cluster */
+	daddr_t	v_lasta;			/* v last allocation  */
+	daddr_t	v_lastw;			/* v last write  */
+	int	v_clen;				/* v length of cur. cluster */
+
+	int	v_holdcnt;			/* i prevents recycling. */
+	int	v_usecount;			/* i ref count of users */
+	u_int	v_iflag;			/* i vnode flags (see below) */
+	u_int	v_vflag;			/* v vnode flags */
+	int	v_writecount;			/* v ref count of writers */
+	u_int	v_hash;
+	enum	vtype v_type;			/* u vnode type */
 };
 
 #endif /* defined(_KERNEL) || defined(_KVM_VNODE) */
@@ -703,8 +705,7 @@ int	vn_io_fault_uiomove(char *data, int xfersize, struct uio *uio);
 
 int	vfs_cache_lookup(struct vop_lookup_args *ap);
 void	vfs_timestamp(struct timespec *);
-void	vfs_write_resume(struct mount *mp);
-void	vfs_write_resume_flags(struct mount *mp, int flags);
+void	vfs_write_resume(struct mount *mp, int flags);
 int	vfs_write_suspend(struct mount *mp);
 int	vop_stdbmap(struct vop_bmap_args *);
 int	vop_stdfsync(struct vop_fsync_args *);
@@ -813,6 +814,7 @@ int	fifo_printinfo(struct vnode *);
 typedef int vfs_hash_cmp_t(struct vnode *vp, void *arg);
 
 int vfs_hash_get(const struct mount *mp, u_int hash, int flags, struct thread *td, struct vnode **vpp, vfs_hash_cmp_t *fn, void *arg);
+u_int vfs_hash_index(struct vnode *vp);
 int vfs_hash_insert(struct vnode *vp, u_int hash, int flags, struct thread *td, struct vnode **vpp, vfs_hash_cmp_t *fn, void *arg);
 void vfs_hash_rehash(struct vnode *vp, u_int hash);
 void vfs_hash_remove(struct vnode *vp);
diff --git a/sys/ufs/ffs/ffs_inode.c b/sys/ufs/ffs/ffs_inode.c
index f8584d5..4571a1c 100644
--- a/sys/ufs/ffs/ffs_inode.c
+++ b/sys/ufs/ffs/ffs_inode.c
@@ -546,9 +546,9 @@ done:
 	 */
 	ip->i_size = length;
 	DIP_SET(ip, i_size, length);
-	DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - blocksreleased);
-
-	if (DIP(ip, i_blocks) < 0)			/* sanity */
+	if (DIP(ip, i_blocks) >= blocksreleased)
+		DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - blocksreleased);
+	else	/* sanity */
 		DIP_SET(ip, i_blocks, 0);
 	ip->i_flag |= IN_CHANGE;
 #ifdef QUOTA
diff --git a/sys/ufs/ffs/ffs_snapshot.c b/sys/ufs/ffs/ffs_snapshot.c
index 5f67ae5..0c653c2 100644
--- a/sys/ufs/ffs/ffs_snapshot.c
+++ b/sys/ufs/ffs/ffs_snapshot.c
@@ -687,7 +687,7 @@ out1:
 	/*
 	 * Resume operation on filesystem.
 	 */
-	vfs_write_resume_flags(vp->v_mount, VR_START_WRITE | VR_NO_SUSPCLR);
+	vfs_write_resume(vp->v_mount, VR_START_WRITE | VR_NO_SUSPCLR);
 	if (collectsnapstats && starttime.tv_sec > 0) {
 		nanotime(&endtime);
 		timespecsub(&endtime, &starttime);
diff --git a/sys/ufs/ffs/ffs_softdep.c b/sys/ufs/ffs/ffs_softdep.c
index 4ee16ab..16fe134 100644
--- a/sys/ufs/ffs/ffs_softdep.c
+++ b/sys/ufs/ffs/ffs_softdep.c
@@ -2802,7 +2802,7 @@ journal_unsuspend(struct ufsmount *ump)
 		jblocks->jb_suspended = 0;
 		FREE_LOCK(&lk);
 		mp->mnt_susp_owner = curthread;
-		vfs_write_resume(mp);
+		vfs_write_resume(mp, 0);
 		ACQUIRE_LOCK(&lk);
 		return (1);
 	}
diff --git a/sys/ufs/ffs/ffs_suspend.c b/sys/ufs/ffs/ffs_suspend.c
index 9d8e2c1..3198c1a 100644
--- a/sys/ufs/ffs/ffs_suspend.c
+++ b/sys/ufs/ffs/ffs_suspend.c
@@ -252,7 +252,7 @@ ffs_susp_dtor(void *data)
 	 */
 	mp->mnt_susp_owner = curthread;
 
-	vfs_write_resume(mp);
+	vfs_write_resume(mp, 0);
 	vfs_unbusy(mp);
 	ump->um_writesuspended = 0;
 
diff --git a/sys/ufs/ffs/ffs_vfsops.c b/sys/ufs/ffs/ffs_vfsops.c
index 83ae202..0204613 100644
--- a/sys/ufs/ffs/ffs_vfsops.c
+++ b/sys/ufs/ffs/ffs_vfsops.c
@@ -292,7 +292,7 @@ ffs_mount(struct mount *mp)
 				error = ffs_flushfiles(mp, flags, td);
 			}
 			if (error) {
-				vfs_write_resume(mp);
+				vfs_write_resume(mp, 0);
 				return (error);
 			}
 			if (fs->fs_pendingblocks != 0 ||
@@ -309,7 +309,7 @@ ffs_mount(struct mount *mp)
 			if ((error = ffs_sbupdate(ump, MNT_WAIT, 0)) != 0) {
 				fs->fs_ronly = 0;
 				fs->fs_clean = 0;
-				vfs_write_resume(mp);
+				vfs_write_resume(mp, 0);
 				return (error);
 			}
 			if (MOUNTEDSOFTDEP(mp))
@@ -330,7 +330,7 @@ ffs_mount(struct mount *mp)
 			 * Allow the writers to note that filesystem
 			 * is ro now.
 			 */
-			vfs_write_resume(mp);
+			vfs_write_resume(mp, 0);
 		}
 		if ((mp->mnt_flag & MNT_RELOAD) &&
 		    (error = ffs_reload(mp, td, 0)) != 0)
@@ -1294,10 +1294,8 @@ ffs_unmount(mp, mntflags)
 			goto fail;
 		}
 	}
-	if (susp) {
-		vfs_write_resume(mp);
-		vn_start_write(NULL, &mp, V_WAIT);
-	}
+	if (susp)
+		vfs_write_resume(mp, VR_START_WRITE);
 	DROP_GIANT();
 	g_topology_lock();
 	if (ump->um_fsckpid > 0) {
@@ -1329,10 +1327,8 @@ ffs_unmount(mp, mntflags)
 	return (error);
 
 fail:
-	if (susp) {
-		vfs_write_resume(mp);
-		vn_start_write(NULL, &mp, V_WAIT);
-	}
+	if (susp)
+		vfs_write_resume(mp, VR_START_WRITE);
 #ifdef UFS_EXTATTR
 	if (e_restart) {
 		ufs_extattr_uepm_init(&ump->um_extattr);
diff --git a/sys/vm/device_pager.c b/sys/vm/device_pager.c
index 2a18b0d..30aaac0 100644
--- a/sys/vm/device_pager.c
+++ b/sys/vm/device_pager.c
@@ -160,6 +160,7 @@ cdev_pager_allocate(void *handle, enum obj_type tp, struct cdev_pager_ops *ops,
 		object1->pg_color = color;
 		object1->handle = handle;
 		object1->un_pager.devp.ops = ops;
+		object1->un_pager.devp.dev = handle;
 		TAILQ_INIT(&object1->un_pager.devp.devp_pglist);
 		mtx_lock(&dev_pager_mtx);
 		object = vm_pager_object_lookup(&dev_pager_object_list, handle);
@@ -237,7 +238,7 @@ dev_pager_dealloc(object)
 	vm_page_t m;
 
 	VM_OBJECT_UNLOCK(object);
-	object->un_pager.devp.ops->cdev_pg_dtor(object->handle);
+	object->un_pager.devp.ops->cdev_pg_dtor(object->un_pager.devp.dev);
 
 	mtx_lock(&dev_pager_mtx);
 	TAILQ_REMOVE(&dev_pager_object_list, object, pager_object_list);
diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c
index 36a149e..2d5b555 100644
--- a/sys/vm/uma_core.c
+++ b/sys/vm/uma_core.c
@@ -1702,7 +1702,7 @@ uma_startup(void *bootmem, int boot_pages)
 
 #ifdef UMA_DEBUG
 	printf("Calculated uma_max_ipers (for OFFPAGE) is %d\n", uma_max_ipers);
-	printf("Calculated uma_max_ipers_slab (for OFFPAGE) is %d\n",
+	printf("Calculated uma_max_ipers_ref (for OFFPAGE) is %d\n",
 	    uma_max_ipers_ref);
 #endif
 
diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
index 2274464..9883dcf 100644
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -937,9 +937,10 @@ vnode_locked:
 	 * Unlock everything, and return
 	 */
 	unlock_and_deallocate(&fs);
-	if (hardfault)
+	if (hardfault) {
+		PCPU_INC(cnt.v_io_faults);
 		curthread->td_ru.ru_majflt++;
-	else
+	} else 
 		curthread->td_ru.ru_minflt++;
 
 	return (KERN_SUCCESS);
diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c
index f87e5b9..26de826 100644
--- a/sys/vm/vm_map.c
+++ b/sys/vm/vm_map.c
@@ -726,12 +726,6 @@ vmspace_resident_count(struct vmspace *vmspace)
 	return pmap_resident_count(vmspace_pmap(vmspace));
 }
 
-long
-vmspace_wired_count(struct vmspace *vmspace)
-{
-	return pmap_wired_count(vmspace_pmap(vmspace));
-}
-
 /*
  *	vm_map_create:
  *
@@ -3281,8 +3275,7 @@ vm_map_stack(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize,
 	}
 
 	if (!old_mlock && map->flags & MAP_WIREFUTURE) {
-		if (ptoa(vmspace_wired_count(curproc->p_vmspace)) +
-		    init_ssize > lmemlim) {
+		if (ptoa(pmap_wired_count(map->pmap)) + init_ssize > lmemlim) {
 			vm_map_unlock(map);
 			return (KERN_NO_SPACE);
 		}
@@ -3505,8 +3498,7 @@ Retry:
 		grow_amount = limit - ctob(vm->vm_ssize);
 #endif
 	if (!old_mlock && map->flags & MAP_WIREFUTURE) {
-		if (ptoa(vmspace_wired_count(p->p_vmspace)) + grow_amount >
-		    lmemlim) {
+		if (ptoa(pmap_wired_count(map->pmap)) + grow_amount > lmemlim) {
 			vm_map_unlock_read(map);
 			rv = KERN_NO_SPACE;
 			goto out;
@@ -3514,7 +3506,7 @@ Retry:
 #ifdef RACCT
 		PROC_LOCK(p);
 		if (racct_set(p, RACCT_MEMLOCK,
-		    ptoa(vmspace_wired_count(p->p_vmspace)) + grow_amount)) {
+		    ptoa(pmap_wired_count(map->pmap)) + grow_amount)) {
 			PROC_UNLOCK(p);
 			vm_map_unlock_read(map);
 			rv = KERN_NO_SPACE;
@@ -3645,7 +3637,7 @@ out:
 		KASSERT(error == 0, ("decreasing RACCT_VMEM failed"));
 		if (!old_mlock) {
 			error = racct_set(p, RACCT_MEMLOCK,
-			    ptoa(vmspace_wired_count(p->p_vmspace)));
+			    ptoa(pmap_wired_count(map->pmap)));
 			KASSERT(error == 0, ("decreasing RACCT_MEMLOCK failed"));
 		}
 	    	error = racct_set(p, RACCT_STACK, ctob(vm->vm_ssize));
diff --git a/sys/vm/vm_map.h b/sys/vm/vm_map.h
index b3b1ad4..135b555 100644
--- a/sys/vm/vm_map.h
+++ b/sys/vm/vm_map.h
@@ -298,7 +298,6 @@ void vm_map_wait_busy(vm_map_t map);
 			_vm_map_lock_downgrade(map, LOCK_FILE, LOCK_LINE)
 
 long vmspace_resident_count(struct vmspace *vmspace);
-long vmspace_wired_count(struct vmspace *vmspace);
 #endif	/* _KERNEL */
 
 
diff --git a/sys/vm/vm_meter.c b/sys/vm/vm_meter.c
index b5bb0fa..05174e9 100644
--- a/sys/vm/vm_meter.c
+++ b/sys/vm/vm_meter.c
@@ -283,6 +283,7 @@ VM_STATS_SYS(v_syscall, "System calls");
 VM_STATS_SYS(v_intr, "Device interrupts");
 VM_STATS_SYS(v_soft, "Software interrupts");
 VM_STATS_VM(v_vm_faults, "Address memory faults");
+VM_STATS_VM(v_io_faults, "Page faults requiring I/O");
 VM_STATS_VM(v_cow_faults, "Copy-on-write faults");
 VM_STATS_VM(v_cow_optim, "Optimized COW faults");
 VM_STATS_VM(v_zfod, "Pages zero-filled on demand");
diff --git a/sys/vm/vm_mmap.c b/sys/vm/vm_mmap.c
index 3ad7c37..4ae1f90 100644
--- a/sys/vm/vm_mmap.c
+++ b/sys/vm/vm_mmap.c
@@ -1044,6 +1044,7 @@ sys_mlock(td, uap)
 	struct proc *proc;
 	vm_offset_t addr, end, last, start;
 	vm_size_t npages, size;
+	vm_map_t map;
 	unsigned long nsize;
 	int error;
 
@@ -1061,8 +1062,9 @@ sys_mlock(td, uap)
 	if (npages > vm_page_max_wired)
 		return (ENOMEM);
 	proc = td->td_proc;
+	map = &proc->p_vmspace->vm_map;
 	PROC_LOCK(proc);
-	nsize = ptoa(npages + vmspace_wired_count(proc->p_vmspace));
+	nsize = ptoa(npages + pmap_wired_count(map->pmap));
 	if (nsize > lim_cur(proc, RLIMIT_MEMLOCK)) {
 		PROC_UNLOCK(proc);
 		return (ENOMEM);
@@ -1077,13 +1079,13 @@ sys_mlock(td, uap)
 	if (error != 0)
 		return (ENOMEM);
 #endif
-	error = vm_map_wire(&proc->p_vmspace->vm_map, start, end,
+	error = vm_map_wire(map, start, end,
 	    VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
 #ifdef RACCT
 	if (error != KERN_SUCCESS) {
 		PROC_LOCK(proc);
 		racct_set(proc, RACCT_MEMLOCK,
-		    ptoa(vmspace_wired_count(proc->p_vmspace)));
+		    ptoa(pmap_wired_count(map->pmap)));
 		PROC_UNLOCK(proc);
 	}
 #endif
@@ -1157,7 +1159,7 @@ sys_mlockall(td, uap)
 	if (error != KERN_SUCCESS) {
 		PROC_LOCK(td->td_proc);
 		racct_set(td->td_proc, RACCT_MEMLOCK,
-		    ptoa(vmspace_wired_count(td->td_proc->p_vmspace)));
+		    ptoa(pmap_wired_count(map->pmap)));
 		PROC_UNLOCK(td->td_proc);
 	}
 #endif
@@ -1491,16 +1493,15 @@ vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot,
 			return (ENOMEM);
 		}
 		if (!old_mlock && map->flags & MAP_WIREFUTURE) {
-			if (ptoa(vmspace_wired_count(td->td_proc->p_vmspace)) +
-			    size > lim_cur(td->td_proc, RLIMIT_MEMLOCK)) {
+			if (ptoa(pmap_wired_count(map->pmap)) + size >
+			    lim_cur(td->td_proc, RLIMIT_MEMLOCK)) {
 				racct_set_force(td->td_proc, RACCT_VMEM,
 				    map->size);
 				PROC_UNLOCK(td->td_proc);
 				return (ENOMEM);
 			}
 			error = racct_set(td->td_proc, RACCT_MEMLOCK,
-			    ptoa(vmspace_wired_count(td->td_proc->p_vmspace)) +
-			    size);
+			    ptoa(pmap_wired_count(map->pmap)) + size);
 			if (error != 0) {
 				racct_set_force(td->td_proc, RACCT_VMEM,
 				    map->size);
diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h
index 6aee2bc..7beee59 100644
--- a/sys/vm/vm_object.h
+++ b/sys/vm/vm_object.h
@@ -138,6 +138,7 @@ struct vm_object {
 		struct {
 			TAILQ_HEAD(, vm_page) devp_pglist;
 			struct cdev_pager_ops *ops;
+			struct cdev *dev;
 		} devp;
 
 		/*
diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index b5e9747..ac593a4 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -151,18 +151,21 @@ static struct mtx vm_daemon_mtx;
 MTX_SYSINIT(vm_daemon, &vm_daemon_mtx, "vm daemon", MTX_DEF);
 #endif
 static int vm_max_launder = 32;
-static int vm_pageout_stats_max=0, vm_pageout_stats_interval = 0;
-static int vm_pageout_full_stats_interval = 0;
-static int vm_pageout_algorithm=0;
-static int defer_swap_pageouts=0;
-static int disable_swap_pageouts=0;
+static int vm_pageout_stats_max;
+static int vm_pageout_stats;
+static int vm_pageout_stats_interval;
+static int vm_pageout_full_stats;
+static int vm_pageout_full_stats_interval;
+static int vm_pageout_algorithm;
+static int defer_swap_pageouts;
+static int disable_swap_pageouts;
 
 #if defined(NO_SWAPPING)
-static int vm_swap_enabled=0;
-static int vm_swap_idle_enabled=0;
+static int vm_swap_enabled = 0;
+static int vm_swap_idle_enabled = 0;
 #else
-static int vm_swap_enabled=1;
-static int vm_swap_idle_enabled=0;
+static int vm_swap_enabled = 1;
+static int vm_swap_idle_enabled = 0;
 #endif
 
 SYSCTL_INT(_vm, VM_PAGEOUT_ALGORITHM, pageout_algorithm,
@@ -174,12 +177,18 @@ SYSCTL_INT(_vm, OID_AUTO, max_launder,
 SYSCTL_INT(_vm, OID_AUTO, pageout_stats_max,
 	CTLFLAG_RW, &vm_pageout_stats_max, 0, "Max pageout stats scan length");
 
-SYSCTL_INT(_vm, OID_AUTO, pageout_full_stats_interval,
-	CTLFLAG_RW, &vm_pageout_full_stats_interval, 0, "Interval for full stats scan");
+SYSCTL_INT(_vm, OID_AUTO, pageout_stats,
+	CTLFLAG_RD, &vm_pageout_stats, 0, "Number of partial stats scans");
 
 SYSCTL_INT(_vm, OID_AUTO, pageout_stats_interval,
 	CTLFLAG_RW, &vm_pageout_stats_interval, 0, "Interval for partial stats scan");
 
+SYSCTL_INT(_vm, OID_AUTO, pageout_full_stats,
+	CTLFLAG_RD, &vm_pageout_full_stats, 0, "Number of full stats scans");
+
+SYSCTL_INT(_vm, OID_AUTO, pageout_full_stats_interval,
+	CTLFLAG_RW, &vm_pageout_full_stats_interval, 0, "Interval for full stats scan");
+
 #if defined(NO_SWAPPING)
 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
 	CTLFLAG_RD, &vm_swap_enabled, 0, "Enable entire process swapout");
@@ -1512,12 +1521,12 @@ vm_pageout_oom(int shortage)
  * helps the situation where paging just starts to occur.
  */
 static void
-vm_pageout_page_stats()
+vm_pageout_page_stats(void)
 {
 	struct vm_pagequeue *pq;
 	vm_object_t object;
-	vm_page_t m,next;
-	int pcount,tpcount;		/* Number of pages to check */
+	vm_page_t m, next;
+	int pcount, tpcount;		/* Number of pages to check */
 	static int fullintervalcount = 0;
 	int page_shortage;
 
@@ -1531,11 +1540,13 @@ vm_pageout_page_stats()
 	pcount = cnt.v_active_count;
 	fullintervalcount += vm_pageout_stats_interval;
 	if (fullintervalcount < vm_pageout_full_stats_interval) {
+		vm_pageout_stats++;
 		tpcount = (int64_t)vm_pageout_stats_max * cnt.v_active_count /
 		    cnt.v_page_count;
 		if (pcount > tpcount)
 			pcount = tpcount;
 	} else {
+		vm_pageout_full_stats++;
 		fullintervalcount = 0;
 	}
 
@@ -1624,7 +1635,7 @@ vm_pageout_page_stats()
  *	vm_pageout is the high level pageout daemon.
  */
 static void
-vm_pageout()
+vm_pageout(void)
 {
 	int error, pass;
 
@@ -1757,7 +1768,7 @@ vm_pageout()
  * the free page queue lock is held until the msleep() is performed.
  */
 void
-pagedaemon_wakeup()
+pagedaemon_wakeup(void)
 {
 
 	if (!vm_pages_needed && curthread->td_proc != pageproc) {
@@ -1782,7 +1793,7 @@ vm_req_vmdaemon(int req)
 }
 
 static void
-vm_daemon()
+vm_daemon(void)
 {
 	struct rlimit rsslim;
 	struct proc *p;
diff --git a/sys/vm/vm_unix.c b/sys/vm/vm_unix.c
index d6da08b..edb6ecc 100644
--- a/sys/vm/vm_unix.c
+++ b/sys/vm/vm_unix.c
@@ -76,6 +76,7 @@ sys_obreak(td, uap)
 	struct obreak_args *uap;
 {
 	struct vmspace *vm = td->td_proc->p_vmspace;
+	vm_map_t map = &vm->vm_map;
 	vm_offset_t new, old, base;
 	rlim_t datalim, lmemlim, vmemlim;
 	int prot, rv;
@@ -90,7 +91,7 @@ sys_obreak(td, uap)
 
 	do_map_wirefuture = FALSE;
 	new = round_page((vm_offset_t)uap->nsize);
-	vm_map_lock(&vm->vm_map);
+	vm_map_lock(map);
 
 	base = round_page((vm_offset_t) vm->vm_daddr);
 	old = base + ctob(vm->vm_dsize);
@@ -103,7 +104,7 @@ sys_obreak(td, uap)
 			error = ENOMEM;
 			goto done;
 		}
-		if (new > vm_map_max(&vm->vm_map)) {
+		if (new > vm_map_max(map)) {
 			error = ENOMEM;
 			goto done;
 		}
@@ -117,14 +118,14 @@ sys_obreak(td, uap)
 		goto done;
 	}
 	if (new > old) {
-		if (!old_mlock && vm->vm_map.flags & MAP_WIREFUTURE) {
-			if (ptoa(vmspace_wired_count(td->td_proc->p_vmspace)) +
+		if (!old_mlock && map->flags & MAP_WIREFUTURE) {
+			if (ptoa(pmap_wired_count(map->pmap)) +
 			    (new - old) > lmemlim) {
 				error = ENOMEM;
 				goto done;
 			}
 		}
-		if (vm->vm_map.size + (new - old) > vmemlim) {
+		if (map->size + (new - old) > vmemlim) {
 			error = ENOMEM;
 			goto done;
 		}
@@ -137,22 +138,21 @@ sys_obreak(td, uap)
 			goto done;
 		}
 		error = racct_set(td->td_proc, RACCT_VMEM,
-		    vm->vm_map.size + (new - old));
+		    map->size + (new - old));
 		if (error != 0) {
 			racct_set_force(td->td_proc, RACCT_DATA, old - base);
 			PROC_UNLOCK(td->td_proc);
 			error = ENOMEM;
 			goto done;
 		}
-		if (!old_mlock && vm->vm_map.flags & MAP_WIREFUTURE) {
+		if (!old_mlock && map->flags & MAP_WIREFUTURE) {
 			error = racct_set(td->td_proc, RACCT_MEMLOCK,
-			    ptoa(vmspace_wired_count(td->td_proc->p_vmspace)) +
-			    (new - old));
+			    ptoa(pmap_wired_count(map->pmap)) + (new - old));
 			if (error != 0) {
 				racct_set_force(td->td_proc, RACCT_DATA,
 				    old - base);
 				racct_set_force(td->td_proc, RACCT_VMEM,
-				    vm->vm_map.size);
+				    map->size);
 				PROC_UNLOCK(td->td_proc);
 				error = ENOMEM;
 				goto done;
@@ -167,17 +167,15 @@ sys_obreak(td, uap)
 			prot |= VM_PROT_EXECUTE;
 #endif
 #endif
-		rv = vm_map_insert(&vm->vm_map, NULL, 0, old, new,
-		    prot, VM_PROT_ALL, 0);
+		rv = vm_map_insert(map, NULL, 0, old, new, prot, VM_PROT_ALL, 0);
 		if (rv != KERN_SUCCESS) {
 #ifdef RACCT
 			PROC_LOCK(td->td_proc);
 			racct_set_force(td->td_proc, RACCT_DATA, old - base);
-			racct_set_force(td->td_proc, RACCT_VMEM, vm->vm_map.size);
-			if (!old_mlock && vm->vm_map.flags & MAP_WIREFUTURE) {
+			racct_set_force(td->td_proc, RACCT_VMEM, map->size);
+			if (!old_mlock && map->flags & MAP_WIREFUTURE) {
 				racct_set_force(td->td_proc, RACCT_MEMLOCK,
-				    ptoa(vmspace_wired_count(
-				    td->td_proc->p_vmspace)));
+				    ptoa(pmap_wired_count(map->pmap)));
 			}
 			PROC_UNLOCK(td->td_proc);
 #endif
@@ -194,13 +192,13 @@ sys_obreak(td, uap)
 		 *
 		 * XXX If the pages cannot be wired, no error is returned.
 		 */
-		if ((vm->vm_map.flags & MAP_WIREFUTURE) == MAP_WIREFUTURE) {
+		if ((map->flags & MAP_WIREFUTURE) == MAP_WIREFUTURE) {
 			if (bootverbose)
 				printf("obreak: MAP_WIREFUTURE set\n");
 			do_map_wirefuture = TRUE;
 		}
 	} else if (new < old) {
-		rv = vm_map_delete(&vm->vm_map, new, old);
+		rv = vm_map_delete(map, new, old);
 		if (rv != KERN_SUCCESS) {
 			error = ENOMEM;
 			goto done;
@@ -209,19 +207,19 @@ sys_obreak(td, uap)
 #ifdef RACCT
 		PROC_LOCK(td->td_proc);
 		racct_set_force(td->td_proc, RACCT_DATA, new - base);
-		racct_set_force(td->td_proc, RACCT_VMEM, vm->vm_map.size);
-		if (!old_mlock && vm->vm_map.flags & MAP_WIREFUTURE) {
+		racct_set_force(td->td_proc, RACCT_VMEM, map->size);
+		if (!old_mlock && map->flags & MAP_WIREFUTURE) {
 			racct_set_force(td->td_proc, RACCT_MEMLOCK,
-			    ptoa(vmspace_wired_count(td->td_proc->p_vmspace)));
+			    ptoa(pmap_wired_count(map->pmap)));
 		}
 		PROC_UNLOCK(td->td_proc);
 #endif
 	}
 done:
-	vm_map_unlock(&vm->vm_map);
+	vm_map_unlock(map);
 
 	if (do_map_wirefuture)
-		(void) vm_map_wire(&vm->vm_map, old, new,
+		(void) vm_map_wire(map, old, new,
 		    VM_MAP_WIRE_USER|VM_MAP_WIRE_NOHOLES);
 
 	return (error);
diff --git a/sys/x86/x86/intr_machdep.c b/sys/x86/x86/intr_machdep.c
index 31cc80b..e21635f 100644
--- a/sys/x86/x86/intr_machdep.c
+++ b/sys/x86/x86/intr_machdep.c
@@ -78,7 +78,7 @@ static int intrcnt_index;
 static struct intsrc *interrupt_sources[NUM_IO_INTS];
 static struct mtx intr_table_lock;
 static struct mtx intrcnt_lock;
-static STAILQ_HEAD(, pic) pics;
+static TAILQ_HEAD(pics_head, pic) pics;
 
 #ifdef SMP
 static int assign_cpu;
@@ -102,7 +102,7 @@ intr_pic_registered(struct pic *pic)
 {
 	struct pic *p;
 
-	STAILQ_FOREACH(p, &pics, pics) {
+	TAILQ_FOREACH(p, &pics, pics) {
 		if (p == pic)
 			return (1);
 	}
@@ -124,7 +124,7 @@ intr_register_pic(struct pic *pic)
 	if (intr_pic_registered(pic))
 		error = EBUSY;
 	else {
-		STAILQ_INSERT_TAIL(&pics, pic, pics);
+		TAILQ_INSERT_TAIL(&pics, pic, pics);
 		error = 0;
 	}
 	mtx_unlock(&intr_table_lock);
@@ -287,7 +287,7 @@ intr_resume(void)
 	atpic_reset();
 #endif
 	mtx_lock(&intr_table_lock);
-	STAILQ_FOREACH(pic, &pics, pics) {
+	TAILQ_FOREACH(pic, &pics, pics) {
 		if (pic->pic_resume != NULL)
 			pic->pic_resume(pic);
 	}
@@ -300,7 +300,7 @@ intr_suspend(void)
 	struct pic *pic;
 
 	mtx_lock(&intr_table_lock);
-	STAILQ_FOREACH(pic, &pics, pics) {
+	TAILQ_FOREACH_REVERSE(pic, &pics, pics_head, pics) {
 		if (pic->pic_suspend != NULL)
 			pic->pic_suspend(pic);
 	}
@@ -381,7 +381,7 @@ intr_init(void *dummy __unused)
 
 	intrcnt_setname("???", 0);
 	intrcnt_index = 1;
-	STAILQ_INIT(&pics);
+	TAILQ_INIT(&pics);
 	mtx_init(&intr_table_lock, "intr sources", NULL, MTX_DEF);
 	mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN);
 }
diff --git a/sys/x86/x86/local_apic.c b/sys/x86/x86/local_apic.c
index e994172..c60db22 100644
--- a/sys/x86/x86/local_apic.c
+++ b/sys/x86/x86/local_apic.c
@@ -227,8 +227,8 @@ lapic_init(vm_paddr_t addr)
 	/* Map the local APIC and setup the spurious interrupt handler. */
 	KASSERT(trunc_page(addr) == addr,
 	    ("local APIC not aligned on a page boundary"));
-	lapic = pmap_mapdev(addr, sizeof(lapic_t));
 	lapic_paddr = addr;
+	lapic = pmap_mapdev(addr, sizeof(lapic_t));
 	setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL,
 	    GSEL_APIC);
 
@@ -1360,11 +1360,19 @@ apic_setup_io(void *dummy __unused)
 
 	if (best_enum == NULL)
 		return;
+
+	/*
+	 * Local APIC must be registered before other PICs and pseudo PICs
+	 * for proper suspend/resume order.
+	 */
+#ifndef XEN
+	intr_register_pic(&lapic_pic);
+#endif
+
 	retval = best_enum->apic_setup_io();
 	if (retval != 0)
 		printf("%s: Failed to setup I/O APICs: returned %d\n",
 		    best_enum->apic_name, retval);
-
 #ifdef XEN
 	return;
 #endif
@@ -1373,7 +1381,6 @@ apic_setup_io(void *dummy __unused)
 	 * properly program the LINT pins.
 	 */
 	lapic_setup(1);
-	intr_register_pic(&lapic_pic);
 	if (bootverbose)
 		lapic_dump("BSP");
 
diff --git a/sys/x86/x86/tsc.c b/sys/x86/x86/tsc.c
index 04231b8..4f67818 100644
--- a/sys/x86/x86/tsc.c
+++ b/sys/x86/x86/tsc.c
@@ -67,6 +67,11 @@ SYSCTL_INT(_kern_timecounter, OID_AUTO, smp_tsc, CTLFLAG_RDTUN, &smp_tsc, 0,
 TUNABLE_INT("kern.timecounter.smp_tsc", &smp_tsc);
 #endif
 
+static int	tsc_shift = 1;
+SYSCTL_INT(_kern_timecounter, OID_AUTO, tsc_shift, CTLFLAG_RDTUN,
+    &tsc_shift, 0, "Shift to pre-apply for the maximum TSC frequency");
+TUNABLE_INT("kern.timecounter.tsc_shift", &tsc_shift);
+
 static int	tsc_disabled;
 SYSCTL_INT(_machdep, OID_AUTO, disable_tsc, CTLFLAG_RDTUN, &tsc_disabled, 0,
     "Disable x86 Time Stamp Counter");
@@ -399,12 +404,12 @@ comp_smp_tsc(void *arg)
 }
 
 static int
-test_smp_tsc(void)
+test_tsc(void)
 {
 	uint64_t *data, *tsc;
 	u_int i, size;
 
-	if (!smp_tsc && !tsc_is_invariant)
+	if ((!smp_tsc && !tsc_is_invariant) || vm_guest)
 		return (-100);
 	size = (mp_maxid + 1) * 3;
 	data = malloc(sizeof(*data) * size * N, M_TEMP, M_WAITOK);
@@ -444,6 +449,19 @@ test_smp_tsc(void)
 
 #undef N
 
+#else
+
+/*
+ * The function is not called, it is provided to avoid linking failure
+ * on uniprocessor kernel.
+ */
+static int
+test_tsc(void)
+{
+
+	return (0);
+}
+
 #endif /* SMP */
 
 static void
@@ -492,41 +510,37 @@ init_TSC_tc(void)
 		goto init;
 	}
 
-#ifdef SMP
 	/*
-	 * We can not use the TSC in SMP mode unless the TSCs on all CPUs are
-	 * synchronized.  If the user is sure that the system has synchronized
-	 * TSCs, set kern.timecounter.smp_tsc tunable to a non-zero value.
-	 * We also limit the frequency even lower to avoid "temporal anomalies"
-	 * as much as possible.  The TSC seems unreliable in virtualized SMP
+	 * We can not use the TSC in SMP mode unless the TSCs on all CPUs
+	 * are synchronized.  If the user is sure that the system has
+	 * synchronized TSCs, set kern.timecounter.smp_tsc tunable to a
+	 * non-zero value.  The TSC seems unreliable in virtualized SMP
 	 * environments, so it is set to a negative quality in those cases.
 	 */
-	if (smp_cpus > 1) {
-		if (vm_guest != 0) {
-			tsc_timecounter.tc_quality = -100;
-		} else {
-			tsc_timecounter.tc_quality = test_smp_tsc();
-			max_freq >>= 8;
-		}
-	} else
-#endif
-	if (tsc_is_invariant)
+	if (mp_ncpus > 1)
+		tsc_timecounter.tc_quality = test_tsc();
+	else if (tsc_is_invariant)
 		tsc_timecounter.tc_quality = 1000;
+	max_freq >>= tsc_shift;
 
 init:
-	for (shift = 0; shift < 31 && (tsc_freq >> shift) > max_freq; shift++)
+	for (shift = 0; shift <= 31 && (tsc_freq >> shift) > max_freq; shift++)
 		;
+	if ((cpu_feature & CPUID_SSE2) != 0 && mp_ncpus > 1) {
+		if (cpu_vendor_id == CPU_VENDOR_AMD) {
+			tsc_timecounter.tc_get_timecount = shift > 0 ?
+			    tsc_get_timecount_low_mfence :
+			    tsc_get_timecount_mfence;
+		} else {
+			tsc_timecounter.tc_get_timecount = shift > 0 ?
+			    tsc_get_timecount_low_lfence :
+			    tsc_get_timecount_lfence;
+		}
+	} else {
+		tsc_timecounter.tc_get_timecount = shift > 0 ?
+		    tsc_get_timecount_low : tsc_get_timecount;
+	}
 	if (shift > 0) {
-		if (cpu_feature & CPUID_SSE2) {
-			if (cpu_vendor_id == CPU_VENDOR_AMD) {
-				tsc_timecounter.tc_get_timecount =
-				    tsc_get_timecount_low_mfence;
-			} else {
-				tsc_timecounter.tc_get_timecount =
-				    tsc_get_timecount_low_lfence;
-			}
-		} else
-			tsc_timecounter.tc_get_timecount = tsc_get_timecount_low;
 		tsc_timecounter.tc_name = "TSC-low";
 		if (bootverbose)
 			printf("TSC timecounter discards lower %d bit(s)\n",