Merge in support for Xen HVM on amd64 architecture.

13 files changed, 569 insertions, 250 deletions

diff --git a/sys/xen/evtchn/evtchn.c b/sys/xen/evtchn/evtchn.c
index 884270c..61b738b 100644
--- a/sys/xen/evtchn/evtchn.c
+++ b/sys/xen/evtchn/evtchn.c
@@ -13,56 +13,28 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
+#include <sys/limits.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/interrupt.h>
 #include <sys/pcpu.h>
+#include <sys/smp.h>
 
 #include <machine/cpufunc.h>
 #include <machine/intr_machdep.h>
+
 #include <machine/xen/xen-os.h>
+#include <machine/xen/xenvar.h>
 #include <xen/xen_intr.h>
 #include <machine/xen/synch_bitops.h>
 #include <xen/evtchn.h>
 #include <xen/hypervisor.h>
 #include <sys/smp.h>
 
-
-
-/* linux helper functions that got sucked in 
- * rename and move XXX
- */
-
-
-static inline int find_first_bit(const unsigned long *addr, unsigned size)
-{
-	int d0, d1;
-	int res;
-
-	/* This looks at memory. Mark it volatile to tell gcc not to move it around */
-	__asm__ __volatile__(
-		"xorl %%eax,%%eax\n\t"
-		"repe; scasl\n\t"
-		"jz 1f\n\t"
-		"leal -4(%%edi),%%edi\n\t"
-		"bsfl (%%edi),%%eax\n"
-		"1:\tsubl %%ebx,%%edi\n\t"
-		"shll $3,%%edi\n\t"
-		"addl %%edi,%%eax"
-		:"=a" (res), "=&c" (d0), "=&D" (d1)
-		:"1" ((size + 31) >> 5), "2" (addr), "b" (addr) : "memory");
-	return res;
-}
-
-#define min_t(type,x,y) \
-	({ type __x = (x); type __y = (y); __x < __y ? __x: __y; })
-#define first_cpu(src) __first_cpu(&(src), NR_CPUS)
-static inline int __first_cpu(const xen_cpumask_t *srcp, int nbits)
-{
-	return min_t(int, nbits, find_first_bit(srcp->bits, nbits));
-}
+#include <xen/xen_intr.h>
+#include <xen/evtchn.h>
 
 static inline unsigned long __ffs(unsigned long word)
 {
@@ -166,7 +138,7 @@ static int irq_bindcount[NR_IRQS];
 #ifdef SMP
 
 static uint8_t cpu_evtchn[NR_EVENT_CHANNELS];
-static unsigned long cpu_evtchn_mask[NR_CPUS][NR_EVENT_CHANNELS/BITS_PER_LONG];
+static unsigned long cpu_evtchn_mask[MAX_VIRT_CPUS][NR_EVENT_CHANNELS/LONG_BIT];
 
 #define active_evtchns(cpu,sh,idx)		\
 	((sh)->evtchn_pending[idx] &		\
@@ -220,7 +192,7 @@ evtchn_do_upcall(struct trapframe *frame)
 	shared_info_t *s;
 	vcpu_info_t   *vcpu_info;
 	
-	cpu = smp_processor_id();
+	cpu = PCPU_GET(cpuid);
 	s = HYPERVISOR_shared_info;
 	vcpu_info = &s->vcpu_info[cpu];
 
@@ -236,7 +208,7 @@ evtchn_do_upcall(struct trapframe *frame)
 		while ((l2 = active_evtchns(cpu, s, l1i)) != 0) {
 			l2i = __ffs(l2);
 
-			port = (l1i * BITS_PER_LONG) + l2i;
+			port = (l1i * LONG_BIT) + l2i;
 			if ((irq = evtchn_to_irq[port]) != -1) {
 				struct intsrc *isrc = intr_lookup_source(irq);
 				/* 
@@ -258,7 +230,7 @@ ipi_pcpu(unsigned int cpu, int vector)
 { 
         int irq;
 
-	irq = per_cpu(ipi_to_irq, cpu)[vector]; 
+	irq = PCPU_GET(ipi_to_irq[vector]);
 	
         notify_remote_via_irq(irq); 
 } 
@@ -310,11 +282,12 @@ bind_local_port_to_irq(unsigned int local_port)
 
         mtx_lock_spin(&irq_mapping_update_lock);
 
-        PANIC_IF(evtchn_to_irq[local_port] != -1);
-
+        KASSERT(evtchn_to_irq[local_port] == -1,
+	    ("evtchn_to_irq inconsistent"));
+	
         if ((irq = find_unbound_irq()) < 0) {
                 struct evtchn_close close = { .port = local_port };
-                PANIC_IF(HYPERVISOR_event_channel_op(EVTCHNOP_close, &close));
+                HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
 		
                 goto out;
         }
@@ -368,21 +341,20 @@ bind_virq_to_irq(unsigned int virq, unsigned int cpu)
 
 	mtx_lock_spin(&irq_mapping_update_lock);
 
-	if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1) {
+	if ((irq = pcpu_find(cpu)->pc_virq_to_irq[virq]) == -1) {
 		if ((irq = find_unbound_irq()) < 0)
 			goto out;
 
 		bind_virq.virq = virq;
 		bind_virq.vcpu = cpu;
-		PANIC_IF(HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
-			&bind_virq) != 0);
+		HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq);
 
 		evtchn = bind_virq.port;
 
 		evtchn_to_irq[evtchn] = irq;
 		irq_info[irq] = mk_irq_info(IRQT_VIRQ, virq, evtchn);
 
-		per_cpu(virq_to_irq, cpu)[virq] = irq;
+		pcpu_find(cpu)->pc_virq_to_irq[virq] = irq;
 
 		bind_evtchn_to_cpu(evtchn, cpu);
 	}
@@ -407,18 +379,18 @@ bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
 
 	mtx_lock_spin(&irq_mapping_update_lock);
 	
-	if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1) {
+	if ((irq = pcpu_find(cpu)->pc_ipi_to_irq[ipi]) == -1) {
 		if ((irq = find_unbound_irq()) < 0)
 			goto out;
 
 		bind_ipi.vcpu = cpu;
-		PANIC_IF(HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi) != 0);
+		HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
 		evtchn = bind_ipi.port;
 
 		evtchn_to_irq[evtchn] = irq;
 		irq_info[irq] = mk_irq_info(IRQT_IPI, ipi, evtchn);
 
-		per_cpu(ipi_to_irq, cpu)[ipi] = irq;
+		pcpu_find(cpu)->pc_ipi_to_irq[ipi] = irq;
 
 		bind_evtchn_to_cpu(evtchn, cpu);
 	}
@@ -432,24 +404,27 @@ out:
 }
 
 
-void 
+static void 
 unbind_from_irq(int irq)
 {
 	struct evtchn_close close;
 	int evtchn = evtchn_from_irq(irq);
+	int cpu;
 
 	mtx_lock_spin(&irq_mapping_update_lock);
 
 	if ((--irq_bindcount[irq] == 0) && VALID_EVTCHN(evtchn)) {
 		close.port = evtchn;
-		PANIC_IF(HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0);
+		HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
 
 		switch (type_from_irq(irq)) {
 		case IRQT_VIRQ:
-			per_cpu(virq_to_irq, cpu_from_evtchn(evtchn))[index_from_irq(irq)] = -1;
+			cpu = cpu_from_evtchn(evtchn);
+			pcpu_find(cpu)->pc_virq_to_irq[index_from_irq(irq)] = -1;
 			break;
 		case IRQT_IPI:
-			per_cpu(ipi_to_irq, cpu_from_evtchn(evtchn))[index_from_irq(irq)] = -1;
+			cpu = cpu_from_evtchn(evtchn);
+			pcpu_find(cpu)->pc_ipi_to_irq[index_from_irq(irq)] = -1;
 			break;
 		default:
 			break;
@@ -467,11 +442,8 @@ unbind_from_irq(int irq)
 
 int 
 bind_caller_port_to_irqhandler(unsigned int caller_port,
-			  const char *devname,
-			  driver_intr_t handler,
-			  void *arg,
-			  unsigned long irqflags,
-                          unsigned int *irqp)
+    const char *devname, driver_intr_t handler, void *arg,
+    unsigned long irqflags, unsigned int *irqp)
 {
 	unsigned int irq;
 	int error;
@@ -493,13 +465,9 @@ bind_caller_port_to_irqhandler(unsigned int caller_port,
 }
 
 int 
-bind_listening_port_to_irqhandler(
-	                  unsigned int remote_domain,
-			  const char *devname,
-			  driver_intr_t handler,
-			  void *arg,
-			  unsigned long irqflags,
-                          unsigned int *irqp)
+bind_listening_port_to_irqhandler(unsigned int remote_domain,
+    const char *devname, driver_intr_t handler, void *arg,
+    unsigned long irqflags, unsigned int *irqp)
 {
 	unsigned int irq;
 	int error;
@@ -519,14 +487,10 @@ bind_listening_port_to_irqhandler(
 }
 
 int 
-bind_interdomain_evtchn_to_irqhandler(
-	                    unsigned int remote_domain,
-			    unsigned int remote_port,
-			    const char *devname,
-			    driver_filter_t filter,
-			    driver_intr_t handler,
-			    unsigned long irqflags,
-			    unsigned int *irqp)
+bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain,
+    unsigned int remote_port, const char *devname,
+    driver_filter_t filter, driver_intr_t handler,
+    unsigned long irqflags, unsigned int *irqp)
 {
 	unsigned int irq;
 	int error;
@@ -546,14 +510,9 @@ bind_interdomain_evtchn_to_irqhandler(
 }
 
 int 
-bind_virq_to_irqhandler(unsigned int virq,
-			unsigned int cpu,
-			const char *devname,
-			driver_filter_t filter,
-			driver_intr_t handler,
-                        void *arg,
-                        unsigned long irqflags,
-                        unsigned int *irqp)
+bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
+    const char *devname, driver_filter_t filter, driver_intr_t handler,
+    unsigned long irqflags, unsigned int *irqp)
 {
 	unsigned int irq;
 	int error;
@@ -573,12 +532,9 @@ bind_virq_to_irqhandler(unsigned int virq,
 }
 
 int 
-bind_ipi_to_irqhandler(unsigned int ipi,
-		       unsigned int cpu,
-		       const char *devname,
-		       driver_filter_t filter,
-                       unsigned long irqflags,
-                       unsigned int *irqp)
+bind_ipi_to_irqhandler(unsigned int ipi, unsigned int cpu,
+    const char *devname, driver_filter_t filter,
+    unsigned long irqflags, unsigned int *irqp)
 {
 	unsigned int irq;
 	int error;
@@ -636,9 +592,9 @@ rebind_irq_to_cpu(unsigned irq, unsigned tcpu)
 
 }
 
-static void set_affinity_irq(unsigned irq, xen_cpumask_t dest)
+static void set_affinity_irq(unsigned irq, cpumask_t dest)
 {
-	unsigned tcpu = first_cpu(dest);
+	unsigned tcpu = ffs(dest) - 1;
 	rebind_irq_to_cpu(irq, tcpu);
 }
 #endif
@@ -656,13 +612,11 @@ static void     xenpic_dynirq_enable_source(struct intsrc *isrc);
 static void     xenpic_dynirq_disable_source(struct intsrc *isrc, int); 
 static void     xenpic_dynirq_eoi_source(struct intsrc *isrc); 
 static void     xenpic_dynirq_enable_intr(struct intsrc *isrc); 
-static void     xenpic_dynirq_disable_intr(struct intsrc *isrc); 
 
 static void     xenpic_pirq_enable_source(struct intsrc *isrc); 
 static void     xenpic_pirq_disable_source(struct intsrc *isrc, int); 
 static void     xenpic_pirq_eoi_source(struct intsrc *isrc); 
 static void     xenpic_pirq_enable_intr(struct intsrc *isrc); 
-static void     xenpic_pirq_disable_intr(struct intsrc *isrc); 
 
 
 static int      xenpic_vector(struct intsrc *isrc); 
@@ -677,7 +631,6 @@ struct pic xenpic_dynirq_template  =  {
 	.pic_disable_source	=	xenpic_dynirq_disable_source,
 	.pic_eoi_source		=	xenpic_dynirq_eoi_source, 
 	.pic_enable_intr	=	xenpic_dynirq_enable_intr, 
-	.pic_disable_intr	=	xenpic_dynirq_disable_intr, 
 	.pic_vector		=	xenpic_vector, 
 	.pic_source_pending	=	xenpic_source_pending,
 	.pic_suspend		=	xenpic_suspend, 
@@ -689,7 +642,6 @@ struct pic xenpic_pirq_template  =  {
 	.pic_disable_source	=	xenpic_pirq_disable_source,
 	.pic_eoi_source		=	xenpic_pirq_eoi_source, 
 	.pic_enable_intr	=	xenpic_pirq_enable_intr, 
-	.pic_disable_intr	=	xenpic_pirq_disable_intr, 
 	.pic_vector		=	xenpic_vector, 
 	.pic_source_pending	=	xenpic_source_pending,
 	.pic_suspend		=	xenpic_suspend, 
@@ -748,20 +700,6 @@ xenpic_dynirq_enable_intr(struct intsrc *isrc)
 }
 
 static void 
-xenpic_dynirq_disable_intr(struct intsrc *isrc)
-{
-	unsigned int irq;
-	struct xenpic_intsrc *xp;
-	
-	xp = (struct xenpic_intsrc *)isrc;	
-	mtx_lock_spin(&irq_mapping_update_lock);
-	xp->xp_masked = 1;
-	irq = xenpic_vector(isrc);
-	mask_evtchn(evtchn_from_irq(irq));
-	mtx_unlock_spin(&irq_mapping_update_lock);
-}
-
-static void 
 xenpic_dynirq_eoi_source(struct intsrc *isrc)
 {
 	unsigned int irq;
@@ -825,7 +763,7 @@ notify_remote_via_irq(int irq)
 	if (VALID_EVTCHN(evtchn))
 		notify_remote_via_evtchn(evtchn);
 	else
-		panic("invalid evtchn");
+		panic("invalid evtchn %d", irq);
 }
 
 /* required for support of physical devices */
@@ -899,32 +837,6 @@ xenpic_pirq_enable_intr(struct intsrc *isrc)
 }
 
 static void 
-xenpic_pirq_disable_intr(struct intsrc *isrc)
-{
-	unsigned int irq;
-	int evtchn;
-	struct evtchn_close close;
-			
-	mtx_lock_spin(&irq_mapping_update_lock);
-	irq = xenpic_vector(isrc);
-	evtchn = evtchn_from_irq(irq);
-
-	if (!VALID_EVTCHN(evtchn)) 
-		goto done;
-	
-	mask_evtchn(evtchn);
-
-	close.port = evtchn;
-	PANIC_IF(HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0);
-
-	bind_evtchn_to_cpu(evtchn, 0);
-	evtchn_to_irq[evtchn] = -1;
-	irq_info[irq] = IRQ_UNBOUND;
- done:
-	mtx_unlock_spin(&irq_mapping_update_lock);
-}
-
-static void 
 xenpic_pirq_enable_source(struct intsrc *isrc)
 {
 	int evtchn;
@@ -998,7 +910,7 @@ void
 unmask_evtchn(int port)
 {
 	shared_info_t *s = HYPERVISOR_shared_info;
-	unsigned int cpu = smp_processor_id();
+	unsigned int cpu = PCPU_GET(cpuid);
 	vcpu_info_t *vcpu_info = &s->vcpu_info[cpu];
 
 	/* Slow path (hypercall) if this is a non-local port. */
@@ -1016,7 +928,7 @@ unmask_evtchn(int port)
 	 * masked.
 	 */
 	if (synch_test_bit(port, &s->evtchn_pending) && 
-	    !synch_test_and_set_bit(port / BITS_PER_LONG,
+	    !synch_test_and_set_bit(port / LONG_BIT,
 				    &vcpu_info->evtchn_pending_sel)) {
 		vcpu_info->evtchn_upcall_pending = 1;
 		if (!vcpu_info->evtchn_upcall_mask)
@@ -1039,15 +951,21 @@ void irq_resume(void)
 		mask_evtchn(evtchn);
 
 	/* Check that no PIRQs are still bound. */
-	for (pirq = 0; pirq < NR_PIRQS; pirq++)
-		PANIC_IF(irq_info[pirq_to_irq(pirq)] != IRQ_UNBOUND);
+	for (pirq = 0; pirq < NR_PIRQS; pirq++) {
+		KASSERT(irq_info[pirq_to_irq(pirq)] == IRQ_UNBOUND,
+		    ("pirq_to_irq inconsistent"));
+	}
 
 	/* Secondary CPUs must have no VIRQ or IPI bindings. */
-	for (cpu = 1; cpu < NR_CPUS; cpu++) {
-		for (virq = 0; virq < NR_VIRQS; virq++)
-			PANIC_IF(per_cpu(virq_to_irq, cpu)[virq] != -1);
-		for (ipi = 0; ipi < NR_IPIS; ipi++)
-			PANIC_IF(per_cpu(ipi_to_irq, cpu)[ipi] != -1);
+	for (cpu = 1; cpu < MAX_VIRT_CPUS; cpu++) {
+		for (virq = 0; virq < NR_VIRQS; virq++) {
+			KASSERT(pcpu_find(cpu)->pc_virq_to_irq[virq] == -1,
+			    ("virq_to_irq inconsistent"));
+		}
+		for (ipi = 0; ipi < NR_IPIS; ipi++) {
+			KASSERT(pcpu_find(cpu)->pc_ipi_to_irq[ipi] == -1,
+			    ("ipi_to_irq inconsistent"));
+		}
 	}
 
 	/* No IRQ <-> event-channel mappings. */
@@ -1058,15 +976,16 @@ void irq_resume(void)
 
 	/* Primary CPU: rebind VIRQs automatically. */
 	for (virq = 0; virq < NR_VIRQS; virq++) {
-		if ((irq = per_cpu(virq_to_irq, 0)[virq]) == -1)
+		if ((irq = pcpu_find(0)->pc_virq_to_irq[virq]) == -1)
 			continue;
 
-		PANIC_IF(irq_info[irq] != mk_irq_info(IRQT_VIRQ, virq, 0));
+		KASSERT(irq_info[irq] == mk_irq_info(IRQT_VIRQ, virq, 0),
+		    ("irq_info inconsistent"));
 
 		/* Get a new binding from Xen. */
 		bind_virq.virq = virq;
 		bind_virq.vcpu = 0;
-		PANIC_IF(HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq) != 0);
+		HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq);
 		evtchn = bind_virq.port;
         
 		/* Record the new mapping. */
@@ -1079,15 +998,16 @@ void irq_resume(void)
 
 	/* Primary CPU: rebind IPIs automatically. */
 	for (ipi = 0; ipi < NR_IPIS; ipi++) {
-		if ((irq = per_cpu(ipi_to_irq, 0)[ipi]) == -1)
+		if ((irq = pcpu_find(0)->pc_ipi_to_irq[ipi]) == -1)
 			continue;
 
-		PANIC_IF(irq_info[irq] != mk_irq_info(IRQT_IPI, ipi, 0));
+		KASSERT(irq_info[irq] == mk_irq_info(IRQT_IPI, ipi, 0),
+		    ("irq_info inconsistent"));
 
 		/* Get a new binding from Xen. */
 		memset(&op, 0, sizeof(op));
 		bind_ipi.vcpu = 0;
-		PANIC_IF(HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi) != 0);
+		HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
 		evtchn = bind_ipi.port;
         
 		/* Record the new mapping. */
@@ -1111,9 +1031,9 @@ evtchn_init(void *dummy __unused)
          /* No VIRQ or IPI bindings. */
 	for (cpu = 0; cpu < mp_ncpus; cpu++) {
 		for (i = 0; i < NR_VIRQS; i++)
-			per_cpu(virq_to_irq, cpu)[i] = -1;
+			pcpu_find(cpu)->pc_virq_to_irq[i] = -1;
 		for (i = 0; i < NR_IPIS; i++)
-			per_cpu(ipi_to_irq, cpu)[i] = -1;
+			pcpu_find(cpu)->pc_ipi_to_irq[i] = -1;
 	}
 
 	/* No event-channel -> IRQ mappings. */
diff --git a/sys/xen/evtchn/evtchn_dev.c b/sys/xen/evtchn/evtchn_dev.c
index ea12860..4253d8a 100644
--- a/sys/xen/evtchn/evtchn_dev.c
+++ b/sys/xen/evtchn/evtchn_dev.c
@@ -23,8 +23,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/fcntl.h>
 #include <sys/ioccom.h>
 
-#include <machine/cpufunc.h>
-#include <machine/intr_machdep.h>
 #include <machine/xen/xen-os.h>
 #include <xen/xen_intr.h>
 #include <machine/bus.h>
@@ -234,14 +232,14 @@ evtchn_ioctl(struct cdev *dev, unsigned long cmd, caddr_t arg,
 		__evtchn_reset_buffer_ring();
 		break;
 	case EVTCHN_BIND:
-		if ( !synch_test_and_set_bit((int)arg, &bound_ports[0]) )
-			unmask_evtchn((int)arg);
+		if ( !synch_test_and_set_bit((uintptr_t)arg, &bound_ports[0]) )
+			unmask_evtchn((uintptr_t)arg);
 		else
 			rc = EINVAL;
 		break;
 	case EVTCHN_UNBIND:
-		if ( synch_test_and_clear_bit((int)arg, &bound_ports[0]) )
-			mask_evtchn((int)arg);
+		if ( synch_test_and_clear_bit((uintptr_t)arg, &bound_ports[0]) )
+			mask_evtchn((uintptr_t)arg);
 		else
 			rc = EINVAL;
 		break;
@@ -383,12 +381,12 @@ evtchn_dev_init(void *dummy __unused)
 	/* (DEVFS) automatically destroy the symlink with its destination. */
 	devfs_auto_unregister(evtchn_miscdev.devfs_handle, symlink_handle);
 #endif
-	printk("Event-channel device installed.\n");
+	if (bootverbose)
+		printf("Event-channel device installed.\n");
 
 	return 0;
 }
 
-
 SYSINIT(evtchn_dev_init, SI_SUB_DRIVERS, SI_ORDER_FIRST, evtchn_dev_init, NULL);
 
 
diff --git a/sys/xen/features.c b/sys/xen/features.c
index 876a7d1..f28fe04 100644
--- a/sys/xen/features.c
+++ b/sys/xen/features.c
@@ -1,10 +1,12 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/systm.h>
+
 #include <machine/xen/xen-os.h>
 #include <xen/hypervisor.h>
-#include <machine/xen/features.h>
+#include <xen/features.h>
 
 uint8_t xen_features[XENFEAT_NR_SUBMAPS * 32] /* __read_mostly */;
 
diff --git a/sys/xen/features.h b/sys/xen/features.h
new file mode 100644
index 0000000..b4cce2f
--- /dev/null
+++ b/sys/xen/features.h
@@ -0,0 +1,20 @@
+/******************************************************************************
+ * features.h
+ *
+ * Query the features reported by Xen.
+ *
+ * Copyright (c) 2006, Ian Campbell
+ */
+
+#ifndef __ASM_XEN_FEATURES_H__
+#define __ASM_XEN_FEATURES_H__
+
+#include <xen/interface/version.h>
+
+extern void setup_xen_features(void);
+
+extern uint8_t xen_features[XENFEAT_NR_SUBMAPS * 32];
+
+#define xen_feature(flag)	(xen_features[flag])
+
+#endif /* __ASM_XEN_FEATURES_H__ */
diff --git a/sys/xen/gnttab.c b/sys/xen/gnttab.c
index 9675655..d05790b 100644
--- a/sys/xen/gnttab.c
+++ b/sys/xen/gnttab.c
@@ -25,28 +25,20 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mman.h>
-#include <vm/vm.h>
-#include <vm/vm_extern.h>
-
-#include <vm/vm_page.h>
-#include <vm/vm_kern.h>
 
+#include <machine/xen/xen-os.h>
 #include <xen/hypervisor.h>
 #include <machine/xen/synch_bitops.h>
-#include <xen/gnttab.h>
 
-#define cmpxchg(a, b, c) atomic_cmpset_int((volatile u_int *)(a),(b),(c))
+#include <xen/hypervisor.h>
+#include <xen/gnttab.h>
 
-#if 1
-#define ASSERT(_p) \
-    if ( !(_p) ) { printk("Assertion '%s': line %d, file %s\n", \
-    #_p , __LINE__, __FILE__); *(int*)0=0; }
-#else
-#define ASSERT(_p) ((void)0)
-#endif
+#include <vm/vm.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_extern.h>
+#include <vm/pmap.h>
 
-#define WPRINTK(fmt, args...) \
-    printk("xen_grant: " fmt, ##args)
+#define cmpxchg(a, b, c) atomic_cmpset_int((volatile u_int *)(a),(b),(c))
 
 /* External tools reserve first few grant table entries. */
 #define NR_RESERVED_ENTRIES 8
@@ -72,14 +64,14 @@ static int gnttab_expand(unsigned int req_entries);
 static int
 get_free_entries(int count, int *entries)
 {
-	int ref, rc;
+	int ref, error;
 	grant_ref_t head;
 	
 	mtx_lock(&gnttab_list_lock);
 	if ((gnttab_free_count < count) &&
-	    ((rc = gnttab_expand(count - gnttab_free_count)) != 0)) {
+	    ((error = gnttab_expand(count - gnttab_free_count)) != 0)) {
 		mtx_unlock(&gnttab_list_lock);
-		return (rc);
+		return (error);
 	}
 	ref = head = gnttab_free_head;
 	gnttab_free_count -= count;
@@ -163,6 +155,7 @@ void
 gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
 				unsigned long frame, int readonly)
 {
+
 	shared[ref].frame = frame;
 	shared[ref].domid = domid;
 	wmb();
@@ -213,7 +206,8 @@ gnttab_end_foreign_access(grant_ref_t ref, void *page)
 }
 
 int
-gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
+gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn,
+    grant_ref_t *result)
 {
 	int error, ref;
 
@@ -223,7 +217,8 @@ gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
 
 	gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
 	
-	return (ref);
+	*result = ref;
+	return (0);
 }
 
 void
@@ -261,7 +256,7 @@ gnttab_end_foreign_transfer_ref(grant_ref_t ref)
 	/* Read the frame number /after/ reading completion status. */
 	rmb();
 	frame = shared[ref].frame;
-	PANIC_IF(frame == 0);
+	KASSERT(frame != 0, ("grant table inconsistent"));
 
 	return (frame);
 }
@@ -320,6 +315,7 @@ gnttab_alloc_grant_references(uint16_t count, grant_ref_t *head)
 int
 gnttab_empty_grant_references(const grant_ref_t *private_head)
 {
+
 	return (*private_head == GNTTAB_LIST_END);
 }
 
@@ -331,20 +327,20 @@ gnttab_claim_grant_reference(grant_ref_t *private_head)
 	if (unlikely(g == GNTTAB_LIST_END))
 		return (ENOSPC);
 	*private_head = gnttab_entry(g);
-
 	return (g);
 }
 
 void
 gnttab_release_grant_reference(grant_ref_t *private_head, grant_ref_t  release)
 {
+
 	gnttab_entry(release) = *private_head;
 	*private_head = release;
 }
 
 void
 gnttab_request_free_callback(struct gnttab_free_callback *callback,
-			     void (*fn)(void *), void *arg, uint16_t count)
+    void (*fn)(void *), void *arg, uint16_t count)
 {
 
 	mtx_lock(&gnttab_list_lock);
@@ -387,7 +383,8 @@ grow_gnttab_list(unsigned int more_frames)
 
 	for (i = nr_grant_frames; i < new_nr_grant_frames; i++)
 	{
-		gnttab_list[i] = (grant_ref_t *)malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
+		gnttab_list[i] = (grant_ref_t *)
+			malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
 
 		if (!gnttab_list[i])
 			goto grow_nomem;
@@ -405,12 +402,12 @@ grow_gnttab_list(unsigned int more_frames)
 
 	check_free_callbacks();
 
-	return 0;
+	return (0);
 	
 grow_nomem:
 	for ( ; i >= nr_grant_frames; i--)
 		free(gnttab_list[i], M_DEVBUF);
-	return (-ENOMEM);
+	return (ENOMEM);
 }
 
 static unsigned int
@@ -464,6 +461,8 @@ unmap_pte_fn(pte_t *pte, struct page *pmd_page,
 }
 #endif
 
+#ifndef XENHVM
+
 static int
 gnttab_map(unsigned int start_idx, unsigned int end_idx)
 {
@@ -486,50 +485,117 @@ gnttab_map(unsigned int start_idx, unsigned int end_idx)
 		free(frames, M_DEVBUF);
 		return (ENOSYS);
 	}
-	PANIC_IF(rc || setup.status);
+	KASSERT(!(rc || setup.status),
+	    ("unexpected result from grant_table_op"));
 
 	if (shared == NULL) {
 		vm_offset_t area;
 		
 		area = kmem_alloc_nofault(kernel_map,
 		    PAGE_SIZE * max_nr_grant_frames());
-		PANIC_IF(area == 0);
+		KASSERT(area, ("can't allocate VM space for grant table"));
 		shared = (grant_entry_t *)area;
 	}
+
 	for (i = 0; i < nr_gframes; i++)
 		PT_SET_MA(((caddr_t)shared) + i*PAGE_SIZE, 
 		    ((vm_paddr_t)frames[i]) << PAGE_SHIFT | PG_RW | PG_V);
 
 	free(frames, M_DEVBUF);
 	
-	return 0;
+	return (0);
 }
 
 int
 gnttab_resume(void)
 {
+
 	if (max_nr_grant_frames() < nr_grant_frames)
-		return -ENOSYS;
-	return gnttab_map(0, nr_grant_frames - 1);
+		return (ENOSYS);
+	return (gnttab_map(0, nr_grant_frames - 1));
 }
 
 int
 gnttab_suspend(void)
 {	
-	int i, pages;
+	int i;
+
+	for (i = 0; i < nr_grant_frames; i++)
+		pmap_kremove((vm_offset_t) shared + i * PAGE_SIZE);
+
+	return (0);
+}
+
+#else /* XENHVM */
 
-	pages = (PAGE_SIZE*nr_grant_frames) >> PAGE_SHIFT;
+#include <dev/xen/xenpci/xenpcivar.h>
 
-	for (i = 0; i < pages; i++)
-		PT_SET_MA(shared + (i*PAGE_SIZE), (vm_paddr_t)0);
+static vm_paddr_t resume_frames;
+
+static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
+{
+	struct xen_add_to_physmap xatp;
+	unsigned int i = end_idx;
+
+	/*
+	 * Loop backwards, so that the first hypercall has the largest index,
+	 * ensuring that the table will grow only once.
+	 */
+	do {
+		xatp.domid = DOMID_SELF;
+		xatp.idx = i;
+		xatp.space = XENMAPSPACE_grant_table;
+		xatp.gpfn = (resume_frames >> PAGE_SHIFT) + i;
+		if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
+			panic("HYPERVISOR_memory_op failed to map gnttab");
+	} while (i-- > start_idx);
+
+	if (shared == NULL) {
+		vm_offset_t area;
+		
+		area = kmem_alloc_nofault(kernel_map,
+		    PAGE_SIZE * max_nr_grant_frames());
+		KASSERT(area, ("can't allocate VM space for grant table"));
+		shared = (grant_entry_t *)area;
+	}
+
+	for (i = start_idx; i <= end_idx; i++) {
+		pmap_kenter((vm_offset_t) shared + i * PAGE_SIZE,
+		    resume_frames + i * PAGE_SIZE);
+	}
 
 	return (0);
 }
 
+int
+gnttab_resume(void)
+{
+	int error;
+	unsigned int max_nr_gframes, nr_gframes;
+
+	nr_gframes = nr_grant_frames;
+	max_nr_gframes = max_nr_grant_frames();
+	if (max_nr_gframes < nr_gframes)
+		return (ENOSYS);
+
+	if (!resume_frames) {
+		error = xenpci_alloc_space(PAGE_SIZE * max_nr_gframes,
+		    &resume_frames);
+		if (error) {
+			printf("error mapping gnttab share frames\n");
+			return (error);
+		}
+	}
+
+	return (gnttab_map(0, nr_gframes - 1));
+}
+
+#endif
+
 static int
 gnttab_expand(unsigned int req_entries)
 {
-	int rc;
+	int error;
 	unsigned int cur, extra;
 
 	cur = nr_grant_frames;
@@ -538,10 +604,11 @@ gnttab_expand(unsigned int req_entries)
 	if (cur + extra > max_nr_grant_frames())
 		return (ENOSPC);
 
-	if ((rc = gnttab_map(cur, cur + extra - 1)) == 0)
-		rc = grow_gnttab_list(extra);
+	error = gnttab_map(cur, cur + extra - 1);
+	if (!error)
+		error = grow_gnttab_list(extra);
 
-	return rc;
+	return (error);
 }
 
 int 
@@ -552,7 +619,7 @@ gnttab_init()
 	unsigned int nr_init_grefs;
 
 	if (!is_running_on_xen())
-		return -ENODEV;
+		return (ENODEV);
 
 	nr_grant_frames = 1;
 	boot_max_nr_grant_frames = __max_nr_grant_frames();
@@ -571,7 +638,8 @@ gnttab_init()
 		return (ENOMEM);
 
 	for (i = 0; i < nr_grant_frames; i++) {
-		gnttab_list[i] = (grant_ref_t *)malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
+		gnttab_list[i] = (grant_ref_t *)
+			malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
 		if (gnttab_list[i] == NULL)
 			goto ini_nomem;
 	}
@@ -588,8 +656,10 @@ gnttab_init()
 	gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
 	gnttab_free_head  = NR_RESERVED_ENTRIES;
 
-	printk("Grant table initialized\n");
-	return 0;
+	if (bootverbose)
+		printf("Grant table initialized\n");
+
+	return (0);
 
 ini_nomem:
 	for (i--; i >= 0; i--)
diff --git a/sys/xen/gnttab.h b/sys/xen/gnttab.h
index bcefbbc..8348af5 100644
--- a/sys/xen/gnttab.h
+++ b/sys/xen/gnttab.h
@@ -36,10 +36,12 @@
 
 #ifndef __ASM_GNTTAB_H__
 
+#include <xen/interface/grant_table.h>
+
 #include <xen/hypervisor.h>
 #include <xen/interface/grant_table.h>
 #include <machine/xen/xen-os.h>
-#include <machine/xen/features.h>
+#include <xen/features.h>
 
 struct gnttab_free_callback {
 	struct gnttab_free_callback *next;
@@ -50,6 +52,10 @@ struct gnttab_free_callback {
 
 int gnttab_init(void);
 
+/*
+ * Allocate a grant table reference and return it in *result. Returns
+ * zero on success or errno on error.
+ */
 int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
     int flags, grant_ref_t *result);
 
@@ -68,7 +74,7 @@ int gnttab_end_foreign_access_ref(grant_ref_t ref);
  */
 void gnttab_end_foreign_access(grant_ref_t ref, void *page);
 
-int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn);
+int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn, grant_ref_t *result);
 
 unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref);
 unsigned long gnttab_end_foreign_transfer(grant_ref_t ref);
@@ -104,6 +110,10 @@ void gnttab_grant_foreign_transfer_ref(grant_ref_t, domid_t domid,
 int gnttab_suspend(void);
 int gnttab_resume(void);
 
+#if 0
+
+#include <xen/features.h>
+
 static inline void
 gnttab_set_map_op(struct gnttab_map_grant_ref *map, vm_paddr_t addr,
 		  uint32_t flags, grant_ref_t ref, domid_t domid)
@@ -149,5 +159,6 @@ gnttab_set_replace_op(struct gnttab_unmap_and_replace *unmap, vm_paddr_t addr,
 
 	unmap->handle = handle;
 }
+#endif
 
 #endif /* __ASM_GNTTAB_H__ */
diff --git a/sys/xen/hypervisor.h b/sys/xen/hypervisor.h
index 369b0c4..0d93f66 100644
--- a/sys/xen/hypervisor.h
+++ b/sys/xen/hypervisor.h
@@ -8,11 +8,19 @@
  * $FreeBSD$
  */
 
-#ifndef __HYPERVISOR_H__
-#define __HYPERVISOR_H__
+#ifndef __XEN_HYPERVISOR_H__
+#define __XEN_HYPERVISOR_H__
+
+#ifdef XENHVM
+
+#define is_running_on_xen()	(HYPERVISOR_shared_info != NULL)
+
+#else
 
 #define is_running_on_xen() 1
 
+#endif
+
 #ifdef PAE
 #ifndef CONFIG_X86_PAE
 #define CONFIG_X86_PAE
@@ -27,6 +35,7 @@
 #include <xen/interface/physdev.h>
 #include <xen/interface/sched.h>
 #include <xen/interface/callback.h>
+#include <xen/interface/memory.h>
 #include <machine/xen/hypercall.h>
 
 #if defined(__amd64__)
@@ -131,7 +140,7 @@ MULTI_update_va_mapping(
     mcl->op = __HYPERVISOR_update_va_mapping;
     mcl->args[0] = va;
 #if defined(__amd64__)
-    mcl->args[1] = new_val.pte;
+    mcl->args[1] = new_val;
 #elif defined(PAE)
     mcl->args[1] = (uint32_t)(new_val & 0xffffffff) ;
     mcl->args[2] = (uint32_t)(new_val >> 32);
@@ -142,4 +151,4 @@ MULTI_update_va_mapping(
     mcl->args[MULTI_UVMFLAGS_INDEX] = flags;
 }
 
-#endif /* __HYPERVISOR_H__ */
+#endif /* __XEN_HYPERVISOR_H__ */
diff --git a/sys/xen/interface/arch-x86/xen.h b/sys/xen/interface/arch-x86/xen.h
index 038048e..2c878ef 100644
--- a/sys/xen/interface/arch-x86/xen.h
+++ b/sys/xen/interface/arch-x86/xen.h
@@ -32,7 +32,8 @@
 #define ___DEFINE_XEN_GUEST_HANDLE(name, type) \
     typedef struct { type *p; } __guest_handle_ ## name
 #else
-#define ___DEFINE_XEN_GUEST_HANDLE(name, type) \
+#error "using old handle"
+#define ___DEFINE_XEN_GUEST_HANDLE(name, type)  \
     typedef type * __guest_handle_ ## name
 #endif
 
@@ -50,7 +51,7 @@
 #if defined(__i386__)
 #include <xen/interface/arch-x86/xen-x86_32.h>
 #elif defined(__x86_64__)
-#include "xen-x86_64.h"
+#include <xen/interface/arch-x86/xen-x86_64.h>
 #endif
 
 #ifndef __ASSEMBLY__
diff --git a/sys/xen/interface/hvm/params.h b/sys/xen/interface/hvm/params.h
index 5f75ed7..6befa78 100644
--- a/sys/xen/interface/hvm/params.h
+++ b/sys/xen/interface/hvm/params.h
@@ -21,7 +21,7 @@
 #ifndef __XEN_PUBLIC_HVM_PARAMS_H__
 #define __XEN_PUBLIC_HVM_PARAMS_H__
 
-#include "hvm_op.h"
+#include <xen/interface/hvm/hvm_op.h>
 
 /*
  * Parameter space for HVMOP_{set,get}_param.
diff --git a/sys/xen/reboot.c b/sys/xen/reboot.c
new file mode 100644
index 0000000..892dfbf
--- /dev/null
+++ b/sys/xen/reboot.c
@@ -0,0 +1,262 @@
+/*
+ *
+ * Copyright (c) 2004 Christian Limpach.
+ * Copyright (c) 2004-2006,2008 Kip Macy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *      This product includes software developed by Christian Limpach.
+ * 4. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/malloc.h>
+#include <sys/kernel.h>
+#include <sys/proc.h>
+#include <sys/reboot.h>
+#include <sys/sched.h>
+#include <sys/smp.h>
+#include <sys/systm.h>
+
+#include <machine/xen/xen-os.h>
+#include <xen/hypervisor.h>
+#include <xen/gnttab.h>
+#include <xen/xen_intr.h>
+#include <xen/xenbus/xenbusvar.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#ifdef XENHVM
+
+#include <dev/xen/xenpci/xenpcivar.h>
+
+#else
+
+static void xen_suspend(void);
+
+#endif
+
+static void 
+shutdown_handler(struct xenbus_watch *watch,
+		 const char **vec, unsigned int len)
+{
+	char *str;
+	struct xenbus_transaction xbt;
+	int error, howto;
+	
+	howto = 0;
+
+ again:
+	error = xenbus_transaction_start(&xbt);
+	if (error)
+		return;
+
+	error = xenbus_read(xbt, "control", "shutdown", NULL, (void **) &str);
+
+	/* Ignore read errors and empty reads. */
+	if (error || strlen(str) == 0) {
+		xenbus_transaction_end(xbt, 1);
+		return;
+	}
+
+	xenbus_write(xbt, "control", "shutdown", "");
+
+	error = xenbus_transaction_end(xbt, 0);
+	if (error == EAGAIN) {
+		free(str, M_DEVBUF);
+		goto again;
+	}
+
+	if (strcmp(str, "reboot") == 0)
+		howto = 0;
+	else if (strcmp(str, "poweroff") == 0)
+		howto |= (RB_POWEROFF | RB_HALT);
+	else if (strcmp(str, "halt") == 0)
+#ifdef XENHVM
+		/*
+		 * We rely on acpi powerdown to halt the VM.
+		 */
+		howto |= (RB_POWEROFF | RB_HALT);
+#else
+		howto |= RB_HALT;
+#endif
+	else if (strcmp(str, "suspend") == 0)
+		howto = -1;
+	else {
+		printf("Ignoring shutdown request: %s\n", str);
+		goto done;
+	}
+
+	if (howto == -1) {
+		xen_suspend();
+		goto done;
+	}
+
+	shutdown_nice(howto);
+ done:
+	free(str, M_DEVBUF);
+}
+
+#ifndef XENHVM
+
+/*
+ * In HV mode, we let acpi take care of halts and reboots.
+ */
+
+static void
+xen_shutdown_final(void *arg, int howto)
+{
+
+	if (howto & (RB_HALT | RB_POWEROFF))
+		HYPERVISOR_shutdown(SHUTDOWN_poweroff);
+	else
+		HYPERVISOR_shutdown(SHUTDOWN_reboot);
+}
+
+#endif
+
+static struct xenbus_watch shutdown_watch = {
+	.node = "control/shutdown",
+	.callback = shutdown_handler
+};
+
+static void
+setup_shutdown_watcher(void *unused)
+{
+
+	if (register_xenbus_watch(&shutdown_watch))
+		printf("Failed to set shutdown watcher\n");
+#ifndef XENHVM
+	EVENTHANDLER_REGISTER(shutdown_final, xen_shutdown_final, NULL,
+	    SHUTDOWN_PRI_LAST);
+#endif
+}
+
+SYSINIT(shutdown, SI_SUB_PSEUDO, SI_ORDER_ANY, setup_shutdown_watcher, NULL);
+
+#ifndef XENHVM
+
+extern void xencons_suspend(void);
+extern void xencons_resume(void);
+
+static void 
+xen_suspend()
+{
+	int i, j, k, fpp;
+	unsigned long max_pfn, start_info_mfn;
+
+#ifdef SMP
+	cpumask_t map;
+	/*
+	 * Bind us to CPU 0 and stop any other VCPUs.
+	 */
+	mtx_lock_spin(&sched_lock);
+	sched_bind(curthread, 0);
+	mtx_unlock_spin(&sched_lock);
+	KASSERT(PCPU_GET(cpuid) == 0, ("xen_suspend: not running on cpu 0"));
+
+	map = PCPU_GET(other_cpus) & ~stopped_cpus;
+	if (map)
+		stop_cpus(map);
+#endif
+
+	if (DEVICE_SUSPEND(root_bus) != 0) {
+		printf("xen_suspend: device_suspend failed\n");
+		if (map)
+			restart_cpus(map);
+		return;
+	}
+
+	local_irq_disable();
+
+	xencons_suspend();
+	gnttab_suspend();
+
+	max_pfn = HYPERVISOR_shared_info->arch.max_pfn;
+
+	void *shared_info = HYPERVISOR_shared_info;
+	HYPERVISOR_shared_info = NULL;
+	pmap_kremove((vm_offset_t) shared_info);
+	PT_UPDATES_FLUSH();
+
+	xen_start_info->store_mfn = MFNTOPFN(xen_start_info->store_mfn);
+	xen_start_info->console.domU.mfn = MFNTOPFN(xen_start_info->console.domU.mfn);
+
+	/*
+	 * We'll stop somewhere inside this hypercall. When it returns,
+	 * we'll start resuming after the restore.
+	 */
+	start_info_mfn = VTOMFN(xen_start_info);
+	pmap_suspend();
+	HYPERVISOR_suspend(start_info_mfn);
+	pmap_resume();
+
+	pmap_kenter_ma((vm_offset_t) shared_info, xen_start_info->shared_info);
+	HYPERVISOR_shared_info = shared_info;
+
+	HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list =
+		VTOMFN(xen_pfn_to_mfn_frame_list_list);
+  
+	fpp = PAGE_SIZE/sizeof(unsigned long);
+	for (i = 0, j = 0, k = -1; i < max_pfn; i += fpp, j++) {
+		if ((j % fpp) == 0) {
+			k++;
+			xen_pfn_to_mfn_frame_list_list[k] = 
+				VTOMFN(xen_pfn_to_mfn_frame_list[k]);
+			j = 0;
+		}
+		xen_pfn_to_mfn_frame_list[k][j] = 
+			VTOMFN(&xen_phys_machine[i]);
+	}
+	HYPERVISOR_shared_info->arch.max_pfn = max_pfn;
+
+	gnttab_resume();
+	irq_resume();
+	local_irq_enable();
+	xencons_resume();
+
+#ifdef CONFIG_SMP
+	for_each_cpu(i)
+		vcpu_prepare(i);
+
+#endif
+	/* 
+	 * Only resume xenbus /after/ we've prepared our VCPUs; otherwise
+	 * the VCPU hotplug callback can race with our vcpu_prepare
+	 */
+	DEVICE_RESUME(root_bus);
+
+#ifdef SMP
+	sched_unbind(curthread);
+	if (map)
+		restart_cpus(map);
+#endif
+}
+
+#endif
diff --git a/sys/xen/xen_intr.h b/sys/xen/xen_intr.h
index 528fa7f..68f5943 100644
--- a/sys/xen/xen_intr.h
+++ b/sys/xen/xen_intr.h
@@ -29,37 +29,63 @@
 #define dynirq_to_irq(_x) ((_x) + DYNIRQ_BASE)
 #define irq_to_dynirq(_x) ((_x) - DYNIRQ_BASE)
 
-/* Dynamic binding of event channels and VIRQ sources to Linux IRQ space. */
-extern void unbind_from_irq(int irq);
+/* 
+ * Dynamic binding of event channels and VIRQ sources to guest IRQ space.
+ */
 
+/*
+ * Bind a caller port event channel to an interrupt handler. If
+ * successful, the guest IRQ number is returned in *irqp. Return zero
+ * on success or errno otherwise.
+ */
 extern int bind_caller_port_to_irqhandler(unsigned int caller_port,
 	const char *devname, driver_intr_t handler, void *arg,
 	unsigned long irqflags, unsigned int *irqp);
+
+/*
+ * Bind a listening port to an interrupt handler. If successful, the
+ * guest IRQ number is returned in *irqp. Return zero on success or
+ * errno otherwise.
+ */
 extern int bind_listening_port_to_irqhandler(unsigned int remote_domain,
-	const char *devname, driver_intr_t handler, void *arg, unsigned long irqflags,
-	unsigned int *irqp);
+	const char *devname, driver_intr_t handler, void *arg,
+	unsigned long irqflags, unsigned int *irqp);
+
+/*
+ * Bind a VIRQ to an interrupt handler. If successful, the guest IRQ
+ * number is returned in *irqp. Return zero on success or errno
+ * otherwise.
+ */
 extern int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
 	const char *devname, driver_filter_t filter, driver_intr_t handler,
 	void *arg, unsigned long irqflags,	unsigned int *irqp);
-extern int bind_ipi_to_irqhandler(unsigned int ipi,
-	unsigned int cpu,
-	const char *devname,
-	driver_filter_t handler,
-	unsigned long irqflags,
-	unsigned int *irqp);
 
-extern int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain,
-	                                             unsigned int remote_port,
-	                                             const char *devname,
-	                                             driver_filter_t filter,
-	                                             driver_intr_t handler,
-	                                             unsigned long irqflags,
-	                                             unsigned int *irqp);
+/*
+ * Bind an IPI to an interrupt handler. If successful, the guest
+ * IRQ number is returned in *irqp. Return zero on success or errno
+ * otherwise.
+ */
+extern int bind_ipi_to_irqhandler(unsigned int ipi, unsigned int cpu,
+	const char *devname, driver_filter_t filter,
+	unsigned long irqflags, unsigned int *irqp);
 
+/*
+ * Bind an interdomain event channel to an interrupt handler. If
+ * successful, the guest IRQ number is returned in *irqp. Return zero
+ * on success or errno otherwise.
+ */
+extern int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain,
+	unsigned int remote_port, const char *devname,
+	driver_filter_t filter, driver_intr_t handler,
+	unsigned long irqflags, unsigned int *irqp);
 
+/*
+ * Unbind an interrupt handler using the guest IRQ number returned
+ * when it was bound.
+ */
+extern void unbind_from_irqhandler(unsigned int irq);
 
-extern void unbind_from_irqhandler(unsigned int evtchn);
-static __inline__ int irq_cannonicalize(int irq)
+static __inline__ int irq_cannonicalize(unsigned int irq)
 {
     return (irq == 2) ? 9 : irq;
 }
diff --git a/sys/xen/xenbus/xenbus_probe.c b/sys/xen/xenbus/xenbus_probe.c
index 3d2cb4b..f04f8ec 100644
--- a/sys/xen/xenbus/xenbus_probe.c
+++ b/sys/xen/xenbus/xenbus_probe.c
@@ -565,7 +565,6 @@ xenbus_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
 	return (ENOENT);
 }
 
-SYSCTL_DECL(_dev);
 SYSCTL_NODE(_dev, OID_AUTO, xen, CTLFLAG_RD, NULL, "Xen");
 SYSCTL_INT(_dev_xen, OID_AUTO, xsd_port, CTLFLAG_RD, &xen_store_evtchn, 0, "");
 SYSCTL_ULONG(_dev_xen, OID_AUTO, xsd_kva, CTLFLAG_RD, (u_long *) &xen_store, 0, "");
diff --git a/sys/xen/xenbus/xenbus_xs.c b/sys/xen/xenbus/xenbus_xs.c
index 9e0f779..8069556 100644
--- a/sys/xen/xenbus/xenbus_xs.c
+++ b/sys/xen/xenbus/xenbus_xs.c
@@ -142,21 +142,17 @@ xs_read_reply(enum xsd_sockmsg_type *type, unsigned int *len, void **result)
 	mtx_lock(&xs_state.reply_lock);
 
 	while (TAILQ_EMPTY(&xs_state.reply_list)) {
-			while (TAILQ_EMPTY(&xs_state.reply_list)) {
-				error = mtx_sleep(&xs_state.reply_waitq,
-				    &xs_state.reply_lock,
-				    PCATCH, "xswait", hz/10);
-				if (error && error != EWOULDBLOCK) {
-					mtx_unlock(&xs_state.reply_lock);
-					return (error);
-				}
-				
+		while (TAILQ_EMPTY(&xs_state.reply_list)) {
+			error = mtx_sleep(&xs_state.reply_waitq,
+			    &xs_state.reply_lock,
+			    PCATCH, "xswait", hz/10);
+			if (error && error != EWOULDBLOCK) {
+				mtx_unlock(&xs_state.reply_lock);
+				return (error);
 			}
-				
-			
 		}
+	}
 
-		
 	msg = TAILQ_FIRST(&xs_state.reply_list);
 	TAILQ_REMOVE(&xs_state.reply_list, msg, list);
 
@@ -202,7 +198,8 @@ xenbus_dev_request_and_reply(struct xsd_sockmsg *msg, void **result)
 
 	sx_xlock(&xs_state.request_mutex);
 
-	error = xb_write(msg, sizeof(*msg) + msg->len, &xs_state.request_mutex.lock_object);
+	error = xb_write(msg, sizeof(*msg) + msg->len,
+	    &xs_state.request_mutex.lock_object);
 	if (error) {
 		msg->type = XS_ERROR;
 	} else {
@@ -243,7 +240,8 @@ xs_talkv(struct xenbus_transaction t, enum xsd_sockmsg_type type,
 
 	sx_xlock(&xs_state.request_mutex);
 
-	error = xb_write(&msg, sizeof(msg), &xs_state.request_mutex.lock_object);
+	error = xb_write(&msg, sizeof(msg),
+	    &xs_state.request_mutex.lock_object);
 	if (error) {
 		sx_xunlock(&xs_state.request_mutex);
 		printf("xs_talkv failed %d\n", error);
@@ -251,7 +249,8 @@ xs_talkv(struct xenbus_transaction t, enum xsd_sockmsg_type type,
 	}
 
 	for (i = 0; i < num_vecs; i++) {
-		error = xb_write(iovec[i].iov_base, iovec[i].iov_len, &xs_state.request_mutex.lock_object);
+		error = xb_write(iovec[i].iov_base, iovec[i].iov_len,
+		    &xs_state.request_mutex.lock_object);
 		if (error) {		
 			sx_xunlock(&xs_state.request_mutex);
 			printf("xs_talkv failed %d\n", error);
@@ -791,7 +790,8 @@ xs_process_msg(enum xsd_sockmsg_type *type)
 		
 	msg = malloc(sizeof(*msg), M_DEVBUF, M_WAITOK);
 	mtx_lock(&xs_state.reply_lock);
-	error = xb_read(&msg->hdr, sizeof(msg->hdr), &xs_state.reply_lock.lock_object);
+	error = xb_read(&msg->hdr, sizeof(msg->hdr),
+	    &xs_state.reply_lock.lock_object);
 	mtx_unlock(&xs_state.reply_lock);
 	if (error) {
 		free(msg, M_DEVBUF);
@@ -800,7 +800,8 @@ xs_process_msg(enum xsd_sockmsg_type *type)
 
 	body = malloc(msg->hdr.len + 1, M_DEVBUF, M_WAITOK);
 	mtx_lock(&xs_state.reply_lock);
-	error = xb_read(body, msg->hdr.len, &xs_state.reply_lock.lock_object); 
+	error = xb_read(body, msg->hdr.len,
+	    &xs_state.reply_lock.lock_object); 
 	mtx_unlock(&xs_state.reply_lock);
 	if (error) {
 		free(body, M_DEVBUF);