Merge in support for Xen HVM on amd64 architecture.

2 files changed, 76 insertions, 158 deletions

diff --git a/sys/xen/evtchn/evtchn.c b/sys/xen/evtchn/evtchn.c
index 884270c..61b738b 100644
--- a/sys/xen/evtchn/evtchn.c
+++ b/sys/xen/evtchn/evtchn.c
@@ -13,56 +13,28 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
+#include <sys/limits.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/interrupt.h>
 #include <sys/pcpu.h>
+#include <sys/smp.h>
 
 #include <machine/cpufunc.h>
 #include <machine/intr_machdep.h>
+
 #include <machine/xen/xen-os.h>
+#include <machine/xen/xenvar.h>
 #include <xen/xen_intr.h>
 #include <machine/xen/synch_bitops.h>
 #include <xen/evtchn.h>
 #include <xen/hypervisor.h>
 #include <sys/smp.h>
 
-
-
-/* linux helper functions that got sucked in 
- * rename and move XXX
- */
-
-
-static inline int find_first_bit(const unsigned long *addr, unsigned size)
-{
-	int d0, d1;
-	int res;
-
-	/* This looks at memory. Mark it volatile to tell gcc not to move it around */
-	__asm__ __volatile__(
-		"xorl %%eax,%%eax\n\t"
-		"repe; scasl\n\t"
-		"jz 1f\n\t"
-		"leal -4(%%edi),%%edi\n\t"
-		"bsfl (%%edi),%%eax\n"
-		"1:\tsubl %%ebx,%%edi\n\t"
-		"shll $3,%%edi\n\t"
-		"addl %%edi,%%eax"
-		:"=a" (res), "=&c" (d0), "=&D" (d1)
-		:"1" ((size + 31) >> 5), "2" (addr), "b" (addr) : "memory");
-	return res;
-}
-
-#define min_t(type,x,y) \
-	({ type __x = (x); type __y = (y); __x < __y ? __x: __y; })
-#define first_cpu(src) __first_cpu(&(src), NR_CPUS)
-static inline int __first_cpu(const xen_cpumask_t *srcp, int nbits)
-{
-	return min_t(int, nbits, find_first_bit(srcp->bits, nbits));
-}
+#include <xen/xen_intr.h>
+#include <xen/evtchn.h>
 
 static inline unsigned long __ffs(unsigned long word)
 {
@@ -166,7 +138,7 @@ static int irq_bindcount[NR_IRQS];
 #ifdef SMP
 
 static uint8_t cpu_evtchn[NR_EVENT_CHANNELS];
-static unsigned long cpu_evtchn_mask[NR_CPUS][NR_EVENT_CHANNELS/BITS_PER_LONG];
+static unsigned long cpu_evtchn_mask[MAX_VIRT_CPUS][NR_EVENT_CHANNELS/LONG_BIT];
 
 #define active_evtchns(cpu,sh,idx)		\
 	((sh)->evtchn_pending[idx] &		\
@@ -220,7 +192,7 @@ evtchn_do_upcall(struct trapframe *frame)
 	shared_info_t *s;
 	vcpu_info_t   *vcpu_info;
 	
-	cpu = smp_processor_id();
+	cpu = PCPU_GET(cpuid);
 	s = HYPERVISOR_shared_info;
 	vcpu_info = &s->vcpu_info[cpu];
 
@@ -236,7 +208,7 @@ evtchn_do_upcall(struct trapframe *frame)
 		while ((l2 = active_evtchns(cpu, s, l1i)) != 0) {
 			l2i = __ffs(l2);
 
-			port = (l1i * BITS_PER_LONG) + l2i;
+			port = (l1i * LONG_BIT) + l2i;
 			if ((irq = evtchn_to_irq[port]) != -1) {
 				struct intsrc *isrc = intr_lookup_source(irq);
 				/* 
@@ -258,7 +230,7 @@ ipi_pcpu(unsigned int cpu, int vector)
 { 
         int irq;
 
-	irq = per_cpu(ipi_to_irq, cpu)[vector]; 
+	irq = PCPU_GET(ipi_to_irq[vector]);
 	
         notify_remote_via_irq(irq); 
 } 
@@ -310,11 +282,12 @@ bind_local_port_to_irq(unsigned int local_port)
 
         mtx_lock_spin(&irq_mapping_update_lock);
 
-        PANIC_IF(evtchn_to_irq[local_port] != -1);
-
+        KASSERT(evtchn_to_irq[local_port] == -1,
+	    ("evtchn_to_irq inconsistent"));
+	
         if ((irq = find_unbound_irq()) < 0) {
                 struct evtchn_close close = { .port = local_port };
-                PANIC_IF(HYPERVISOR_event_channel_op(EVTCHNOP_close, &close));
+                HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
 		
                 goto out;
         }
@@ -368,21 +341,20 @@ bind_virq_to_irq(unsigned int virq, unsigned int cpu)
 
 	mtx_lock_spin(&irq_mapping_update_lock);
 
-	if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1) {
+	if ((irq = pcpu_find(cpu)->pc_virq_to_irq[virq]) == -1) {
 		if ((irq = find_unbound_irq()) < 0)
 			goto out;
 
 		bind_virq.virq = virq;
 		bind_virq.vcpu = cpu;
-		PANIC_IF(HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
-			&bind_virq) != 0);
+		HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq);
 
 		evtchn = bind_virq.port;
 
 		evtchn_to_irq[evtchn] = irq;
 		irq_info[irq] = mk_irq_info(IRQT_VIRQ, virq, evtchn);
 
-		per_cpu(virq_to_irq, cpu)[virq] = irq;
+		pcpu_find(cpu)->pc_virq_to_irq[virq] = irq;
 
 		bind_evtchn_to_cpu(evtchn, cpu);
 	}
@@ -407,18 +379,18 @@ bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
 
 	mtx_lock_spin(&irq_mapping_update_lock);
 	
-	if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1) {
+	if ((irq = pcpu_find(cpu)->pc_ipi_to_irq[ipi]) == -1) {
 		if ((irq = find_unbound_irq()) < 0)
 			goto out;
 
 		bind_ipi.vcpu = cpu;
-		PANIC_IF(HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi) != 0);
+		HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
 		evtchn = bind_ipi.port;
 
 		evtchn_to_irq[evtchn] = irq;
 		irq_info[irq] = mk_irq_info(IRQT_IPI, ipi, evtchn);
 
-		per_cpu(ipi_to_irq, cpu)[ipi] = irq;
+		pcpu_find(cpu)->pc_ipi_to_irq[ipi] = irq;
 
 		bind_evtchn_to_cpu(evtchn, cpu);
 	}
@@ -432,24 +404,27 @@ out:
 }
 
 
-void 
+static void 
 unbind_from_irq(int irq)
 {
 	struct evtchn_close close;
 	int evtchn = evtchn_from_irq(irq);
+	int cpu;
 
 	mtx_lock_spin(&irq_mapping_update_lock);
 
 	if ((--irq_bindcount[irq] == 0) && VALID_EVTCHN(evtchn)) {
 		close.port = evtchn;
-		PANIC_IF(HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0);
+		HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
 
 		switch (type_from_irq(irq)) {
 		case IRQT_VIRQ:
-			per_cpu(virq_to_irq, cpu_from_evtchn(evtchn))[index_from_irq(irq)] = -1;
+			cpu = cpu_from_evtchn(evtchn);
+			pcpu_find(cpu)->pc_virq_to_irq[index_from_irq(irq)] = -1;
 			break;
 		case IRQT_IPI:
-			per_cpu(ipi_to_irq, cpu_from_evtchn(evtchn))[index_from_irq(irq)] = -1;
+			cpu = cpu_from_evtchn(evtchn);
+			pcpu_find(cpu)->pc_ipi_to_irq[index_from_irq(irq)] = -1;
 			break;
 		default:
 			break;
@@ -467,11 +442,8 @@ unbind_from_irq(int irq)
 
 int 
 bind_caller_port_to_irqhandler(unsigned int caller_port,
-			  const char *devname,
-			  driver_intr_t handler,
-			  void *arg,
-			  unsigned long irqflags,
-                          unsigned int *irqp)
+    const char *devname, driver_intr_t handler, void *arg,
+    unsigned long irqflags, unsigned int *irqp)
 {
 	unsigned int irq;
 	int error;
@@ -493,13 +465,9 @@ bind_caller_port_to_irqhandler(unsigned int caller_port,
 }
 
 int 
-bind_listening_port_to_irqhandler(
-	                  unsigned int remote_domain,
-			  const char *devname,
-			  driver_intr_t handler,
-			  void *arg,
-			  unsigned long irqflags,
-                          unsigned int *irqp)
+bind_listening_port_to_irqhandler(unsigned int remote_domain,
+    const char *devname, driver_intr_t handler, void *arg,
+    unsigned long irqflags, unsigned int *irqp)
 {
 	unsigned int irq;
 	int error;
@@ -519,14 +487,10 @@ bind_listening_port_to_irqhandler(
 }
 
 int 
-bind_interdomain_evtchn_to_irqhandler(
-	                    unsigned int remote_domain,
-			    unsigned int remote_port,
-			    const char *devname,
-			    driver_filter_t filter,
-			    driver_intr_t handler,
-			    unsigned long irqflags,
-			    unsigned int *irqp)
+bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain,
+    unsigned int remote_port, const char *devname,
+    driver_filter_t filter, driver_intr_t handler,
+    unsigned long irqflags, unsigned int *irqp)
 {
 	unsigned int irq;
 	int error;
@@ -546,14 +510,9 @@ bind_interdomain_evtchn_to_irqhandler(
 }
 
 int 
-bind_virq_to_irqhandler(unsigned int virq,
-			unsigned int cpu,
-			const char *devname,
-			driver_filter_t filter,
-			driver_intr_t handler,
-                        void *arg,
-                        unsigned long irqflags,
-                        unsigned int *irqp)
+bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
+    const char *devname, driver_filter_t filter, driver_intr_t handler,
+    unsigned long irqflags, unsigned int *irqp)
 {
 	unsigned int irq;
 	int error;
@@ -573,12 +532,9 @@ bind_virq_to_irqhandler(unsigned int virq,
 }
 
 int 
-bind_ipi_to_irqhandler(unsigned int ipi,
-		       unsigned int cpu,
-		       const char *devname,
-		       driver_filter_t filter,
-                       unsigned long irqflags,
-                       unsigned int *irqp)
+bind_ipi_to_irqhandler(unsigned int ipi, unsigned int cpu,
+    const char *devname, driver_filter_t filter,
+    unsigned long irqflags, unsigned int *irqp)
 {
 	unsigned int irq;
 	int error;
@@ -636,9 +592,9 @@ rebind_irq_to_cpu(unsigned irq, unsigned tcpu)
 
 }
 
-static void set_affinity_irq(unsigned irq, xen_cpumask_t dest)
+static void set_affinity_irq(unsigned irq, cpumask_t dest)
 {
-	unsigned tcpu = first_cpu(dest);
+	unsigned tcpu = ffs(dest) - 1;
 	rebind_irq_to_cpu(irq, tcpu);
 }
 #endif
@@ -656,13 +612,11 @@ static void     xenpic_dynirq_enable_source(struct intsrc *isrc);
 static void     xenpic_dynirq_disable_source(struct intsrc *isrc, int); 
 static void     xenpic_dynirq_eoi_source(struct intsrc *isrc); 
 static void     xenpic_dynirq_enable_intr(struct intsrc *isrc); 
-static void     xenpic_dynirq_disable_intr(struct intsrc *isrc); 
 
 static void     xenpic_pirq_enable_source(struct intsrc *isrc); 
 static void     xenpic_pirq_disable_source(struct intsrc *isrc, int); 
 static void     xenpic_pirq_eoi_source(struct intsrc *isrc); 
 static void     xenpic_pirq_enable_intr(struct intsrc *isrc); 
-static void     xenpic_pirq_disable_intr(struct intsrc *isrc); 
 
 
 static int      xenpic_vector(struct intsrc *isrc); 
@@ -677,7 +631,6 @@ struct pic xenpic_dynirq_template  =  {
 	.pic_disable_source	=	xenpic_dynirq_disable_source,
 	.pic_eoi_source		=	xenpic_dynirq_eoi_source, 
 	.pic_enable_intr	=	xenpic_dynirq_enable_intr, 
-	.pic_disable_intr	=	xenpic_dynirq_disable_intr, 
 	.pic_vector		=	xenpic_vector, 
 	.pic_source_pending	=	xenpic_source_pending,
 	.pic_suspend		=	xenpic_suspend, 
@@ -689,7 +642,6 @@ struct pic xenpic_pirq_template  =  {
 	.pic_disable_source	=	xenpic_pirq_disable_source,
 	.pic_eoi_source		=	xenpic_pirq_eoi_source, 
 	.pic_enable_intr	=	xenpic_pirq_enable_intr, 
-	.pic_disable_intr	=	xenpic_pirq_disable_intr, 
 	.pic_vector		=	xenpic_vector, 
 	.pic_source_pending	=	xenpic_source_pending,
 	.pic_suspend		=	xenpic_suspend, 
@@ -748,20 +700,6 @@ xenpic_dynirq_enable_intr(struct intsrc *isrc)
 }
 
 static void 
-xenpic_dynirq_disable_intr(struct intsrc *isrc)
-{
-	unsigned int irq;
-	struct xenpic_intsrc *xp;
-	
-	xp = (struct xenpic_intsrc *)isrc;	
-	mtx_lock_spin(&irq_mapping_update_lock);
-	xp->xp_masked = 1;
-	irq = xenpic_vector(isrc);
-	mask_evtchn(evtchn_from_irq(irq));
-	mtx_unlock_spin(&irq_mapping_update_lock);
-}
-
-static void 
 xenpic_dynirq_eoi_source(struct intsrc *isrc)
 {
 	unsigned int irq;
@@ -825,7 +763,7 @@ notify_remote_via_irq(int irq)
 	if (VALID_EVTCHN(evtchn))
 		notify_remote_via_evtchn(evtchn);
 	else
-		panic("invalid evtchn");
+		panic("invalid evtchn %d", irq);
 }
 
 /* required for support of physical devices */
@@ -899,32 +837,6 @@ xenpic_pirq_enable_intr(struct intsrc *isrc)
 }
 
 static void 
-xenpic_pirq_disable_intr(struct intsrc *isrc)
-{
-	unsigned int irq;
-	int evtchn;
-	struct evtchn_close close;
-			
-	mtx_lock_spin(&irq_mapping_update_lock);
-	irq = xenpic_vector(isrc);
-	evtchn = evtchn_from_irq(irq);
-
-	if (!VALID_EVTCHN(evtchn)) 
-		goto done;
-	
-	mask_evtchn(evtchn);
-
-	close.port = evtchn;
-	PANIC_IF(HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0);
-
-	bind_evtchn_to_cpu(evtchn, 0);
-	evtchn_to_irq[evtchn] = -1;
-	irq_info[irq] = IRQ_UNBOUND;
- done:
-	mtx_unlock_spin(&irq_mapping_update_lock);
-}
-
-static void 
 xenpic_pirq_enable_source(struct intsrc *isrc)
 {
 	int evtchn;
@@ -998,7 +910,7 @@ void
 unmask_evtchn(int port)
 {
 	shared_info_t *s = HYPERVISOR_shared_info;
-	unsigned int cpu = smp_processor_id();
+	unsigned int cpu = PCPU_GET(cpuid);
 	vcpu_info_t *vcpu_info = &s->vcpu_info[cpu];
 
 	/* Slow path (hypercall) if this is a non-local port. */
@@ -1016,7 +928,7 @@ unmask_evtchn(int port)
 	 * masked.
 	 */
 	if (synch_test_bit(port, &s->evtchn_pending) && 
-	    !synch_test_and_set_bit(port / BITS_PER_LONG,
+	    !synch_test_and_set_bit(port / LONG_BIT,
 				    &vcpu_info->evtchn_pending_sel)) {
 		vcpu_info->evtchn_upcall_pending = 1;
 		if (!vcpu_info->evtchn_upcall_mask)
@@ -1039,15 +951,21 @@ void irq_resume(void)
 		mask_evtchn(evtchn);
 
 	/* Check that no PIRQs are still bound. */
-	for (pirq = 0; pirq < NR_PIRQS; pirq++)
-		PANIC_IF(irq_info[pirq_to_irq(pirq)] != IRQ_UNBOUND);
+	for (pirq = 0; pirq < NR_PIRQS; pirq++) {
+		KASSERT(irq_info[pirq_to_irq(pirq)] == IRQ_UNBOUND,
+		    ("pirq_to_irq inconsistent"));
+	}
 
 	/* Secondary CPUs must have no VIRQ or IPI bindings. */
-	for (cpu = 1; cpu < NR_CPUS; cpu++) {
-		for (virq = 0; virq < NR_VIRQS; virq++)
-			PANIC_IF(per_cpu(virq_to_irq, cpu)[virq] != -1);
-		for (ipi = 0; ipi < NR_IPIS; ipi++)
-			PANIC_IF(per_cpu(ipi_to_irq, cpu)[ipi] != -1);
+	for (cpu = 1; cpu < MAX_VIRT_CPUS; cpu++) {
+		for (virq = 0; virq < NR_VIRQS; virq++) {
+			KASSERT(pcpu_find(cpu)->pc_virq_to_irq[virq] == -1,
+			    ("virq_to_irq inconsistent"));
+		}
+		for (ipi = 0; ipi < NR_IPIS; ipi++) {
+			KASSERT(pcpu_find(cpu)->pc_ipi_to_irq[ipi] == -1,
+			    ("ipi_to_irq inconsistent"));
+		}
 	}
 
 	/* No IRQ <-> event-channel mappings. */
@@ -1058,15 +976,16 @@ void irq_resume(void)
 
 	/* Primary CPU: rebind VIRQs automatically. */
 	for (virq = 0; virq < NR_VIRQS; virq++) {
-		if ((irq = per_cpu(virq_to_irq, 0)[virq]) == -1)
+		if ((irq = pcpu_find(0)->pc_virq_to_irq[virq]) == -1)
 			continue;
 
-		PANIC_IF(irq_info[irq] != mk_irq_info(IRQT_VIRQ, virq, 0));
+		KASSERT(irq_info[irq] == mk_irq_info(IRQT_VIRQ, virq, 0),
+		    ("irq_info inconsistent"));
 
 		/* Get a new binding from Xen. */
 		bind_virq.virq = virq;
 		bind_virq.vcpu = 0;
-		PANIC_IF(HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq) != 0);
+		HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq);
 		evtchn = bind_virq.port;
         
 		/* Record the new mapping. */
@@ -1079,15 +998,16 @@ void irq_resume(void)
 
 	/* Primary CPU: rebind IPIs automatically. */
 	for (ipi = 0; ipi < NR_IPIS; ipi++) {
-		if ((irq = per_cpu(ipi_to_irq, 0)[ipi]) == -1)
+		if ((irq = pcpu_find(0)->pc_ipi_to_irq[ipi]) == -1)
 			continue;
 
-		PANIC_IF(irq_info[irq] != mk_irq_info(IRQT_IPI, ipi, 0));
+		KASSERT(irq_info[irq] == mk_irq_info(IRQT_IPI, ipi, 0),
+		    ("irq_info inconsistent"));
 
 		/* Get a new binding from Xen. */
 		memset(&op, 0, sizeof(op));
 		bind_ipi.vcpu = 0;
-		PANIC_IF(HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi) != 0);
+		HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
 		evtchn = bind_ipi.port;
         
 		/* Record the new mapping. */
@@ -1111,9 +1031,9 @@ evtchn_init(void *dummy __unused)
          /* No VIRQ or IPI bindings. */
 	for (cpu = 0; cpu < mp_ncpus; cpu++) {
 		for (i = 0; i < NR_VIRQS; i++)
-			per_cpu(virq_to_irq, cpu)[i] = -1;
+			pcpu_find(cpu)->pc_virq_to_irq[i] = -1;
 		for (i = 0; i < NR_IPIS; i++)
-			per_cpu(ipi_to_irq, cpu)[i] = -1;
+			pcpu_find(cpu)->pc_ipi_to_irq[i] = -1;
 	}
 
 	/* No event-channel -> IRQ mappings. */
diff --git a/sys/xen/evtchn/evtchn_dev.c b/sys/xen/evtchn/evtchn_dev.c
index ea12860..4253d8a 100644
--- a/sys/xen/evtchn/evtchn_dev.c
+++ b/sys/xen/evtchn/evtchn_dev.c
@@ -23,8 +23,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/fcntl.h>
 #include <sys/ioccom.h>
 
-#include <machine/cpufunc.h>
-#include <machine/intr_machdep.h>
 #include <machine/xen/xen-os.h>
 #include <xen/xen_intr.h>
 #include <machine/bus.h>
@@ -234,14 +232,14 @@ evtchn_ioctl(struct cdev *dev, unsigned long cmd, caddr_t arg,
 		__evtchn_reset_buffer_ring();
 		break;
 	case EVTCHN_BIND:
-		if ( !synch_test_and_set_bit((int)arg, &bound_ports[0]) )
-			unmask_evtchn((int)arg);
+		if ( !synch_test_and_set_bit((uintptr_t)arg, &bound_ports[0]) )
+			unmask_evtchn((uintptr_t)arg);
 		else
 			rc = EINVAL;
 		break;
 	case EVTCHN_UNBIND:
-		if ( synch_test_and_clear_bit((int)arg, &bound_ports[0]) )
-			mask_evtchn((int)arg);
+		if ( synch_test_and_clear_bit((uintptr_t)arg, &bound_ports[0]) )
+			mask_evtchn((uintptr_t)arg);
 		else
 			rc = EINVAL;
 		break;
@@ -383,12 +381,12 @@ evtchn_dev_init(void *dummy __unused)
 	/* (DEVFS) automatically destroy the symlink with its destination. */
 	devfs_auto_unregister(evtchn_miscdev.devfs_handle, symlink_handle);
 #endif
-	printk("Event-channel device installed.\n");
+	if (bootverbose)
+		printf("Event-channel device installed.\n");
 
 	return 0;
 }
 
-
 SYSINIT(evtchn_dev_init, SI_SUB_DRIVERS, SI_ORDER_FIRST, evtchn_dev_init, NULL);