[SPARC64]: Move over to GENERIC_HARDIRQS.

This is the long overdue conversion of sparc64 over to
the generic IRQ layer.

The kernel image is slightly larger, but the BSS is ~60K
smaller due to the reduced size of struct ino_bucket.

A lot of IRQ implementation details, including ino_bucket,
were moved out of asm-sparc64/irq.h and are now private to
arch/sparc64/kernel/irq.c, and most of the code in irq.c
totally disappeared.

One thing that's different at the moment is IRQ distribution,
we do it at enable_irq() time.  If the cpu mask is ALL then
we round-robin using a global rotating cpu counter, else
we pick the first cpu in the mask to support single cpu
targetting.  This is similar to what powerpc's XICS IRQ
support code does.

This works fine on my UP SB1000, and the SMP build goes
fine and runs on that machine, but lots of testing on
different setups is needed.

Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 284 insertions, 673 deletions

diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c
index 49ad9cd..a8c9dc8 100644
--- a/arch/sparc64/kernel/irq.c
+++ b/arch/sparc64/kernel/irq.c
@@ -22,6 +22,7 @@
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
 #include <linux/bootmem.h>
+#include <linux/irq.h>
 
 #include <asm/ptrace.h>
 #include <asm/processor.h>
@@ -42,10 +43,6 @@
 #include <asm/auxio.h>
 #include <asm/head.h>
 
-#ifdef CONFIG_SMP
-static void distribute_irqs(void);
-#endif
-
 /* UPA nodes send interrupt packet to UltraSparc with first data reg
  * value low 5 (7 on Starfire) bits holding the IRQ identifier being
  * delivered.  We must translate this into a non-vector IRQ so we can
@@ -57,10 +54,29 @@ static void distribute_irqs(void);
  * The IVEC handler does not need to act atomically, the PIL dispatch
  * code uses CAS to get an atomic snapshot of the list and clear it
  * at the same time.
+ *
+ * If you make changes to ino_bucket, please update hand coded assembler
+ * of the vectored interrupt trap handler(s) in entry.S and sun4v_ivec.S
  */
+struct ino_bucket {
+	/* Next handler in per-CPU IRQ worklist.  We know that
+	 * bucket pointers have the high 32-bits clear, so to
+	 * save space we only store the bits we need.
+	 */
+/*0x00*/unsigned int irq_chain;
 
+	/* Virtual interrupt number assigned to this INO.  */
+/*0x04*/unsigned int virt_irq;
+};
+
+#define NUM_IVECS	(IMAP_INR + 1)
 struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BYTES)));
 
+#define __irq_ino(irq) \
+        (((struct ino_bucket *)(unsigned long)(irq)) - &ivector_table[0])
+#define __bucket(irq) ((struct ino_bucket *)(unsigned long)(irq))
+#define __irq(bucket) ((unsigned int)(unsigned long)(bucket))
+
 /* This has to be in the main kernel image, it cannot be
  * turned into per-cpu data.  The reason is that the main
  * kernel image is locked into the TLB and this structure
@@ -70,18 +86,6 @@ struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BY
  */
 #define irq_work(__cpu)	&(trap_block[(__cpu)].irq_worklist)
 
-static struct irqaction timer_irq_action = {
-	.name = "timer",
-};
-static struct irqaction *irq_action[NR_IRQS] = { &timer_irq_action, };
-
-/* This only synchronizes entities which modify IRQ handler
- * state and some selected user-level spots that want to
- * read things in the table.  IRQ handler processing orders
- * its' accesses such that no locking is needed.
- */
-static DEFINE_SPINLOCK(irq_action_lock);
-
 static unsigned int virt_to_real_irq_table[NR_IRQS];
 static unsigned char virt_irq_cur = 1;
 
@@ -117,69 +121,45 @@ static unsigned int virt_to_real_irq(unsigned char virt_irq)
 	return virt_to_real_irq_table[virt_irq];
 }
 
-void irq_install_pre_handler(int virt_irq,
-			     void (*func)(struct ino_bucket *, void *, void *),
-			     void *arg1, void *arg2)
-{
-	unsigned int real_irq = virt_to_real_irq(virt_irq);
-	struct ino_bucket *bucket;
-	struct irq_desc *d;
-
-	if (unlikely(!real_irq))
-		return;
-
-	bucket = __bucket(real_irq);
-	d = bucket->irq_info;
-	d->pre_handler = func;
-	d->pre_handler_arg1 = arg1;
-	d->pre_handler_arg2 = arg2;
-}
-
-static void register_irq_proc (unsigned int irq);
-
 /*
- * Upper 2b of irqaction->flags holds the ino.
- * irqaction->mask holds the smp affinity information.
+ * /proc/interrupts printing:
  */
-#define put_ino_in_irqaction(action, irq) \
-	action->flags &= 0xffffffffffffUL; \
-	action->flags |= __irq_ino(irq) << 48;
-
-#define get_ino_in_irqaction(action)	(action->flags >> 48)
-
-#define put_smpaff_in_irqaction(action, smpaff)	(action)->mask = (smpaff)
-#define get_smpaff_in_irqaction(action) 	((action)->mask)
 
 int show_interrupts(struct seq_file *p, void *v)
 {
+	int i = *(loff_t *) v, j;
+	struct irqaction * action;
 	unsigned long flags;
-	int i = *(loff_t *) v;
-	struct irqaction *action;
-#ifdef CONFIG_SMP
-	int j;
-#endif
 
-	spin_lock_irqsave(&irq_action_lock, flags);
-	if (i <= NR_IRQS) {
-		if (!(action = *(i + irq_action)))
-			goto out_unlock;
-		seq_printf(p, "%3d: ", i);
+	if (i == 0) {
+		seq_printf(p, "           ");
+		for_each_online_cpu(j)
+			seq_printf(p, "CPU%d       ",j);
+		seq_putc(p, '\n');
+	}
+
+	if (i < NR_IRQS) {
+		spin_lock_irqsave(&irq_desc[i].lock, flags);
+		action = irq_desc[i].action;
+		if (!action)
+			goto skip;
+		seq_printf(p, "%3d: ",i);
 #ifndef CONFIG_SMP
 		seq_printf(p, "%10u ", kstat_irqs(i));
 #else
-		for_each_online_cpu(j) {
-			seq_printf(p, "%10u ",
-				   kstat_cpu(j).irqs[i]);
-		}
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", kstat_cpu(j).irqs[i]);
 #endif
-		seq_printf(p, " %s", action->name);
-		for (action = action->next; action; action = action->next)
+		seq_printf(p, " %9s", irq_desc[i].handler->typename);
+		seq_printf(p, "  %s", action->name);
+
+		for (action=action->next; action; action = action->next)
 			seq_printf(p, ", %s", action->name);
+
 		seq_putc(p, '\n');
+skip:
+		spin_unlock_irqrestore(&irq_desc[i].lock, flags);
 	}
-out_unlock:
-	spin_unlock_irqrestore(&irq_action_lock, flags);
-
 	return 0;
 }
 
@@ -220,509 +200,321 @@ static unsigned int sun4u_compute_tid(unsigned long imap, unsigned long cpuid)
 	return tid;
 }
 
-void enable_irq(unsigned int virt_irq)
-{
-	unsigned int real_irq = virt_to_real_irq(virt_irq);
-	struct ino_bucket *bucket;
-	unsigned long imap, cpuid;
-
-	if (unlikely(!real_irq))
-		return;
+struct irq_handler_data {
+	unsigned long	iclr;
+	unsigned long	imap;
 
-	bucket = __bucket(real_irq);
-	imap = bucket->imap;
-	if (unlikely(imap == 0UL))
-		return;
-
-	preempt_disable();
-
-	/* This gets the physical processor ID, even on uniprocessor,
-	 * so we can always program the interrupt target correctly.
-	 */
-	cpuid = real_hard_smp_processor_id();
-
-	if (tlb_type == hypervisor) {
-		unsigned int ino = __irq_ino(real_irq);
-		int err;
-
-		err = sun4v_intr_settarget(ino, cpuid);
-		if (err != HV_EOK)
-			printk("sun4v_intr_settarget(%x,%lu): err(%d)\n",
-			       ino, cpuid, err);
-		err = sun4v_intr_setenabled(ino, HV_INTR_ENABLED);
-		if (err != HV_EOK)
-			printk("sun4v_intr_setenabled(%x): err(%d)\n",
-			       ino, err);
-	} else {
-		unsigned int tid = sun4u_compute_tid(imap, cpuid);
-
-		/* NOTE NOTE NOTE, IGN and INO are read-only, IGN is a product
-		 * of this SYSIO's preconfigured IGN in the SYSIO Control
-		 * Register, the hardware just mirrors that value here.
-		 * However for Graphics and UPA Slave devices the full
-		 * IMAP_INR field can be set by the programmer here.
-		 *
-		 * Things like FFB can now be handled via the new IRQ
-		 * mechanism.
-		 */
-		upa_writel(tid | IMAP_VALID, imap);
-	}
-
-	preempt_enable();
-}
+	void		(*pre_handler)(unsigned int, void *, void *);
+	void		*pre_handler_arg1;
+	void		*pre_handler_arg2;
+};
 
-void disable_irq(unsigned int virt_irq)
+static inline struct ino_bucket *virt_irq_to_bucket(unsigned int virt_irq)
 {
 	unsigned int real_irq = virt_to_real_irq(virt_irq);
-	struct ino_bucket *bucket;
-	unsigned long imap;
+	struct ino_bucket *bucket = NULL;
 
-	if (unlikely(!real_irq))
-		return;
-
-	bucket = __bucket(real_irq);
-	imap = bucket->imap;
-	if (unlikely(imap == 0UL))
-		return;
-
-	if (tlb_type == hypervisor) {
-		unsigned int ino = __irq_ino(real_irq);
-		int err;
+	if (likely(real_irq))
+		bucket = __bucket(real_irq);
 
-		err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED);
-		if (err != HV_EOK)
-			printk("sun4v_intr_setenabled(%x): "
-			       "err(%d)\n", ino, err);
-	} else {
-		u32 tmp;
-
-		/* NOTE: We do not want to futz with the IRQ clear registers
-		 *       and move the state to IDLE, the SCSI code does call
-		 *       disable_irq() to assure atomicity in the queue cmd
-		 *       SCSI adapter driver code.  Thus we'd lose interrupts.
-		 */
-		tmp = upa_readl(imap);
-		tmp &= ~IMAP_VALID;
-		upa_writel(tmp, imap);
-	}
+	return bucket;
 }
 
-static void build_irq_error(const char *msg, unsigned int ino, int inofixup,
-			    unsigned long iclr, unsigned long imap,
-			    struct ino_bucket *bucket)
+#ifdef CONFIG_SMP
+static int irq_choose_cpu(unsigned int virt_irq)
 {
-	prom_printf("IRQ: INO %04x (%016lx:%016lx) --> "
-		    "(%d:%016lx:%016lx), halting...\n",
-		    ino, bucket->iclr, bucket->imap,
-		    inofixup, iclr, imap);
-	prom_halt();
-}
+	cpumask_t mask = irq_affinity[virt_irq];
+	int cpuid;
 
-unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap, unsigned char flags)
-{
-	struct ino_bucket *bucket;
-	int ino;
+	if (cpus_equal(mask, CPU_MASK_ALL)) {
+		static int irq_rover;
+		static DEFINE_SPINLOCK(irq_rover_lock);
+		unsigned long flags;
 
-	BUG_ON(tlb_type == hypervisor);
+		/* Round-robin distribution... */
+	do_round_robin:
+		spin_lock_irqsave(&irq_rover_lock, flags);
 
-	/* RULE: Both must be specified. */
-	if (iclr == 0UL || imap == 0UL) {
-		prom_printf("Invalid build_irq %d %016lx %016lx\n",
-			    inofixup, iclr, imap);
-		prom_halt();
-	}
-	
-	ino = (upa_readl(imap) & (IMAP_IGN | IMAP_INO)) + inofixup;
-	if (ino > NUM_IVECS) {
-		prom_printf("Invalid INO %04x (%d:%016lx:%016lx)\n",
-			    ino, inofixup, iclr, imap);
-		prom_halt();
-	}
+		while (!cpu_online(irq_rover)) {
+			if (++irq_rover >= NR_CPUS)
+				irq_rover = 0;
+		}
+		cpuid = irq_rover;
+		do {
+			if (++irq_rover >= NR_CPUS)
+				irq_rover = 0;
+		} while (!cpu_online(irq_rover));
 
-	bucket = &ivector_table[ino];
-	if (bucket->flags & IBF_ACTIVE)
-		build_irq_error("IRQ: Trying to build active INO bucket.\n",
-				ino, inofixup, iclr, imap, bucket);
+		spin_unlock_irqrestore(&irq_rover_lock, flags);
+	} else {
+		cpumask_t tmp;
 
-	if (bucket->irq_info) {
-		if (bucket->imap != imap || bucket->iclr != iclr)
-			build_irq_error("IRQ: Trying to reinit INO bucket.\n",
-					ino, inofixup, iclr, imap, bucket);
+		cpus_and(tmp, cpu_online_map, mask);
 
-		goto out;
-	}
+		if (cpus_empty(tmp))
+			goto do_round_robin;
 
-	bucket->irq_info = kzalloc(sizeof(struct irq_desc), GFP_ATOMIC);
-	if (!bucket->irq_info) {
-		prom_printf("IRQ: Error, kmalloc(irq_desc) failed.\n");
-		prom_halt();
+		cpuid = first_cpu(tmp);
 	}
 
-	/* Ok, looks good, set it up.  Don't touch the irq_chain or
-	 * the pending flag.
-	 */
-	bucket->imap  = imap;
-	bucket->iclr  = iclr;
-	if (!bucket->virt_irq)
-		bucket->virt_irq = virt_irq_alloc(__irq(bucket));
-	bucket->flags = flags;
-
-out:
-	return bucket->virt_irq;
+	return cpuid;
+}
+#else
+static int irq_choose_cpu(unsigned int virt_irq)
+{
+	return real_hard_smp_processor_id();
 }
+#endif
 
-unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino, unsigned char flags)
+static void sun4u_irq_enable(unsigned int virt_irq)
 {
-	struct ino_bucket *bucket;
-	unsigned long sysino;
+	irq_desc_t *desc = irq_desc + virt_irq;
+	struct irq_handler_data *data = desc->handler_data;
 
-	sysino = sun4v_devino_to_sysino(devhandle, devino);
+	if (likely(data)) {
+		unsigned long cpuid, imap;
+		unsigned int tid;
 
-	bucket = &ivector_table[sysino];
+		cpuid = irq_choose_cpu(virt_irq);
+		imap = data->imap;
 
-	/* Catch accidental accesses to these things.  IMAP/ICLR handling
-	 * is done by hypervisor calls on sun4v platforms, not by direct
-	 * register accesses.
-	 *
-	 * But we need to make them look unique for the disable_irq() logic
-	 * in free_irq().
-	 */
-	bucket->imap = ~0UL - sysino;
-	bucket->iclr = ~0UL - sysino;
-	if (!bucket->virt_irq)
-		bucket->virt_irq = virt_irq_alloc(__irq(bucket));
-	bucket->flags = flags;
+		tid = sun4u_compute_tid(imap, cpuid);
 
-	bucket->irq_info = kzalloc(sizeof(struct irq_desc), GFP_ATOMIC);
-	if (!bucket->irq_info) {
-		prom_printf("IRQ: Error, kmalloc(irq_desc) failed.\n");
-		prom_halt();
+		upa_writel(tid | IMAP_VALID, imap);
 	}
-
-	return bucket->virt_irq;
 }
 
-static void atomic_bucket_insert(struct ino_bucket *bucket)
+static void sun4u_irq_disable(unsigned int virt_irq)
 {
-	unsigned long pstate;
-	unsigned int *ent;
+	irq_desc_t *desc = irq_desc + virt_irq;
+	struct irq_handler_data *data = desc->handler_data;
 
-	__asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate));
-	__asm__ __volatile__("wrpr %0, %1, %%pstate"
-			     : : "r" (pstate), "i" (PSTATE_IE));
-	ent = irq_work(smp_processor_id());
-	bucket->irq_chain = *ent;
-	*ent = __irq(bucket);
-	__asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate));
-}
+	if (likely(data)) {
+		unsigned long imap = data->imap;
+		u32 tmp = upa_readl(imap);
 
-static int check_irq_sharing(int pil, unsigned long irqflags)
-{
-	struct irqaction *action;
-
-	action = *(irq_action + pil);
-	if (action) {
-		if (!(action->flags & SA_SHIRQ) || !(irqflags & SA_SHIRQ))
-			return -EBUSY;
+		tmp &= ~IMAP_VALID;
+		upa_writel(tmp, imap);
 	}
-	return 0;
 }
 
-static void append_irq_action(int pil, struct irqaction *action)
+static void sun4u_irq_end(unsigned int virt_irq)
 {
-	struct irqaction **pp = irq_action + pil;
+	irq_desc_t *desc = irq_desc + virt_irq;
+	struct irq_handler_data *data = desc->handler_data;
 
-	while (*pp)
-		pp = &((*pp)->next);
-	*pp = action;
+	if (likely(data))
+		upa_writel(ICLR_IDLE, data->iclr);
 }
 
-static struct irqaction *get_action_slot(struct ino_bucket *bucket)
+static void sun4v_irq_enable(unsigned int virt_irq)
 {
-	struct irq_desc *desc = bucket->irq_info;
-	int max_irq, i;
+	struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq);
+	unsigned int ino = bucket - &ivector_table[0];
 
-	max_irq = 1;
-	if (bucket->flags & IBF_PCI)
-		max_irq = MAX_IRQ_DESC_ACTION;
-	for (i = 0; i < max_irq; i++) {
-		struct irqaction *p = &desc->action[i];
-		u32 mask = (1 << i);
+	if (likely(bucket)) {
+		unsigned long cpuid;
+		int err;
 
-		if (desc->action_active_mask & mask)
-			continue;
+		cpuid = irq_choose_cpu(virt_irq);
 
-		desc->action_active_mask |= mask;
-		return p;
+		err = sun4v_intr_settarget(ino, cpuid);
+		if (err != HV_EOK)
+			printk("sun4v_intr_settarget(%x,%lu): err(%d)\n",
+			       ino, cpuid, err);
+		err = sun4v_intr_setenabled(ino, HV_INTR_ENABLED);
+		if (err != HV_EOK)
+			printk("sun4v_intr_setenabled(%x): err(%d)\n",
+			       ino, err);
 	}
-	return NULL;
 }
 
-int request_irq(unsigned int virt_irq,
-		irqreturn_t (*handler)(int, void *, struct pt_regs *),
-		unsigned long irqflags, const char *name, void *dev_id)
+static void sun4v_irq_disable(unsigned int virt_irq)
 {
-	struct irqaction *action;
-	struct ino_bucket *bucket;
-	unsigned long flags;
-	unsigned int real_irq;
-	int pending = 0;
-
-	real_irq = virt_to_real_irq(virt_irq);
-	if (unlikely(!real_irq))
-		return -EINVAL;
-
-	if (unlikely(!handler))
-		return -EINVAL;
-
-	bucket = __bucket(real_irq);
-	if (unlikely(!bucket->irq_info))
-		return -ENODEV;
-
-	if (irqflags & SA_SAMPLE_RANDOM) {
-		/*
-		 * This function might sleep, we want to call it first,
-		 * outside of the atomic block.
-		 * Yes, this might clear the entropy pool if the wrong
-		 * driver is attempted to be loaded, without actually
-		 * installing a new handler, but is this really a problem,
-		 * only the sysadmin is able to do this.
-		 */
-		rand_initialize_irq(virt_irq);
-	}
-
-	spin_lock_irqsave(&irq_action_lock, flags);
+	struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq);
+	unsigned int ino = bucket - &ivector_table[0];
 
-	if (check_irq_sharing(virt_irq, irqflags)) {
-		spin_unlock_irqrestore(&irq_action_lock, flags);
-		return -EBUSY;
-	}
+	if (likely(bucket)) {
+		int err;
 
-	action = get_action_slot(bucket);
-	if (!action) { 
-		spin_unlock_irqrestore(&irq_action_lock, flags);
-		return -ENOMEM;
+		err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED);
+		if (err != HV_EOK)
+			printk("sun4v_intr_setenabled(%x): "
+			       "err(%d)\n", ino, err);
 	}
+}
 
-	bucket->flags |= IBF_ACTIVE;
-	pending = bucket->pending;
-	if (pending)
-		bucket->pending = 0;
-
-	action->handler = handler;
-	action->flags = irqflags;
-	action->name = name;
-	action->next = NULL;
-	action->dev_id = dev_id;
-	put_ino_in_irqaction(action, __irq_ino(real_irq));
-	put_smpaff_in_irqaction(action, CPU_MASK_NONE);
-
-	append_irq_action(virt_irq, action);
+static void sun4v_irq_end(unsigned int virt_irq)
+{
+	struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq);
+	unsigned int ino = bucket - &ivector_table[0];
 
-	enable_irq(virt_irq);
+	if (likely(bucket)) {
+		int err;
 
-	/* We ate the IVEC already, this makes sure it does not get lost. */
-	if (pending) {
-		atomic_bucket_insert(bucket);
-		set_softint(1 << PIL_DEVICE_IRQ);
+		err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE);
+		if (err != HV_EOK)
+			printk("sun4v_intr_setstate(%x): "
+			       "err(%d)\n", ino, err);
 	}
-
-	spin_unlock_irqrestore(&irq_action_lock, flags);
-
-	register_irq_proc(virt_irq);
-
-#ifdef CONFIG_SMP
-	distribute_irqs();
-#endif
-	return 0;
 }
 
-EXPORT_SYMBOL(request_irq);
-
-static struct irqaction *unlink_irq_action(unsigned int virt_irq, void *dev_id)
+static void run_pre_handler(unsigned int virt_irq)
 {
-	struct irqaction *action, **pp;
-
-	pp = irq_action + virt_irq;
-	action = *pp;
-	if (unlikely(!action))
-		return NULL;
+	struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq);
+	irq_desc_t *desc = irq_desc + virt_irq;
+	struct irq_handler_data *data = desc->handler_data;
 
-	if (unlikely(!action->handler)) {
-		printk("Freeing free IRQ %d\n", virt_irq);
-		return NULL;
+	if (likely(data->pre_handler)) {
+		data->pre_handler(__irq_ino(__irq(bucket)),
+				  data->pre_handler_arg1,
+				  data->pre_handler_arg2);
 	}
-
-	while (action && action->dev_id != dev_id) {
-		pp = &action->next;
-		action = *pp;
-	}
-
-	if (likely(action))
-		*pp = action->next;
-
-	return action;
 }
 
-void free_irq(unsigned int virt_irq, void *dev_id)
-{
-	struct irqaction *action;
-	struct ino_bucket *bucket;
-	struct irq_desc *desc;
-	unsigned long flags;
-	unsigned int real_irq;
-	int ent, i;
-
-	real_irq = virt_to_real_irq(virt_irq);
-	if (unlikely(!real_irq))
-		return;
+static struct hw_interrupt_type sun4u_irq = {
+	.typename	= "sun4u",
+	.enable		= sun4u_irq_enable,
+	.disable	= sun4u_irq_disable,
+	.end		= sun4u_irq_end,
+};
 
-	spin_lock_irqsave(&irq_action_lock, flags);
+static struct hw_interrupt_type sun4u_irq_ack = {
+	.typename	= "sun4u+ack",
+	.enable		= sun4u_irq_enable,
+	.disable	= sun4u_irq_disable,
+	.ack		= run_pre_handler,
+	.end		= sun4u_irq_end,
+};
 
-	action = unlink_irq_action(virt_irq, dev_id);
+static struct hw_interrupt_type sun4v_irq = {
+	.typename	= "sun4v",
+	.enable		= sun4v_irq_enable,
+	.disable	= sun4v_irq_disable,
+	.end		= sun4v_irq_end,
+};
 
-	spin_unlock_irqrestore(&irq_action_lock, flags);
+static struct hw_interrupt_type sun4v_irq_ack = {
+	.typename	= "sun4v+ack",
+	.enable		= sun4v_irq_enable,
+	.disable	= sun4v_irq_disable,
+	.ack		= run_pre_handler,
+	.end		= sun4v_irq_end,
+};
 
-	if (unlikely(!action))
-		return;
+void irq_install_pre_handler(int virt_irq,
+			     void (*func)(unsigned int, void *, void *),
+			     void *arg1, void *arg2)
+{
+	irq_desc_t *desc = irq_desc + virt_irq;
+	struct irq_handler_data *data = desc->handler_data;
 
-	synchronize_irq(virt_irq);
+	data->pre_handler = func;
+	data->pre_handler_arg1 = arg1;
+	data->pre_handler_arg2 = arg2;
 
-	spin_lock_irqsave(&irq_action_lock, flags);
+	desc->handler = (desc->handler == &sun4u_irq ?
+			 &sun4u_irq_ack : &sun4v_irq_ack);
+}
 
-	bucket = __bucket(real_irq);
-	desc = bucket->irq_info;
+unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap)
+{
+	struct ino_bucket *bucket;
+	struct irq_handler_data *data;
+	irq_desc_t *desc;
+	int ino;
 
-	for (i = 0; i < MAX_IRQ_DESC_ACTION; i++) {
-		struct irqaction *p = &desc->action[i];
+	BUG_ON(tlb_type == hypervisor);
 
-		if (p == action) {
-			desc->action_active_mask &= ~(1 << i);
-			break;
-		}
+	ino = (upa_readl(imap) & (IMAP_IGN | IMAP_INO)) + inofixup;
+	bucket = &ivector_table[ino];
+	if (!bucket->virt_irq) {
+		bucket->virt_irq = virt_irq_alloc(__irq(bucket));
+		irq_desc[bucket->virt_irq].handler = &sun4u_irq;
 	}
 
-	if (!desc->action_active_mask) {
-		unsigned long imap = bucket->imap;
-
-		/* This unique interrupt source is now inactive. */
-		bucket->flags &= ~IBF_ACTIVE;
-
-		/* See if any other buckets share this bucket's IMAP
-		 * and are still active.
-		 */
-		for (ent = 0; ent < NUM_IVECS; ent++) {
-			struct ino_bucket *bp = &ivector_table[ent];
-			if (bp != bucket	&&
-			    bp->imap == imap	&&
-			    (bp->flags & IBF_ACTIVE) != 0)
-				break;
-		}
+	desc = irq_desc + bucket->virt_irq;
+	if (unlikely(desc->handler_data))
+		goto out;
 
-		/* Only disable when no other sub-irq levels of
-		 * the same IMAP are active.
-		 */
-		if (ent == NUM_IVECS)
-			disable_irq(virt_irq);
+	data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC);
+	if (unlikely(!data)) {
+		prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n");
+		prom_halt();
 	}
+	desc->handler_data = data;
 
-	spin_unlock_irqrestore(&irq_action_lock, flags);
-}
+	data->imap  = imap;
+	data->iclr  = iclr;
 
-EXPORT_SYMBOL(free_irq);
+out:
+	return bucket->virt_irq;
+}
 
-#ifdef CONFIG_SMP
-void synchronize_irq(unsigned int virt_irq)
+unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino)
 {
-	unsigned int real_irq = virt_to_real_irq(virt_irq);
 	struct ino_bucket *bucket;
+	struct irq_handler_data *data;
+	unsigned long sysino;
+	irq_desc_t *desc;
 
-	if (unlikely(!real_irq))
-		return;
+	BUG_ON(tlb_type != hypervisor);
 
-	bucket = __bucket(real_irq);
-#if 0
-	/* The following is how I wish I could implement this.
-	 * Unfortunately the ICLR registers are read-only, you can
-	 * only write ICLR_foo values to them.  To get the current
-	 * IRQ status you would need to get at the IRQ diag registers
-	 * in the PCI/SBUS controller and the layout of those vary
-	 * from one controller to the next, sigh... -DaveM
-	 */
-	unsigned long iclr = bucket->iclr;
-
-	while (1) {
-		u32 tmp = upa_readl(iclr);
-		
-		if (tmp == ICLR_TRANSMIT ||
-		    tmp == ICLR_PENDING) {
-			cpu_relax();
-			continue;
-		}
-		break;
+	sysino = sun4v_devino_to_sysino(devhandle, devino);
+	bucket = &ivector_table[sysino];
+	if (!bucket->virt_irq) {
+		bucket->virt_irq = virt_irq_alloc(__irq(bucket));
+		irq_desc[bucket->virt_irq].handler = &sun4v_irq;
 	}
-#else
-	/* So we have to do this with a INPROGRESS bit just like x86.  */
-	while (bucket->flags & IBF_INPROGRESS)
-		cpu_relax();
-#endif
-}
-#endif /* CONFIG_SMP */
-
-static void process_bucket(struct ino_bucket *bp, struct pt_regs *regs)
-{
-	struct irq_desc *desc = bp->irq_info;
-	unsigned char flags = bp->flags;
-	u32 action_mask, i;
-	int random;
-
-	bp->flags |= IBF_INPROGRESS;
 
-	if (unlikely(!(flags & IBF_ACTIVE))) {
-		bp->pending = 1;
+	desc = irq_desc + bucket->virt_irq;
+	if (unlikely(desc->handler_data))
 		goto out;
-	}
-
-	if (desc->pre_handler)
-		desc->pre_handler(bp,
-				  desc->pre_handler_arg1,
-				  desc->pre_handler_arg2);
 
-	action_mask = desc->action_active_mask;
-	random = 0;
-	for (i = 0; i < MAX_IRQ_DESC_ACTION; i++) {
-		struct irqaction *p = &desc->action[i];
-		u32 mask = (1 << i);
+	data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC);
+	if (unlikely(!data)) {
+		prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n");
+		prom_halt();
+	}
+	desc->handler_data = data;
 
-		if (!(action_mask & mask))
-			continue;
+	/* Catch accidental accesses to these things.  IMAP/ICLR handling
+	 * is done by hypervisor calls on sun4v platforms, not by direct
+	 * register accesses.
+	 */
+	data->imap = ~0UL;
+	data->iclr = ~0UL;
 
-		action_mask &= ~mask;
+out:
+	return bucket->virt_irq;
+}
 
-		if (p->handler(bp->virt_irq, p->dev_id, regs) == IRQ_HANDLED)
-			random |= p->flags;
+void hw_resend_irq(struct hw_interrupt_type *handler, unsigned int virt_irq)
+{
+	struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq);
+	unsigned long pstate;
+	unsigned int *ent;
 
-		if (!action_mask)
-			break;
-	}
+	__asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate));
+	__asm__ __volatile__("wrpr %0, %1, %%pstate"
+			     : : "r" (pstate), "i" (PSTATE_IE));
+	ent = irq_work(smp_processor_id());
+	bucket->irq_chain = *ent;
+	*ent = __irq(bucket);
+	set_softint(1 << PIL_DEVICE_IRQ);
+	__asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate));
+}
 
-	if (tlb_type == hypervisor) {
-		unsigned int ino = __irq_ino(bp);
-		int err;
+void ack_bad_irq(unsigned int virt_irq)
+{
+	struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq);
+	unsigned int ino = 0xdeadbeef;
 
-		err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE);
-		if (err != HV_EOK)
-			printk("sun4v_intr_setstate(%x): "
-			       "err(%d)\n", ino, err);
-	} else {
-		upa_writel(ICLR_IDLE, bp->iclr);
-	}
+	if (bucket)
+		ino = bucket - &ivector_table[0];
 
-	/* Test and add entropy */
-	if (random & SA_SAMPLE_RANDOM)
-		add_interrupt_randomness(bp->virt_irq);
-out:
-	bp->flags &= ~IBF_INPROGRESS;
+	printk(KERN_CRIT "Unexpected IRQ from ino[%x] virt_irq[%u]\n",
+	       ino, virt_irq);
 }
 
 #ifndef CONFIG_SMP
@@ -740,35 +532,33 @@ void timer_irq(int irq, struct pt_regs *regs)
 	clear_softint(clr_mask);
 
 	irq_enter();
+
 	kstat_this_cpu.irqs[0]++;
 	timer_interrupt(irq, NULL, regs);
+
 	irq_exit();
 }
 #endif
 
 void handler_irq(int irq, struct pt_regs *regs)
 {
-	struct ino_bucket *bp;
-	int cpu = smp_processor_id();
+	struct ino_bucket *bucket;
 
-	/* XXX at this point we should be able to assert that
-	 * XXX irq is PIL_DEVICE_IRQ...
-	 */
 	clear_softint(1 << irq);
 
 	irq_enter();
 
 	/* Sliiiick... */
-	bp = __bucket(xchg32(irq_work(cpu), 0));
-	while (bp) {
-		struct ino_bucket *nbp = __bucket(bp->irq_chain);
+	bucket = __bucket(xchg32(irq_work(smp_processor_id()), 0));
+	while (bucket) {
+		struct ino_bucket *next = __bucket(bucket->irq_chain);
 
-		kstat_this_cpu.irqs[bp->virt_irq]++;
+		bucket->irq_chain = 0;
+		__do_IRQ(bucket->virt_irq, regs);
 
-		bp->irq_chain = 0;
-		process_bucket(bp, regs);
-		bp = nbp;
+		bucket = next;
 	}
+
 	irq_exit();
 }
 
@@ -833,74 +623,6 @@ main_interrupt:
 EXPORT_SYMBOL(sparc_floppy_irq);
 #endif
 
-/* We really don't need these at all on the Sparc.  We only have
- * stubs here because they are exported to modules.
- */
-unsigned long probe_irq_on(void)
-{
-	return 0;
-}
-
-EXPORT_SYMBOL(probe_irq_on);
-
-int probe_irq_off(unsigned long mask)
-{
-	return 0;
-}
-
-EXPORT_SYMBOL(probe_irq_off);
-
-#ifdef CONFIG_SMP
-static int retarget_one_irq(struct irqaction *p, int goal_cpu)
-{
-	struct ino_bucket *bucket = get_ino_in_irqaction(p) + ivector_table;
-
-	while (!cpu_online(goal_cpu)) {
-		if (++goal_cpu >= NR_CPUS)
-			goal_cpu = 0;
-	}
-
-	if (tlb_type == hypervisor) {
-		unsigned int ino = __irq_ino(bucket);
-
-		sun4v_intr_settarget(ino, goal_cpu);
-		sun4v_intr_setenabled(ino, HV_INTR_ENABLED);
-	} else {
-		unsigned long imap = bucket->imap;
-		unsigned int tid = sun4u_compute_tid(imap, goal_cpu);
-
-		upa_writel(tid | IMAP_VALID, imap);
-	}
-
-	do {
-		if (++goal_cpu >= NR_CPUS)
-			goal_cpu = 0;
-	} while (!cpu_online(goal_cpu));
-
-	return goal_cpu;
-}
-
-/* Called from request_irq. */
-static void distribute_irqs(void)
-{
-	unsigned long flags;
-	int cpu, level;
-
-	spin_lock_irqsave(&irq_action_lock, flags);
-	cpu = 0;
-
-	for (level = 1; level < NR_IRQS; level++) {
-		struct irqaction *p = irq_action[level];
-
-		while(p) {
-			cpu = retarget_one_irq(p, cpu);
-			p = p->next;
-		}
-	}
-	spin_unlock_irqrestore(&irq_action_lock, flags);
-}
-#endif
-
 struct sun5_timer {
 	u64	count0;
 	u64	limit0;
@@ -1076,6 +798,10 @@ void __cpuinit sun4v_init_mondo_queues(int use_bootmem, int cpu, int alloc, int
 	}
 }
 
+static struct irqaction timer_irq_action = {
+	.name = "timer",
+};
+
 /* Only invoked on boot processor. */
 void __init init_IRQ(void)
 {
@@ -1103,121 +829,6 @@ void __init init_IRQ(void)
 			     : /* No outputs */
 			     : "i" (PSTATE_IE)
 			     : "g1");
-}
 
-static struct proc_dir_entry *root_irq_dir;
-static struct proc_dir_entry *irq_dir[NR_IRQS];
-
-#ifdef CONFIG_SMP
-
-static int irq_affinity_read_proc(char *page, char **start, off_t off,
-				  int count, int *eof, void *data)
-{
-	struct ino_bucket *bp = ivector_table + (long)data;
-	struct irq_desc *desc = bp->irq_info;
-	struct irqaction *ap = desc->action;
-	cpumask_t mask;
-	int len;
-
-	mask = get_smpaff_in_irqaction(ap);
-	if (cpus_empty(mask))
-		mask = cpu_online_map;
-
-	len = cpumask_scnprintf(page, count, mask);
-	if (count - len < 2)
-		return -EINVAL;
-	len += sprintf(page + len, "\n");
-	return len;
+	irq_desc[0].action = &timer_irq_action;
 }
-
-static inline void set_intr_affinity(int virt_irq, cpumask_t hw_aff)
-{
-	struct ino_bucket *bp;
-	struct irq_desc *desc;
-	struct irqaction *ap;
-	unsigned int real_irq;
-
-	real_irq = virt_to_real_irq(virt_irq);
-	if (unlikely(!real_irq))
-		return;
-
-	bp = __bucket(real_irq);
-	desc = bp->irq_info;
-	ap = desc->action;
-
-	/* Users specify affinity in terms of hw cpu ids.
-	 * As soon as we do this, handler_irq() might see and take action.
-	 */
-	put_smpaff_in_irqaction(ap, hw_aff);
-
-	/* Migration is simply done by the next cpu to service this
-	 * interrupt.
-	 *
-	 * XXX Broken, this doesn't happen anymore...
-	 */
-}
-
-static int irq_affinity_write_proc(struct file *file,
-				   const char __user *buffer,
-				   unsigned long count, void *data)
-{
-	int virt_irq = (long) data, full_count = count, err;
-	cpumask_t new_value;
-
-	err = cpumask_parse(buffer, count, new_value);
-
-	/*
-	 * Do not allow disabling IRQs completely - it's a too easy
-	 * way to make the system unusable accidentally :-) At least
-	 * one online CPU still has to be targeted.
-	 */
-	cpus_and(new_value, new_value, cpu_online_map);
-	if (cpus_empty(new_value))
-		return -EINVAL;
-
-	set_intr_affinity(virt_irq, new_value);
-
-	return full_count;
-}
-
-#endif
-
-#define MAX_NAMELEN 10
-
-static void register_irq_proc(unsigned int virt_irq)
-{
-	char name [MAX_NAMELEN];
-
-	if (!root_irq_dir || irq_dir[virt_irq])
-		return;
-
-	memset(name, 0, MAX_NAMELEN);
-	sprintf(name, "%d", virt_irq);
-
-	/* create /proc/irq/1234 */
-	irq_dir[virt_irq] = proc_mkdir(name, root_irq_dir);
-
-#ifdef CONFIG_SMP
-	/* XXX SMP affinity not supported on starfire yet. */
-	if (this_is_starfire == 0) {
-		struct proc_dir_entry *entry;
-
-		/* create /proc/irq/1234/smp_affinity */
-		entry = create_proc_entry("smp_affinity", 0600, irq_dir[irq]);
-
-		if (entry) {
-			entry->nlink = 1;
-			entry->data = (void *)(long)virt_irq;
-			entry->read_proc = irq_affinity_read_proc;
-			entry->write_proc = irq_affinity_write_proc;
-		}
-	}
-#endif
-}
-
-void init_irq_proc(void)
-{
-	/* create /proc/irq */
-	root_irq_dir = proc_mkdir("irq", NULL);
-}
-